fix:bio

Author: cullenwatson

examples/upload_staff_to_clay.py

@@ -0,0 +1,12 @@
+from staffspy import LinkedInAccount
+from staffspy.utils.utils import upload_to_clay
+
+session_file = "session.pkl"
+account = LinkedInAccount(session_file=session_file, log_level=2)
+
+connections = account.scrape_connections(extra_profile_data=True, max_results=3)
+
+clay_webhook_url = (
+    "https://api.clay.com/v3/sources/webhook/pull-in-data-from-a-webhook-XXXXXXXXXXXXXX"
+)
+upload_to_clay(webhook_url=clay_webhook_url, data=connections)

staffspy/__init__.py

@@ -15,7 +15,14 @@
     extract_emails_from_text,
     clean_df,
 )
-from staffspy.utils.driver_type import DriverType
+from staffspy.utils.driver_type import DriverType, BrowserType
+
+__all__ = [
+    "LinkedInAccount",
+    "SolverType",
+    "DriverType",
+    "BrowserType",
+]
 
 
 class LinkedInAccount:

staffspy/linkedin/employee_bio.py

@@ -9,7 +9,7 @@
 class EmployeeBioFetcher:
     def __init__(self, session):
         self.session = session
-        self.endpoint = "https://www.linkedin.com/voyager/api/graphql?queryId=voyagerIdentityDashProfileComponents.9117695ef207012719e3e0681c667e14&queryName=ProfileComponentsBySectionType&variables=(tabIndex:0,sectionType:languages,profileUrn:urn%3Ali%3Afsd_profile%3A{employee_id},count:50)"
+        self.endpoint = "https://www.linkedin.com/voyager/api/graphql?queryId=voyagerIdentityDashProfileCards.9ad2590cb61a073ad514922fa752f566&queryName=ProfileTabInitialCards&variables=(count:50,profileUrn:urn%3Ali%3Afsd_profile%3A{employee_id})"
 
     def fetch_employee_bio(self, base_staff):
         ep = self.endpoint.format(employee_id=base_staff.id)
@@ -18,36 +18,19 @@ def fetch_employee_bio(self, base_staff):
         if res.status_code == 429:
             return TooManyRequests("429 Too Many Requests")
         if not res.ok:
-            logger.debug(res.text[:200])
+            logger.debug(res.text)
             return False
         try:
-            res_json = res.json()
+            data = res.json()
         except json.decoder.JSONDecodeError:
-            logger.debug(res.text[:200])
+            logger.debug(res.text)
             return False
 
         try:
-            employee_json = list(
-                filter(
-                    lambda x: ",ABOUT," in x["entityUrn"],
-                    res_json["data"]["identityDashProfileCardsByInitialCards"][
-                        "elements"
-                    ],
-                )
-            )
+            base_staff.bio = data["data"]["identityDashProfileCardsByInitialCards"][
+                "elements"
+            ][3]["topComponents"][1]["components"]["textComponent"]["text"]["text"]
         except (KeyError, IndexError, TypeError):
-            logger.debug(res_json)
             return False
 
-        self.parse_emp_bio(base_staff, employee_json)
         return True
-
-    def parse_emp_bio(self, emp, emp_dict):
-        """Parse the employee data from the employee profile."""
-        try:
-            bio = emp_dict[0]["topComponents"][1]["components"]["textComponent"][
-                "text"
-            ]["text"]
-        except:
-            bio = None
-        emp.bio = bio

staffspy/utils/utils.py

@@ -447,5 +447,23 @@ def clean_df(staff_df):
     return staff_df
 
 
+def upload_to_clay(webhook_url: str, data: pd.DataFrame):
+    records = data.to_dict("records")
+
+    responses = []
+    for i, row in enumerate(records, start=1):
+        try:
+            response = requests.post(
+                webhook_url, headers={"Accept": "application/json"}, json=row
+            )
+            response.raise_for_status()
+            logger.info(f"Uploaded row to Clay: {i} / {len(records)}")
+        except requests.exceptions.RequestException as e:
+            logger.error(f"Failed to upload row to Clay: {str(e)}")
+            responses.append({"error": str(e), "data": row})
+
+    return responses
+
+
 if __name__ == "__main__":
     p = parse_dates("May 2018 - Jun 2024")