Transport layer tests

.mocharc.js

@@ -0,0 +1,3 @@
+module.exports = {
+  require: ["test/hooks.js"]
+}

Makefile

@@ -4,10 +4,19 @@ test:
 	./node_modules/.bin/mocha --recursive --timeout 10000 --exit ./test/unit ./test/integration
 
 test-bail:
-	./node_modules/.bin/mocha --bail --recursive --timeout 10000 ./test/unit ./test/integration
+	./node_modules/.bin/mocha --bail --recursive --timeout 10000 --exit ./test/unit ./test/integration
 
 test-integration:
-	./node_modules/.bin/mocha --recursive --timeout 10000 ./test/integration
+	./node_modules/.bin/mocha --recursive --timeout 10000 --exit ./test/integration
+
+test-stats:
+	./node_modules/.bin/mocha --timeout 30000 ./test/integration/stats.js
+
+test-data:
+	./node_modules/.bin/mocha --timeout 60000 --exit --bail ./test/integration/data.js
+
+test-synergy:
+	./node_modules/.bin/mocha --timeout 10000 --exit --bail ./test/integration/synergy.js
 
 test-unit:
 	./node_modules/.bin/mocha --recursive ./test/unit/

package.json

@@ -18,6 +18,7 @@
     "url": "https://github.com/bttmly/nba/issues"
   },
   "dependencies": {
+    "axios": "^0.21.1",
     "camel-case": "^3.0.0",
     "lodash.find": "^3.2.0",
     "lodash.findwhere": "^3.1.0",
@@ -29,6 +30,7 @@
     "minimist": "^1.2.0",
     "nba-client-template": "4.5.0",
     "node-fetch": "2.6.1",
+    "puppeteer": "^5.5.0",
     "url": "^0.11.0"
   },
   "devDependencies": {

scripts/run-transport.js

@@ -0,0 +1,20 @@
+const { NBA_URL, TRANSPORT = "basic" } = process.env;
+const { defaultTransport, setDefaultTransport } = require("../src/transport");
+const { transport: puppeteerTransport } = require("../src/PuppeteerTransport");
+const { URL } = require("url");
+const transforms = require("../src/transforms");
+
+if (!NBA_URL) throw new Error("must provide NBA_URL");
+
+(async () => {
+  if (TRANSPORT !== "basic") {
+    setDefaultTransport(puppeteerTransport);
+  }
+
+
+  let url = new URL(NBA_URL).toString();
+  url = url.split("\\").join("");
+  const result = await defaultTransport(url);
+  console.log(transforms.general(result));
+  // console.log(JSON.stringify(result));
+})();

src/PuppeteerTransport.js

@@ -0,0 +1,138 @@
+const puppeteer = require("puppeteer");
+const { URL } = require("url");
+
+const USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";
+
+const delay = (ms) => new Promise(r => setTimeout(r, ms));
+
+class PuppeteerTransport {
+  static async create () {
+    const browser = await puppeteer.launch({
+      handleSIGINT: false,
+      handleSIGTERM: false,
+      args: [
+        "--disable-web-security",
+      ],
+    });
+    return new PuppeteerTransport(browser);
+  }
+
+  constructor (browser) {
+    this.browser = browser;
+  }
+
+  async _createPage () {
+    const page = await this.browser.newPage();
+    await page.setRequestInterception(true);
+    page.on("request", req => {
+      const type = req.resourceType();
+      switch (type) {
+        case "document":
+        case "fetch":
+          req.continue();
+          break;
+        default:
+          req.abort();
+      }
+    });
+    page.on("framenavigated", (frame) => {
+      console.log("NAVIGATION:", frame.url(), "main", frame === page.mainFrame());
+    });
+    await page.setUserAgent(USER_AGENT);
+    await page.goto("https://www.nba.com/stats/", { waitUntil: "domcontentloaded", timeout: 8 * 1000 });
+    return page;
+  }
+
+  async _getPage () {
+    if (this.pageP) {
+      return this.pageP;
+    }
+    this.pageP = this._createPage();
+    return this.pageP;
+  }
+
+  async run (_url) {
+    const page = await this._getPage();
+    const result = await page.evaluate(async (url) => {
+      const headers = {
+        "Accept-Encoding": "gzip, deflate, br",
+        "Accept-Language": "en,en-US;q=0.9",
+        Accept: "application/json, text/plain, */*",
+        Referer: "https://www.nba.com/",
+        Connection: "keep-alive",
+        "Cache-Control": "no-cache",
+        Origin: "http://www.nba.com",
+        // "x-nba-stats-origin": "stats",
+        // "x-nba-stats-token": "true",
+      };
+
+      try {
+        const res = await fetch(url, { headers });
+        if (res.ok) {
+          const data = await res.json();
+          return { data, ok: true };
+        }
+        const text = await res.text();
+        return {
+          ok: false,
+          data: { text, status: res.status },
+        };
+      } catch (err) {
+        console.log(err);
+        return { ok: false, data: { text: err.toString() }};
+      }
+    }, _url);
+    return result;
+  }
+
+  close () {
+    console.trace("browser close");
+    return this.browser.close();
+  }
+};
+
+let instanceP = null;
+
+async function transport (baseURL, query = {}) {
+  if (instanceP == null) {
+    instanceP = PuppeteerTransport.create();
+    await delay(1);
+  }
+  const instance = await instanceP;
+
+  const u = new URL(baseURL);
+  for (const [key, value] of Object.entries(query)) {
+    u.searchParams.append(key, value);
+  }
+  u.protocol = "https:";
+  const result = await instance.run(u.toString());
+  if (result.ok) return result.data;
+  throw new Error(`${result.data.text} – ${u.toString()}`);
+}
+
+module.exports.transport = transport;
+module.exports.closeTransport = async () => {
+  if (instanceP == null) return;
+  const instance = await instanceP;
+  await instance.browser.close();
+};
+module.exports.PuppeteerTransport = PuppeteerTransport;
+
+// (async () => {
+//   const urls = [
+//     "https://stats.nba.com/stats/leaguedashteamstats?Conference=&DateFrom=&DateTo=&Division=&GameScope=&GameSegment=&LastNGames=0&LeagueID=00&Location=&MeasureType=Advanced&Month=0&OpponentTeamID=0&Outcome=&PORound=0&PaceAdjust=N&PerMode=PerGame&Period=0&PlayerExperience=&PlayerPosition=&PlusMinus=N&Rank=N&Season=2020-21&SeasonSegment=&SeasonType=Regular+Season&ShotClockRange=&StarterBench=&TeamID=0&TwoWay=0&VsConference=&VsDivision=",
+//     "http://stats.nba.com/stats/playerprofilev2?LeagueID=00&PerMode=PerGame&PlayerID=201939&Season=2017-18",
+//   ];
+
+//   const b = await puppeteer.launch({
+//     args: [
+//       "--disable-web-security",
+//     ],
+//   });
+//   const p = new PuppeteerTransport(b);
+//   for (const u of urls) {
+//     const { ok, data } = await p.run(u);
+//     console.log(ok, data.text, u.split("?")[0]);
+//   }
+//   await p.close();
+// })();

src/data.js

@@ -1,78 +1,96 @@
 // this includes endpoints at data.nba.com
 
-let transport = require("./get-json");
+const { defaultTransport } = require("./transport");
 const { interpolate } = require("./util/string");
 
-const scoreboardURL = interpolate("http://data.nba.com/data/5s/json/cms/noseason/scoreboard/__date__/games.json");
-const boxScoreURL = interpolate("http://data.nba.com/data/5s/json/cms/noseason/game/__date__/__gameId__/boxscore.json");
-const playByPlayURL = interpolate("http://data.nba.com/data/5s/json/cms/noseason/game/__date__/__gameId__/pbp_all.json");
-const scheduleURL = interpolate("http://data.nba.com/data/10s/prod/v1/__season__/schedule.json");
-const teamScheduleURL = interpolate("http://data.nba.com/data/10s/prod/v1/__season__/teams/__teamId__/schedule.json");
-const previewArticleURL = interpolate("http://data.nba.com/data/10s/prod/v1/__date__/__gameId___preview_article.json");
-const recapArticleURL = interpolate("http://data.nba.com/data/10s/prod/v1/__date__/__gameId___recap_article.json");
-const leadTrackerURL = interpolate("http://data.nba.com/data/10s/prod/v1/__date__/__gameId___lead_tracker___period__.json");
-const playoffsBracketURL = interpolate("http://data.nba.com/data/10s/prod/v1/__season__/playoffsBracket.json");
-const teamLeadersURL = interpolate("http://data.nba.com/data/10s/prod/v1/__season__/teams/__teamId__/leaders.json");
-const teamStatsRankingsURL = interpolate("http://data.nba.com/data/10s/prod/v1/__season__/team_stats_rankings.json");
-const coachesURL = interpolate("http://data.nba.com/data/10s/prod/v1/__season__/coaches.json");
-const teamsURL = interpolate("http://data.nba.net/data/10s/prod/v1/__year__/teams.json");
-
-const calendarURL = "http://data.nba.net/data/10s/prod/v1/calendar.json";
-const standingsURL = "http://data.nba.net/data/10s/prod/v1/current/standings_all.json";
-
-const withTransport = (newTransport) => {
-  transport = newTransport;
-};
+const scoreboardURL = interpolate("https://data.nba.com/data/5s/json/cms/noseason/scoreboard/__date__/games.json");
+const boxScoreURL = interpolate("https://data.nba.com/data/5s/json/cms/noseason/game/__date__/__gameId__/boxscore.json");
+const playByPlayURL = interpolate("https://data.nba.com/data/5s/json/cms/noseason/game/__date__/__gameId__/pbp_all.json");
+const scheduleURL = interpolate("https://data.nba.com/data/10s/prod/v1/__season__/schedule.json");
+const teamScheduleURL = interpolate("https://data.nba.com/data/10s/prod/v1/__season__/teams/__teamId__/schedule.json");
+const previewArticleURL = interpolate("https://data.nba.com/data/10s/prod/v1/__date__/__gameId___preview_article.json");
+const recapArticleURL = interpolate("https://data.nba.com/data/10s/prod/v1/__date__/__gameId___recap_article.json");
+const leadTrackerURL = interpolate("https://data.nba.com/data/10s/prod/v1/__date__/__gameId___lead_tracker___period__.json");
+const playoffsBracketURL = interpolate("https://data.nba.com/data/10s/prod/v1/__season__/playoffsBracket.json");
+const teamLeadersURL = interpolate("https://data.nba.com/data/10s/prod/v1/__season__/teams/__teamId__/leaders.json");
+const teamStatsRankingsURL = interpolate("https://data.nba.com/data/10s/prod/v1/__season__/team_stats_rankings.json");
+const coachesURL = interpolate("https://data.nba.com/data/10s/prod/v1/__season__/coaches.json");
+const teamsURL = interpolate("https://data.nba.net/data/10s/prod/v1/__year__/teams.json");
+
+const calendarURL = "https://data.nba.net/data/10s/prod/v1/calendar.json";
+const standingsURL = "https://data.nba.net/data/10s/prod/v1/current/standings_all.json";
 
 // NOTE: the 'date' argument should be a string in format like "20181008" (which indicates Oct 8 2018)
 // You *can* pass a Date object but beware of timezone weirdness!
 
 // NOTE: the 'season' argument is the first year of the NBA season e.g. "2018" for the 2018-19 season
 
-const scoreboard = date => transport(scoreboardURL({ date: dateToYYYYMMDD(date) }));
-scoreboard.defaults = { date: null };
+const withTransport = (transportOverride) => {
+  const transport = transportOverride || defaultTransport;
+
+  const scoreboard = date => transport(scoreboardURL({ date: dateToYYYYMMDD(date) }));
+  scoreboard.defaults = { date: null };
 
-const boxScore = (date, gameId) => transport(boxScoreURL({ date: dateToYYYYMMDD(date), gameId }));
-boxScore.defaults = { date: null, gameId: null };
+  const boxScore = (date, gameId) => transport(boxScoreURL({ date: dateToYYYYMMDD(date), gameId }));
+  boxScore.defaults = { date: null, gameId: null };
 
-const playByPlay = (date, gameId) => transport(playByPlayURL({ date: dateToYYYYMMDD(date), gameId }));
-playByPlay.defaults = { date: null, gameId: null };
+  const playByPlay = (date, gameId) => transport(playByPlayURL({ date: dateToYYYYMMDD(date), gameId }));
+  playByPlay.defaults = { date: null, gameId: null };
 
-const schedule = (season) => transport(scheduleURL({ season }));
-schedule.defaults = { season: null };
+  const schedule = (season) => transport(scheduleURL({ season }));
+  schedule.defaults = { season: null };
 
-const teamSchedule = (season, teamId) => transport(teamScheduleURL({ season, teamId }));
-teamSchedule.defaults = { season: null, teamId: null };
+  const teamSchedule = (season, teamId) => transport(teamScheduleURL({ season, teamId }));
+  teamSchedule.defaults = { season: null, teamId: null };
 
-const previewArticle = (date, gameId) => transport(previewArticleURL({date: dateToYYYYMMDD(date), gameId }));
-previewArticle.defaults = { date: null, gameId: null };
+  const previewArticle = (date, gameId) => transport(previewArticleURL({date: dateToYYYYMMDD(date), gameId }));
+  previewArticle.defaults = { date: null, gameId: null };
 
-const recapArticle = (date, gameId) => transport(recapArticleURL({date: dateToYYYYMMDD(date), gameId }));
-recapArticle.defaults = { date: null, gameId: null };
+  const recapArticle = (date, gameId) => transport(recapArticleURL({date: dateToYYYYMMDD(date), gameId }));
+  recapArticle.defaults = { date: null, gameId: null };
 
-const leadTracker = (date, gameId, period) => transport(leadTrackerURL({date: dateToYYYYMMDD(date), gameId, period }));
-leadTracker.defaults = { date: null, gameId: null, period: null };
+  const leadTracker = (date, gameId, period) => transport(leadTrackerURL({date: dateToYYYYMMDD(date), gameId, period }));
+  leadTracker.defaults = { date: null, gameId: null, period: null };
 
-const playoffsBracket = (season) => transport(playoffsBracketURL({ season }));
-playoffsBracket.defaults = { season: null };
+  const playoffsBracket = (season) => transport(playoffsBracketURL({ season }));
+  playoffsBracket.defaults = { season: null };
 
-const teamLeaders = (season, teamId) => transport(teamLeadersURL({ season, teamId }));
-teamLeaders.defaults = { season: null, teamId: null };
+  const teamLeaders = (season, teamId) => transport(teamLeadersURL({ season, teamId }));
+  teamLeaders.defaults = { season: null, teamId: null };
 
-const teamStatsRankings = (season) => transport(teamStatsRankingsURL({ season }));
-teamStatsRankings.defaults = { season: null };
+  const teamStatsRankings = (season) => transport(teamStatsRankingsURL({ season }));
+  teamStatsRankings.defaults = { season: null };
 
-const coaches = (season) => transport(coachesURL({ season }));
-coaches.defaults = { season: null };
+  const coaches = (season) => transport(coachesURL({ season }));
+  coaches.defaults = { season: null };
 
-const teams = (year = "2019") => transport(teamsURL({ year }));
-teams.defaults = { year: null };
+  const teams = (year = "2019") => transport(teamsURL({ year }));
+  teams.defaults = { year: null };
 
-const calendar = () => transport(calendarURL);
-calendar.defaults = {};
+  const calendar = () => transport(calendarURL);
+  calendar.defaults = {};
 
-const standings = () => transport(standingsURL);
-standings.defaults = {};
+  const standings = () => transport(standingsURL);
+  standings.defaults = {};
+
+  return {
+    scoreboard,
+    boxScore,
+    playByPlay,
+    schedule,
+    teamSchedule,
+    previewArticle,
+    recapArticle,
+    leadTracker,
+    playoffsBracket,
+    teamLeaders,
+    teamStatsRankings,
+    coaches,
+    teams,
+    calendar,
+    standings,
+  };
+};
 
 function dateToYYYYMMDD (date) {
   if (date instanceof Date) {
@@ -82,27 +100,12 @@ function dateToYYYYMMDD (date) {
       String(date.getDate()).padStart(2, 0),
     ].join("");
   }
-
-  // TODO: better checking here?
-
   return date;
 }
 
+const client = withTransport();
+
 module.exports = {
-  scoreboard,
-  boxScore,
-  playByPlay,
-  schedule,
-  teamSchedule,
-  previewArticle,
-  recapArticle,
-  leadTracker,
-  playoffsBracket,
-  teamLeaders,
-  teamStatsRankings,
-  coaches,
-  teams,
-  calendar,
-  standings,
+  ...client,
   withTransport,
 };