src, scripts: use kv to cache directories

Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com>

Author: flakey5

.env.example

@@ -0,0 +1,30 @@
+# Your Cloudflare account tag.
+# 
+# Needed for:
+#  - Directory cache scripts
+CLOUDFLARE_ACCOUNT_ID=
+
+# Cloudflare V4 API token.
+# 
+# Needed for:
+#  - Directory cache scripts
+# 
+# Required permissions:
+#  - `Workers KV Storage`: Edit
+#  - `Workers R2 Storage`: Read
+# 
+# See https://developers.cloudflare.com/fundamentals/api/get-started/create-token/
+CLOUDFLARE_API_TOKEN=
+
+# S3 credentials for your R2 bucket.
+# 
+# Needed for:
+#  - Directory listings in the worker.
+#  - Directory cache scripts
+# 
+# Required permissions:
+#  - `Object Read Only`
+# 
+# See https://dash.cloudflare.com/?account=/r2/api-tokens
+S3_ACCESS_KEY_ID=
+S3_ACCESS_KEY_SECRET=

.github/workflows/update-links.yml

@@ -6,6 +6,11 @@ permissions:
 on:
   # Triggered by https://github.com/nodejs/node/blob/main/.github/workflows/update-release-links.yml
   workflow_dispatch:
+    inputs:
+      version:
+        description: 'Node.js version (ex/ `v20.0.0`)'
+        required: true
+        type: string
   schedule:
     - cron: '0 0 * * *'
 
@@ -81,6 +86,15 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GH_BOT_TOKEN }}
 
+      # Do this last for now so we avoid breaking releases if anything breaks
+      - name: Update Directory Cache
+        run: node scripts/update-directory-cache.mjs "$VERSION_INPUT"
+        env:
+          VERSION_INPUT: '${{ inputs.version }}'
+          CLOUDFLARE_API_TOKEN: ${{ secrets.CF_API_TOKEN }}
+          S3_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }}
+          S3_ACCESS_KEY_SECRET: ${{ secrets.CF_SECRET_ACCESS_KEY }}
+
       - name: Alert on Failure
         if: failure() && github.repository == 'nodejs/release-cloudflare-worker'
         uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # 2.3.3

.gitignore

@@ -3,3 +3,4 @@ node_modules/
 dist/
 .dev.vars
 .sentryclirc
+.env

common/README.md

@@ -0,0 +1,3 @@
+# Common
+
+Utilities used in local scripts and in the deployed worker.

src/constants/limits.ts common/limits.mjssrc/constants/limits.ts renamed to common/limits.mjs

@@ -1,8 +1,13 @@
 /**
- * Max amount of retries for R2 requests
+ * Max amount of retries for requests to R2
  */
 export const R2_RETRY_LIMIT = 5;
 
+/**
+ * Max amount of retries for requests to KV
+ */
+export const KV_RETRY_LIMIT = 5;
+
 /**
  * Max amount of keys to be returned in a S3 request
  */

common/listR2Directory.mjs

@@ -0,0 +1,107 @@
+import { ListObjectsV2Command } from '@aws-sdk/client-s3';
+import { R2_RETRY_LIMIT, S3_MAX_KEYS } from './limits';
+
+/**
+ * List the contents of a directory in R2.
+ *
+ * @param {import('@aws-sdk/client-s3').S3Client} client
+ * @param {string} bucket
+ * @param {string | undefined} [directory=undefined]
+ * @param {number} retryCount
+ * @returns {Promise<import('../src/providers/provider.js').ReadDirectoryResult | undefined>}
+ */
+export async function listR2Directory(
+  client,
+  bucket,
+  directory = undefined,
+  retryCount = R2_RETRY_LIMIT
+) {
+  /**
+   * @type {Set<string>}
+   */
+  const subdirectories = new Set();
+
+  /**
+   * @type {Set<import('../src/providers/provider.js').File>}
+   */
+  const files = new Set();
+
+  let hasIndexHtmlFile = false;
+  let directoryLastModified = new Date(0);
+
+  let isTruncated;
+  let continuationToken;
+  do {
+    /**
+     * @type {import('@aws-sdk/client-s3').ListObjectsV2Output | undefined}
+     */
+    let data = undefined;
+
+    let retriesLeft = retryCount;
+    while (retriesLeft) {
+      try {
+        data = await client.send(
+          new ListObjectsV2Command({
+            Bucket: bucket,
+            Delimiter: '/',
+            Prefix: directory,
+            ContinuationToken: continuationToken,
+            MaxKeys: S3_MAX_KEYS,
+          })
+        );
+
+        break;
+      } catch (err) {
+        retriesLeft--;
+
+        if (retriesLeft === 0) {
+          throw new Error('exhausted R2 retries', { cause: err });
+        }
+      }
+    }
+
+    if (!data) {
+      return undefined;
+    }
+
+    isTruncated = data.IsTruncated;
+    continuationToken = data.NextContinuationToken;
+
+    data.CommonPrefixes?.forEach(subdirectory => {
+      if (subdirectory.Prefix) {
+        subdirectories.add(
+          subdirectory.Prefix.substring(directory?.length ?? 0)
+        );
+      }
+    });
+
+    data.Contents?.forEach(file => {
+      if (!file.Key) {
+        return;
+      }
+
+      if (!hasIndexHtmlFile && file.Key.match(/index.htm(?:l)$/)) {
+        hasIndexHtmlFile = true;
+      }
+
+      files.add({
+        name: file.Key.substring(directory?.length ?? 0),
+        lastModified: file.LastModified,
+        size: file.Size,
+      });
+
+      // Set the directory's last modified date to be the same as the most
+      // recently updated file
+      if (file.LastModified > directoryLastModified) {
+        directoryLastModified = file.LastModified;
+      }
+    });
+  } while (isTruncated);
+
+  return {
+    subdirectories: Array.from(subdirectories),
+    hasIndexHtmlFile,
+    files: Array.from(files),
+    lastModified: directoryLastModified,
+  };
+}

common/listR2Directory.test.ts

@@ -0,0 +1,177 @@
+import { test, expect } from 'vitest';
+import { ListObjectsV2Command } from '@aws-sdk/client-s3';
+import { listR2Directory } from './listR2Directory.mjs';
+import { R2_RETRY_LIMIT } from './limits.mjs';
+
+test('adds subdirectories and files properly', async () => {
+  const now = new Date();
+
+  // Add a second so we can check the directory's last modified is determined properly
+  const directoryLastModified = new Date(now.getTime() + 1000);
+
+  const client = {
+    async send(cmd: ListObjectsV2Command) {
+      expect(cmd.input.Bucket).toStrictEqual('dist-prod');
+      expect(cmd.input.Prefix).toStrictEqual('some/directory/');
+
+      return {
+        IsTruncated: false,
+        NextContinuationToken: undefined,
+        CommonPrefixes: [
+          { Prefix: 'some/directory/subdirectory1/' },
+          { Prefix: 'some/directory/subdirectory2/' },
+          { Prefix: 'some/directory/subdirectory3/' },
+        ],
+        Contents: [
+          {
+            Key: 'some/directory/file.txt',
+            LastModified: now,
+            Size: 1,
+          },
+          {
+            Key: 'some/directory/file2.txt',
+            LastModified: directoryLastModified,
+            Size: 2,
+          },
+          {
+            Key: 'some/directory/file3.txt',
+            LastModified: now,
+            Size: 3,
+          },
+        ],
+      };
+    },
+  };
+
+  // @ts-expect-error don't need full client
+  const result = await listR2Directory(client, 'dist-prod', 'some/directory/');
+
+  expect(result).toStrictEqual({
+    subdirectories: ['subdirectory1/', 'subdirectory2/', 'subdirectory3/'],
+    hasIndexHtmlFile: false,
+    files: [
+      {
+        name: 'file.txt',
+        lastModified: now,
+        size: 1,
+      },
+      {
+        name: 'file2.txt',
+        lastModified: directoryLastModified,
+        size: 2,
+      },
+      {
+        name: 'file3.txt',
+        lastModified: now,
+        size: 3,
+      },
+    ],
+    lastModified: directoryLastModified,
+  });
+});
+
+test('handles truncation properly', async () => {
+  const now = new Date();
+
+  const client = {
+    async send(cmd: ListObjectsV2Command) {
+      expect(cmd.input.Bucket).toStrictEqual('dist-prod');
+      expect(cmd.input.Prefix).toStrictEqual('some/directory/');
+
+      switch (cmd.input.ContinuationToken) {
+        case undefined: {
+          return {
+            IsTruncated: true,
+            NextContinuationToken: '1',
+            CommonPrefixes: [{ Prefix: 'some/directory/subdirectory1/' }],
+            Contents: [
+              {
+                Key: 'some/directory/file.txt',
+                LastModified: now,
+                Size: 1,
+              },
+            ],
+          };
+        }
+        case '1': {
+          return {
+            IsTruncated: true,
+            NextContinuationToken: '2',
+            CommonPrefixes: [{ Prefix: 'some/directory/subdirectory2/' }],
+            Contents: [
+              {
+                Key: 'some/directory/file2.txt',
+                LastModified: now,
+                Size: 2,
+              },
+            ],
+          };
+        }
+        case '2': {
+          return {
+            IsTruncated: false,
+            NextContinuationToken: undefined,
+            CommonPrefixes: [{ Prefix: 'some/directory/subdirectory3/' }],
+            Contents: [
+              {
+                Key: 'some/directory/file3.txt',
+                LastModified: now,
+                Size: 3,
+              },
+            ],
+          };
+        }
+      }
+    },
+  };
+
+  // @ts-expect-error don't need full client
+  const result = await listR2Directory(client, 'dist-prod', 'some/directory/');
+
+  expect(result).toStrictEqual({
+    subdirectories: ['subdirectory1/', 'subdirectory2/', 'subdirectory3/'],
+    hasIndexHtmlFile: false,
+    files: [
+      {
+        name: 'file.txt',
+        lastModified: now,
+        size: 1,
+      },
+      {
+        name: 'file2.txt',
+        lastModified: now,
+        size: 2,
+      },
+      {
+        name: 'file3.txt',
+        lastModified: now,
+        size: 3,
+      },
+    ],
+    lastModified: now,
+  });
+});
+
+test('retries properly', async () => {
+  let retries = R2_RETRY_LIMIT;
+
+  let requestsSent = 0;
+  const client = {
+    async send() {
+      requestsSent++;
+
+      throw new TypeError('dummy');
+    },
+  };
+
+  const result = listR2Directory(
+    // @ts-expect-error don't need full client
+    client,
+    'dist-prod',
+    'some/directory/',
+    retries
+  );
+
+  await expect(result).rejects.toThrow('exhausted R2 retries');
+  expect(requestsSent).toBe(retries);
+});