From aeac46b126dafa41cc567156bcf635e35caa00c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Sun, 8 Oct 2023 22:59:44 +0100 Subject: [PATCH 01/16] feat!: add `getStore` method --- .eslintrc.cjs | 1 + README.md | 163 +++++++++++----- src/client.ts | 124 ++++++++++++ src/main.test.ts | 479 +++++++++++++++++++++++++---------------------- src/main.ts | 259 +------------------------ src/retry.ts | 7 +- src/store.ts | 201 ++++++++++++++++++++ src/types.ts | 7 + 8 files changed, 708 insertions(+), 533 deletions(-) create mode 100644 src/client.ts create mode 100644 src/store.ts create mode 100644 src/types.ts diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 216de88..04799d4 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -14,6 +14,7 @@ module.exports = { 'max-lines-per-function': 'off', 'max-statements': 'off', 'node/no-missing-import': 'off', + 'n/no-missing-import': 'off', 'no-magic-numbers': 'off', 'no-shadow': 'off', 'no-use-before-define': 'off', diff --git a/README.md b/README.md index 9048da6..1cb003e 100644 --- a/README.md +++ b/README.md @@ -15,71 +15,134 @@ npm install @netlify/blobs ## Usage -To use the blob store, import the module and create an instance of the `Blobs` class. The constructor accepts an object -with the following properties: +To start reading and writing data, you must first get a reference to a store using the `getStore` method. -| Property | Description | Required | -| ---------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -| `authentication` | An object containing authentication credentials (see [Authentication](#authentication)) | **Yes** | -| `siteID` | The Netlify site ID | **Yes** | -| `context` | The [deploy context](https://docs.netlify.com/site-deploys/overview/#deploy-contexts) to use (defaults to `production`) | No | -| `fetcher` | An implementation of a [fetch-compatible](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) module for making HTTP requests (defaults to `globalThis.fetch`) | No | +This method takes an options object that lets you configure the store for different access modes. -### Example +### API access -```javascript -import assert from 'node:assert' -import { Blobs } from '@netlify/blobs' - -const store = new Blobs({ - authentication: { - token: 'YOUR_NETLIFY_AUTH_TOKEN', - }, - siteID: 'YOUR_NETLIFY_SITE_ID', -}) +You can interact with the blob store through the [Netlify API](https://docs.netlify.com/api/get-started). This is the +recommended method if you're looking for a strong-consistency way of accessing data, where latency is not mission +critical (since requests will always go to a non-distributed origin). + +Create a store for API access by calling `getStore` with the following parameters: + +- `name` (string): Name of the store +- `siteID` (string): ID of the Netlify site +- `token` (string): [Personal access + token]([your Netlify API credentials](https://docs.netlify.com/api/get-started/#authentication)) to access the Netlify + API +- `apiURL` (string): URL of the Netlify API (optional, defaults to `https://api.netlify.com`) -await store.set('some-key', 'Hello!') +```ts +import { getStore } from '@netlify/blobs' -const item = await store.get('some-key') +const store = getStore({ + name: 'my-store', + siteID: 'MY_SITE_ID', + token: 'MY_TOKEN', +}) -assert.strictEqual(item, 'Hello!') +console.log(await store.get('some-key')) ``` -### Authentication +### Edge access + +You can also interact with the blob store using a distributed network that caches entries at the edge. This is the +recommended method if you're looking for fast reads across multiple locations, knowing that reads will be +eventually-consistent with a drift of up to 60 seconds. -Authentication with the blob storage is done in one of two ways: +Create a store for edge access by calling `getStore` with the following parameters: -- Using a [Netlify API token](https://docs.netlify.com/api/get-started/#authentication) +- `name` (string): Name of the store +- `siteID` (string): ID of the Netlify site +- `token` (string): [Personal access + token]([your Netlify API credentials](https://docs.netlify.com/api/get-started/#authentication)) to access the Netlify + API +- `edgeURL` (string): URL of the edge endpoint - ```javascript - import { Blobs } from '@netlify/blobs' +```ts +import { Buffer } from 'node:buffer' - const store = new Blobs({ - authentication: { - token: 'YOUR_NETLIFY_API_TOKEN', - }, - siteID: 'YOUR_NETLIFY_SITE_ID', +import { getStore } from '@netlify/blobs' + +// Serverless function using the Lambda compatibility mode +export const handler = async (event, context) => { + const rawData = Buffer.from(context.clientContext.custom.blobs, 'base64') + const data = JSON.parse(rawData.toString('ascii')) + const store = getStore({ + edgeURL: data.url, + name: 'my-store', + token: data.token, + siteID: 'MY_SITE_ID', }) - ``` - -- Using a context object injected in Netlify Functions - - ```javascript - import { Blobs } from '@netlify/blobs' - import type { Handler, HandlerEvent, HandlerContext } from '@netlify/functions' - - export const handler: Handler = async (event: HandlerEvent, context: HandlerContext) => { - const store = new Blobs({ - authentication: { - contextURL: context.blobs.url, - token: context.blobs.token, - }, - siteID: 'YOUR_NETLIFY_SITE_ID', - }) + const item = await store.get('some-key') + + return { + statusCode: 200, + body: item, } - ``` +} +``` + +### Environment-based configuration + +Rather than explicitly passing the configuration context to the `getStore` method, it can be read from the execution +environment. This is particularly useful for setups where the configuration data is held by one system and the data +needs to be accessed in another system, with no direct communication between the two. + +To do this, the system that holds the configuration data should set an environment variable called `NETLIFY_BLOBS_1` +with a Base64-encoded, JSON-stringified representation of an object with the following properties: + +- `apiURL` (optional) or `edgeURL`: URL of the Netlify API (for [API access](#api-access)) or the edge endpoint (for + [Edge access](#edge-access)) +- `token`: Access token for the corresponding access mode +- `siteID`: ID of the Netlify site + +With this in place, the `getStore` method can be called just with the store name. No configuration object is required, +since it'll be read from the environment. + +```ts +import { getStore } from '@netlify/blobs' + +const store = getStore('my-store') + +console.log(await store.get('my-key')) +``` + +### Deploy scope + +By default, stores exist at the site level, which means that data can be read and written across different deploys and +deploy contexts. Users are responsible for managing that data, since the platform doesn't have enough information to +know whether an item is still relevant or safe to delete. + +But sometimes it's useful to have data pegged to a specific deploy, and shift to the platform the responsibility of +managing that data — keep it as long as the deploy is around, and wipe it if the deploy is deleted. + +You can opt-in to this behavior by supplying a `deployID` instead of a `name` to the `getStore` method. + +```ts +import { assert } from 'node:assert' + +import { getStore } from '@netlify/blobs' + +// Using API access +const store1 = getStore({ + deployID: 'MY_DEPLOY_ID', + token: 'MY_API_TOKEN', +}) + +await store1.set('my-key', 'my value') + +// Using environment-based configuration +const store2 = getStore({ + deployID: 'MY_DEPLOY_ID', +}) + +assert.equal(await store2.get('my-key'), 'my value') +``` -## API +## Store API ### `get(key: string, { type: string }): Promise` diff --git a/src/client.ts b/src/client.ts new file mode 100644 index 0000000..ce71235 --- /dev/null +++ b/src/client.ts @@ -0,0 +1,124 @@ +import { Buffer } from 'node:buffer' +import { env } from 'node:process' + +import { fetchAndRetry } from './retry.ts' +import { BlobInput, HTTPMethod } from './types.ts' + +// The name of the environment variable that holds the context in a Base64, +// JSON-encoded object. If we ever need to change the encoding or the shape +// of this object, we should bump the version and create a new variable, so +// that the client knows how to consume the data and can advise the user to +// update the client if needed. +export const NETLIFY_CONTEXT_VARIABLE = 'NETLIFY_BLOBS_1' + +export interface Context { + apiURL?: string + edgeURL?: string + siteID: string + token: string +} + +interface MakeStoreRequestOptions { + body?: BlobInput | null + headers?: Record + key: string + method: HTTPMethod + storeName: string +} + +export class Client { + private context?: Context + + constructor(context?: Context) { + this.context = context + } + + private getContext() { + if (this.context) { + return this.context + } + + if (!env[NETLIFY_CONTEXT_VARIABLE]) { + return + } + + const data = Buffer.from(env[NETLIFY_CONTEXT_VARIABLE], 'base64').toString() + + try { + return JSON.parse(data) as Context + } catch { + // no-op + } + } + + private static async getFinalRequest(context: Context, storeName: string, key: string, method: string) { + const encodedKey = encodeURIComponent(key) + + if ('edgeURL' in context) { + return { + headers: { + authorization: `Bearer ${context.token}`, + }, + url: `${context.edgeURL}/${context.siteID}/${storeName}/${encodedKey}`, + } + } + + const apiURL = `${context.apiURL ?? 'https://api.netlify.com'}/api/v1/sites/${ + context.siteID + }/blobs/${encodedKey}?context=${storeName}` + const headers = { authorization: `Bearer ${context.token}` } + const res = await fetch(apiURL, { headers, method }) + + if (res.status !== 200) { + throw new Error(`${method} operation has failed: API returned a ${res.status} response`) + } + + const { url } = await res.json() + + return { + url, + } + } + + async makeRequest({ body, headers: extraHeaders, key, method, storeName }: MakeStoreRequestOptions) { + const context = this.getContext() + + if (!context || !context.token || !context.siteID) { + throw new Error("The blob store is unavailable because it's missing required configuration properties") + } + + const { headers: baseHeaders = {}, url } = await Client.getFinalRequest(context, storeName, key, method) + const headers: Record = { + ...baseHeaders, + ...extraHeaders, + } + + if (method === HTTPMethod.Put) { + headers['cache-control'] = 'max-age=0, stale-while-revalidate=60' + } + + const options: RequestInit = { + body, + headers, + method, + } + + if (body instanceof ReadableStream) { + // @ts-expect-error Part of the spec, but not typed: + // https://fetch.spec.whatwg.org/#enumdef-requestduplex + options.duplex = 'half' + } + + const res = await fetchAndRetry(url, options) + + if (res.status === 404 && method === HTTPMethod.Get) { + return null + } + + if (res.status !== 200) { + throw new Error(`${method} operation has failed: store returned a ${res.status} response`) + } + + return res + } +} diff --git a/src/main.test.ts b/src/main.test.ts index 336741f..7601dae 100644 --- a/src/main.test.ts +++ b/src/main.test.ts @@ -1,5 +1,6 @@ +import { Buffer } from 'node:buffer' import { writeFile } from 'node:fs/promises' -import { version as nodeVersion } from 'node:process' +import { env, version as nodeVersion } from 'node:process' import semver from 'semver' import tmp from 'tmp-promise' @@ -8,7 +9,7 @@ import { describe, test, expect, beforeAll } from 'vitest' import { MockFetch } from '../test/mock_fetch.js' import { streamToString } from '../test/util.js' -import { Blobs } from './main.js' +import { getStore } from './main.js' beforeAll(async () => { if (semver.lt(nodeVersion, '18.0.0')) { @@ -25,6 +26,7 @@ beforeAll(async () => { } }) +const deployID = 'abcdef' const siteID = '12345' const key = '54321' const complexKey = '/artista/canção' @@ -37,7 +39,7 @@ const edgeURL = 'https://cloudfront.url' describe('get', () => { describe('With API credentials', () => { test('Reads from the blob store', async () => { - const store = new MockFetch() + const mockStore = new MockFetch() .get({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), @@ -68,11 +70,11 @@ describe('get', () => { url: signedURL, }) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) @@ -85,11 +87,11 @@ describe('get', () => { const string2 = await blobs.get(complexKey) expect(string2).toBe(value) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) test('Returns `null` when the pre-signed URL returns a 404', async () => { - const store = new MockFetch() + const mockStore = new MockFetch() .get({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), @@ -100,41 +102,41 @@ describe('get', () => { url: signedURL, }) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) expect(await blobs.get(key)).toBeNull() - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) test('Throws when the API returns a non-200 status code', async () => { - const store = new MockFetch().get({ + const mockStore = new MockFetch().get({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(null, { status: 401 }), url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`, }) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) expect(async () => await blobs.get(key)).rejects.toThrowError( 'get operation has failed: API returned a 401 response', ) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) test('Throws when a pre-signed URL returns a non-200 status code', async () => { - const store = new MockFetch() + const mockStore = new MockFetch() .get({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), @@ -145,11 +147,11 @@ describe('get', () => { url: signedURL, }) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) @@ -157,11 +159,11 @@ describe('get', () => { 'get operation has failed: store returned a 401 response', ) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) test('Returns `null` when the blob entry contains an expiry date in the past', async () => { - const store = new MockFetch() + const mockStore = new MockFetch() .get({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), @@ -176,22 +178,22 @@ describe('get', () => { url: signedURL, }) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) expect(await blobs.get(key)).toBeNull() - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) }) describe('With context credentials', () => { test('Reads from the blob store', async () => { - const store = new MockFetch() + const mockStore = new MockFetch() .get({ headers: { authorization: `Bearer ${edgeToken}` }, response: new Response(value), @@ -203,12 +205,12 @@ describe('get', () => { url: `${edgeURL}/${siteID}/production/${key}`, }) - const blobs = new Blobs({ - authentication: { - contextURL: edgeURL, - token: edgeToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + edgeURL, + name: 'production', + token: edgeToken, siteID, }) @@ -218,42 +220,42 @@ describe('get', () => { const stream = await blobs.get(key, { type: 'stream' }) expect(await streamToString(stream as unknown as NodeJS.ReadableStream)).toBe(value) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) test('Returns `null` when the edge URL returns a 404', async () => { - const store = new MockFetch().get({ + const mockStore = new MockFetch().get({ headers: { authorization: `Bearer ${edgeToken}` }, response: new Response(null, { status: 404 }), url: `${edgeURL}/${siteID}/production/${key}`, }) - const blobs = new Blobs({ - authentication: { - contextURL: edgeURL, - token: edgeToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + edgeURL, + name: 'production', + token: edgeToken, siteID, }) expect(await blobs.get(key)).toBeNull() - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) test('Throws when an edge URL returns a non-200 status code', async () => { - const store = new MockFetch().get({ + const mockStore = new MockFetch().get({ headers: { authorization: `Bearer ${edgeToken}` }, response: new Response(null, { status: 401 }), url: `${edgeURL}/${siteID}/production/${key}`, }) - const blobs = new Blobs({ - authentication: { - contextURL: edgeURL, - token: edgeToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + edgeURL, + name: 'production', + token: edgeToken, siteID, }) @@ -261,32 +263,27 @@ describe('get', () => { 'get operation has failed: store returned a 401 response', ) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) }) test('Throws when the instance is missing required configuration properties', async () => { const { fetcher } = new MockFetch() - const blobs1 = new Blobs({ - authentication: { - token: '', - }, - fetcher, - siteID, - }) + globalThis.fetch = fetcher - const blobs2 = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher, - siteID: '', - }) + const blobs1 = getStore('production') expect(async () => await blobs1.get(key)).rejects.toThrowError( `The blob store is unavailable because it's missing required configuration properties`, ) + + const blobs2 = getStore({ + name: 'production', + token: apiToken, + siteID: '', + }) + expect(async () => await blobs2.get(key)).rejects.toThrowError( `The blob store is unavailable because it's missing required configuration properties`, ) @@ -296,7 +293,7 @@ describe('get', () => { describe('set', () => { describe('With API credentials', () => { test('Writes to the blob store', async () => { - const store = new MockFetch() + const mockStore = new MockFetch() .put({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), @@ -322,23 +319,23 @@ describe('set', () => { url: signedURL, }) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) await blobs.set(key, value) await blobs.set(complexKey, value) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) test('Accepts an `expiration` parameter', async () => { const expiration = new Date(Date.now() + 15_000) - const store = new MockFetch() + const mockStore = new MockFetch() .put({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), @@ -354,24 +351,24 @@ describe('set', () => { url: signedURL, }) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) await blobs.set(key, value, { expiration }) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) // We need `Readable.toWeb` to be available, which needs Node 16+. if (semver.gte(nodeVersion, '16.0.0')) { test('Accepts a file', async () => { const fileContents = 'Hello from a file' - const store = new MockFetch() + const mockStore = new MockFetch() .put({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), @@ -388,21 +385,21 @@ describe('set', () => { url: signedURL, }) + globalThis.fetch = mockStore.fetcher + const { cleanup, path } = await tmp.file() await writeFile(path, fileContents) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) await blobs.setFile(key, path) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() await cleanup() }) @@ -411,7 +408,7 @@ describe('set', () => { const contents = ['Hello from key-0', 'Hello from key-1', 'Hello from key-2'] const signedURLs = ['https://signed-url.aws/0', 'https://signed-url.aws/1', 'https://signed-url.aws/2'] - const store = new MockFetch() + const mockStore = new MockFetch() .put({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURLs[0] })), @@ -458,6 +455,8 @@ describe('set', () => { url: signedURLs[2], }) + globalThis.fetch = mockStore.fetcher + const writes = await Promise.all( contents.map(async (content) => { const { cleanup, path } = await tmp.file() @@ -472,45 +471,43 @@ describe('set', () => { path, })) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) await blobs.setFiles(files) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() await Promise.all(writes.map(({ cleanup }) => cleanup())) }) } test('Throws when the API returns a non-200 status code', async () => { - const store = new MockFetch().put({ + const mockStore = new MockFetch().put({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(null, { status: 401 }), url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`, }) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) expect(async () => await blobs.set(key, 'value')).rejects.toThrowError( 'put operation has failed: API returned a 401 response', ) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) test('Retries failed operations', async () => { - const store = new MockFetch() + const mockStore = new MockFetch() .put({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), @@ -549,23 +546,23 @@ describe('set', () => { url: signedURL, }) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) await blobs.set(key, value) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) }) describe('With context credentials', () => { test('Writes to the blob store', async () => { - const store = new MockFetch() + const mockStore = new MockFetch() .put({ body: value, headers: { authorization: `Bearer ${edgeToken}`, 'cache-control': 'max-age=0, stale-while-revalidate=60' }, @@ -579,35 +576,35 @@ describe('set', () => { url: `${edgeURL}/${siteID}/production/${encodeURIComponent(complexKey)}`, }) - const blobs = new Blobs({ - authentication: { - contextURL: edgeURL, - token: edgeToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + edgeURL, + name: 'production', + token: edgeToken, siteID, }) await blobs.set(key, value) await blobs.set(complexKey, value) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) test('Throws when the edge URL returns a non-200 status code', async () => { - const store = new MockFetch().put({ + const mockStore = new MockFetch().put({ body: value, headers: { authorization: `Bearer ${edgeToken}`, 'cache-control': 'max-age=0, stale-while-revalidate=60' }, response: new Response(null, { status: 401 }), url: `${edgeURL}/${siteID}/production/${key}`, }) - const blobs = new Blobs({ - authentication: { - contextURL: edgeURL, - token: edgeToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + edgeURL, + name: 'production', + token: edgeToken, siteID, }) @@ -615,11 +612,11 @@ describe('set', () => { 'put operation has failed: store returned a 401 response', ) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) test('Retries failed operations', async () => { - const store = new MockFetch() + const mockStore = new MockFetch() .put({ body: value, headers: { authorization: `Bearer ${edgeToken}`, 'cache-control': 'max-age=0, stale-while-revalidate=60' }, @@ -645,43 +642,38 @@ describe('set', () => { url: `${edgeURL}/${siteID}/production/${key}`, }) - const blobs = new Blobs({ - authentication: { - contextURL: edgeURL, - token: edgeToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + edgeURL, + name: 'production', + token: edgeToken, siteID, }) await blobs.set(key, value) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) }) test('Throws when the instance is missing required configuration properties', async () => { const { fetcher } = new MockFetch() - const blobs1 = new Blobs({ - authentication: { - token: '', - }, - fetcher, - siteID, - }) + globalThis.fetch = fetcher - const blobs2 = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher, - siteID: '', - }) + const blobs1 = getStore('production') expect(async () => await blobs1.set(key, value)).rejects.toThrowError( `The blob store is unavailable because it's missing required configuration properties`, ) + + const blobs2 = getStore({ + name: 'production', + token: apiToken, + siteID: '', + }) + expect(async () => await blobs2.set(key, value)).rejects.toThrowError( `The blob store is unavailable because it's missing required configuration properties`, ) @@ -691,7 +683,7 @@ describe('set', () => { describe('setJSON', () => { describe('With API credentials', () => { test('Writes to the blob store', async () => { - const store = new MockFetch() + const mockStore = new MockFetch() .put({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), @@ -706,46 +698,46 @@ describe('setJSON', () => { url: signedURL, }) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) await blobs.setJSON(key, { value }) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) }) describe('With context credentials', () => { test('Writes to the blob store', async () => { - const store = new MockFetch().put({ + const mockStore = new MockFetch().put({ body: JSON.stringify({ value }), headers: { authorization: `Bearer ${edgeToken}`, 'cache-control': 'max-age=0, stale-while-revalidate=60' }, response: new Response(null), url: `${edgeURL}/${siteID}/production/${key}`, }) - const blobs = new Blobs({ - authentication: { - contextURL: edgeURL, - token: edgeToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + edgeURL, + name: 'production', + token: edgeToken, siteID, }) await blobs.setJSON(key, { value }) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) test('Accepts an `expiration` parameter', async () => { const expiration = new Date(Date.now() + 15_000) - const store = new MockFetch() + const mockStore = new MockFetch() .put({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), @@ -761,17 +753,17 @@ describe('setJSON', () => { url: signedURL, }) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) await blobs.setJSON(key, { value }, { expiration }) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) }) }) @@ -779,7 +771,7 @@ describe('setJSON', () => { describe('delete', () => { describe('With API credentials', () => { test('Deletes from the blob store', async () => { - const store = new MockFetch() + const mockStore = new MockFetch() .delete({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(JSON.stringify({ url: signedURL })), @@ -801,77 +793,77 @@ describe('delete', () => { url: signedURL, }) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) await blobs.delete(key) await blobs.delete(complexKey) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) test('Throws when the API returns a non-200 status code', async () => { - const store = new MockFetch().delete({ + const mockStore = new MockFetch().delete({ headers: { authorization: `Bearer ${apiToken}` }, response: new Response(null, { status: 401 }), url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`, }) - const blobs = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + name: 'production', + token: apiToken, siteID, }) expect(async () => await blobs.delete(key)).rejects.toThrowError( 'delete operation has failed: API returned a 401 response', ) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) }) describe('With context credentials', () => { test('Deletes from the blob store', async () => { - const store = new MockFetch().delete({ + const mockStore = new MockFetch().delete({ headers: { authorization: `Bearer ${edgeToken}` }, response: new Response(null), url: `${edgeURL}/${siteID}/production/${key}`, }) - const blobs = new Blobs({ - authentication: { - contextURL: edgeURL, - token: edgeToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + edgeURL, + name: 'production', + token: edgeToken, siteID, }) await blobs.delete(key) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) test('Throws when the edge URL returns a non-200 status code', async () => { - const store = new MockFetch().delete({ + const mockStore = new MockFetch().delete({ headers: { authorization: `Bearer ${edgeToken}` }, response: new Response(null, { status: 401 }), url: `${edgeURL}/${siteID}/production/${key}`, }) - const blobs = new Blobs({ - authentication: { - contextURL: edgeURL, - token: edgeToken, - }, - fetcher: store.fetcher, + globalThis.fetch = mockStore.fetcher + + const blobs = getStore({ + edgeURL, + name: 'production', + token: edgeToken, siteID, }) @@ -879,34 +871,79 @@ describe('delete', () => { 'delete operation has failed: store returned a 401 response', ) - expect(store.fulfilled).toBeTruthy() + expect(mockStore.fulfilled).toBeTruthy() }) }) test('Throws when the instance is missing required configuration properties', async () => { const { fetcher } = new MockFetch() - const blobs1 = new Blobs({ - authentication: { - token: '', - }, - fetcher, - siteID, - }) + globalThis.fetch = fetcher - const blobs2 = new Blobs({ - authentication: { - token: apiToken, - }, - fetcher, - siteID: '', - }) + const blobs1 = getStore('production') expect(async () => await blobs1.delete(key)).rejects.toThrowError( `The blob store is unavailable because it's missing required configuration properties`, ) + + const blobs2 = getStore({ + name: 'production', + token: apiToken, + siteID: '', + }) + expect(async () => await blobs2.delete(key)).rejects.toThrowError( `The blob store is unavailable because it's missing required configuration properties`, ) }) }) + +describe('Global client', () => { + test('Reads from the blob store', async () => { + const tokens = ['some-token-1', 'another-token-2'] + const mockStore = new MockFetch() + .get({ + headers: { authorization: `Bearer ${tokens[0]}` }, + response: new Response(value), + url: `${edgeURL}/${siteID}/images/${key}`, + }) + .get({ + headers: { authorization: `Bearer ${tokens[0]}` }, + response: new Response(value), + url: `${edgeURL}/${siteID}/images/${key}`, + }) + .get({ + headers: { authorization: `Bearer ${tokens[1]}` }, + response: new Response(value), + url: `${edgeURL}/${siteID}/images/${key}`, + }) + .get({ + headers: { authorization: `Bearer ${tokens[1]}` }, + response: new Response(value), + url: `${edgeURL}/${siteID}/images/${key}`, + }) + + globalThis.fetch = mockStore.fetcher + + for (let index = 0; index <= 1; index++) { + const context = { + edgeURL, + deployID, + siteID, + token: tokens[index], + } + + env.NETLIFY_BLOBS_1 = Buffer.from(JSON.stringify(context)).toString('base64') + + const store = getStore('images') + + const string = await store.get(key) + expect(string).toBe(value) + + const stream = await store.get(key, { type: 'stream' }) + expect(await streamToString(stream as unknown as NodeJS.ReadableStream)).toBe(value) + } + + expect(mockStore.fulfilled).toBeTruthy() + }) +}) diff --git a/src/main.ts b/src/main.ts index 459392d..9cbe45a 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,258 +1 @@ -import { createReadStream } from 'node:fs' -import { stat } from 'node:fs/promises' -import { Readable } from 'node:stream' - -import pMap from 'p-map' - -import { fetchAndRetry } from './retry.ts' - -interface APICredentials { - apiURL?: string - token: string -} - -interface ContextCredentials { - contextURL: string - token: string -} - -interface BlobsOptions { - authentication: APICredentials | ContextCredentials - context?: string - fetcher?: typeof globalThis.fetch - siteID: string -} - -enum HTTPMethod { - Delete = 'delete', - Get = 'get', - Put = 'put', -} - -interface SetOptions { - expiration?: Date | number -} - -interface SetFilesItem extends SetOptions { - key: string - path: string -} - -interface SetFilesOptions { - concurrency?: number -} - -type BlobInput = ReadableStream | string | ArrayBuffer | Blob - -const EXPIRY_HEADER = 'x-nf-expires-at' - -export class Blobs { - private authentication: APICredentials | ContextCredentials - private context: string - private fetcher: typeof globalThis.fetch - private siteID: string - - constructor({ authentication, context, fetcher, siteID }: BlobsOptions) { - this.context = context ?? 'production' - this.fetcher = fetcher ?? globalThis.fetch - this.siteID = siteID - - if ('contextURL' in authentication) { - this.authentication = authentication - } else { - this.authentication = { - apiURL: authentication.apiURL ?? 'https://api.netlify.com', - token: authentication.token, - } - } - - if (fetcher) { - this.fetcher = fetcher - } else if (globalThis.fetch) { - this.fetcher = globalThis.fetch - } else { - throw new Error('You must specify a fetch-compatible `fetcher` parameter when `fetch` is not available globally') - } - } - - private async getFinalRequest(key: string, method: string) { - const encodedKey = encodeURIComponent(key) - - if ('contextURL' in this.authentication) { - return { - headers: { - authorization: `Bearer ${this.authentication.token}`, - }, - url: `${this.authentication.contextURL}/${this.siteID}/${this.context}/${encodedKey}`, - } - } - - const apiURL = `${this.authentication.apiURL}/api/v1/sites/${this.siteID}/blobs/${encodedKey}?context=${this.context}` - const headers = { authorization: `Bearer ${this.authentication.token}` } - const res = await this.fetcher(apiURL, { headers, method }) - - if (res.status !== 200) { - throw new Error(`${method} operation has failed: API returned a ${res.status} response`) - } - - const { url } = await res.json() - - return { - url, - } - } - - private static getExpirationHeaders(expiration: Date | number | undefined): Record { - if (typeof expiration === 'number') { - return { - [EXPIRY_HEADER]: (Date.now() + expiration).toString(), - } - } - - if (expiration instanceof Date) { - return { - [EXPIRY_HEADER]: expiration.getTime().toString(), - } - } - - if (expiration === undefined) { - return {} - } - - throw new TypeError(`'expiration' value must be a number or a Date, ${typeof expiration} found.`) - } - - private isConfigured() { - return Boolean(this.authentication?.token) && Boolean(this.siteID) - } - - private async makeStoreRequest( - key: string, - method: HTTPMethod, - extraHeaders?: Record, - body?: BlobInput | null, - ) { - if (!this.isConfigured()) { - throw new Error("The blob store is unavailable because it's missing required configuration properties") - } - - const { headers: baseHeaders = {}, url } = await this.getFinalRequest(key, method) - const headers: Record = { - ...baseHeaders, - ...extraHeaders, - } - - if (method === HTTPMethod.Put) { - headers['cache-control'] = 'max-age=0, stale-while-revalidate=60' - } - - const options: RequestInit = { - body, - headers, - method, - } - - if (body instanceof ReadableStream) { - // @ts-expect-error Part of the spec, but not typed: - // https://fetch.spec.whatwg.org/#enumdef-requestduplex - options.duplex = 'half' - } - - const res = await fetchAndRetry(this.fetcher, url, options) - - if (res.status === 404 && method === HTTPMethod.Get) { - return null - } - - if (res.status !== 200) { - throw new Error(`${method} operation has failed: store returned a ${res.status} response`) - } - - return res - } - - async delete(key: string) { - await this.makeStoreRequest(key, HTTPMethod.Delete) - } - - async get(key: string): Promise - async get(key: string, { type }: { type: 'arrayBuffer' }): Promise - async get(key: string, { type }: { type: 'blob' }): Promise - // eslint-disable-next-line @typescript-eslint/no-explicit-any - async get(key: string, { type }: { type: 'json' }): Promise - async get(key: string, { type }: { type: 'stream' }): Promise - async get(key: string, { type }: { type: 'text' }): Promise - async get( - key: string, - options?: { type: 'arrayBuffer' | 'blob' | 'json' | 'stream' | 'text' }, - ): Promise { - const { type } = options ?? {} - const res = await this.makeStoreRequest(key, HTTPMethod.Get) - const expiration = res?.headers.get(EXPIRY_HEADER) - - if (typeof expiration === 'string') { - const expirationTS = Number.parseInt(expiration) - - if (!Number.isNaN(expirationTS) && expirationTS <= Date.now()) { - return null - } - } - - if (res === null) { - return res - } - - if (type === undefined || type === 'text') { - return res.text() - } - - if (type === 'arrayBuffer') { - return res.arrayBuffer() - } - - if (type === 'blob') { - return res.blob() - } - - if (type === 'json') { - return res.json() - } - - if (type === 'stream') { - return res.body - } - - throw new Error(`Invalid 'type' property: ${type}. Expected: arrayBuffer, blob, json, stream, or text.`) - } - - async set(key: string, data: BlobInput, { expiration }: SetOptions = {}) { - const headers = Blobs.getExpirationHeaders(expiration) - - await this.makeStoreRequest(key, HTTPMethod.Put, headers, data) - } - - async setFile(key: string, path: string, { expiration }: SetOptions = {}) { - const { size } = await stat(path) - const file = Readable.toWeb(createReadStream(path)) - const headers = { - ...Blobs.getExpirationHeaders(expiration), - 'content-length': size.toString(), - } - - await this.makeStoreRequest(key, HTTPMethod.Put, headers, file as ReadableStream) - } - - setFiles(files: SetFilesItem[], { concurrency = 5 }: SetFilesOptions = {}) { - return pMap(files, ({ key, path, ...options }) => this.setFile(key, path, options), { concurrency }) - } - - async setJSON(key: string, data: unknown, { expiration }: SetOptions = {}) { - const payload = JSON.stringify(data) - const headers = { - ...Blobs.getExpirationHeaders(expiration), - 'content-type': 'application/json', - } - - await this.makeStoreRequest(key, HTTPMethod.Put, headers, payload) - } -} +export { getStore } from './store.ts' diff --git a/src/retry.ts b/src/retry.ts index 288cd74..10b86a8 100644 --- a/src/retry.ts +++ b/src/retry.ts @@ -4,20 +4,19 @@ const MAX_RETRY = 5 const RATE_LIMIT_HEADER = 'X-RateLimit-Reset' export const fetchAndRetry = async ( - fetcher: typeof globalThis.fetch, url: string, options: RequestInit, attemptsLeft = MAX_RETRY, ): ReturnType => { try { - const res = await fetcher(url, options) + const res = await fetch(url, options) if (attemptsLeft > 0 && (res.status === 429 || res.status >= 500)) { const delay = getDelay(res.headers.get(RATE_LIMIT_HEADER)) await sleep(delay) - return fetchAndRetry(fetcher, url, options, attemptsLeft - 1) + return fetchAndRetry(url, options, attemptsLeft - 1) } return res @@ -30,7 +29,7 @@ export const fetchAndRetry = async ( await sleep(delay) - return fetchAndRetry(fetcher, url, options, attemptsLeft - 1) + return fetchAndRetry(url, options, attemptsLeft - 1) } } diff --git a/src/store.ts b/src/store.ts new file mode 100644 index 0000000..26713d0 --- /dev/null +++ b/src/store.ts @@ -0,0 +1,201 @@ +import { createReadStream } from 'node:fs' +import { stat } from 'node:fs/promises' +import { Readable } from 'node:stream' + +import pMap from 'p-map' + +import { Client, Context } from './client.js' +import { BlobInput, HTTPMethod } from './types.js' + +interface BaseStoreOptions { + client: Client +} + +interface DeployStoreOptions extends BaseStoreOptions { + deployID: string +} + +interface NamedStoreOptions extends BaseStoreOptions { + name: string +} + +type StoreOptions = DeployStoreOptions | NamedStoreOptions + +interface SetOptions { + expiration?: Date | number +} + +interface SetFilesItem extends SetOptions { + key: string + path: string +} + +interface SetFilesOptions { + concurrency?: number +} + +const EXPIRY_HEADER = 'x-nf-expires-at' + +class Store { + private client: Client + private name: string + + constructor(options: StoreOptions) { + this.client = options.client + this.name = 'deployID' in options ? `deploy:${options.deployID}` : options.name + } + + private static getExpirationHeaders(expiration: Date | number | undefined): Record { + if (typeof expiration === 'number') { + return { + [EXPIRY_HEADER]: (Date.now() + expiration).toString(), + } + } + + if (expiration instanceof Date) { + return { + [EXPIRY_HEADER]: expiration.getTime().toString(), + } + } + + if (expiration === undefined) { + return {} + } + + throw new TypeError(`'expiration' value must be a number or a Date, ${typeof expiration} found.`) + } + + async delete(key: string) { + await this.client.makeRequest({ key, method: HTTPMethod.Delete, storeName: this.name }) + } + + async get(key: string): Promise + async get(key: string, { type }: { type: 'arrayBuffer' }): Promise + async get(key: string, { type }: { type: 'blob' }): Promise + // eslint-disable-next-line @typescript-eslint/no-explicit-any + async get(key: string, { type }: { type: 'json' }): Promise + async get(key: string, { type }: { type: 'stream' }): Promise + async get(key: string, { type }: { type: 'text' }): Promise + async get( + key: string, + options?: { type: 'arrayBuffer' | 'blob' | 'json' | 'stream' | 'text' }, + ): Promise { + const { type } = options ?? {} + const res = await this.client.makeRequest({ key, method: HTTPMethod.Get, storeName: this.name }) + const expiration = res?.headers.get(EXPIRY_HEADER) + + if (typeof expiration === 'string') { + const expirationTS = Number.parseInt(expiration) + + if (!Number.isNaN(expirationTS) && expirationTS <= Date.now()) { + return null + } + } + + if (res === null) { + return res + } + + if (type === undefined || type === 'text') { + return res.text() + } + + if (type === 'arrayBuffer') { + return res.arrayBuffer() + } + + if (type === 'blob') { + return res.blob() + } + + if (type === 'json') { + return res.json() + } + + if (type === 'stream') { + return res.body + } + + throw new Error(`Invalid 'type' property: ${type}. Expected: arrayBuffer, blob, json, stream, or text.`) + } + + async set(key: string, data: BlobInput, { expiration }: SetOptions = {}) { + const headers = Store.getExpirationHeaders(expiration) + + await this.client.makeRequest({ + body: data, + headers, + key, + method: HTTPMethod.Put, + storeName: this.name, + }) + } + + async setFile(key: string, path: string, { expiration }: SetOptions = {}) { + const { size } = await stat(path) + const file = Readable.toWeb(createReadStream(path)) + const headers = { + ...Store.getExpirationHeaders(expiration), + 'content-length': size.toString(), + } + + await this.client.makeRequest({ + body: file as ReadableStream, + headers, + key, + method: HTTPMethod.Put, + storeName: this.name, + }) + } + + setFiles(files: SetFilesItem[], { concurrency = 5 }: SetFilesOptions = {}) { + return pMap(files, ({ key, path, ...options }) => this.setFile(key, path, options), { concurrency }) + } + + async setJSON(key: string, data: unknown, { expiration }: SetOptions = {}) { + const payload = JSON.stringify(data) + const headers = { + ...Store.getExpirationHeaders(expiration), + 'content-type': 'application/json', + } + + await this.client.makeRequest({ + body: payload, + headers, + key, + method: HTTPMethod.Put, + storeName: this.name, + }) + } +} + +interface GetDeployStoreOptions extends Context { + deployID: string +} + +interface GetNamedStoreOptions extends Context { + name: string +} + +export const getStore: { + (name: string): Store + (options: GetDeployStoreOptions | GetNamedStoreOptions): Store +} = (input) => { + if (typeof input === 'string') { + const client = new Client() + + return new Store({ client, name: input }) + } + + if ('deployID' in input) { + const { deployID, ...context } = input + const client = new Client(context) + + return new Store({ client, name: deployID }) + } + + const { name, ...context } = input + const client = new Client(context) + + return new Store({ client, name }) +} diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..50967a9 --- /dev/null +++ b/src/types.ts @@ -0,0 +1,7 @@ +export type BlobInput = ReadableStream | string | ArrayBuffer | Blob + +export enum HTTPMethod { + Delete = 'delete', + Get = 'get', + Put = 'put', +} From f40de5b1df821819ba38fae1dbde0ca0ba242070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Sun, 8 Oct 2023 23:06:04 +0100 Subject: [PATCH 02/16] chore: add requirements to README --- README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1cb003e..98c3356 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,12 @@ You can install `@netlify/blobs` via npm: npm install @netlify/blobs ``` +### Requirements + +- Deno 1.30 and above or Node.js 16.0.0 and above +- `fetch` in the global scope with a [fetch-compatible](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) + interface + ## Usage To start reading and writing data, you must first get a reference to a store using the `getStore` method. @@ -29,9 +35,8 @@ Create a store for API access by calling `getStore` with the following parameter - `name` (string): Name of the store - `siteID` (string): ID of the Netlify site -- `token` (string): [Personal access - token]([your Netlify API credentials](https://docs.netlify.com/api/get-started/#authentication)) to access the Netlify - API +- `token` (string): [Personal access token](https://docs.netlify.com/api/get-started/#authentication) to access the + Netlify API - `apiURL` (string): URL of the Netlify API (optional, defaults to `https://api.netlify.com`) ```ts @@ -56,9 +61,8 @@ Create a store for edge access by calling `getStore` with the following paramete - `name` (string): Name of the store - `siteID` (string): ID of the Netlify site -- `token` (string): [Personal access - token]([your Netlify API credentials](https://docs.netlify.com/api/get-started/#authentication)) to access the Netlify - API +- `token` (string): [Personal access token](https://docs.netlify.com/api/get-started/#authentication) to access the + Netlify API - `edgeURL` (string): URL of the edge endpoint ```ts From 84aae508dd04801c1e463196eebcec99415590a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Sun, 8 Oct 2023 23:08:07 +0100 Subject: [PATCH 03/16] chore: update heading --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 98c3356..760e34b 100644 --- a/README.md +++ b/README.md @@ -146,7 +146,7 @@ const store2 = getStore({ assert.equal(await store2.get('my-key'), 'my value') ``` -## Store API +## Store API reference ### `get(key: string, { type: string }): Promise` From 7fe28c54ee03031d8b4430362647fe0b7f7d0608 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Mon, 9 Oct 2023 10:43:32 +0100 Subject: [PATCH 04/16] chore: update README --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 760e34b..1bc428f 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,9 @@ with a Base64-encoded, JSON-stringified representation of an object with the fol - `token`: Access token for the corresponding access mode - `siteID`: ID of the Netlify site +This environment variable is automatically populated by Netlify in the execution environment for both serverless and +edge functions. + With this in place, the `getStore` method can be called just with the store name. No configuration object is required, since it'll be read from the environment. From 4fc010e721ebc203365a84f2c0e91d6c0b47719c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Mon, 9 Oct 2023 10:44:39 +0100 Subject: [PATCH 05/16] chore: words --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 1bc428f..61abdba 100644 --- a/README.md +++ b/README.md @@ -103,8 +103,7 @@ with a Base64-encoded, JSON-stringified representation of an object with the fol - `token`: Access token for the corresponding access mode - `siteID`: ID of the Netlify site -This environment variable is automatically populated by Netlify in the execution environment for both serverless and -edge functions. +This data is automatically populated by Netlify in the execution environment for both serverless and edge functions. With this in place, the `getStore` method can be called just with the store name. No configuration object is required, since it'll be read from the environment. From 8abdfc5a8ac55cc74fd77b11cde3128afc2ec58e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Thu, 12 Oct 2023 15:09:21 +0100 Subject: [PATCH 06/16] refactor: update variable name --- README.md | 5 +- src/client.ts | 2 +- src/main.test.ts | 116 +++++++++++++++++++++++++++++++++++------------ src/store.ts | 5 +- 4 files changed, 93 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 61abdba..7248e31 100644 --- a/README.md +++ b/README.md @@ -95,8 +95,9 @@ Rather than explicitly passing the configuration context to the `getStore` metho environment. This is particularly useful for setups where the configuration data is held by one system and the data needs to be accessed in another system, with no direct communication between the two. -To do this, the system that holds the configuration data should set an environment variable called `NETLIFY_BLOBS_1` -with a Base64-encoded, JSON-stringified representation of an object with the following properties: +To do this, the system that holds the configuration data should set an environment variable called +`NETLIFY_BLOBS_CONTEXT` with a Base64-encoded, JSON-stringified representation of an object with the following +properties: - `apiURL` (optional) or `edgeURL`: URL of the Netlify API (for [API access](#api-access)) or the edge endpoint (for [Edge access](#edge-access)) diff --git a/src/client.ts b/src/client.ts index ce71235..13abdad 100644 --- a/src/client.ts +++ b/src/client.ts @@ -9,7 +9,7 @@ import { BlobInput, HTTPMethod } from './types.ts' // of this object, we should bump the version and create a new variable, so // that the client knows how to consume the data and can advise the user to // update the client if needed. -export const NETLIFY_CONTEXT_VARIABLE = 'NETLIFY_BLOBS_1' +export const NETLIFY_CONTEXT_VARIABLE = 'NETLIFY_BLOBS_CONTEXT' export interface Context { apiURL?: string diff --git a/src/main.test.ts b/src/main.test.ts index 7601dae..4359af5 100644 --- a/src/main.test.ts +++ b/src/main.test.ts @@ -4,7 +4,7 @@ import { env, version as nodeVersion } from 'node:process' import semver from 'semver' import tmp from 'tmp-promise' -import { describe, test, expect, beforeAll } from 'vitest' +import { describe, test, expect, beforeAll, afterEach } from 'vitest' import { MockFetch } from '../test/mock_fetch.js' import { streamToString } from '../test/util.js' @@ -26,8 +26,12 @@ beforeAll(async () => { } }) -const deployID = 'abcdef' -const siteID = '12345' +afterEach(() => { + delete env.NETLIFY_BLOBS_CONTEXT +}) + +const deployID = '6527dfab35be400008332a1d' +const siteID = '9a003659-aaaa-0000-aaaa-63d3720d8621' const key = '54321' const complexKey = '/artista/canção' const value = 'some value' @@ -191,7 +195,7 @@ describe('get', () => { }) }) - describe('With context credentials', () => { + describe('With edge credentials', () => { test('Reads from the blob store', async () => { const mockStore = new MockFetch() .get({ @@ -265,6 +269,53 @@ describe('get', () => { expect(mockStore.fulfilled).toBeTruthy() }) + + test('Loads credentials from the environment', async () => { + const tokens = ['some-token-1', 'another-token-2'] + const mockStore = new MockFetch() + .get({ + headers: { authorization: `Bearer ${tokens[0]}` }, + response: new Response(value), + url: `${edgeURL}/${siteID}/images/${key}`, + }) + .get({ + headers: { authorization: `Bearer ${tokens[0]}` }, + response: new Response(value), + url: `${edgeURL}/${siteID}/images/${key}`, + }) + .get({ + headers: { authorization: `Bearer ${tokens[1]}` }, + response: new Response(value), + url: `${edgeURL}/${siteID}/images/${key}`, + }) + .get({ + headers: { authorization: `Bearer ${tokens[1]}` }, + response: new Response(value), + url: `${edgeURL}/${siteID}/images/${key}`, + }) + + globalThis.fetch = mockStore.fetcher + + for (let index = 0; index <= 1; index++) { + const context = { + edgeURL, + siteID, + token: tokens[index], + } + + env.NETLIFY_BLOBS_CONTEXT = Buffer.from(JSON.stringify(context)).toString('base64') + + const store = getStore('images') + + const string = await store.get(key) + expect(string).toBe(value) + + const stream = await store.get(key, { type: 'stream' }) + expect(await streamToString(stream as unknown as NodeJS.ReadableStream)).toBe(value) + } + + expect(mockStore.fulfilled).toBeTruthy() + }) }) test('Throws when the instance is missing required configuration properties', async () => { @@ -560,7 +611,7 @@ describe('set', () => { }) }) - describe('With context credentials', () => { + describe('With edge credentials', () => { test('Writes to the blob store', async () => { const mockStore = new MockFetch() .put({ @@ -712,7 +763,7 @@ describe('setJSON', () => { }) }) - describe('With context credentials', () => { + describe('With edge credentials', () => { test('Writes to the blob store', async () => { const mockStore = new MockFetch().put({ body: JSON.stringify({ value }), @@ -829,7 +880,7 @@ describe('delete', () => { }) }) - describe('With context credentials', () => { + describe('With edge credentials', () => { test('Deletes from the blob store', async () => { const mockStore = new MockFetch().delete({ headers: { authorization: `Bearer ${edgeToken}` }, @@ -898,51 +949,56 @@ describe('delete', () => { }) }) -describe('Global client', () => { - test('Reads from the blob store', async () => { - const tokens = ['some-token-1', 'another-token-2'] +describe.only('Deploy scope', () => { + test('Returns a deploy-scoped store if the `deployID` parameter is supplied', async () => { + const mockToken = 'some-token' const mockStore = new MockFetch() .get({ - headers: { authorization: `Bearer ${tokens[0]}` }, + headers: { authorization: `Bearer ${mockToken}` }, response: new Response(value), url: `${edgeURL}/${siteID}/images/${key}`, }) .get({ - headers: { authorization: `Bearer ${tokens[0]}` }, + headers: { authorization: `Bearer ${mockToken}` }, response: new Response(value), url: `${edgeURL}/${siteID}/images/${key}`, }) .get({ - headers: { authorization: `Bearer ${tokens[1]}` }, + headers: { authorization: `Bearer ${mockToken}` }, response: new Response(value), - url: `${edgeURL}/${siteID}/images/${key}`, + url: `${edgeURL}/${siteID}/${deployID}/${key}`, }) .get({ - headers: { authorization: `Bearer ${tokens[1]}` }, + headers: { authorization: `Bearer ${mockToken}` }, response: new Response(value), - url: `${edgeURL}/${siteID}/images/${key}`, + url: `${edgeURL}/${siteID}/${deployID}/${key}`, }) globalThis.fetch = mockStore.fetcher - for (let index = 0; index <= 1; index++) { - const context = { - edgeURL, - deployID, - siteID, - token: tokens[index], - } + const context = { + edgeURL, + siteID, + token: mockToken, + } - env.NETLIFY_BLOBS_1 = Buffer.from(JSON.stringify(context)).toString('base64') + env.NETLIFY_BLOBS_CONTEXT = Buffer.from(JSON.stringify(context)).toString('base64') - const store = getStore('images') + const siteStore = getStore('images') - const string = await store.get(key) - expect(string).toBe(value) + const string1 = await siteStore.get(key) + expect(string1).toBe(value) - const stream = await store.get(key, { type: 'stream' }) - expect(await streamToString(stream as unknown as NodeJS.ReadableStream)).toBe(value) - } + const stream1 = await siteStore.get(key, { type: 'stream' }) + expect(await streamToString(stream1 as unknown as NodeJS.ReadableStream)).toBe(value) + + const deployStore = getStore({ deployID }) + + const string2 = await deployStore.get(key) + expect(string2).toBe(value) + + const stream2 = await deployStore.get(key, { type: 'stream' }) + expect(await streamToString(stream2 as unknown as NodeJS.ReadableStream)).toBe(value) expect(mockStore.fulfilled).toBeTruthy() }) diff --git a/src/store.ts b/src/store.ts index 26713d0..483195d 100644 --- a/src/store.ts +++ b/src/store.ts @@ -179,7 +179,7 @@ interface GetNamedStoreOptions extends Context { export const getStore: { (name: string): Store - (options: GetDeployStoreOptions | GetNamedStoreOptions): Store + (options: GetDeployStoreOptions | GetNamedStoreOptions | { deployID: string }): Store } = (input) => { if (typeof input === 'string') { const client = new Client() @@ -188,7 +188,8 @@ export const getStore: { } if ('deployID' in input) { - const { deployID, ...context } = input + const { deployID, ...otherProps } = input + const context = 'siteID' in otherProps && 'token' in otherProps ? otherProps : undefined const client = new Client(context) return new Store({ client, name: deployID }) From bca8ebed9ae2489a6298a8d3586c4a9ed86717aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Thu, 12 Oct 2023 15:11:48 +0100 Subject: [PATCH 07/16] chore: reenable tests --- src/main.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.test.ts b/src/main.test.ts index 4359af5..48f4c24 100644 --- a/src/main.test.ts +++ b/src/main.test.ts @@ -949,7 +949,7 @@ describe('delete', () => { }) }) -describe.only('Deploy scope', () => { +describe('Deploy scope', () => { test('Returns a deploy-scoped store if the `deployID` parameter is supplied', async () => { const mockToken = 'some-token' const mockStore = new MockFetch() From 043a339145f0a99cb167239e23c1b6cac977f19b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Thu, 12 Oct 2023 19:17:48 +0100 Subject: [PATCH 08/16] feat: add back `fetcher` --- .eslintrc.cjs | 1 + README.md | 20 +++++++++++++++++++ src/client.ts | 51 +++++++++++++++++++++++++++++++----------------- src/main.test.ts | 29 +++++++++++++++++++++++++++ src/retry.ts | 9 ++++++--- src/store.ts | 33 ++++++++++++++++--------------- src/types.ts | 2 ++ 7 files changed, 108 insertions(+), 37 deletions(-) diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 04799d4..66cc2ba 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -14,6 +14,7 @@ module.exports = { 'max-lines-per-function': 'off', 'max-statements': 'off', 'node/no-missing-import': 'off', + 'import/no-unresolved': 'off', 'n/no-missing-import': 'off', 'no-magic-numbers': 'off', 'no-shadow': 'off', diff --git a/README.md b/README.md index 7248e31..442560a 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,26 @@ const store2 = getStore({ assert.equal(await store2.get('my-key'), 'my value') ``` +### Custom fetcher + +The client uses [the web platform `fetch()`](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) to make HTTP +calls. By default, it will use any globally-defined instance of `fetch`, but you can choose to provide your own. + +You can do this by supplying a `fetcher` property to the `getStore` method. + +```ts +import { fetch } from 'whatwg-fetch' + +import { getStore } from '@netlify/blobs' + +const store = getStore({ + fetcher: fetch, + name: 'my-store', +}) + +console.log(await store.get('my-key')) +``` + ## Store API reference ### `get(key: string, { type: string }): Promise` diff --git a/src/client.ts b/src/client.ts index 13abdad..f984ceb 100644 --- a/src/client.ts +++ b/src/client.ts @@ -2,7 +2,7 @@ import { Buffer } from 'node:buffer' import { env } from 'node:process' import { fetchAndRetry } from './retry.ts' -import { BlobInput, HTTPMethod } from './types.ts' +import { BlobInput, Fetcher, HTTPMethod } from './types.ts' // The name of the environment variable that holds the context in a Base64, // JSON-encoded object. If we ever need to change the encoding or the shape @@ -14,6 +14,11 @@ export const NETLIFY_CONTEXT_VARIABLE = 'NETLIFY_BLOBS_CONTEXT' export interface Context { apiURL?: string edgeURL?: string + siteID?: string + token?: string +} + +interface PopulatedContext extends Context { siteID: string token: string } @@ -28,16 +33,16 @@ interface MakeStoreRequestOptions { export class Client { private context?: Context + private fetcher?: Fetcher - constructor(context?: Context) { - this.context = context - } + constructor(context?: Context, fetcher?: Fetcher) { + this.context = context ?? {} + this.fetcher = fetcher - private getContext() { - if (this.context) { - return this.context - } + console.log('---> FETCHER', this.fetcher) + } + private static getEnvironmentContext() { if (!env[NETLIFY_CONTEXT_VARIABLE]) { return } @@ -51,7 +56,21 @@ export class Client { } } - private static async getFinalRequest(context: Context, storeName: string, key: string, method: string) { + private getContext() { + const context = { + ...Client.getEnvironmentContext(), + ...this.context, + } + + if (!context.siteID || !context.token) { + throw new Error(`The blob store is unavailable because it's missing required configuration properties`) + } + + return context as PopulatedContext + } + + private async getFinalRequest(storeName: string, key: string, method: string) { + const context = this.getContext() const encodedKey = encodeURIComponent(key) if ('edgeURL' in context) { @@ -67,7 +86,8 @@ export class Client { context.siteID }/blobs/${encodedKey}?context=${storeName}` const headers = { authorization: `Bearer ${context.token}` } - const res = await fetch(apiURL, { headers, method }) + const fetcher = this.fetcher ?? globalThis.fetch + const res = await fetcher(apiURL, { headers, method }) if (res.status !== 200) { throw new Error(`${method} operation has failed: API returned a ${res.status} response`) @@ -81,13 +101,7 @@ export class Client { } async makeRequest({ body, headers: extraHeaders, key, method, storeName }: MakeStoreRequestOptions) { - const context = this.getContext() - - if (!context || !context.token || !context.siteID) { - throw new Error("The blob store is unavailable because it's missing required configuration properties") - } - - const { headers: baseHeaders = {}, url } = await Client.getFinalRequest(context, storeName, key, method) + const { headers: baseHeaders = {}, url } = await this.getFinalRequest(storeName, key, method) const headers: Record = { ...baseHeaders, ...extraHeaders, @@ -109,7 +123,8 @@ export class Client { options.duplex = 'half' } - const res = await fetchAndRetry(url, options) + const fetcher = this.fetcher ?? globalThis.fetch + const res = await fetchAndRetry(fetcher, url, options) if (res.status === 404 && method === HTTPMethod.Get) { return null diff --git a/src/main.test.ts b/src/main.test.ts index 48f4c24..cc370a5 100644 --- a/src/main.test.ts +++ b/src/main.test.ts @@ -1003,3 +1003,32 @@ describe('Deploy scope', () => { expect(mockStore.fulfilled).toBeTruthy() }) }) + +describe('Customer fetcher', () => { + test('Uses a custom implementation of `fetch` if the `fetcher` parameter is supplied', async () => { + globalThis.fetch = () => { + throw new Error('I should not be called') + } + + const mockToken = 'some-token' + const mockStore = new MockFetch().get({ + headers: { authorization: `Bearer ${mockToken}` }, + response: new Response(value), + url: `${edgeURL}/${siteID}/images/${key}`, + }) + const context = { + edgeURL, + siteID, + token: mockToken, + } + + env.NETLIFY_BLOBS_CONTEXT = Buffer.from(JSON.stringify(context)).toString('base64') + + const store = getStore({ fetcher: mockStore.fetcher, name: 'images' }) + + const string = await store.get(key) + expect(string).toBe(value) + + expect(mockStore.fulfilled).toBeTruthy() + }) +}) diff --git a/src/retry.ts b/src/retry.ts index 10b86a8..e52c76e 100644 --- a/src/retry.ts +++ b/src/retry.ts @@ -1,22 +1,25 @@ +import type { Fetcher } from './types.ts' + const DEFAULT_RETRY_DELAY = 5000 const MIN_RETRY_DELAY = 1000 const MAX_RETRY = 5 const RATE_LIMIT_HEADER = 'X-RateLimit-Reset' export const fetchAndRetry = async ( + fetcher: Fetcher, url: string, options: RequestInit, attemptsLeft = MAX_RETRY, ): ReturnType => { try { - const res = await fetch(url, options) + const res = await fetcher(url, options) if (attemptsLeft > 0 && (res.status === 429 || res.status >= 500)) { const delay = getDelay(res.headers.get(RATE_LIMIT_HEADER)) await sleep(delay) - return fetchAndRetry(url, options, attemptsLeft - 1) + return fetchAndRetry(fetcher, url, options, attemptsLeft - 1) } return res @@ -29,7 +32,7 @@ export const fetchAndRetry = async ( await sleep(delay) - return fetchAndRetry(url, options, attemptsLeft - 1) + return fetchAndRetry(fetcher, url, options, attemptsLeft - 1) } } diff --git a/src/store.ts b/src/store.ts index 483195d..385f967 100644 --- a/src/store.ts +++ b/src/store.ts @@ -5,7 +5,7 @@ import { Readable } from 'node:stream' import pMap from 'p-map' import { Client, Context } from './client.js' -import { BlobInput, HTTPMethod } from './types.js' +import { BlobInput, Fetcher, HTTPMethod } from './types.js' interface BaseStoreOptions { client: Client @@ -169,17 +169,15 @@ class Store { } } -interface GetDeployStoreOptions extends Context { - deployID: string -} - -interface GetNamedStoreOptions extends Context { - name: string +interface GetStoreOptions extends Context { + deployID?: string + fetcher?: Fetcher + name?: string } export const getStore: { (name: string): Store - (options: GetDeployStoreOptions | GetNamedStoreOptions | { deployID: string }): Store + (options: GetStoreOptions): Store } = (input) => { if (typeof input === 'string') { const client = new Client() @@ -187,16 +185,19 @@ export const getStore: { return new Store({ client, name: input }) } - if ('deployID' in input) { - const { deployID, ...otherProps } = input - const context = 'siteID' in otherProps && 'token' in otherProps ? otherProps : undefined - const client = new Client(context) + if (typeof input.name === 'string') { + const { fetcher, name, ...context } = input + const client = new Client(context, fetcher) - return new Store({ client, name: deployID }) + return new Store({ client, name }) } - const { name, ...context } = input - const client = new Client(context) + if (typeof input.deployID === 'string') { + const { fetcher, deployID, ...context } = input + const client = new Client(context, fetcher) + + return new Store({ client, name: deployID }) + } - return new Store({ client, name }) + throw new Error('`getStore()` requires a `name` or `siteID` properties.') } diff --git a/src/types.ts b/src/types.ts index 50967a9..f0f6277 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,7 @@ export type BlobInput = ReadableStream | string | ArrayBuffer | Blob +export type Fetcher = typeof globalThis.fetch + export enum HTTPMethod { Delete = 'delete', Get = 'get', From 9cbbab8c5efb86566793cddf05362fe0dfd0e221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Thu, 12 Oct 2023 19:21:55 +0100 Subject: [PATCH 09/16] refactor: remove `setFile` and `setFiles` --- package-lock.json | 59 ---------------------- package.json | 4 -- src/main.test.ts | 123 ---------------------------------------------- src/store.ts | 36 -------------- 4 files changed, 222 deletions(-) diff --git a/package-lock.json b/package-lock.json index c003fb3..710e715 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,9 +8,6 @@ "name": "@netlify/blobs", "version": "2.0.0", "license": "MIT", - "dependencies": { - "p-map": "^6.0.0" - }, "devDependencies": { "@commitlint/cli": "^17.0.0", "@commitlint/config-conventional": "^17.0.0", @@ -20,7 +17,6 @@ "husky": "^8.0.0", "node-fetch": "^3.3.1", "semver": "^7.5.3", - "tmp-promise": "^3.0.3", "typescript": "^5.0.0", "vitest": "^0.34.0" }, @@ -6292,17 +6288,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-map": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-6.0.0.tgz", - "integrity": "sha512-T8BatKGY+k5rU+Q/GTYgrEf2r4xRMevAN5mtXc2aPc4rS1j3s+vWTaO2Wag94neXuCAUAs8cxBL9EeB5EA6diw==", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -7461,27 +7446,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", - "dev": true, - "dependencies": { - "rimraf": "^3.0.0" - }, - "engines": { - "node": ">=8.17.0" - } - }, - "node_modules/tmp-promise": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", - "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", - "dev": true, - "dependencies": { - "tmp": "^0.2.0" - } - }, "node_modules/to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -12980,11 +12944,6 @@ "p-limit": "^3.0.2" } }, - "p-map": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-6.0.0.tgz", - "integrity": "sha512-T8BatKGY+k5rU+Q/GTYgrEf2r4xRMevAN5mtXc2aPc4rS1j3s+vWTaO2Wag94neXuCAUAs8cxBL9EeB5EA6diw==" - }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -13807,24 +13766,6 @@ "integrity": "sha512-KxVu8EYHDPBdUYdKZdKtU2aj2XfEx9AfjXxE/Aj0vT06w2icA09Vus1rh6eSu1y01akYg6BjIK/hxyLJINoMLQ==", "dev": true }, - "tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", - "dev": true, - "requires": { - "rimraf": "^3.0.0" - } - }, - "tmp-promise": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", - "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", - "dev": true, - "requires": { - "tmp": "^0.2.0" - } - }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", diff --git a/package.json b/package.json index dbda329..866db8e 100644 --- a/package.json +++ b/package.json @@ -56,14 +56,10 @@ "husky": "^8.0.0", "node-fetch": "^3.3.1", "semver": "^7.5.3", - "tmp-promise": "^3.0.3", "typescript": "^5.0.0", "vitest": "^0.34.0" }, "engines": { "node": "^14.16.0 || >=16.0.0" - }, - "dependencies": { - "p-map": "^6.0.0" } } diff --git a/src/main.test.ts b/src/main.test.ts index cc370a5..b54d043 100644 --- a/src/main.test.ts +++ b/src/main.test.ts @@ -1,9 +1,7 @@ import { Buffer } from 'node:buffer' -import { writeFile } from 'node:fs/promises' import { env, version as nodeVersion } from 'node:process' import semver from 'semver' -import tmp from 'tmp-promise' import { describe, test, expect, beforeAll, afterEach } from 'vitest' import { MockFetch } from '../test/mock_fetch.js' @@ -415,127 +413,6 @@ describe('set', () => { expect(mockStore.fulfilled).toBeTruthy() }) - // We need `Readable.toWeb` to be available, which needs Node 16+. - if (semver.gte(nodeVersion, '16.0.0')) { - test('Accepts a file', async () => { - const fileContents = 'Hello from a file' - const mockStore = new MockFetch() - .put({ - headers: { authorization: `Bearer ${apiToken}` }, - response: new Response(JSON.stringify({ url: signedURL })), - url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`, - }) - .put({ - body: async (body) => { - expect(await streamToString(body as unknown as NodeJS.ReadableStream)).toBe(fileContents) - }, - headers: { - 'cache-control': 'max-age=0, stale-while-revalidate=60', - }, - response: new Response(null), - url: signedURL, - }) - - globalThis.fetch = mockStore.fetcher - - const { cleanup, path } = await tmp.file() - - await writeFile(path, fileContents) - - const blobs = getStore({ - name: 'production', - token: apiToken, - siteID, - }) - - await blobs.setFile(key, path) - - expect(mockStore.fulfilled).toBeTruthy() - - await cleanup() - }) - - test('Accepts multiple files concurrently', async () => { - const contents = ['Hello from key-0', 'Hello from key-1', 'Hello from key-2'] - const signedURLs = ['https://signed-url.aws/0', 'https://signed-url.aws/1', 'https://signed-url.aws/2'] - - const mockStore = new MockFetch() - .put({ - headers: { authorization: `Bearer ${apiToken}` }, - response: new Response(JSON.stringify({ url: signedURLs[0] })), - url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/key-0?context=production`, - }) - .put({ - body: async (body) => { - expect(await streamToString(body as unknown as NodeJS.ReadableStream)).toBe(contents[0]) - }, - headers: { - 'cache-control': 'max-age=0, stale-while-revalidate=60', - }, - response: new Response(null), - url: signedURLs[0], - }) - .put({ - headers: { authorization: `Bearer ${apiToken}` }, - response: new Response(JSON.stringify({ url: signedURLs[1] })), - url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/key-1?context=production`, - }) - .put({ - body: async (body) => { - expect(await streamToString(body as unknown as NodeJS.ReadableStream)).toBe(contents[1]) - }, - headers: { - 'cache-control': 'max-age=0, stale-while-revalidate=60', - }, - response: new Response(null), - url: signedURLs[1], - }) - .put({ - headers: { authorization: `Bearer ${apiToken}` }, - response: new Response(JSON.stringify({ url: signedURLs[2] })), - url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/key-2?context=production`, - }) - .put({ - body: async (body) => { - expect(await streamToString(body as unknown as NodeJS.ReadableStream)).toBe(contents[2]) - }, - headers: { - 'cache-control': 'max-age=0, stale-while-revalidate=60', - }, - response: new Response(null), - url: signedURLs[2], - }) - - globalThis.fetch = mockStore.fetcher - - const writes = await Promise.all( - contents.map(async (content) => { - const { cleanup, path } = await tmp.file() - - await writeFile(path, content) - - return { cleanup, path } - }), - ) - const files = writes.map(({ path }, idx) => ({ - key: `key-${idx}`, - path, - })) - - const blobs = getStore({ - name: 'production', - token: apiToken, - siteID, - }) - - await blobs.setFiles(files) - - expect(mockStore.fulfilled).toBeTruthy() - - await Promise.all(writes.map(({ cleanup }) => cleanup())) - }) - } - test('Throws when the API returns a non-200 status code', async () => { const mockStore = new MockFetch().put({ headers: { authorization: `Bearer ${apiToken}` }, diff --git a/src/store.ts b/src/store.ts index 385f967..8a91b3d 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1,9 +1,3 @@ -import { createReadStream } from 'node:fs' -import { stat } from 'node:fs/promises' -import { Readable } from 'node:stream' - -import pMap from 'p-map' - import { Client, Context } from './client.js' import { BlobInput, Fetcher, HTTPMethod } from './types.js' @@ -25,15 +19,6 @@ interface SetOptions { expiration?: Date | number } -interface SetFilesItem extends SetOptions { - key: string - path: string -} - -interface SetFilesOptions { - concurrency?: number -} - const EXPIRY_HEADER = 'x-nf-expires-at' class Store { @@ -131,27 +116,6 @@ class Store { }) } - async setFile(key: string, path: string, { expiration }: SetOptions = {}) { - const { size } = await stat(path) - const file = Readable.toWeb(createReadStream(path)) - const headers = { - ...Store.getExpirationHeaders(expiration), - 'content-length': size.toString(), - } - - await this.client.makeRequest({ - body: file as ReadableStream, - headers, - key, - method: HTTPMethod.Put, - storeName: this.name, - }) - } - - setFiles(files: SetFilesItem[], { concurrency = 5 }: SetFilesOptions = {}) { - return pMap(files, ({ key, path, ...options }) => this.setFile(key, path, options), { concurrency }) - } - async setJSON(key: string, data: unknown, { expiration }: SetOptions = {}) { const payload = JSON.stringify(data) const headers = { From 474454c2556ad5ff7abc97960151e4ee45cadcaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Thu, 12 Oct 2023 20:48:15 +0100 Subject: [PATCH 10/16] chore: remove unused deps --- package-lock.json | 59 ----------------------------------------------- package.json | 2 -- 2 files changed, 61 deletions(-) diff --git a/package-lock.json b/package-lock.json index 954222b..01f9c27 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,9 +8,6 @@ "name": "@netlify/blobs", "version": "2.1.0", "license": "MIT", - "dependencies": { - "p-map": "^6.0.0" - }, "devDependencies": { "@commitlint/cli": "^17.0.0", "@commitlint/config-conventional": "^17.0.0", @@ -20,7 +17,6 @@ "husky": "^8.0.0", "node-fetch": "^3.3.1", "semver": "^7.5.3", - "tmp-promise": "^3.0.3", "tsup": "^7.2.0", "typescript": "^5.0.0", "vitest": "^0.34.0" @@ -6452,17 +6448,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-map": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-6.0.0.tgz", - "integrity": "sha512-T8BatKGY+k5rU+Q/GTYgrEf2r4xRMevAN5mtXc2aPc4rS1j3s+vWTaO2Wag94neXuCAUAs8cxBL9EeB5EA6diw==", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -7746,27 +7731,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", - "dev": true, - "dependencies": { - "rimraf": "^3.0.0" - }, - "engines": { - "node": ">=8.17.0" - } - }, - "node_modules/tmp-promise": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", - "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", - "dev": true, - "dependencies": { - "tmp": "^0.2.0" - } - }, "node_modules/to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -13866,11 +13830,6 @@ "p-limit": "^3.0.2" } }, - "p-map": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-6.0.0.tgz", - "integrity": "sha512-T8BatKGY+k5rU+Q/GTYgrEf2r4xRMevAN5mtXc2aPc4rS1j3s+vWTaO2Wag94neXuCAUAs8cxBL9EeB5EA6diw==" - }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -14776,24 +14735,6 @@ "integrity": "sha512-KxVu8EYHDPBdUYdKZdKtU2aj2XfEx9AfjXxE/Aj0vT06w2icA09Vus1rh6eSu1y01akYg6BjIK/hxyLJINoMLQ==", "dev": true }, - "tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", - "dev": true, - "requires": { - "rimraf": "^3.0.0" - } - }, - "tmp-promise": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", - "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", - "dev": true, - "requires": { - "tmp": "^0.2.0" - } - }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", diff --git a/package.json b/package.json index 987e094..84404e8 100644 --- a/package.json +++ b/package.json @@ -73,9 +73,7 @@ "esbuild": "^0.19.0", "husky": "^8.0.0", "node-fetch": "^3.3.1", - "p-map": "^6.0.0", "semver": "^7.5.3", - "tmp-promise": "^3.0.3", "tsup": "^7.2.0", "typescript": "^5.0.0", "vitest": "^0.34.0" From 9c1db4a7acf0995d1a1a35be79aed28d8d4d3fa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Thu, 12 Oct 2023 21:07:48 +0100 Subject: [PATCH 11/16] refactor: rewrite imports --- src/client.ts | 2 -- src/store.ts | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/client.ts b/src/client.ts index f984ceb..a1ec97d 100644 --- a/src/client.ts +++ b/src/client.ts @@ -38,8 +38,6 @@ export class Client { constructor(context?: Context, fetcher?: Fetcher) { this.context = context ?? {} this.fetcher = fetcher - - console.log('---> FETCHER', this.fetcher) } private static getEnvironmentContext() { diff --git a/src/store.ts b/src/store.ts index 8a91b3d..028611c 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1,5 +1,5 @@ -import { Client, Context } from './client.js' -import { BlobInput, Fetcher, HTTPMethod } from './types.js' +import { Client, Context } from './client.ts' +import { BlobInput, Fetcher, HTTPMethod } from './types.ts' interface BaseStoreOptions { client: Client From 7bbe0fabdda97e226682f58521b837c129944c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Thu, 12 Oct 2023 21:12:51 +0100 Subject: [PATCH 12/16] refactor: add comment --- src/store.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/store.ts b/src/store.ts index 028611c..afcd462 100644 --- a/src/store.ts +++ b/src/store.ts @@ -16,6 +16,10 @@ interface NamedStoreOptions extends BaseStoreOptions { type StoreOptions = DeployStoreOptions | NamedStoreOptions interface SetOptions { + /** + * Accepts an absolute date as a `Date` object, or a relative date as the + * number of seconds from the current date. + */ expiration?: Date | number } From d69f97721d192cf048eb0ae32beb06a5167e027b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Fri, 13 Oct 2023 11:07:17 +0100 Subject: [PATCH 13/16] refactor: update enum --- src/client.ts | 4 ++-- src/store.ts | 8 ++++---- src/types.ts | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/client.ts b/src/client.ts index a1ec97d..7bf8548 100644 --- a/src/client.ts +++ b/src/client.ts @@ -105,7 +105,7 @@ export class Client { ...extraHeaders, } - if (method === HTTPMethod.Put) { + if (method === HTTPMethod.PUT) { headers['cache-control'] = 'max-age=0, stale-while-revalidate=60' } @@ -124,7 +124,7 @@ export class Client { const fetcher = this.fetcher ?? globalThis.fetch const res = await fetchAndRetry(fetcher, url, options) - if (res.status === 404 && method === HTTPMethod.Get) { + if (res.status === 404 && method === HTTPMethod.GET) { return null } diff --git a/src/store.ts b/src/store.ts index afcd462..6ac373d 100644 --- a/src/store.ts +++ b/src/store.ts @@ -55,7 +55,7 @@ class Store { } async delete(key: string) { - await this.client.makeRequest({ key, method: HTTPMethod.Delete, storeName: this.name }) + await this.client.makeRequest({ key, method: HTTPMethod.DELETE, storeName: this.name }) } async get(key: string): Promise @@ -70,7 +70,7 @@ class Store { options?: { type: 'arrayBuffer' | 'blob' | 'json' | 'stream' | 'text' }, ): Promise { const { type } = options ?? {} - const res = await this.client.makeRequest({ key, method: HTTPMethod.Get, storeName: this.name }) + const res = await this.client.makeRequest({ key, method: HTTPMethod.GET, storeName: this.name }) const expiration = res?.headers.get(EXPIRY_HEADER) if (typeof expiration === 'string') { @@ -115,7 +115,7 @@ class Store { body: data, headers, key, - method: HTTPMethod.Put, + method: HTTPMethod.PUT, storeName: this.name, }) } @@ -131,7 +131,7 @@ class Store { body: payload, headers, key, - method: HTTPMethod.Put, + method: HTTPMethod.PUT, storeName: this.name, }) } diff --git a/src/types.ts b/src/types.ts index f0f6277..d8c45b9 100644 --- a/src/types.ts +++ b/src/types.ts @@ -3,7 +3,7 @@ export type BlobInput = ReadableStream | string | ArrayBuffer | Blob export type Fetcher = typeof globalThis.fetch export enum HTTPMethod { - Delete = 'delete', - Get = 'get', - Put = 'put', + DELETE = 'delete', + GET = 'get', + PUT = 'put', } From 479af4eafc1483450ddbf9e12e144650faec831a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Tue, 17 Oct 2023 12:38:05 +0100 Subject: [PATCH 14/16] chore: update lock file --- package-lock.json | 59 ----------------------------------------------- 1 file changed, 59 deletions(-) diff --git a/package-lock.json b/package-lock.json index 73fd612..eb13aff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,9 +16,7 @@ "esbuild": "^0.19.0", "husky": "^8.0.0", "node-fetch": "^3.3.1", - "p-map": "^6.0.0", "semver": "^7.5.3", - "tmp-promise": "^3.0.3", "tsup": "^7.2.0", "typescript": "^5.0.0", "vitest": "^0.34.0" @@ -6450,18 +6448,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/p-map": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-6.0.0.tgz", - "integrity": "sha512-T8BatKGY+k5rU+Q/GTYgrEf2r4xRMevAN5mtXc2aPc4rS1j3s+vWTaO2Wag94neXuCAUAs8cxBL9EeB5EA6diw==", - "dev": true, - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -7745,27 +7731,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", - "dev": true, - "dependencies": { - "rimraf": "^3.0.0" - }, - "engines": { - "node": ">=8.17.0" - } - }, - "node_modules/tmp-promise": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", - "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", - "dev": true, - "dependencies": { - "tmp": "^0.2.0" - } - }, "node_modules/to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -13865,12 +13830,6 @@ "p-limit": "^3.0.2" } }, - "p-map": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/p-map/-/p-map-6.0.0.tgz", - "integrity": "sha512-T8BatKGY+k5rU+Q/GTYgrEf2r4xRMevAN5mtXc2aPc4rS1j3s+vWTaO2Wag94neXuCAUAs8cxBL9EeB5EA6diw==", - "dev": true - }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -14776,24 +14735,6 @@ "integrity": "sha512-KxVu8EYHDPBdUYdKZdKtU2aj2XfEx9AfjXxE/Aj0vT06w2icA09Vus1rh6eSu1y01akYg6BjIK/hxyLJINoMLQ==", "dev": true }, - "tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", - "dev": true, - "requires": { - "rimraf": "^3.0.0" - } - }, - "tmp-promise": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/tmp-promise/-/tmp-promise-3.0.3.tgz", - "integrity": "sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==", - "dev": true, - "requires": { - "tmp": "^0.2.0" - } - }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", From 99a0e4ca26a8e8d350457f01ac598d278c7fa51a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Tue, 17 Oct 2023 16:18:43 +0100 Subject: [PATCH 15/16] chore: update comment --- src/client.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/client.ts b/src/client.ts index 7bf8548..c5a6e02 100644 --- a/src/client.ts +++ b/src/client.ts @@ -4,11 +4,13 @@ import { env } from 'node:process' import { fetchAndRetry } from './retry.ts' import { BlobInput, Fetcher, HTTPMethod } from './types.ts' -// The name of the environment variable that holds the context in a Base64, -// JSON-encoded object. If we ever need to change the encoding or the shape -// of this object, we should bump the version and create a new variable, so -// that the client knows how to consume the data and can advise the user to -// update the client if needed. +/** + * The name of the environment variable that holds the context in a Base64, + * JSON-encoded object. If we ever need to change the encoding or the shape + * of this object, we should bump the version and create a new variable, so + * that the client knows how to consume the data and can advise the user to + * update the client if needed. + */ export const NETLIFY_CONTEXT_VARIABLE = 'NETLIFY_BLOBS_CONTEXT' export interface Context { From 1a951aa3dd257051224b2abcb13e905fe37d1199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Eduardo=20Bou=C3=A7as?= Date: Tue, 17 Oct 2023 16:43:24 +0100 Subject: [PATCH 16/16] refactor: rename `fetcher` to `fetch` --- README.md | 6 ++--- src/client.ts | 14 +++++----- src/main.test.ts | 66 +++++++++++++++++++++++----------------------- src/retry.ts | 8 +++--- src/store.ts | 10 +++---- test/mock_fetch.ts | 2 +- 6 files changed, 53 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 442560a..a1f64d6 100644 --- a/README.md +++ b/README.md @@ -149,12 +149,12 @@ const store2 = getStore({ assert.equal(await store2.get('my-key'), 'my value') ``` -### Custom fetcher +### Custom `fetch` The client uses [the web platform `fetch()`](https://developer.mozilla.org/en-US/docs/Web/API/Fetch_API) to make HTTP calls. By default, it will use any globally-defined instance of `fetch`, but you can choose to provide your own. -You can do this by supplying a `fetcher` property to the `getStore` method. +You can do this by supplying a `fetch` property to the `getStore` method. ```ts import { fetch } from 'whatwg-fetch' @@ -162,7 +162,7 @@ import { fetch } from 'whatwg-fetch' import { getStore } from '@netlify/blobs' const store = getStore({ - fetcher: fetch, + fetch, name: 'my-store', }) diff --git a/src/client.ts b/src/client.ts index c5a6e02..c267215 100644 --- a/src/client.ts +++ b/src/client.ts @@ -35,11 +35,11 @@ interface MakeStoreRequestOptions { export class Client { private context?: Context - private fetcher?: Fetcher + private fetch?: Fetcher - constructor(context?: Context, fetcher?: Fetcher) { + constructor(context?: Context, fetch?: Fetcher) { this.context = context ?? {} - this.fetcher = fetcher + this.fetch = fetch } private static getEnvironmentContext() { @@ -86,8 +86,8 @@ export class Client { context.siteID }/blobs/${encodedKey}?context=${storeName}` const headers = { authorization: `Bearer ${context.token}` } - const fetcher = this.fetcher ?? globalThis.fetch - const res = await fetcher(apiURL, { headers, method }) + const fetch = this.fetch ?? globalThis.fetch + const res = await fetch(apiURL, { headers, method }) if (res.status !== 200) { throw new Error(`${method} operation has failed: API returned a ${res.status} response`) @@ -123,8 +123,8 @@ export class Client { options.duplex = 'half' } - const fetcher = this.fetcher ?? globalThis.fetch - const res = await fetchAndRetry(fetcher, url, options) + const fetch = this.fetch ?? globalThis.fetch + const res = await fetchAndRetry(fetch, url, options) if (res.status === 404 && method === HTTPMethod.GET) { return null diff --git a/src/main.test.ts b/src/main.test.ts index b54d043..c4df92c 100644 --- a/src/main.test.ts +++ b/src/main.test.ts @@ -72,7 +72,7 @@ describe('get', () => { url: signedURL, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ name: 'production', @@ -104,7 +104,7 @@ describe('get', () => { url: signedURL, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ name: 'production', @@ -123,7 +123,7 @@ describe('get', () => { url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ name: 'production', @@ -149,7 +149,7 @@ describe('get', () => { url: signedURL, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ name: 'production', @@ -180,7 +180,7 @@ describe('get', () => { url: signedURL, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ name: 'production', @@ -207,7 +207,7 @@ describe('get', () => { url: `${edgeURL}/${siteID}/production/${key}`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ edgeURL, @@ -232,7 +232,7 @@ describe('get', () => { url: `${edgeURL}/${siteID}/production/${key}`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ edgeURL, @@ -252,7 +252,7 @@ describe('get', () => { url: `${edgeURL}/${siteID}/production/${key}`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ edgeURL, @@ -292,7 +292,7 @@ describe('get', () => { url: `${edgeURL}/${siteID}/images/${key}`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch for (let index = 0; index <= 1; index++) { const context = { @@ -317,9 +317,9 @@ describe('get', () => { }) test('Throws when the instance is missing required configuration properties', async () => { - const { fetcher } = new MockFetch() + const { fetch } = new MockFetch() - globalThis.fetch = fetcher + globalThis.fetch = fetch const blobs1 = getStore('production') @@ -368,7 +368,7 @@ describe('set', () => { url: signedURL, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ name: 'production', @@ -400,7 +400,7 @@ describe('set', () => { url: signedURL, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ name: 'production', @@ -420,7 +420,7 @@ describe('set', () => { url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ name: 'production', @@ -474,7 +474,7 @@ describe('set', () => { url: signedURL, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ name: 'production', @@ -504,7 +504,7 @@ describe('set', () => { url: `${edgeURL}/${siteID}/production/${encodeURIComponent(complexKey)}`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ edgeURL, @@ -527,7 +527,7 @@ describe('set', () => { url: `${edgeURL}/${siteID}/production/${key}`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ edgeURL, @@ -570,7 +570,7 @@ describe('set', () => { url: `${edgeURL}/${siteID}/production/${key}`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ edgeURL, @@ -586,9 +586,9 @@ describe('set', () => { }) test('Throws when the instance is missing required configuration properties', async () => { - const { fetcher } = new MockFetch() + const { fetch } = new MockFetch() - globalThis.fetch = fetcher + globalThis.fetch = fetch const blobs1 = getStore('production') @@ -626,7 +626,7 @@ describe('setJSON', () => { url: signedURL, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ name: 'production', @@ -649,7 +649,7 @@ describe('setJSON', () => { url: `${edgeURL}/${siteID}/production/${key}`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ edgeURL, @@ -681,7 +681,7 @@ describe('setJSON', () => { url: signedURL, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ name: 'production', @@ -721,7 +721,7 @@ describe('delete', () => { url: signedURL, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ name: 'production', @@ -742,7 +742,7 @@ describe('delete', () => { url: `https://api.netlify.com/api/v1/sites/${siteID}/blobs/${key}?context=production`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ name: 'production', @@ -765,7 +765,7 @@ describe('delete', () => { url: `${edgeURL}/${siteID}/production/${key}`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ edgeURL, @@ -786,7 +786,7 @@ describe('delete', () => { url: `${edgeURL}/${siteID}/production/${key}`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const blobs = getStore({ edgeURL, @@ -804,9 +804,9 @@ describe('delete', () => { }) test('Throws when the instance is missing required configuration properties', async () => { - const { fetcher } = new MockFetch() + const { fetch } = new MockFetch() - globalThis.fetch = fetcher + globalThis.fetch = fetch const blobs1 = getStore('production') @@ -851,7 +851,7 @@ describe('Deploy scope', () => { url: `${edgeURL}/${siteID}/${deployID}/${key}`, }) - globalThis.fetch = mockStore.fetcher + globalThis.fetch = mockStore.fetch const context = { edgeURL, @@ -881,8 +881,8 @@ describe('Deploy scope', () => { }) }) -describe('Customer fetcher', () => { - test('Uses a custom implementation of `fetch` if the `fetcher` parameter is supplied', async () => { +describe('Custom `fetch`', () => { + test('Uses a custom implementation of `fetch` if the `fetch` parameter is supplied', async () => { globalThis.fetch = () => { throw new Error('I should not be called') } @@ -901,7 +901,7 @@ describe('Customer fetcher', () => { env.NETLIFY_BLOBS_CONTEXT = Buffer.from(JSON.stringify(context)).toString('base64') - const store = getStore({ fetcher: mockStore.fetcher, name: 'images' }) + const store = getStore({ fetch: mockStore.fetch, name: 'images' }) const string = await store.get(key) expect(string).toBe(value) diff --git a/src/retry.ts b/src/retry.ts index e52c76e..336a6b5 100644 --- a/src/retry.ts +++ b/src/retry.ts @@ -6,20 +6,20 @@ const MAX_RETRY = 5 const RATE_LIMIT_HEADER = 'X-RateLimit-Reset' export const fetchAndRetry = async ( - fetcher: Fetcher, + fetch: Fetcher, url: string, options: RequestInit, attemptsLeft = MAX_RETRY, ): ReturnType => { try { - const res = await fetcher(url, options) + const res = await fetch(url, options) if (attemptsLeft > 0 && (res.status === 429 || res.status >= 500)) { const delay = getDelay(res.headers.get(RATE_LIMIT_HEADER)) await sleep(delay) - return fetchAndRetry(fetcher, url, options, attemptsLeft - 1) + return fetchAndRetry(fetch, url, options, attemptsLeft - 1) } return res @@ -32,7 +32,7 @@ export const fetchAndRetry = async ( await sleep(delay) - return fetchAndRetry(fetcher, url, options, attemptsLeft - 1) + return fetchAndRetry(fetch, url, options, attemptsLeft - 1) } } diff --git a/src/store.ts b/src/store.ts index 6ac373d..6724c28 100644 --- a/src/store.ts +++ b/src/store.ts @@ -139,7 +139,7 @@ class Store { interface GetStoreOptions extends Context { deployID?: string - fetcher?: Fetcher + fetch?: Fetcher name?: string } @@ -154,15 +154,15 @@ export const getStore: { } if (typeof input.name === 'string') { - const { fetcher, name, ...context } = input - const client = new Client(context, fetcher) + const { fetch, name, ...context } = input + const client = new Client(context, fetch) return new Store({ client, name }) } if (typeof input.deployID === 'string') { - const { fetcher, deployID, ...context } = input - const client = new Client(context, fetcher) + const { fetch, deployID, ...context } = input + const client = new Client(context, fetch) return new Store({ client, name: deployID }) } diff --git a/test/mock_fetch.ts b/test/mock_fetch.ts index 6da7cb9..7248a0a 100644 --- a/test/mock_fetch.ts +++ b/test/mock_fetch.ts @@ -53,7 +53,7 @@ export class MockFetch { return this.addExpectedRequest({ ...options, method: 'put' }) } - get fetcher() { + get fetch() { // eslint-disable-next-line require-await return async (...args: Parameters) => { const [url, options] = args