diff --git a/src/metadata.ts b/src/metadata.ts index 5ba1ad1..288a70d 100644 --- a/src/metadata.ts +++ b/src/metadata.ts @@ -22,18 +22,12 @@ export const encodeMetadata = (metadata?: Metadata) => { return payload } -export const decodeMetadata = (headers?: Headers): Metadata => { - if (!headers) { +export const decodeMetadata = (header: string | null): Metadata => { + if (!header || !header.startsWith(BASE64_PREFIX)) { return {} } - const metadataHeader = headers.get(METADATA_HEADER_INTERNAL) - - if (!metadataHeader || !metadataHeader.startsWith(BASE64_PREFIX)) { - return {} - } - - const encodedData = metadataHeader.slice(BASE64_PREFIX.length) + const encodedData = header.slice(BASE64_PREFIX.length) const decodedData = Buffer.from(encodedData, 'base64').toString() const metadata = JSON.parse(decodedData) diff --git a/src/server.test.ts b/src/server.test.ts index 9765a27..4ac564c 100644 --- a/src/server.test.ts +++ b/src/server.test.ts @@ -3,7 +3,7 @@ import { env, version as nodeVersion } from 'node:process' import semver from 'semver' import tmp from 'tmp-promise' -import { describe, test, expect, beforeAll, afterEach } from 'vitest' +import { test, expect, beforeAll, afterEach } from 'vitest' import { getStore } from './main.js' import { BlobsServer } from './server.js' @@ -28,87 +28,190 @@ afterEach(() => { }) const siteID = '9a003659-aaaa-0000-aaaa-63d3720d8621' -const key = '54321' const token = 'my-very-secret-token' -describe('Local server', () => { - test('Reads and writes from the file system', async () => { - const directory = await tmp.dir() - const server = new BlobsServer({ - directory: directory.path, - token, - }) - const { port } = await server.start() - const blobs = getStore({ - edgeURL: `http://localhost:${port}`, - name: 'mystore', - token, - siteID, - }) - - await blobs.set(key, 'value 1') - expect(await blobs.get(key)).toBe('value 1') - - await blobs.set(key, 'value 2') - expect(await blobs.get(key)).toBe('value 2') - - await blobs.delete(key) - expect(await blobs.get(key)).toBe(null) - - await server.stop() - await fs.rm(directory.path, { force: true, recursive: true }) +test('Reads and writes from the file system', async () => { + const directory = await tmp.dir() + const server = new BlobsServer({ + directory: directory.path, + token, }) + const { port } = await server.start() + const blobs = getStore({ + edgeURL: `http://localhost:${port}`, + name: 'mystore', + token, + siteID, + }) + const metadata = { + features: { + blobs: true, + functions: true, + }, + name: 'Netlify', + } + + await blobs.set('simple-key', 'value 1') + expect(await blobs.get('simple-key')).toBe('value 1') + + await blobs.set('simple-key', 'value 2', { metadata }) + expect(await blobs.get('simple-key')).toBe('value 2') + + await blobs.set('parent/child', 'value 3') + expect(await blobs.get('parent/child')).toBe('value 3') + expect(await blobs.get('parent')).toBe(null) + + const entry = await blobs.getWithMetadata('simple-key') + expect(entry.metadata).toEqual(metadata) + + await blobs.delete('simple-key') + expect(await blobs.get('simple-key')).toBe(null) + + await server.stop() + await fs.rm(directory.path, { force: true, recursive: true }) +}) + +test('Separates keys from different stores', async () => { + const directory = await tmp.dir() + const server = new BlobsServer({ + directory: directory.path, + token, + }) + const { port } = await server.start() + + const store1 = getStore({ + edgeURL: `http://localhost:${port}`, + name: 'mystore1', + token, + siteID, + }) + const store2 = getStore({ + edgeURL: `http://localhost:${port}`, + name: 'mystore2', + token, + siteID, + }) + const key = 'my-key' + + await store1.set(key, 'value 1 for store 1') + await store2.set(key, 'value 1 for store 2') - test('Separates keys from different stores', async () => { - const directory = await tmp.dir() - const server = new BlobsServer({ - directory: directory.path, - token, - }) - const { port } = await server.start() - - const store1 = getStore({ - edgeURL: `http://localhost:${port}`, - name: 'mystore1', - token, - siteID, - }) - const store2 = getStore({ - edgeURL: `http://localhost:${port}`, - name: 'mystore2', - token, - siteID, - }) - - await store1.set(key, 'value 1 for store 1') - await store2.set(key, 'value 1 for store 2') - - expect(await store1.get(key)).toBe('value 1 for store 1') - expect(await store2.get(key)).toBe('value 1 for store 2') - - await server.stop() - await fs.rm(directory.path, { force: true, recursive: true }) + expect(await store1.get(key)).toBe('value 1 for store 1') + expect(await store2.get(key)).toBe('value 1 for store 2') + + await server.stop() + await fs.rm(directory.path, { force: true, recursive: true }) +}) + +test('If a token is set, rejects any requests with an invalid `authorization` header', async () => { + const directory = await tmp.dir() + const server = new BlobsServer({ + directory: directory.path, + token, + }) + const { port } = await server.start() + const blobs = getStore({ + edgeURL: `http://localhost:${port}`, + name: 'mystore', + token: 'another token', + siteID, }) - test('If a token is set, rejects any requests with an invalid `authorization` header', async () => { - const directory = await tmp.dir() - const server = new BlobsServer({ - directory: directory.path, - token, - }) - const { port } = await server.start() - const blobs = getStore({ - edgeURL: `http://localhost:${port}`, - name: 'mystore', - token: 'another token', - siteID, - }) - - await expect(async () => await blobs.get(key)).rejects.toThrowError( - 'Netlify Blobs has generated an internal error: 403 response', - ) - - await server.stop() - await fs.rm(directory.path, { force: true, recursive: true }) + await expect(async () => await blobs.get('some-key')).rejects.toThrowError( + 'Netlify Blobs has generated an internal error: 403 response', + ) + + await server.stop() + await fs.rm(directory.path, { force: true, recursive: true }) +}) + +test('Lists entries', async () => { + const directory = await tmp.dir() + const server = new BlobsServer({ + directory: directory.path, + token, + }) + const { port } = await server.start() + const blobs = getStore({ + edgeURL: `http://localhost:${port}`, + name: 'mystore', + token, + siteID, }) + const songs: Record = { + 'coldplay/parachutes/shiver': "I'll always be waiting for you", + 'coldplay/parachutes/spies': 'And the spies came out of the water', + 'coldplay/parachutes/trouble': 'And I:I never meant to cause you trouble', + 'coldplay/a-rush-of-blood-to-the-head/politik': 'Give me heart and give me soul', + 'coldplay/a-rush-of-blood-to-the-head/in-my-place': 'How long must you wait for it?', + 'coldplay/a-rush-of-blood-to-the-head/the-scientist': 'Questions of science, science and progress', + 'phoenix/united/too-young': "Oh rainfalls and hard times coming they won't leave me tonight", + 'phoenix/united/party-time': 'Summertime is gone', + 'phoenix/ti-amo/j-boy': 'Something in the middle of the side of the store', + 'phoenix/ti-amo/fleur-de-lys': 'No rest till I get to you, no rest till I get to you', + } + + for (const title in songs) { + await blobs.set(title, songs[title]) + } + + const allSongs = await blobs.list() + + for (const title in songs) { + const match = allSongs.blobs.find((blob) => blob.key === title) + + expect(match).toBeTruthy() + } + + const coldplaySongs = await blobs.list({ prefix: 'coldplay/' }) + + for (const title in songs) { + if (!title.startsWith('coldplay/')) { + continue + } + + const match = coldplaySongs.blobs.find((blob) => blob.key === title) + + expect(match).toBeTruthy() + } + + const parachutesSongs = await blobs.list({ prefix: 'coldplay/parachutes/' }) + + for (const title in songs) { + if (!title.startsWith('coldplay/parachutes/')) { + continue + } + + const match = parachutesSongs.blobs.find((blob) => blob.key === title) + + expect(match).toBeTruthy() + } + + const fooFightersSongs = await blobs.list({ prefix: 'foo-fighters/' }) + + expect(fooFightersSongs.blobs).toEqual([]) + + const artists = await blobs.list({ directories: true }) + + expect(artists.blobs).toEqual([]) + expect(artists.directories).toEqual(['coldplay', 'phoenix']) + + const coldplayAlbums = await blobs.list({ directories: true, prefix: 'coldplay/' }) + + expect(coldplayAlbums.blobs).toEqual([]) + expect(coldplayAlbums.directories).toEqual(['coldplay/a-rush-of-blood-to-the-head', 'coldplay/parachutes']) + + const parachutesSongs2 = await blobs.list({ directories: true, prefix: 'coldplay/parachutes/' }) + + for (const title in songs) { + if (!title.startsWith('coldplay/parachutes/')) { + continue + } + + const match = parachutesSongs2.blobs.find((blob) => blob.key === title) + + expect(match).toBeTruthy() + } + + expect(parachutesSongs2.directories).toEqual([]) }) diff --git a/src/server.ts b/src/server.ts index 9dc5d94..73f6df5 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,8 +1,10 @@ import { createReadStream, createWriteStream, promises as fs } from 'node:fs' import http from 'node:http' import { tmpdir } from 'node:os' -import { basename, dirname, join, resolve } from 'node:path' +import { dirname, join, relative, resolve, sep } from 'node:path' +import { ListResponse } from './backend/list.ts' +import { decodeMetadata, encodeMetadata, METADATA_HEADER_EXTERNAL, METADATA_HEADER_INTERNAL } from './metadata.ts' import { isNodeError, Logger } from './util.ts' interface BlobsServerOptions { @@ -35,6 +37,7 @@ interface BlobsServerOptions { } export class BlobsServer { + private address: string private debug: boolean private directory: string private logger: Logger @@ -43,6 +46,7 @@ export class BlobsServer { private token?: string constructor({ debug, directory, logger, port, token }: BlobsServerOptions) { + this.address = '' this.debug = debug === true this.directory = directory this.logger = logger ?? console.log @@ -59,9 +63,10 @@ export class BlobsServer { } async delete(req: http.IncomingMessage, res: http.ServerResponse) { - const { dataPath } = this.getFilePathFromURL(req.url) + const url = new URL(req.url ?? '', this.address) + const { dataPath, key } = this.getLocalPaths(url) - if (!dataPath) { + if (!dataPath || !key) { return this.sendResponse(req, res, 400) } @@ -78,33 +83,89 @@ export class BlobsServer { return this.sendResponse(req, res, 200) } - get(req: http.IncomingMessage, res: http.ServerResponse) { - const { dataPath } = this.getFilePathFromURL(req.url) + async get(req: http.IncomingMessage, res: http.ServerResponse) { + const url = new URL(req.url ?? '', this.address) + const { dataPath, key, metadataPath, rootPath } = this.getLocalPaths(url) - if (!dataPath) { + if (!dataPath || !metadataPath) { return this.sendResponse(req, res, 400) } + // If there is no key in the URL, it means a `list` operation. + if (!key) { + return this.list({ dataPath, metadataPath, rootPath, req, res, url }) + } + + const headers: Record = {} + + try { + const rawData = await fs.readFile(metadataPath, 'utf8') + const metadata = JSON.parse(rawData) + const encodedMetadata = encodeMetadata(metadata) + + if (encodedMetadata) { + headers[METADATA_HEADER_INTERNAL] = encodedMetadata + } + } catch (error) { + this.logDebug('Could not read metadata file:', error) + } + + for (const name in headers) { + res.setHeader(name, headers[name]) + } + const stream = createReadStream(dataPath) stream.on('error', (error: NodeJS.ErrnoException) => { - if (error.code === 'ENOENT') { + if (error.code === 'EISDIR' || error.code === 'ENOENT') { return this.sendResponse(req, res, 404) } return this.sendResponse(req, res, 500) }) - stream.on('finish', () => this.sendResponse(req, res, 200)) stream.pipe(res) } + async list(options: { + dataPath: string + metadataPath: string + rootPath: string + req: http.IncomingMessage + res: http.ServerResponse + url: URL + }) { + const { dataPath, rootPath, req, res, url } = options + const directories = url.searchParams.get('directories') === 'true' + const prefix = url.searchParams.get('prefix') ?? '' + const result: ListResponse = { + blobs: [], + directories: [], + } + + try { + await BlobsServer.walk({ directories, path: dataPath, prefix, rootPath, result }) + } catch (error) { + this.logDebug('Could not perform list:', error) + + return this.sendResponse(req, res, 500) + } + + res.setHeader('content-type', 'application/json') + + return this.sendResponse(req, res, 200, JSON.stringify(result)) + } + async put(req: http.IncomingMessage, res: http.ServerResponse) { - const { dataPath } = this.getFilePathFromURL(req.url) + const url = new URL(req.url ?? '', this.address) + const { dataPath, key, metadataPath } = this.getLocalPaths(url) - if (!dataPath) { + if (!dataPath || !key || !metadataPath) { return this.sendResponse(req, res, 400) } + const metadataHeader = req.headers[METADATA_HEADER_EXTERNAL] + const metadata = decodeMetadata(Array.isArray(metadataHeader) ? metadataHeader[0] : metadataHeader ?? null) + try { // We can't have multiple requests writing to the same file, which could // lead to corrupted data. Ideally we'd have a mechanism where the last @@ -112,17 +173,23 @@ export class BlobsServer { // now, we address this by writing data to a temporary file and then // moving it to the right path after the write has succeeded. const tempDirectory = await fs.mkdtemp(join(tmpdir(), 'netlify-blobs')) - const tempPath = join(tempDirectory, basename(dataPath)) + const relativeDataPath = relative(this.directory, dataPath) + const tempDataPath = join(tempDirectory, relativeDataPath) + + await fs.mkdir(dirname(tempDataPath), { recursive: true }) await new Promise((resolve, reject) => { - req.pipe(createWriteStream(tempPath)) + req.pipe(createWriteStream(tempDataPath)) req.on('end', resolve) req.on('error', reject) }) await fs.mkdir(dirname(dataPath), { recursive: true }) - await fs.rename(tempPath, dataPath) + await fs.rename(tempDataPath, dataPath) await fs.rm(tempDirectory, { force: true, recursive: true }) + + await fs.mkdir(dirname(metadataPath), { recursive: true }) + await fs.writeFile(metadataPath, JSON.stringify(metadata)) } catch (error) { this.logDebug('Error when writing data:', error) @@ -133,23 +200,25 @@ export class BlobsServer { } /** - * Returns the path to the local file associated with a given combination of - * site ID, store name, and object, which are extracted from a URL path. + * Parses the URL and returns the filesystem paths where entries and metadata + * should be stored. */ - getFilePathFromURL(urlPath?: string) { - if (!urlPath) { + getLocalPaths(url?: URL) { + if (!url) { return {} } - const [, siteID, storeName, key] = urlPath.split('/') + const [, siteID, storeName, ...key] = url.pathname.split('/') - if (!siteID || !storeName || !key) { + if (!siteID || !storeName) { return {} } - const dataPath = resolve(this.directory, 'entries', siteID, storeName, key) + const rootPath = resolve(this.directory, 'entries', siteID, storeName) + const dataPath = resolve(rootPath, ...key) + const metadataPath = resolve(this.directory, 'metadata', siteID, storeName, ...key) - return { dataPath } + return { dataPath, key: key.join('/'), metadataPath, rootPath } } handleRequest(req: http.IncomingMessage, res: http.ServerResponse) { @@ -172,11 +241,11 @@ export class BlobsServer { } } - sendResponse(req: http.IncomingMessage, res: http.ServerResponse, status: number) { + sendResponse(req: http.IncomingMessage, res: http.ServerResponse, status: number, body?: string) { this.logDebug(`${req.method} ${req.url}: ${status}`) res.writeHead(status) - res.end() + res.end(body) } async start(): Promise<{ address: string; family: string; port: number }> { @@ -194,6 +263,8 @@ export class BlobsServer { return reject(new Error('Server cannot be started on a pipe or Unix socket')) } + this.address = `http://localhost:${address.port}` + resolve(address) }) }) @@ -229,4 +300,73 @@ export class BlobsServer { return parts[1] === this.token } + + /** + * Traverses a path and collects both blobs and directories into a `result` + * object, taking into account the `directories` and `prefix` parameters. + */ + private static async walk(options: { + directories: boolean + path: string + prefix: string + result: ListResponse + rootPath: string + }) { + const { directories, path, prefix, result, rootPath } = options + const entries = await fs.readdir(path) + + for (const entry of entries) { + const entryPath = join(path, entry) + const stat = await fs.stat(entryPath) + + let key = relative(rootPath, entryPath) + + // Normalize keys to use `/` as delimiter regardless of OS. + if (sep !== '/') { + key = key.split(sep).join('/') + } + + // To match the key against the prefix, we start by creating a "mask", + // which consists of the subset of the key up to the length of the + // prefix. + const mask = key.slice(0, prefix.length) + + // There is a match if the mask matches the prefix. + const isMatch = prefix.startsWith(mask) + + if (!isMatch) { + continue + } + + // If the entry is a file, add it to the `blobs` bucket. + if (!stat.isDirectory()) { + // We don't support conditional requests in the local server, so we + // generate a random ETag for each entry. + const etag = Math.random().toString().slice(2) + + result.blobs?.push({ + etag, + key, + last_modified: stat.mtime.toISOString(), + size: stat.size, + }) + + continue + } + + // The entry is a directory. We push it to the `directories` bucket only + // if the `directories` parameter is enabled and we're at the same level + // as the prefix. For example, if the prefix is `animals/cats/` and the + // key we're processing is `animals`, we don't want to push it to the + // `directories` bucket. We want to traverse it. + if (directories && key.startsWith(prefix)) { + result.directories?.push(key) + + continue + } + + // Call this method recursively with the directory as the starting point. + await BlobsServer.walk({ directories, path: entryPath, prefix, rootPath, result }) + } + } } diff --git a/src/store.ts b/src/store.ts index 50ac421..5b8a809 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1,6 +1,6 @@ import { ListResponse, ListResponseBlob } from './backend/list.ts' import { Client } from './client.ts' -import { decodeMetadata, Metadata } from './metadata.ts' +import { decodeMetadata, Metadata, METADATA_HEADER_INTERNAL } from './metadata.ts' import { BlobInput, HTTPMethod } from './types.ts' import { BlobsInternalError } from './util.ts' @@ -189,7 +189,7 @@ export class Store { let metadata: Metadata = {} try { - metadata = decodeMetadata(res?.headers) + metadata = decodeMetadata(res?.headers.get(METADATA_HEADER_INTERNAL)) } catch { throw new Error( 'An internal error occurred while trying to retrieve the metadata for an entry. Please try updating to the latest version of the Netlify Blobs client.',