File storage (R2)

Files people upload should not live on the server's disk: a new deploy is a new container, and the disk goes with the old one. spider.r2 keeps them in a bucket of Cloudflare R2, and lets a browser send and fetch files there directly, without passing them through the app.

The code on this page is from examples/guide (source): a small server kept as tests, for what the example apps do not use. Its tests cover what needs no network: the client, the names and the signed addresses. The calls that talk to R2 are compiled, not run.

Turning it on

In build.zig, where the Spider dependency is declared:

zig

const std = @import("std");

// Builds nothing to run: `zig build test` is all there is. See README.md.
pub fn build(b: *std.Build) void {
    const target = b.standardTargetOptions(.{});
    const optimize = b.standardOptimizeOption(.{});

    const spider_dep = b.dependency("spider", .{
        .target = target,
        .optimize = optimize,
        // The parts of Spider an app asks for by name.
        .xlsx = true,
        .qrcode = true,
        .r2 = true,
    });
    const spider_mod = spider_dep.module("spider");
    spider_mod.addImport("spider_config", b.createModule(.{
        .root_source_file = b.path("spider.config.zig"),
        .imports = &.{.{ .name = "spider", .module = spider_mod }},
    }));

    const tests = b.addTest(.{ .root_module = b.createModule(.{
        .root_source_file = b.path("src/tests.zig"),
        .target = target,
        .optimize = optimize,
        .imports = &.{.{ .name = "spider", .module = spider_mod }},
    }) });
    b.step("test", "Run the examples").dependOn(&b.addRunArtifact(tests).step);

    // Signing in at Keycloak needs a Keycloak: a step of its own, run by
    // login-test.sh, which starts one for the occasion.
    const login_tests = b.addTest(.{ .root_module = b.createModule(.{
        .root_source_file = b.path("login/tests.zig"),
        .target = target,
        .optimize = optimize,
        .imports = &.{.{ .name = "spider", .module = spider_mod }},
    }) });
    b.step("test-login", "Run the login examples (needs login/docker-compose.yml up)").dependOn(&b.addRunArtifact(login_tests).step);
}

One client, made at start

zig

/// What a main() would do to serve. The tests start it once
/// (`spider.testing.start(run)`) and share it.
pub fn run() !void {
    // One client for the bucket, made once. In an app the values come from
    // the environment: `spider.r2.R2.initFromEnv(io)`.
    // (`io` is the one main() received; this package has no main.)
    const io = std.Options.debug_io;
    store = try spider.r2.R2.init(io, .{
        .account_id = "0a1b2c3d4e5f",
        .access_key = "an-access-key-id",
        .secret_key = "a-secret-access-key",
        .bucket = "guide-files",
        .pub_url = "https://files.guide.example",
    });
    defer store.deinit();

    var server = spider.app(.{});
    defer server.deinit();

    try server
        .use(spider.loggerWith(.{ .quiet_paths = &.{ "/up", "/assets/**" } }))
        .use(spider.forceHttps(.{ .default_base_url = "https://guide.example" }))
        .use(spider.varyHtmx)
        .get("/whoami", whoami, .{ .public = true })
        .post("/notes", addNote, .{ .public = true })
        .post("/hooks/payments", paymentHook, .{ .public = true, .allow_http = true })
        .post("/photos", addPhoto, .{ .public = true })
        .get("/photos/report", photoReport, .{ .public = true })
        .sseWith("/ticks", ticks, .{ .public = true })
        .wsWith("/rooms/:room/chat", chat, .{ .public = true })
        .wsWith("/staff/feed", staffFeed, .{ .authenticated = true })
        .get("/orders.xlsx", exportOrders, .{ .public = true })
        .post("/orders/import", importOrders, .{ .public = true })
        .get("/tickets/:code/qr.svg", ticketQr, .{ .public = true })
        .post("/avatars/sign", signAvatarUpload, .{ .public = true })
        .get("/invoices/:number", invoice, .{ .public = true })
        .post("/reports", keepReport, .{ .public = true })
        .get("/auth/google", googleLogin, .{ .public = true })
        .get("/auth/google/callback", googleCallback, .{ .public = true })
        .get("/shop", shop, .{ .public = true })
        .get("/shop/typo", shopTypo, .{ .public = true })
        .get("/shop/receipt", receipt, .{ .public = true })
        .get("/tasks", tasks, .{ .public = true })
        .post("/tasks", addTask, .{ .public = true })
        .post("/tasks/:id/archive", archiveTask, .{ .public = true })
        .onError(spider.errorHandler(.{}))
        .listen(.{});
}

An app reads the values from the environment instead: spider.r2.R2.initFromEnv(io) takes R2_ACCOUNT_ID, R2_ACCESS_KEY, R2_SECRET_KEY, R2_BUCKET and, for a bucket with a public domain, R2_PUBLIC_URL. The keys are those of an R2 API token.

Making the client calls nobody, so wrong keys show at the first operation. Missing ones show right away:

zig

test "storage: a client without its account, keys or bucket is refused when it is made" {
    try std.testing.expectError(error.R2ConfigMissing, spider.r2.R2.init(std.testing.io, .{
        .account_id = "0a1b2c3d4e5f",
        .access_key = "an-access-key-id",
        .secret_key = "",
        .bucket = "guide-files",
    }));
}

Make one client for the whole app and share it, as store here: it keeps its connections open between calls.

The browser uploads straight to the bucket

The app does not receive the file. It gives the browser an address that accepts one upload for a few minutes, and the browser sends the file there with a PUT:

zig

var store: spider.r2.R2 = undefined;

/// Where the browser may send one avatar, straight to the bucket: the file
/// never passes through the app.
fn signAvatarUpload(c: *spider.Ctx) !spider.Response {
    // The app names the object: nothing of the name comes from the visitor.
    const key = try store.objectKey(c.arena, "account-7", "avatars", try std.fmt.allocPrint(c.arena, "{s}.png", .{c.requestId()}));
    // Good for five minutes, for a PNG and nothing else.
    const url = try store.presignedPut(c.arena, key, "image/png", 300);
    return c.json(.{ .key = key, .upload_to = url, .shown_at = try store.publicUrl(c.arena, key) }, .{});
}

zig

test "storage: the browser gets an address to upload to, signed for a few minutes" {
    const app = try spider.testing.start(run);
    var res = try app.request(.{ .method = "POST", .target = "/avatars/sign", .body = "" });
    defer res.deinit();
    try res.expectStatus(200);

    // The object's name is the app's, under the account's folder.
    try res.expectContains("\"key\":\"account-7/avatars/");
    // The address is the bucket's, and carries the signature and how long it lasts.
    try res.expectContains("\"upload_to\":\"https://0a1b2c3d4e5f.r2.cloudflarestorage.com/guide-files/account-7/avatars/");
    try res.expectContains("X-Amz-Expires=300");
    try res.expectContains("X-Amz-Signature=");
    // The secret key signs it and is not in it.
    try res.expectNotContains("a-secret-access-key");
    // Where the file will be seen, on the bucket's public domain.
    try res.expectContains("\"shown_at\":\"https://files.guide.example/account-7/avatars/");
}

presignedPut(allocator, key, content_type, seconds) signs the address. The upload has to send the same Content-Type: it is part of the signature, so an address signed for a PNG takes nothing else.
The app chooses the key. Nothing of it should come from the visitor: see Uploads and downloads for why.
objectKey(allocator, account, category, filename) joins the three with /: one folder per account keeps the files of each apart.
publicUrl(allocator, key) is the address of the object on the bucket's public domain, for files anyone may see.

The app's own limit on request bodies does not apply to an upload that goes to the bucket, which is the point for large files. Allow the site's address in the bucket's CORS settings, or the browser refuses to send.

A private file

For a file only its owner may see, the bucket has no public domain. The app checks who is asking, then sends them to an address that works for a short time:

zig

/// A private file: the visitor is sent to an address that works for one
/// minute, after the app checked that the invoice is theirs.
fn invoice(c: *spider.Ctx) !spider.Response {
    const number = c.params.get("number") orelse return error.NotFound;
    const key = try std.fmt.allocPrint(c.arena, "account-7/invoices/{s}.pdf", .{number});
    return c.redirect(try store.presignedGet(c.arena, key, 60));
}

zig

test "storage: a private file is reached through an address that lasts a minute" {
    const app = try spider.testing.start(run);
    var res = try app.get("/invoices/2026-0042");
    defer res.deinit();
    try res.expectStatus(302);
    const to = res.header("Location").?;
    try std.testing.expect(std.mem.startsWith(u8, to, "https://0a1b2c3d4e5f.r2.cloudflarestorage.com/guide-files/account-7/invoices/2026-0042.pdf?"));
    try std.testing.expect(std.mem.indexOf(u8, to, "X-Amz-Expires=60") != null);
}

Signing calls nobody, so an address is made for a key that does not exist too: the visitor then gets R2's own "not found".

Files the app handles itself

For what the app makes or reads (a report, an export, a file it has to look into), the client talks to R2:

zig

/// A file the app itself makes and keeps. This one talks to R2, so the
/// tests of this package do not call it.
fn keepReport(c: *spider.Ctx) !spider.Response {
    const csv = "caption,bytes\nThe team,20480\n";
    try store.put(c, "account-7/reports/photos.csv", csv, "text/csv");

    if (try store.head(c, "account-7/reports/old.csv")) {
        try store.delete(c, "account-7/reports/old.csv");
    }
    const back = try store.get(c, "account-7/reports/photos.csv");
    return c.text(back, .{});
}

put(c, key, bytes, content_type) writes an object, replacing one that is there.
get(c, key) reads one whole into the request's memory; error.NotFound when there is none, which a handler can let through as a 404.
head(c, key) says whether it exists, without reading it.
delete(c, key) removes it, and copyObject(c, from, to) copies it inside the bucket without downloading.

Each takes the request's c. A failure is an error by name (error.R2PutFailed, error.R2GetFailed and so on), with R2's answer in the log.