Zig 0.17.0

HTTP client

spider.http_client calls other servers: an API, a webhook, a payment provider.

The code on this page is from Bookings, the example app in examples/bookings (source). When a booking is confirmed it tells an address of the office's choosing, so that another system can follow along.

A request

zig
/// What the webhook receives, as JSON.
const Event = struct {
    event: []const u8 = "booking.confirmed",
    id: i64,
    room: []const u8,
    day: []const u8,
    starts: []const u8,
    ends: []const u8,
    title: []const u8,
    booked_by: []const u8,
};

fn call(c: *spider.Ctx, url: []const u8, booking: model.Booking) !void {
    const body = try std.json.Stringify.valueAlloc(c.arena, Event{
        .id = booking.id,
        .room = booking.room,
        .day = booking.day,
        .starts = booking.starts,
        .ends = booking.ends,
        .title = booking.title,
        .booked_by = booking.booked_by,
    }, .{});
    // Three seconds, then the request gives up: someone else's server
    // being slow is not a reason to keep this visitor waiting.
    var res = try spider.http_client.post(c.io(), c.arena, url, .{
        .body = .{ .json = body },
        .timeout_ms = 3000,
    });
    defer res.deinit();
    if (@intFromEnum(res.status) >= 300) return error.WebhookRefused;
}

post takes the server's io, where to keep the answer, the address and the options. In a handler those are c.io() and c.arena. There is one function for each method: get, post, put, patch, delete and head.

The options used most:

  • .body: .{ .json = text }, .{ .form = pairs } or .{ .raw = bytes }. Only post, put and patch take one.
  • .headers: &.{.{ .name = "Authorization", .value = token }}.
  • .query: pairs added to the address as its query string, encoded for you.
  • .timeout_ms: see below.

The answer

The answer has a status and the body as body_text. res.json(T) parses the body as JSON into a struct, which is then in its .value. Call res.deinit() when done with it.

An answer of 400 or 500 is still an answer: the call succeeds and the status says what happened. Check it, as call does above. The call itself fails only when there was no answer to have: the server could not be reached, or the time ran out.

A redirect is not followed. The client goes where you send it and nowhere else.

Always say how long to wait

Without .timeout_ms a request waits for as long as the other server takes, and so does the visitor whose request made it. Give every call a limit that fits what it is for: Bookings gives its webhook three seconds. Past the limit the call fails with error.Timeout.

Then decide what a failure means. Someone else's server being down should rarely stop your own from doing its job:

zig
/// Tells the owner, and the webhook, that `booking` is confirmed.
pub fn confirmed(c: *spider.Ctx, booking: model.Booking, owner: ?Owner, day_label: []const u8) void {
    if (owner) |to| mail(c, booking, to, day_label) catch |err| {
        std.log.warn("bookings: the confirmation of booking {d} was not mailed: {s}", .{ booking.id, @errorName(err) });
    };
    if (webhook_url) |url| call(c, url, booking) catch |err| {
        std.log.warn("bookings: the webhook was not told about booking {d}: {s}", .{ booking.id, @errorName(err) });
    };
}

A test points the address at a port nobody listens on, and the booking is confirmed all the same:

zig
test "booking: a webhook that is down does not undo the confirmation" {
    const app = try app_test.running();
    const notify = features.bookings.notify;
    var id_buf: [20]u8 = undefined;
    const cedar = try roomId(&id_buf, "Cedar");
    var buf: [1024]u8 = undefined;
    const ana = app.with(&.{try app_test.signUp(app, &buf, "Ana Ribeiro", "ana.hookdown@example.com")});

    const webhook_before = notify.webhook_url;
    defer notify.webhook_url = webhook_before;
    // A port nothing listens on.
    var probe = try Hook.listen(std.testing.io);
    const dead_port = probe.port;
    probe.server.deinit(std.testing.io);
    var url_buf: [64]u8 = undefined;
    notify.webhook_url = try std.fmt.bufPrint(&url_buf, "http://127.0.0.1:{d}/hooks", .{dead_port});

    var made = try ana.postForm("/bookings/create", Booking{ .room = cedar, .date = "2032-07-26", .start = "10:00", .end = "11:00", .title = "Sprint demo" });
    defer made.deinit();
    const location = made.header("Location").?;
    var confirm_buf: [64]u8 = undefined;
    var confirmed = try ana.postForm(try std.fmt.bufPrint(&confirm_buf, "{s}/confirm", .{location}), .{});
    defer confirmed.deinit();
    try confirmed.expectRedirect(location);
    var page = try ana.get(location);
    defer page.deinit();
    try page.expectContains("Confirmed");
}

Many calls to the same server

Each of the functions above opens a connection and closes it. An app that keeps calling the same server makes a client once and reuses its connections:

zig
test "http client: one client for many calls to the same server" {
    const app = try app_test.running();
    var url_buf: [64]u8 = undefined;
    const base_url = try std.fmt.bufPrint(&url_buf, "http://127.0.0.1:{d}", .{app.port});

    // Made once and kept: its connections are reused from call to call.
    var client = try spider.http_client.Client.init(std.testing.io, std.testing.allocator, .{ .base_url = base_url });
    defer client.deinit();

    var up = try client.get("/up", .{ .timeout_ms = 3000 });
    defer up.deinit();
    try std.testing.expectEqual(std.http.Status.ok, up.status);

    var rooms = try client.get("/rooms", .{ .timeout_ms = 3000 });
    defer rooms.deinit();
    try std.testing.expect(std.mem.indexOf(u8, rooms.body_text, "Atlas") != null);

    // An address that is nothing is an answer too: the call succeeds, the status says 404.
    var missing = try client.get("/nowhere", .{ .timeout_ms = 3000 });
    defer missing.deinit();
    try std.testing.expectEqual(std.http.Status.not_found, missing.status);
}

Testing

The test of the webhook starts a small server of its own, points the app at it, and reads what arrived:

zig
/// A server that takes one request and answers 204: where the webhook
/// goes in the test. It keeps what it was sent.
const Hook = struct {
    server: std.Io.net.Server,
    port: u16,
    received: [2048]u8 = undefined,
    len: usize = 0,

    fn listen(io: std.Io) !Hook {
        const address = try std.Io.net.IpAddress.parse("127.0.0.1", 0);
        var server = try address.listen(io, .{ .reuse_address = true });
        return .{ .port = server.socket.address.getPort(), .server = server };
    }

    fn serveOne(self: *Hook) void {
        const io = std.testing.io;
        var stream = self.server.accept(io) catch return;
        defer stream.close(io);
        var read_buf: [2048]u8 = undefined;
        var reader = stream.reader(io, &read_buf);
        // The head, then as many bytes as Content-Length says.
        while (true) {
            const chunk = reader.interface.peekGreedy(1) catch break;
            const n = @min(chunk.len, self.received.len - self.len);
            @memcpy(self.received[self.len..][0..n], chunk[0..n]);
            self.len += n;
            reader.interface.toss(chunk.len);
            const so_far = self.received[0..self.len];
            const head_end = std.mem.indexOf(u8, so_far, "\r\n\r\n") orelse continue;
            const at = std.ascii.findIgnoreCase(so_far, "content-length:") orelse break;
            const line_end = std.mem.indexOfPos(u8, so_far, at, "\r\n") orelse break;
            const length = std.fmt.parseInt(usize, std.mem.trim(u8, so_far[at + "content-length:".len .. line_end], " "), 10) catch break;
            if (self.len >= head_end + 4 + length) break;
        }
        var write_buf: [128]u8 = undefined;
        var writer = stream.writer(io, &write_buf);
        writer.interface.writeAll("HTTP/1.1 204 No Content\r\nConnection: close\r\n\r\n") catch {};
        writer.interface.flush() catch {};
    }
};

Through a proxy

Some networks let nothing out except through a proxy, and some services only accept calls from an address they know, which a proxy with a fixed address provides. proxy_url sends a call through one:

zig
test "proxy: the call asks the proxy for a way to the server, then goes through it" {
    // The proxy of the test, which is also the server at the other end.
    var proxy = try Stub.listen("200 OK", "1 EUR = 6.21 BRL");
    const thread = try std.Thread.spawn(.{}, Stub.serveThroughTunnel, .{&proxy});

    var url_buf: [64]u8 = undefined;
    const proxy_url = try std.fmt.bufPrint(&url_buf, "http://ana:s3cret@127.0.0.1:{d}", .{proxy.port});

    var res = try spider.http_client.get(std.testing.io, std.testing.allocator, "http://rates.example/today?base=EUR", .{
        .proxy_url = proxy_url,
        .timeout_ms = 3000,
    });
    defer res.deinit();
    thread.join();

    try std.testing.expectEqualStrings("1 EUR = 6.21 BRL", res.body_text);
    // First, the way: which server, and who is asking ("ana:s3cret").
    try std.testing.expect(proxy.got("CONNECT rates.example:80 HTTP/1.1"));
    try std.testing.expect(proxy.got("Proxy-Authorization: Basic YW5hOnMzY3JldA=="));
    // Then the request itself, as the server would get it without a proxy.
    try std.testing.expect(proxy.got("GET /today?base=EUR HTTP/1.1"));
}
  • The address is http://host:port for an HTTP proxy, or socks5://host:port for a SOCKS5 one. A user and a password go in it: http://user:pass@host:port.
  • The call asks the proxy for a tunnel to the server and talks to the server through it. With https, the proxy carries the bytes and cannot read them.
  • Without proxy_url, the environment decides: http_proxy, https_proxy and all_proxy, with no_proxy for the hosts that go direct. With none of them set there is no proxy.

A Client takes its proxy when it is made, in Client.init(io, allocator, .{ .base_url, .proxy_url }), and keeps it:

zig
test "proxy: a client is tied to the proxy it was made with" {
    var client = try spider.http_client.Client.init(std.testing.io, std.testing.allocator, .{
        .base_url = "http://rates.example",
        .proxy_url = "http://127.0.0.1:3128",
    });
    defer client.deinit();

    try std.testing.expectError(error.ProxyMismatch, client.get("/today", .{ .proxy_url = "http://127.0.0.1:9999" }));
}

Showing a certificate (mTLS)

Some APIs, banks above all, want the caller to prove who it is with a certificate of its own. That is another function, spider.http_client_mtls.request:

zig
test "client certificate: a server that is not there is an error by name" {
    const gpa = std.testing.allocator;
    const certificate: spider.http_client_mtls.MtlsCert = .{ .pem = .{
        .cert = "-----BEGIN CERTIFICATE-----\n...\n-----END CERTIFICATE-----\n",
        .key = "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----\n",
    } };

    // Nothing listens on port 1.
    const result = spider.http_client_mtls.request(gpa, "https://127.0.0.1:1/accounts", certificate, .{
        .method = .POST,
        .headers = &.{.{ "Content-Type", "application/json" }},
        .body = "{\"account\":7}",
        .timeout_ms = 2000,
    });
    if (result) |ok| {
        var res = ok;
        res.deinit(gpa);
        return error.TestUnexpectedResult;
    } else |err| {
        try std.testing.expectEqual(error.MtlsTransportFailed, err);
    }
}
  • The certificate is .pem (the certificate and its key, as text) or .pkcs12 (the bytes of a .pfx or .p12 file and its password), the format certificate authorities usually hand out.
  • The answer has status and body, and res.json(T, allocator) parses it. A 4xx or a 5xx is an answer, not an error. Free it with res.deinit(allocator).
  • error.MtlsTransportFailed: the server could not be reached or the handshake failed, a refused certificate included; the log has the reason. error.MtlsTimeout after timeout_ms, 30 seconds unless the call says otherwise.

It works differently from the rest of the client, and it shows:

  • It runs the curl program, so curl has to be in the container. Zig's own TLS cannot present a certificate yet.
  • The certificate and the key are written to files only the app's user can read, in /tmp, for the time of the call.
  • The call blocks its thread until the answer comes, and it does not use a Client: every call opens a connection.
  • The server's own certificate has to be one the system trusts. There is no option to trust a private authority.

The test above covers a server that is not there. A call that succeeds needs a server that asks for a certificate, which the tests of this site do not have.