Architecture

This page describes the internal architecture of compressionz for contributors and those interested in implementation details.

High-Level Structure

compressionz/
├── src/
│   ├── root.zig           # Public API entry point
│   ├── codec.zig          # Codec enum and detection
│   ├── level.zig          # Compression levels
│   ├── error.zig          # Error types
│   ├── stream.zig         # Streaming wrappers
│   │
│   ├── lz4/               # Pure Zig LZ4
│   │   ├── lz4.zig        # Module entry
│   │   ├── block.zig      # LZ4 block compression
│   │   └── frame.zig      # LZ4 frame format
│   │
│   ├── snappy/            # Pure Zig Snappy
│   │   └── snappy.zig     # Snappy compression
│   │
│   ├── zstd.zig           # Zstd C bindings
│   ├── gzip.zig           # Gzip C bindings
│   ├── zlib_codec.zig     # Zlib/Deflate C bindings
│   ├── brotli.zig         # Brotli C bindings
│   │
│   └── archive/           # Archive formats
│       ├── archive.zig    # Archive entry point
│       ├── zip.zig        # ZIP reader/writer
│       └── tar.zig        # TAR reader/writer
│
├── vendor/                # Vendored C libraries
│   ├── zstd/
│   ├── zlib/
│   └── brotli/
│
└── benchmarks/            # Benchmark suite

Layer Architecture

┌─────────────────────────────────────────────────────────────┐
│                      User Application                        │
└─────────────────────────────────────────────────────────────┘
                              │
                              ▼
┌─────────────────────────────────────────────────────────────┐
│                     Unified API (root.zig)                   │
│  compress() / decompress() / compressWithOptions() / etc.   │
└─────────────────────────────────────────────────────────────┘
                              │
                              ▼
┌─────────────────────────────────────────────────────────────┐
│                    Codec Dispatch Layer                      │
│         Routes calls to appropriate implementation           │
└─────────────────────────────────────────────────────────────┘
                              │
        ┌─────────────────────┼─────────────────────┐
        │                     │                     │
        ▼                     ▼                     ▼
┌───────────────┐    ┌───────────────┐    ┌───────────────┐
│  Pure Zig     │    │  C Bindings   │    │   Archive     │
│  LZ4, Snappy  │    │  Zstd, Gzip,  │    │   ZIP, TAR    │
│               │    │  Brotli       │    │               │
└───────────────┘    └───────────────┘    └───────────────┘
                              │
                              ▼
                    ┌───────────────┐
                    │ Vendored C    │
                    │ Libraries     │
                    └───────────────┘

Unified API Design

The Problem

Each compression library has its own API:

// zstd
ZSTD_compress(dst, dstSize, src, srcSize, level);

// zlib
deflateInit(&stream, level);
deflate(&stream, Z_FINISH);

// brotli
BrotliEncoderCompress(quality, window, mode, input_size, input, &output_size, output);

The Solution

compressionz provides a single interface:

// All codecs use the same signature
pub fn compress(
    codec: Codec,
    input: []const u8,
    allocator: std.mem.Allocator,
) Error![]u8 {
    switch (codec) {
        .lz4 => return lz4.frame.compress(input, allocator, .{}),
        .snappy => return snappy.compress(input, allocator),
        .zstd => return zstd.compress(input, allocator, .default),
        .gzip => return gzip.compress(input, allocator, .default),
        .brotli => return brotli.compress(input, allocator, .default),
        // ...
    }
}

Pure Zig Implementations

LZ4 Architecture

pub const block = @import("block.zig");
pub const frame = @import("frame.zig");

// lz4/block.zig
pub fn compress(input: []const u8, allocator: Allocator) ![]u8 {
    // 1. Build hash table for match finding
    // 2. Scan input for matches
    // 3. Encode literals and match references
    // 4. Return compressed output
}

// lz4/frame.zig
pub fn compress(input: []const u8, allocator: Allocator, options: Options) ![]u8 {
    // 1. Write frame header (magic, flags, content size)
    // 2. Compress data blocks using block.compress()
    // 3. Write frame footer (checksum)
}

Snappy Architecture

pub fn compress(input: []const u8, allocator: Allocator) ![]u8 {
    // 1. Write uncompressed length (varint)
    // 2. Build hash table
    // 3. Find matches and emit literals/copies
    // 4. Return compressed output
}

C Bindings Architecture

Pattern

All C bindings follow the same pattern:

// 1. Import C symbols
const c = @cImport({
    @cInclude("zstd.h");
});

// 2. Wrap with Zig-friendly API
pub fn compress(input: []const u8, allocator: Allocator, level: Level) ![]u8 {
    // Allocate output buffer
    const bound = c.ZSTD_compressBound(input.len);
    const output = try allocator.alloc(u8, bound);
    errdefer allocator.free(output);

    // Call C function
    const result = c.ZSTD_compress(
        output.ptr,
        output.len,
        input.ptr,
        input.len,
        level.toZstdLevel(),
    );

    // Handle errors
    if (c.ZSTD_isError(result) != 0) {
        allocator.free(output);
        return error.InvalidData;
    }

    // Shrink to actual size
    return allocator.realloc(output, result);
}

Memory Management

C libraries are given Zig allocator callbacks:

fn zigAlloc(opaque: ?*anyopaque, size: usize) callconv(.C) ?*anyopaque {
    const allocator = @as(*Allocator, @ptrCast(@alignCast(opaque)));
    const slice = allocator.alloc(u8, size) catch return null;
    return slice.ptr;
}

fn zigFree(opaque: ?*anyopaque, ptr: ?*anyopaque) callconv(.C) void {
    const allocator = @as(*Allocator, @ptrCast(@alignCast(opaque)));
    if (ptr) |p| {
        allocator.free(@ptrCast(p));
    }
}

Streaming Architecture

pub fn Decompressor(comptime ReaderType: type) type {
    return struct {
        inner: union(enum) {
            gzip: GzipDecompressor(ReaderType),
            zstd: ZstdDecompressor(ReaderType),
            // ...
        },

        pub fn reader(self: *@This()) Reader {
            return .{ .context = self };
        }

        fn read(self: *@This(), buffer: []u8) !usize {
            return switch (self.inner) {
                .gzip => |*d| d.read(buffer),
                .zstd => |*d| d.read(buffer),
                // ...
            };
        }
    };
}

Error Handling

Unified Error Type

pub const Error = error{
    InvalidData,
    ChecksumMismatch,
    OutputTooSmall,
    OutputTooLarge,
    UnexpectedEof,
    UnsupportedFeature,
    OutOfMemory,
    DictionaryMismatch,
    InvalidParameter,
};

Error Mapping

Each implementation maps its errors:

fn mapZstdError(code: usize) Error {
    const err = c.ZSTD_getErrorCode(code);
    return switch (err) {
        c.ZSTD_error_memory_allocation => error.OutOfMemory,
        c.ZSTD_error_corruption_detected => error.InvalidData,
        c.ZSTD_error_checksum_wrong => error.ChecksumMismatch,
        c.ZSTD_error_dictionary_wrong => error.DictionaryMismatch,
        else => error.InvalidData,
    };
}

Build System

Vendored Libraries

pub fn build(b: *std.Build) void {
    const lib = b.addStaticLibrary(.{
        .name = "compressionz",
        // ...
    });

    // Add vendored zstd
    lib.addCSourceFiles(&zstd_sources, &zstd_flags);
    lib.addIncludePath(.{ .path = "vendor/zstd/lib" });

    // Add vendored zlib
    lib.addCSourceFiles(&zlib_sources, &zlib_flags);
    lib.addIncludePath(.{ .path = "vendor/zlib" });

    // Add vendored brotli
    lib.addCSourceFiles(&brotli_sources, &brotli_flags);
    lib.addIncludePath(.{ .path = "vendor/brotli/include" });
}

Cross-Platform

Works on all Zig-supported targets:

No system library dependencies
C code compiled with Zig’s C compiler
Platform-specific SIMD handled by vendored libraries

Testing Architecture

test {
    _ = @import("codec.zig");
    _ = @import("lz4/lz4.zig");
    _ = @import("snappy/snappy.zig");
    _ = @import("zstd.zig");
    _ = @import("gzip.zig");
    _ = @import("brotli.zig");
    _ = @import("archive/archive.zig");
}

// Individual tests in each file
test "compress and decompress zstd" {
    const input = "test data";
    const compressed = try compress(.zstd, input, testing.allocator);
    defer testing.allocator.free(compressed);

    const decompressed = try decompress(.zstd, compressed, testing.allocator);
    defer testing.allocator.free(decompressed);

    try testing.expectEqualStrings(input, decompressed);
}