diff --git a/.gitignore b/.gitignore index 1f632399..7280d58b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Program binaries /carbonapi /carbonzipper +/.build/ # Booking.com internals .deploy diff --git a/Makefile b/Makefile index df6efd08..7d8a653f 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ debug: GCFLAGS += -gcflags=all='-l -N' LDFLAGS = -ldflags '-X main.BuildVersion=$(VERSION)' -TAGS := -tags cairo,nativegzip +TAGS := -tags cairo,nativegzip,zlibng nocairo: TAGS = ### Targets ### @@ -29,7 +29,7 @@ nocairo: build debug: build .PHONY: build -build: +build: native-deps $(PKGCONF) go build $(TAGS) $(LDFLAGS) $(GCFLAGS) $(PKG_CARBONAPI) .PHONY: check @@ -41,8 +41,15 @@ lint: GOMEMLIMIT=1024MiB golangci-lint run -v .PHONY: test -test: +test: native-deps $(PKGCONF) go test $(TAGS) -timeout 10s -race ./... + @case '$(TAGS)' in *zlibng*) \ + go test -tags nativegzip -timeout 10s -race ./pkg/backend/net && \ + go test -timeout 10s -race ./pkg/backend/net ;; esac + +.PHONY: native-deps +native-deps: + @case '$(TAGS)' in *zlibng*) ./scripts/build-zlib-ng.sh ;; esac .PHONY: test-e2e test-e2e: @@ -55,6 +62,7 @@ test-hurl: .PHONY: clean clean: rm -f carbonapi carbonzipper + rm -rf .build/zlib-ng .PHONY: authors authors: diff --git a/README.md b/README.md index 96f8d5da..521cd570 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,18 @@ To build both the `carbonapi` and `carbonzipper` binaries, run: make ``` +The default build links a pinned zlib-ng 2.3.3 static library for backend gzip +decoding. The first build needs a C compiler, Make, curl, tar and internet access; +the source archive is verified by SHA-256 and its tests run before installation +under `.build/zlib-ng`. No system libraries are installed or replaced. The native +`zng_*` API leaves response compression and Cairo/libpng on their existing paths. +Remove `.build/zlib-ng` (or run `make clean`) to rebuild the dependency. + +`make build TAGS='-tags cairo,nativegzip'` retains system-zlib decoding. +`make nocairo` retains the portable Go decoder without either native dependency. +The existing `CARBONAPI_NATIVE_GZIP` and `CARBONAPI_STREAM_RENDER` startup flags +still control use of native backend decoding. + To build the binaries with debug symbols, run: ``` diff --git a/docker/carbonapi/Dockerfile b/docker/carbonapi/Dockerfile index 92a3a12b..b574f45f 100644 --- a/docker/carbonapi/Dockerfile +++ b/docker/carbonapi/Dockerfile @@ -1,7 +1,7 @@ FROM golang:1.26 RUN apt-get update -RUN apt-get install -y libcairo2-dev +RUN apt-get install -y libcairo2-dev curl WORKDIR /go/src/github.com/bookingcom/carbonapi COPY . . diff --git a/pkg/backend/net/native_gzip.go b/pkg/backend/net/native_gzip.go index 05d80ed5..e142fec4 100644 --- a/pkg/backend/net/native_gzip.go +++ b/pkg/backend/net/native_gzip.go @@ -3,19 +3,32 @@ package net /* -#cgo LDFLAGS: -lz +#cgo !zlibng LDFLAGS: -lz +#cgo zlibng CFLAGS: -DCARBONAPI_ZLIBNG -I${SRCDIR}/../../../.build/zlib-ng/include +#cgo zlibng LDFLAGS: ${SRCDIR}/../../../.build/zlib-ng/lib/libz-ng.a #include +#ifdef CARBONAPI_ZLIBNG +// The native API has separate symbols: Cairo/libpng continue using system zlib. +#include +typedef zng_stream carbonapi_z_stream; +#define inflateInit2 zng_inflateInit2 +#define inflateReset2 zng_inflateReset2 +#define inflateEnd zng_inflateEnd +#define inflate zng_inflate +#else #include +typedef z_stream carbonapi_z_stream; +#endif typedef struct { unsigned int used, produced; int code; } carbonapi_inflate_result; -static z_stream *carbonapi_inflate_new(void) { - z_stream *s = (z_stream *)calloc(1, sizeof(z_stream)); +static carbonapi_z_stream *carbonapi_inflate_new(void) { + carbonapi_z_stream *s = (carbonapi_z_stream *)calloc(1, sizeof(carbonapi_z_stream)); if (s && inflateInit2(s, 15 + 16) != Z_OK) { free(s); return NULL; } return s; } -static int carbonapi_inflate_reset(z_stream *s) { return inflateReset2(s, 15 + 16); } -static void carbonapi_inflate_free(z_stream *s) { if (s) { inflateEnd(s); free(s); } } -static carbonapi_inflate_result carbonapi_inflate_step(z_stream *s, unsigned char *in, +static int carbonapi_inflate_reset(carbonapi_z_stream *s) { return inflateReset2(s, 15 + 16); } +static void carbonapi_inflate_free(carbonapi_z_stream *s) { if (s) { inflateEnd(s); free(s); } } +static carbonapi_inflate_result carbonapi_inflate_step(carbonapi_z_stream *s, unsigned char *in, unsigned int input_size, unsigned char *out, unsigned int output_size) { s->next_in = in; s->avail_in = input_size; s->next_out = out; s->avail_out = output_size; @@ -38,7 +51,7 @@ import ( ) type nativeGzipReader struct { - state *C.z_stream + state *C.carbonapi_z_stream src io.Reader input []byte pos, end int diff --git a/pkg/backend/net/stream_render_test.go b/pkg/backend/net/stream_render_test.go index b7ada044..efc45f2e 100644 --- a/pkg/backend/net/stream_render_test.go +++ b/pkg/backend/net/stream_render_test.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "math" + "math/rand" "net/http" "net/http/httptest" "reflect" @@ -41,6 +42,46 @@ func gzipData(t testing.TB, body []byte) []byte { return b.Bytes() } +func TestNativeInflaterCompressionVariants(t *testing.T) { + noise := make([]byte, 65537) + _, _ = rand.New(rand.NewSource(42)).Read(noise) + for _, level := range []int{gzip.NoCompression, gzip.BestSpeed, gzip.BestCompression, gzip.HuffmanOnly} { + for _, body := range [][]byte{noise, bytes.Repeat([]byte("repeated backend points\x00\xff"), 4096)} { + var encoded bytes.Buffer + w, err := gzip.NewWriterLevel(&encoded, level) + if err != nil { + t.Fatal(err) + } + w.Name, w.Comment, w.Extra = "backend", "optional gzip header", []byte{0, 1, 255} + if _, err := w.Write(body); err != nil { + t.Fatal(err) + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + for _, chunk := range []int{1, 37, 32768} { + base, err := gzip.NewReader(bytes.NewReader(encoded.Bytes())) + if err != nil { + t.Fatal(err) + } + want, err := io.ReadAll(base) + if err != nil { + t.Fatal(err) + } + r, err := newNativeInflater(&gzipFragments{data: encoded.Bytes(), chunk: chunk}) + if err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(r) + _ = r.Close() + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("level=%d chunk=%d: %v; bytes=%d/%d", level, chunk, err, len(got), len(want)) + } + } + } + } +} + func TestNativeInflaterDifferential(t *testing.T) { for _, size := range []int{0, 1, 31, 4096, 65536, 1 << 20} { body := bytes.Repeat([]byte("some repeated protobuf-like bytes\x00\x01\xff"), (size+38)/39) diff --git a/scripts/build-zlib-ng.sh b/scripts/build-zlib-ng.sh new file mode 100755 index 00000000..7c5155c6 --- /dev/null +++ b/scripts/build-zlib-ng.sh @@ -0,0 +1,34 @@ +#!/bin/sh +set -eu + +# zlib-ng 2.3.3. Use its native, namespaced API, never replace system zlib. +revision=12731092979c6d07f42da27da673a9f6c7b13586 +checksum=a0d2a5d122c84b56a793a1553a9c3327fb2eb7469bf7a86b79e3c7be5d92e8d6 +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +prefix="$root/.build/zlib-ng" +if test -f "$prefix/revision" && test "$(cat "$prefix/revision")" = "$revision" && + test -f "$prefix/lib/libz-ng.a" && test -f "$prefix/include/zlib-ng.h"; then + exit 0 +fi + +mkdir -p "$root/.build" +work=$(mktemp -d "$root/.build/zlib-ng-build.XXXXXX") +trap 'rm -rf "$work"' EXIT HUP INT TERM +curl --fail --location --retry 3 --connect-timeout 10 --max-time 120 --output "$work/source.tar.gz" \ + "https://codeload.github.com/zlib-ng/zlib-ng/tar.gz/$revision" +if command -v sha256sum >/dev/null 2>&1; then + actual=$(sha256sum "$work/source.tar.gz" | cut -d ' ' -f 1) +else + actual=$(shasum -a 256 "$work/source.tar.gz" | cut -d ' ' -f 1) +fi +test "$actual" = "$checksum" || { echo 'zlib-ng source checksum mismatch' >&2; exit 1; } +mkdir "$work/source" +tar -xzf "$work/source.tar.gz" -C "$work/source" --strip-components=1 +cd "$work/source" +# Upstream's default build uses runtime CPU dispatch, not -march=native. +./configure --static --prefix="$prefix" +make -j "${ZLIBNG_JOBS:-2}" +make check +make install +cp LICENSE.md "$prefix/LICENSE.md" +printf '%s\n' "$revision" > "$prefix/revision"