From 33cd6823a7640029348b17a3648171c785595863 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Thu, 8 Oct 2026 11:00:50 +0200 Subject: [PATCH 1/7] fix(jsonld): time out remote context fetches and remember failures A remote JSON-LD context fetch had no timeout. In non-strict mode, a context server that never answered blocked network synchronization at the first credential referring to it. Fetches now go through the node's strict HTTP client with a 5 second timeout, and a failed URL is not fetched again for 5 minutes. Fixes #4615 Assisted by AI --- .../deployment/verifiable-credentials.rst | 2 + docs/pages/release_notes.rst | 1 + jsonld/ldutils.go | 101 +++++++++++++- jsonld/ldutils_test.go | 127 ++++++++++++++++++ 4 files changed, 228 insertions(+), 3 deletions(-) diff --git a/docs/pages/deployment/verifiable-credentials.rst b/docs/pages/deployment/verifiable-credentials.rst index 866962550c..e1d93d5cd5 100644 --- a/docs/pages/deployment/verifiable-credentials.rst +++ b/docs/pages/deployment/verifiable-credentials.rst @@ -53,6 +53,8 @@ Fetching & Caching ^^^^^^^^^^^^^^^^^^ During startup of the node, remote contexts are fetched and cached. If the contents of a remote context changes, the node must be restarted in order for these changes to have effect. Only remote context listed in the `remoteallowlist` are fetched. +When strict mode is disabled, the node also fetches contexts that are not on the `remoteallowlist` from the internet, when a credential it processes refers to them. Use strict mode outside of development. +A remote context fetch fails after 5 seconds. A context that failed to load is not fetched again for 5 minutes: credentials referring to it fail immediately during that time, and are retried later. Local mappings can be used to pin a version of a context, so no unseen changes can be made. Working with local mappings is also useful for developing purposes when the remote context is older or non-existent. When you work with local mappings, make sure all nodes involved in the use-case have the same custom context configured. Searching and indexing diff --git a/docs/pages/release_notes.rst b/docs/pages/release_notes.rst index 865bcccc0e..81a71161df 100644 --- a/docs/pages/release_notes.rst +++ b/docs/pages/release_notes.rst @@ -14,6 +14,7 @@ Unreleased * #4233: ``request-credential`` API gains an optional ``credential_request_params`` JSON object overlaid on top of the OpenID4VCI Credential Request body sent to the issuer. Lets the wallet talk to issuers that accept additional fields, or to override the credential request entirely. ## Minor fixes/changes +* #4615: JSON-LD: fetching a remote context now times out after 5 seconds, and a context that failed to load is not fetched again for 5 minutes. Previously the fetch had no timeout: in non-strict mode a context server that never answered stopped network synchronization at the first credential referring to it, without logging an error. Remote contexts are now fetched with the node's HTTP client, so the strict-mode URL checks apply and the node identifies with its own User-Agent. By @reinkrul * Network: a peer that rejects an outbound connection with ``already connected`` is now retried with exponential backoff instead of every 1 to 5 seconds. By @stevenvegt in https://github.com/nuts-foundation/nuts-node/pull/4467 * Network: the default of ``network.maxbackoff`` is lowered from ``24h`` to ``1h``. The backoff is persisted across restarts and only reset when a peer's NutsComm address changes, so a peer that was unreachable for a few days could previously go unattempted for up to a day after it came back. By @stevenvegt in https://github.com/nuts-foundation/nuts-node/pull/4467 * Network: failed connection attempts are now logged at debug level instead of warning level. A warning is logged once, on the attempt that reaches ``network.maxbackoff``, so an unreachable peer no longer repeats the same warning on every retry. By @stevenvegt in https://github.com/nuts-foundation/nuts-node/pull/4467 diff --git a/jsonld/ldutils.go b/jsonld/ldutils.go index 4cce2c8f1e..1ff365d944 100644 --- a/jsonld/ldutils.go +++ b/jsonld/ldutils.go @@ -24,13 +24,25 @@ import ( "errors" "fmt" ssi "github.com/nuts-foundation/go-did" + "github.com/nuts-foundation/nuts-node/v6/http/client" "github.com/nuts-foundation/nuts-node/v6/jsonld/log" "github.com/nuts-foundation/nuts-node/v6/vcr/assets" "github.com/piprate/json-gold/ld" "io/fs" + "net/http" "net/url" + "sync" + "time" ) +// remoteContextTimeout bounds a single fetch of a remote JSON-LD context document. +// Contexts are fetched while processing network transactions, so a server that never answers must not block that processing. +var remoteContextTimeout = 5 * time.Second + +// remoteContextFailureTTL is how long a failed remote context fetch is remembered. +// During that time, loads of the same URL fail immediately with the original error instead of fetching again. +var remoteContextFailureTTL = 5 * time.Minute + // ContextsConfig contains config for json-ld document loader type ContextsConfig struct { // RemoteAllowList A list with urls as string which are allowed to request @@ -193,9 +205,11 @@ func NewContextLoader(allowUnlistedExternalCalls bool, contexts ContextsConfig) ld.NewCachingDocumentLoader( // Handle all embedded file system files NewEmbeddedFSDocumentLoader(assets.Assets, - // Last in the chain is the defaultLoader which can resolve - // local files and remote (via http) context documents - ld.NewDefaultDocumentLoader(nil)))) + // Remember failed remote fetches, so a dead context server costs one timeout instead of one per document + newFailureCachingLoader(remoteContextFailureTTL, + // Last in the chain is the defaultLoader which can resolve + // local files and remote (via http) context documents + ld.NewDefaultDocumentLoader(newRemoteContextHTTPClient()))))) // If unlisted calls are not allowed, filter all calls to the defaultLoader if !allowUnlistedExternalCalls { @@ -219,6 +233,87 @@ func NewContextLoader(allowUnlistedExternalCalls bool, contexts ContextsConfig) return loader, nil } +// newRemoteContextHTTPClient returns the HTTP client used to fetch remote JSON-LD contexts. +// json-gold needs a *http.Client, so the node's strict HTTP client (timeout, SSRF guard, URL checks, response size limit) +// is wrapped as its transport. +func newRemoteContextHTTPClient() *http.Client { + return &http.Client{Transport: strictClientTransport{client: client.New(remoteContextTimeout)}} +} + +// strictClientTransport adapts a client.StrictHTTPClient to http.RoundTripper. +// The strict client follows redirects itself, so the outer http.Client receives the final response. +type strictClientTransport struct { + client *client.StrictHTTPClient +} + +func (s strictClientTransport) RoundTrip(request *http.Request) (*http.Response, error) { + // RoundTrip must not modify the request, but the strict client sets the User-Agent header. + return s.client.Do(request.Clone(request.Context())) +} + +type failedLoad struct { + err error + until time.Time +} + +// failureCachingLoader remembers failed loads per URL for a fixed time, and returns the remembered error without +// calling the next loader during that time. Successful loads are not cached here (see ld.NewCachingDocumentLoader). +type failureCachingLoader struct { + ttl time.Duration + nextLoader ld.DocumentLoader + mutex sync.Mutex + failures map[string]failedLoad +} + +func newFailureCachingLoader(ttl time.Duration, nextLoader ld.DocumentLoader) *failureCachingLoader { + return &failureCachingLoader{ + ttl: ttl, + nextLoader: nextLoader, + failures: map[string]failedLoad{}, + } +} + +// LoadDocument returns the remembered error if loading u failed less than ttl ago, otherwise it calls the next loader. +func (f *failureCachingLoader) LoadDocument(u string) (*ld.RemoteDocument, error) { + if err := f.recentFailure(u); err != nil { + log.Logger().Debugf("Not fetching JSON-LD context, it failed recently (url=%s)", u) + return nil, err + } + document, err := f.nextLoader.LoadDocument(u) + if err != nil { + f.recordFailure(u, err) + log.Logger().WithError(err).Warnf("Failed to load JSON-LD context, not retrying for %s (url=%s)", f.ttl, u) + } + return document, err +} + +func (f *failureCachingLoader) recentFailure(u string) error { + f.mutex.Lock() + defer f.mutex.Unlock() + failure, ok := f.failures[u] + if !ok { + return nil + } + if time.Now().After(failure.until) { + delete(f.failures, u) + return nil + } + return failure.err +} + +func (f *failureCachingLoader) recordFailure(u string, err error) { + f.mutex.Lock() + defer f.mutex.Unlock() + now := time.Now() + // Prune expired entries, so the map stays bounded by the number of URLs that failed within the last ttl. + for failedURL, failure := range f.failures { + if now.After(failure.until) { + delete(f.failures, failedURL) + } + } + f.failures[u] = failedLoad{err: err, until: now.Add(f.ttl)} +} + // LDUtil package a set of often used JSON-LD operations for re-usability. type LDUtil struct { LDDocumentLoader ld.DocumentLoader diff --git a/jsonld/ldutils_test.go b/jsonld/ldutils_test.go index c3ca96d805..88fa32d09a 100644 --- a/jsonld/ldutils_test.go +++ b/jsonld/ldutils_test.go @@ -24,11 +24,15 @@ import ( "fmt" "net/http" "net/http/httptest" + "sync/atomic" "testing" + "time" ssi "github.com/nuts-foundation/go-did" + "github.com/nuts-foundation/nuts-node/v6/core" "github.com/piprate/json-gold/ld" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) //go:embed test/* @@ -273,3 +277,126 @@ func Test_filteredDocumentLoader(t *testing.T) { assert.False(t, mockLoader.Called) }) } + +func TestNewContextLoader_RemoteContexts(t *testing.T) { + t.Run("fails when the server does not answer within the timeout", func(t *testing.T) { + timeout := remoteContextTimeout + remoteContextTimeout = 100 * time.Millisecond + defer func() { remoteContextTimeout = timeout }() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // never answer, until the client gives up + <-r.Context().Done() + })) + defer srv.Close() + loader, err := NewContextLoader(true, DefaultContextConfig()) + require.NoError(t, err) + + start := time.Now() + _, err = loader.LoadDocument(srv.URL + "/context.jsonld") + + var jsonLDError *ld.JsonLdError + require.ErrorAs(t, err, &jsonLDError) + assert.Equal(t, ld.LoadingDocumentFailed, jsonLDError.Code) + assert.Less(t, time.Since(start), 5*time.Second) + }) + t.Run("does not fetch a context again after it failed", func(t *testing.T) { + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + loader, err := NewContextLoader(true, DefaultContextConfig()) + require.NoError(t, err) + + _, err1 := loader.LoadDocument(srv.URL + "/context.jsonld") + _, err2 := loader.LoadDocument(srv.URL + "/context.jsonld") + + require.Error(t, err1) + assert.Same(t, err1, err2) + assert.Equal(t, int32(1), requests.Load()) + }) + t.Run("identifies as the Nuts node", func(t *testing.T) { + var userAgent string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + userAgent = r.UserAgent() + w.Header().Set("Content-Type", "application/ld+json") + _, _ = w.Write([]byte(`{"@context":{}}`)) + })) + defer srv.Close() + loader, err := NewContextLoader(true, DefaultContextConfig()) + require.NoError(t, err) + + _, err = loader.LoadDocument(srv.URL + "/context.jsonld") + + require.NoError(t, err) + assert.Equal(t, core.UserAgent(), userAgent) + }) +} + +type countingLoader struct { + calls int + err error +} + +func (c *countingLoader) LoadDocument(u string) (*ld.RemoteDocument, error) { + c.calls++ + if c.err != nil { + return nil, c.err + } + return &ld.RemoteDocument{DocumentURL: u}, nil +} + +func Test_failureCachingLoader(t *testing.T) { + const contextURL = "https://example.com/context.jsonld" + t.Run("returns the remembered error without calling the next loader", func(t *testing.T) { + next := &countingLoader{err: ld.NewJsonLdError(ld.LoadingDocumentFailed, "boom")} + sut := newFailureCachingLoader(time.Minute, next) + + _, err1 := sut.LoadDocument(contextURL) + _, err2 := sut.LoadDocument(contextURL) + + assert.Same(t, err1, err2) + assert.Equal(t, 1, next.calls) + }) + t.Run("calls the next loader again after the failure expired", func(t *testing.T) { + next := &countingLoader{err: ld.NewJsonLdError(ld.LoadingDocumentFailed, "boom")} + sut := newFailureCachingLoader(time.Minute, next) + _, _ = sut.LoadDocument(contextURL) + sut.failures[contextURL] = failedLoad{err: next.err, until: time.Now().Add(-time.Second)} + + _, _ = sut.LoadDocument(contextURL) + + assert.Equal(t, 2, next.calls) + }) + t.Run("does not remember successful loads", func(t *testing.T) { + next := &countingLoader{} + sut := newFailureCachingLoader(time.Minute, next) + + _, err1 := sut.LoadDocument(contextURL) + _, err2 := sut.LoadDocument(contextURL) + + assert.NoError(t, err1) + assert.NoError(t, err2) + assert.Equal(t, 2, next.calls) + }) + t.Run("remembers failures per URL", func(t *testing.T) { + next := &countingLoader{err: ld.NewJsonLdError(ld.LoadingDocumentFailed, "boom")} + sut := newFailureCachingLoader(time.Minute, next) + + _, _ = sut.LoadDocument(contextURL) + _, _ = sut.LoadDocument("https://example.com/other.jsonld") + + assert.Equal(t, 2, next.calls) + }) + t.Run("prunes expired failures when recording a new one", func(t *testing.T) { + next := &countingLoader{err: ld.NewJsonLdError(ld.LoadingDocumentFailed, "boom")} + sut := newFailureCachingLoader(time.Minute, next) + sut.failures["https://example.com/expired.jsonld"] = failedLoad{err: next.err, until: time.Now().Add(-time.Second)} + + _, _ = sut.LoadDocument(contextURL) + + assert.Len(t, sut.failures, 1) + assert.Contains(t, sut.failures, contextURL) + }) +} From 4b306689b0b6f036dfd43a971a87d0f3f6016157 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Thu, 8 Oct 2026 11:01:16 +0200 Subject: [PATCH 2/7] docs: link PR in release note Assisted by AI --- docs/pages/release_notes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/release_notes.rst b/docs/pages/release_notes.rst index 81a71161df..be3da38758 100644 --- a/docs/pages/release_notes.rst +++ b/docs/pages/release_notes.rst @@ -14,7 +14,7 @@ Unreleased * #4233: ``request-credential`` API gains an optional ``credential_request_params`` JSON object overlaid on top of the OpenID4VCI Credential Request body sent to the issuer. Lets the wallet talk to issuers that accept additional fields, or to override the credential request entirely. ## Minor fixes/changes -* #4615: JSON-LD: fetching a remote context now times out after 5 seconds, and a context that failed to load is not fetched again for 5 minutes. Previously the fetch had no timeout: in non-strict mode a context server that never answered stopped network synchronization at the first credential referring to it, without logging an error. Remote contexts are now fetched with the node's HTTP client, so the strict-mode URL checks apply and the node identifies with its own User-Agent. By @reinkrul +* #4615: JSON-LD: fetching a remote context now times out after 5 seconds, and a context that failed to load is not fetched again for 5 minutes. Previously the fetch had no timeout: in non-strict mode a context server that never answered stopped network synchronization at the first credential referring to it, without logging an error. Remote contexts are now fetched with the node's HTTP client, so the strict-mode URL checks apply and the node identifies with its own User-Agent. By @reinkrul in https://github.com/nuts-foundation/nuts-node/pull/4617 * Network: a peer that rejects an outbound connection with ``already connected`` is now retried with exponential backoff instead of every 1 to 5 seconds. By @stevenvegt in https://github.com/nuts-foundation/nuts-node/pull/4467 * Network: the default of ``network.maxbackoff`` is lowered from ``24h`` to ``1h``. The backoff is persisted across restarts and only reset when a peer's NutsComm address changes, so a peer that was unreachable for a few days could previously go unattempted for up to a day after it came back. By @stevenvegt in https://github.com/nuts-foundation/nuts-node/pull/4467 * Network: failed connection attempts are now logged at debug level instead of warning level. A warning is logged once, on the attempt that reaches ``network.maxbackoff``, so an unreachable peer no longer repeats the same warning on every retry. By @stevenvegt in https://github.com/nuts-foundation/nuts-node/pull/4467 From 43d807484058aecd41a5228181bdc61975add5ae Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Thu, 12 Nov 2026 07:56:14 +0100 Subject: [PATCH 3/7] refactor(jsonld): use a strict *http.Client and go-cache for remote contexts - Add client.NewHTTPClient: an *http.Client whose transport applies the strict checks (URL check, User-Agent, response size limit) on every request, including redirects. Replaces the jsonld adapter that nested a StrictHTTPClient inside an outer http.Client. - Use go-cache for the failure cache instead of a hand-written TTL map. Assisted by AI --- http/client/client.go | 41 +++++++++++++++++++ http/client/client_test.go | 82 ++++++++++++++++++++++++++++++++++++++ jsonld/ldutils.go | 66 ++++-------------------------- jsonld/ldutils_test.go | 58 +++++++++++++++++++++------ 4 files changed, 176 insertions(+), 71 deletions(-) diff --git a/http/client/client.go b/http/client/client.go index 62f82e0600..1478c5443b 100644 --- a/http/client/client.go +++ b/http/client/client.go @@ -169,6 +169,47 @@ func NewWithTLSConfig(timeout time.Duration, tlsConfig *tls.Config, options ...O }, options) } +// NewHTTPClient creates an *http.Client with the same protections as New, for libraries that require an *http.Client. +// The protections are applied by the transport, so they apply to every request, including redirects. +func NewHTTPClient(timeout time.Duration) *http.Client { + return &http.Client{ + Transport: strictTransport{next: getTransport(SafeHttpTransport), maxResponseSize: DefaultMaxHttpResponseSize}, + Timeout: timeout, + } +} + +// strictTransport applies the checks of StrictHTTPClient.Do to every request it sends: +// the target URL check, the User-Agent header and the response size limit. +type strictTransport struct { + next http.RoundTripper + maxResponseSize int64 +} + +func (s strictTransport) RoundTrip(req *http.Request) (*http.Response, error) { + // IP-literal hosts are left to the dial guard (see denyNonPublicAddr): it is aware of + // http.client.allowedinternalcidrs/deniedcidrs, this check is not. + if _, err := core.ParsePublicURLAllowIP(req.URL.String(), StrictMode); err != nil { + return nil, fmt.Errorf("invalid target URL: %w", err) + } + // RoundTrip must not modify the request, so set the User-Agent on a copy. + req = req.Clone(req.Context()) + req.Header.Set("User-Agent", core.UserAgent()) + result, err := s.next.RoundTrip(req) + if err != nil { + return nil, err + } + if result.Body != nil { + originalBody := result.Body + defer originalBody.Close() + body, err := limitedReadAll(originalBody, s.maxResponseSize) + if err != nil { + return nil, err + } + result.Body = io.NopCloser(bytes.NewReader(body)) + } + return result, nil +} + type StrictHTTPClient struct { client *http.Client maxResponseSize int64 diff --git a/http/client/client_test.go b/http/client/client_test.go index 997d47459b..dc146f1b46 100644 --- a/http/client/client_test.go +++ b/http/client/client_test.go @@ -31,6 +31,7 @@ import ( "testing" "time" + "github.com/nuts-foundation/nuts-node/v6/core" "github.com/nuts-foundation/nuts-node/v6/tracing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -476,3 +477,84 @@ func TestNew(t *testing.T) { assert.Equal(t, SafeHttpTransport, client.client.Transport) }) } + +func TestNewHTTPClient(t *testing.T) { + oldStrictMode := StrictMode + t.Cleanup(func() { StrictMode = oldStrictMode }) + newClient := func(next http.RoundTripper, maxResponseSize int64) *http.Client { + client := NewHTTPClient(time.Second) + client.Transport = strictTransport{next: next, maxResponseSize: maxResponseSize} + return client + } + t.Run("strict mode rejects HTTP target", func(t *testing.T) { + StrictMode = true + rt := &stubRoundTripper{} + request, _ := http.NewRequest(http.MethodGet, "http://nuts.nl", nil) + + _, err := newClient(rt, DefaultMaxHttpResponseSize).Do(request) + + assert.ErrorContains(t, err, "invalid target URL") + assert.Equal(t, 0, rt.invocations) + }) + t.Run("strict mode rejects redirect to HTTP target", func(t *testing.T) { + StrictMode = true + rt := &stubRoundTripper{ + statusCode: http.StatusFound, + headers: map[string]string{"Location": "http://nuts.nl/other"}, + } + request, _ := http.NewRequest(http.MethodGet, "https://nuts.nl", nil) + + _, err := newClient(rt, DefaultMaxHttpResponseSize).Do(request) + + assert.ErrorContains(t, err, "invalid target URL") + assert.Equal(t, 1, rt.invocations) + }) + t.Run("sets User-Agent without modifying the request", func(t *testing.T) { + StrictMode = false + var userAgent string + rt := &stubRoundTripper{ + statusCode: http.StatusOK, + dataFn: func(r *http.Request) []byte { + userAgent = r.UserAgent() + return nil + }, + } + request, _ := http.NewRequest(http.MethodGet, "https://example.com", nil) + + _, err := newClient(rt, DefaultMaxHttpResponseSize).Do(request) + + require.NoError(t, err) + assert.Equal(t, core.UserAgent(), userAgent) + assert.Empty(t, request.Header.Get("User-Agent")) + }) + t.Run("rejects response that exceeds the limit, and closes its body", func(t *testing.T) { + StrictMode = false + transport := &closeRecordingTransport{base: &stubRoundTripper{statusCode: http.StatusOK, data: []byte("12345678901")}} + request, _ := http.NewRequest(http.MethodGet, "https://example.com", nil) + + _, err := newClient(transport, 10).Do(request) + + assert.ErrorContains(t, err, "data to read exceeds max. safety limit of 10 bytes") + assert.Equal(t, int32(1), transport.closed.Load()) + }) + t.Run("returns response within the limit", func(t *testing.T) { + StrictMode = false + request, _ := http.NewRequest(http.MethodGet, "https://example.com", nil) + + response, err := newClient(&stubRoundTripper{statusCode: http.StatusOK, data: []byte("1234567890")}, 10).Do(request) + + require.NoError(t, err) + data, _ := io.ReadAll(response.Body) + assert.Equal(t, "1234567890", string(data)) + }) + t.Run("uses SafeHttpTransport and the given timeout", func(t *testing.T) { + original := tracing.Enabled() + tracing.SetEnabled(false) + t.Cleanup(func() { tracing.SetEnabled(original) }) + + client := NewHTTPClient(5 * time.Second) + + assert.Equal(t, strictTransport{next: SafeHttpTransport, maxResponseSize: DefaultMaxHttpResponseSize}, client.Transport) + assert.Equal(t, 5*time.Second, client.Timeout) + }) +} diff --git a/jsonld/ldutils.go b/jsonld/ldutils.go index 1ff365d944..11f8549d62 100644 --- a/jsonld/ldutils.go +++ b/jsonld/ldutils.go @@ -27,11 +27,10 @@ import ( "github.com/nuts-foundation/nuts-node/v6/http/client" "github.com/nuts-foundation/nuts-node/v6/jsonld/log" "github.com/nuts-foundation/nuts-node/v6/vcr/assets" + gocache "github.com/patrickmn/go-cache" "github.com/piprate/json-gold/ld" "io/fs" - "net/http" "net/url" - "sync" "time" ) @@ -209,7 +208,7 @@ func NewContextLoader(allowUnlistedExternalCalls bool, contexts ContextsConfig) newFailureCachingLoader(remoteContextFailureTTL, // Last in the chain is the defaultLoader which can resolve // local files and remote (via http) context documents - ld.NewDefaultDocumentLoader(newRemoteContextHTTPClient()))))) + ld.NewDefaultDocumentLoader(client.NewHTTPClient(remoteContextTimeout)))))) // If unlisted calls are not allowed, filter all calls to the defaultLoader if !allowUnlistedExternalCalls { @@ -233,87 +232,36 @@ func NewContextLoader(allowUnlistedExternalCalls bool, contexts ContextsConfig) return loader, nil } -// newRemoteContextHTTPClient returns the HTTP client used to fetch remote JSON-LD contexts. -// json-gold needs a *http.Client, so the node's strict HTTP client (timeout, SSRF guard, URL checks, response size limit) -// is wrapped as its transport. -func newRemoteContextHTTPClient() *http.Client { - return &http.Client{Transport: strictClientTransport{client: client.New(remoteContextTimeout)}} -} - -// strictClientTransport adapts a client.StrictHTTPClient to http.RoundTripper. -// The strict client follows redirects itself, so the outer http.Client receives the final response. -type strictClientTransport struct { - client *client.StrictHTTPClient -} - -func (s strictClientTransport) RoundTrip(request *http.Request) (*http.Response, error) { - // RoundTrip must not modify the request, but the strict client sets the User-Agent header. - return s.client.Do(request.Clone(request.Context())) -} - -type failedLoad struct { - err error - until time.Time -} - // failureCachingLoader remembers failed loads per URL for a fixed time, and returns the remembered error without // calling the next loader during that time. Successful loads are not cached here (see ld.NewCachingDocumentLoader). type failureCachingLoader struct { ttl time.Duration nextLoader ld.DocumentLoader - mutex sync.Mutex - failures map[string]failedLoad + failures *gocache.Cache } func newFailureCachingLoader(ttl time.Duration, nextLoader ld.DocumentLoader) *failureCachingLoader { return &failureCachingLoader{ ttl: ttl, nextLoader: nextLoader, - failures: map[string]failedLoad{}, + failures: gocache.New(ttl, ttl), } } // LoadDocument returns the remembered error if loading u failed less than ttl ago, otherwise it calls the next loader. func (f *failureCachingLoader) LoadDocument(u string) (*ld.RemoteDocument, error) { - if err := f.recentFailure(u); err != nil { + if failure, ok := f.failures.Get(u); ok { log.Logger().Debugf("Not fetching JSON-LD context, it failed recently (url=%s)", u) - return nil, err + return nil, failure.(error) } document, err := f.nextLoader.LoadDocument(u) if err != nil { - f.recordFailure(u, err) + f.failures.SetDefault(u, err) log.Logger().WithError(err).Warnf("Failed to load JSON-LD context, not retrying for %s (url=%s)", f.ttl, u) } return document, err } -func (f *failureCachingLoader) recentFailure(u string) error { - f.mutex.Lock() - defer f.mutex.Unlock() - failure, ok := f.failures[u] - if !ok { - return nil - } - if time.Now().After(failure.until) { - delete(f.failures, u) - return nil - } - return failure.err -} - -func (f *failureCachingLoader) recordFailure(u string, err error) { - f.mutex.Lock() - defer f.mutex.Unlock() - now := time.Now() - // Prune expired entries, so the map stays bounded by the number of URLs that failed within the last ttl. - for failedURL, failure := range f.failures { - if now.After(failure.until) { - delete(f.failures, failedURL) - } - } - f.failures[u] = failedLoad{err: err, until: now.Add(f.ttl)} -} - // LDUtil package a set of often used JSON-LD operations for re-usability. type LDUtil struct { LDDocumentLoader ld.DocumentLoader diff --git a/jsonld/ldutils_test.go b/jsonld/ldutils_test.go index 88fa32d09a..10e30f9d97 100644 --- a/jsonld/ldutils_test.go +++ b/jsonld/ldutils_test.go @@ -30,6 +30,7 @@ import ( ssi "github.com/nuts-foundation/go-did" "github.com/nuts-foundation/nuts-node/v6/core" + "github.com/nuts-foundation/nuts-node/v6/http/client" "github.com/piprate/json-gold/ld" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -332,6 +333,49 @@ func TestNewContextLoader_RemoteContexts(t *testing.T) { require.NoError(t, err) assert.Equal(t, core.UserAgent(), userAgent) }) + t.Run("returns the URL of the final request after a redirect", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/old.jsonld" { + http.Redirect(w, r, "/new.jsonld", http.StatusFound) + return + } + w.Header().Set("Content-Type", "application/ld+json") + _, _ = w.Write([]byte(`{"@context":{}}`)) + })) + defer srv.Close() + loader, err := NewContextLoader(true, DefaultContextConfig()) + require.NoError(t, err) + + document, err := loader.LoadDocument(srv.URL + "/old.jsonld") + + require.NoError(t, err) + assert.Equal(t, srv.URL+"/new.jsonld", document.DocumentURL) + }) + t.Run("fails on invalid JSON", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`not JSON`)) + })) + defer srv.Close() + loader, err := NewContextLoader(true, DefaultContextConfig()) + require.NoError(t, err) + + _, err = loader.LoadDocument(srv.URL + "/context.jsonld") + + var jsonLDError *ld.JsonLdError + require.ErrorAs(t, err, &jsonLDError) + assert.Equal(t, ld.LoadingDocumentFailed, jsonLDError.Code) + }) + t.Run("strict mode rejects an HTTP URL", func(t *testing.T) { + strictMode := client.StrictMode + client.StrictMode = true + defer func() { client.StrictMode = strictMode }() + loader, err := NewContextLoader(true, DefaultContextConfig()) + require.NoError(t, err) + + _, err = loader.LoadDocument("http://example.com/context.jsonld") + + assert.ErrorContains(t, err, "invalid target URL") + }) } type countingLoader struct { @@ -361,9 +405,9 @@ func Test_failureCachingLoader(t *testing.T) { }) t.Run("calls the next loader again after the failure expired", func(t *testing.T) { next := &countingLoader{err: ld.NewJsonLdError(ld.LoadingDocumentFailed, "boom")} - sut := newFailureCachingLoader(time.Minute, next) + sut := newFailureCachingLoader(time.Millisecond, next) _, _ = sut.LoadDocument(contextURL) - sut.failures[contextURL] = failedLoad{err: next.err, until: time.Now().Add(-time.Second)} + time.Sleep(10 * time.Millisecond) _, _ = sut.LoadDocument(contextURL) @@ -389,14 +433,4 @@ func Test_failureCachingLoader(t *testing.T) { assert.Equal(t, 2, next.calls) }) - t.Run("prunes expired failures when recording a new one", func(t *testing.T) { - next := &countingLoader{err: ld.NewJsonLdError(ld.LoadingDocumentFailed, "boom")} - sut := newFailureCachingLoader(time.Minute, next) - sut.failures["https://example.com/expired.jsonld"] = failedLoad{err: next.err, until: time.Now().Add(-time.Second)} - - _, _ = sut.LoadDocument(contextURL) - - assert.Len(t, sut.failures, 1) - assert.Contains(t, sut.failures, contextURL) - }) } From 9f9c4a952060ffe2193be133c02e26efd2f482a9 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Thu, 12 Nov 2026 08:20:09 +0100 Subject: [PATCH 4/7] refactor(http): apply StrictHTTPClient checks in its transport StrictHTTPClient.Do now delegates to its http.Client, whose transport (strictTransport) applies the URL check, User-Agent and response size limit on every request. NewHTTPClient returns that same client, so there is one implementation of the strict checks. Errors are now wrapped by http.Client (*url.Error), e.g. 'Get "": invalid target URL: ...'. Assisted by AI --- http/client/client.go | 35 +++++++---------------------------- http/client/client_test.go | 12 ++++++------ vcr/vcr_test.go | 2 +- 3 files changed, 14 insertions(+), 35 deletions(-) diff --git a/http/client/client.go b/http/client/client.go index 1478c5443b..5495715bc0 100644 --- a/http/client/client.go +++ b/http/client/client.go @@ -71,9 +71,8 @@ var StrictMode bool // checkRedirect is the http.Client.CheckRedirect policy for the strict HTTP client. // Setting CheckRedirect replaces the standard library's default policy, so the // 10-redirect cap is reimplemented here. It also re-runs core.ParsePublicURLAllowIP on -// every redirect target: the check in Do only guards the first hop, so without this a -// valid remote host could redirect the client onto a plaintext or otherwise disallowed -// endpoint. IP-literal hosts are left to the dial guard (see denyNonPublicAddr), which +// every redirect target, so a disallowed redirect is refused before it is followed +// (strictTransport checks every hop as well). IP-literal hosts are left to the dial guard (see denyNonPublicAddr), which // runs on every hop already and is aware of http.client.allowedinternalcidrs/deniedcidrs; // rejecting them here too would make that allowlist unreachable. func checkRedirect(req *http.Request, via []*http.Request) error { @@ -131,6 +130,7 @@ func newStrictHTTPClient(httpClient *http.Client, options []Option) *StrictHTTPC for _, option := range options { option(result) } + httpClient.Transport = strictTransport{next: httpClient.Transport, maxResponseSize: result.maxResponseSize} return result } @@ -172,13 +172,10 @@ func NewWithTLSConfig(timeout time.Duration, tlsConfig *tls.Config, options ...O // NewHTTPClient creates an *http.Client with the same protections as New, for libraries that require an *http.Client. // The protections are applied by the transport, so they apply to every request, including redirects. func NewHTTPClient(timeout time.Duration) *http.Client { - return &http.Client{ - Transport: strictTransport{next: getTransport(SafeHttpTransport), maxResponseSize: DefaultMaxHttpResponseSize}, - Timeout: timeout, - } + return New(timeout).client } -// strictTransport applies the checks of StrictHTTPClient.Do to every request it sends: +// strictTransport applies the strict checks to every request it sends, including redirects: // the target URL check, the User-Agent header and the response size limit. type strictTransport struct { next http.RoundTripper @@ -215,25 +212,7 @@ type StrictHTTPClient struct { maxResponseSize int64 } +// Do sends the request. The strict checks are applied by the client's transport (see strictTransport). func (s *StrictHTTPClient) Do(req *http.Request) (*http.Response, error) { - // IP-literal hosts are left to the dial guard (see denyNonPublicAddr): it is aware of - // http.client.allowedinternalcidrs/deniedcidrs, this check is not. - if _, err := core.ParsePublicURLAllowIP(req.URL.String(), StrictMode); err != nil { - return nil, fmt.Errorf("invalid target URL: %w", err) - } - req.Header.Set("User-Agent", core.UserAgent()) - result, err := s.client.Do(req) - if err != nil { - return nil, err - } - if result.Body != nil { - originalBody := result.Body - defer originalBody.Close() - body, err := limitedReadAll(originalBody, s.maxResponseSize) - if err != nil { - return nil, err - } - result.Body = io.NopCloser(bytes.NewReader(body)) - } - return result, nil + return s.client.Do(req) } diff --git a/http/client/client_test.go b/http/client/client_test.go index dc146f1b46..7b058e2aff 100644 --- a/http/client/client_test.go +++ b/http/client/client_test.go @@ -86,7 +86,7 @@ func TestStrictHTTPClient(t *testing.T) { client := NewWithTLSConfig(time.Second, &tls.Config{ InsecureSkipVerify: true, }) - ts := client.client.Transport.(*http.Transport) + ts := client.client.Transport.(strictTransport).next.(*http.Transport) assert.True(t, ts.TLSClientConfig.InsecureSkipVerify) }) }) @@ -265,7 +265,7 @@ func TestWithMaxResponseSize(t *testing.T) { _, err := New(time.Second).Do(request) - assert.EqualError(t, err, "data to read exceeds max. safety limit of 1048576 bytes") + assert.ErrorContains(t, err, "data to read exceeds max. safety limit of 1048576 bytes") }) t.Run("raised limit accepts response", func(t *testing.T) { request, _ := http.NewRequest(http.MethodGet, server.URL, nil) @@ -303,7 +303,7 @@ func TestStrictHTTPClient_ClosesResponseBody(t *testing.T) { _, err := client.Do(request) - assert.EqualError(t, err, "data to read exceeds max. safety limit of 10 bytes") + assert.ErrorContains(t, err, "data to read exceeds max. safety limit of 10 bytes") assert.Equal(t, int32(1), transport.closed.Load()) }) t.Run("response within limit", func(t *testing.T) { @@ -464,7 +464,7 @@ func TestNew(t *testing.T) { client := New(time.Second) // Transport should be wrapped (not equal to SafeHttpTransport) - assert.NotEqual(t, SafeHttpTransport, client.client.Transport) + assert.NotEqual(t, SafeHttpTransport, client.client.Transport.(strictTransport).next) }) t.Run("uses SafeHttpTransport when tracing disabled", func(t *testing.T) { @@ -474,7 +474,7 @@ func TestNew(t *testing.T) { client := New(time.Second) - assert.Equal(t, SafeHttpTransport, client.client.Transport) + assert.Equal(t, SafeHttpTransport, client.client.Transport.(strictTransport).next) }) } @@ -506,7 +506,7 @@ func TestNewHTTPClient(t *testing.T) { _, err := newClient(rt, DefaultMaxHttpResponseSize).Do(request) - assert.ErrorContains(t, err, "invalid target URL") + assert.ErrorContains(t, err, "invalid redirect target") assert.Equal(t, 1, rt.invocations) }) t.Run("sets User-Agent without modifying the request", func(t *testing.T) { diff --git a/vcr/vcr_test.go b/vcr/vcr_test.go index 479409b369..c208010382 100644 --- a/vcr/vcr_test.go +++ b/vcr/vcr_test.go @@ -108,7 +108,7 @@ func TestVCR_Configure(t *testing.T) { require.NoError(t, err) err = issuer.OfferCredential(context.Background(), testVC, "http://example.com") - assert.ErrorContains(t, err, "http request error: invalid target URL") + assert.ErrorContains(t, err, "invalid target URL") assert.ErrorContains(t, err, "scheme must be https") }) } From 9571f1cfd66dd3f3e852eb17f2d7922c4df80226 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Thu, 12 Nov 2026 09:52:48 +0100 Subject: [PATCH 5/7] refactor(http): order strictTransport after StrictHTTPClient for a smaller diff Assisted by AI --- http/client/client.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/http/client/client.go b/http/client/client.go index 5495715bc0..5fd39c0f1c 100644 --- a/http/client/client.go +++ b/http/client/client.go @@ -175,6 +175,16 @@ func NewHTTPClient(timeout time.Duration) *http.Client { return New(timeout).client } +type StrictHTTPClient struct { + client *http.Client + maxResponseSize int64 +} + +// Do sends the request. The strict checks are applied by the client's transport (see strictTransport). +func (s *StrictHTTPClient) Do(req *http.Request) (*http.Response, error) { + return s.client.Do(req) +} + // strictTransport applies the strict checks to every request it sends, including redirects: // the target URL check, the User-Agent header and the response size limit. type strictTransport struct { @@ -206,13 +216,3 @@ func (s strictTransport) RoundTrip(req *http.Request) (*http.Response, error) { } return result, nil } - -type StrictHTTPClient struct { - client *http.Client - maxResponseSize int64 -} - -// Do sends the request. The strict checks are applied by the client's transport (see strictTransport). -func (s *StrictHTTPClient) Do(req *http.Request) (*http.Response, error) { - return s.client.Do(req) -} From 0350cf4a0e906cefdc2c14beac4fd77a1f9097c1 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Thu, 12 Nov 2026 09:54:13 +0100 Subject: [PATCH 6/7] test(http): drop NewHTTPClient tests already covered by StrictHTTPClient tests Assisted by AI --- http/client/client_test.go | 95 +++++++++----------------------------- 1 file changed, 22 insertions(+), 73 deletions(-) diff --git a/http/client/client_test.go b/http/client/client_test.go index 7b058e2aff..0580902dbe 100644 --- a/http/client/client_test.go +++ b/http/client/client_test.go @@ -478,83 +478,32 @@ func TestNew(t *testing.T) { }) } -func TestNewHTTPClient(t *testing.T) { - oldStrictMode := StrictMode - t.Cleanup(func() { StrictMode = oldStrictMode }) - newClient := func(next http.RoundTripper, maxResponseSize int64) *http.Client { - client := NewHTTPClient(time.Second) - client.Transport = strictTransport{next: next, maxResponseSize: maxResponseSize} - return client +func TestStrictHTTPClient_UserAgent(t *testing.T) { + var userAgent string + rt := &stubRoundTripper{ + statusCode: http.StatusOK, + dataFn: func(r *http.Request) []byte { + userAgent = r.UserAgent() + return nil + }, } - t.Run("strict mode rejects HTTP target", func(t *testing.T) { - StrictMode = true - rt := &stubRoundTripper{} - request, _ := http.NewRequest(http.MethodGet, "http://nuts.nl", nil) - - _, err := newClient(rt, DefaultMaxHttpResponseSize).Do(request) - - assert.ErrorContains(t, err, "invalid target URL") - assert.Equal(t, 0, rt.invocations) - }) - t.Run("strict mode rejects redirect to HTTP target", func(t *testing.T) { - StrictMode = true - rt := &stubRoundTripper{ - statusCode: http.StatusFound, - headers: map[string]string{"Location": "http://nuts.nl/other"}, - } - request, _ := http.NewRequest(http.MethodGet, "https://nuts.nl", nil) - - _, err := newClient(rt, DefaultMaxHttpResponseSize).Do(request) - - assert.ErrorContains(t, err, "invalid redirect target") - assert.Equal(t, 1, rt.invocations) - }) - t.Run("sets User-Agent without modifying the request", func(t *testing.T) { - StrictMode = false - var userAgent string - rt := &stubRoundTripper{ - statusCode: http.StatusOK, - dataFn: func(r *http.Request) []byte { - userAgent = r.UserAgent() - return nil - }, - } - request, _ := http.NewRequest(http.MethodGet, "https://example.com", nil) - - _, err := newClient(rt, DefaultMaxHttpResponseSize).Do(request) + request, _ := http.NewRequest(http.MethodGet, "https://nuts.nl", nil) - require.NoError(t, err) - assert.Equal(t, core.UserAgent(), userAgent) - assert.Empty(t, request.Header.Get("User-Agent")) - }) - t.Run("rejects response that exceeds the limit, and closes its body", func(t *testing.T) { - StrictMode = false - transport := &closeRecordingTransport{base: &stubRoundTripper{statusCode: http.StatusOK, data: []byte("12345678901")}} - request, _ := http.NewRequest(http.MethodGet, "https://example.com", nil) - - _, err := newClient(transport, 10).Do(request) - - assert.ErrorContains(t, err, "data to read exceeds max. safety limit of 10 bytes") - assert.Equal(t, int32(1), transport.closed.Load()) - }) - t.Run("returns response within the limit", func(t *testing.T) { - StrictMode = false - request, _ := http.NewRequest(http.MethodGet, "https://example.com", nil) + _, err := newStrictHTTPClient(&http.Client{Transport: rt}, nil).Do(request) - response, err := newClient(&stubRoundTripper{statusCode: http.StatusOK, data: []byte("1234567890")}, 10).Do(request) + require.NoError(t, err) + assert.Equal(t, core.UserAgent(), userAgent) + assert.Empty(t, request.Header.Get("User-Agent"), "the caller's request must not be modified") +} - require.NoError(t, err) - data, _ := io.ReadAll(response.Body) - assert.Equal(t, "1234567890", string(data)) - }) - t.Run("uses SafeHttpTransport and the given timeout", func(t *testing.T) { - original := tracing.Enabled() - tracing.SetEnabled(false) - t.Cleanup(func() { tracing.SetEnabled(original) }) +func TestNewHTTPClient(t *testing.T) { + original := tracing.Enabled() + tracing.SetEnabled(false) + t.Cleanup(func() { tracing.SetEnabled(original) }) - client := NewHTTPClient(5 * time.Second) + client := NewHTTPClient(5 * time.Second) - assert.Equal(t, strictTransport{next: SafeHttpTransport, maxResponseSize: DefaultMaxHttpResponseSize}, client.Transport) - assert.Equal(t, 5*time.Second, client.Timeout) - }) + assert.Equal(t, strictTransport{next: SafeHttpTransport, maxResponseSize: DefaultMaxHttpResponseSize}, client.Transport) + assert.Equal(t, 5*time.Second, client.Timeout) + assert.NotNil(t, client.CheckRedirect) } From 616897da89c703fd7c1035dfc2dadfa3b65e7211 Mon Sep 17 00:00:00 2001 From: Rein Krul Date: Thu, 12 Nov 2026 10:23:43 +0100 Subject: [PATCH 7/7] fix(http): StrictHTTPClient.Do returns no response with an error http.Client returns the last response together with the error when CheckRedirect refuses a redirect. Do used to return only the error; keep it that way. Also clarify that only credentials received over the network are retried, and re-wrap the checkRedirect comment. Assisted by AI --- docs/pages/deployment/verifiable-credentials.rst | 2 +- http/client/client.go | 11 ++++++++--- http/client/client_test.go | 3 ++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/pages/deployment/verifiable-credentials.rst b/docs/pages/deployment/verifiable-credentials.rst index e1d93d5cd5..e698531c9c 100644 --- a/docs/pages/deployment/verifiable-credentials.rst +++ b/docs/pages/deployment/verifiable-credentials.rst @@ -54,7 +54,7 @@ Fetching & Caching During startup of the node, remote contexts are fetched and cached. If the contents of a remote context changes, the node must be restarted in order for these changes to have effect. Only remote context listed in the `remoteallowlist` are fetched. When strict mode is disabled, the node also fetches contexts that are not on the `remoteallowlist` from the internet, when a credential it processes refers to them. Use strict mode outside of development. -A remote context fetch fails after 5 seconds. A context that failed to load is not fetched again for 5 minutes: credentials referring to it fail immediately during that time, and are retried later. +A remote context fetch fails after 5 seconds. A context that failed to load is not fetched again for 5 minutes: credentials referring to it fail immediately during that time. Credentials received over the network are retried later. Local mappings can be used to pin a version of a context, so no unseen changes can be made. Working with local mappings is also useful for developing purposes when the remote context is older or non-existent. When you work with local mappings, make sure all nodes involved in the use-case have the same custom context configured. Searching and indexing diff --git a/http/client/client.go b/http/client/client.go index 5fd39c0f1c..0ab7348f8f 100644 --- a/http/client/client.go +++ b/http/client/client.go @@ -72,8 +72,8 @@ var StrictMode bool // Setting CheckRedirect replaces the standard library's default policy, so the // 10-redirect cap is reimplemented here. It also re-runs core.ParsePublicURLAllowIP on // every redirect target, so a disallowed redirect is refused before it is followed -// (strictTransport checks every hop as well). IP-literal hosts are left to the dial guard (see denyNonPublicAddr), which -// runs on every hop already and is aware of http.client.allowedinternalcidrs/deniedcidrs; +// (strictTransport checks every hop as well). IP-literal hosts are left to the dial +// guard (see denyNonPublicAddr), which runs on every hop already and is aware of http.client.allowedinternalcidrs/deniedcidrs; // rejecting them here too would make that allowlist unreachable. func checkRedirect(req *http.Request, via []*http.Request) error { if len(via) >= 10 { @@ -182,7 +182,12 @@ type StrictHTTPClient struct { // Do sends the request. The strict checks are applied by the client's transport (see strictTransport). func (s *StrictHTTPClient) Do(req *http.Request) (*http.Response, error) { - return s.client.Do(req) + result, err := s.client.Do(req) + if err != nil { + // http.Client also returns the last response when CheckRedirect fails; callers only get the error. + return nil, err + } + return result, nil } // strictTransport applies the strict checks to every request it sends, including redirects: diff --git a/http/client/client_test.go b/http/client/client_test.go index 0580902dbe..d155cabc8b 100644 --- a/http/client/client_test.go +++ b/http/client/client_test.go @@ -218,9 +218,10 @@ func TestStrictHTTPClient_RedirectScheme(t *testing.T) { client := NewWithTLSConfig(time.Second, tlsConfig) req, _ := http.NewRequest("GET", redirector.URL, nil) - _, err := client.Do(req) + response, err := client.Do(req) assert.Error(t, err, "strict mode must not follow a redirect from HTTPS to HTTP") + assert.Nil(t, response, "no response must be returned with the error") assert.False(t, reached.Load(), "the plaintext endpoint must not be contacted") }) t.Run("non-strict mode follows redirect to HTTP", func(t *testing.T) {