diff --git a/docs/pages/deployment/verifiable-credentials.rst b/docs/pages/deployment/verifiable-credentials.rst index 866962550..e698531c9 100644 --- a/docs/pages/deployment/verifiable-credentials.rst +++ b/docs/pages/deployment/verifiable-credentials.rst @@ -53,6 +53,8 @@ Fetching & Caching ^^^^^^^^^^^^^^^^^^ During startup of the node, remote contexts are fetched and cached. If the contents of a remote context changes, the node must be restarted in order for these changes to have effect. Only remote context listed in the `remoteallowlist` are fetched. +When strict mode is disabled, the node also fetches contexts that are not on the `remoteallowlist` from the internet, when a credential it processes refers to them. Use strict mode outside of development. +A remote context fetch fails after 5 seconds. A context that failed to load is not fetched again for 5 minutes: credentials referring to it fail immediately during that time. Credentials received over the network are retried later. Local mappings can be used to pin a version of a context, so no unseen changes can be made. Working with local mappings is also useful for developing purposes when the remote context is older or non-existent. When you work with local mappings, make sure all nodes involved in the use-case have the same custom context configured. Searching and indexing diff --git a/http/client/client.go b/http/client/client.go index 62f82e060..0ab7348f8 100644 --- a/http/client/client.go +++ b/http/client/client.go @@ -71,10 +71,9 @@ var StrictMode bool // checkRedirect is the http.Client.CheckRedirect policy for the strict HTTP client. // Setting CheckRedirect replaces the standard library's default policy, so the // 10-redirect cap is reimplemented here. It also re-runs core.ParsePublicURLAllowIP on -// every redirect target: the check in Do only guards the first hop, so without this a -// valid remote host could redirect the client onto a plaintext or otherwise disallowed -// endpoint. IP-literal hosts are left to the dial guard (see denyNonPublicAddr), which -// runs on every hop already and is aware of http.client.allowedinternalcidrs/deniedcidrs; +// every redirect target, so a disallowed redirect is refused before it is followed +// (strictTransport checks every hop as well). IP-literal hosts are left to the dial +// guard (see denyNonPublicAddr), which runs on every hop already and is aware of http.client.allowedinternalcidrs/deniedcidrs; // rejecting them here too would make that allowlist unreachable. func checkRedirect(req *http.Request, via []*http.Request) error { if len(via) >= 10 { @@ -131,6 +130,7 @@ func newStrictHTTPClient(httpClient *http.Client, options []Option) *StrictHTTPC for _, option := range options { option(result) } + httpClient.Transport = strictTransport{next: httpClient.Transport, maxResponseSize: result.maxResponseSize} return result } @@ -169,19 +169,44 @@ func NewWithTLSConfig(timeout time.Duration, tlsConfig *tls.Config, options ...O }, options) } +// NewHTTPClient creates an *http.Client with the same protections as New, for libraries that require an *http.Client. +// The protections are applied by the transport, so they apply to every request, including redirects. +func NewHTTPClient(timeout time.Duration) *http.Client { + return New(timeout).client +} + type StrictHTTPClient struct { client *http.Client maxResponseSize int64 } +// Do sends the request. The strict checks are applied by the client's transport (see strictTransport). func (s *StrictHTTPClient) Do(req *http.Request) (*http.Response, error) { + result, err := s.client.Do(req) + if err != nil { + // http.Client also returns the last response when CheckRedirect fails; callers only get the error. + return nil, err + } + return result, nil +} + +// strictTransport applies the strict checks to every request it sends, including redirects: +// the target URL check, the User-Agent header and the response size limit. +type strictTransport struct { + next http.RoundTripper + maxResponseSize int64 +} + +func (s strictTransport) RoundTrip(req *http.Request) (*http.Response, error) { // IP-literal hosts are left to the dial guard (see denyNonPublicAddr): it is aware of // http.client.allowedinternalcidrs/deniedcidrs, this check is not. if _, err := core.ParsePublicURLAllowIP(req.URL.String(), StrictMode); err != nil { return nil, fmt.Errorf("invalid target URL: %w", err) } + // RoundTrip must not modify the request, so set the User-Agent on a copy. + req = req.Clone(req.Context()) req.Header.Set("User-Agent", core.UserAgent()) - result, err := s.client.Do(req) + result, err := s.next.RoundTrip(req) if err != nil { return nil, err } diff --git a/http/client/client_test.go b/http/client/client_test.go index 997d47459..d155cabc8 100644 --- a/http/client/client_test.go +++ b/http/client/client_test.go @@ -31,6 +31,7 @@ import ( "testing" "time" + "github.com/nuts-foundation/nuts-node/v6/core" "github.com/nuts-foundation/nuts-node/v6/tracing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -85,7 +86,7 @@ func TestStrictHTTPClient(t *testing.T) { client := NewWithTLSConfig(time.Second, &tls.Config{ InsecureSkipVerify: true, }) - ts := client.client.Transport.(*http.Transport) + ts := client.client.Transport.(strictTransport).next.(*http.Transport) assert.True(t, ts.TLSClientConfig.InsecureSkipVerify) }) }) @@ -217,9 +218,10 @@ func TestStrictHTTPClient_RedirectScheme(t *testing.T) { client := NewWithTLSConfig(time.Second, tlsConfig) req, _ := http.NewRequest("GET", redirector.URL, nil) - _, err := client.Do(req) + response, err := client.Do(req) assert.Error(t, err, "strict mode must not follow a redirect from HTTPS to HTTP") + assert.Nil(t, response, "no response must be returned with the error") assert.False(t, reached.Load(), "the plaintext endpoint must not be contacted") }) t.Run("non-strict mode follows redirect to HTTP", func(t *testing.T) { @@ -264,7 +266,7 @@ func TestWithMaxResponseSize(t *testing.T) { _, err := New(time.Second).Do(request) - assert.EqualError(t, err, "data to read exceeds max. safety limit of 1048576 bytes") + assert.ErrorContains(t, err, "data to read exceeds max. safety limit of 1048576 bytes") }) t.Run("raised limit accepts response", func(t *testing.T) { request, _ := http.NewRequest(http.MethodGet, server.URL, nil) @@ -302,7 +304,7 @@ func TestStrictHTTPClient_ClosesResponseBody(t *testing.T) { _, err := client.Do(request) - assert.EqualError(t, err, "data to read exceeds max. safety limit of 10 bytes") + assert.ErrorContains(t, err, "data to read exceeds max. safety limit of 10 bytes") assert.Equal(t, int32(1), transport.closed.Load()) }) t.Run("response within limit", func(t *testing.T) { @@ -463,7 +465,7 @@ func TestNew(t *testing.T) { client := New(time.Second) // Transport should be wrapped (not equal to SafeHttpTransport) - assert.NotEqual(t, SafeHttpTransport, client.client.Transport) + assert.NotEqual(t, SafeHttpTransport, client.client.Transport.(strictTransport).next) }) t.Run("uses SafeHttpTransport when tracing disabled", func(t *testing.T) { @@ -473,6 +475,36 @@ func TestNew(t *testing.T) { client := New(time.Second) - assert.Equal(t, SafeHttpTransport, client.client.Transport) + assert.Equal(t, SafeHttpTransport, client.client.Transport.(strictTransport).next) }) } + +func TestStrictHTTPClient_UserAgent(t *testing.T) { + var userAgent string + rt := &stubRoundTripper{ + statusCode: http.StatusOK, + dataFn: func(r *http.Request) []byte { + userAgent = r.UserAgent() + return nil + }, + } + request, _ := http.NewRequest(http.MethodGet, "https://nuts.nl", nil) + + _, err := newStrictHTTPClient(&http.Client{Transport: rt}, nil).Do(request) + + require.NoError(t, err) + assert.Equal(t, core.UserAgent(), userAgent) + assert.Empty(t, request.Header.Get("User-Agent"), "the caller's request must not be modified") +} + +func TestNewHTTPClient(t *testing.T) { + original := tracing.Enabled() + tracing.SetEnabled(false) + t.Cleanup(func() { tracing.SetEnabled(original) }) + + client := NewHTTPClient(5 * time.Second) + + assert.Equal(t, strictTransport{next: SafeHttpTransport, maxResponseSize: DefaultMaxHttpResponseSize}, client.Transport) + assert.Equal(t, 5*time.Second, client.Timeout) + assert.NotNil(t, client.CheckRedirect) +} diff --git a/jsonld/ldutils.go b/jsonld/ldutils.go index 4cce2c8f1..11f8549d6 100644 --- a/jsonld/ldutils.go +++ b/jsonld/ldutils.go @@ -24,13 +24,24 @@ import ( "errors" "fmt" ssi "github.com/nuts-foundation/go-did" + "github.com/nuts-foundation/nuts-node/v6/http/client" "github.com/nuts-foundation/nuts-node/v6/jsonld/log" "github.com/nuts-foundation/nuts-node/v6/vcr/assets" + gocache "github.com/patrickmn/go-cache" "github.com/piprate/json-gold/ld" "io/fs" "net/url" + "time" ) +// remoteContextTimeout bounds a single fetch of a remote JSON-LD context document. +// Contexts are fetched while processing network transactions, so a server that never answers must not block that processing. +var remoteContextTimeout = 5 * time.Second + +// remoteContextFailureTTL is how long a failed remote context fetch is remembered. +// During that time, loads of the same URL fail immediately with the original error instead of fetching again. +var remoteContextFailureTTL = 5 * time.Minute + // ContextsConfig contains config for json-ld document loader type ContextsConfig struct { // RemoteAllowList A list with urls as string which are allowed to request @@ -193,9 +204,11 @@ func NewContextLoader(allowUnlistedExternalCalls bool, contexts ContextsConfig) ld.NewCachingDocumentLoader( // Handle all embedded file system files NewEmbeddedFSDocumentLoader(assets.Assets, - // Last in the chain is the defaultLoader which can resolve - // local files and remote (via http) context documents - ld.NewDefaultDocumentLoader(nil)))) + // Remember failed remote fetches, so a dead context server costs one timeout instead of one per document + newFailureCachingLoader(remoteContextFailureTTL, + // Last in the chain is the defaultLoader which can resolve + // local files and remote (via http) context documents + ld.NewDefaultDocumentLoader(client.NewHTTPClient(remoteContextTimeout)))))) // If unlisted calls are not allowed, filter all calls to the defaultLoader if !allowUnlistedExternalCalls { @@ -219,6 +232,36 @@ func NewContextLoader(allowUnlistedExternalCalls bool, contexts ContextsConfig) return loader, nil } +// failureCachingLoader remembers failed loads per URL for a fixed time, and returns the remembered error without +// calling the next loader during that time. Successful loads are not cached here (see ld.NewCachingDocumentLoader). +type failureCachingLoader struct { + ttl time.Duration + nextLoader ld.DocumentLoader + failures *gocache.Cache +} + +func newFailureCachingLoader(ttl time.Duration, nextLoader ld.DocumentLoader) *failureCachingLoader { + return &failureCachingLoader{ + ttl: ttl, + nextLoader: nextLoader, + failures: gocache.New(ttl, ttl), + } +} + +// LoadDocument returns the remembered error if loading u failed less than ttl ago, otherwise it calls the next loader. +func (f *failureCachingLoader) LoadDocument(u string) (*ld.RemoteDocument, error) { + if failure, ok := f.failures.Get(u); ok { + log.Logger().Debugf("Not fetching JSON-LD context, it failed recently (url=%s)", u) + return nil, failure.(error) + } + document, err := f.nextLoader.LoadDocument(u) + if err != nil { + f.failures.SetDefault(u, err) + log.Logger().WithError(err).Warnf("Failed to load JSON-LD context, not retrying for %s (url=%s)", f.ttl, u) + } + return document, err +} + // LDUtil package a set of often used JSON-LD operations for re-usability. type LDUtil struct { LDDocumentLoader ld.DocumentLoader diff --git a/jsonld/ldutils_test.go b/jsonld/ldutils_test.go index c3ca96d80..10e30f9d9 100644 --- a/jsonld/ldutils_test.go +++ b/jsonld/ldutils_test.go @@ -24,11 +24,16 @@ import ( "fmt" "net/http" "net/http/httptest" + "sync/atomic" "testing" + "time" ssi "github.com/nuts-foundation/go-did" + "github.com/nuts-foundation/nuts-node/v6/core" + "github.com/nuts-foundation/nuts-node/v6/http/client" "github.com/piprate/json-gold/ld" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) //go:embed test/* @@ -273,3 +278,159 @@ func Test_filteredDocumentLoader(t *testing.T) { assert.False(t, mockLoader.Called) }) } + +func TestNewContextLoader_RemoteContexts(t *testing.T) { + t.Run("fails when the server does not answer within the timeout", func(t *testing.T) { + timeout := remoteContextTimeout + remoteContextTimeout = 100 * time.Millisecond + defer func() { remoteContextTimeout = timeout }() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // never answer, until the client gives up + <-r.Context().Done() + })) + defer srv.Close() + loader, err := NewContextLoader(true, DefaultContextConfig()) + require.NoError(t, err) + + start := time.Now() + _, err = loader.LoadDocument(srv.URL + "/context.jsonld") + + var jsonLDError *ld.JsonLdError + require.ErrorAs(t, err, &jsonLDError) + assert.Equal(t, ld.LoadingDocumentFailed, jsonLDError.Code) + assert.Less(t, time.Since(start), 5*time.Second) + }) + t.Run("does not fetch a context again after it failed", func(t *testing.T) { + var requests atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusNotFound) + })) + defer srv.Close() + loader, err := NewContextLoader(true, DefaultContextConfig()) + require.NoError(t, err) + + _, err1 := loader.LoadDocument(srv.URL + "/context.jsonld") + _, err2 := loader.LoadDocument(srv.URL + "/context.jsonld") + + require.Error(t, err1) + assert.Same(t, err1, err2) + assert.Equal(t, int32(1), requests.Load()) + }) + t.Run("identifies as the Nuts node", func(t *testing.T) { + var userAgent string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + userAgent = r.UserAgent() + w.Header().Set("Content-Type", "application/ld+json") + _, _ = w.Write([]byte(`{"@context":{}}`)) + })) + defer srv.Close() + loader, err := NewContextLoader(true, DefaultContextConfig()) + require.NoError(t, err) + + _, err = loader.LoadDocument(srv.URL + "/context.jsonld") + + require.NoError(t, err) + assert.Equal(t, core.UserAgent(), userAgent) + }) + t.Run("returns the URL of the final request after a redirect", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/old.jsonld" { + http.Redirect(w, r, "/new.jsonld", http.StatusFound) + return + } + w.Header().Set("Content-Type", "application/ld+json") + _, _ = w.Write([]byte(`{"@context":{}}`)) + })) + defer srv.Close() + loader, err := NewContextLoader(true, DefaultContextConfig()) + require.NoError(t, err) + + document, err := loader.LoadDocument(srv.URL + "/old.jsonld") + + require.NoError(t, err) + assert.Equal(t, srv.URL+"/new.jsonld", document.DocumentURL) + }) + t.Run("fails on invalid JSON", func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`not JSON`)) + })) + defer srv.Close() + loader, err := NewContextLoader(true, DefaultContextConfig()) + require.NoError(t, err) + + _, err = loader.LoadDocument(srv.URL + "/context.jsonld") + + var jsonLDError *ld.JsonLdError + require.ErrorAs(t, err, &jsonLDError) + assert.Equal(t, ld.LoadingDocumentFailed, jsonLDError.Code) + }) + t.Run("strict mode rejects an HTTP URL", func(t *testing.T) { + strictMode := client.StrictMode + client.StrictMode = true + defer func() { client.StrictMode = strictMode }() + loader, err := NewContextLoader(true, DefaultContextConfig()) + require.NoError(t, err) + + _, err = loader.LoadDocument("http://example.com/context.jsonld") + + assert.ErrorContains(t, err, "invalid target URL") + }) +} + +type countingLoader struct { + calls int + err error +} + +func (c *countingLoader) LoadDocument(u string) (*ld.RemoteDocument, error) { + c.calls++ + if c.err != nil { + return nil, c.err + } + return &ld.RemoteDocument{DocumentURL: u}, nil +} + +func Test_failureCachingLoader(t *testing.T) { + const contextURL = "https://example.com/context.jsonld" + t.Run("returns the remembered error without calling the next loader", func(t *testing.T) { + next := &countingLoader{err: ld.NewJsonLdError(ld.LoadingDocumentFailed, "boom")} + sut := newFailureCachingLoader(time.Minute, next) + + _, err1 := sut.LoadDocument(contextURL) + _, err2 := sut.LoadDocument(contextURL) + + assert.Same(t, err1, err2) + assert.Equal(t, 1, next.calls) + }) + t.Run("calls the next loader again after the failure expired", func(t *testing.T) { + next := &countingLoader{err: ld.NewJsonLdError(ld.LoadingDocumentFailed, "boom")} + sut := newFailureCachingLoader(time.Millisecond, next) + _, _ = sut.LoadDocument(contextURL) + time.Sleep(10 * time.Millisecond) + + _, _ = sut.LoadDocument(contextURL) + + assert.Equal(t, 2, next.calls) + }) + t.Run("does not remember successful loads", func(t *testing.T) { + next := &countingLoader{} + sut := newFailureCachingLoader(time.Minute, next) + + _, err1 := sut.LoadDocument(contextURL) + _, err2 := sut.LoadDocument(contextURL) + + assert.NoError(t, err1) + assert.NoError(t, err2) + assert.Equal(t, 2, next.calls) + }) + t.Run("remembers failures per URL", func(t *testing.T) { + next := &countingLoader{err: ld.NewJsonLdError(ld.LoadingDocumentFailed, "boom")} + sut := newFailureCachingLoader(time.Minute, next) + + _, _ = sut.LoadDocument(contextURL) + _, _ = sut.LoadDocument("https://example.com/other.jsonld") + + assert.Equal(t, 2, next.calls) + }) +} diff --git a/vcr/vcr_test.go b/vcr/vcr_test.go index 479409b36..c20801038 100644 --- a/vcr/vcr_test.go +++ b/vcr/vcr_test.go @@ -108,7 +108,7 @@ func TestVCR_Configure(t *testing.T) { require.NoError(t, err) err = issuer.OfferCredential(context.Background(), testVC, "http://example.com") - assert.ErrorContains(t, err, "http request error: invalid target URL") + assert.ErrorContains(t, err, "invalid target URL") assert.ErrorContains(t, err, "scheme must be https") }) }