Skip to content
2 changes: 2 additions & 0 deletions docs/pages/deployment/verifiable-credentials.rst
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ Fetching & Caching
^^^^^^^^^^^^^^^^^^

During startup of the node, remote contexts are fetched and cached. If the contents of a remote context changes, the node must be restarted in order for these changes to have effect. Only remote context listed in the `remoteallowlist` are fetched.
When strict mode is disabled, the node also fetches contexts that are not on the `remoteallowlist` from the internet, when a credential it processes refers to them. Use strict mode outside of development.
A remote context fetch fails after 5 seconds. A context that failed to load is not fetched again for 5 minutes: credentials referring to it fail immediately during that time. Credentials received over the network are retried later.
Local mappings can be used to pin a version of a context, so no unseen changes can be made. Working with local mappings is also useful for developing purposes when the remote context is older or non-existent. When you work with local mappings, make sure all nodes involved in the use-case have the same custom context configured.

Searching and indexing
Expand Down
35 changes: 30 additions & 5 deletions http/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,9 @@ var StrictMode bool
// checkRedirect is the http.Client.CheckRedirect policy for the strict HTTP client.
// Setting CheckRedirect replaces the standard library's default policy, so the
// 10-redirect cap is reimplemented here. It also re-runs core.ParsePublicURLAllowIP on
// every redirect target: the check in Do only guards the first hop, so without this a
// valid remote host could redirect the client onto a plaintext or otherwise disallowed
// endpoint. IP-literal hosts are left to the dial guard (see denyNonPublicAddr), which
// runs on every hop already and is aware of http.client.allowedinternalcidrs/deniedcidrs;
// every redirect target, so a disallowed redirect is refused before it is followed
// (strictTransport checks every hop as well). IP-literal hosts are left to the dial
// guard (see denyNonPublicAddr), which runs on every hop already and is aware of http.client.allowedinternalcidrs/deniedcidrs;
// rejecting them here too would make that allowlist unreachable.
func checkRedirect(req *http.Request, via []*http.Request) error {
if len(via) >= 10 {
Expand Down Expand Up @@ -131,6 +130,7 @@ func newStrictHTTPClient(httpClient *http.Client, options []Option) *StrictHTTPC
for _, option := range options {
option(result)
}
httpClient.Transport = strictTransport{next: httpClient.Transport, maxResponseSize: result.maxResponseSize}
return result
}

Expand Down Expand Up @@ -169,19 +169,44 @@ func NewWithTLSConfig(timeout time.Duration, tlsConfig *tls.Config, options ...O
}, options)
}

// NewHTTPClient creates an *http.Client with the same protections as New, for libraries that require an *http.Client.
// The protections are applied by the transport, so they apply to every request, including redirects.
func NewHTTPClient(timeout time.Duration) *http.Client {
return New(timeout).client
}

type StrictHTTPClient struct {
client *http.Client
maxResponseSize int64
}

// Do sends the request. The strict checks are applied by the client's transport (see strictTransport).
func (s *StrictHTTPClient) Do(req *http.Request) (*http.Response, error) {
result, err := s.client.Do(req)
if err != nil {
// http.Client also returns the last response when CheckRedirect fails; callers only get the error.
return nil, err
}
return result, nil
}

// strictTransport applies the strict checks to every request it sends, including redirects:
// the target URL check, the User-Agent header and the response size limit.
type strictTransport struct {
next http.RoundTripper
maxResponseSize int64
}

func (s strictTransport) RoundTrip(req *http.Request) (*http.Response, error) {
// IP-literal hosts are left to the dial guard (see denyNonPublicAddr): it is aware of
// http.client.allowedinternalcidrs/deniedcidrs, this check is not.
if _, err := core.ParsePublicURLAllowIP(req.URL.String(), StrictMode); err != nil {
return nil, fmt.Errorf("invalid target URL: %w", err)
}
// RoundTrip must not modify the request, so set the User-Agent on a copy.
req = req.Clone(req.Context())
req.Header.Set("User-Agent", core.UserAgent())
result, err := s.client.Do(req)
result, err := s.next.RoundTrip(req)
if err != nil {
return nil, err
}
Expand Down
44 changes: 38 additions & 6 deletions http/client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"testing"
"time"

"github.com/nuts-foundation/nuts-node/v6/core"
"github.com/nuts-foundation/nuts-node/v6/tracing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -85,7 +86,7 @@ func TestStrictHTTPClient(t *testing.T) {
client := NewWithTLSConfig(time.Second, &tls.Config{
InsecureSkipVerify: true,
})
ts := client.client.Transport.(*http.Transport)
ts := client.client.Transport.(strictTransport).next.(*http.Transport)
assert.True(t, ts.TLSClientConfig.InsecureSkipVerify)
})
})
Expand Down Expand Up @@ -217,9 +218,10 @@ func TestStrictHTTPClient_RedirectScheme(t *testing.T) {

client := NewWithTLSConfig(time.Second, tlsConfig)
req, _ := http.NewRequest("GET", redirector.URL, nil)
_, err := client.Do(req)
response, err := client.Do(req)

assert.Error(t, err, "strict mode must not follow a redirect from HTTPS to HTTP")
assert.Nil(t, response, "no response must be returned with the error")
assert.False(t, reached.Load(), "the plaintext endpoint must not be contacted")
})
t.Run("non-strict mode follows redirect to HTTP", func(t *testing.T) {
Expand Down Expand Up @@ -264,7 +266,7 @@ func TestWithMaxResponseSize(t *testing.T) {

_, err := New(time.Second).Do(request)

assert.EqualError(t, err, "data to read exceeds max. safety limit of 1048576 bytes")
assert.ErrorContains(t, err, "data to read exceeds max. safety limit of 1048576 bytes")
})
t.Run("raised limit accepts response", func(t *testing.T) {
request, _ := http.NewRequest(http.MethodGet, server.URL, nil)
Expand Down Expand Up @@ -302,7 +304,7 @@ func TestStrictHTTPClient_ClosesResponseBody(t *testing.T) {

_, err := client.Do(request)

assert.EqualError(t, err, "data to read exceeds max. safety limit of 10 bytes")
assert.ErrorContains(t, err, "data to read exceeds max. safety limit of 10 bytes")
assert.Equal(t, int32(1), transport.closed.Load())
})
t.Run("response within limit", func(t *testing.T) {
Expand Down Expand Up @@ -463,7 +465,7 @@ func TestNew(t *testing.T) {
client := New(time.Second)

// Transport should be wrapped (not equal to SafeHttpTransport)
assert.NotEqual(t, SafeHttpTransport, client.client.Transport)
assert.NotEqual(t, SafeHttpTransport, client.client.Transport.(strictTransport).next)
})

t.Run("uses SafeHttpTransport when tracing disabled", func(t *testing.T) {
Expand All @@ -473,6 +475,36 @@ func TestNew(t *testing.T) {

client := New(time.Second)

assert.Equal(t, SafeHttpTransport, client.client.Transport)
assert.Equal(t, SafeHttpTransport, client.client.Transport.(strictTransport).next)
})
}

func TestStrictHTTPClient_UserAgent(t *testing.T) {
var userAgent string
rt := &stubRoundTripper{
statusCode: http.StatusOK,
dataFn: func(r *http.Request) []byte {
userAgent = r.UserAgent()
return nil
},
}
request, _ := http.NewRequest(http.MethodGet, "https://nuts.nl", nil)

_, err := newStrictHTTPClient(&http.Client{Transport: rt}, nil).Do(request)

require.NoError(t, err)
assert.Equal(t, core.UserAgent(), userAgent)
assert.Empty(t, request.Header.Get("User-Agent"), "the caller's request must not be modified")
}

func TestNewHTTPClient(t *testing.T) {
original := tracing.Enabled()
tracing.SetEnabled(false)
t.Cleanup(func() { tracing.SetEnabled(original) })

client := NewHTTPClient(5 * time.Second)

assert.Equal(t, strictTransport{next: SafeHttpTransport, maxResponseSize: DefaultMaxHttpResponseSize}, client.Transport)
assert.Equal(t, 5*time.Second, client.Timeout)
assert.NotNil(t, client.CheckRedirect)
}
49 changes: 46 additions & 3 deletions jsonld/ldutils.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,24 @@ import (
"errors"
"fmt"
ssi "github.com/nuts-foundation/go-did"
"github.com/nuts-foundation/nuts-node/v6/http/client"
"github.com/nuts-foundation/nuts-node/v6/jsonld/log"
"github.com/nuts-foundation/nuts-node/v6/vcr/assets"
gocache "github.com/patrickmn/go-cache"
"github.com/piprate/json-gold/ld"
"io/fs"
"net/url"
"time"
)

// remoteContextTimeout bounds a single fetch of a remote JSON-LD context document.
// Contexts are fetched while processing network transactions, so a server that never answers must not block that processing.
var remoteContextTimeout = 5 * time.Second

// remoteContextFailureTTL is how long a failed remote context fetch is remembered.
// During that time, loads of the same URL fail immediately with the original error instead of fetching again.
var remoteContextFailureTTL = 5 * time.Minute

// ContextsConfig contains config for json-ld document loader
type ContextsConfig struct {
// RemoteAllowList A list with urls as string which are allowed to request
Expand Down Expand Up @@ -193,9 +204,11 @@ func NewContextLoader(allowUnlistedExternalCalls bool, contexts ContextsConfig)
ld.NewCachingDocumentLoader(
// Handle all embedded file system files
NewEmbeddedFSDocumentLoader(assets.Assets,
// Last in the chain is the defaultLoader which can resolve
// local files and remote (via http) context documents
ld.NewDefaultDocumentLoader(nil))))
// Remember failed remote fetches, so a dead context server costs one timeout instead of one per document
newFailureCachingLoader(remoteContextFailureTTL,
// Last in the chain is the defaultLoader which can resolve
// local files and remote (via http) context documents
ld.NewDefaultDocumentLoader(client.NewHTTPClient(remoteContextTimeout))))))

// If unlisted calls are not allowed, filter all calls to the defaultLoader
if !allowUnlistedExternalCalls {
Expand All @@ -219,6 +232,36 @@ func NewContextLoader(allowUnlistedExternalCalls bool, contexts ContextsConfig)
return loader, nil
}

// failureCachingLoader remembers failed loads per URL for a fixed time, and returns the remembered error without
// calling the next loader during that time. Successful loads are not cached here (see ld.NewCachingDocumentLoader).
type failureCachingLoader struct {
ttl time.Duration
nextLoader ld.DocumentLoader
failures *gocache.Cache
}

func newFailureCachingLoader(ttl time.Duration, nextLoader ld.DocumentLoader) *failureCachingLoader {
return &failureCachingLoader{
ttl: ttl,
nextLoader: nextLoader,
failures: gocache.New(ttl, ttl),
}
}

// LoadDocument returns the remembered error if loading u failed less than ttl ago, otherwise it calls the next loader.
func (f *failureCachingLoader) LoadDocument(u string) (*ld.RemoteDocument, error) {
if failure, ok := f.failures.Get(u); ok {
log.Logger().Debugf("Not fetching JSON-LD context, it failed recently (url=%s)", u)
return nil, failure.(error)
}
document, err := f.nextLoader.LoadDocument(u)
if err != nil {
f.failures.SetDefault(u, err)
log.Logger().WithError(err).Warnf("Failed to load JSON-LD context, not retrying for %s (url=%s)", f.ttl, u)
}
return document, err
}

// LDUtil package a set of often used JSON-LD operations for re-usability.
type LDUtil struct {
LDDocumentLoader ld.DocumentLoader
Expand Down
Loading
Loading