Discovery: update client copy of Discovery Services

README.rst

@@ -208,6 +208,7 @@ The following options can be configured on the server:
     crypto.vault.timeout                                5s                                                                                                                                                                                                                                                                                                                                                                                                     Timeout of client calls to Vault, in Golang time.Duration string format (e.g. 1s).
     crypto.vault.token                                                                                                                                                                                                                                                                                                                                                                                                                                         The Vault token. If set it overwrites the VAULT_TOKEN env var.
     **Discovery**
+    discovery.client.refresh_interval                   10m0s                                                                                                                                                                                                                                                                                                                                                                                                  How often to check for new Verifiable Presentations on the Discovery Services to update the local copy. Specified as Golang duration (e.g. 1m, 1h30m).
     discovery.client.registration_refresh_interval      10m0s                                                                                                                                                                                                                                                                                                                                                                                                  Interval at which the client should refresh checks for registrations to refresh on the configured Discovery Services,in Golang time.Duration string format (e.g. 1s). Note that it only will actually refresh registrations that about to expire (less than 1/4th of their lifetime left).
     discovery.definitions.directory                                                                                                                                                                                                                                                                                                                                                                                                                            Directory to load Discovery Service Definitions from. If not set, the discovery service will be disabled. If the directory contains JSON files that can't be parsed as service definition, the node will fail to start.
     discovery.server.definition_ids                     []                                                                                                                                                                                                                                                                                                                                                                                                     IDs of the Discovery Service Definitions for which to act as server. If an ID does not map to a loaded service definition, the node will fail to start.
@@ -228,7 +229,7 @@ The following options can be configured on the server:
     http.default.auth.type                                                                                                                                                                                                                                                                                                                                                                                                                                     Whether to enable authentication for the default interface, specify 'token_v2' for bearer token mode or 'token' for legacy bearer token mode.
     http.default.cors.origin                            []                                                                                                                                                                                                                                                                                                                                                                                                     When set, enables CORS from the specified origins on the default HTTP interface.
     **JSONLD**
-    jsonld.contexts.localmapping                        [https://w3id.org/vc/status-list/2021/v1=assets/contexts/w3c-statuslist2021.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson,https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson]      This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist.
+    jsonld.contexts.localmapping                        [https://nuts.nl/credentials/v1=assets/contexts/nuts.ldjson,https://www.w3.org/2018/credentials/v1=assets/contexts/w3c-credentials-v1.ldjson,https://w3id.org/vc/status-list/2021/v1=assets/contexts/w3c-statuslist2021.ldjson,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json=assets/contexts/lds-jws2020-v1.ldjson,https://schema.org=assets/contexts/schema-org-v13.ldjson]      This setting allows mapping external URLs to local files for e.g. preventing external dependencies. These mappings have precedence over those in remoteallowlist.
     jsonld.contexts.remoteallowlist                     [https://schema.org,https://www.w3.org/2018/credentials/v1,https://w3c-ccg.github.io/lds-jws2020/contexts/lds-jws2020-v1.json,https://w3id.org/vc/status-list/2021/v1]                                                                                                                                                                                                                                 In strict mode, fetching external JSON-LD contexts is not allowed except for context-URLs listed here.
     **Network**
     network.bootstrapnodes                              []                                                                                                                                                                                                                                                                                                                                                                                                     List of bootstrap nodes ('<host>:<port>') which the node initially connect to.

discovery/api/v1/client/http.go

@@ -65,29 +65,29 @@ func (h DefaultHTTPClient) Register(ctx context.Context, serviceEndpointURL stri
 	return nil
 }
 
-func (h DefaultHTTPClient) Get(ctx context.Context, serviceEndpointURL string, tag *string) ([]vc.VerifiablePresentation, *string, error) {
+func (h DefaultHTTPClient) Get(ctx context.Context, serviceEndpointURL string, tag string) ([]vc.VerifiablePresentation, string, error) {
 	httpRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, serviceEndpointURL, nil)
-	if tag != nil {
-		httpRequest.URL.RawQuery = url.Values{"tag": []string{*tag}}.Encode()
+	if tag != "" {
+		httpRequest.URL.RawQuery = url.Values{"tag": []string{tag}}.Encode()
 	}
 	if err != nil {
-		return nil, nil, err
+		return nil, "", err
 	}
 	httpResponse, err := h.client.Do(httpRequest)
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to invoke remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
+		return nil, "", fmt.Errorf("failed to invoke remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
 	}
 	defer httpResponse.Body.Close()
 	if err := core.TestResponseCode(200, httpResponse); err != nil {
-		return nil, nil, fmt.Errorf("non-OK response from remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
+		return nil, "", fmt.Errorf("non-OK response from remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
 	}
 	responseData, err := io.ReadAll(httpResponse.Body)
 	if err != nil {
-		return nil, nil, fmt.Errorf("failed to read response from remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
+		return nil, "", fmt.Errorf("failed to read response from remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
 	}
 	var result model.PresentationsResponse
 	if err := json.Unmarshal(responseData, &result); err != nil {
-		return nil, nil, fmt.Errorf("failed to unmarshal response from remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
+		return nil, "", fmt.Errorf("failed to unmarshal response from remote Discovery Service (url=%s): %w", serviceEndpointURL, err)
 	}
-	return result.Entries, &result.Tag, nil
+	return result.Entries, result.Tag, nil
 }

discovery/api/v1/client/http_test.go

@@ -72,12 +72,12 @@ func TestHTTPInvoker_Get(t *testing.T) {
 		server := httptest.NewServer(handler)
 		client := New(false, time.Minute, server.TLS)
 
-		presentations, tag, err := client.Get(context.Background(), server.URL, nil)
+		presentations, tag, err := client.Get(context.Background(), server.URL, "")
 
 		assert.NoError(t, err)
 		assert.Len(t, presentations, 1)
 		assert.Empty(t, handler.RequestQuery.Get("tag"))
-		assert.Equal(t, serverTag, *tag)
+		assert.Equal(t, serverTag, tag)
 	})
 	t.Run("tag provided by client", func(t *testing.T) {
 		handler := &testHTTP.Handler{StatusCode: http.StatusOK}
@@ -88,20 +88,19 @@ func TestHTTPInvoker_Get(t *testing.T) {
 		server := httptest.NewServer(handler)
 		client := New(false, time.Minute, server.TLS)
 
-		inputTag := clientTag
-		presentations, tag, err := client.Get(context.Background(), server.URL, &inputTag)
+		presentations, tag, err := client.Get(context.Background(), server.URL, clientTag)
 
 		assert.NoError(t, err)
 		assert.Len(t, presentations, 1)
 		assert.Equal(t, clientTag, handler.RequestQuery.Get("tag"))
-		assert.Equal(t, serverTag, *tag)
+		assert.Equal(t, serverTag, tag)
 	})
 	t.Run("server returns invalid status code", func(t *testing.T) {
 		handler := &testHTTP.Handler{StatusCode: http.StatusInternalServerError}
 		server := httptest.NewServer(handler)
 		client := New(false, time.Minute, server.TLS)
 
-		_, _, err := client.Get(context.Background(), server.URL, nil)
+		_, _, err := client.Get(context.Background(), server.URL, "")
 
 		assert.ErrorContains(t, err, "non-OK response from remote Discovery Service")
 	})
@@ -111,7 +110,7 @@ func TestHTTPInvoker_Get(t *testing.T) {
 		server := httptest.NewServer(handler)
 		client := New(false, time.Minute, server.TLS)
 
-		_, _, err := client.Get(context.Background(), server.URL, nil)
+		_, _, err := client.Get(context.Background(), server.URL, "")
 
 		assert.ErrorContains(t, err, "failed to unmarshal response from remote Discovery Service")
 	})

discovery/api/v1/client/interface.go

@@ -30,6 +30,6 @@ type HTTPClient interface {
 
 	// Get retrieves Verifiable Presentations from the remote Discovery Service, that were added since the given tag.
 	// If the call succeeds it returns the Verifiable Presentations and the tag that was returned by the server.
-	// If tag is nil, all Verifiable Presentations are retrieved.
-	Get(ctx context.Context, serviceEndpointURL string, tag *string) ([]vc.VerifiablePresentation, *string, error)
+	// If tag is empty, all Verifiable Presentations are retrieved.
+	Get(ctx context.Context, serviceEndpointURL string, tag string) ([]vc.VerifiablePresentation, string, error)
 }

discovery/client.go

@@ -196,3 +196,66 @@ func (r *defaultClientRegistrationManager) refresh(ctx context.Context, interval
 		}
 	}
 }
+
+// clientUpdater is responsible for updating the presentations for the given services, at the given interval.
+// Callers should only call update().
+type clientUpdater struct {
+	services map[string]ServiceDefinition
+	store    *sqlStore
+	client   client.HTTPClient
+	verifier presentationVerifier
+}
+
+func newClientUpdater(services map[string]ServiceDefinition, store *sqlStore, verifier presentationVerifier, client client.HTTPClient) *clientUpdater {
+	return &clientUpdater{
+		services: services,
+		store:    store,
+		client:   client,
+		verifier: verifier,
+	}
+}
+
+// update starts a blocking loop that updates the presentations for the given services, at the given interval.
+// It returns when the context is cancelled.
+func (u *clientUpdater) update(ctx context.Context, interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	u.doUpdate(ctx)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			u.doUpdate(ctx)
+		}
+	}
+}
+
+func (u *clientUpdater) doUpdate(ctx context.Context) {
+	for _, service := range u.services {
+		if err := u.updateService(ctx, service); err != nil {
+			log.Logger().Errorf("Failed to update service (id=%s): %s", service.ID, err)
+		}
+	}
+}
+
+func (u *clientUpdater) updateService(ctx context.Context, service ServiceDefinition) error {
+	currentTag, err := u.store.getTag(service.ID)
+	if err != nil {
+		return err
+	}
+	presentations, tag, err := u.client.Get(ctx, service.Endpoint, string(currentTag))
+	if err != nil {
+		return fmt.Errorf("failed to get presentations from discovery service (id=%s): %w", service.ID, err)
+	}
+	for _, presentation := range presentations {
+		if err := u.verifier(service, presentation); err != nil {
+			log.Logger().WithError(err).Warnf("Presentation verification failed, not adding it (service=%s, id=%s)", service.ID, presentation.ID)
+			continue
+		}
+		if err := u.store.add(service.ID, presentation, Tag(tag)); err != nil {
+			return fmt.Errorf("failed to store presentation (service=%s, id=%s): %w", service.ID, presentation.ID, err)
+		}
+	}
+	return nil
+}