Update IPFS library (#45)

dipdup-io · Jun 15, 2023 · dc99c17 · dc99c17
1 parent 457918d
commit dc99c17
Show file tree

Hide file tree

Showing 17 changed files with 511 additions and 1,283 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -11,20 +11,20 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.19'
+          go-version: '1.20'
       - uses: actions/checkout@v3
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
           version: v1.51.2
-          args: --timeout=3m
+          args: --timeout=4m
   test:
     runs-on: ubuntu-latest
     steps:
       - name: install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.19.x
+          go-version: 1.20.x
       - name: checkout code
         uses: actions/checkout@v2
       - uses: actions/cache@v2

diff --git a/build/api/Dockerfile b/build/api/Dockerfile
@@ -1,7 +1,7 @@
 # ---------------------------------------------------------------------
 #  The first stage container, for building the application
 # ---------------------------------------------------------------------
-FROM golang:1.19-alpine as builder
+FROM golang:1.20-alpine as builder
 
 ENV CGO_ENABLED=1
 ENV GO111MODULE=on

diff --git a/build/dipdup.yml b/build/dipdup.yml
@@ -32,6 +32,73 @@ metadata:
       timeout: ${IPFS_TIMEOUT:-10}
       fallback: ${IPFS_NODE_URI}
       delay: ${IPFS_DELAY:-10}
+      providers:
+        # Pinata
+        - id: QmWaik1eJcGHq1ybTWe7sezRfqKNcDRNkeBaLnGwQJz1Cj
+          addr: /dnsaddr/fra1-1.hostnodes.pinata.cloud
+        - id: QmNfpLrQQZr5Ns9FAJKpyzgnDL2GgC6xBug1yUZozKFgu4
+          addr: /dnsaddr/fra1-2.hostnodes.pinata.cloud
+        - id: QmPo1ygpngghu5it8u4Mr3ym6SEU2Wp2wA66Z91Y1S1g29
+          addr: /dnsaddr/fra1-3.hostnodes.pinata.cloud
+        - id: QmRjLSisUCHVpFa5ELVvX3qVPfdxajxWJEHs9kN3EcxAW6
+          addr: /dnsaddr/nyc1-1.hostnodes.pinata.cloud
+        - id: QmPySsdmbczdZYBpbi2oq2WMJ8ErbfxtkG8Mo192UHkfGP
+          addr: /dnsaddr/nyc1-2.hostnodes.pinata.cloud
+        - id: QmSarArpxemsPESa6FNkmuu9iSE1QWqPX2R3Aw6f5jq4D5
+          addr: /dnsaddr/nyc1-3.hostnodes.pinata.cloud
+        - id: QmSarArpxemsPESa6FNkmuu9iSE1QWqPX2R3Aw6f5jq4D5
+          addr: /dnsaddr/nyc1-3.hostnodes.pinata.cloud
+        - id: Qma8ddFEQWEU8ijWvdxXm3nxU7oHsRtCykAaVz8WUYhiKn
+          addr: /dns4/production-ipfs-peer.pinata.cloud
+        # Cloudflare
+        - id: QmcFf2FH3CEgTNHeMRGhN7HNHU1EXAxoEk6EFuSyXCsvRE
+          addr: /dnsaddr/node-1.ingress.cloudflare-ipfs.com
+        - id: QmcFmLd5ySfk2WZuJ1mfSWLDjdmHZq7rSAua4GoeSQfs1z
+          addr: /dnsaddr/node-2.ingress.cloudflare-ipfs.com
+        - id: QmcfFmzSDVbwexQ9Au2pt5YEXHK5xajwgaU6PpkbLWerMa
+          addr: /dnsaddr/node-3.ingress.cloudflare-ipfs.com
+        - id: QmcfJeB3Js1FG7T8YaZATEiaHqNKVdQfybYYkbT1knUswx
+          addr: /dnsaddr/node-4.ingress.cloudflare-ipfs.com
+        - id: QmcfVvzK4tMdFmpJjEKDUoqRgP4W9FnmJoziYX5GXJJ8eZ
+          addr: /dnsaddr/node-5.ingress.cloudflare-ipfs.com
+        - id: QmcfZD3VKrUxyP9BbyUnZDpbqDnT7cQ4WjPP8TRLXaoE7G
+          addr: /dnsaddr/node-6.ingress.cloudflare-ipfs.com
+        - id: QmcfZP2LuW4jxviTeG8fi28qjnZScACb8PEgHAc17ZEri3
+          addr: /dnsaddr/node-7.ingress.cloudflare-ipfs.com
+        - id: QmcfgsJsMtx6qJb74akCw1M24X1zFwgGo11h1cuhwQjtJP
+          addr: /dnsaddr/node-8.ingress.cloudflare-ipfs.com
+        - id: Qmcfr2FC7pFzJbTSDfYaSy1J8Uuy8ccGLeLyqJCKJvTHMi
+          addr: /dnsaddr/node-9.ingress.cloudflare-ipfs.com
+        - id: QmcfR3V5YAtHBzxVACWCzXTt26SyEkxdwhGJ6875A8BuWx
+          addr: /dnsaddr/node-10.ingress.cloudflare-ipfs.com
+        - id: Qmcfuo1TM9uUiJp6dTbm915Rf1aTqm3a3dnmCdDQLHgvL5
+          addr: /dnsaddr/node-11.ingress.cloudflare-ipfs.com
+        - id: QmcfV2sg9zaq7UUHVCGuSvT2M2rnLBAPsiE79vVyK3Cuev
+          addr: /dnsaddr/node-12.ingress.cloudflare-ipfs.com
+        # web3 storage
+        - id: bafzbeibhqavlasjc7dvbiopygwncnrtvjd2xmryk5laib7zyjor6kf3avm
+          addr: /dns4/elastic.dag.house
+        # European EPC Competence Center
+        - id: 12D3KooWGaHbxpDWn4JVYud899Wcpa4iHPa3AMYydfxQDb3MhDME
+          addr: /dnsaddr/ipfs.ssi.eecc.de
+        # Estuary
+        - id: 12D3KooWCVXs8P7iq6ao4XhfAmKWrEeuKFWCJgqe9jGDMTqHYBjw
+          addr: /ip4/139.178.68.217/tcp/6744
+        - id: 12D3KooWGBWx9gyUFTVQcKMTenQMSyE2ad9m7c9fpjS4NMjoDien
+          addr: /ip4/147.75.49.71/tcp/6745
+        - id: 12D3KooWFrnuj5o3tx4fGD2ZVJRyDqTdzGnU3XYXmBbWbc8Hs8Nd
+          addr: /ip4/147.75.86.255/tcp/6745
+        # Textile
+        - id: QmR69wtWUMm1TWnmuD4JqC1TWLZcc8iR2KrTenfZZbiztd
+          addr: /ip4/104.210.43.77
+        # 8api.sh
+        - id: 12D3KooWGASC2jm3pmohEJXUhuStkxDitPgzvs4qMuFPaiD9x1BA
+          addr: /ip4/78.46.108.24
+        - id: 12D3KooWRbWZN3GvLf9CHmozq4vnTzDD4EEoiqtRJxg5FV6Gfjmm
+          addr: /ip4/65.109.19.136
+        # 4EVERLAND
+        - id: 12D3KooWQ85aSCFwFkByr5e3pUCQeuheVhobVxGSSs1DrRQHGv1t
+          addr: /dnsaddr/node-1.ipfs.4everland.net
     http_timeout: 5
     max_retry_count_on_error: ${MAX_RETRY_COUNT:-5}
     contract_service_workers: 15

diff --git a/build/metadata/Dockerfile b/build/metadata/Dockerfile
@@ -1,7 +1,7 @@
 # ---------------------------------------------------------------------
 #  The first stage container, for building the application
 # ---------------------------------------------------------------------
-FROM golang:1.19-alpine as builder
+FROM golang:1.20-alpine as builder
 
 ENV CGO_ENABLED=0
 ENV GO111MODULE=on

diff --git a/cmd/metadata/config/config.go b/cmd/metadata/config/config.go
@@ -2,6 +2,7 @@ package config
 
 import (
 	"github.com/dipdup-net/go-lib/config"
+	"github.com/dipdup-net/metadata/internal/ipfs"
 	"github.com/pkg/errors"
 )
 
@@ -119,11 +120,12 @@ type Thumbnail struct {
 
 // IPFS -
 type IPFS struct {
-	Dir       string   `yaml:"dir"`
-	Bootstrap []string `yaml:"bootstrap"`
-	Gateways  []string `yaml:"gateways" validate:"min=1,dive,url"`
-	Blacklist []string `yaml:"blacklist"`
-	Timeout   uint64   `yaml:"timeout" validate:"min=1"`
-	Fallback  string   `yaml:"fallback" validate:"url"`
-	Delay     int      `yaml:"delay" validate:"min=1"`
+	Dir       string          `yaml:"dir"`
+	Bootstrap []string        `yaml:"bootstrap"`
+	Gateways  []string        `yaml:"gateways" validate:"min=1,dive,url"`
+	Blacklist []string        `yaml:"blacklist"`
+	Timeout   uint64          `yaml:"timeout" validate:"min=1"`
+	Fallback  string          `yaml:"fallback" validate:"url"`
+	Delay     int             `yaml:"delay" validate:"min=1"`
+	Providers []ipfs.Provider `yaml:"providers" validate:"omitempty"`
 }
diff --git a/cmd/metadata/contract.go b/cmd/metadata/contract.go
@@ -50,6 +50,12 @@ func (indexer *Indexer) resolveContractMetadata(ctx context.Context, cm *models.
 		if e, ok := err.(resolver.ResolvingError); ok {
 			indexer.prom.IncrementErrorCounter(indexer.network, e)
 			err = e.Err
+
+			if e.Type == resolver.ErrorInvalidHTTPURI ||
+				e.Type == resolver.ErrorTypeInvalidJSON ||
+				e.Type == resolver.ErrorInvalidCID {
+				cm.RetryCount = int8(indexer.settings.MaxRetryCountOnError)
+			}
 		}
 
 		if cm.RetryCount < int8(indexer.settings.MaxRetryCountOnError) {

diff --git a/cmd/metadata/main.go b/cmd/metadata/main.go
@@ -86,7 +86,7 @@ func main() {
 		return
 	}
 
-	ipfsNode, err := ipfs.NewNode(ctx, cfg.Metadata.Settings.IPFS.Dir, 1024*1024, cfg.Metadata.Settings.IPFS.Blacklist)
+	ipfsNode, err := ipfs.NewNode(ctx, cfg.Metadata.Settings.IPFS.Dir, 1024*1024, cfg.Metadata.Settings.IPFS.Blacklist, cfg.Metadata.Settings.IPFS.Providers)
 	if err != nil {
 		log.Err(err).Msg("ipfs.NewNode")
 		return

diff --git a/cmd/metadata/resolver/ipfs.go b/cmd/metadata/resolver/ipfs.go
@@ -2,11 +2,11 @@ package resolver
 
 import (
 	"context"
-	"errors"
 	"strings"
 	"time"
 
 	"github.com/dipdup-net/metadata/internal/ipfs"
+	"github.com/pkg/errors"
 
 	shell "github.com/ipfs/go-ipfs-api"
 )
@@ -96,6 +96,9 @@ func (s Ipfs) Resolve(ctx context.Context, network, address, link string) (ipfs.
 
 		data, err = s.pool.GetFromNode(requestCtx, link, s.fallback)
 		if err != nil {
+			if errors.Is(err, ipfs.ErrInvalidCID) {
+				return data, newResolvingError(0, ErrorTypeReceiving, errors.Wrap(ErrInvalidURI, err.Error()))
+			}
 			return data, err
 		}
 	}

diff --git a/cmd/metadata/resolver/resolver.go b/cmd/metadata/resolver/resolver.go
@@ -37,6 +37,7 @@ const (
 	ErrorTypeTezosURIParsing ErrorType = "tezos_uri_parsing"
 	ErrorTypeInvalidJSON     ErrorType = "invalid_json"
 	ErrorInvalidHTTPURI      ErrorType = "invalid_http_uri"
+	ErrorInvalidCID          ErrorType = "invalid_ipfs_cid"
 )
 
 // ResolvingError -
@@ -103,6 +104,9 @@ func (r Receiver) Resolve(ctx context.Context, network, address, link string) (r
 		resolved.By = ResolverTypeIPFS
 		data, err := r.ipfs.Resolve(ctx, network, address, link)
 		if err != nil {
+			if errors.Is(err, ipfs.ErrInvalidCID) {
+				return resolved, newResolvingError(0, ErrorInvalidCID, err)
+			}
 			return resolved, err
 		}
 		resolved.Data = data.Raw

diff --git a/cmd/metadata/token.go b/cmd/metadata/token.go
@@ -137,7 +137,9 @@ func (indexer *Indexer) resolveTokenMetadata(ctx context.Context, tm *models.Tok
 			indexer.prom.IncrementErrorCounter(indexer.network, e)
 			err = e.Err
 
-			if e.Type == resolver.ErrorInvalidHTTPURI || e.Type == resolver.ErrorTypeInvalidJSON {
+			if e.Type == resolver.ErrorInvalidHTTPURI ||
+				e.Type == resolver.ErrorTypeInvalidJSON ||
+				e.Type == resolver.ErrorInvalidCID {
 				tm.RetryCount = int8(indexer.settings.MaxRetryCountOnError)
 			}
 		}