diff --git a/blockchain/orginzationMetadata.go b/blockchain/orginzationMetadata.go index 407331d8..d868bdeb 100644 --- a/blockchain/orginzationMetadata.go +++ b/blockchain/orginzationMetadata.go @@ -98,17 +98,16 @@ type PaymentChannelStorageClient struct { Endpoints []string `json:"endpoints"` } -// Construct the Organization metadata from the JSON Passed -func InitOrganizationMetaDataFromJson(jsonData string) (metaData *OrganizationMetaData, err error) { +// InitOrganizationMetaDataFromJson Construct the Organization metadata from the JSON Passed +func InitOrganizationMetaDataFromJson(jsonData []byte) (metaData *OrganizationMetaData, err error) { metaData = new(OrganizationMetaData) - err = json.Unmarshal([]byte(jsonData), &metaData) + err = json.Unmarshal(jsonData, &metaData) if err != nil { zap.L().Error("Error in unmarshalling metadata json", zap.Error(err), zap.Any("jsondata", jsonData)) return nil, err } // Check for mandatory validations - if err = setDerivedAttributes(metaData); err != nil { zap.L().Error("Error in setting derived attributes", zap.Error(err)) return nil, err @@ -166,7 +165,7 @@ func GetOrganizationMetaData() *OrganizationMetaData { var err error if config.GetBool(config.BlockchainEnabledKey) { ipfsHash := string(getMetaDataURI()) - metadata, err = GetOrganizationMetaDataFromIPFS(FormatHash(ipfsHash)) + metadata, err = GetOrganizationMetaDataFromIPFS(ipfsHash) } else { metadata = &OrganizationMetaData{daemonGroup: &Group{}} } @@ -177,7 +176,10 @@ func GetOrganizationMetaData() *OrganizationMetaData { } func GetOrganizationMetaDataFromIPFS(hash string) (*OrganizationMetaData, error) { - jsondata := ipfsutils.GetIpfsFile(hash) + jsondata, err := ipfsutils.ReadFile(hash) + if err != nil { + return nil, err + } return InitOrganizationMetaDataFromJson(jsondata) } diff --git a/blockchain/serviceMetadata.go b/blockchain/serviceMetadata.go index 7f22dacf..84c5dd2d 100644 --- a/blockchain/serviceMetadata.go +++ b/blockchain/serviceMetadata.go @@ -143,18 +143,18 @@ import ( } */ const ( - IpfsPrefix = "ipfs://" serviceProto = "service.proto" ) type ServiceMetadata struct { - Version int `json:"version"` - DisplayName string `json:"display_name"` - Encoding string `json:"encoding"` - ServiceType string `json:"service_type"` - Groups []OrganizationGroup `json:"groups"` - ModelIpfsHash string `json:"model_ipfs_hash"` - MpeAddress string `json:"mpe_address"` + Version int `json:"version"` + DisplayName string `json:"display_name"` + Encoding string `json:"encoding"` + ServiceType string `json:"service_type"` + Groups []OrganizationGroup `json:"groups"` + ModelIpfsHash string `json:"model_ipfs_hash"` + ServiceApiSource string `json:"service_api_source"` + MpeAddress string `json:"mpe_address"` multiPartyEscrowAddress common.Address defaultPricing Pricing @@ -196,6 +196,7 @@ type Subscriptions struct { IsActive string `json:"isActive"` Subscription []Subscription `json:"subscription"` } + type Tier struct { Type string `json:"type"` PlanName string `json:"planName"` @@ -205,6 +206,7 @@ type Tier struct { DetailsURL string `json:"detailsUrl"` IsActive string `json:"isActive"` } + type Licenses struct { Subscriptions Subscriptions `json:"subscriptions,omitempty"` Tiers []Tier `json:"tiers"` @@ -252,7 +254,7 @@ func ServiceMetaData() *ServiceMetadata { var err error if config.GetBool(config.BlockchainEnabledKey) { ipfsHash := string(getServiceMetaDataUrifromRegistry()) - metadata, err = GetServiceMetaDataFromIPFS(FormatHash(ipfsHash)) + metadata, err = GetServiceMetaDataFromIPFS(ipfsHash) if err != nil { zap.L().Panic("error on determining service metadata from file", zap.Error(err)) } @@ -268,9 +270,7 @@ func ReadServiceMetaDataFromLocalFile(filename string) (*ServiceMetadata, error) if err != nil { return nil, errors.Wrapf(err, "could not read file: %v", filename) } - strJson := string(file) - metadata, err := InitServiceMetaDataFromJson(strJson) - + metadata, err := InitServiceMetaDataFromJson(file) if err != nil { return nil, fmt.Errorf("error reading local file service_metadata.json ") } @@ -317,13 +317,16 @@ func getServiceMetaDataUrifromRegistry() []byte { } func GetServiceMetaDataFromIPFS(hash string) (*ServiceMetadata, error) { - jsondata := ipfsutils.GetIpfsFile(hash) + jsondata, err := ipfsutils.ReadFile(hash) + if err != nil { + return nil, err + } return InitServiceMetaDataFromJson(jsondata) } -func InitServiceMetaDataFromJson(jsonData string) (*ServiceMetadata, error) { +func InitServiceMetaDataFromJson(jsonData []byte) (*ServiceMetadata, error) { metaData := new(ServiceMetadata) - err := json.Unmarshal([]byte(jsonData), &metaData) + err := json.Unmarshal(jsonData, &metaData) if err != nil { zap.L().Error(err.Error(), zap.Any("jsondata", jsonData)) return nil, err @@ -350,7 +353,7 @@ func InitServiceMetaDataFromJson(jsonData string) (*ServiceMetadata, error) { zap.L().Error(err.Error()) } - zap.L().Debug("Traning method", zap.String("json", string(trainingMethodsJson))) + zap.L().Debug("Training method", zap.String("json", string(trainingMethodsJson))) return metaData, err } @@ -488,8 +491,22 @@ func isElementInArray(a string, list []string) bool { func setServiceProto(metaData *ServiceMetadata) (err error) { metaData.DynamicPriceMethodMapping = make(map[string]string, 0) metaData.TrainingMethods = make([]string, 0) - //This is to handler the scenario where there could be multiple protos associated with the service proto - protoFiles, err := ipfsutils.ReadFilesCompressed(ipfsutils.GetIpfsFile(metaData.ModelIpfsHash)) + var rawFile []byte + + // for backwards compatibility + if metaData.ModelIpfsHash != "" { + rawFile, err = ipfsutils.GetIpfsFile(metaData.ServiceApiSource) + } + + if metaData.ServiceApiSource != "" { + rawFile, err = ipfsutils.ReadFile(metaData.ServiceApiSource) + } + + if err != nil { + zap.L().Error("Error in retrieving file from filecoin/ipfs", zap.Error(err)) + } + + protoFiles, err := ipfsutils.ReadFilesCompressed(rawFile) if err != nil { return err } diff --git a/blockchain/utils.go b/blockchain/utils.go index dea7e943..6fb47b36 100644 --- a/blockchain/utils.go +++ b/blockchain/utils.go @@ -3,9 +3,6 @@ package blockchain import ( "encoding/base64" "fmt" - "regexp" - "strings" - "github.com/ethereum/go-ethereum/common" "go.uber.org/zap" ) @@ -52,11 +49,6 @@ func StringToBytes32(str string) [32]byte { return byte32 } -func RemoveSpecialCharactersfromHash(pString string) string { - reg := regexp.MustCompile("[^a-zA-Z0-9=]") - return reg.ReplaceAllString(pString, "") -} - func ConvertBase64Encoding(str string) ([32]byte, error) { var byte32 [32]byte data, err := base64.StdEncoding.DecodeString(str) @@ -68,14 +60,6 @@ func ConvertBase64Encoding(str string) ([32]byte, error) { return byte32, nil } -func FormatHash(ipfsHash string) string { - zap.L().Debug("Before Formatting", zap.String("metadataHash", ipfsHash)) - ipfsHash = strings.Replace(ipfsHash, IpfsPrefix, "", -1) - ipfsHash = RemoveSpecialCharactersfromHash(ipfsHash) - zap.L().Debug("After Formatting", zap.String("metadataUri", ipfsHash)) - return ipfsHash -} - func ToChecksumAddress(hexAddress string) string { address := common.HexToAddress(hexAddress) mixedAddress := common.NewMixedcaseAddress(address) diff --git a/blockchain/utils_test.go b/blockchain/utils_test.go index e25b0bef..e56456ab 100644 --- a/blockchain/utils_test.go +++ b/blockchain/utils_test.go @@ -11,49 +11,13 @@ func TestBytesToBase64(t *testing.T) { assert.Equal(t, "AQL+/w==", base64) } -func TestFormatHash(t *testing.T) { - s2 := []byte("ipfs://Here is a string....+=") - hash := FormatHash(string(s2)) - assert.Equal(t, hash, "Hereisastring=") - s2 = []byte("QmaGnQ3iVZPuPwdam2rEeQcCSoCYRpxjnZhQ6Z2oeeRSrp") - - b4 := append(s2, make([]byte, 3)...) - assert.NotEqual(t, "QmaGnQ3iVZPuPwdam2rEeQcCSoCYRpxjnZhQ6Z2oeeRSrp", string(b4)) - assert.Equal(t, "QmaGnQ3iVZPuPwdam2rEeQcCSoCYRpxjnZhQ6Z2oeeRSrp", FormatHash(string(b4))) -} func TestConvertBase64Encoding(t *testing.T) { - if _, err := ConvertBase64Encoding("n@@###zNEetD1kzU3PZqR4nHPS8erDkrUK0hN4iCBQ4vH5U"); err != nil { assert.Equal(t, err.Error(), "illegal base64 data at input byte 1") } - } func TestToChecksumAddress(t *testing.T) { - assert.Equal(t, "0xE9D09A6C296ACDd4C01b21F407aC93FDfC63e78c", ToChecksumAddress("0xe9d09A6C296aCdd4c01b21f407ac93fdfC63E78C")) - assert.Equal(t, "0xE9D09A6C296ACDd4C01b21F407aC93FDfC63e78c", ToChecksumAddress("0xe9d09A6C296aCdd4c01b21f407ac93fdfC63E78C")) } - -func TestRemoveSpecialCharactersfromHash(t *testing.T) { - testCases := []struct { - input string - expectedOutput string - }{ - {"abc123", "abc123"}, - {"abc123!@#", "abc123"}, - {"a1b2c3 ~`!@#$%^&*()_+-={}[]|\\:;\"'<>,.?/", "a1b2c3="}, - {"abc=123", "abc=123"}, - {"a1!b2@c3#=4", "a1b2c3=4"}, - } - - for _, tc := range testCases { - t.Run(tc.input, func(t *testing.T) { - output := RemoveSpecialCharactersfromHash(tc.input) - if output != tc.expectedOutput { - t.Errorf("RemoveSpecialCharactersfromHash(%q) = %q; want %q", tc.input, output, tc.expectedOutput) - } - }) - } -} diff --git a/escrow/payment_channel_storage_test.go b/escrow/payment_channel_storage_test.go index 4ac97112..4b8b26e4 100644 --- a/escrow/payment_channel_storage_test.go +++ b/escrow/payment_channel_storage_test.go @@ -163,19 +163,20 @@ func (suite *BlockchainChannelReaderSuite) TestGetChannelState() { assert.Equal(suite.T(), suite.channel(), channel) } -func (suite *BlockchainChannelReaderSuite) TestGetChannelStateIncorrectRecipeintAddress() { +func (suite *BlockchainChannelReaderSuite) TestGetChannelStateIncorrectRecipientAddress() { reader := suite.reader reader.recipientPaymentAddress = func() common.Address { return crypto.PubkeyToAddress(GenerateTestPrivateKey().PublicKey) } channel, ok, err := reader.GetChannelStateFromBlockchain(suite.channelKey()) - assert.Equal(suite.T(), errors.New("recipient Address from service metadata does not Match on what was retrieved from Channel"), err) + assert.Equal(suite.T(), errors.New("recipient Address from org metadata does not Match on what was retrieved from Channel"), err) assert.False(suite.T(), ok) assert.Nil(suite.T(), channel) } func (suite *PaymentChannelStorageSuite) TestNewPaymentChannelStorage() { mpeStorage := storage.NewPrefixedAtomicStorage(storage.NewPrefixedAtomicStorage(suite.memoryStorage, "path1"), "path2") - mpeStorage.Put("key1", "value1") + err := mpeStorage.Put("key1", "value1") + assert.Nil(suite.T(), err) value, _, _ := mpeStorage.Get("key1") assert.Equal(suite.T(), value, "value1") values, err := suite.memoryStorage.GetByKeyPrefix("path1") diff --git a/etcddb/etcddb_client.go b/etcddb/etcddb_client.go index b2c383cd..a2e46512 100644 --- a/etcddb/etcddb_client.go +++ b/etcddb/etcddb_client.go @@ -65,7 +65,8 @@ func NewEtcdClientFromVip(vip *viper.Viper, metaData *blockchain.OrganizationMet var etcdv3 *clientv3.Client if utils.CheckIfHttps(metaData.GetPaymentStorageEndPoints()) { - tlsConfig, err := getTlsConfig() + var tlsConfig *tls.Config + tlsConfig, err = getTlsConfig() if err != nil { return nil, err } diff --git a/go.mod b/go.mod index ca128242..84905942 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/OneOfOne/go-utils v0.0.0-20180319162427-6019ff89a94e github.com/bufbuild/protocompile v0.14.1 github.com/emicklei/proto v1.13.2 - github.com/ethereum/go-ethereum v1.14.9 + github.com/ethereum/go-ethereum v1.14.11 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/gorilla/handlers v1.5.2 @@ -30,7 +30,7 @@ require ( go.etcd.io/etcd/client/v3 v3.5.16 go.etcd.io/etcd/server/v3 v3.5.16 go.uber.org/zap v1.27.0 - golang.org/x/crypto v0.27.0 + golang.org/x/crypto v0.28.0 golang.org/x/net v0.29.0 golang.org/x/time v0.6.0 google.golang.org/grpc v1.66.2 @@ -45,7 +45,7 @@ require ( github.com/VictoriaMetrics/fastcache v1.12.2 // indirect github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.13.0 // indirect + github.com/bits-and-blooms/bitset v1.14.3 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect @@ -56,20 +56,20 @@ require ( github.com/cockroachdb/pebble v1.1.2 // indirect github.com/cockroachdb/redact v1.1.5 // indirect github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect - github.com/consensys/bavard v0.1.13 // indirect - github.com/consensys/gnark-crypto v0.12.1 // indirect + github.com/consensys/bavard v0.1.22 // indirect + github.com/consensys/gnark-crypto v0.14.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.4 // indirect github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect - github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c // indirect - github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect + github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect + github.com/crate-crypto/go-kzg-4844 v1.1.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ethereum/c-kzg-4844 v1.0.2 // indirect + github.com/ethereum/c-kzg-4844 v1.0.3 // indirect github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 // indirect github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect @@ -184,7 +184,7 @@ require ( github.com/spf13/afero v1.11.0 // indirect github.com/status-im/keycard-go v0.2.0 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/supranational/blst v0.3.12 // indirect + github.com/supranational/blst v0.3.13 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect @@ -217,8 +217,8 @@ require ( golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/mod v0.19.0 // indirect golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect golang.org/x/tools v0.23.0 // indirect golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect gonum.org/v1/gonum v0.15.0 // indirect diff --git a/go.sum b/go.sum index 5862aa61..f6ddceb8 100644 --- a/go.sum +++ b/go.sum @@ -67,8 +67,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE= -github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= +github.com/bits-and-blooms/bitset v1.14.3 h1:Gd2c8lSNf9pKXom5JtD7AaKO8o7fGQ2LtFj1436qilA= +github.com/bits-and-blooms/bitset v1.14.3/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ= @@ -108,18 +108,19 @@ github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/e github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.1 h1:XnKU22oiCLy2Xn8vp1re67cXg4SAasg/WDt1NtcRFaw= -github.com/cockroachdb/pebble v1.1.1/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= -github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= -github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= -github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/consensys/bavard v0.1.17 h1:53CdY/g35YSH9oRoa/b29tZinaiOEJYBmf9vydozPpE= +github.com/consensys/bavard v0.1.17/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/bavard v0.1.22 h1:Uw2CGvbXSZWhqK59X0VG/zOjpTFuOMcPLStrp1ihI0A= +github.com/consensys/bavard v0.1.22/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= +github.com/consensys/gnark-crypto v0.14.0 h1:DDBdl4HaBtdQsq/wfMwJvZNE80sHidrK3Nfrefatm0E= +github.com/consensys/gnark-crypto v0.14.0/go.mod h1:CU4UijNPsHawiVGNxe9co07FkzCeWHHrb1li/n1XoU0= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -134,10 +135,10 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4 h1:wfIWP927BUkWJb2NmU/kNDYIBTh/ziUX91+lV github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 h1:ZFUue+PNxmHlu7pYv+IYMtqlaO/0VwaGEqKepZf9JpA= github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= -github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= -github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= -github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= -github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= +github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= +github.com/crate-crypto/go-kzg-4844 v1.1.0 h1:EN/u9k2TF6OWSHrCCDBBU6GLNMq88OspHHlMnHfoyU4= +github.com/crate-crypto/go-kzg-4844 v1.1.0/go.mod h1:JolLjpSff1tCCJKaJx4psrlEdlXuJEC996PL3tTAFks= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= @@ -179,14 +180,14 @@ github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymF github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/ethereum/c-kzg-4844 v1.0.2 h1:8tV84BCEiPeOkiVgW9mpYBeBUir2bkCNVqxPwwVeO+s= -github.com/ethereum/c-kzg-4844 v1.0.2/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/ethereum/go-ethereum v1.14.8 h1:NgOWvXS+lauK+zFukEvi85UmmsS/OkV0N23UZ1VTIig= -github.com/ethereum/go-ethereum v1.14.8/go.mod h1:TJhyuDq0JDppAkFXgqjwpdlQApywnu/m10kFPxh8vvs= +github.com/ethereum/c-kzg-4844 v1.0.3 h1:IEnbOHwjixW2cTvKRUlAAUOeleV7nNM/umJR+qy4WDs= +github.com/ethereum/c-kzg-4844 v1.0.3/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.14.9 h1:J7iwXDrtUyE9FUjUYbd4c9tyzwMh6dTJsKzo9i6SrwA= github.com/ethereum/go-ethereum v1.14.9/go.mod h1:QeW+MtTpRdBEm2pUFoonByee8zfHv7kGp0wK0odvU1I= -github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 h1:KrE8I4reeVvf7C1tm8elRjj4BdscTYzz/WAbYyf/JI4= -github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0/go.mod h1:D9AJLVXSyZQXJQVk8oh1EwjISE+sJTn2duYIZC0dy3w= +github.com/ethereum/go-ethereum v1.14.10 h1:kC24WjYeRjDy86LVo6MfF5Xs7nnUu+XG4AjaYIaZYko= +github.com/ethereum/go-ethereum v1.14.10/go.mod h1:+l/fr42Mma+xBnhefL/+z11/hcmJ2egl+ScIVPjhc7E= +github.com/ethereum/go-ethereum v1.14.11 h1:8nFDCUUE67rPc6AKxFj7JKaOa2W/W1Rse3oS6LvvxEY= +github.com/ethereum/go-ethereum v1.14.11/go.mod h1:+l/fr42Mma+xBnhefL/+z11/hcmJ2egl+ScIVPjhc7E= github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 h1:8NfxH2iXvJ60YRB8ChToFTUzl8awsc3cJ8CbLjGIl/A= github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= @@ -558,8 +559,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4= +github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= @@ -921,8 +922,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/supranational/blst v0.3.12 h1:Vfas2U2CFHhniv2QkUm2OVa1+pGTdqtpqm9NnhUUbZ8= -github.com/supranational/blst v0.3.12/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.13 h1:AYeSxdOMacwu7FBmpfloBz5pbFXDmJL33RuwnKtmTjk= +github.com/supranational/blst v0.3.13/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= @@ -1069,6 +1070,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1218,6 +1221,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -1231,6 +1236,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1319,8 +1326,6 @@ google.golang.org/genproto v0.0.0-20240722135656-d784300faade h1:lKFsS7wpngDgSCe google.golang.org/genproto v0.0.0-20240722135656-d784300faade/go.mod h1:FfBgJBJg9GcpPvKIuHSZ/aE1g2ecGL74upMzGZjiGEY= google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade h1:WxZOF2yayUHpHSbUE6NMzumUzBxYc3YGwo0YHnbzsJY= google.golang.org/genproto/googleapis/api v0.0.0-20240722135656-d784300faade/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade h1:oCRSWfwGXQsqlVdErcyTt4A93Y8fo0/9D4b1gnI++qo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240722135656-d784300faade/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1340,8 +1345,6 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.66.1 h1:hO5qAXR19+/Z44hmvIM4dQFMSYX9XcWsByfoxutBpAM= -google.golang.org/grpc v1.66.1/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/grpc v1.66.2 h1:3QdXkuq3Bkh7w+ywLdLvM56cmGvQHUMZpiCzt6Rqaoo= google.golang.org/grpc v1.66.2/go.mod h1:s3/l6xSSCURdVfAnL+TqCNMyTDAGN6+lZeVxnZR128Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/handler/intergrationtests/grpc_stream_test.proto b/handler/intergrationtests/grpc_stream_test.proto new file mode 100644 index 00000000..689facb1 --- /dev/null +++ b/handler/intergrationtests/grpc_stream_test.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package handler; + +option go_package = "../intergrationtests"; + +service ExampleStreamingService { + rpc Stream(stream InStream) returns (stream OutStream) {} +} + +message InStream { + string message = 1; +} + +message OutStream { + string message = 1; +} \ No newline at end of file diff --git a/handler/intergrationtests/grpc_to_grpc_test.go b/handler/intergrationtests/grpc_to_grpc_test.go new file mode 100644 index 00000000..71f110a3 --- /dev/null +++ b/handler/intergrationtests/grpc_to_grpc_test.go @@ -0,0 +1,236 @@ +//go:generate protoc grpc_stream_test.proto --go-grpc_out=. --go_out=. + +package intergrationtests + +import ( + "context" + "io" + "net" + "testing" + "time" + + "github.com/singnet/snet-daemon/blockchain" + "github.com/singnet/snet-daemon/config" + "github.com/singnet/snet-daemon/handler" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/spf13/viper" + "go.uber.org/zap" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +type integrationTestEnvironment struct { + serverA *grpc.Server + listenerA *net.Listener + serverB *grpc.Server + listenerB *net.Listener +} + +func setupTestConfig() { + testConfigJson := ` +{ + "blockchain_enabled": true, + "blockchain_network_selected": "sepolia", + "daemon_end_point": "127.0.0.1:8080", + "daemon_group_name":"default_group", + "payment_channel_storage_type": "etcd", + "ipfs_end_point": "http://ipfs.singularitynet.io:80", + "ipfs_timeout" : 30, + "passthrough_enabled": true, + "passthrough_endpoint":"http://127.0.0.1:5002", + "service_id": "ExampleServiceId", + "organization_id": "ExampleOrganizationId", + "metering_enabled": false, + "ssl_cert": "", + "ssl_key": "", + "max_message_size_in_mb" : 4, + "daemon_type": "grpc", + "enable_dynamic_pricing":false, + "allowed_user_flag" :false, + "auto_ssl_domain": "", + "auto_ssl_cache_dir": ".certs", + "private_key": "", + "log": { + "level": "info", + "timezone": "UTC", + "formatter": { + "type": "text", + "timestamp_format": "2006-01-02T15:04:05.999Z07:00" + }, + "output": { + "type": ["file", "stdout"], + "file_pattern": "./snet-daemon.%Y%m%d.log", + "current_link": "./snet-daemon.log", + "max_size_in_mb": 10, + "max_age_in_days": 7, + "rotation_count": 0 + }, + "hooks": [] + }, + "payment_channel_storage_client": { + "connection_timeout": "0s", + "request_timeout": "0s", + "hot_reload": true + }, + "payment_channel_storage_server": { + "id": "storage-1", + "scheme": "http", + "host" : "127.0.0.1", + "client_port": 2379, + "peer_port": 2380, + "token": "unique-token", + "cluster": "storage-1=http://127.0.0.1:2380", + "startup_timeout": "1m", + "data_dir": "storage-data-dir-1.etcd", + "log_level": "info", + "log_outputs": ["./etcd-server.log"], + "enabled": false + }, + "alerts_email": "", + "service_heartbeat_type": "http", + "token_expiry_in_minutes": 1440, + "model_training_enabled": false +}` + + var testConfig = viper.New() + err := config.ReadConfigFromJsonString(testConfig, testConfigJson) + if err != nil { + zap.L().Fatal("Error in reading config") + } + + config.SetVip(testConfig) +} + +func startServerA(port string, h grpc.StreamHandler) (*grpc.Server, *net.Listener) { + lis, err := net.Listen("tcp", port) + if err != nil { + zap.L().Fatal("Failed to listen", zap.Error(err)) + } + + grpcServer := grpc.NewServer() + RegisterExampleStreamingServiceServer(grpcServer, &ServiceA{h: h}) + + go func() { + if err := grpcServer.Serve(lis); err != nil { + zap.L().Fatal("Failed to serve", zap.Error(err)) + } + }() + return grpcServer, &lis +} + +func startServerB(port string) (*grpc.Server, *net.Listener) { + lis, err := net.Listen("tcp", port) + if err != nil { + zap.L().Fatal("Failed to listen", zap.Error(err)) + } + + grpcServer := grpc.NewServer() + RegisterExampleStreamingServiceServer(grpcServer, &ServiceB{}) + + go func() { + if err := grpcServer.Serve(lis); err != nil { + zap.L().Fatal("Failed to serve", zap.Error(err)) + } + }() + return grpcServer, &lis +} + +func setupEnvironment() *integrationTestEnvironment { + setupTestConfig() + serviceMetadata := &blockchain.ServiceMetadata{ + ServiceType: "grpc", + Encoding: "proto", + } + grpcToGrpc := handler.NewGrpcHandler(serviceMetadata) + grpcServerA, listenerA := startServerA(":5001", grpcToGrpc) + grpcServerB, listenerB := startServerB(":5002") + + testEnv := &integrationTestEnvironment{ + serverA: grpcServerA, + listenerA: listenerA, + serverB: grpcServerB, + listenerB: listenerB, + } + + return testEnv +} + +func teardownEnvironment(env *integrationTestEnvironment) { + env.serverA.Stop() + env.serverB.Stop() + (*env.listenerA).Close() + (*env.listenerB).Close() +} + +type ServiceA struct { + UnimplementedExampleStreamingServiceServer + h grpc.StreamHandler +} + +type ServiceB struct { + UnimplementedExampleStreamingServiceServer +} + +func (s *ServiceA) Stream(stream ExampleStreamingService_StreamServer) error { + // Forward the stream to grpcToGrpc handler + err := s.h(nil, stream) + if err != nil { + return err + } + return nil +} + +func (s *ServiceB) Stream(stream ExampleStreamingService_StreamServer) error { + for { + // Receive the input from the proxied stream + req, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + // Send back the response to the stream + err = stream.Send(&OutStream{ + Message: "Response from Server B: " + req.Message, + }) + if err != nil { + return err + } + } +} + +func TestGrpcToGrpc_StreamingIntegration(t *testing.T) { + // Setup test environment with real servers + testEnv := setupEnvironment() + defer teardownEnvironment(testEnv) + + // Create a gRPC client to interact with Server A + connA, err := grpc.NewClient((*testEnv.listenerA).Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) + require.NoError(t, err) + defer connA.Close() + + clientA := NewExampleStreamingServiceClient(connA) + + // Simulate a bidirectional streaming RPC call from clientA to Server A + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + stream, err := clientA.Stream(ctx) + require.NoError(t, err) + + // Send a message from the client A to Server A + err = stream.Send(&InStream{Message: "Hello from client A"}) + require.NoError(t, err) + + // Receive the response from Server B (proxied by grpcToGrpc) + resp, err := stream.Recv() + require.NoError(t, err) + assert.Equal(t, "Response from Server B: Hello from client A", resp.Message) + + // Close the stream + stream.CloseSend() +} diff --git a/ipfsutils/common.go b/ipfsutils/common.go new file mode 100644 index 00000000..418959cb --- /dev/null +++ b/ipfsutils/common.go @@ -0,0 +1,35 @@ +package ipfsutils + +import ( + "go.uber.org/zap" + "regexp" + "strings" +) + +const ( + IpfsPrefix = "ipfs://" + FilecoinPrefix = "filecoin://" +) + +func ReadFile(hash string) (rawFile []byte, err error) { + if strings.HasPrefix(hash, FilecoinPrefix) { + rawFile, err = GetLighthouseFile(formatHash(hash)) + } else { + rawFile, err = GetIpfsFile(formatHash(hash)) + } + return rawFile, err +} + +func formatHash(hash string) string { + zap.L().Debug("Before Formatting", zap.String("metadataHash", hash)) + hash = strings.Replace(hash, IpfsPrefix, "", -1) + hash = strings.Replace(hash, FilecoinPrefix, "", -1) + hash = removeSpecialCharacters(hash) + zap.L().Debug("After Formatting", zap.String("metadataHash", hash)) + return hash +} + +func removeSpecialCharacters(pString string) string { + reg := regexp.MustCompile("[^a-zA-Z0-9=]") + return reg.ReplaceAllString(pString, "") +} diff --git a/ipfsutils/common_test.go b/ipfsutils/common_test.go new file mode 100644 index 00000000..25cea6e3 --- /dev/null +++ b/ipfsutils/common_test.go @@ -0,0 +1,45 @@ +package ipfsutils + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestReadFile(t *testing.T) { + file, err := ReadFile("filecoin://bafkreibk4ham7y6mwad2qxwyrhpmxh2fho7xwvsfw26lcra5bt5r5fvcwe") + assert.Nil(t, err) + assert.NotNil(t, file) +} + +func TestFormatHash(t *testing.T) { + s2 := []byte("ipfs://Here is a string....+=") + hash := formatHash(string(s2)) + assert.Equal(t, hash, "Hereisastring=") + + s2 = []byte("filecoin://QmaGnQ3iVZPuPwdam2rEeQcCSoCYRpxjnZhQ6Z2oeeRSrp") + b4 := append(s2, make([]byte, 3)...) + assert.NotEqual(t, "QmaGnQ3iVZPuPwdam2rEeQcCSoCYRpxjnZhQ6Z2oeeRSrp", string(b4)) + assert.Equal(t, "QmaGnQ3iVZPuPwdam2rEeQcCSoCYRpxjnZhQ6Z2oeeRSrp", formatHash(string(b4))) +} + +func TestRemoveSpecialCharactersfromHash(t *testing.T) { + testCases := []struct { + input string + expectedOutput string + }{ + {"abc123", "abc123"}, + {"abc123!@#", "abc123"}, + {"a1b2c3 ~`!@#$%^&*()_+-={}[]|\\:;\"'<>,.?/", "a1b2c3="}, + {"abc=123", "abc=123"}, + {"a1!b2@c3#=4", "a1b2c3=4"}, + } + + for _, tc := range testCases { + t.Run(tc.input, func(t *testing.T) { + output := removeSpecialCharacters(tc.input) + if output != tc.expectedOutput { + t.Errorf("RemoveSpecialCharactersfromHash(%q) = %q; want %q", tc.input, output, tc.expectedOutput) + } + }) + } +} diff --git a/ipfsutils/compressed.go b/ipfsutils/compressed.go new file mode 100644 index 00000000..f1943518 --- /dev/null +++ b/ipfsutils/compressed.go @@ -0,0 +1,52 @@ +package ipfsutils + +import ( + "archive/tar" + "bytes" + "fmt" + "go.uber.org/zap" + "io" +) + +// ReadFilesCompressed - read all files which have been compressed, there can be more than one file +// We need to start reading the proto files associated with the service. +// proto files are compressed and stored as modelipfsHash +func ReadFilesCompressed(compressedFile []byte) (protofiles []string, err error) { + f := bytes.NewReader(compressedFile) + tarReader := tar.NewReader(f) + protofiles = make([]string, 0) + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + zap.L().Error(err.Error()) + return nil, err + } + name := header.Name + switch header.Typeflag { + case tar.TypeDir: + zap.L().Debug("Directory name", zap.String("name", name)) + case tar.TypeReg: + zap.L().Debug("File name", zap.String("name", name)) + data := make([]byte, header.Size) + _, err := tarReader.Read(data) + if err != nil && err != io.EOF { + zap.L().Error(err.Error()) + return nil, err + } + protofiles = append(protofiles, string(data)) + default: + err = fmt.Errorf(fmt.Sprintf("%s : %c %s %s\n", + "Unknown file Type ", + header.Typeflag, + "in file", + name, + )) + zap.L().Error(err.Error()) + return nil, err + } + } + return protofiles, nil +} diff --git a/ipfsutils/ipfsutils.go b/ipfsutils/ipfsutils.go index f2fbbc47..87348316 100644 --- a/ipfsutils/ipfsutils.go +++ b/ipfsutils/ipfsutils.go @@ -1,10 +1,8 @@ package ipfsutils import ( - "archive/tar" "context" - "fmt" - + "errors" "github.com/ipfs/go-cid" "github.com/ipfs/kubo/client/rpc" "github.com/singnet/snet-daemon/config" @@ -12,54 +10,10 @@ import ( "io" "net/http" - "strings" "time" ) -// ReadFilesCompressed - read all files which have been compressed, there can be more than one file -// We need to start reading the proto files associated with the service. -// proto files are compressed and stored as modelipfsHash -func ReadFilesCompressed(compressedFile string) (protofiles []string, err error) { - f := strings.NewReader(compressedFile) - tarReader := tar.NewReader(f) - protofiles = make([]string, 0) - for { - header, err := tarReader.Next() - if err == io.EOF { - break - } - if err != nil { - zap.L().Error(err.Error()) - return nil, err - } - name := header.Name - switch header.Typeflag { - case tar.TypeDir: - zap.L().Debug("Directory name", zap.String("name", name)) - case tar.TypeReg: - zap.L().Debug("File name", zap.String("name", name)) - data := make([]byte, header.Size) - _, err := tarReader.Read(data) - if err != nil && err != io.EOF { - zap.L().Error(err.Error()) - return nil, err - } - protofiles = append(protofiles, string(data)) - default: - err = fmt.Errorf(fmt.Sprintf("%s : %c %s %s\n", - "Unknown file Type ", - header.Typeflag, - "in file", - name, - )) - zap.L().Error(err.Error()) - return nil, err - } - } - return protofiles, nil -} - -func GetIpfsFile(hash string) (content string) { +func GetIpfsFile(hash string) (content []byte, err error) { zap.L().Debug("Hash Used to retrieve from IPFS", zap.String("hash", hash)) @@ -67,29 +21,35 @@ func GetIpfsFile(hash string) (content string) { cID, err := cid.Parse(hash) if err != nil { - zap.L().Fatal("error parsing the ipfs hash", zap.String("hashFromMetaData", hash), zap.Error(err)) + zap.L().Error("error parsing the ipfs hash", zap.String("hashFromMetaData", hash), zap.Error(err)) + return nil, err } req := ipfsClient.Request("cat", cID.String()) - if err != nil { - zap.L().Fatal("error executing the cat command in ipfs", zap.String("hashFromMetaData", hash), zap.Error(err)) - return + if req == nil { + zap.L().Error("error executing the cat command in ipfs: req is nil", zap.String("hashFromMetaData", hash)) + return nil, err } resp, err := req.Send(context.Background()) if err != nil { - zap.L().Fatal("error executing the cat command in ipfs", zap.String("hashFromMetaData", hash), zap.Error(err)) - return + zap.L().Error("error executing the cat command in ipfs", zap.String("hashFromMetaData", hash), zap.Error(err)) + return nil, err } - defer resp.Close() + defer func(resp *rpc.Response) { + err := resp.Close() + if err != nil { + zap.L().Error(err.Error()) + } + }(resp) if resp.Error != nil { - zap.L().Fatal("error executing the cat command in ipfs", zap.String("hashFromMetaData", hash), zap.Error(err)) - return + zap.L().Error("error executing the cat command in ipfs", zap.String("hashFromMetaData", hash), zap.Error(err)) + return nil, err } fileContent, err := io.ReadAll(resp.Output) if err != nil { - zap.L().Fatal("error: in Reading the meta data file", zap.Error(err), zap.String("hashFromMetaData", hash)) - return + zap.L().Error("error: in Reading the meta data file", zap.Error(err), zap.String("hashFromMetaData", hash)) + return nil, err } // log.WithField("hash", hash).WithField("blob", string(fileContent)).Debug("Blob of IPFS file with hash") @@ -97,18 +57,19 @@ func GetIpfsFile(hash string) (content string) { // Create a cid manually to check cid _, c, err := cid.CidFromBytes(append(cID.Bytes(), fileContent...)) if err != nil { - zap.L().Fatal("error generating ipfs hash", zap.String("hashFromMetaData", hash), zap.Error(err)) - return + zap.L().Error("error generating ipfs hash", zap.String("hashFromMetaData", hash), zap.Error(err)) + return nil, err } // To test if two cid's are equivalent, be sure to use the 'Equals' method: if !c.Equals(cID) { - zap.L().Fatal("IPFS hash verification failed. Generated hash doesnt match with expected hash", + zap.L().Error("IPFS hash verification failed. Generated hash doesnt match with expected hash", zap.String("expectedHash", hash), zap.String("hashFromIPFSContent", c.String())) + return nil, errors.New("IPFS hash doesnt match with expected hash") } - return string(fileContent) + return fileContent, nil } func GetIPFSClient() *rpc.HttpApi { @@ -117,7 +78,7 @@ func GetIPFSClient() *rpc.HttpApi { } ifpsClient, err := rpc.NewURLApiWithClient(config.GetString(config.IpfsEndPoint), &httpClient) if err != nil { - zap.L().Panic("Connection failed to IPFS", zap.String("IPFS", config.GetString(config.IpfsEndPoint)), zap.Error(err)) + zap.L().Fatal("Connection failed to IPFS", zap.String("IPFS", config.GetString(config.IpfsEndPoint)), zap.Error(err)) } return ifpsClient } diff --git a/ipfsutils/ipfsutils_test.go b/ipfsutils/ipfsutils_test.go index ddd4caef..abd1f406 100644 --- a/ipfsutils/ipfsutils_test.go +++ b/ipfsutils/ipfsutils_test.go @@ -27,8 +27,9 @@ func (suite *IpfsUtilsTestSuite) BeforeTest() { func (suite *IpfsUtilsTestSuite) TestReadFiles() { // For testing purposes, a hash is used from the calculator service. hash := "QmeyrQkEyba8dd4rc3jrLd5pEwsxHutfH2RvsSaeSMqTtQ" - data := GetIpfsFile(hash) + data, err := GetIpfsFile(hash) assert.NotNil(suite.T(), data) + assert.Nil(suite.T(), err) protoFiles, err := ReadFilesCompressed(data) diff --git a/ipfsutils/lighthouse.go b/ipfsutils/lighthouse.go new file mode 100644 index 00000000..4849e5a9 --- /dev/null +++ b/ipfsutils/lighthouse.go @@ -0,0 +1,20 @@ +package ipfsutils + +import ( + "io" + "net/http" +) + +const lighthouseURL = "https://gateway.lighthouse.storage/ipfs/" + +func GetLighthouseFile(cID string) ([]byte, error) { + resp, err := http.Get(lighthouseURL + cID) + if err != nil { + return nil, err + } + file, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return file, nil +} diff --git a/metrics/clients_test.go b/metrics/clients_test.go index 3f641f7e..f02d2715 100644 --- a/metrics/clients_test.go +++ b/metrics/clients_test.go @@ -7,12 +7,13 @@ package metrics import ( "context" + "net" + "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "google.golang.org/grpc" pb "google.golang.org/grpc/health/grpc_health_v1" - "net" - "testing" ) // server is used to implement api.HeartbeatServer @@ -101,14 +102,16 @@ func (suite *ClientTestSuite) Test_callHTTPServiceHeartbeat() { assert.True(suite.T(), err != nil) } -func (suite *ClientTestSuite) Test_callRegisterService() { +// TODO: refactor register service - daemonID := GetDaemonID() +// func (suite *ClientTestSuite) Test_callRegisterService() { - result := callRegisterService(daemonID, suite.serviceURL+"/register") - assert.Equal(suite.T(), true, result) +// daemonID := GetDaemonID() - result = callRegisterService(daemonID, suite.serviceURL+"/registererror") - assert.Equal(suite.T(), false, result) +// result := callRegisterService(daemonID, suite.serviceURL+"/register") +// assert.Equal(suite.T(), true, result) -} +// result = callRegisterService(daemonID, suite.serviceURL+"/registererror") +// assert.Equal(suite.T(), false, result) + +// } diff --git a/metrics/register_test.go b/metrics/register_test.go index 78982257..9192feed 100644 --- a/metrics/register_test.go +++ b/metrics/register_test.go @@ -6,9 +6,10 @@ package metrics import ( + "testing" + "github.com/stretchr/testify/suite" "google.golang.org/grpc" - "testing" "github.com/stretchr/testify/assert" ) @@ -40,15 +41,17 @@ func TestGetDaemonID(t *testing.T) { assert.NotEqual(t, "48d343313a1e06093c81830103b45496cc7c277f321049e9ee632fd03207d042", daemonID) } -func (suite *RegisterTestSuite) TestRegisterDaemon() { +// TODO: refactor register service - result := RegisterDaemon(suite.serviceURL + "/register") - assert.Equal(suite.T(), true, result) +// func (suite *RegisterTestSuite) TestRegisterDaemon() { - wrongserviceURL := "https://localhost:9999/registererror" - result = RegisterDaemon(wrongserviceURL) - assert.Equal(suite.T(), false, result) -} +// result := RegisterDaemon(suite.serviceURL + "/register") +// assert.Equal(suite.T(), true, result) + +// wrongserviceURL := "https://localhost:9999/registererror" +// result = RegisterDaemon(wrongserviceURL) +// assert.Equal(suite.T(), false, result) +// } func (suite *RegisterTestSuite) TestSetDaemonGrpId() { grpid := "group01" diff --git a/snetd/cmd/serve.go b/snetd/cmd/serve.go index 4d453ce8..31dc1720 100644 --- a/snetd/cmd/serve.go +++ b/snetd/cmd/serve.go @@ -147,14 +147,19 @@ func newDaemon(components *Components) (daemon, error) { func (d *daemon) start() { var tlsConfig *tls.Config + var certReloader *CertReloader - certReloader := CertReloader{ - CertFile: config.GetString(config.SSLCertPathKey), - KeyFile: config.GetString(config.SSLKeyPathKey), - mutex: new(sync.Mutex), + if config.GetString(config.SSLCertPathKey) != "" { + certReloader = &CertReloader{ + CertFile: config.GetString(config.SSLCertPathKey), + KeyFile: config.GetString(config.SSLKeyPathKey), + mutex: new(sync.Mutex), + } } - certReloader.Listen() + if certReloader != nil { + certReloader.Listen() + } if d.autoSSLDomain != "" { zap.L().Debug("enabling automatic SSL support")