Merge branch 'main' into chore/sign-CLA

Author: DevlinRocha

LOCAL_DEVELOPMENT.md

@@ -17,10 +17,9 @@ A minimal local Lilypad network consists of the following pieces of infrastructu
 - One bacalhau node
 - One postgres database
 - One solver service
-- One job creator service
 - One resource provider service
 
-Order matters because the `solver`, `job creator` and `resource provider` will right away try to connect to the `blockchain node`. First, the `solver` will update the state of a known smart contract to publish a URL where other services can connect to it. Then, the `job creator`, and `resource provider` will fetch from the `blockchain` the URL for the `solver` and try to connect to it.
+Order matters because the `solver` and `resource provider` will right away try to connect to the `blockchain node`. First, the `solver` will update the state of a known smart contract to publish a URL where other services can connect to it. Then, the `resource provider` will fetch from the `blockchain` the URL for the `solver` and try to connect to it.
 
 ## Minimal local setup
 
@@ -39,11 +38,7 @@ A helper script is in place to verify balances on the accounts: `cd hardhat && n
 
 This process can be executed directly if Golang has been installed or in a docker container. The commands are `./stack solver`,`./stack solver-docker-build` and `./stack solver-docker-run` respectively. The `solver` service will output a log line that reads that "the solver has been registered successfully" or "the solver already exists". It is best to wait for this output before starting the services that will try to connect to the `solver`.
 
-### 3. Job creator
-
-This process can be executed directly if Golang has been installed or in a docker container. The commands are `./stack job-creator`,`./stack job-creator-docker-build` and `./stack job-creator-docker-run` respectively. The `job-creator` service's main function is to listen to events from the blockchain to execute jobs and when it receives such an event it will relay the payload to the `solver`. So think about the `job-creator` as the "on-chain solver".
-
-### 4. Resource provider
+### 3. Resource provider
 
 For the time being this process has to be executed directly and needs Golang to be installed. This is the command to execute the service: `./stack resource-provider`. If you have a GPU you can use the following flag to use it: `./stack resource-provider --offer-gpu 1`
 
@@ -76,7 +71,25 @@ Run `./stack compose-down`.
 
 ## Running a job
 
-Once all the services are up and running this command can be used to trigger an on-chain job: `./stack run-cowsay-onchain`
+Once all the services are up, run a cowsay job with:
+
+```sh
+./stack run cowsay:v0.0.4 -i Message="Hello!"
+```
+
+The `cowsay:v0.0.4` specifies the module to run with a short code and tag, but a module URL and tag can also be used:
+
+```sh
+./stack run github.com/Lilypad-Tech/lilypad-module-cowsay:v0.0.4 -i Message="Hello!"
+```
+
+Lastly, a module URL and git hash can be used:
+
+```sh
+./stack run github.com/Lilypad-Tech/lilypad-module-cowsay:cb8b670805b06206bd63603a8ba582638a619fe5 -i Message="Hello!"
+```
+
+The `-i Message="Hello!"` states the input to the module. `Message="Hello!"` is an input expected by the cowsay module. Other modules may expect a different set of input key-value pairs.
 
 ### Tests
 
@@ -111,6 +124,12 @@ This can be addressed by doing the following:
 
 - Open your Docker Desktop app, go to `Volumes` and delete `lilypad_chain-data` as there might be stale data in the volume not allowing you to properly execute all the transactions `chain-boot` executes
 
+## Onchain job creator
+
+This onchain job creator can be run directly if Golang has been installed. Run the onchain job creator with `./stack job-creator`. The onchain job creator service's main function is to listen to events from the blockchain to execute jobs and when it receives such an event it will relay the payload to the solver.
+
+We have an example script that submits a job the blockchain for the onchain job creator to pick up and run. Use `./stack run-cowsay-onchain` to run an onchain job. Note that all the services listed above must also be running.
+
 ### Issues running onchain cowsay
 
 If you find that you have issues with the Job Creator not picking up your `run-cowsay-onchain` command while running the Lilypad stack through Docker, do the following:

cmd/lilypad/run.go

@@ -142,11 +142,11 @@ func runJob(cmd *cobra.Command, options jobcreator.JobCreatorOptions, network st
 		return err
 	}
 	spinner.Stop()
-	fmt.Printf("\n🍂 Lilypad job completed, try 👇\n    open %s\n    cat %s/stdout\n    cat %s/stderr\n    https://ipfs.io/ipfs/%s\n",
+	fmt.Printf("🆔  Data ID: %s\n", result.Result.DataID)
+	fmt.Printf("\n🍂 Lilypad job completed, try 👇\n    open %s\n    cat %s/stdout\n    cat %s/stderr\n",
 		solver.GetDownloadsFilePath(result.JobOffer.DealID),
 		solver.GetDownloadsFilePath(result.JobOffer.DealID),
 		solver.GetDownloadsFilePath(result.JobOffer.DealID),
-		result.Result.DataID,
 	)
 	return err
 }

go.mod

@@ -211,6 +211,7 @@ require (
 	k8s.io/klog/v2 v2.110.1 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	lukechampine.com/blake3 v1.3.0 // indirect
+	pgregory.net/rapid v1.1.0 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect
 )

go.sum

@@ -1355,6 +1355,8 @@ modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6
 modernc.org/strutil v1.1.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs=
 modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
 modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I=
+pgregory.net/rapid v1.1.0 h1:CMa0sjHSru3puNx+J0MIAuiiEV4N0qj8/cMWGBBCsjw=
+pgregory.net/rapid v1.1.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

pkg/data/types.go renamed to pkg/data/data.go

@@ -1,6 +1,8 @@
 package data
 
 import (
+	"encoding/json"
+
 	"github.com/lilypad-tech/lilypad/pkg/data/bacalhau"
 )
 
@@ -76,11 +78,62 @@ type Result struct {
 	ID     string `json:"id"`
 	DealID string `json:"deal_id"`
 	// the CID of the actual results
-	DataID           string `json:"results_id"`
+	DataID           string `json:"data_id"`
 	Error            string `json:"error"`
 	InstructionCount uint64 `json:"instruction_count"`
 }
 
+// Provides compatibility for older clients that expect the results_id field
+func (r Result) MarshalJSON() ([]byte, error) {
+	// TODO(bgins) Remove when older clients have been deprecated
+
+	// Create an auxiliary type to avoid recursively calling json.Marshal
+	// https://stackoverflow.com/a/23046869
+	type ResultAux Result
+
+	// Add results_id field to the existing Result fields and marshal
+	return json.Marshal(struct {
+		ResultAux
+		ResultsID string `json:"results_id"`
+	}{
+		ResultAux: ResultAux(r),
+		ResultsID: r.DataID,
+	})
+}
+
+// Provides compatibility for newer clients that expect the data_id field
+func (r *Result) UnmarshalJSON(data []byte) error {
+	// TODO(bgins) Remove when older clients have been deprecated
+
+	// Create an auxiliary type to avoid recursively calling json.Unmarshal
+	// https://stackoverflow.com/a/52433660
+	type ResultAux Result
+
+	// Unmarshal into auxiliary type
+	var aux ResultAux
+	if err := json.Unmarshal(data, &aux); err != nil {
+		return err
+	}
+
+	// Cast the auxilliary type to Result
+	*r = Result(aux)
+
+	// Create a raw map to capture the results_id field
+	var rawMap map[string]interface{}
+	if err := json.Unmarshal(data, &rawMap); err != nil {
+		return err
+	}
+
+	// Check if results_id exists and assign it to DataID if so
+	if resultsID, ok := rawMap["results_id"]; ok {
+		if strID, ok := resultsID.(string); ok {
+			r.DataID = strID
+		}
+	}
+
+	return nil
+}
+
 // MarketPrice means - get me the best deal
 // job creators will do this by default i.e. "just buy me the cheapest"
 // FixedPrice means - take it or leave it

pkg/data/data_test.go

@@ -0,0 +1,106 @@
+//go:build unit
+
+package data
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/mr-tron/base58"
+	"pgregory.net/rapid"
+)
+
+func TestResultJSONRoundtrip(t *testing.T) {
+	rapid.Check(t, func(t *rapid.T) {
+		// Generate a random Result
+		original := Result{
+			ID:     generateCID(t),
+			DealID: generateCID(t),
+			DataID: generateCID(t),
+		}
+
+		// Test marshaling to JSON
+		data, err := json.Marshal(original)
+		if err != nil {
+			t.Fatalf("Marshal failed: %v", err)
+		}
+
+		// Test unmarshaling back
+		var decoded Result
+		err = json.Unmarshal(data, &decoded)
+		if err != nil {
+			t.Fatalf("Unmarshal failed: %v", err)
+		}
+
+		// Verify the roundtrip preserved values
+		if original != decoded {
+			t.Errorf("Roundtrip failed: got %v, want %v", decoded, original)
+		}
+
+		// Verify both fields exist in JSON
+		var rawData map[string]interface{}
+		err = json.Unmarshal(data, &rawData)
+		if err != nil {
+			t.Fatalf("Raw unmarshal failed: %v", err)
+		}
+
+		// Check both data_id and results_id exist and match
+		dataID, hasDataID := rawData["data_id"]
+		resultsID, hasResultsID := rawData["results_id"]
+
+		if !hasDataID {
+			t.Error("data_id field missing")
+		}
+		if !hasResultsID {
+			t.Error("results_id field missing")
+		}
+		if dataID != resultsID {
+			t.Errorf("data_id and results_id mismatch: %v != %v", dataID, resultsID)
+		}
+	})
+}
+
+func TestResultJSONBackwardsCompatibility(t *testing.T) {
+	rapid.Check(t, func(t *rapid.T) {
+		expectedDataID := generateCID(t)
+
+		// Test old client format (results_id)
+		oldClientJSON := map[string]interface{}{
+			"id":         generateCID(t),
+			"results_id": expectedDataID,
+		}
+		oldClientData, _ := json.Marshal(oldClientJSON)
+
+		var resultFromOld Result
+		err := json.Unmarshal(oldClientData, &resultFromOld)
+		if err != nil {
+			t.Fatalf("Unmarshal of old format failed: %v", err)
+		}
+		if resultFromOld.DataID != expectedDataID {
+			t.Errorf("Old format: got DataID %v, want %v", resultFromOld.DataID, expectedDataID)
+		}
+
+		// Test new client format (data_id)
+		newClientJSON := map[string]interface{}{
+			"id":      generateCID(t),
+			"data_id": expectedDataID,
+		}
+		newClientData, _ := json.Marshal(newClientJSON)
+
+		var resultFromNew Result
+		err = json.Unmarshal(newClientData, &resultFromNew)
+		if err != nil {
+			t.Fatalf("Unmarshal of new format failed: %v", err)
+		}
+		if resultFromNew.DataID != expectedDataID {
+			t.Errorf("New format: got DataID %v, want %v", resultFromNew.DataID, expectedDataID)
+		}
+	})
+}
+
+// Generators
+
+func generateCID(t *rapid.T) string {
+	bytes := rapid.SliceOfN(rapid.Byte(), 32, 32).Draw(t, "bytes")
+	return "Qm" + base58.Encode(bytes)
+}

pkg/solver/matcher/match.go

@@ -81,6 +81,50 @@ func (result ramMismatch) attributes() []attribute.KeyValue {
 	}
 }
 
+type vramMismatch struct {
+	resourceOffer data.ResourceOffer
+	jobOffer      data.JobOffer
+}
+
+func (_ vramMismatch) matched() bool   { return false }
+func (_ vramMismatch) message() string { return "did not match VRAM" }
+func (result vramMismatch) attributes() []attribute.KeyValue {
+	var resourceOfferVRAMs []int
+	for _, gpu := range result.resourceOffer.Spec.GPUs {
+		resourceOfferVRAMs = append(resourceOfferVRAMs, gpu.VRAM)
+	}
+
+	var jobOfferVRAMS []int
+	for _, gpu := range result.jobOffer.Spec.GPUs {
+		jobOfferVRAMS = append(jobOfferVRAMS, gpu.VRAM)
+	}
+
+	return []attribute.KeyValue{
+		attribute.String("match_result", fmt.Sprintf("%T", result)),
+		attribute.Bool("match_result.matched", result.matched()),
+		attribute.String("match_result.message", result.message()),
+		attribute.IntSlice("match_result.resource_offer.spec.gpus.vram", resourceOfferVRAMs),
+		attribute.IntSlice("match_result.job_offer.spec.gpus.vram", jobOfferVRAMS),
+	}
+}
+
+type diskSpaceMismatch struct {
+	resourceOffer data.ResourceOffer
+	jobOffer      data.JobOffer
+}
+
+func (_ diskSpaceMismatch) matched() bool   { return false }
+func (_ diskSpaceMismatch) message() string { return "did not match disk space" }
+func (result diskSpaceMismatch) attributes() []attribute.KeyValue {
+	return []attribute.KeyValue{
+		attribute.String("match_result", fmt.Sprintf("%T", result)),
+		attribute.Bool("match_result.matched", result.matched()),
+		attribute.String("match_result.message", result.message()),
+		attribute.Int("match_result.job_offer.spec.disk", result.jobOffer.Spec.Disk),
+		attribute.Int("match_result.resource_offer.spec.disk", result.resourceOffer.Spec.Disk),
+	}
+}
+
 type moduleIDError struct {
 	resourceOffer data.ResourceOffer
 	jobOffer      data.JobOffer
@@ -227,6 +271,34 @@ func matchOffers(
 		}
 	}
 
+	// Skip VRAM check when job offer does not request VRAM
+	if len(jobOffer.Spec.GPUs) > 0 {
+		// Mismatch if job offer requests VRAM but resource provider has none
+		if len(resourceOffer.Spec.GPUs) == 0 {
+			return &vramMismatch{
+				jobOffer:      jobOffer,
+				resourceOffer: resourceOffer,
+			}
+		}
+
+		// Mismatch if job offer largest VRAM is greater than resource offer largest VRAM
+		largestResourceOfferVRAM := getLargestVRAM(resourceOffer.Spec.GPUs)
+		largestJobOfferVRAM := getLargestVRAM(jobOffer.Spec.GPUs)
+		if largestResourceOfferVRAM < largestJobOfferVRAM {
+			return &vramMismatch{
+				jobOffer:      jobOffer,
+				resourceOffer: resourceOffer,
+			}
+		}
+	}
+
+	if resourceOffer.Spec.Disk < jobOffer.Spec.Disk {
+		return &diskSpaceMismatch{
+			jobOffer:      jobOffer,
+			resourceOffer: resourceOffer,
+		}
+	}
+
 	moduleID, err := data.GetModuleID(jobOffer.Module)
 	if err != nil {
 		return &moduleIDError{
@@ -295,6 +367,16 @@ func matchOffers(
 	}
 }
 
+func getLargestVRAM(gpus []data.GPUSpec) int {
+	largestVRAM := 0
+	for _, gpu := range gpus {
+		if gpu.VRAM > largestVRAM {
+			largestVRAM = gpu.VRAM
+		}
+	}
+	return largestVRAM
+}
+
 func logMatch(result matchResult) {
 	switch r := result.(type) {
 	case offersMatched: