From f47e5c9bba1635aadd38f3c52e5d839319e68de3 Mon Sep 17 00:00:00 2001 From: alvin-reyes Date: Sat, 5 Aug 2023 15:56:46 -0400 Subject: [PATCH] move the db objects to the delta main --- api/admin.go | 2 +- api/deal.go | 2 +- api/node.go | 2 +- api/open_info.go | 2 +- api/open_stats.go | 2 +- api/repair_retry.go | 2 +- api/router.go | 45 +-- api/stats.go | 2 +- api/websocket.go | 2 +- cmd/daemon.go | 16 +- core/libp2p.go | 2 +- core/node.go | 18 +- core/replication.go | 2 +- core/stats.go | 2 +- core/status_logger.go | 2 +- core/wallet.go | 2 +- core/wallet_test.go | 2 +- go.mod | 8 +- go.sum | 4 +- jobs/clean_up.go | 2 +- jobs/clean_up_content.go | 2 +- jobs/data_transfer_restart.go | 2 +- jobs/data_transfer_status.go | 2 +- jobs/deal_status_check.go | 2 +- jobs/instance_meta.go | 2 +- jobs/log_event.go | 54 ++-- jobs/miner_check.go | 2 +- jobs/piece_commp_compute.go | 2 +- jobs/retry.go | 2 +- jobs/storage_deal_maker.go | 2 +- models/base.go | 28 ++ models/batch_import_content.go | 107 +++++++ models/content.go | 75 +++++ models/content_deal.go | 92 ++++++ models/content_deal_proposal.go | 61 ++++ models/content_deal_proposal_parameters.go | 72 +++++ models/content_miner.go | 57 ++++ models/content_wallet.go | 56 ++++ models/database.go | 102 +++++++ models/instance_meta.go | 91 ++++++ models/log_event.go | 1 + models/log_models.go | 323 +++++++++++++++++++++ models/piece_commitment.go | 66 +++++ models/repair_request.go | 13 + models/wallet.go | 66 +++++ utils/constants.go | 6 - utils/model_utils.go | 2 +- 47 files changed, 1272 insertions(+), 139 deletions(-) create mode 100644 models/base.go create mode 100644 models/batch_import_content.go create mode 100644 models/content.go create mode 100644 models/content_deal.go create mode 100644 models/content_deal_proposal.go create mode 100644 models/content_deal_proposal_parameters.go create mode 100644 models/content_miner.go create mode 100644 models/content_wallet.go create mode 100644 models/database.go create mode 100644 models/instance_meta.go create mode 100644 models/log_event.go create mode 100644 models/log_models.go create mode 100644 models/piece_commitment.go create mode 100644 models/repair_request.go create mode 100644 models/wallet.go diff --git a/api/admin.go b/api/admin.go index 5cccee4..2c6e0dc 100644 --- a/api/admin.go +++ b/api/admin.go @@ -7,7 +7,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" "github.com/google/uuid" diff --git a/api/deal.go b/api/deal.go index f8f2e85..644247b 100644 --- a/api/deal.go +++ b/api/deal.go @@ -9,7 +9,7 @@ import ( "delta/utils" "encoding/json" "fmt" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "github.com/google/uuid" "github.com/ipfs/go-cid" "github.com/multiformats/go-multiaddr" diff --git a/api/node.go b/api/node.go index 06fd45e..d9abbb6 100644 --- a/api/node.go +++ b/api/node.go @@ -3,7 +3,7 @@ package api import ( "context" "delta/core" - "github.com/application-research/delta-db/db_models" + "delta/models" "github.com/labstack/echo/v4" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" diff --git a/api/open_info.go b/api/open_info.go index 31be846..743cc07 100644 --- a/api/open_info.go +++ b/api/open_info.go @@ -3,7 +3,7 @@ package api import ( "context" "delta/core" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" "github.com/labstack/echo/v4" diff --git a/api/open_stats.go b/api/open_stats.go index 5993ad8..8027327 100644 --- a/api/open_stats.go +++ b/api/open_stats.go @@ -4,7 +4,7 @@ import ( "delta/core" "delta/jobs" "delta/utils" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "github.com/labstack/echo/v4" "strconv" ) diff --git a/api/repair_retry.go b/api/repair_retry.go index 2fceacc..dc25efa 100644 --- a/api/repair_retry.go +++ b/api/repair_retry.go @@ -5,7 +5,7 @@ import ( "delta/jobs" "delta/utils" "encoding/json" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "github.com/labstack/echo/v4" "strings" "time" diff --git a/api/router.go b/api/router.go index 3f2d015..f40f32a 100644 --- a/api/router.go +++ b/api/router.go @@ -3,7 +3,7 @@ package api import ( "delta/config" "delta/core" - "delta/utils" + _ "delta/docs/swagger" "encoding/json" "fmt" "net/http" @@ -11,10 +11,7 @@ import ( "os/signal" "strings" "syscall" - "time" - _ "delta/docs/swagger" - "github.com/application-research/delta-db/messaging" logging "github.com/ipfs/go-log/v2" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" @@ -268,46 +265,6 @@ func ValidateRequestBody() echo.MiddlewareFunc { // ErrorHandler It's a function that is called when an error occurs. func ErrorHandler(err error, c echo.Context) { - ip := DeltaNodeConfig.Node.AnnounceAddrIP - - // get the request body and log it - - s := struct { - RemoteIP string `json:"remote_ip"` - PublicIP string `json:"public_ip"` - Host string `json:"host"` - Referer string `json:"referer"` - Request string `json:"request"` - Path string `json:"path"` - ErrorDetails string `json:"details"` - }{ - RemoteIP: c.RealIP(), - PublicIP: ip, - Host: c.Request().Host, - Referer: c.Request().Referer(), - Request: c.Request().RequestURI, - Path: c.Path(), - ErrorDetails: err.Error(), - } - - b, errM := json.Marshal(s) - if errM != nil { - log.Error(errM) - } - - // It's sending the error to the log server. - utils.GlobalDeltaDataReporter.TraceLog( - messaging.LogEvent{ - LogEventType: "Error: " + core.GetHostname() + " " + c.Request().Method + " " + c.Path(), - SourceHost: core.GetHostname(), - SourceIP: ip, - LogEventObject: b, - LogEvent: c.Path(), - DeltaUuid: DeltaNodeConfig.Node.InstanceUuid, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }) - var httpRespErr *HttpError if xerrors.As(err, &httpRespErr) { log.Errorf("handler error: %s", err) diff --git a/api/stats.go b/api/stats.go index 59a1c46..effae2b 100644 --- a/api/stats.go +++ b/api/stats.go @@ -2,7 +2,7 @@ package api import ( "delta/core" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "strings" "github.com/labstack/echo/v4" diff --git a/api/websocket.go b/api/websocket.go index 6498aee..9c4669e 100644 --- a/api/websocket.go +++ b/api/websocket.go @@ -3,7 +3,7 @@ package api import ( "delta/core" "fmt" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "github.com/gorilla/websocket" "github.com/labstack/echo/v4" "sync" diff --git a/cmd/daemon.go b/cmd/daemon.go index 35aa92e..942aa62 100644 --- a/cmd/daemon.go +++ b/cmd/daemon.go @@ -4,16 +4,13 @@ import ( "delta/api" c "delta/config" "delta/core" + _ "delta/models" "delta/utils" "fmt" - "github.com/application-research/delta-db/db_models" - _ "github.com/application-research/delta-db/db_models" - "github.com/application-research/delta-db/messaging" "github.com/filecoin-project/go-address" "github.com/jasonlvhit/gocron" "github.com/urfave/cli/v2" "runtime" - "time" ) // DaemonCmd Creating a new command called `daemon` that will run the API node. @@ -186,17 +183,6 @@ By: Protocol Labs - Outercore Engineering ` + utils.Reset + utils.Red + "version: " + cfg.Common.Version + utils.Reset) fmt.Println(utils.Blue + "Reporting Delta startup logs" + utils.Reset) - utils.GlobalDeltaDataReporter.Trace(messaging.DeltaMetricsBaseMessage{ - ObjectType: "DeltaStartupLogs", - Object: db_models.DeltaStartupLogs{ - NodeInfo: core.GetHostname(), - OSDetails: runtime.GOARCH + " " + runtime.GOOS, - IPAddress: ip, - DeltaNodeUuid: cfg.Node.InstanceUuid, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - }) fmt.Println(utils.Blue + "Reporting Delta startup logs... DONE" + utils.Reset) fmt.Println("----------------------------------") fmt.Println(utils.Green + "Welcome! Delta daemon is running..." + utils.Reset) diff --git a/core/libp2p.go b/core/libp2p.go index 0598283..0f7714c 100644 --- a/core/libp2p.go +++ b/core/libp2p.go @@ -4,7 +4,7 @@ import ( "context" "delta/utils" "fmt" - model "github.com/application-research/delta-db/db_models" + model "delta/models" fc "github.com/application-research/filclient" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/ipfs/go-cid" diff --git a/core/node.go b/core/node.go index d1ad687..adb4f05 100644 --- a/core/node.go +++ b/core/node.go @@ -3,12 +3,11 @@ package core import ( "context" c "delta/config" + model "delta/models" "delta/utils" "encoding/base64" "encoding/json" "fmt" - model "github.com/application-research/delta-db/db_models" - "github.com/application-research/delta-db/messaging" fc "github.com/application-research/filclient" "github.com/application-research/filclient/keystore" "github.com/application-research/whypfs-core" @@ -30,7 +29,6 @@ import ( "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" "github.com/multiformats/go-multiaddr" - trace2 "go.opencensus.io/trace" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/sdk/trace" "gorm.io/gorm" @@ -75,19 +73,13 @@ type DeltaNode struct { Dispatcher *Dispatcher MetaInfo *model.InstanceMeta - DeltaEventEmitter *DeltaEventEmitter - DeltaMetricsTracer *DeltaMetricsTracer + DeltaEventEmitter *DeltaEventEmitter } type DeltaEventEmitter struct { WebsocketBroadcast WebsocketBroadcast } -type DeltaMetricsTracer struct { - Tracer trace2.Tracer - DeltaDataReporter *messaging.DeltaMetricsTracer -} - // WebsocketBroadcast `WebsocketBroadcast` is a struct that contains three channels, one for each type of message that can be broadcasted. // @property {ContentChannel} ContentChannel - This is the channel that will be used to send content to the client. // @property {PieceCommitmentChannel} PieceCommitmentChannel - This is a channel that will be used to send piece @@ -266,9 +258,6 @@ func NewLightNode(repo NewLightNodeParams) (*DeltaNode, error) { // job dispatcher dispatcher := CreateNewDispatcher() - // delta metrics tracer - dataTracer := messaging.NewDeltaMetricsTracer() - openTelemetryTracerProvider := trace.NewTracerProvider(trace.WithSampler(trace.AlwaysSample())) defer openTelemetryTracerProvider.Shutdown(context.Background()) @@ -285,9 +274,6 @@ func NewLightNode(repo NewLightNodeParams) (*DeltaNode, error) { Dispatcher: dispatcher, LotusApiNode: api, Config: repo.Config, - DeltaMetricsTracer: &DeltaMetricsTracer{ - DeltaDataReporter: dataTracer, - }, }, nil } diff --git a/core/replication.go b/core/replication.go index aadce6a..851b2a6 100644 --- a/core/replication.go +++ b/core/replication.go @@ -3,7 +3,7 @@ package core // //import ( // "fmt" -// model "github.com/application-research/delta-db/db_models" +// model "delta/models" // "gorm.io/gorm" // "time" //) diff --git a/core/stats.go b/core/stats.go index 40f626f..2ace72d 100644 --- a/core/stats.go +++ b/core/stats.go @@ -1,6 +1,6 @@ package core -import model "github.com/application-research/delta-db/db_models" +import model "delta/models" type StatsService struct { DeltaNode *DeltaNode diff --git a/core/status_logger.go b/core/status_logger.go index 4e6aeee..2ff3ca1 100644 --- a/core/status_logger.go +++ b/core/status_logger.go @@ -1,6 +1,6 @@ package core -import model "github.com/application-research/delta-db/db_models" +import model "delta/models" // StatusLogger is used to change the status of each system objects in a async manner. // Each state content, commp and transfers needs to be updated and logged in the database. diff --git a/core/wallet.go b/core/wallet.go index f8f5c78..81d2ed0 100644 --- a/core/wallet.go +++ b/core/wallet.go @@ -7,7 +7,7 @@ import ( "encoding/hex" "encoding/json" "fmt" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/wallet" diff --git a/core/wallet_test.go b/core/wallet_test.go index d147530..db04de6 100644 --- a/core/wallet_test.go +++ b/core/wallet_test.go @@ -2,7 +2,7 @@ package core import ( "context" - "github.com/application-research/delta-db/db_models" + "delta/models" "reflect" "testing" ) diff --git a/go.mod b/go.mod index 4e06680..e0c9c91 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 - github.com/application-research/delta-db v0.0.2-0.20230504034024-a5e4190148d6 + github.com/application-research/delta-db v0.0.1 github.com/application-research/filclient v0.5.0-rc1.0.20230331195738-9826f79f0648 github.com/application-research/whypfs-core v0.1.1 github.com/caarlos0/env/v6 v6.10.1 @@ -49,10 +49,11 @@ require ( github.com/swaggo/echo-swagger v1.4.0 github.com/swaggo/swag v1.16.1 github.com/urfave/cli/v2 v2.24.4 - go.opencensus.io v0.24.0 go.opentelemetry.io/otel v1.14.0 go.opentelemetry.io/otel/sdk v1.14.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 + gorm.io/driver/postgres v1.4.8 + gorm.io/driver/sqlite v1.4.4 gorm.io/gorm v1.24.5 ) @@ -287,6 +288,7 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/zondax/hid v0.9.1 // indirect github.com/zondax/ledger-go v0.12.1 // indirect + go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/trace v1.14.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.16.1 // indirect @@ -309,8 +311,6 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - gorm.io/driver/postgres v1.4.8 // indirect - gorm.io/driver/sqlite v1.4.4 // indirect howett.net/plist v0.0.0-20181124034731-591f970eefbb // indirect lukechampine.com/blake3 v1.1.7 // indirect ) diff --git a/go.sum b/go.sum index d4110d3..c89dfe8 100644 --- a/go.sum +++ b/go.sum @@ -92,8 +92,8 @@ github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYU github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/application-research/delta-db v0.0.2-0.20230504034024-a5e4190148d6 h1:LGMPeFvr8n+gRorQ1mX/PuvdXG4e0wfCLUVebPkyDcI= -github.com/application-research/delta-db v0.0.2-0.20230504034024-a5e4190148d6/go.mod h1:LkFnhccvtKmSNHCatQNlKNjEAV/30w/e50n+A9hzCaU= +github.com/application-research/delta-db v0.0.1 h1:rvqhS/nvXhl6M8mlbb2bEnN9JBiwy8TGnqn+7hDBXxw= +github.com/application-research/delta-db v0.0.1/go.mod h1:LkFnhccvtKmSNHCatQNlKNjEAV/30w/e50n+A9hzCaU= github.com/application-research/filclient v0.5.0-rc1.0.20230331195738-9826f79f0648 h1:58+odb1yvlrhuWbmpcAu7IKN4/0+LOP8XjKP3JxOPGo= github.com/application-research/filclient v0.5.0-rc1.0.20230331195738-9826f79f0648/go.mod h1:HSCZh+v53XTGMwHGyoKpedQH+PlwcmKkMFCYx/Q3hMk= github.com/application-research/whypfs-core v0.1.1 h1:BY40N15Ge9BlyAT3f4T4B/CAlaL7A1RYaRrNS3HIyEk= diff --git a/jobs/clean_up.go b/jobs/clean_up.go index ba6198a..b98c71a 100644 --- a/jobs/clean_up.go +++ b/jobs/clean_up.go @@ -5,7 +5,7 @@ import ( "delta/core" "delta/utils" "fmt" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "github.com/ipfs/go-cid" "time" ) diff --git a/jobs/clean_up_content.go b/jobs/clean_up_content.go index af11b0c..4a8679b 100644 --- a/jobs/clean_up_content.go +++ b/jobs/clean_up_content.go @@ -4,7 +4,7 @@ import ( "context" "delta/core" "fmt" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "github.com/ipfs/go-cid" ) diff --git a/jobs/data_transfer_restart.go b/jobs/data_transfer_restart.go index 4058f56..c8b358b 100644 --- a/jobs/data_transfer_restart.go +++ b/jobs/data_transfer_restart.go @@ -5,7 +5,7 @@ import ( "delta/core" "delta/utils" "fmt" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "github.com/application-research/filclient" ) diff --git a/jobs/data_transfer_status.go b/jobs/data_transfer_status.go index 467c02b..1e4ea5b 100644 --- a/jobs/data_transfer_status.go +++ b/jobs/data_transfer_status.go @@ -4,7 +4,7 @@ import ( "delta/core" "delta/utils" "fmt" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "github.com/application-research/filclient" datatransfer "github.com/filecoin-project/go-data-transfer" "strconv" diff --git a/jobs/deal_status_check.go b/jobs/deal_status_check.go index f1ff2d5..6342973 100644 --- a/jobs/deal_status_check.go +++ b/jobs/deal_status_check.go @@ -3,9 +3,9 @@ package jobs import ( "context" "delta/core" + model "delta/models" "encoding/base64" "fmt" - model "github.com/application-research/delta-db/db_models" fc "github.com/application-research/filclient" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-fil-markets/storagemarket" diff --git a/jobs/instance_meta.go b/jobs/instance_meta.go index 1fbbde0..09fa857 100644 --- a/jobs/instance_meta.go +++ b/jobs/instance_meta.go @@ -2,7 +2,7 @@ package jobs import ( "delta/core" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "runtime" "syscall" ) diff --git a/jobs/log_event.go b/jobs/log_event.go index 6198e24..0da324f 100644 --- a/jobs/log_event.go +++ b/jobs/log_event.go @@ -1,29 +1,29 @@ package jobs -import ( - "delta/core" - "github.com/application-research/delta-db/messaging" -) - -// LogEventProcessor `LogEventProcessor` is a struct that contains a `LightNode` and a `LogEvent`. -// @property LightNode - The node that the event is being processed for. -// @property LogEvent - This is the event that we want to process. -type LogEventProcessor struct { - LightNode *core.DeltaNode - LogEvent messaging.LogEvent -} - -// NewLogEvent > This function creates a new LogEventProcessor object and returns it -func NewLogEvent(ln *core.DeltaNode, logEvent messaging.LogEvent) IProcessor { - return &LogEventProcessor{ - LightNode: ln, - LogEvent: logEvent, - } -} - -// Run Saving the log event to the database. -func (l LogEventProcessor) Run() error { - // save log event - l.LightNode.DB.Create(&l.LogEvent) - return nil -} +//import ( +// "delta/core" +// "github.com/application-research/delta-db/messaging" +//) +// +//// LogEventProcessor `LogEventProcessor` is a struct that contains a `LightNode` and a `LogEvent`. +//// @property LightNode - The node that the event is being processed for. +//// @property LogEvent - This is the event that we want to process. +//type LogEventProcessor struct { +// LightNode *core.DeltaNode +// LogEvent messaging.LogEvent +//} +// +//// NewLogEvent > This function creates a new LogEventProcessor object and returns it +//func NewLogEvent(ln *core.DeltaNode, logEvent messaging.LogEvent) IProcessor { +// return &LogEventProcessor{ +// LightNode: ln, +// LogEvent: logEvent, +// } +//} +// +//// Run Saving the log event to the database. +//func (l LogEventProcessor) Run() error { +// // save log event +// l.LightNode.DB.Create(&l.LogEvent) +// return nil +//} diff --git a/jobs/miner_check.go b/jobs/miner_check.go index 27df552..8e98f9c 100644 --- a/jobs/miner_check.go +++ b/jobs/miner_check.go @@ -4,7 +4,7 @@ import ( "delta/core" "encoding/json" "fmt" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "gorm.io/gorm" "net/http" ) diff --git a/jobs/piece_commp_compute.go b/jobs/piece_commp_compute.go index 906f68f..818e905 100644 --- a/jobs/piece_commp_compute.go +++ b/jobs/piece_commp_compute.go @@ -7,7 +7,7 @@ import ( "io" "time" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "github.com/application-research/filclient" "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" diff --git a/jobs/retry.go b/jobs/retry.go index 39661ad..6a2d50b 100644 --- a/jobs/retry.go +++ b/jobs/retry.go @@ -5,7 +5,7 @@ import ( "delta/core" "delta/utils" "fmt" - model "github.com/application-research/delta-db/db_models" + model "delta/models" "github.com/ipfs/go-cid" "time" ) diff --git a/jobs/storage_deal_maker.go b/jobs/storage_deal_maker.go index 57f86b8..6823265 100644 --- a/jobs/storage_deal_maker.go +++ b/jobs/storage_deal_maker.go @@ -7,7 +7,7 @@ import ( "encoding/base64" "encoding/json" "fmt" - model "github.com/application-research/delta-db/db_models" + model "delta/models" fc "github.com/application-research/filclient" smtypes "github.com/filecoin-project/boost/storagemarket/types" "github.com/filecoin-project/boost/transport/httptransport" diff --git a/models/base.go b/models/base.go new file mode 100644 index 0000000..a590c0c --- /dev/null +++ b/models/base.go @@ -0,0 +1,28 @@ +package db_models + +import ( + "io/ioutil" + "net/http" + "os" +) + +func GetPublicIP() (string, error) { + resp, err := http.Get("https://ifconfig.me") // important to get the public ip if possible. + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(body), nil +} + +func GetHostname() string { + hostname, err := os.Hostname() + if err != nil { + return "unknown" + } + return hostname +} diff --git a/models/batch_import_content.go b/models/batch_import_content.go new file mode 100644 index 0000000..ea99f44 --- /dev/null +++ b/models/batch_import_content.go @@ -0,0 +1,107 @@ +package db_models + +import ( + "time" +) + +// BatchImport create an entry first +type BatchImport struct { + ID int64 `gorm:"primaryKey"` + Uuid string `json:"uuid" gorm:"index:,option:CONCURRENTLY"` + Status string `json:"status"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +//func (u *BatchImport) AfterSave(tx *gorm.DB) (err error) { +// +// var instanceFromDb InstanceMeta +// // get the latest instance uuid +// tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) +// +// tx.Model(&InstanceMeta{}).Where("id > 0").First(&instanceFromDb) +// +// if instanceFromDb.ID == 0 { +// return +// } +// +// // get instance info +// ip, err := GetPublicIP() +// if err != nil { +// return +// } +// +// log := BatchImportLog{ +// Uuid: u.Uuid, +// Status: u.Status, +// NodeInfo: GetHostname(), +// RequesterInfo: ip, +// DeltaNodeUuid: instanceFromDb.InstanceUuid, +// SystemBatchImportId: u.ID, +// CreatedAt: time.Now(), +// UpdatedAt: time.Now(), +// } +// +// deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ +// ObjectType: "BatchImportLog", +// Object: log, +// } +// +// messageBytes, err := json.Marshal(deltaMetricsBaseMessage) +// if err != nil { +// return err +// } +// producer.Publish(messageBytes) +// return +//} + +// BatchContent associate the content to a batch +type BatchImportContent struct { + ID int64 `gorm:"primaryKey"` + BatchImportID int64 `json:"batch_import_id" gorm:"index:,option:CONCURRENTLY"` + ContentID int64 `json:"content_id" gorm:"index:,option:CONCURRENTLY"` // check status of the content + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +//func (u *BatchImportContent) AfterSave(tx *gorm.DB) (err error) { +// +// var instanceFromDb InstanceMeta +// // get the latest instance uuid +// tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) +// +// tx.Model(&InstanceMeta{}).Where("id > 0").First(&instanceFromDb) +// +// if instanceFromDb.ID == 0 { +// return +// } +// +// // get instance info +// ip, err := GetPublicIP() +// if err != nil { +// return +// } +// +// log := BatchImportContentLog{ +// BatchImportID: u.BatchImportID, +// ContentID: u.ContentID, +// NodeInfo: GetHostname(), +// RequesterInfo: ip, +// DeltaNodeUuid: instanceFromDb.InstanceUuid, +// SystemBatchContentId: u.ID, +// CreatedAt: time.Now(), +// UpdatedAt: time.Now(), +// } +// +// deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ +// ObjectType: "BatchImportContentLog", +// Object: log, +// } +// +// messageBytes, err := json.Marshal(deltaMetricsBaseMessage) +// if err != nil { +// return err +// } +// producer.Publish(messageBytes) +// return +//} diff --git a/models/content.go b/models/content.go new file mode 100644 index 0000000..91b87a6 --- /dev/null +++ b/models/content.go @@ -0,0 +1,75 @@ +package db_models + +import ( + "time" +) + +type Content struct { + ID int64 `gorm:"primaryKey"` + Name string `json:"name"` + Size int64 `json:"size"` + Cid string `json:"cid"` + RequestingApiKey string `json:"requesting_api_key,omitempty"` + PieceCommitmentId int64 `json:"piece_commitment_id,omitempty"` + Status string `json:"status"` + RequestType string `json:"request_type"` // default signed, or unsigned + ConnectionMode string `json:"connection_mode"` // offline or online + AutoRetry bool `json:"auto_retry"` + LastMessage string `json:"last_message"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +//func (u *Content) AfterSave(tx *gorm.DB) (err error) { +// +// var instanceFromDb InstanceMeta +// // get the latest instance info based on created_at +// tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) +// +// if instanceFromDb.ID == 0 { +// return +// } +// +// var contentFromDb Content +// tx.Model(&Content{}).Where("id = ?", u.ID).First(&contentFromDb) +// +// if contentFromDb.ID == 0 { +// return +// } +// // get instance info +// ip, err := GetPublicIP() +// if err != nil { +// return +// } +// +// log := ContentLog{ +// Name: u.Name, +// Size: u.Size, +// Cid: u.Cid, +// RequestingApiKey: u.RequestingApiKey, +// PieceCommitmentId: u.PieceCommitmentId, +// Status: u.Status, +// ConnectionMode: u.ConnectionMode, +// LastMessage: u.LastMessage, +// AutoRetry: u.AutoRetry, +// NodeInfo: GetHostname(), +// RequesterInfo: ip, +// DeltaNodeUuid: instanceFromDb.InstanceUuid, +// SystemContentId: u.ID, +// CreatedAt: time.Now(), +// UpdatedAt: time.Now(), +// } +// +// deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ +// ObjectType: "ContentLog", +// Object: log, +// } +// +// messageBytes, err := json.Marshal(deltaMetricsBaseMessage) +// if err != nil { +// return err +// } +// producer.Publish(messageBytes) +// +// return +//} diff --git a/models/content_deal.go b/models/content_deal.go new file mode 100644 index 0000000..ddda975 --- /dev/null +++ b/models/content_deal.go @@ -0,0 +1,92 @@ +package db_models + +import ( + "time" +) + +type ContentDeal struct { + ID int64 `gorm:"primaryKey"` + Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` + //Content Content `gorm:"references:ID"` + PropCid string `json:"propCid"` + DealUUID string `json:"dealUuid"` + Miner string `json:"miner"` + DealID int64 `json:"dealId"` + Failed bool `json:"failed"` + Verified bool `json:"verified"` + Slashed bool `json:"slashed"` + FailedAt time.Time `json:"failedAt,omitempty"` + DTChan string `json:"dtChan" gorm:"index"` + TransferStarted time.Time `json:"transferStarted"` + TransferFinished time.Time `json:"transferFinished"` + OnChainAt time.Time `json:"onChainAt"` + SealedAt time.Time `json:"sealedAt"` + LastMessage string `json:"lastMessage"` + DealProtocolVersion string `json:"deal_protocol_version"` + MinerVersion string `json:"miner_version,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +//func (u *ContentDeal) AfterSave(tx *gorm.DB) (err error) { +// +// var instanceFromDb InstanceMeta +// // get the latest instance uuid +// tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) +// +// tx.Model(&InstanceMeta{}).Where("id > 0").First(&instanceFromDb) +// +// if instanceFromDb.ID == 0 { +// return +// } +// +// var contentDealLog ContentDeal +// tx.Model(&ContentDeal{}).Where("id = ?", u.ID).First(&contentDealLog) +// +// if contentDealLog.ID == 0 { +// return +// } +// // get instance info +// ip, err := GetPublicIP() +// if err != nil { +// return +// } +// +// log := ContentDealLog{ +// Content: u.Content, +// PropCid: u.PropCid, +// DealUUID: u.DealUUID, +// Miner: u.Miner, +// DealID: u.DealID, +// Failed: u.Failed, +// Verified: u.Verified, +// Slashed: u.Slashed, +// FailedAt: u.FailedAt, +// DTChan: u.DTChan, +// TransferStarted: u.TransferStarted, +// TransferFinished: u.TransferFinished, +// OnChainAt: u.OnChainAt, +// SealedAt: u.SealedAt, +// LastMessage: u.LastMessage, +// DealProtocolVersion: u.DealProtocolVersion, +// MinerVersion: u.MinerVersion, +// DeltaNodeUuid: instanceFromDb.InstanceUuid, +// NodeInfo: GetHostname(), +// RequesterInfo: ip, +// SystemContentDealId: u.ID, +// CreatedAt: time.Now(), +// UpdatedAt: time.Now(), +// } +// +// deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ +// ObjectType: "ContentDealLog", +// Object: log, +// } +// +// messageBytes, err := json.Marshal(deltaMetricsBaseMessage) +// if err != nil { +// return err +// } +// producer.Publish(messageBytes) +// return +//} diff --git a/models/content_deal_proposal.go b/models/content_deal_proposal.go new file mode 100644 index 0000000..6667ffb --- /dev/null +++ b/models/content_deal_proposal.go @@ -0,0 +1,61 @@ +package db_models + +import ( + "time" +) + +type ContentDealProposal struct { + ID int64 `gorm:"primaryKey"` + Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` + Unsigned string `json:"unsigned"` + Signed string `json:"signed"` + Meta string `json:"meta"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +//func (u *ContentDealProposal) AfterSave(tx *gorm.DB) (err error) { +// +// var instanceFromDb InstanceMeta +// tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) +// +// if instanceFromDb.ID == 0 { +// return +// } +// +// var contentDealProposal ContentDealProposal +// tx.Model(&ContentDealProposal{}).Where("id = ?", u.ID).First(&contentDealProposal) +// +// if contentDealProposal.ID == 0 { +// return +// } +// +// // get instance info +// ip, err := GetPublicIP() +// if err != nil { +// return +// } +// +// log := ContentDealProposalLog{ +// Content: u.Content, +// Unsigned: u.Unsigned, +// Signed: u.Signed, +// Meta: u.Meta, +// NodeInfo: GetHostname(), +// RequesterInfo: ip, +// DeltaNodeUuid: instanceFromDb.InstanceUuid, +// SystemContentDealProposalId: u.ID, +// CreatedAt: time.Now(), +// UpdatedAt: time.Now(), +// } +// +// deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ +// ObjectType: "ContentDealProposalLog", +// Object: log, +// } +// +// messageBytes, err := json.Marshal(deltaMetricsBaseMessage) +// producer.Publish(messageBytes) +// +// return +//} diff --git a/models/content_deal_proposal_parameters.go b/models/content_deal_proposal_parameters.go new file mode 100644 index 0000000..2fead3d --- /dev/null +++ b/models/content_deal_proposal_parameters.go @@ -0,0 +1,72 @@ +package db_models + +import ( + "time" +) + +type ContentDealProposalParameters struct { + ID int64 `gorm:"primaryKey"` + Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` + Label string `json:"label,omitempty"` + Duration int64 `json:"duration,omitempty"` + StartEpoch int64 `json:"start_epoch,omitempty"` + EndEpoch int64 `json:"end_epoch,omitempty"` + TransferParams string `json:"transfer_params,omitempty"` + RemoveUnsealedCopy bool `json:"remove_unsealed_copy"` + SkipIPNIAnnounce bool `json:"skip_ipni_announce"` + VerifiedDeal bool `json:"verified_deal"` + UnverifiedDealMaxPrice string `json:"unverified_deal_max_price"` + CreatedAt time.Time `json:"created_at" json:"created-at"` + UpdatedAt time.Time `json:"updated_at" json:"updated-at"` +} + +//func (u *ContentDealProposalParameters) AfterSave(tx *gorm.DB) (err error) { +// +// var instanceFromDb InstanceMeta +// tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) +// +// if instanceFromDb.ID == 0 { +// return +// } +// +// var contentDealProposalParams ContentDealProposalParameters +// tx.Model(&ContentDealProposalParameters{}).Where("id = ?", u.ID).First(&contentDealProposalParams) +// +// if contentDealProposalParams.ID == 0 { +// return +// } +// +// // get instance info +// ip, err := GetPublicIP() +// if err != nil { +// return +// } +// log := ContentDealProposalParametersLog{ +// Content: u.Content, +// Label: u.Label, +// Duration: u.Duration, +// StartEpoch: u.StartEpoch, +// EndEpoch: u.EndEpoch, +// TransferParams: u.TransferParams, +// RemoveUnsealedCopy: u.RemoveUnsealedCopy, +// SkipIPNIAnnounce: u.SkipIPNIAnnounce, +// VerifiedDeal: u.VerifiedDeal, +// UnverifiedDealMaxPrice: u.UnverifiedDealMaxPrice, +// NodeInfo: GetHostname(), +// RequesterInfo: ip, +// DeltaNodeUuid: instanceFromDb.InstanceUuid, +// SystemContentDealProposalParametersId: u.ID, +// CreatedAt: time.Now(), +// UpdatedAt: time.Now(), +// } +// +// deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ +// ObjectType: "ContentDealProposalParametersLog", +// Object: log, +// } +// +// messageBytes, err := json.Marshal(deltaMetricsBaseMessage) +// producer.Publish(messageBytes) +// +// return +//} diff --git a/models/content_miner.go b/models/content_miner.go new file mode 100644 index 0000000..60a45bb --- /dev/null +++ b/models/content_miner.go @@ -0,0 +1,57 @@ +package db_models + +import ( + "time" +) + +type ContentMiner struct { + ID int64 `gorm:"primaryKey"` + Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` + Miner string `json:"miner"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +//func (u *ContentMiner) AfterSave(tx *gorm.DB) (err error) { +// +// var instanceFromDb InstanceMeta +// tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) +// +// if instanceFromDb.ID == 0 { +// return +// } +// +// var contentMiner ContentMiner +// tx.Model(&ContentMiner{}).Where("id = ?", u.ID).First(&contentMiner) +// +// if contentMiner.ID == 0 { +// return +// } +// +// // get instance info +// ip, err := GetPublicIP() +// if err != nil { +// return +// } +// +// log := ContentMinerLog{ +// Content: u.Content, +// Miner: u.Miner, +// NodeInfo: GetHostname(), +// RequesterInfo: ip, +// SystemContentMinerId: u.ID, +// DeltaNodeUuid: instanceFromDb.InstanceUuid, +// CreatedAt: time.Now(), +// UpdatedAt: time.Now(), +// } +// +// deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ +// ObjectType: "ContentMinerLog", +// Object: log, +// } +// +// messageBytes, err := json.Marshal(deltaMetricsBaseMessage) +// producer.Publish(messageBytes) +// +// return +//} diff --git a/models/content_wallet.go b/models/content_wallet.go new file mode 100644 index 0000000..7d12249 --- /dev/null +++ b/models/content_wallet.go @@ -0,0 +1,56 @@ +package db_models + +import ( + "time" +) + +type ContentWallet struct { + ID int64 `gorm:"primaryKey"` + Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` + WalletId int64 `json:"wallet_id" gorm:"index:,option:CONCURRENTLY"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +//func (u *ContentWallet) AfterSave(tx *gorm.DB) (err error) { +// +// var instanceFromDb InstanceMeta +// tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) +// +// if instanceFromDb.ID == 0 { +// return +// } +// +// var contentWallet ContentWallet +// tx.Model(&ContentWallet{}).Where("id = ?", u.ID).First(&contentWallet) +// +// if contentWallet.ID == 0 { +// return +// } +// +// // get instance info +// ip, err := GetPublicIP() +// if err != nil { +// return +// } +// log := ContentWalletLog{ +// Content: u.Content, +// WalletId: u.WalletId, +// NodeInfo: GetHostname(), +// RequesterInfo: ip, +// SystemContentWalletId: u.ID, +// DeltaNodeUuid: instanceFromDb.InstanceUuid, +// CreatedAt: time.Now(), +// UpdatedAt: time.Now(), +// } +// +// deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ +// ObjectType: "ContentWalletLog", +// Object: log, +// } +// +// messageBytes, err := json.Marshal(deltaMetricsBaseMessage) +// producer.Publish(messageBytes) +// +// return +//} diff --git a/models/database.go b/models/database.go new file mode 100644 index 0000000..7d38382 --- /dev/null +++ b/models/database.go @@ -0,0 +1,102 @@ +package db_models + +import ( + "fmt" + "github.com/application-research/delta-db/messaging" + "gorm.io/gorm/logger" + "time" + + "gorm.io/driver/postgres" + "gorm.io/driver/sqlite" + "gorm.io/gorm" +) + +var producer *messaging.DeltaMetricsMessageProducer + +type DeltaMetricsBaseMessage struct { + ObjectType string `json:"object_type"` + Object interface{} `json:"object"` +} + +func init() { + producer = messaging.NewDeltaMetricsMessageProducer() +} + +func OpenDatabase(dbDsn string) (*gorm.DB, error) { + // use postgres + var DB *gorm.DB + var err error + + if dbDsn[:8] == "postgres" { + DB, err = gorm.Open(postgres.Open(dbDsn), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + } else { + DB, err = gorm.Open(sqlite.Open(dbDsn), &gorm.Config{ + Logger: logger.Default.LogMode(logger.Silent), + }) + } + + // generate new models. + ConfigureModels(DB) // create models. + + if err != nil { + return nil, err + } + return DB, nil +} + +func ConfigureModels(db *gorm.DB) { + db.AutoMigrate(&Content{}, &ContentDeal{}, &PieceCommitment{}, &MinerInfo{}, &MinerPrice{}, &messaging.LogEvent{}, &ContentMiner{}, &ProcessContentCounter{}, &ContentWallet{}, &ContentDealProposalParameters{}, &Wallet{}, &ContentDealProposal{}, &InstanceMeta{}, &RetryDealCount{}, &BatchImport{}, &BatchImportContent{}) +} + +type ProcessContentCounter struct { + ID int64 `gorm:"primaryKey"` + Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` + Counter int64 `json:"counter"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type MinerInfo struct { + ID int64 `gorm:"primaryKey"` + Addr string `json:"addr"` // same as Miner from MinerPrice + Name string `json:"name"` + Suspended bool `json:"suspended"` + Version string `json:"version"` + ChainInfo string `json:"chain_info"` + SuspendedReason string `json:"suspendedReason,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type MinerPrice struct { + ID int64 `gorm:"primaryKey"` + Miner string `json:"miner"` + Price string `json:"price"` + VerifiedPrice string `json:"verifiedPrice"` + MinPieceSize int64 `json:"minPieceSize"` + MaxPieceSize int64 `json:"maxPieceSize"` + MinerVersion string `json:"miner_version"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type AdminUser struct { + ID int64 `gorm:"primaryKey"` + Username string `json:"username"` + Password string `json:"password"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type RetryDealCount struct { + ID int64 `gorm:"primaryKey"` + Type string `json:"type"` + OldId int64 `json:"old_id"` + NewId int64 `json:"new_id"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +var ErrNoChannelID = fmt.Errorf("no data transfer channel id in deal") diff --git a/models/instance_meta.go b/models/instance_meta.go new file mode 100644 index 0000000..a185c3f --- /dev/null +++ b/models/instance_meta.go @@ -0,0 +1,91 @@ +package db_models + +import ( + "time" +) + +type InstanceMeta struct { + // gorm id + ID int64 `gorm:"primary_key" json:"id"` + InstanceUuid string `json:"instance_uuid"` + InstanceHostName string `json:"instance_host_name"` + InstanceNodeName string `json:"instance_node_name"` + OSDetails string `json:"os_details"` + PublicIp string `json:"public_ip"` + MemoryLimit uint64 `json:"memory_limit"` + CpuLimit uint64 `json:"cpu_limit"` + StorageLimit uint64 `json:"storage_limit"` + DisableRequest bool `json:"disable_requests"` + DisableCommitmentPieceGeneration bool `json:"disable_commitment_piece_generation"` + DisableStorageDeal bool `json:"disable_storage_deal"` + DisableOnlineDeals bool `json:"disable_online_deals"` + DisableOfflineDeals bool `json:"disable_offline_deals"` + NumberOfCpus uint64 `json:"number_of_cpus"` + StorageInBytes uint64 `json:"storage_in_bytes"` + SystemMemory uint64 `json:"system_memory"` + HeapMemory uint64 `json:"heap_memory"` + HeapInUse uint64 `json:"heap_in_use"` + StackInUse uint64 `json:"stack_in_use"` + InstanceStart time.Time `json:"instance_start"` + BytesPerCpu uint64 `json:"bytes_per_cpu"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +//func (u *InstanceMeta) AfterSave(tx *gorm.DB) (err error) { +// +// var contentFromDb Content +// tx.Model(&Content{}).Where("id = ?", u.ID).First(&contentFromDb) +// +// if contentFromDb.ID == 0 { +// return +// } +// // get instance info +// ip, err := GetPublicIP() +// if err != nil { +// return +// } +// +// log := InstanceMetaLog{ +// InstanceUuid: u.InstanceUuid, +// InstanceHostName: u.InstanceHostName, +// InstanceNodeName: u.InstanceNodeName, +// OSDetails: u.OSDetails, +// PublicIp: u.PublicIp, +// MemoryLimit: u.MemoryLimit, +// CpuLimit: u.CpuLimit, +// StorageLimit: u.StorageLimit, +// DisableRequest: u.DisableRequest, +// DisableCommitmentPieceGeneration: u.DisableCommitmentPieceGeneration, +// DisableStorageDeal: u.DisableStorageDeal, +// DisableOnlineDeals: u.DisableOnlineDeals, +// DisableOfflineDeals: u.DisableOfflineDeals, +// NumberOfCpus: u.NumberOfCpus, +// StorageInBytes: u.StorageInBytes, +// SystemMemory: u.SystemMemory, +// HeapMemory: u.HeapMemory, +// HeapInUse: u.HeapInUse, +// StackInUse: u.StackInUse, +// InstanceStart: u.InstanceStart, +// BytesPerCpu: u.BytesPerCpu, +// NodeInfo: GetHostname(), +// RequesterInfo: ip, +// DeltaNodeUuid: u.InstanceUuid, +// SystemInstanceMetaId: u.ID, +// CreatedAt: time.Now(), +// UpdatedAt: time.Now(), +// } +// +// deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ +// ObjectType: "InstanceMetaLog", +// Object: log, +// } +// +// messageBytes, err := json.Marshal(deltaMetricsBaseMessage) +// if err != nil { +// return err +// } +// producer.Publish(messageBytes) +// +// return +//} diff --git a/models/log_event.go b/models/log_event.go new file mode 100644 index 0000000..297ebb4 --- /dev/null +++ b/models/log_event.go @@ -0,0 +1 @@ +package db_models diff --git a/models/log_models.go b/models/log_models.go new file mode 100644 index 0000000..4308f73 --- /dev/null +++ b/models/log_models.go @@ -0,0 +1,323 @@ +package db_models + +// +//// DeltaStartupLogs action events +//type DeltaStartupLogs struct { +// ID int64 `gorm:"primaryKey"` // auto increment +// NodeInfo string `json:"node_info"` +// OSDetails string `json:"os_details"` +// IPAddress string `json:"ip_address"` +// DeltaNodeUuid string `json:"delta_node_uuid"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type DealEndpointRequestLog struct { +// ID int64 `gorm:"primaryKey"` // auto increment +// NodeInfo string `json:"node_info"` +// Information string `json:"information"` +//} +// +//type DealContentRequestLog struct { +// ID int64 `gorm:"primaryKey"` // auto increment +// NodeInfo string `json:"node_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// EventMessage string `json:"event_message"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type DealPieceCommitmentRequestLog struct { +// ID int64 `gorm:"primaryKey"` // auto increment +// NodeInfo string `json:"node_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// EventMessage string `json:"event_message"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type ContentPrepareLog struct { +// ID int64 `gorm:"primaryKey"` // auto increment +// NodeInfo string `json:"node_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// EventMessage string `json:"event_message"` +// ContentDealProposal ContentDealProposal `json:"content_deal_proposal"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +//type ContentAnnounceLog struct { +// ID int64 `gorm:"primaryKey"` // auto increment +// NodeInfo string `json:"node_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// EventMessage string `json:"event_message"` +// ContentDealProposal ContentDealProposal `json:"content_deal_proposal"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type OpenStatsLog struct { +// ID int64 `gorm:"primaryKey"` // auto increment +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type RepairRequestLog struct { +// ID int64 `gorm:"primaryKey"` // auto increment +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// EventMessage string `json:"event_message"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type NodeRequestLog struct { +// ID int64 `gorm:"primaryKey"` // auto increment +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//// job events +//type PieceCommitmentJobLog struct { +// ID int64 `gorm:"primaryKey"` // auto increment +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type StorageDealMakeJobLog struct { +// ID int64 `gorm:"primaryKey"` // auto increment +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type DataTransferStatusJobLog struct { +// ID int64 `gorm:"primaryKey"` // auto increment +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type InstanceMetaJobLog struct { +// ID int64 +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// DeltaNodeUuid string `json:"delta_node_uuid"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//// ContentLog time series log events +//type ContentLog struct { +// ID int64 `gorm:"primaryKey"` // auto increment +// Name string `json:"name"` +// Size int64 `json:"size"` +// Cid string `json:"cid"` +// RequestingApiKey string `json:"requesting_api_key,omitempty"` +// PieceCommitmentId int64 `json:"piece_commitment_id,omitempty"` +// Status string `json:"status"` +// ConnectionMode string `json:"connection_mode"` // offline or online +// AutoRetry bool `json:"auto_retry"` +// LastMessage string `json:"last_message"` +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// DeltaNodeUuid string `json:"delta_node_uuid"` +// SystemContentId int64 `json:"system_content_id"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//// ContentDealLog time series content deal events +//type ContentDealLog struct { +// ID int64 `gorm:"primaryKey"` +// Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` +// PropCid string `json:"propCid"` +// DealUUID string `json:"dealUuid"` +// Miner string `json:"miner"` +// DealID int64 `json:"dealId"` +// Failed bool `json:"failed"` +// Verified bool `json:"verified"` +// Slashed bool `json:"slashed"` +// FailedAt time.Time `json:"failedAt,omitempty"` +// DTChan string `json:"dtChan" gorm:"index"` +// TransferStarted time.Time `json:"transferStarted"` +// TransferFinished time.Time `json:"transferFinished"` +// OnChainAt time.Time `json:"onChainAt"` +// SealedAt time.Time `json:"sealedAt"` +// LastMessage string `json:"lastMessage"` +// DealProtocolVersion string `json:"deal_protocol_version"` +// MinerVersion string `json:"miner_version,omitempty"` +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// DeltaNodeUuid string `json:"delta_node_uuid"` +// SystemContentDealId int64 `json:"system_content_deal_id"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type ContentMinerLog struct { +// ID int64 `gorm:"primaryKey"` +// Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` +// Miner string `json:"miner"` +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// DeltaNodeUuid string `json:"delta_node_uuid"` +// SystemContentMinerId int64 `json:"system_content_miner_id"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type ContentWalletLog struct { +// ID int64 `gorm:"primaryKey"` +// Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` +// WalletId int64 `json:"wallet_id" gorm:"index:,option:CONCURRENTLY"` +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// DeltaNodeUuid string `json:"delta_node_uuid"` +// SystemContentWalletId int64 `json:"system_content_miner_id"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type PieceCommitmentLog struct { +// ID int64 `gorm:"primaryKey"` +// Cid string `json:"cid"` +// Piece string `json:"piece"` +// Size int64 `json:"size"` +// PaddedPieceSize uint64 `json:"padded_piece_size"` +// UnPaddedPieceSize uint64 `json:"unnpadded_piece_size"` +// Status string `json:"status"` // open, in-progress, completed (closed). +// LastMessage string `json:"last_message"` +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// DeltaNodeUuid string `json:"delta_node_uuid"` +// SystemContentPieceCommitmentId int64 `json:"system_content_piece_commitment_id"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type ContentDealProposalLog struct { +// ID int64 `gorm:"primaryKey"` +// Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` +// Unsigned string `json:"unsigned"` +// Signed string `json:"signed"` +// Meta string `json:"meta"` +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// DeltaNodeUuid string `json:"delta_node_uuid"` +// SystemContentDealProposalId int64 `json:"system_content_deal_proposal_id"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type ContentDealProposalParametersLog struct { +// ID int64 `gorm:"primaryKey"` +// Content int64 `json:"content" gorm:"index:,option:CONCURRENTLY"` +// Label string `json:"label,omitempty"` +// Duration int64 `json:"duration,omitempty"` +// StartEpoch int64 `json:"start_epoch,omitempty"` +// EndEpoch int64 `json:"end_epoch,omitempty"` +// TransferParams string `json:"transfer_params,omitempty"` +// RemoveUnsealedCopy bool `json:"remove_unsealed_copy"` +// SkipIPNIAnnounce bool `json:"skip_ipni_announce"` +// VerifiedDeal bool `json:"verified_deal"` +// UnverifiedDealMaxPrice string `json:"unverified_deal_max_price,omitempty"` +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// DeltaNodeUuid string `json:"delta_node_uuid"` +// SystemContentDealProposalParametersId int64 `json:"system_content_deal_proposal_id"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type WalletLog struct { +// ID int64 `gorm:"primaryKey"` +// UuId string `json:"uuid"` +// Addr string `json:"addr"` +// Owner string `json:"owner"` +// KeyType string `json:"key_type"` +// PrivateKey string `json:"private_key"` +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// RequestingApiKey string `json:"requesting_api_key"` +// DeltaNodeUuid string `json:"delta_node_uuid"` +// SystemWalletId int64 `json:"system_content_deal_proposal_id"` +// CreatedAt time.Time `json:"created_at"` // auto set +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type InstanceMetaLog struct { +// // gorm id +// ID int64 `gorm:"primary_key" json:"id"` +// InstanceUuid string `json:"instance_uuid"` +// InstanceHostName string `json:"instance_host_name"` +// InstanceNodeName string `json:"instance_node_name"` +// OSDetails string `json:"os_details"` +// PublicIp string `json:"public_ip"` +// MemoryLimit uint64 `json:"memory_limit"` +// CpuLimit uint64 `json:"cpu_limit"` +// StorageLimit uint64 `json:"storage_limit"` +// DisableRequest bool `json:"disable_requests"` +// DisableCommitmentPieceGeneration bool `json:"disable_commitment_piece_generation"` +// DisableStorageDeal bool `json:"disable_storage_deal"` +// DisableOnlineDeals bool `json:"disable_online_deals"` +// DisableOfflineDeals bool `json:"disable_offline_deals"` +// NumberOfCpus uint64 `json:"number_of_cpus"` +// StorageInBytes uint64 `json:"storage_in_bytes"` +// SystemMemory uint64 `json:"system_memory"` +// HeapMemory uint64 `json:"heap_memory"` +// HeapInUse uint64 `json:"heap_in_use"` +// StackInUse uint64 `json:"stack_in_use"` +// InstanceStart time.Time `json:"instance_start"` +// BytesPerCpu uint64 `json:"bytes_per_cpu"` +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// DeltaNodeUuid string `json:"delta_node_uuid"` +// SystemInstanceMetaId int64 `json:"system_instance_meta_id"` +// CreatedAt time.Time `json:"created_at"` +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//type BatchImportLog struct { +// ID int64 `gorm:"primaryKey"` +// Uuid string `json:"uuid" gorm:"index:,option:CONCURRENTLY"` +// Status string `json:"status"` +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// DeltaNodeUuid string `json:"delta_node_uuid"` +// SystemBatchImportId int64 `json:"system_batch_import_id"` +// CreatedAt time.Time `json:"created_at"` +// UpdatedAt time.Time `json:"updated_at"` +//} +// +//// BatchContent associate the content to a batch +//type BatchImportContentLog struct { +// ID int64 `gorm:"primaryKey"` +// BatchImportID int64 `json:"batch_import_id" gorm:"index:,option:CONCURRENTLY"` +// ContentID int64 `json:"content_id" gorm:"index:,option:CONCURRENTLY"` // check status of the content +// NodeInfo string `json:"node_info"` +// RequesterInfo string `json:"requester_info"` +// DeltaNodeUuid string `json:"delta_node_uuid"` +// SystemBatchContentId int64 `json:"system_batch_content_id"` +// CreatedAt time.Time `json:"created_at"` +// UpdatedAt time.Time `json:"updated_at"` +//} diff --git a/models/piece_commitment.go b/models/piece_commitment.go new file mode 100644 index 0000000..24c9c41 --- /dev/null +++ b/models/piece_commitment.go @@ -0,0 +1,66 @@ +package db_models + +import ( + "time" +) + +type PieceCommitment struct { + ID int64 `gorm:"primaryKey"` + Cid string `json:"cid"` + Piece string `json:"piece"` + Size int64 `json:"size"` + PaddedPieceSize uint64 `json:"padded_piece_size"` + UnPaddedPieceSize uint64 `json:"unnpadded_piece_size"` + Status string `json:"status"` // open, in-progress, completed (closed). + LastMessage string `json:"last_message"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +//func (u *PieceCommitment) AfterSave(tx *gorm.DB) (err error) { +// +// var instanceFromDb InstanceMeta +// tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) +// +// if instanceFromDb.ID == 0 { +// return +// } +// +// var pieceComm PieceCommitment +// tx.Model(&PieceCommitment{}).Where("id = ?", u.ID).First(&pieceComm) +// +// if pieceComm.ID == 0 { +// return +// } +// +// // get instance info +// ip, err := GetPublicIP() +// if err != nil { +// return +// } +// log := PieceCommitmentLog{ +// Cid: u.Cid, +// Piece: u.Piece, +// Size: u.Size, +// PaddedPieceSize: u.PaddedPieceSize, +// UnPaddedPieceSize: u.UnPaddedPieceSize, +// Status: u.Status, +// LastMessage: u.LastMessage, +// NodeInfo: GetHostname(), +// RequesterInfo: ip, +// SystemContentPieceCommitmentId: u.ID, +// DeltaNodeUuid: instanceFromDb.InstanceUuid, +// CreatedAt: time.Now(), +// UpdatedAt: time.Now(), +// } +// +// deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ +// ObjectType: "PieceCommitmentLog", +// Object: log, +// } +// +// messageBytes, err := json.Marshal(deltaMetricsBaseMessage) +// producer.Publish(messageBytes) +// +// return +//} diff --git a/models/repair_request.go b/models/repair_request.go new file mode 100644 index 0000000..e4a8014 --- /dev/null +++ b/models/repair_request.go @@ -0,0 +1,13 @@ +package db_models + +import ( + "time" +) + +type RepairRequest struct { + ID int64 `gorm:"primaryKey"` + ObjectId int64 `json:"object_id"` + Type string `json:"type"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} diff --git a/models/wallet.go b/models/wallet.go new file mode 100644 index 0000000..31748a3 --- /dev/null +++ b/models/wallet.go @@ -0,0 +1,66 @@ +package db_models + +import ( + "time" +) + +// Wallet time series log events +type Wallet struct { + ID int64 `gorm:"primaryKey"` + UuId string `json:"uuid"` + Addr string `json:"addr"` + Owner string `json:"owner"` + KeyType string `json:"key_type"` + PrivateKey string `json:"private_key"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +//func (7u *Wallet) AfterSave(tx *gorm.DB) (err error) { +// +// var instanceFromDb InstanceMeta +// tx.Raw("SELECT * FROM instance_meta ORDER BY id DESC LIMIT 1").Scan(&instanceFromDb) +// +// if instanceFromDb.ID == 0 { +// return +// } +// +// var walletFromDb Wallet +// tx.Model(&Wallet{}).Where("id = ?", u.ID).First(&walletFromDb) +// +// if walletFromDb.ID == 0 { +// return +// } +// // get instance info +// ip, err := GetPublicIP() +// if err != nil { +// return +// } +// +// log := WalletLog{ +// UuId: u.UuId, +// Addr: u.Addr, +// Owner: u.Owner, +// KeyType: "REDACTED", +// PrivateKey: "REDACTED", +// NodeInfo: GetHostname(), +// RequesterInfo: ip, +// SystemWalletId: u.ID, +// DeltaNodeUuid: instanceFromDb.InstanceUuid, +// CreatedAt: time.Now(), +// UpdatedAt: time.Now(), +// } +// +// deltaMetricsBaseMessage := DeltaMetricsBaseMessage{ +// ObjectType: "WalletLog", +// Object: log, +// } +// +// messageBytes, err := json.Marshal(deltaMetricsBaseMessage) +// if err != nil { +// return err +// } +// producer.Publish(messageBytes) +// +// return +//} diff --git a/utils/constants.go b/utils/constants.go index 315d4ad..37965d0 100644 --- a/utils/constants.go +++ b/utils/constants.go @@ -1,8 +1,6 @@ // A package that is used to define the constants used in the project. package utils -import "github.com/application-research/delta-db/messaging" - var Reset = "\033[0m" var Red = "\033[31m" var Green = "\033[32m" @@ -13,10 +11,6 @@ var Cyan = "\033[36m" var Gray = "\033[37m" var White = "\033[97m" -// GlobalDeltaDataReporter the global metrics tracer -// helps us improve the product. -var GlobalDeltaDataReporter = messaging.NewDeltaMetricsTracer() - const ( DELTA_LABEL string = "seal-the-delta-deal" CONTENT_PINNED string = "pinned" diff --git a/utils/model_utils.go b/utils/model_utils.go index 2259f99..eea93d2 100644 --- a/utils/model_utils.go +++ b/utils/model_utils.go @@ -2,7 +2,7 @@ package utils import ( "fmt" - "github.com/application-research/delta-db/db_models" + "delta/models" "github.com/application-research/filclient" datatransfer "github.com/filecoin-project/go-data-transfer" )