-
Notifications
You must be signed in to change notification settings - Fork 13
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
13 changed files
with
682 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| // SPDX-FileCopyrightText: 2024 PK Lab AG <contact@pklab.io> | ||
| // SPDX-License-Identifier: MIT | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "os" | ||
| "strconv" | ||
|
|
||
| "github.com/celenium-io/celestia-indexer/pkg/indexer/config" | ||
| goLibConfig "github.com/dipdup-net/go-lib/config" | ||
| "github.com/rs/zerolog" | ||
| "github.com/rs/zerolog/log" | ||
| ) | ||
|
|
||
| func init() { | ||
| log.Logger = log.Output(zerolog.ConsoleWriter{ | ||
| Out: os.Stdout, | ||
| TimeFormat: "2006-01-02 15:04:05", | ||
| }) | ||
| } | ||
|
|
||
| func initConfig() (*config.Config, error) { | ||
| configPath := rootCmd.PersistentFlags().StringP("config", "c", "dipdup.yml", "path to YAML config file") | ||
| if err := rootCmd.Execute(); err != nil { | ||
| log.Panic().Err(err).Msg("command line execute") | ||
| return nil, err | ||
| } | ||
|
|
||
| if err := rootCmd.MarkFlagRequired("config"); err != nil { | ||
| log.Panic().Err(err).Msg("config command line arg is required") | ||
| return nil, err | ||
| } | ||
|
|
||
| var cfg config.Config | ||
| if err := goLibConfig.Parse(*configPath, &cfg); err != nil { | ||
| log.Panic().Err(err).Msg("parsing config file") | ||
| return nil, err | ||
| } | ||
|
|
||
| if cfg.LogLevel == "" { | ||
| cfg.LogLevel = zerolog.LevelInfoValue | ||
| } | ||
|
|
||
| return &cfg, nil | ||
| } | ||
|
|
||
| func initLogger(level string) error { | ||
| logLevel, err := zerolog.ParseLevel(level) | ||
| if err != nil { | ||
| log.Panic().Err(err).Msg("parsing log level") | ||
| return err | ||
| } | ||
| zerolog.SetGlobalLevel(logLevel) | ||
| zerolog.CallerMarshalFunc = func(pc uintptr, file string, line int) string { | ||
| short := file | ||
| for i := len(file) - 1; i > 0; i-- { | ||
| if file[i] == '/' { | ||
| short = file[i+1:] | ||
| break | ||
| } | ||
| } | ||
| file = short | ||
| return file + ":" + strconv.Itoa(line) | ||
| } | ||
| log.Logger = log.Logger.With().Caller().Logger() | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| // SPDX-FileCopyrightText: 2024 PK Lab AG <contact@pklab.io> | ||
| // SPDX-License-Identifier: MIT | ||
|
|
||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| indexer "github.com/celenium-io/celestia-indexer/pkg/tvl" | ||
| "github.com/dipdup-net/indexer-sdk/pkg/modules/stopper" | ||
|
|
||
| "github.com/rs/zerolog/log" | ||
| "github.com/spf13/cobra" | ||
| "os" | ||
| "os/signal" | ||
| "syscall" | ||
| ) | ||
|
|
||
| var rootCmd = &cobra.Command{ | ||
| Use: "indexer", | ||
| Short: "DipDup Verticals | Celenium TVL Scanner", | ||
| } | ||
|
|
||
| func main() { | ||
| cfg, err := initConfig() | ||
| if err != nil { | ||
| return | ||
| } | ||
|
|
||
| if err = initLogger(cfg.LogLevel); err != nil { | ||
| return | ||
| } | ||
|
|
||
| ctx, cancel := context.WithCancel(context.Background()) | ||
|
|
||
| notifyCtx, notifyCancel := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM, syscall.SIGINT) | ||
| defer notifyCancel() | ||
|
|
||
| stopperModule := stopper.NewModule(cancel) | ||
| indexerModule, err := indexer.New(ctx, *cfg, &stopperModule) | ||
| if err != nil { | ||
| log.Panic().Err(err).Msg("error during indexer module creation") | ||
| return | ||
| } | ||
|
|
||
| stopperModule.Start(ctx) | ||
| indexerModule.Start(ctx) | ||
|
|
||
| <-notifyCtx.Done() | ||
| cancel() | ||
|
|
||
| log.Info().Msg("stopped") | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,103 @@ | ||
| // SPDX-FileCopyrightText: 2024 PK Lab AG <contact@pklab.io> | ||
| // SPDX-License-Identifier: MIT | ||
|
|
||
| package l2beat | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "github.com/dipdup-net/go-lib/config" | ||
| "github.com/pkg/errors" | ||
| "github.com/rs/zerolog" | ||
| "github.com/rs/zerolog/log" | ||
| "golang.org/x/time/rate" | ||
| "io" | ||
| "net/http" | ||
| "net/url" | ||
| "time" | ||
| ) | ||
|
|
||
| type API struct { | ||
| client *http.Client | ||
| cfg config.DataSource | ||
| rateLimit *rate.Limiter | ||
| log zerolog.Logger | ||
| } | ||
|
|
||
| func NewAPI(cfg config.DataSource) API { | ||
| rps := cfg.RequestsPerSecond | ||
| if cfg.RequestsPerSecond < 1 || cfg.RequestsPerSecond > 100 { | ||
| rps = 10 | ||
| } | ||
|
|
||
| t := http.DefaultTransport.(*http.Transport).Clone() | ||
| t.MaxIdleConns = rps | ||
| t.MaxConnsPerHost = rps | ||
| t.MaxIdleConnsPerHost = rps | ||
|
|
||
| return API{ | ||
| client: &http.Client{ | ||
| Transport: t, | ||
| }, | ||
| cfg: cfg, | ||
| rateLimit: rate.NewLimiter(rate.Every(time.Second/time.Duration(rps)), rps), | ||
| log: log.With().Str("module", "L2Beat api").Logger(), | ||
| } | ||
| } | ||
|
|
||
| func (api *API) get(ctx context.Context, path string, args map[string]string, output any) error { | ||
| u, err := url.Parse(api.cfg.URL) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| u.Path, err = url.JoinPath(u.Path, path) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| values := u.Query() | ||
| for key, value := range args { | ||
| values.Add(key, value) | ||
| } | ||
| u.RawQuery = values.Encode() | ||
|
|
||
| if api.rateLimit != nil { | ||
| if err := api.rateLimit.Wait(ctx); err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| start := time.Now() | ||
|
|
||
| req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| response, err := api.client.Do(req) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer closeWithLogError(response.Body, api.log) | ||
|
|
||
| api.log.Trace(). | ||
| Int64("ms", time.Since(start).Milliseconds()). | ||
| Str("url", u.String()). | ||
| Msg("request") | ||
|
|
||
| if response.StatusCode != http.StatusOK { | ||
| return errors.Errorf("invalid status: %d", response.StatusCode) | ||
| } | ||
|
|
||
| err = json.NewDecoder(response.Body).Decode(output) | ||
| return err | ||
| } | ||
|
|
||
| func closeWithLogError(stream io.ReadCloser, log zerolog.Logger) { | ||
| if _, err := io.Copy(io.Discard, stream); err != nil { | ||
| log.Err(err).Msg("L2Beat api copy GET body response to discard") | ||
| } | ||
| if err := stream.Close(); err != nil { | ||
| log.Err(err).Msg("L2Beat api close GET body request") | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| // SPDX-FileCopyrightText: 2024 PK Lab AG <contact@pklab.io> | ||
| // SPDX-License-Identifier: MIT | ||
|
|
||
| package l2beat | ||
|
|
||
| import "context" | ||
|
|
||
| //go:generate mockgen -source=$GOFILE -destination=mock/$GOFILE -package=mock -typed | ||
| type IApi interface { | ||
| TVL(ctx context.Context, arguments *TVLArgs) (result TVLResponse, err error) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package l2beat | ||
|
|
||
| import ( | ||
| "context" | ||
| ) | ||
|
|
||
| type Data struct { | ||
| JSON [][]interface{} `json:"json"` | ||
| } | ||
|
|
||
| type Result struct { | ||
| Data Data `json:"data"` | ||
| } | ||
|
|
||
| type TVLResponse []struct { | ||
| Result Result `json:"result"` | ||
| } | ||
|
|
||
| type TVLArgs struct { | ||
| Batch int64 `json:"batch"` | ||
| Input JsonData `json:"input"` | ||
| } | ||
|
|
||
| type JsonData struct { | ||
| Filter Filter `json:"filter"` | ||
| Range string `json:"range"` | ||
| ExcludeAssociatedTokens bool `json:"excludeAssociatedTokens"` | ||
| } | ||
|
|
||
| type Filter struct { | ||
| Type string `json:"type"` | ||
| ProjectIds []string `json:"projectIds"` | ||
| } | ||
|
|
||
| func (api API) TVL(ctx context.Context, arguments *TVLArgs) (result TVLResponse, err error) { | ||
| // TODO: cast args to query params | ||
| args := map[string]string{} | ||
| err = api.get(ctx, "trpc/tvl.chart", args, &result) | ||
| return | ||
| } |
Oops, something went wrong.