Skip to content

Commit b3d3d6e

Browse files
author
Ken Erwin
committed
✨ New backend now syncing to s3
1 parent fb5a522 commit b3d3d6e

File tree

5 files changed

+208
-16
lines changed

5 files changed

+208
-16
lines changed

‎new-backend/go.mod

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,24 @@ require (
1818
require (
1919
github.com/Code-Hex/dd v1.1.0 // indirect
2020
github.com/Microsoft/go-winio v0.6.2 // indirect
21+
github.com/aws/aws-sdk-go-v2 v1.31.0 // indirect
22+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 // indirect
23+
github.com/aws/aws-sdk-go-v2/config v1.27.38 // indirect
24+
github.com/aws/aws-sdk-go-v2/credentials v1.17.36 // indirect
25+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 // indirect
26+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect
27+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 // indirect
28+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
29+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 // indirect
30+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5 // indirect
31+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 // indirect
32+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 // indirect
33+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 // indirect
34+
github.com/aws/aws-sdk-go-v2/service/s3 v1.63.2 // indirect
35+
github.com/aws/aws-sdk-go-v2/service/sso v1.23.2 // indirect
36+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.2 // indirect
37+
github.com/aws/aws-sdk-go-v2/service/sts v1.31.2 // indirect
38+
github.com/aws/smithy-go v1.21.0 // indirect
2139
github.com/bits-and-blooms/bitset v1.14.3 // indirect
2240
github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
2341
github.com/consensys/bavard v0.1.15 // indirect

‎new-backend/go.sum

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,42 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo
66
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
77
github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
88
github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI=
9+
github.com/aws/aws-sdk-go-v2 v1.31.0 h1:3V05LbxTSItI5kUqNwhJrrrY1BAXxXt0sN0l72QmG5U=
10+
github.com/aws/aws-sdk-go-v2 v1.31.0/go.mod h1:ztolYtaEUtdpf9Wftr31CJfLVjOnD/CVRkKOOYgF8hA=
11+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 h1:xDAuZTn4IMm8o1LnBZvmrL8JA1io4o3YWNXgohbf20g=
12+
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5/go.mod h1:wYSv6iDS621sEFLfKvpPE2ugjTuGlAG7iROg0hLOkfc=
13+
github.com/aws/aws-sdk-go-v2/config v1.27.38 h1:mMVyJJuSUdbD4zKXoxDgWrgM60QwlFEg+JhihCq6wCw=
14+
github.com/aws/aws-sdk-go-v2/config v1.27.38/go.mod h1:6xOiNEn58bj/64MPKx89r6G/el9JZn8pvVbquSqTKK4=
15+
github.com/aws/aws-sdk-go-v2/credentials v1.17.36 h1:zwI5WrT+oWWfzSKoTNmSyeBKQhsFRJRv+PGW/UZW+Yk=
16+
github.com/aws/aws-sdk-go-v2/credentials v1.17.36/go.mod h1:3AG/sY1rc9NJrNWcN/3KPU4SIDPGTrd/qegKB0TnFdE=
17+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 h1:C/d03NAmh8C4BZXhuRNboF/DqhBkBCeDiJDcaqIT5pA=
18+
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14/go.mod h1:7I0Ju7p9mCIdlrfS+JCgqcYD0VXz/N4yozsox+0o078=
19+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 h1:kYQ3H1u0ANr9KEKlGs/jTLrBFPo8P8NaH/w7A01NeeM=
20+
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18/go.mod h1:r506HmK5JDUh9+Mw4CfGJGSSoqIiLCndAuqXuhbv67Y=
21+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18 h1:Z7IdFUONvTcvS7YuhtVxN99v2cCoHRXOS4mTr0B/pUc=
22+
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.18/go.mod h1:DkKMmksZVVyat+Y+r1dEOgJEfUeA7UngIHWeKsi0yNc=
23+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ=
24+
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc=
25+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18 h1:OWYvKL53l1rbsUmW7bQyJVsYU/Ii3bbAAQIIFNbM0Tk=
26+
github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.18/go.mod h1:CUx0G1v3wG6l01tUB+j7Y8kclA8NSqK4ef0YG79a4cg=
27+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5 h1:QFASJGfT8wMXtuP3D5CRmMjARHv9ZmzFUMJznHDOY3w=
28+
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.5/go.mod h1:QdZ3OmoIjSX+8D1OPAzPxDfjXASbBMDsz9qvtyIhtik=
29+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 h1:rTWjG6AvWekO2B1LHeM3ktU7MqyX9rzWQ7hgzneZW7E=
30+
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20/go.mod h1:RGW2DDpVc8hu6Y6yG8G5CHVmVOAn1oV8rNKOHRJyswg=
31+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 h1:Xbwbmk44URTiHNx6PNo0ujDE6ERlsCKJD3u1zfnzAPg=
32+
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20/go.mod h1:oAfOFzUB14ltPZj1rWwRc3d/6OgD76R8KlvU3EqM9Fg=
33+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 h1:eb+tFOIl9ZsUe2259/BKPeniKuz4/02zZFH/i4Nf8Rg=
34+
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18/go.mod h1:GVCC2IJNJTmdlyEsSmofEy7EfJncP7DNnXDzRjJ5Keg=
35+
github.com/aws/aws-sdk-go-v2/service/s3 v1.63.2 h1:1iXmXy8SJzQVMGvo40TSzBYS9ig6BSyXfRIMzLfmBfE=
36+
github.com/aws/aws-sdk-go-v2/service/s3 v1.63.2/go.mod h1:NLTqRLe3pUNu3nTEHI6XlHLKYmc8fbHUdMxAB6+s41Q=
37+
github.com/aws/aws-sdk-go-v2/service/sso v1.23.2 h1:yzi/y/vKlLyzOfG7pSu5ONNGRxHIgLeDrV4w2AMRCo0=
38+
github.com/aws/aws-sdk-go-v2/service/sso v1.23.2/go.mod h1:XRlMvmad0ZNL+75C5FYdMvbbLkd6qiqz6foR1nA1PXY=
39+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.2 h1:3gb6pYhYLjo8rB1h2Tqs61wpjRd3rQymYcVq/pp0yxI=
40+
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.2/go.mod h1:FnvDM4sfa+isJ3kDXIzAB9GAwVSzFzSy97uZ3IsHo4E=
41+
github.com/aws/aws-sdk-go-v2/service/sts v1.31.2 h1:O6tyji8mXmBGsHvTCB0VIhrDw19lGTUSbKIyjnw79s8=
42+
github.com/aws/aws-sdk-go-v2/service/sts v1.31.2/go.mod h1:yMWe0F+XG0DkRZK5ODZhG7BEFYhLXi2dqGsv6tX0cgI=
43+
github.com/aws/smithy-go v1.21.0 h1:H7L8dtDRk0P1Qm6y0ji7MCYMQObJ5R9CRpyPhRUkLYA=
44+
github.com/aws/smithy-go v1.21.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
945
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
1046
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
1147
github.com/bits-and-blooms/bitset v1.14.3 h1:Gd2c8lSNf9pKXom5JtD7AaKO8o7fGQ2LtFj1436qilA=

‎new-backend/internal/ingestor/constants.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package ingestor
2+
3+
const (
4+
startBlockNumber = 2641527
5+
blockRangeSize = 10000
6+
safetyBlockOffset = 10
7+
constructorMethodID = "0x60606040"
8+
imageSize = 512
9+
maxPostgresNumeric = 1e3 // Display max of 1000 eth
10+
11+
EventTypeImageRender = "image_render"
12+
EventTypeDiscordNotification = "discord_notification"
13+
)

‎new-backend/internal/ingestor/ingestor.go

Lines changed: 17 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -24,22 +24,6 @@ import (
2424
utils "pixelmap.io/backend/internal/utils"
2525
)
2626

27-
const (
28-
startBlockNumber = 2641527
29-
blockRangeSize = 10000
30-
safetyBlockOffset = 10
31-
constructorMethodID = "0x60606040"
32-
imageSize = 512
33-
maxPostgresNumeric = 1e3 // Display max of 1000 eth
34-
)
35-
36-
// Event types
37-
const (
38-
EventTypeImageRender = "image_render"
39-
EventTypeDiscordNotification = "discord_notification"
40-
// Add more event types as needed
41-
)
42-
4327
type Event struct {
4428
Type string
4529
Payload json.RawMessage
@@ -85,10 +69,16 @@ type Ingestor struct {
8569
isRendering atomic.Bool
8670
maxRetries int
8771
baseDelay time.Duration
72+
s3Syncer *S3Syncer
8873
}
8974

9075
func NewIngestor(logger *zap.Logger, sqlDB *sql.DB, apiKey string) *Ingestor {
9176
pubSub := NewPubSub()
77+
s3Syncer, err := NewS3Syncer(logger, "cache")
78+
if err != nil {
79+
logger.Error("Failed to create S3Syncer", zap.Error(err))
80+
}
81+
9282
ingestor := &Ingestor{
9383
logger: logger,
9484
queries: db.New(sqlDB),
@@ -97,6 +87,7 @@ func NewIngestor(logger *zap.Logger, sqlDB *sql.DB, apiKey string) *Ingestor {
9787
renderSignal: make(chan struct{}, 1),
9888
maxRetries: 5,
9989
baseDelay: time.Second,
90+
s3Syncer: s3Syncer,
10091
}
10192

10293
// Start the continuous rendering process
@@ -629,6 +620,16 @@ func (i *Ingestor) processDataHistory(ctx context.Context) error {
629620
utils.RenderFullMap(tiles, "cache/tilemap.png")
630621

631622
i.logger.Info("Finished processing data history", zap.Int("count", len(history)))
623+
624+
// Sync with S3 after processing
625+
if i.s3Syncer != nil {
626+
err := i.s3Syncer.SyncWithS3(ctx)
627+
if err != nil {
628+
i.logger.Error("Failed to sync with S3", zap.Error(err))
629+
// Note: We're not returning this error as it shouldn't stop the main process
630+
}
631+
}
632+
632633
return nil
633634
}
634635

‎new-backend/internal/ingestor/s3_syncer.go

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
package ingestor
2+
3+
import (
4+
"context"
5+
"crypto/md5"
6+
"encoding/hex"
7+
"io"
8+
"os"
9+
"path/filepath"
10+
"strings"
11+
"sync"
12+
13+
"github.com/aws/aws-sdk-go-v2/aws"
14+
"github.com/aws/aws-sdk-go-v2/config"
15+
"github.com/aws/aws-sdk-go-v2/service/s3"
16+
"go.uber.org/zap"
17+
)
18+
19+
type S3Syncer struct {
20+
client *s3.Client
21+
bucketName string
22+
cacheDir string
23+
logger *zap.Logger
24+
syncMu sync.Mutex
25+
fileHashes map[string]string
26+
}
27+
28+
func NewS3Syncer(logger *zap.Logger, cacheDir string) (*S3Syncer, error) {
29+
cfg, err := config.LoadDefaultConfig(context.TODO())
30+
if err != nil {
31+
return nil, err
32+
}
33+
34+
client := s3.NewFromConfig(cfg)
35+
36+
return &S3Syncer{
37+
client: client,
38+
bucketName: "pixelmap.art",
39+
cacheDir: cacheDir,
40+
logger: logger,
41+
fileHashes: make(map[string]string),
42+
}, nil
43+
}
44+
45+
// Modify SyncWithS3 to accept a context
46+
func (s *S3Syncer) SyncWithS3(ctx context.Context) error {
47+
s.syncMu.Lock()
48+
defer s.syncMu.Unlock()
49+
50+
err := filepath.Walk(s.cacheDir, func(path string, info os.FileInfo, err error) error {
51+
if err != nil {
52+
return err
53+
}
54+
55+
if info.IsDir() {
56+
return nil
57+
}
58+
59+
relPath, err := filepath.Rel(s.cacheDir, path)
60+
if err != nil {
61+
return err
62+
}
63+
64+
s3Key := strings.ReplaceAll(relPath, string(os.PathSeparator), "/")
65+
66+
fileHash, err := s.calculateMD5(path)
67+
if err != nil {
68+
s.logger.Error("Failed to calculate MD5", zap.Error(err), zap.String("path", path))
69+
return nil
70+
}
71+
72+
if storedHash, ok := s.fileHashes[s3Key]; !ok || storedHash != fileHash {
73+
if err := s.uploadToS3(ctx, path, s3Key); err != nil {
74+
s.logger.Error("Failed to upload file to S3", zap.Error(err), zap.String("path", path))
75+
} else {
76+
s.fileHashes[s3Key] = fileHash
77+
}
78+
}
79+
80+
return nil
81+
})
82+
83+
if err != nil {
84+
s.logger.Error("Error walking through cache directory", zap.Error(err))
85+
}
86+
87+
return err
88+
}
89+
90+
// Update uploadToS3 to accept a context
91+
func (s *S3Syncer) uploadToS3(ctx context.Context, filePath, s3Key string) error {
92+
file, err := os.Open(filePath)
93+
if err != nil {
94+
return err
95+
}
96+
defer file.Close()
97+
98+
_, err = s.client.PutObject(ctx, &s3.PutObjectInput{
99+
Bucket: aws.String(s.bucketName),
100+
Key: aws.String(s3Key),
101+
Body: file,
102+
})
103+
104+
if err == nil {
105+
s.logger.Info("File uploaded to S3", zap.String("key", s3Key))
106+
}
107+
108+
return err
109+
}
110+
111+
func (s *S3Syncer) calculateMD5(filePath string) (string, error) {
112+
file, err := os.Open(filePath)
113+
if err != nil {
114+
return "", err
115+
}
116+
defer file.Close()
117+
118+
hash := md5.New()
119+
if _, err := io.Copy(hash, file); err != nil {
120+
return "", err
121+
}
122+
123+
return hex.EncodeToString(hash.Sum(nil)), nil
124+
}

0 commit comments

Comments
 (0)