✨ Now generating tiledata.sjon as well, but with terrible efficiency

Author: Ken Erwin

new-backend/internal/ingestor/ingestor.go

@@ -74,9 +74,15 @@ type Ingestor struct {
 
 func NewIngestor(logger *zap.Logger, sqlDB *sql.DB, apiKey string) *Ingestor {
 	pubSub := NewPubSub()
-	s3Syncer, err := NewS3Syncer(logger, "cache")
-	if err != nil {
-		logger.Error("Failed to create S3Syncer", zap.Error(err))
+	var s3Syncer *S3Syncer
+
+	// Check if SYNC_TO_AWS environment variable is set
+	if os.Getenv("SYNC_TO_AWS") == "true" {
+		var err error
+		s3Syncer, err = NewS3Syncer(logger, "cache")
+		if err != nil {
+			logger.Error("Failed to create S3Syncer", zap.Error(err))
+		}
 	}
 
 	ingestor := &Ingestor{
@@ -439,6 +445,14 @@ func (i *Ingestor) processTransaction(ctx context.Context, tx *EtherscanTransact
 			if err != nil {
 				return fmt.Errorf("failed to insert wrapping history: %w", err)
 			}
+			err = i.queries.UpdateTile(ctx, db.UpdateTileParams{
+				ID:      int32(location.Int64()),
+				Owner:   tx.From,
+				Wrapped: wrapped == "true",
+			})
+			if err != nil {
+				return fmt.Errorf("failed to update tile owner: %w", err)
+			}
 		}
 	case "unwrap":
 		i.logger.Info("wrap called",
@@ -465,6 +479,14 @@ func (i *Ingestor) processTransaction(ctx context.Context, tx *EtherscanTransact
 			if err != nil {
 				return fmt.Errorf("failed to insert wrapping history: %w", err)
 			}
+			err = i.queries.UpdateTile(ctx, db.UpdateTileParams{
+				ID:      int32(location.Int64()),
+				Owner:   tx.From,
+				Wrapped: wrapped == "false",
+			})
+			if err != nil {
+				return fmt.Errorf("failed to update tile owner: %w", err)
+			}
 		}
 	case "transferFrom", "safeTransferFrom", "safeTransferFrom0":
 		if err := i.processTransfer(ctx, args, tx, timeStamp.Int64(), blockNumber.Int64(), int32(transactionIndex)); err != nil {
@@ -633,9 +655,20 @@ func (i *Ingestor) processDataHistory(ctx context.Context) error {
 	}
 	utils.RenderFullMap(tiles, "cache/tilemap.png")
 
+	// get all tiles
+	all_tiles, err := i.queries.ListTiles(ctx, db.ListTilesParams{
+		Limit:  int32(3970),
+		Offset: 0,
+	})
+
+	if err != nil {
+		return fmt.Errorf("failed to get all tiles: %w", err)
+	}
+	GenerateTiledataJSON(all_tiles)
+
 	i.logger.Info("Finished processing data history", zap.Int("count", len(history)))
 
-	// Sync with S3 after processing
+	// Sync with S3 after processing only if s3Syncer is initialized
 	if i.s3Syncer != nil {
 		err := i.s3Syncer.SyncWithS3(ctx)
 		if err != nil {
@@ -764,6 +797,19 @@ func (i *Ingestor) processTileUpdate(ctx context.Context, location *big.Int, ima
 		return fmt.Errorf("failed to insert data history: %w", err)
 	}
 
+	// Update the tile in the database
+	err := i.queries.UpdateTile(ctx, db.UpdateTileParams{
+		ID:    int32(location.Int64()),
+		Price: priceEthStr,
+		Url:   url,
+		Image: image,
+	})
+	if err != nil {
+		i.logger.Error("Failed to update tile", zap.Error(err), zap.String("location", location.String()))
+		os.Exit(1)
+		return fmt.Errorf("failed to update tile: %w", err)
+	}
+
 	// Signal that new data is available to render
 	i.signalNewData()
 

new-backend/internal/ingestor/metadata.go

@@ -3,6 +3,7 @@ package ingestor
 import (
 	"encoding/json"
 	"fmt"
+	"log"
 	"os"
 	"path/filepath"
 	"strings"
@@ -12,6 +13,8 @@ import (
 	utils "pixelmap.io/backend/internal/utils"
 )
 
+var logger = log.New(os.Stdout, "metadata", log.LstdFlags)
+
 type MetadataPixelMapTile struct {
 	ID               int             `json:"id"`
 	Image            string          `json:"image"`
@@ -24,6 +27,47 @@ type MetadataPixelMapTile struct {
 	HistoricalImages []PixelMapImage `json:"historical_images"`
 }
 
+// GenerateTiledataJSON generates the tiledata.json file
+func GenerateTiledataJSON(tiles []db.Tile) error {
+
+	logger.Println("Generating tiledata.json")
+	tiledataJSON := make([]map[string]interface{}, 3970)
+
+	for i := 0; i <= 3969; i++ {
+		tile := tiles[i]
+		if err := updateTileMetadata(tile, nil); err != nil {
+			return fmt.Errorf("error updating tile metadata: %w", err)
+		}
+
+		tiledataJSON[i] = map[string]interface{}{
+			"id":           tile.ID,
+			"url":          tile.Url,
+			"image":        tile.Image,
+			"owner":        tile.Owner,
+			"price":        tile.Price,
+			"wrapped":      tile.Wrapped,
+			"openseaPrice": tile.OpenseaPrice,
+			"lastUpdated":  time.Date(2021, time.December, 13, 1, 1, 0, 0, time.UTC),
+			"ens":          tile.Ens,
+		}
+	}
+
+	jsonData, err := json.MarshalIndent(tiledataJSON, "", "  ")
+	if err != nil {
+		return fmt.Errorf("error marshaling tiledata JSON: %w", err)
+	}
+
+	if err := os.MkdirAll(filepath.Dir("cache/tiledata.json"), os.ModePerm); err != nil {
+		return fmt.Errorf("error creating cache directory: %w", err)
+	}
+
+	if err := os.WriteFile("cache/tiledata.json", jsonData, 0644); err != nil {
+		return fmt.Errorf("error writing tiledata.json file: %w", err)
+	}
+
+	return nil
+}
+
 // updateTileMetadata updates the metadata for a given tile
 func updateTileMetadata(tile db.Tile, dataHistory []db.DataHistory) error {
 	tileMetaData := map[string]interface{}{