feat: [CI-15661]: Add CacheMetadata file for telemetry (#128)

* feat: [CI-15661]: Add CacheMetadata file for telemetry

* feat: [CI-15661]: Add CacheMetadata file for telemetry

* feat: [CI-15661]: Add CacheMetadata file for telemetry file

* feat: [CI-15661]: Add CacheMetadata file for telemetry file

cache/cache.go

@@ -27,7 +27,7 @@ type Rebuilder interface {
 // Restorer is an interface represents a restore action.
 type Restorer interface {
 	// Restore restores files from the cache provided with given paths.
-	Restore(srcs []string) error
+	Restore(srcs []string, cacheFileName string) error
 }
 
 // Flusher is an interface represents a flush action.
@@ -42,6 +42,11 @@ type cache struct {
 	Flusher
 }
 
+type CacheMetadata struct {
+	CacheSizeBytes uint64 `json:"cache_size_bytes,omitempty"`
+	Dstpath        string `json:"dst_path,omitempty"`
+}
+
 // New creates a new cache with given parameters.
 func New(logger log.Logger, s storage.Storage, a archive.Archive, g key.Generator, backend, accountID string, opts ...Option) Cache {
 	options := options{}

cache/restorer.go

@@ -1,8 +1,10 @@
 package cache
 
 import (
+	"encoding/json"
 	"fmt"
 	"io"
+	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -35,13 +37,15 @@ type restorer struct {
 	accountID               string
 }
 
+var cacheFileMutex sync.Mutex // To ensure thread-safe writes to the file
+
 // NewRestorer creates a new cache.Restorer.
 func NewRestorer(logger log.Logger, s storage.Storage, a archive.Archive, g key.Generator, fg key.Generator, namespace string, failIfKeyNotPresent bool, enableCacheKeySeparator bool, backend, accountID string) Restorer { // nolint:lll
 	return restorer{logger, a, s, g, fg, namespace, failIfKeyNotPresent, enableCacheKeySeparator, backend, accountID}
 }
 
 // Restore restores files from the cache provided with given paths.
-func (r restorer) Restore(dsts []string) error {
+func (r restorer) Restore(dsts []string, cacheFileName string) error {
 	level.Info(r.logger).Log("msg", "restoring cache")
 
 	now := time.Now()
@@ -94,7 +98,7 @@ func (r restorer) Restore(dsts []string) error {
 		go func(src, dst string) {
 			defer wg.Done()
 
-			if err := r.restore(src, dst); err != nil {
+			if err := r.restore(src, dst, cacheFileName); err != nil {
 				errs.Add(fmt.Errorf("download from <%s> to <%s>, %w", src, dst, err))
 			}
 		}(src, dst)
@@ -112,7 +116,7 @@ func (r restorer) Restore(dsts []string) error {
 }
 
 // restore fetches the archived file from the cache and restores to the host machine's file system.
-func (r restorer) restore(src, dst string) (err error) {
+func (r restorer) restore(src, dst, cacheFileName string) (err error) {
 	pr, pw := io.Pipe()
 	defer internal.CloseWithErrCapturef(&err, pr, "rebuild, pr close <%s>", dst)
 
@@ -140,6 +144,11 @@ func (r restorer) restore(src, dst string) (err error) {
 		return err
 	}
 
+	err = writeCacheMetadata(CacheMetadata{CacheSizeBytes: uint64(written), Dstpath: dst}, cacheFileName)
+	if err != nil {
+		level.Error(r.logger).Log("msg", "writeCacheMetadata", "err", err)
+	}
+
 	level.Info(r.logger).Log("msg", "downloaded to local", "directory", dst, "cache size", humanize.Bytes(uint64(written)))
 
 	level.Debug(r.logger).Log(
@@ -179,3 +188,54 @@ func getSeparator() string {
 
 	return "/"
 }
+
+func writeCacheMetadata(data CacheMetadata, filename string) error {
+	// Lock the mutex to prevent concurrent writes
+	cacheFileMutex.Lock()
+	defer cacheFileMutex.Unlock()
+
+	// Ensure the directory exists
+	dir := filepath.Dir(filename)
+	err := os.MkdirAll(dir, 0755)
+	if err != nil {
+		return fmt.Errorf("failed to create directory %s: %w", dir, err)
+	}
+
+	// Initialize a slice to hold the cache metadata
+	var cacheData []CacheMetadata
+
+	// Check if the file exists
+	if _, err := os.Stat(filename); err == nil {
+		// File exists, read and unmarshal its contents
+		fileContent, err := os.ReadFile(filename)
+		if err != nil {
+			return fmt.Errorf("failed to read cache file %s: %w", filename, err)
+		}
+
+		// Unmarshal the JSON into the slice
+		if len(fileContent) > 0 {
+			err = json.Unmarshal(fileContent, &cacheData)
+			if err != nil {
+				return fmt.Errorf("failed to unmarshal existing cache data: %w", err)
+			}
+		}
+	}
+
+	// Append the new metadata to the slice
+	cacheData = append(cacheData, data)
+
+	// Marshal the updated slice back to JSON
+	updatedData, err := json.MarshalIndent(cacheData, "", "\t")
+	if err != nil {
+		return fmt.Errorf("failed to marshal updated cache data: %w", err)
+	}
+
+	// Write the updated JSON to the file
+	err = os.WriteFile(filename, updatedData, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to write updated cache data to file %s: %w", filename, err)
+	}
+
+	fmt.Println("Successfully updated cache metrics to", filename)
+	return nil
+}

internal/plugin/config.go

@@ -16,6 +16,7 @@ type Config struct {
 	ArchiveFormat    string
 	Backend          string
 	CacheKeyTemplate string
+	MetricsFile      string
 	RemoteRoot       string
 	LocalRoot        string
 	AccountID        string

internal/plugin/plugin.go

@@ -177,7 +177,7 @@ func (p *Plugin) Exec() error { // nolint:funlen
 	}
 
 	if cfg.Restore {
-		if err := c.Restore(p.Config.Mount); err != nil {
+		if err := c.Restore(p.Config.Mount, p.Config.MetricsFile); err != nil {
 			level.Debug(p.logger).Log("err", fmt.Sprintf("%+v\n", err))
 			return Error(fmt.Sprintf("[IMPORTANT] restore cache, %+v\n", err))
 		}

main.go

@@ -252,6 +252,11 @@ func main() {
 			Usage:   "cache key to use for the cache directories",
 			EnvVars: []string{"PLUGIN_CACHE_KEY"},
 		},
+		&cli.StringFlag{
+			Name:    "metrics-file, chf",
+			Usage:   "cache file to use for generating cache file metrics",
+			EnvVars: []string{"PLUGIN_CACHE_INTEL_METRICS_FILE"},
+		},
 		&cli.StringFlag{
 			Name:    "remote-root, rr",
 			Usage:   "remote root directory to contain all the cache files created (default repo.name)",
@@ -610,6 +615,7 @@ func run(c *cli.Context) error {
 		ArchiveFormat:              c.String("archive-format"),
 		Backend:                    c.String("backend"),
 		CacheKeyTemplate:           c.String("cache-key"),
+		MetricsFile:                c.String("metrics-file"),
 		CompressionLevel:           c.Int("compression-level"),
 		Debug:                      c.Bool("debug"),
 		Mount:                      c.StringSlice("mount"),