From d372e2159507775174dcda5399258c887fa6b2fb Mon Sep 17 00:00:00 2001 From: Soumyajit Das Date: Mon, 27 Jan 2025 18:27:58 +0530 Subject: [PATCH] feat: [CI-15661]: Add CacheMetadata file for telemetry (#128) * feat: [CI-15661]: Add CacheMetadata file for telemetry * feat: [CI-15661]: Add CacheMetadata file for telemetry * feat: [CI-15661]: Add CacheMetadata file for telemetry file * feat: [CI-15661]: Add CacheMetadata file for telemetry file --- cache/cache.go | 7 ++++- cache/restorer.go | 66 +++++++++++++++++++++++++++++++++++++-- internal/plugin/config.go | 1 + internal/plugin/plugin.go | 2 +- main.go | 6 ++++ 5 files changed, 77 insertions(+), 5 deletions(-) diff --git a/cache/cache.go b/cache/cache.go index 5fe96828..44ad012e 100644 --- a/cache/cache.go +++ b/cache/cache.go @@ -27,7 +27,7 @@ type Rebuilder interface { // Restorer is an interface represents a restore action. type Restorer interface { // Restore restores files from the cache provided with given paths. - Restore(srcs []string) error + Restore(srcs []string, cacheFileName string) error } // Flusher is an interface represents a flush action. @@ -42,6 +42,11 @@ type cache struct { Flusher } +type CacheMetadata struct { + CacheSizeBytes uint64 `json:"cache_size_bytes,omitempty"` + Dstpath string `json:"dst_path,omitempty"` +} + // New creates a new cache with given parameters. func New(logger log.Logger, s storage.Storage, a archive.Archive, g key.Generator, backend, accountID string, opts ...Option) Cache { options := options{} diff --git a/cache/restorer.go b/cache/restorer.go index 9a44d6c0..f4399421 100644 --- a/cache/restorer.go +++ b/cache/restorer.go @@ -1,8 +1,10 @@ package cache import ( + "encoding/json" "fmt" "io" + "os" "path/filepath" "runtime" "strings" @@ -35,13 +37,15 @@ type restorer struct { accountID string } +var cacheFileMutex sync.Mutex // To ensure thread-safe writes to the file + // NewRestorer creates a new cache.Restorer. func NewRestorer(logger log.Logger, s storage.Storage, a archive.Archive, g key.Generator, fg key.Generator, namespace string, failIfKeyNotPresent bool, enableCacheKeySeparator bool, backend, accountID string) Restorer { // nolint:lll return restorer{logger, a, s, g, fg, namespace, failIfKeyNotPresent, enableCacheKeySeparator, backend, accountID} } // Restore restores files from the cache provided with given paths. -func (r restorer) Restore(dsts []string) error { +func (r restorer) Restore(dsts []string, cacheFileName string) error { level.Info(r.logger).Log("msg", "restoring cache") now := time.Now() @@ -94,7 +98,7 @@ func (r restorer) Restore(dsts []string) error { go func(src, dst string) { defer wg.Done() - if err := r.restore(src, dst); err != nil { + if err := r.restore(src, dst, cacheFileName); err != nil { errs.Add(fmt.Errorf("download from <%s> to <%s>, %w", src, dst, err)) } }(src, dst) @@ -112,7 +116,7 @@ func (r restorer) Restore(dsts []string) error { } // restore fetches the archived file from the cache and restores to the host machine's file system. -func (r restorer) restore(src, dst string) (err error) { +func (r restorer) restore(src, dst, cacheFileName string) (err error) { pr, pw := io.Pipe() defer internal.CloseWithErrCapturef(&err, pr, "rebuild, pr close <%s>", dst) @@ -140,6 +144,11 @@ func (r restorer) restore(src, dst string) (err error) { return err } + err = writeCacheMetadata(CacheMetadata{CacheSizeBytes: uint64(written), Dstpath: dst}, cacheFileName) + if err != nil { + level.Error(r.logger).Log("msg", "writeCacheMetadata", "err", err) + } + level.Info(r.logger).Log("msg", "downloaded to local", "directory", dst, "cache size", humanize.Bytes(uint64(written))) level.Debug(r.logger).Log( @@ -179,3 +188,54 @@ func getSeparator() string { return "/" } + +func writeCacheMetadata(data CacheMetadata, filename string) error { + // Lock the mutex to prevent concurrent writes + cacheFileMutex.Lock() + defer cacheFileMutex.Unlock() + + // Ensure the directory exists + dir := filepath.Dir(filename) + err := os.MkdirAll(dir, 0755) + if err != nil { + return fmt.Errorf("failed to create directory %s: %w", dir, err) + } + + // Initialize a slice to hold the cache metadata + var cacheData []CacheMetadata + + // Check if the file exists + if _, err := os.Stat(filename); err == nil { + // File exists, read and unmarshal its contents + fileContent, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read cache file %s: %w", filename, err) + } + + // Unmarshal the JSON into the slice + if len(fileContent) > 0 { + err = json.Unmarshal(fileContent, &cacheData) + if err != nil { + return fmt.Errorf("failed to unmarshal existing cache data: %w", err) + } + } + } + + // Append the new metadata to the slice + cacheData = append(cacheData, data) + + // Marshal the updated slice back to JSON + updatedData, err := json.MarshalIndent(cacheData, "", "\t") + if err != nil { + return fmt.Errorf("failed to marshal updated cache data: %w", err) + } + + // Write the updated JSON to the file + err = os.WriteFile(filename, updatedData, 0644) + if err != nil { + return fmt.Errorf("failed to write updated cache data to file %s: %w", filename, err) + } + + fmt.Println("Successfully updated cache metrics to", filename) + return nil +} diff --git a/internal/plugin/config.go b/internal/plugin/config.go index 79188e42..8d04c82c 100644 --- a/internal/plugin/config.go +++ b/internal/plugin/config.go @@ -16,6 +16,7 @@ type Config struct { ArchiveFormat string Backend string CacheKeyTemplate string + MetricsFile string RemoteRoot string LocalRoot string AccountID string diff --git a/internal/plugin/plugin.go b/internal/plugin/plugin.go index 4af6e07a..7e8d8a8b 100644 --- a/internal/plugin/plugin.go +++ b/internal/plugin/plugin.go @@ -177,7 +177,7 @@ func (p *Plugin) Exec() error { // nolint:funlen } if cfg.Restore { - if err := c.Restore(p.Config.Mount); err != nil { + if err := c.Restore(p.Config.Mount, p.Config.MetricsFile); err != nil { level.Debug(p.logger).Log("err", fmt.Sprintf("%+v\n", err)) return Error(fmt.Sprintf("[IMPORTANT] restore cache, %+v\n", err)) } diff --git a/main.go b/main.go index 719ba3d3..e181d477 100644 --- a/main.go +++ b/main.go @@ -252,6 +252,11 @@ func main() { Usage: "cache key to use for the cache directories", EnvVars: []string{"PLUGIN_CACHE_KEY"}, }, + &cli.StringFlag{ + Name: "metrics-file, chf", + Usage: "cache file to use for generating cache file metrics", + EnvVars: []string{"PLUGIN_CACHE_INTEL_METRICS_FILE"}, + }, &cli.StringFlag{ Name: "remote-root, rr", Usage: "remote root directory to contain all the cache files created (default repo.name)", @@ -610,6 +615,7 @@ func run(c *cli.Context) error { ArchiveFormat: c.String("archive-format"), Backend: c.String("backend"), CacheKeyTemplate: c.String("cache-key"), + MetricsFile: c.String("metrics-file"), CompressionLevel: c.Int("compression-level"), Debug: c.Bool("debug"), Mount: c.StringSlice("mount"),