From 9fe0d76aaf8bbaeadee48f54e87f8a5a5783f6a2 Mon Sep 17 00:00:00 2001 From: Aviral Takkar Date: Fri, 12 Apr 2024 15:49:14 -0700 Subject: [PATCH] refactor: move cache to pkg --- build/package/peerd-helm/Chart.yaml | 2 +- internal/files/store/file_test.go | 2 +- internal/files/store/main_test.go | 2 +- internal/files/store/mockstore.go | 2 +- internal/files/store/store.go | 4 ++-- internal/handlers/files/main_test.go | 2 +- internal/remote/interface.go | 2 +- .../files/cache/cache.go => pkg/cache/filecache.go | 6 +++--- .../cache_test.go => pkg/cache/filecache_test.go | 12 ++++++++---- {internal/files => pkg}/cache/interface.go | 8 ++++---- {internal/files => pkg}/cache/item.go | 0 {internal/files => pkg}/cache/item_test.go | 0 {internal/files => pkg}/cache/main_test.go | 0 {internal/files => pkg}/cache/syncmap.go | 0 {internal/files => pkg}/cache/syncmap_test.go | 0 pkg/peernet/network.go | 2 +- 16 files changed, 24 insertions(+), 20 deletions(-) rename internal/files/cache/cache.go => pkg/cache/filecache.go (95%) rename internal/files/cache/cache_test.go => pkg/cache/filecache_test.go (95%) rename {internal/files => pkg}/cache/interface.go (76%) rename {internal/files => pkg}/cache/item.go (100%) rename {internal/files => pkg}/cache/item_test.go (100%) rename {internal/files => pkg}/cache/main_test.go (100%) rename {internal/files => pkg}/cache/syncmap.go (100%) rename {internal/files => pkg}/cache/syncmap_test.go (100%) diff --git a/build/package/peerd-helm/Chart.yaml b/build/package/peerd-helm/Chart.yaml index a0e059c..49da0fc 100644 --- a/build/package/peerd-helm/Chart.yaml +++ b/build/package/peerd-helm/Chart.yaml @@ -21,4 +21,4 @@ version: 0.0.3-alpha # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.0.6-alpha" +appVersion: "0.0.7-alpha" diff --git a/internal/files/store/file_test.go b/internal/files/store/file_test.go index a305f19..0c377f3 100644 --- a/internal/files/store/file_test.go +++ b/internal/files/store/file_test.go @@ -10,8 +10,8 @@ import ( "testing" "github.com/azure/peerd/internal/files" - "github.com/azure/peerd/internal/files/cache" remotetests "github.com/azure/peerd/internal/remote/tests" + "github.com/azure/peerd/pkg/cache" "github.com/azure/peerd/pkg/discovery/routing/tests" ) diff --git a/internal/files/store/main_test.go b/internal/files/store/main_test.go index 21268e9..26d00d9 100644 --- a/internal/files/store/main_test.go +++ b/internal/files/store/main_test.go @@ -9,7 +9,7 @@ import ( "os" "testing" - "github.com/azure/peerd/internal/files/cache" + "github.com/azure/peerd/pkg/cache" "github.com/azure/peerd/pkg/metrics" ) diff --git a/internal/files/store/mockstore.go b/internal/files/store/mockstore.go index e7a7785..383772b 100644 --- a/internal/files/store/mockstore.go +++ b/internal/files/store/mockstore.go @@ -5,7 +5,7 @@ package store import ( "context" - "github.com/azure/peerd/internal/files/cache" + "github.com/azure/peerd/pkg/cache" "github.com/azure/peerd/pkg/discovery/routing" ) diff --git a/internal/files/store/store.go b/internal/files/store/store.go index 7150669..53e60b2 100644 --- a/internal/files/store/store.go +++ b/internal/files/store/store.go @@ -11,8 +11,8 @@ import ( p2pcontext "github.com/azure/peerd/internal/context" "github.com/azure/peerd/internal/files" - "github.com/azure/peerd/internal/files/cache" "github.com/azure/peerd/internal/remote" + "github.com/azure/peerd/pkg/cache" "github.com/azure/peerd/pkg/discovery/routing" "github.com/azure/peerd/pkg/metrics" "github.com/azure/peerd/pkg/urlparser" @@ -25,7 +25,7 @@ import ( func NewFilesStore(ctx context.Context, r routing.Router) (FilesStore, error) { fs := &store{ metricsRecorder: metrics.FromContext(ctx), - cache: cache.New(ctx), + cache: cache.New(ctx, int64(files.CacheBlockSize)), prefetchChan: make(chan prefetchableSegment, PrefetchWorkers), prefetchable: PrefetchWorkers > 0, router: r, diff --git a/internal/handlers/files/main_test.go b/internal/handlers/files/main_test.go index 9298e97..c79d83f 100644 --- a/internal/handlers/files/main_test.go +++ b/internal/handlers/files/main_test.go @@ -8,7 +8,7 @@ import ( "os" "testing" - "github.com/azure/peerd/internal/files/cache" + "github.com/azure/peerd/pkg/cache" ) func TestMain(m *testing.M) { diff --git a/internal/remote/interface.go b/internal/remote/interface.go index 56c93bb..66004b4 100644 --- a/internal/remote/interface.go +++ b/internal/remote/interface.go @@ -20,7 +20,7 @@ type Reader interface { Log() *zerolog.Logger } -// Error describes an error that occured during a remote operation. +// Error describes an error that occurred during a remote operation. type Error struct { *http.Response error diff --git a/internal/files/cache/cache.go b/pkg/cache/filecache.go similarity index 95% rename from internal/files/cache/cache.go rename to pkg/cache/filecache.go index fe4db73..4455470 100644 --- a/internal/files/cache/cache.go +++ b/pkg/cache/filecache.go @@ -13,7 +13,6 @@ import ( "sync/atomic" "time" - "github.com/azure/peerd/internal/files" "github.com/dgraph-io/ristretto" "github.com/rs/zerolog" ) @@ -149,7 +148,8 @@ func waitForSet() { } // New creates a new cache of files. -func New(ctx context.Context) Cache { +// cacheBlockSize is the fixed size of the cache block in bytes, and is used to evaluate the cost of each item in the cache. +func New(ctx context.Context, cacheBlockSize int64) Cache { log := zerolog.Ctx(ctx).With().Str("component", "cache").Logger() atomic.StoreInt32(&fdCnt, 0) @@ -176,7 +176,7 @@ func New(ctx context.Context) Cache { }, Cost: func(val interface{}) int64 { - return int64(files.CacheBlockSize) + return cacheBlockSize }, }); err != nil { // This will call os.Exit(1) diff --git a/internal/files/cache/cache_test.go b/pkg/cache/filecache_test.go similarity index 95% rename from internal/files/cache/cache_test.go rename to pkg/cache/filecache_test.go index d430ccb..3ae70a9 100644 --- a/internal/files/cache/cache_test.go +++ b/pkg/cache/filecache_test.go @@ -18,10 +18,14 @@ import ( "golang.org/x/sync/errgroup" ) +var ( + cacheBlockSize = int64(1 * 1024 * 1024) +) + func TestGetKey(t *testing.T) { name := newRandomStringN(10) offset := int64(100) - c := New(context.Background()) + c := New(context.Background(), cacheBlockSize) got := c.(*fileCache).getKey(name, offset) want := fmt.Sprintf("%v/%v/%v", Path, name, offset) if got != want { @@ -30,7 +34,7 @@ func TestGetKey(t *testing.T) { } func TestExists(t *testing.T) { - c := New(context.Background()) + c := New(context.Background(), cacheBlockSize) filesThatExist := []string{} for i := 0; i < 5; i++ { @@ -113,7 +117,7 @@ func TestExists(t *testing.T) { } func TestPutAndGetSize(t *testing.T) { - c := New(context.Background()) + c := New(context.Background(), cacheBlockSize) var eg errgroup.Group for i := 0; i < 1000; i++ { @@ -150,7 +154,7 @@ func TestPutAndGetSize(t *testing.T) { func TestGetOrCreate(t *testing.T) { zerolog.TimeFieldFormat = time.RFC3339 //c := New(zerolog.New(os.Stdout).With().Timestamp().Logger().WithContext(context.Background())) - c := New(context.Background()) + c := New(context.Background(), cacheBlockSize) var eg errgroup.Group fileNames := new(sync.Map) diff --git a/internal/files/cache/interface.go b/pkg/cache/interface.go similarity index 76% rename from internal/files/cache/interface.go rename to pkg/cache/interface.go index 9ee1b09..854eb44 100644 --- a/internal/files/cache/interface.go +++ b/pkg/cache/interface.go @@ -2,7 +2,7 @@ // Licensed under the MIT License. package cache -// Cache describes the cache of files. +// Cache describes a cache of files. type Cache interface { // Size gets the size of the file. Size(path string) (int64, bool) @@ -18,12 +18,12 @@ type Cache interface { } var ( - // FilesCacheMaxCost is the capacity of the files cache in any unit. + // FilesCacheMaxCost is the capacity of the files cache. FilesCacheMaxCost int64 = 4 * 1024 * 1024 * 1024 // 4 Gib - // MemoryCacheMaxCost is the capacity of the memory cache in any unit. + // MemoryCacheMaxCost is the capacity of the memory cache. MemoryCacheMaxCost int64 = 1 * 1024 * 1024 * 1024 // 1 Gib // Path is the path to the cache directory. - Path string = "/tmp/distribution/p2p/cache" + Path string = "/tmp/distribution/peerd/cache" ) diff --git a/internal/files/cache/item.go b/pkg/cache/item.go similarity index 100% rename from internal/files/cache/item.go rename to pkg/cache/item.go diff --git a/internal/files/cache/item_test.go b/pkg/cache/item_test.go similarity index 100% rename from internal/files/cache/item_test.go rename to pkg/cache/item_test.go diff --git a/internal/files/cache/main_test.go b/pkg/cache/main_test.go similarity index 100% rename from internal/files/cache/main_test.go rename to pkg/cache/main_test.go diff --git a/internal/files/cache/syncmap.go b/pkg/cache/syncmap.go similarity index 100% rename from internal/files/cache/syncmap.go rename to pkg/cache/syncmap.go diff --git a/internal/files/cache/syncmap_test.go b/pkg/cache/syncmap_test.go similarity index 100% rename from internal/files/cache/syncmap_test.go rename to pkg/cache/syncmap_test.go diff --git a/pkg/peernet/network.go b/pkg/peernet/network.go index 80bae88..31e2e06 100644 --- a/pkg/peernet/network.go +++ b/pkg/peernet/network.go @@ -48,7 +48,7 @@ type network struct { var _ Network = &network{} // DefaultTLSConfig creates a TLS config to use for this server. -// This config does not require client certificate verification and is resuable. +// This config does not require client certificate verification and is reusable. func (n *network) DefaultTLSConfig() *tls.Config { return n.defaultTLSConfig }