diff --git a/go.mod b/go.mod
index d1d5c56c..5f7f5e99 100644
--- a/go.mod
+++ b/go.mod
@@ -10,6 +10,7 @@ require (
 	github.com/docker/cli v23.0.3+incompatible
 	github.com/dustin/go-humanize v1.0.1
 	github.com/google/uuid v1.3.0
+	github.com/hashicorp/golang-lru/v2 v2.0.5
 	github.com/labstack/echo-contrib v0.14.1
 	github.com/labstack/echo/v4 v4.10.2
 	github.com/labstack/gommon v0.4.0
diff --git a/go.sum b/go.sum
index 3c5622cc..e400b3cc 100644
--- a/go.sum
+++ b/go.sum
@@ -151,6 +151,8 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4=
+github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
diff --git a/pkg/content/cache.go b/pkg/content/cache.go
new file mode 100644
index 00000000..34a2bf0a
--- /dev/null
+++ b/pkg/content/cache.go
@@ -0,0 +1,124 @@
+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package content
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"sort"
+	"strconv"
+
+	"github.com/containerd/containerd/leases"
+	"github.com/containerd/containerd/namespaces"
+	lru "github.com/hashicorp/golang-lru/v2"
+)
+
+// This is not thread-safe, which means it will depend on the parent implementation to do the locking mechanism.
+type leaseCache struct {
+	caches      map[string]*lru.Cache[string, any]
+	cachesIndex []int
+	size        int
+}
+
+// newleaseCache return new empty leaseCache
+func newLeaseCache() *leaseCache {
+	return &leaseCache{
+		caches:      make(map[string]*lru.Cache[string, any]),
+		cachesIndex: make([]int, 0),
+		size:        0,
+	}
+}
+
+// Init leaseCache by leases manager from db
+func (leaseCache *leaseCache) Init(lm leases.Manager) error {
+	ls, err := lm.List(namespaces.WithNamespace(context.Background(), accelerationServiceNamespace))
+	if err != nil {
+		return err
+	}
+	sort.Slice(ls, func(i, j int) bool {
+		return ls[i].Labels[usedAtLabel] > ls[j].Labels[usedAtLabel]
+	})
+	for _, lease := range ls {
+		if err := leaseCache.Add(lease.ID, lease.Labels[usedCountLabel]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Add the key into cache
+func (leaseCache *leaseCache) Add(key string, usedCount string) error {
+	count, err := strconv.Atoi(usedCount)
+	if err != nil {
+		return err
+	}
+	if cache, ok := leaseCache.caches[usedCount]; ok {
+		cache.Add(key, nil)
+	} else {
+		cache, err := lru.New[string, any](math.MaxInt)
+		if err != nil {
+			return err
+		}
+		cache.Add(key, nil)
+		leaseCache.caches[usedCount] = cache
+		usedCount, err := strconv.Atoi(usedCount)
+		if err != nil {
+			return err
+		}
+		leaseCache.cachesIndex = append(leaseCache.cachesIndex, usedCount)
+		sort.Ints(leaseCache.cachesIndex)
+	}
+	// remove old cache
+	if usedCount != "1" {
+		if cache, ok := leaseCache.caches[strconv.Itoa(count-1)]; ok {
+			if cache.Contains(key) {
+				leaseCache.remove(key, strconv.Itoa(count-1))
+				leaseCache.size--
+			}
+		}
+	}
+	leaseCache.size++
+	return nil
+}
+
+// Remove oldest key from cache
+func (leaseCache *leaseCache) Remove() (string, error) {
+	if key, _, ok := leaseCache.caches[strconv.Itoa(leaseCache.cachesIndex[0])].GetOldest(); ok {
+		leaseCache.remove(key, strconv.Itoa(leaseCache.cachesIndex[0]))
+		leaseCache.size--
+		return key, nil
+	}
+	return "", fmt.Errorf("leaseCache have empty cache with cachesIndex")
+}
+
+func (leaseCache *leaseCache) remove(key string, usedCount string) {
+	leaseCache.caches[usedCount].Remove(key)
+	if leaseCache.caches[usedCount].Len() == 0 {
+		delete(leaseCache.caches, usedCount)
+		var newCachesIndex []int
+		for _, index := range leaseCache.cachesIndex {
+			if usedCount != strconv.Itoa(index) {
+				newCachesIndex = append(newCachesIndex, index)
+			}
+		}
+		leaseCache.cachesIndex = newCachesIndex
+	}
+}
+
+// Len return the size of leaseCache
+func (leaseCache *leaseCache) Len() int {
+	return leaseCache.size
+}
diff --git a/pkg/content/content.go b/pkg/content/content.go
index ac4b8122..5759133b 100644
--- a/pkg/content/content.go
+++ b/pkg/content/content.go
@@ -16,8 +16,8 @@ package content
 
 import (
 	"context"
+	"fmt"
 	"path/filepath"
-	"sort"
 	"strconv"
 	"time"
 
@@ -43,6 +43,8 @@ type Content struct {
 	lm leases.Manager
 	// gcSingleflight help to resolve concurrent gc
 	gcSingleflight *singleflight.Group
+	// lc cache the used count and reference order of lease
+	lc *leaseCache
 	// store is the local content store wrapped inner db
 	store content.Store
 	// threshold is the maximum capacity of the local caches storage
@@ -69,10 +71,16 @@ func NewContent(contentDir string, databaseDir string, threshold string) (*Conte
 	if err != nil {
 		return nil, err
 	}
+	lm := metadata.NewLeaseManager(db)
+	lc := newLeaseCache()
+	if err := lc.Init(lm); err != nil {
+		return nil, err
+	}
 	content := Content{
 		db:             db,
 		lm:             metadata.NewLeaseManager(db),
 		gcSingleflight: &singleflight.Group{},
+		lc:             lc,
 		store:          db.ContentStore(),
 		threshold:      int64(t),
 	}
@@ -136,17 +144,16 @@ func (content *Content) garbageCollect(ctx context.Context, size int64) error {
 
 // cleanLeases use lease to manage content blob, delete lease of content which should be gc
 func (content *Content) cleanLeases(ctx context.Context, size int64) error {
-	ls, err := content.lm.List(ctx)
-	if err != nil {
-		return err
-	}
-	// TODO: now the order of gc is LRU, should update to LRFU by usedCountLabel
-	sort.Slice(ls, func(i, j int) bool {
-		return ls[i].Labels[usedAtLabel] < ls[j].Labels[usedAtLabel]
-	})
-	for _, lease := range ls {
+	for size > 0 {
+		if content.lc.Len() == 0 {
+			return fmt.Errorf("cleanLeases: leaseCache is empty, error caches")
+		}
+		digest, err := content.lc.Remove()
+		if err != nil {
+			return err
+		}
 		if err := content.db.View(func(tx *bolt.Tx) error {
-			blobsize, err := blobSize(getBlobsBucket(tx).Bucket([]byte(lease.ID)))
+			blobsize, err := blobSize(getBlobsBucket(tx).Bucket([]byte(digest)))
 			if err != nil {
 				return err
 			}
@@ -155,15 +162,16 @@ func (content *Content) cleanLeases(ctx context.Context, size int64) error {
 		}); err != nil {
 			return err
 		}
-		if err := content.lm.Delete(ctx, lease); err != nil {
+		contentLease, err := content.lm.List(ctx, "id=="+digest)
+		if err != nil {
 			return err
 		}
-		if size <= 0 {
-			break
+		if len(contentLease) != 1 {
+			return fmt.Errorf("cleanLeases: find lease by digest failed")
+		}
+		if err := content.lm.Delete(ctx, contentLease[0]); err != nil {
+			return err
 		}
-	}
-	if err != nil {
-		return err
 	}
 	return nil
 }
@@ -208,6 +216,9 @@ func (content *Content) updateLease(digest *digest.Digest) error {
 			}
 			usedCount++
 		}
+		if err := content.lc.Add(digest.String(), strconv.Itoa(usedCount)); err != nil {
+			return err
+		}
 		// write the new labels into lease bucket
 		return boltutil.WriteLabels(bucket, map[string]string{
 			usedCountLabel: strconv.Itoa(usedCount),