Skip to content

Commit

Permalink
Merge pull request #158 from Desiki-high/refactor-store
Browse files Browse the repository at this point in the history
feat: add new type Store in content
  • Loading branch information
imeoer authored Jul 13, 2023
2 parents c893d36 + 6a1b79b commit 5acde6c
Show file tree
Hide file tree
Showing 5 changed files with 110 additions and 35 deletions.
22 changes: 19 additions & 3 deletions pkg/content/content.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@ import (
"context"
"encoding/binary"
"fmt"
"path/filepath"
"time"

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/metadata"
"github.com/dustin/go-humanize"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
bolt "go.etcd.io/bbolt"
)
Expand All @@ -42,15 +44,29 @@ type Content struct {
threshold int64
}

func NewContent(db *metadata.DB, threshold string) (*Content, error) {
func NewContent(contentDir string, databaseDir string, threshold string) (*Content, error) {
store, err := newStore(contentDir)
if err != nil {
return nil, errors.Wrap(err, "create local provider content store")
}
bdb, err := bolt.Open(filepath.Join(databaseDir, "meta.db"), 0655, nil)
if err != nil {
return nil, errors.Wrap(err, "create local provider database")
}
db := metadata.NewDB(bdb, store, nil)
if err := db.Init(context.Background()); err != nil {
return nil, err
}
t, err := humanize.ParseBytes(threshold)
if err != nil {
return nil, err
}
return &Content{
content := Content{
db: db,
threshold: int64(t),
}, nil
}
store.Init(&content)
return &content, nil
}

// return the content store in db
Expand Down
12 changes: 1 addition & 11 deletions pkg/content/content_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,26 +15,16 @@
package content

import (
"context"
"os"
"testing"

"github.com/containerd/containerd/content/local"
"github.com/containerd/containerd/metadata"
"github.com/stretchr/testify/require"
bolt "go.etcd.io/bbolt"
)

func TestContenSize(t *testing.T) {
os.MkdirAll("./tmp", 0755)
defer os.RemoveAll("./tmp")
store, err := local.NewStore("./tmp")
require.NoError(t, err)
bdb, err := bolt.Open("./tmp/metadata.db", 0655, nil)
require.NoError(t, err)
db := metadata.NewDB(bdb, store, nil)
require.NoError(t, db.Init(context.Background()))
content, err := NewContent(db, "1000MB")
content, err := NewContent("./tmp", "./tmp", "1000MB")
require.NoError(t, err)
size, err := content.Size()
require.NoError(t, err)
Expand Down
19 changes: 2 additions & 17 deletions pkg/content/local.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,12 @@ import (

"github.com/containerd/containerd"
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/content/local"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/metadata"
"github.com/containerd/containerd/platforms"
"github.com/containerd/containerd/remotes"
"github.com/goharbor/acceleration-service/pkg/remote"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
)

type LocalProvider struct {
Expand All @@ -52,19 +49,7 @@ func NewLocalProvider(
if err := os.MkdirAll(contentDir, 0755); err != nil {
return nil, nil, errors.Wrap(err, "create local provider work directory")
}
store, err := local.NewLabeledStore(contentDir, newMemoryLabelStore())
if err != nil {
return nil, nil, errors.Wrap(err, "create local provider content store")
}
bdb, err := bolt.Open(filepath.Join(workDir, "meta.db"), 0655, nil)
if err != nil {
return nil, nil, errors.Wrap(err, "create local provider database")
}
db := metadata.NewDB(bdb, store, nil)
if err := db.Init(context.Background()); err != nil {
return nil, nil, err
}
content, err := NewContent(db, threshold)
content, err := NewContent(contentDir, workDir, threshold)
if err != nil {
return nil, nil, errors.Wrap(err, "create local provider content")
}
Expand Down Expand Up @@ -100,7 +85,7 @@ func (pvd *LocalProvider) Pull(ctx context.Context, ref string) error {
PlatformMatcher: pvd.platformMC,
}

img, err := fetch(ctx, pvd.ContentStore(), rc, ref, 0, pvd.content)
img, err := fetch(ctx, pvd.ContentStore(), rc, ref, 0)
if err != nil {
return errors.Wrap(err, "pull source image")
}
Expand Down
8 changes: 4 additions & 4 deletions pkg/content/ported.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ var fetchSingleflight = &singleflight.Group{}

// Ported from containerd project, copyright The containerd Authors.
// github.com/containerd/containerd/blob/main/pull.go
func fetch(ctx context.Context, store content.Store, rCtx *containerd.RemoteContext, ref string, limit int, content *Content) (images.Image, error) {
func fetch(ctx context.Context, store content.Store, rCtx *containerd.RemoteContext, ref string, limit int) (images.Image, error) {
name, desc, err := rCtx.Resolver.Resolve(ctx, ref)
if err != nil {
return images.Image{}, fmt.Errorf("failed to resolve reference %q: %w", ref, err)
Expand Down Expand Up @@ -107,7 +107,7 @@ func fetch(ctx context.Context, store content.Store, rCtx *containerd.RemoteCont
}

handlers := append(rCtx.BaseHandlers,
fetchHandler(store, fetcher, content),
fetchHandler(store, fetcher),
convertibleHandler,
childrenHandler,
appendDistSrcLabelHandler,
Expand Down Expand Up @@ -147,7 +147,7 @@ func fetch(ctx context.Context, store content.Store, rCtx *containerd.RemoteCont

// Ported from containerd project, copyright The containerd Authors.
// https://github.com/containerd/containerd/blob/main/remotes/handlers.go
func fetchHandler(ingester content.Ingester, fetcher remotes.Fetcher, content *Content) images.HandlerFunc {
func fetchHandler(ingester content.Ingester, fetcher remotes.Fetcher) images.HandlerFunc {
return func(ctx context.Context, desc ocispec.Descriptor) (subdescs []ocispec.Descriptor, err error) {
ctx = log.WithLogger(ctx, log.G(ctx).WithFields(log.Fields{
"digest": desc.Digest,
Expand All @@ -163,7 +163,7 @@ func fetchHandler(ingester content.Ingester, fetcher remotes.Fetcher, content *C
return nil, remotes.Fetch(ctx, ingester, fetcher, desc)
})
if errdefs.IsAlreadyExists(err) {
return nil, content.UpdateTime(&desc.Digest)
return nil, nil
}
return nil, err
}
Expand Down
84 changes: 84 additions & 0 deletions pkg/content/store.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package content

import (
"context"

containerdContent "github.com/containerd/containerd/content"
"github.com/containerd/containerd/content/local"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)

// Store wrap the content store to complete custom feature
type Store struct {
// store is content store
store containerdContent.Store
// content is related to database
content *Content
}

// newStore returns a content store
func newStore(contentDir string) (*Store, error) {
store, err := local.NewLabeledStore(contentDir, newMemoryLabelStore())
return &Store{
store: store,
}, err
}

func (store *Store) Init(content *Content) {
store.content = content
}

func (store *Store) Info(ctx context.Context, dgst digest.Digest) (containerdContent.Info, error) {
return store.store.Info(ctx, dgst)
}

func (store *Store) Update(ctx context.Context, info containerdContent.Info, fieldpaths ...string) (containerdContent.Info, error) {
return store.store.Update(ctx, info, fieldpaths...)
}

func (store *Store) Walk(ctx context.Context, fn containerdContent.WalkFunc, filters ...string) error {
return store.store.Walk(ctx, fn, filters...)
}

func (store *Store) Delete(ctx context.Context, dgst digest.Digest) error {
return store.store.Delete(ctx, dgst)
}

func (store *Store) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (containerdContent.ReaderAt, error) {
readerAt, err := store.store.ReaderAt(ctx, desc)
if err != nil {
return readerAt, err
}
return readerAt, store.content.UpdateTime(&desc.Digest)
}

func (store *Store) Status(ctx context.Context, ref string) (containerdContent.Status, error) {
return store.store.Status(ctx, ref)
}

func (store *Store) ListStatuses(ctx context.Context, filters ...string) ([]containerdContent.Status, error) {
return store.store.ListStatuses(ctx, filters...)
}

func (store *Store) Abort(ctx context.Context, ref string) error {
return store.store.Abort(ctx, ref)
}

func (store *Store) Writer(ctx context.Context, opts ...containerdContent.WriterOpt) (containerdContent.Writer, error) {
return store.store.Writer(ctx, opts...)
}

0 comments on commit 5acde6c

Please sign in to comment.