From aa96fc9d27a6b1c41aebfce0c8efa0f850a40349 Mon Sep 17 00:00:00 2001 From: bufdev <4228796+bufdev@users.noreply.github.com> Date: Thu, 29 Jun 2023 12:27:55 -0400 Subject: [PATCH] Add buf beta graph (#2151) --- private/buf/bufcli/bufcli.go | 48 ++-- private/buf/cmd/buf/buf.go | 2 + .../buf/cmd/buf/command/beta/graph/graph.go | 186 +++++++++++++++ .../cmd/buf/command/beta/graph/usage.gen.go | 19 ++ private/bufpkg/bufgraph/bufgraph.go | 88 ++++++++ private/bufpkg/bufgraph/builder.go | 213 ++++++++++++++++++ private/bufpkg/bufgraph/usage.gen.go | 19 ++ 7 files changed, 551 insertions(+), 24 deletions(-) create mode 100644 private/buf/cmd/buf/command/beta/graph/graph.go create mode 100644 private/buf/cmd/buf/command/beta/graph/usage.gen.go create mode 100644 private/bufpkg/bufgraph/bufgraph.go create mode 100644 private/bufpkg/bufgraph/builder.go create mode 100644 private/bufpkg/bufgraph/usage.gen.go diff --git a/private/buf/bufcli/bufcli.go b/private/buf/bufcli/bufcli.go index b53e5d5ea8..b92acfe6f6 100644 --- a/private/buf/bufcli/bufcli.go +++ b/private/buf/bufcli/bufcli.go @@ -413,7 +413,7 @@ func NewWireImageConfigReader( return bufwire.NewImageConfigReader( logger, storageosProvider, - newFetchReader(logger, storageosProvider, runner, moduleResolver, moduleReader), + NewFetchReader(logger, storageosProvider, runner, moduleResolver, moduleReader), bufmodulebuild.NewModuleBucketBuilder(), bufimagebuild.NewBuilder(logger, moduleReader), ), nil @@ -438,7 +438,7 @@ func NewWireModuleConfigReader( return bufwire.NewModuleConfigReader( logger, storageosProvider, - newFetchReader(logger, storageosProvider, runner, moduleResolver, moduleReader), + NewFetchReader(logger, storageosProvider, runner, moduleResolver, moduleReader), bufmodulebuild.NewModuleBucketBuilder(), ), nil } @@ -460,7 +460,7 @@ func NewWireModuleConfigReaderForModuleReader( return bufwire.NewModuleConfigReader( logger, storageosProvider, - newFetchReader(logger, storageosProvider, runner, moduleResolver, moduleReader), + NewFetchReader(logger, storageosProvider, runner, moduleResolver, moduleReader), bufmodulebuild.NewModuleBucketBuilder(), ), nil } @@ -484,7 +484,7 @@ func NewWireFileLister( return bufwire.NewFileLister( logger, storageosProvider, - newFetchReader(logger, storageosProvider, runner, moduleResolver, moduleReader), + NewFetchReader(logger, storageosProvider, runner, moduleResolver, moduleReader), bufmodulebuild.NewModuleBucketBuilder(), bufimagebuild.NewBuilder(logger, moduleReader), ), nil @@ -638,6 +638,26 @@ func NewConnectClientConfigWithToken(container appflag.Container, token string) ) } +// NewFetchReader creates a new buffetch.Reader with the default HTTP client +// and git cloner. +func NewFetchReader( + logger *zap.Logger, + storageosProvider storageos.Provider, + runner command.Runner, + moduleResolver bufmodule.ModuleResolver, + moduleReader bufmodule.ModuleReader, +) buffetch.Reader { + return buffetch.NewReader( + logger, + storageosProvider, + defaultHTTPClient, + defaultHTTPAuthenticator, + git.NewCloner(logger, storageosProvider, runner, defaultGitClonerOptions), + moduleResolver, + moduleReader, + ) +} + // PromptUserForDelete is used to receive user confirmation that a specific // entity should be deleted. If the user's answer does not match the expected // answer, an error is returned. @@ -983,26 +1003,6 @@ func promptUser(container app.Container, prompt string, isPassword bool) (string return "", NewTooManyEmptyAnswersError(userPromptAttempts) } -// newFetchReader creates a new buffetch.Reader with the default HTTP client -// and git cloner. -func newFetchReader( - logger *zap.Logger, - storageosProvider storageos.Provider, - runner command.Runner, - moduleResolver bufmodule.ModuleResolver, - moduleReader bufmodule.ModuleReader, -) buffetch.Reader { - return buffetch.NewReader( - logger, - storageosProvider, - defaultHTTPClient, - defaultHTTPAuthenticator, - git.NewCloner(logger, storageosProvider, runner, defaultGitClonerOptions), - moduleResolver, - moduleReader, - ) -} - // newFetchSourceReader creates a new buffetch.SourceReader with the default HTTP client // and git cloner. func newFetchSourceReader( diff --git a/private/buf/cmd/buf/buf.go b/private/buf/cmd/buf/buf.go index 4062c873e1..7a8de4f827 100644 --- a/private/buf/cmd/buf/buf.go +++ b/private/buf/cmd/buf/buf.go @@ -29,6 +29,7 @@ import ( "github.com/bufbuild/buf/private/buf/cmd/buf/command/alpha/registry/token/tokenlist" "github.com/bufbuild/buf/private/buf/cmd/buf/command/alpha/repo/reposync" "github.com/bufbuild/buf/private/buf/cmd/buf/command/alpha/workspace/workspacepush" + "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/graph" "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/migratev1beta1" "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/price" "github.com/bufbuild/buf/private/buf/cmd/buf/command/beta/registry/commit/commitget" @@ -134,6 +135,7 @@ func NewRootCommand(name string) *appcmd.Command { Use: "beta", Short: "Beta commands. Unstable and likely to change", SubCommands: []*appcmd.Command{ + graph.NewCommand("graph", builder), price.NewCommand("price", builder), stats.NewCommand("stats", builder), migratev1beta1.NewCommand("migrate-v1beta1", builder), diff --git a/private/buf/cmd/buf/command/beta/graph/graph.go b/private/buf/cmd/buf/command/beta/graph/graph.go new file mode 100644 index 0000000000..4bb34c5403 --- /dev/null +++ b/private/buf/cmd/buf/command/beta/graph/graph.go @@ -0,0 +1,186 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +import ( + "context" + "fmt" + + "github.com/bufbuild/buf/private/buf/bufcli" + "github.com/bufbuild/buf/private/buf/buffetch" + "github.com/bufbuild/buf/private/buf/bufwire" + "github.com/bufbuild/buf/private/bufpkg/bufanalysis" + "github.com/bufbuild/buf/private/bufpkg/bufapimodule" + "github.com/bufbuild/buf/private/bufpkg/bufgraph" + "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmodulebuild" + "github.com/bufbuild/buf/private/pkg/app/appcmd" + "github.com/bufbuild/buf/private/pkg/app/appflag" + "github.com/bufbuild/buf/private/pkg/command" + "github.com/bufbuild/buf/private/pkg/stringutil" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +const ( + errorFormatFlagName = "error-format" + configFlagName = "config" + disableSymlinksFlagName = "disable-symlinks" +) + +// NewCommand returns a new Command. +func NewCommand( + name string, + builder appflag.Builder, +) *appcmd.Command { + flags := newFlags() + return &appcmd.Command{ + Use: name + " ", + Short: "Print the dependency graph in DOT format", + Long: bufcli.GetSourceOrModuleLong(`the source or module to print for`), + Args: cobra.MaximumNArgs(1), + Run: builder.NewRunFunc( + func(ctx context.Context, container appflag.Container) error { + return run(ctx, container, flags) + }, + bufcli.NewErrorInterceptor(), + ), + BindFlags: flags.Bind, + } +} + +type flags struct { + ErrorFormat string + Config string + DisableSymlinks bool + // special + InputHashtag string +} + +func newFlags() *flags { + return &flags{} +} + +func (f *flags) Bind(flagSet *pflag.FlagSet) { + bufcli.BindInputHashtag(flagSet, &f.InputHashtag) + bufcli.BindDisableSymlinks(flagSet, &f.DisableSymlinks, disableSymlinksFlagName) + flagSet.StringVar( + &f.ErrorFormat, + errorFormatFlagName, + "text", + fmt.Sprintf( + "The format for build errors printed to stderr. Must be one of %s", + stringutil.SliceToString(bufanalysis.AllFormatStrings), + ), + ) + flagSet.StringVar( + &f.Config, + configFlagName, + "", + `The file or data to use to use for configuration`, + ) +} + +func run( + ctx context.Context, + container appflag.Container, + flags *flags, +) error { + if err := bufcli.ValidateErrorFormatFlag(flags.ErrorFormat, errorFormatFlagName); err != nil { + return err + } + input, err := bufcli.GetInputValue(container, flags.InputHashtag, ".") + if err != nil { + return err + } + sourceOrModuleRef, err := buffetch.NewRefParser(container.Logger()).GetSourceOrModuleRef(ctx, input) + if err != nil { + return err + } + storageosProvider := bufcli.NewStorageosProvider(flags.DisableSymlinks) + runner := command.NewRunner() + clientConfig, err := bufcli.NewConnectClientConfig(container) + if err != nil { + return err + } + moduleResolver := bufapimodule.NewModuleResolver( + container.Logger(), + bufapimodule.NewRepositoryCommitServiceClientFactory(clientConfig), + ) + moduleReader, err := bufcli.NewModuleReaderAndCreateCacheDirs(container, clientConfig) + if err != nil { + return err + } + moduleConfigReader := bufwire.NewModuleConfigReader( + container.Logger(), + storageosProvider, + bufcli.NewFetchReader(container.Logger(), storageosProvider, runner, moduleResolver, moduleReader), + bufmodulebuild.NewModuleBucketBuilder(), + ) + if err != nil { + return err + } + graphBuilder := bufgraph.NewBuilder( + container.Logger(), + moduleResolver, + moduleReader, + ) + moduleConfigSet, err := moduleConfigReader.GetModuleConfigSet( + ctx, + container, + sourceOrModuleRef, + flags.Config, + nil, + nil, + false, + ) + if err != nil { + return err + } + moduleConfigs := moduleConfigSet.ModuleConfigs() + modules := make([]bufmodule.Module, len(moduleConfigs)) + for i, moduleConfig := range moduleConfigs { + modules[i] = moduleConfig.Module() + } + graph, fileAnnotations, err := graphBuilder.Build( + ctx, + modules, + bufgraph.BuildWithWorkspace(moduleConfigSet.Workspace()), + ) + if err != nil { + return err + } + if len(fileAnnotations) > 0 { + // stderr since we do output to stdout potentially + if err := bufanalysis.PrintFileAnnotations( + container.Stderr(), + fileAnnotations, + flags.ErrorFormat, + ); err != nil { + return err + } + return bufcli.ErrFileAnnotation + } + dotString, err := graph.DOTString( + func(node bufgraph.Node) string { + return node.String() + }, + ) + if err != nil { + return err + } + _, err = fmt.Fprintln(container.Stdout(), dotString) + return err +} diff --git a/private/buf/cmd/buf/command/beta/graph/usage.gen.go b/private/buf/cmd/buf/command/beta/graph/usage.gen.go new file mode 100644 index 0000000000..f674903503 --- /dev/null +++ b/private/buf/cmd/buf/command/beta/graph/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package graph + +import _ "github.com/bufbuild/buf/private/usage" diff --git a/private/bufpkg/bufgraph/bufgraph.go b/private/bufpkg/bufgraph/bufgraph.go new file mode 100644 index 0000000000..851134172b --- /dev/null +++ b/private/bufpkg/bufgraph/bufgraph.go @@ -0,0 +1,88 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufgraph + +import ( + "context" + + "github.com/bufbuild/buf/private/bufpkg/bufanalysis" + "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/pkg/dag" + "go.uber.org/zap" +) + +// Node is a node in a dependency graph. +// +// This is a struct because this needs to be comparable for the *dag.Graph. +// +// TODO: Don't have the duplication across Node and ImageModuleDependency. +type Node struct { + // Required, + Remote string + // Required. + Owner string + // Required. + Repository string + // Optional. Will not bet set for modules read from workspaces. + Commit string +} + +// IdentityString prints remote/owner/repository. +func (n *Node) IdentityString() string { + return n.Remote + "/" + n.Owner + "/" + n.Repository +} + +// String prints remote/owner/repository[:commit]. +func (n *Node) String() string { + s := n.IdentityString() + if n.Commit != "" { + return s + ":" + n.Commit + } + return s +} + +// Builder builds dependency graphs. +type Builder interface { + // Build builds the dependency graph. + Build( + ctx context.Context, + modules []bufmodule.Module, + options ...BuildOption, + ) (*dag.Graph[Node], []bufanalysis.FileAnnotation, error) +} + +// NewBuilder returns a new Builder. +func NewBuilder( + logger *zap.Logger, + moduleResolver bufmodule.ModuleResolver, + moduleReader bufmodule.ModuleReader, +) Builder { + return newBuilder( + logger, + moduleResolver, + moduleReader, + ) +} + +// BuildOption is an option for Build. +type BuildOption func(*buildOptions) + +// BuildWithWorkspace returns a new BuildOption that specifies a workspace +// that is being operated on. +func BuildWithWorkspace(workspace bufmodule.Workspace) BuildOption { + return func(buildOptions *buildOptions) { + buildOptions.workspace = workspace + } +} diff --git a/private/bufpkg/bufgraph/builder.go b/private/bufpkg/bufgraph/builder.go new file mode 100644 index 0000000000..891225ce1f --- /dev/null +++ b/private/bufpkg/bufgraph/builder.go @@ -0,0 +1,213 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bufgraph + +import ( + "context" + "fmt" + + "github.com/bufbuild/buf/private/bufpkg/bufanalysis" + "github.com/bufbuild/buf/private/bufpkg/bufimage" + "github.com/bufbuild/buf/private/bufpkg/bufimage/bufimagebuild" + "github.com/bufbuild/buf/private/bufpkg/bufmodule" + "github.com/bufbuild/buf/private/bufpkg/bufmodule/bufmoduleref" + "github.com/bufbuild/buf/private/pkg/dag" + "go.uber.org/zap" +) + +type builder struct { + logger *zap.Logger + moduleResolver bufmodule.ModuleResolver + moduleReader bufmodule.ModuleReader + imageBuilder bufimagebuild.Builder +} + +func newBuilder( + logger *zap.Logger, + moduleResolver bufmodule.ModuleResolver, + moduleReader bufmodule.ModuleReader, +) *builder { + return &builder{ + logger: logger, + moduleResolver: moduleResolver, + moduleReader: moduleReader, + imageBuilder: bufimagebuild.NewBuilder( + logger, + moduleReader, + ), + } +} + +func (b *builder) Build( + ctx context.Context, + modules []bufmodule.Module, + options ...BuildOption, +) (*dag.Graph[Node], []bufanalysis.FileAnnotation, error) { + buildOptions := newBuildOptions() + for _, option := range options { + option(buildOptions) + } + return b.build( + ctx, + modules, + buildOptions.workspace, + ) +} + +func (b *builder) build( + ctx context.Context, + modules []bufmodule.Module, + workspace bufmodule.Workspace, +) (*dag.Graph[Node], []bufanalysis.FileAnnotation, error) { + graph := dag.NewGraph[Node]() + for _, module := range modules { + fileAnnotations, err := b.buildForModule( + ctx, + module, + newNodeForModule(module), + workspace, + graph, + ) + if err != nil { + return nil, nil, err + } + if len(fileAnnotations) > 0 { + return nil, fileAnnotations, nil + } + } + return graph, nil, nil +} + +func (b *builder) buildForModule( + ctx context.Context, + module bufmodule.Module, + node Node, + workspace bufmodule.Workspace, + graph *dag.Graph[Node], +) ([]bufanalysis.FileAnnotation, error) { + image, fileAnnotations, err := b.imageBuilder.Build( + ctx, + module, + bufimagebuild.WithWorkspace(workspace), + ) + if err != nil { + return nil, err + } + if len(fileAnnotations) > 0 { + return fileAnnotations, nil + } + for _, imageModuleDependency := range bufimage.ImageModuleDependencies(image) { + dependencyNode := newNodeForImageModuleDependency(imageModuleDependency) + if imageModuleDependency.IsDirect() { + graph.AddEdge(node, dependencyNode) + } + dependencyModule, err := b.getModuleForImageModuleDependency( + ctx, + imageModuleDependency, + workspace, + ) + if err != nil { + return nil, err + } + // TODO: deal with the case where there are differing commits for a given ModuleIdentity. + fileAnnotations, err := b.buildForModule( + ctx, + dependencyModule, + dependencyNode, + workspace, + graph, + ) + if err != nil { + return nil, err + } + if len(fileAnnotations) > 0 { + return fileAnnotations, nil + } + } + return nil, nil +} + +func (b *builder) getModuleForImageModuleDependency( + ctx context.Context, + imageModuleDependency bufimage.ImageModuleDependency, + workspace bufmodule.Workspace, +) (bufmodule.Module, error) { + moduleIdentity := imageModuleDependency.ModuleIdentity() + commit := imageModuleDependency.Commit() + if workspace != nil { + module, ok := workspace.GetModule(moduleIdentity) + if ok { + return module, nil + } + } + if commit == "" { + // TODO: can we error here? The only + // case we should likely not have a commit is when we are using a workspace. + // There's no enforcement of this property, so erroring here is a bit weird, + // but it might be better to check our assumptions and figure out if there + // are exceptions after the fact, as opposed to resolving a ModulePin for + // main when we don't know if main is what we want. + return nil, fmt.Errorf("had ModuleIdentity %v with no associated commit, but did not have the module in a workspace", moduleIdentity) + } + moduleReference, err := bufmoduleref.NewModuleReference( + moduleIdentity.Remote(), + moduleIdentity.Owner(), + moduleIdentity.Repository(), + commit, + ) + if err != nil { + return nil, err + } + modulePin, err := b.moduleResolver.GetModulePin( + ctx, + moduleReference, + ) + if err != nil { + return nil, err + } + return b.moduleReader.GetModule( + ctx, + modulePin, + ) +} + +func newNodeForImageModuleDependency(imageModuleDependency bufimage.ImageModuleDependency) Node { + return Node{ + Remote: imageModuleDependency.ModuleIdentity().Remote(), + Owner: imageModuleDependency.ModuleIdentity().Owner(), + Repository: imageModuleDependency.ModuleIdentity().Repository(), + Commit: imageModuleDependency.Commit(), + } +} + +func newNodeForModule(module bufmodule.Module) Node { + // TODO: deal with unnamed Modules + var node Node + if moduleIdentity := module.ModuleIdentity(); moduleIdentity != nil { + node.Remote = moduleIdentity.Remote() + node.Owner = moduleIdentity.Owner() + node.Repository = moduleIdentity.Repository() + node.Commit = module.Commit() + } + return node +} + +type buildOptions struct { + workspace bufmodule.Workspace +} + +func newBuildOptions() *buildOptions { + return &buildOptions{} +} diff --git a/private/bufpkg/bufgraph/usage.gen.go b/private/bufpkg/bufgraph/usage.gen.go new file mode 100644 index 0000000000..e9c1d1b315 --- /dev/null +++ b/private/bufpkg/bufgraph/usage.gen.go @@ -0,0 +1,19 @@ +// Copyright 2020-2023 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated. DO NOT EDIT. + +package bufgraph + +import _ "github.com/bufbuild/buf/private/usage"