Skip to content

Commit e065fbb

Browse files
emcfarlaneemcfarlane
authored
Add support for local Wasm plugins (#3349)
1 parent 23c7df4 commit e065fbb

File tree

37 files changed

+865
-49
lines changed

37 files changed

+865
-49
lines changed

‎CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
they satisfy the field constraints, and are only present if constraints are present.
77
- Update the `PROTOVALIDATE` lint rule to check predefined rules. Predefined rules will be checked
88
that they compile.
9+
- Add support for a WebAssembly (Wasm) runtime for custom lint and breaking changes plugins. Use the
10+
`.wasm` file extension to specify a path to a Wasm plugin.
911

1012
## [v1.43.0] - 2024-09-30
1113

‎go.mod

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ require (
3535
github.com/spf13/cobra v1.8.1
3636
github.com/spf13/pflag v1.0.5
3737
github.com/stretchr/testify v1.9.0
38+
github.com/tetratelabs/wazero v1.8.0
3839
go.lsp.dev/jsonrpc2 v0.10.0
3940
go.lsp.dev/protocol v0.12.0
4041
go.opentelemetry.io/otel v1.30.0

‎go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -252,6 +252,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
252252
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
253253
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
254254
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
255+
github.com/tetratelabs/wazero v1.8.0 h1:iEKu0d4c2Pd+QSRieYbnQC9yiFlMS9D+Jr0LsRmcF4g=
256+
github.com/tetratelabs/wazero v1.8.0/go.mod h1:yAI0XTsMBhREkM/YDAK/zNou3GoiAce1P6+rp/wQhjs=
255257
github.com/vbatts/tar-split v0.11.6 h1:4SjTW5+PU11n6fZenf2IPoV8/tz3AaYHMWjf23envGs=
256258
github.com/vbatts/tar-split v0.11.6/go.mod h1:dqKNtesIOr2j2Qv3W/cHjnvk9I8+G7oAkFDFN6TCBEI=
257259
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

‎make/buf/all.mk

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ GO_TEST_BINS := $(GO_TEST_BINS) \
2424
private/bufpkg/bufcheck/internal/cmd/buf-plugin-rpc-ext \
2525
private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-category \
2626
private/bufpkg/bufcheck/internal/cmd/buf-plugin-duplicate-rule
27+
GO_TEST_WASM_BINS := $(GO_TEST_WASM_BINS) \
28+
private/bufpkg/bufcheck/internal/cmd/buf-plugin-suffix
2729
GO_MOD_VERSION := 1.22
2830
DOCKER_BINS := $(DOCKER_BINS) buf
2931
FILE_IGNORES := $(FILE_IGNORES) \

‎make/go/go.mk

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ GO_BINS ?=
1616
# Settable
1717
GO_TEST_BINS ?=
1818
# Settable
19+
GO_TEST_WASM_BINS ?=
20+
# Settable
1921
GO_GET_PKGS ?=
2022
# Settable
2123
GO_MOD_VERSION ?= 1.22
@@ -142,7 +144,7 @@ build: prebuild ## Run go build.
142144
pretest::
143145

144146
.PHONY: test
145-
test: pretest installtest ## Run all go tests.
147+
test: pretest installtest installtestwasm ## Run all go tests.
146148
go test $(GO_TEST_FLAGS) $(GOPKGS)
147149

148150
.PHONY: testrace
@@ -203,3 +205,17 @@ endef
203205

204206
$(foreach gobin,$(sort $(GO_TEST_BINS)),$(eval $(call gotestbinfunc,$(gobin))))
205207
$(foreach gobin,$(sort $(GO_TEST_BINS)),$(eval FILE_IGNORES := $(FILE_IGNORES) $(gobin)/$(notdir $(gobin))))
208+
209+
.PHONY: installtestwasm
210+
installtestwasm::
211+
212+
define gotestwasmfunc
213+
.PHONY: installtestwasm$(notdir $(1))
214+
installtestwasm$(notdir $(1)):
215+
GOOS=wasip1 GOARCH=wasm go build -o $(GOBIN)/$(notdir $(1)).wasm ./$(1)
216+
217+
installtestwasm:: installtestwasm$(notdir $(1))
218+
endef
219+
220+
$(foreach gobin,$(sort $(GO_TEST_WASM_BINS)),$(eval $(call gotestwasmfunc,$(gobin))))
221+
$(foreach gobin,$(sort $(GO_TEST_WASM_BINS)),$(eval FILE_IGNORES := $(FILE_IGNORES) $(gobin)/$(notdir $(gobin))))

‎private/buf/bufcli/cache.go

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,11 @@ var (
103103
//
104104
// Normalized.
105105
v3CacheModuleLockRelDirPath = normalpath.Join("v3", "modulelocks")
106+
// v3CacheWasmRuntimeRelDirPath is the relative path to the Wasm runtime cache directory in its newest iteration.
107+
// This directory is used to store the Wasm runtime cache. This is an implementation specific cache and opaque outside of the runtime.
108+
//
109+
// Normalized.
110+
v3CacheWasmRuntimeRelDirPath = normalpath.Join("v3", "wasmruntime")
106111
)
107112

108113
// NewModuleDataProvider returns a new ModuleDataProvider while creating the
@@ -135,6 +140,19 @@ func NewCommitProvider(container appext.Container) (bufmodule.CommitProvider, er
135140
)
136141
}
137142

143+
// CreateWasmRuntimeCacheDir creates the cache directory for the Wasm runtime.
144+
//
145+
// This is used by the Wasm runtime to cache compiled Wasm plugins. This is an
146+
// implementation specific cache and opaque outside of the runtime. The runtime
147+
// will manage the cache versioning itself within this directory.
148+
func CreateWasmRuntimeCacheDir(container appext.Container) (string, error) {
149+
if err := createCacheDir(container.CacheDirPath(), v3CacheWasmRuntimeRelDirPath); err != nil {
150+
return "", err
151+
}
152+
fullCacheDirPath := normalpath.Join(container.CacheDirPath(), v3CacheWasmRuntimeRelDirPath)
153+
return fullCacheDirPath, nil
154+
}
155+
138156
// newWKTStore returns a new bufwktstore.Store while creating the required cache directories.
139157
func newWKTStore(container appext.Container) (bufwktstore.Store, error) {
140158
if err := createCacheDir(container.CacheDirPath(), v3CacheWKTRelDirPath); err != nil {

‎private/buf/buflsp/buflsp.go

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import (
2626
"github.com/bufbuild/buf/private/bufpkg/bufcheck"
2727
"github.com/bufbuild/buf/private/bufpkg/bufimage"
2828
"github.com/bufbuild/buf/private/pkg/app/appext"
29-
"github.com/bufbuild/buf/private/pkg/command"
3029
"github.com/bufbuild/buf/private/pkg/storage"
3130
"github.com/bufbuild/buf/private/pkg/storage/storageos"
3231
"github.com/bufbuild/buf/private/pkg/tracing"
@@ -43,6 +42,7 @@ func Serve(
4342
ctx context.Context,
4443
container appext.Container,
4544
controller bufctl.Controller,
45+
checkClient bufcheck.Client,
4646
stream jsonrpc2.Stream,
4747
) (jsonrpc2.Conn, error) {
4848
// The LSP protocol deals with absolute filesystem paths. This requires us to
@@ -57,12 +57,6 @@ func Serve(
5757
return nil, err
5858
}
5959

60-
tracer := tracing.NewTracer(container.Tracer())
61-
checkClient, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr()))
62-
if err != nil {
63-
return nil, err
64-
}
65-
6660
conn := jsonrpc2.NewConn(stream)
6761
lsp := &lsp{
6862
conn: conn,
@@ -71,7 +65,7 @@ func Serve(
7165
zap.NewNop(), // The logging from protocol itself isn't very good, we've replaced it with connAdapter here.
7266
),
7367
logger: container.Logger(),
74-
tracer: tracer,
68+
tracer: tracing.NewTracer(container.Tracer()),
7569
controller: controller,
7670
checkClient: checkClient,
7771
rootBucket: bucket,

‎private/buf/bufmigrate/migrator.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"github.com/bufbuild/buf/private/pkg/storage"
3333
"github.com/bufbuild/buf/private/pkg/storage/storagemem"
3434
"github.com/bufbuild/buf/private/pkg/tracing"
35+
"github.com/bufbuild/buf/private/pkg/wasm"
3536
"github.com/google/uuid"
3637
"go.uber.org/multierr"
3738
"go.uber.org/zap"
@@ -712,7 +713,7 @@ func equivalentCheckConfigInV2(
712713
) (bufconfig.CheckConfig, error) {
713714
// No need for custom lint/breaking plugins since there's no plugins to migrate from <=v1.
714715
// TODO: If we ever need v3, then we will have to deal with this.
715-
client, err := bufcheck.NewClient(logger, tracer, bufcheck.NewRunnerProvider(runner))
716+
client, err := bufcheck.NewClient(logger, tracer, bufcheck.NewRunnerProvider(runner, wasm.UnimplementedRuntime))
716717
if err != nil {
717718
return nil, err
718719
}

‎private/buf/cmd/buf/buf_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import (
4545
"github.com/bufbuild/buf/private/pkg/storage/storageos"
4646
"github.com/bufbuild/buf/private/pkg/storage/storagetesting"
4747
"github.com/bufbuild/buf/private/pkg/tracing"
48+
"github.com/bufbuild/buf/private/pkg/wasm"
4849
"github.com/stretchr/testify/assert"
4950
"github.com/stretchr/testify/require"
5051
"go.uber.org/zap"
@@ -1349,7 +1350,7 @@ func TestCheckLsBreakingRulesFromConfigExceptDeprecated(t *testing.T) {
13491350
t.Run(version.String(), func(t *testing.T) {
13501351
t.Parallel()
13511352
// Do not need any custom lint/breaking plugins here.
1352-
client, err := bufcheck.NewClient(zap.NewNop(), tracing.NopTracer, bufcheck.NewRunnerProvider(command.NewRunner()))
1353+
client, err := bufcheck.NewClient(zap.NewNop(), tracing.NopTracer, bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime))
13531354
require.NoError(t, err)
13541355
allRules, err := client.AllRules(context.Background(), check.RuleTypeBreaking, version)
13551356
require.NoError(t, err)

‎private/buf/cmd/buf/command/beta/lsp/lsp.go

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,16 @@ import (
2525

2626
"github.com/bufbuild/buf/private/buf/bufcli"
2727
"github.com/bufbuild/buf/private/buf/buflsp"
28+
"github.com/bufbuild/buf/private/bufpkg/bufcheck"
2829
"github.com/bufbuild/buf/private/pkg/app/appcmd"
2930
"github.com/bufbuild/buf/private/pkg/app/appext"
31+
"github.com/bufbuild/buf/private/pkg/command"
3032
"github.com/bufbuild/buf/private/pkg/ioext"
33+
"github.com/bufbuild/buf/private/pkg/tracing"
34+
"github.com/bufbuild/buf/private/pkg/wasm"
3135
"github.com/spf13/pflag"
3236
"go.lsp.dev/jsonrpc2"
37+
"go.uber.org/multierr"
3338
)
3439

3540
const (
@@ -77,7 +82,7 @@ func run(
7782
ctx context.Context,
7883
container appext.Container,
7984
flags *flags,
80-
) error {
85+
) (retErr error) {
8186
bufcli.WarnBetaCommand(ctx, container)
8287

8388
transport, err := dial(container, flags)
@@ -90,7 +95,28 @@ func run(
9095
return err
9196
}
9297

93-
conn, err := buflsp.Serve(ctx, container, controller, jsonrpc2.NewStream(transport))
98+
wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container)
99+
if err != nil {
100+
return err
101+
}
102+
wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir))
103+
if err != nil {
104+
return err
105+
}
106+
defer func() {
107+
retErr = multierr.Append(retErr, wasmRuntime.Close(ctx))
108+
}()
109+
checkClient, err := bufcheck.NewClient(
110+
container.Logger(),
111+
tracing.NewTracer(container.Tracer()),
112+
bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime),
113+
bufcheck.ClientWithStderr(container.Stderr()),
114+
)
115+
if err != nil {
116+
return err
117+
}
118+
119+
conn, err := buflsp.Serve(ctx, container, controller, checkClient, jsonrpc2.NewStream(transport))
94120
if err != nil {
95121
return err
96122
}

‎private/buf/cmd/buf/command/breaking/breaking.go

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@ import (
3131
"github.com/bufbuild/buf/private/pkg/slicesext"
3232
"github.com/bufbuild/buf/private/pkg/stringutil"
3333
"github.com/bufbuild/buf/private/pkg/tracing"
34+
"github.com/bufbuild/buf/private/pkg/wasm"
3435
"github.com/spf13/pflag"
36+
"go.uber.org/multierr"
3537
)
3638

3739
const (
@@ -145,7 +147,7 @@ func run(
145147
ctx context.Context,
146148
container appext.Container,
147149
flags *flags,
148-
) error {
150+
) (retErr error) {
149151
if err := bufcli.ValidateRequiredFlag(againstFlagName, flags.Against); err != nil {
150152
return err
151153
}
@@ -206,10 +208,26 @@ func run(
206208
len(againstImageWithConfigs),
207209
)
208210
}
211+
wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container)
212+
if err != nil {
213+
return err
214+
}
215+
wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir))
216+
if err != nil {
217+
return err
218+
}
219+
defer func() {
220+
retErr = multierr.Append(retErr, wasmRuntime.Close(ctx))
221+
}()
209222
tracer := tracing.NewTracer(container.Tracer())
210223
var allFileAnnotations []bufanalysis.FileAnnotation
211224
for i, imageWithConfig := range imageWithConfigs {
212-
client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr()))
225+
client, err := bufcheck.NewClient(
226+
container.Logger(),
227+
tracer,
228+
bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime),
229+
bufcheck.ClientWithStderr(container.Stderr()),
230+
)
213231
if err != nil {
214232
return err
215233
}

‎private/buf/cmd/buf/command/config/internal/internal.go

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@ import (
3232
"github.com/bufbuild/buf/private/pkg/stringutil"
3333
"github.com/bufbuild/buf/private/pkg/syserror"
3434
"github.com/bufbuild/buf/private/pkg/tracing"
35+
"github.com/bufbuild/buf/private/pkg/wasm"
3536
"github.com/spf13/pflag"
37+
"go.uber.org/multierr"
3638
)
3739

3840
const (
@@ -149,7 +151,7 @@ func lsRun(
149151
flags *flags,
150152
commandName string,
151153
ruleType check.RuleType,
152-
) error {
154+
) (retErr error) {
153155
if flags.ConfiguredOnly {
154156
if flags.Version != "" {
155157
return appcmd.NewInvalidArgumentErrorf("--%s cannot be specified if --%s is specified", versionFlagName, configFlagName)
@@ -184,8 +186,24 @@ func lsRun(
184186
return err
185187
}
186188
}
189+
wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container)
190+
if err != nil {
191+
return err
192+
}
193+
wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir))
194+
if err != nil {
195+
return err
196+
}
197+
defer func() {
198+
retErr = multierr.Append(retErr, wasmRuntime.Close(ctx))
199+
}()
187200
tracer := tracing.NewTracer(container.Tracer())
188-
client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr()))
201+
client, err := bufcheck.NewClient(
202+
container.Logger(),
203+
tracer,
204+
bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime),
205+
bufcheck.ClientWithStderr(container.Stderr()),
206+
)
189207
if err != nil {
190208
return err
191209
}

‎private/buf/cmd/buf/command/lint/lint.go

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,9 @@ import (
2828
"github.com/bufbuild/buf/private/pkg/command"
2929
"github.com/bufbuild/buf/private/pkg/stringutil"
3030
"github.com/bufbuild/buf/private/pkg/tracing"
31+
"github.com/bufbuild/buf/private/pkg/wasm"
3132
"github.com/spf13/pflag"
33+
"go.uber.org/multierr"
3234
)
3335

3436
const (
@@ -131,10 +133,26 @@ func run(
131133
if err != nil {
132134
return err
133135
}
136+
wasmRuntimeCacheDir, err := bufcli.CreateWasmRuntimeCacheDir(container)
137+
if err != nil {
138+
return err
139+
}
140+
wasmRuntime, err := wasm.NewRuntime(ctx, wasm.WithLocalCacheDir(wasmRuntimeCacheDir))
141+
if err != nil {
142+
return err
143+
}
144+
defer func() {
145+
retErr = multierr.Append(retErr, wasmRuntime.Close(ctx))
146+
}()
134147
tracer := tracing.NewTracer(container.Tracer())
135148
var allFileAnnotations []bufanalysis.FileAnnotation
136149
for _, imageWithConfig := range imageWithConfigs {
137-
client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr()))
150+
client, err := bufcheck.NewClient(
151+
container.Logger(),
152+
tracer,
153+
bufcheck.NewRunnerProvider(command.NewRunner(), wasmRuntime),
154+
bufcheck.ClientWithStderr(container.Stderr()),
155+
)
138156
if err != nil {
139157
return err
140158
}

‎private/buf/cmd/buf/command/mod/internal/internal.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"github.com/bufbuild/buf/private/pkg/stringutil"
3232
"github.com/bufbuild/buf/private/pkg/syserror"
3333
"github.com/bufbuild/buf/private/pkg/tracing"
34+
"github.com/bufbuild/buf/private/pkg/wasm"
3435
"github.com/spf13/pflag"
3536
)
3637

@@ -175,7 +176,12 @@ func lsRun(
175176
}
176177
// BufYAMLFiles <=v1 never had plugins.
177178
tracer := tracing.NewTracer(container.Tracer())
178-
client, err := bufcheck.NewClient(container.Logger(), tracer, bufcheck.NewRunnerProvider(command.NewRunner()), bufcheck.ClientWithStderr(container.Stderr()))
179+
client, err := bufcheck.NewClient(
180+
container.Logger(),
181+
tracer,
182+
bufcheck.NewRunnerProvider(command.NewRunner(), wasm.UnimplementedRuntime),
183+
bufcheck.ClientWithStderr(container.Stderr()),
184+
)
179185
if err != nil {
180186
return err
181187
}

0 commit comments

Comments
 (0)