diff --git a/go.mod b/go.mod index 347dc7b1c..7ee2f735b 100644 --- a/go.mod +++ b/go.mod @@ -54,6 +54,7 @@ require ( github.com/osteele/liquid v1.3.0 github.com/pelletier/go-toml v1.9.4 github.com/pivotal/image-relocation v0.0.0-20191111101224-e94aff6df06c + github.com/pkg/errors v0.9.1 github.com/spf13/afero v1.5.1 github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 @@ -173,7 +174,6 @@ require ( github.com/opencontainers/runc v1.1.2 // indirect github.com/osteele/tuesday v1.0.3 // indirect github.com/pierrec/lz4/v4 v4.0.3 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.12.1 // indirect github.com/prometheus/client_model v0.2.0 // indirect diff --git a/go1.18.3.linux-amd64.tar.gz b/go1.18.3.linux-amd64.tar.gz new file mode 100644 index 000000000..b830f85f5 Binary files /dev/null and b/go1.18.3.linux-amd64.tar.gz differ diff --git a/pkg/cnab/config-adapter/adapter.go b/pkg/cnab/config-adapter/adapter.go index 6991ec9e1..fefe0c091 100644 --- a/pkg/cnab/config-adapter/adapter.go +++ b/pkg/cnab/config-adapter/adapter.go @@ -3,6 +3,7 @@ package configadapter import ( "context" "fmt" + "os" "path" "strings" @@ -80,7 +81,6 @@ func (c *ManifestConverter) ToBundle(ctx context.Context) (cnab.ExtendedBundle, } b.Custom = custom b.RequiredExtensions = c.generateRequiredExtensions(b) - b.Custom[config.CustomPorterKey] = stamp return b, nil @@ -203,8 +203,11 @@ func (c *ManifestConverter) generateBundleParameters(ctx context.Context, defs * } if param.Type == nil { - // Default to a file type if the param is stored in a file - if param.Destination.Path != "" { + // Default to a directory type if the param is a directory + if param.Destination.Path != "" && strings.HasSuffix(param.Destination.Path, string(os.PathSeparator)) { + param.Type = "directory" + } else if param.Destination.Path != "" { + // If the path could refer to a file assume that it does unless specified explicity param.Type = "file" } else { // Assume it's a string otherwise @@ -298,12 +301,8 @@ func (c *ManifestConverter) addDefinition(name string, kind string, def definiti defName = name + "-" + kind } - // file is a porter specific type, swap it out for something CNAB understands - if def.Type == "file" { - def.Type = "string" - def.ContentEncoding = "base64" - } - + // Type may be a porter specific type, swap it out for something CNAB understands + MakeCNABCompatible(&def) (*defs)[defName] = &def return defName @@ -467,16 +466,31 @@ func (c *ManifestConverter) generateParameterSources(b *cnab.ExtendedBundle) cna // 3. directly when they use `source` on a parameter // Directly wired outputs to parameters - for _, p := range c.Manifest.Parameters { - // Skip parameters that aren't set from an output - if p.Source.Output == "" { + for k, p := range c.Manifest.Parameters { + // Skip parameters that aren't set from an output or from a directory source + if (!p.Source.IsDirSource()) && p.Source.Output == "" { continue } var pso cnab.ParameterSource - if p.Source.Dependency == "" { + if p.Source.IsDirSource() { + // If it's a directory handle it accordingly + defName := fmt.Sprintf("%s-parameter", p.Name) + pso = c.generateDirectoryParameterSource(p.Source.Mount, p.Name, p.Destination.Path) + def := c.generateDirectoryParameterSchema(*b, defName) + // Make sure that the destination is changed to an env var instead of a path + // Otherwise cnab will attempt to place the path into the container which will fail + if pb, ok := b.Parameters[k]; ok { + c.sanitizeDirParameters(pb.Destination, k) + b.Parameters[k] = pb + } + b.Definitions[defName] = &def + + } else if p.Source.Dependency == "" { + // If it's not a directory and it doesn't have a dependency, it's a standard output pso = c.generateOutputParameterSource(p.Source.Output) } else { + // Otherwise it must be a dependency ref := manifest.DependencyOutputReference{ Dependency: p.Source.Dependency, Output: p.Source.Output, @@ -519,6 +533,28 @@ func (c *ManifestConverter) generateParameterSources(b *cnab.ExtendedBundle) cna return ps } +func (c *ManifestConverter) generateDirectoryParameterSchema(b cnab.ExtendedBundle, name string) definition.Schema { + var def definition.Schema + pdef, ok := b.Definitions[name] + if ok { + MakeCNABCompatible(b.Definitions[name]) + def = *pdef + } else { + def = definition.Schema{} + def.Type = "directory" + MakeCNABCompatible(&def) + } + def.ID = "https://porter.sh/generated-bundle/#porter-parameter-source-definition" + return def +} + +// Remove the path value from directory parameters so they aren't assumed to be files +// By the cnab.io package. Apply the destination to an env var "directory-parameters.[name]" +func(c *ManifestConverter) sanitizeDirParameters(destination *bundle.Location, name string) { + destination.Path = "" + destination.EnvironmentVariable = cnab.DirectoryExtensionShortHand + "." + name +} + // generateOutputWiringParameter creates an internal parameter used only by porter, it won't be visible to the user. // The parameter exists solely so that Porter can inject an output back into the bundle, using a parameter source. // The parameter's definition is a copy of the output's definition, with the ID set so we know that it was generated by porter. @@ -580,6 +616,30 @@ func (c *ManifestConverter) generateOutputParameterSource(outputName string) cna } } +// Pass the inferred info from the parameter to the parameter source +func (c *ManifestConverter) generateDirectoryParameterSource(source interface{}, name string, target string) cnab.ParameterSource { + switch source.(type) { + case cnab.MountParameterSourceDefn: + return c.generateMountParameterSource(source.(cnab.MountParameterSourceDefn), name, target) + default: + return cnab.ParameterSource{} + } +} + +// generateMountParameterSource builds a parameter source that connects a parameter to a mount. +func (c *ManifestConverter) generateMountParameterSource(mount cnab.MountParameterSourceDefn, name string, target string) cnab.ParameterSource { + return cnab.ParameterSource{ + Priority: []string{cnab.ParameterSourceTypeMount}, + Sources: map[string]cnab.ParameterSourceDefinition{ + cnab.ParameterSourceTypeMount: func() cnab.MountParameterSourceDefn { + mount.Name = name + mount.Target = target + return mount + }(), + }, + } +} + // generateDependencyOutputParameterSource builds a parameter source that connects a dependency output to a parameter. func (c *ManifestConverter) generateDependencyOutputParameterSource(ref manifest.DependencyOutputReference) cnab.ParameterSource { return cnab.ParameterSource{ @@ -630,6 +690,13 @@ func (c *ManifestConverter) generateCustomExtensions(b *cnab.ExtendedBundle) (ma customExtensions[cnab.ParameterSourcesExtensionKey] = ps } + // Add the directory extension + if dirs, err := c.generateDirectoryExtension(ps); err == nil && len(dirs) > 0 { + customExtensions[cnab.DirectoryParameterExtensionKey] = dirs + } else if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + } + // Add entries for user-specified required extensions, like docker for _, ext := range c.Manifest.Required { customExtensions[lookupExtensionKey(ext.Name)] = ext.Config @@ -638,6 +705,29 @@ func (c *ManifestConverter) generateCustomExtensions(b *cnab.ExtendedBundle) (ma return customExtensions, nil } +func (c *ManifestConverter) generateDirectoryExtension(ps cnab.ParameterSources) (map[string]cnab.DirectoryDetails, error) { + dirs := make(map[string]cnab.DirectoryDetails, 0) + for name, param := range ps { + for _, src := range param.Sources { + switch src.(type) { + case cnab.MountParameterSourceDefn: + dirs[name] = cnab.DirectoryDetails{ + DirectorySources: cnab.DirectorySources{ + Mount: src.(cnab.MountParameterSourceDefn), + }, + DirectoryParameterDefinition: c.Manifest.Parameters[name].DirectoryParameterDefinition, + Kind: cnab.ParameterSourceTypeMount, + } + break + default: + continue + } + } + } + + return dirs, nil +} + func (c *ManifestConverter) generateRequiredExtensions(b cnab.ExtendedBundle) []string { requiredExtensions := []string{cnab.FileParameterExtensionKey} @@ -651,6 +741,10 @@ func (c *ManifestConverter) generateRequiredExtensions(b cnab.ExtendedBundle) [] requiredExtensions = append(requiredExtensions, cnab.ParameterSourcesExtensionKey) } + if b.HasDirectoryParameters() { + requiredExtensions = append(requiredExtensions, cnab.DirectoryParameterExtensionKey) + } + // Add all under required section of manifest for _, ext := range c.Manifest.Required { requiredExtensions = append(requiredExtensions, lookupExtensionKey(ext.Name)) diff --git a/pkg/cnab/config-adapter/helpers.go b/pkg/cnab/config-adapter/helpers.go index 318257f96..f0fa18dce 100644 --- a/pkg/cnab/config-adapter/helpers.go +++ b/pkg/cnab/config-adapter/helpers.go @@ -6,6 +6,7 @@ import ( "get.porter.sh/porter/pkg/cnab" "get.porter.sh/porter/pkg/config" "get.porter.sh/porter/pkg/manifest" + "github.com/cnabio/cnab-go/bundle/definition" ) // ConvertToTestBundle is suitable for taking a test manifest (porter.yaml) @@ -15,3 +16,19 @@ func ConvertToTestBundle(ctx context.Context, cfg *config.Config, manifest *mani converter := NewManifestConverter(cfg, manifest, nil, nil) return converter.ToBundle(ctx) } + +// MakeCNABCompatible receives a schema with possible porter specific parameters +// and converts those parameters to CNAB compatible versions. +// Returns true if values were replaced and false otherwise. +func MakeCNABCompatible(schema *definition.Schema) bool { + if v, ok := schema.Type.(string); ok { + if c, ok := config.PorterParamMap[v]; ok { + schema.Type = c.Type + schema.ContentEncoding = c.Encoding + schema.Comment = c.Comment + return ok + } + } + + return false +} \ No newline at end of file diff --git a/pkg/cnab/directory_parameter.go b/pkg/cnab/directory_parameter.go new file mode 100644 index 000000000..240390379 --- /dev/null +++ b/pkg/cnab/directory_parameter.go @@ -0,0 +1,104 @@ +package cnab + +import ( + "encoding/json" + + "github.com/cnabio/cnab-go/bundle/definition" + "github.com/docker/docker/api/types/mount" + "github.com/pkg/errors" +) + +const ( + DirectoryExtensionShortHand = "directory-parameter" + DirectoryParameterExtensionKey = PorterExtensionsPrefix + DirectoryExtensionShortHand +) + +// DirectoryParameterDefinition represents those parameter options +// That apply exclusively to the directory parameter type +type DirectoryParameterDefinition struct { + Writeable bool `yaml:"writeable,omitempty"` + // UID and GID should be ints, however 0 is the default value for int type + // But is also a realistic value for UID/GID thus we need to make the type interface + // To detect the case that the values weren't set + GID interface{} `yaml:"gid,omitempty" json:"gid,omitempty"` + UID interface{} `yaml:"uid,omitempty" json:"uid,omitempty"` +} + +// MountParameterSource represents a parameter using a docker mount +// As a its source with the provided options +type MountParameterSourceDefn struct { + mount.Mount `yaml:",inline"` + Name string `json:"name,omitempty" yaml:"name,omitempty"` +} + +// DirectorySources represents the sources available to the directory parameter type +// Currently only mount has been specified, but this could change in the future +type DirectorySources struct { + Mount MountParameterSourceDefn `yaml:"mount,omitempty" json:"mount,omitempty"` +} +type DirectoryDetails struct { + DirectorySources + DirectoryParameterDefinition + Kind string `json:"kind,omitempty"` +} + +// DirectoryParameterExtension indicates that Directory support is required +var DirectoryParameterExtension = RequiredExtension{ + Shorthand: DirectoryExtensionShortHand, + Key: DirectoryParameterExtensionKey, + Reader: DirectoryParameterReader, +} + +// SupportsDirectoryParameters returns true if the bundle supports the +// Directory parameter extension +func (b ExtendedBundle) SupportsDirectoryParameters() bool { + return b.SupportsExtension(DirectoryParameterExtensionKey) +} + +// IsDirType determines if the parameter/credential is of type "directory". +func (b ExtendedBundle) IsDirType(def *definition.Schema) bool { + return b.SupportsDirectoryParameters() && def.Type == "string" && def.Comment == DirectoryParameterExtensionKey +} + +// DirectoryParameterReader is a Reader for the DirectoryParameterExtension. +// The extension maintains the list of directory parameters in the bundle +func DirectoryParameterReader(b ExtendedBundle) (interface{}, error) { + return b.DirectoryParameterReader() +} + +// DirectoryParameterReader is a Reader for the DirectoryParameterExtension. +// This method generates the list of directory parameter names in the bundle. +// The Directory Parameter extension maintains the list of directory parameters in the bundle +func (b ExtendedBundle) DirectoryParameterReader() (interface{}, error) { + bytes, err := json.Marshal(b.Custom[DirectoryParameterExtensionKey]) + if err != nil { + return nil, errors.Wrapf(err, "Failed to marshal custom extension %s", DirectoryParameterExtensionKey) + } + var dd map[string]DirectoryDetails + if err = errors.Wrapf(json.Unmarshal(bytes, &dd), "Failed to unmarshal custom extension %s %s", DirectoryParameterExtensionKey, string(bytes)); err != nil { + return nil, err + } + dirs := make([]DirectoryDetails, len(dd)) + i := 0 + for _, dir := range dd { + dirs[i] = dir + i++ + } + return dirs, nil +} + +// DirectoryParameterSupport checks if the Directory parameter extension is present +func (e ProcessedExtensions) DirectoryParameterSupport() bool { + _, extensionRequired := e[DirectoryParameterExtensionKey] + return extensionRequired +} + +// IDToInt converts an interface to an integer. If the id is coercable to an int, returns the value +// Otherwise returns -1 +func IDToInt(id interface{}) int { + if i, ok := id.(int); ok { + return i + } + + return -1 +} diff --git a/pkg/cnab/extended_bundle.go b/pkg/cnab/extended_bundle.go index a72069527..4c64c8f50 100644 --- a/pkg/cnab/extended_bundle.go +++ b/pkg/cnab/extended_bundle.go @@ -88,6 +88,8 @@ func (b ExtendedBundle) GetParameterType(def *definition.Schema) string { return fmt.Sprintf("%v", def.Type) } + + // IsFileType determines if the parameter/credential is of type "file". func (b ExtendedBundle) IsFileType(def *definition.Schema) bool { return b.SupportsFileParameters() && diff --git a/pkg/cnab/file_parameter.go b/pkg/cnab/file_parameter.go index cd3fc1b5b..65c025da0 100644 --- a/pkg/cnab/file_parameter.go +++ b/pkg/cnab/file_parameter.go @@ -16,13 +16,13 @@ var FileParameterExtension = RequiredExtension{ Reader: FileParameterReader, } + // FileParameterReader is a Reader for the FileParameterExtension. // The extension does not have any data, its presence indicates that // parameters of type "file" should be supported by the tooling. func FileParameterReader(b ExtendedBundle) (interface{}, error) { return b.FileParameterReader() } - // FileParameterReader is a Reader for the FileParameterExtension. // The extension does not have any data, its presence indicates that // parameters of type "file" should be supported by the tooling. @@ -45,4 +45,4 @@ func (b ExtendedBundle) SupportsFileParameters() bool { func (e ProcessedExtensions) FileParameterSupport() bool { _, extensionRequired := e[FileParameterExtensionKey] return extensionRequired -} +} \ No newline at end of file diff --git a/pkg/cnab/parameter_sources.go b/pkg/cnab/parameter_sources.go index 10f589920..874f897ae 100644 --- a/pkg/cnab/parameter_sources.go +++ b/pkg/cnab/parameter_sources.go @@ -2,7 +2,7 @@ package cnab import ( "encoding/json" - "errors" + "github.com/pkg/errors" "fmt" ) @@ -20,6 +20,9 @@ const ( // ParameterSourceTypeDependencyOutput defines a type of parameter source that is provided by a bundle's dependency // output. ParameterSourceTypeDependencyOutput = "dependencies.output" + + // ParameterSouceTypeMount defines a type of parameter source that is provided by a docker mount + ParameterSourceTypeMount = "docker.mount" ) // ParameterSourcesExtension represents a required extension that specifies how @@ -127,6 +130,14 @@ func (m *ParameterSourceMap) UnmarshalJSON(data []byte) error { return fmt.Errorf("invalid parameter source definition for key %s: %w", sourceKey, err) } (*m)[ParameterSourceTypeDependencyOutput] = depOutput + case ParameterSourceTypeMount: + var src MountParameterSourceDefn + err := json.Unmarshal(rawDef, &src) + if err != nil { + return errors.Wrapf(err, "invalid parameter source definition for key %s", sourceKey) + } + (*m)[ParameterSourceTypeMount] = src + default: return fmt.Errorf("unsupported parameter source key %s", sourceKey) } @@ -224,6 +235,12 @@ func (b ExtendedBundle) HasParameterSources() bool { return ok } +// HasDirectoryParameters returns whether or not the bundle has directory parameters defined. +func (b ExtendedBundle) HasDirectoryParameters() bool { + _, ok := b.Custom[DirectoryParameterExtensionKey] + return ok +} + // ParameterHasSource determines if the specified parameter has a parameter // source defined. func (b ExtendedBundle) ParameterHasSource(paramName string) bool { diff --git a/pkg/cnab/provider/driver.go b/pkg/cnab/provider/driver.go index 71a03a5fe..984753d36 100644 --- a/pkg/cnab/provider/driver.go +++ b/pkg/cnab/provider/driver.go @@ -1,6 +1,8 @@ package cnabprovider import ( + "os" + "strings" "fmt" "get.porter.sh/porter/pkg/cnab" @@ -8,6 +10,7 @@ import ( "github.com/cnabio/cnab-go/driver" "github.com/cnabio/cnab-go/driver/docker" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/mount" ) const ( @@ -45,6 +48,47 @@ func (r *Runtime) newDriver(driverName string, args ActionArguments) (driver.Dri return nil, err } + // Handle Directory support for the docker driver + // TODO: Handle directory support for other runtimes -- how? + if driverName == "docker" && r.Extensions.DirectoryParameterSupport() { + d := driverImpl.(*docker.Driver) + for _, _dd := range r.Extensions[cnab.DirectoryParameterExtension.Key].([]cnab.DirectoryDetails) { + switch _dd.Kind { + case cnab.ParameterSourceTypeMount: + // preserve the closure context by running in an immediate closure + // AddConfigurationOptions executes its closure asynchronously and so only picks up + // The last value for dd unless we do it this way + func () { + dd := _dd + d.AddConfigurationOptions(func(cfg *container.Config, hostCfg *container.HostConfig) error { + x := dd.Mount + x.Type = "bind" + x.ReadOnly = !dd.Writeable + pairs := make([]string, len(os.Environ())*2) + for i, env := range os.Environ() { + parts := strings.Split(env, "=") + pairs[i*2] = "$" + parts[0] + pairs[i*2+1] = parts[1] + } + + rep := strings.NewReplacer(pairs...) + x.Source = rep.Replace(x.Source) + x.Target = rep.Replace(x.Target) + if hostCfg.Mounts == nil || len(hostCfg.Mounts) < 1 { + hostCfg.Mounts = []mount.Mount{ + x.Mount, + } + } else { + hostCfg.Mounts = append(hostCfg.Mounts, x.Mount) + } + return nil + }) + + }() + } + } + } + if configurable, ok := driverImpl.(driver.Configurable); ok { driverCfg := make(map[string]string) // Load any driver-specific config out of the environment diff --git a/pkg/cnab/required.go b/pkg/cnab/required.go index 98888eba6..35037dbe1 100644 --- a/pkg/cnab/required.go +++ b/pkg/cnab/required.go @@ -17,6 +17,7 @@ var SupportedExtensions = []RequiredExtension{ DockerExtension, FileParameterExtension, ParameterSourcesExtension, + DirectoryParameterExtension, } // ProcessedExtensions represents a map of the extension name to the diff --git a/pkg/config/config.go b/pkg/config/config.go index 692bb641a..dc07e8e8b 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -56,6 +56,27 @@ const ( EnvPorterInstallationName = "PORTER_INSTALLATION_NAME" ) +type CustomParam struct { + Type string + Encoding string + Comment string +} + +// PorterParamMap maps custom porter parameter types to a CNAB compatible alternative +// Comment is specified to indicate the original type, which is required to differentiate +// between multiple custom param types +var PorterParamMap = map[string]CustomParam { + "file": { + Type: "string", + Encoding: "base64", + }, + "directory": { + Type: "string", + Comment: fmt.Sprintf("%s.directory-parameter", CustomPorterKey), + }, +} + + // These are functions that afero doesn't support, so this lets us stub them out for tests to set the // location of the current executable porter binary and resolve PORTER_HOME. var getExecutable = os.Executable diff --git a/pkg/manifest/helpers.go b/pkg/manifest/helpers.go new file mode 100644 index 000000000..cb8c40bab --- /dev/null +++ b/pkg/manifest/helpers.go @@ -0,0 +1,18 @@ +package manifest + +import "get.porter.sh/porter/pkg/config" + +// MakeCNABCompatible receives a Paramaeter Definition with possible porter specific parameters +// and converts those parameters to CNAB compatible versions. +// Returns true if values were replaced and false otherwise. +func MakeCNABCompatible(def *ParameterDefinition) bool { + if v, ok := def.Type.(string); ok { + if c, ok := config.PorterParamMap[v]; ok { + def.Type = c.Type + def.ContentEncoding = c.Encoding + return ok + } + } + + return false +} diff --git a/pkg/manifest/manifest.go b/pkg/manifest/manifest.go index 725ead784..14fb39474 100644 --- a/pkg/manifest/manifest.go +++ b/pkg/manifest/manifest.go @@ -320,11 +320,14 @@ func (pd *ParameterDefinitions) UnmarshalYAML(unmarshal func(interface{}) error) var _ bundle.Scoped = &ParameterDefinition{} + + // ParameterDefinition defines a single parameter for a CNAB bundle type ParameterDefinition struct { - Name string `yaml:"name"` - Sensitive bool `yaml:"sensitive"` - Source ParameterSource `yaml:"source,omitempty"` + cnab.DirectoryParameterDefinition `yaml:",inline"` + Name string `yaml:"name"` + Sensitive bool `yaml:"sensitive"` + Source ParameterSource `yaml:"source,omitempty"` // These fields represent a subset of bundle.Parameter as defined in cnabio/cnab-go, // minus the 'Description' field (definition.Schema's will be used) and `Definition` field @@ -348,11 +351,13 @@ func (pd *ParameterDefinition) Validate() error { result = multierror.Append(result, errors.New("parameter name is required")) } - // Porter supports declaring a parameter of type: "file", + // Porter supports declaring a parameter of types: "file" and "directory", // which we will convert to the appropriate bundle.Parameter type in adapter.go // Here, we copy the ParameterDefinition and make the same modification before validation pdCopy := pd.DeepCopy() - if pdCopy.Type == "file" { + if MakeCNABCompatible(pdCopy) { + // TODO: Currently all custom parameter types require a path property. This may not be the case in the future, + // Instead, we should do the validation separately for each type. if pd.Destination.Path == "" { result = multierror.Append(result, fmt.Errorf("no destination path supplied for parameter %s", pd.Name)) } @@ -420,8 +425,9 @@ func (pd *ParameterDefinition) UpdateApplyTo(m *Manifest) { } type ParameterSource struct { - Dependency string `yaml:"dependency,omitempty"` - Output string `yaml:"output"` + cnab.DirectorySources `yaml:",inline"` + Dependency string `yaml:"dependency,omitempty"` + Output string `yaml:"output"` } // CredentialDefinitions allows us to represent credentials as a list in the YAML @@ -438,6 +444,11 @@ func (cd CredentialDefinitions) MarshalYAML() (interface{}, error) { return raw, nil } +// IsDirSource returns true if the Parameter Source is a Directory Source +func (p *ParameterSource) IsDirSource() bool { + return p.DirectorySources != cnab.DirectorySources{} +} + func (cd *CredentialDefinitions) UnmarshalYAML(unmarshal func(interface{}) error) error { var raw []CredentialDefinition err := unmarshal(&raw) diff --git a/pkg/porter/lifecycle.go b/pkg/porter/lifecycle.go index 340db54e3..e10a25840 100644 --- a/pkg/porter/lifecycle.go +++ b/pkg/porter/lifecycle.go @@ -372,6 +372,8 @@ func (p *Porter) BuildActionArgs(ctx context.Context, installation storage.Insta return cnabprovider.ActionArguments{}, log.Error(err) } + + args := cnabprovider.ActionArguments{ Action: action.GetAction(), Installation: installation, diff --git a/pkg/porter/parameters.go b/pkg/porter/parameters.go index 4825f9693..e73a2f596 100644 --- a/pkg/porter/parameters.go +++ b/pkg/porter/parameters.go @@ -4,7 +4,7 @@ import ( "context" "encoding/base64" "encoding/json" - "errors" + "github.com/pkg/errors" "fmt" "path/filepath" "sort" @@ -635,6 +635,7 @@ func (p *Porter) resolveParameterSources(ctx context.Context, bun cnab.ExtendedB for _, rawSource := range parameterSource.ListSourcesByPriority() { var installationName string var outputName string + var mount *cnab.MountParameterSourceDefn switch source := rawSource.(type) { case cnab.OutputParameterSource: installationName = installation.Name @@ -643,17 +644,18 @@ func (p *Porter) resolveParameterSources(ctx context.Context, bun cnab.ExtendedB // TODO(carolynvs): does this need to take namespace into account installationName = depsv1.BuildPrerequisiteInstallationName(installation.Name, source.Dependency) outputName = source.OutputName + case cnab.MountParameterSourceDefn: + installationName = installation.Name + mount = &source } output, err := p.Installations.GetLastOutput(ctx, installation.Namespace, installationName, outputName) if err != nil { - // When we can't find the output, skip it and let the parameter be set another way - if errors.Is(err, storage.ErrNotFound{}) { - span.Debugf("No previous output found for %s from %s/%s", outputName, installation.Namespace, installationName) - continue + // When we can't find the output, it may be a directory parameter, or we may have to find it some other way + if !errors.Is(err, storage.ErrNotFound{}) { + // Otherwise, something else has happened, perhaps bad data or connectivity problems, we can't ignore it + return nil, errors.Wrapf(err, "could not set parameter %s from output %s of %s", parameterName, outputName, installation) } - // Otherwise, something else has happened, perhaps bad data or connectivity problems, we can't ignore it - return nil, span.Error(fmt.Errorf("could not set parameter %s from output %s of %s: %w", parameterName, outputName, installation, err)) } if output.Key != "" { @@ -676,6 +678,15 @@ func (p *Porter) resolveParameterSources(ctx context.Context, bun cnab.ExtendedB if bun.IsFileType(def) { values[parameterName] = base64.StdEncoding.EncodeToString(output.Value) + } else if bun.IsDirType(def) && mount != nil { + p := bun.Parameters[parameterName] + // p.Definition = config.CustomPorterKey + ".directory" + bun.Parameters[parameterName] = p + if bytes, err := json.Marshal(mount); err == nil { + values[parameterName] = string(bytes) + } else { + return nil, fmt.Errorf("Could not marshal source for definition %s", param.Definition) + } } else { values[parameterName] = string(output.Value) } diff --git a/pkg/runtime/runtime-manifest.go b/pkg/runtime/runtime-manifest.go index e06e55395..d773ee497 100644 --- a/pkg/runtime/runtime-manifest.go +++ b/pkg/runtime/runtime-manifest.go @@ -13,7 +13,7 @@ import ( "strings" "get.porter.sh/porter/pkg" - "get.porter.sh/porter/pkg/cnab" + cnab "get.porter.sh/porter/pkg/cnab" "get.porter.sh/porter/pkg/config" "get.porter.sh/porter/pkg/manifest" "get.porter.sh/porter/pkg/portercontext" @@ -22,6 +22,7 @@ import ( "github.com/cnabio/cnab-go/bundle" "github.com/cnabio/cnab-to-oci/relocation" "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" yaml3 "gopkg.in/yaml.v3" ) @@ -141,7 +142,20 @@ func (m *RuntimeManifest) loadDependencyDefinitions() error { return nil } -func (m *RuntimeManifest) resolveParameter(pd manifest.ParameterDefinition) string { +func (m *RuntimeManifest) resolveParameter(pd manifest.ParameterDefinition) interface{} { + // Handle directories separately + if pd.Schema.Comment == cnab.DirectoryParameterExtensionKey { + if pd.Source.Mount.Source != "" { + pd.Source.Mount.Target = pd.Destination.Path + } + return map[string]interface{}{ + "path": pd.Destination.Path, + "uid": cnab.IDToInt(pd.UID), + "gid": cnab.IDToInt(pd.GID), + "writeable": pd.Writeable, + "source": pd.Source, + } + } if pd.Destination.EnvironmentVariable != "" { return m.Getenv(pd.Destination.EnvironmentVariable) } @@ -180,6 +194,27 @@ func (m *RuntimeManifest) GetSensitiveValues() []string { return m.sensitiveValues } +// setRecursiveSensitiveValue handles the case of a sensitive parameter being a map of strings +// of arbitrary depth +func (m *RuntimeManifest) setRecursiveSensitiveValue(v interface{}, key string) error { + var ok bool = true + for ok { + v, ok = v.(map[string]interface{}) + for k, i := range v.(map[string]interface{}) { + key += "." + k + m.setRecursiveSensitiveValue(i, key) + } + } + + if s, ok := v.(string); ok { + m.setSensitiveValue(s) + return nil + } + + return errors.Wrapf(errors.New("Conversion Error"), "Could not convert %s to string", key) + +} + func (m *RuntimeManifest) setSensitiveValue(val string) { exists := false for _, item := range m.sensitiveValues { @@ -283,7 +318,14 @@ func (m *RuntimeManifest) buildSourceData() (map[string]interface{}, error) { pe := param.Name val := m.resolveParameter(param) if param.Sensitive { - m.setSensitiveValue(val) + if val, ok := val.(string); ok { + m.setSensitiveValue(val) + } else { + err := m.setRecursiveSensitiveValue(val, "parameters") + if err != nil { + return nil, err + } + } } params[pe] = val } @@ -497,6 +539,14 @@ func (m *RuntimeManifest) Initialize() error { continue } + if m.bundle.IsDirType(def) { + // Update the manifest schema so we can detect directory type + // when we inject parameter values + p := m.Parameters[paramName] + p.Schema = *def + m.Parameters[paramName] = p + } + if m.bundle.IsFileType(def) { if param.Destination.Path == "" { return fmt.Errorf("destination path is not supplied for parameter %s", paramName) diff --git a/scratch.go b/scratch.go new file mode 100644 index 000000000..fbde42352 --- /dev/null +++ b/scratch.go @@ -0,0 +1,242 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" +) + +// Type represents the type of a mount. +type Type string + +// Type constants +const ( + // TypeBind is the type for mounting host dir + TypeBind Type = "bind" + // TypeVolume is the type for remote storage volumes + TypeVolume Type = "volume" + // TypeTmpfs is the type for mounting tmpfs + TypeTmpfs Type = "tmpfs" + // TypeNamedPipe is the type for mounting Windows named pipes + TypeNamedPipe Type = "npipe" +) + +// Mount represents a mount (volume). +type Mount struct { + Type Type `json:""` + // Source specifies the name of the mount. Depending on mount type, this + // may be a volume name or a host path, or even ignored. + // Source is not supported for tmpfs (must be an empty value) + Source string `json:""` + Target string `json:""` + ReadOnly bool `json:""` + Consistency Consistency `json:""` + + BindOptions *BindOptions `json:""` + VolumeOptions *VolumeOptions `json:""` + TmpfsOptions *TmpfsOptions `json:""` +} + +// Propagation represents the propagation of a mount. +type Propagation string + +const ( + // PropagationRPrivate RPRIVATE + PropagationRPrivate Propagation = "rprivate" + // PropagationPrivate PRIVATE + PropagationPrivate Propagation = "private" + // PropagationRShared RSHARED + PropagationRShared Propagation = "rshared" + // PropagationShared SHARED + PropagationShared Propagation = "shared" + // PropagationRSlave RSLAVE + PropagationRSlave Propagation = "rslave" + // PropagationSlave SLAVE + PropagationSlave Propagation = "slave" +) + +// Propagations is the list of all valid mount propagations +var Propagations = []Propagation{ + PropagationRPrivate, + PropagationPrivate, + PropagationRShared, + PropagationShared, + PropagationRSlave, + PropagationSlave, +} + +// Consistency represents the consistency requirements of a mount. +type Consistency string + +const ( + // ConsistencyFull guarantees bind mount-like consistency + ConsistencyFull Consistency = "consistent" + // ConsistencyCached mounts can cache read data and FS structure + ConsistencyCached Consistency = "cached" + // ConsistencyDelegated mounts can cache read and written data and structure + ConsistencyDelegated Consistency = "delegated" + // ConsistencyDefault provides "consistent" behavior unless overridden + ConsistencyDefault Consistency = "default" +) + +// BindOptions defines options specific to mounts of type "bind". +type BindOptions struct { + Propagation Propagation `json:""` + NonRecursive bool `json:""` +} + +// VolumeOptions represents the options for a mount of type volume. +type VolumeOptions struct { + NoCopy bool `json:""` + Labels map[string]string `json:""` + DriverConfig *Driver `json:""` +} + +// Driver represents a volume driver. +type Driver struct { + Name string `json:""` + Options map[string]string `json:""` +} + +// TmpfsOptions defines options specific to mounts of type "tmpfs". +type TmpfsOptions struct { + // Size sets the size of the tmpfs, in bytes. + // + // This will be converted to an operating system specific value + // depending on the host. For example, on linux, it will be converted to + // use a 'k', 'm' or 'g' syntax. BSD, though not widely supported with + // docker, uses a straight byte value. + // + // Percentages are not supported. + SizeBytes int64 `json:""` + // Mode of the tmpfs upon creation + Mode os.FileMode `json:""` + + // TODO(stevvooe): There are several more tmpfs flags, specified in the + // daemon, that are accepted. Only the most basic are added for now. + // + // From https://github.com/moby/sys/blob/mount/v0.1.1/mount/flags.go#L47-L56 + // + // var validFlags = map[string]bool{ + // "": true, + // "size": true, X + // "mode": true, X + // "uid": true, + // "gid": true, + // "nr_inodes": true, + // "nr_blocks": true, + // "mpol": true, + // } + // + // Some of these may be straightforward to add, but others, such as + // uid/gid have implications in a clustered system. +} + +type PortBinding struct { + // HostIP is the host IP Address + HostIP string `json:"HostIp"` + // HostPort is the host port number + HostPort string +} + +// PortMap is a collection of PortBinding indexed by Port +type PortMap map[Port][]PortBinding + +// PortSet is a collection of structs indexed by Port +type PortSet map[Port]struct{} + +// Port is a string containing port number and protocol in the format "80/tcp" +type Port string + +// RestartPolicy represents the restart policies of the container. +type RestartPolicy struct { + Name string + MaximumRetryCount int +} + +// Docker describes the set of custom extension metadata associated with the Docker extension +type Docker struct { + // Privileged represents whether or not the Docker container should run as --privileged + Privileged bool `json:"privileged"` + // Mounts represent mounts to be attached to the host machine with all configurable options. + Mounts []Mount `json:"mounts"` + // Network represents the network type applied to the container "host,bridged,etc" + Network string `json:"network"` + // CapAdd represents the capabilities available to the container kernel + CapAdd []string `json:"capadd"` + // CapDrop represents capabilities to exclude from the container kernel + CapDrop []string `json:"capdrop"` + // Ports to bind between the host and the container + PortBindings []PortMap `json:"portBindings"` + // Restart policy to be used for the container + // This may be useful in some rare cases + RestartPolicy RestartPolicy `json:"restartPolicy"` +} + +func PrettyStruct(data interface{}) (string, error) { + val, err := json.MarshalIndent(data, "", " ") + if err != nil { + return "", err + } + return string(val), nil +} + +func main() { + docker := Docker{ + Privileged: false, + Mounts: []Mount{ + { + Type: "", + Source: "", + Target: "", + ReadOnly: false, + Consistency: "", + BindOptions: &BindOptions{ + Propagation: "", + NonRecursive: false, + }, + VolumeOptions: &VolumeOptions{ + NoCopy: false, + Labels: map[string]string{ + "": "", + }, + DriverConfig: &Driver{ + Name: "", + Options: map[string]string{ + "": "", + }, + }, + }, + TmpfsOptions: &TmpfsOptions{ + SizeBytes: 0, + Mode: 0, + }, + }, + }, + Network: "", + CapAdd: []string{ + "c1", + "c2", + }, + CapDrop: []string{ + "c3", + "c4", + }, + PortBindings: []PortMap{ + { + "80": []PortBinding{ + { + HostIP: "0.0.0.0", + HostPort: "8080", + }, + }, + }, + }, + RestartPolicy: RestartPolicy{ + Name: "", + MaximumRetryCount: 0, + }, + } + + fmt.Println(PrettyStruct(docker)) +} diff --git a/scratch.json b/scratch.json new file mode 100644 index 000000000..6e646478f --- /dev/null +++ b/scratch.json @@ -0,0 +1,58 @@ +{ + "required": + { + "docker": { + "privileged": false, + "mounts": [ + { + "Type": "", + "Source": "", + "Target": "", + "ReadOnly": false, + "Consistency": "", + "BindOptions": { + "Propagation": "", + "NonRecursive": false + }, + "VolumeOptions": { + "NoCopy": false, + "Labels": { + "": "" + }, + "DriverConfig": { + "Name": "", + "Options": { + "": "" + } + } + }, + "TmpfsOptions": { + "SizeBytes": 0, + "Mode": 0 + } + } + ], + "network": "", + "capadd": [ + "c1", + "c2" + ], + "capdrop": [ + "c3", + "c4" + ], + "portBindings": [ + { + "80": [ + { + "HostIp": "0.0.0.0", + "HostPort": "8080" + } + ] + } + ], + "restartPolicy": { + "Name": "", + "MaximumRetryCount": 0 + } + }}} \ No newline at end of file