Add example manifest generation pipeline #4
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,301 @@ | ||
| /* | ||
| Copyright 2022 Upbound Inc. | ||
| */ | ||
|
|
||
| package pipeline | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io/ioutil" | ||
| "os" | ||
| "path/filepath" | ||
| "regexp" | ||
| "strings" | ||
|
|
||
| "github.com/crossplane/crossplane-runtime/pkg/fieldpath" | ||
| "github.com/pkg/errors" | ||
| "sigs.k8s.io/yaml" | ||
|
|
||
| "github.com/upbound/upjet/pkg/config" | ||
| "github.com/upbound/upjet/pkg/resource/json" | ||
| tjtypes "github.com/upbound/upjet/pkg/types" | ||
| ) | ||
|
|
||
| var ( | ||
| reRef = regexp.MustCompile(`\${(.+)}`) | ||
| reFile = regexp.MustCompile(`file\("(.+)"\)`) | ||
| ) | ||
|
|
||
| type pavedWithManifest struct { | ||
| manifestPath string | ||
| paved *fieldpath.Paved | ||
| refsResolved bool | ||
| } | ||
|
|
||
| // ResourceExample represents the scraped example HCL configuration | ||
| // for a Terraform resource | ||
| type ResourceExample struct { | ||
| Manifest string `yaml:"manifest"` | ||
| References map[string]string `yaml:"references,omitempty"` | ||
| } | ||
|
|
||
| // Resource represents the scraped metadata for a Terraform resource | ||
| type Resource struct { | ||
| SubCategory string `yaml:"subCategory"` | ||
| Description string `yaml:"description,omitempty"` | ||
| Name string `yaml:"name"` | ||
| TitleName string `yaml:"titleName"` | ||
| Examples []ResourceExample `yaml:"examples,omitempty"` | ||
| ArgumentDocs map[string]string `yaml:"argumentDocs"` | ||
| ImportStatements []string `yaml:"importStatements"` | ||
| } | ||
|
|
||
| // ProviderMetadata metadata for a Terraform native provider | ||
| type ProviderMetadata struct { | ||
| Name string `yaml:"name"` | ||
| Resources map[string]*Resource `yaml:"resources"` | ||
| } | ||
|
|
||
| // NewProviderMetadataFromFile loads metadata from the specified YAML-formatted file | ||
| func NewProviderMetadataFromFile(path string) (*ProviderMetadata, error) { | ||
| buff, err := ioutil.ReadFile(filepath.Clean(path)) | ||
| if err != nil { | ||
| return nil, errors.Wrapf(err, "failed to read metadata file %q", path) | ||
| } | ||
|
|
||
| metadata := &ProviderMetadata{} | ||
| return metadata, errors.Wrap(yaml.Unmarshal(buff, metadata), "failed to unmarshal provider metadata") | ||
| } | ||
|
|
||
| // ExampleGenerator represents a pipeline for generating example manifests. | ||
| // Generates example manifests for Terraform resources under examples-generated. | ||
| type ExampleGenerator struct { | ||
| rootDir string | ||
| resourceMeta map[string]*Resource | ||
| resources map[string]*pavedWithManifest | ||
| } | ||
|
|
||
| // NewExampleGenerator returns a configured ExampleGenerator | ||
| func NewExampleGenerator(rootDir string, resourceMeta map[string]*Resource) *ExampleGenerator { | ||
| return &ExampleGenerator{ | ||
| rootDir: rootDir, | ||
| resourceMeta: resourceMeta, | ||
| resources: make(map[string]*pavedWithManifest), | ||
| } | ||
| } | ||
|
|
||
| // StoreExamples stores the generated example manifests under examples-generated in | ||
| // their respective API groups. | ||
| func (eg *ExampleGenerator) StoreExamples() error { | ||
| for n, pm := range eg.resources { | ||
| if err := eg.resolveReferencesOfPaved(pm); err != nil { | ||
| return errors.Wrapf(err, "cannot resolve references for resource: %s", n) | ||
| } | ||
| u := pm.paved.UnstructuredContent() | ||
| delete(u["spec"].(map[string]interface{})["forProvider"].(map[string]interface{}), "depends_on") | ||
| buff, err := yaml.Marshal(u) | ||
| if err != nil { | ||
| return errors.Wrapf(err, "cannot marshal example manifest for resource: %s", n) | ||
| } | ||
| manifestDir := filepath.Dir(pm.manifestPath) | ||
| if err := os.MkdirAll(manifestDir, 0750); err != nil { | ||
| return errors.Wrapf(err, "cannot mkdir %s", manifestDir) | ||
| } | ||
| // no sensitive info in the example manifest | ||
| if err := ioutil.WriteFile(pm.manifestPath, buff, 0644); err != nil { // nolint:gosec | ||
| return errors.Wrapf(err, "cannot write example manifest file %s for resource %s", pm.manifestPath, n) | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func (eg *ExampleGenerator) resolveReferencesOfPaved(pm *pavedWithManifest) error { | ||
| if pm.refsResolved { | ||
| return nil | ||
| } | ||
| pm.refsResolved = true | ||
| return errors.Wrap(eg.resolveReferences(pm.paved.UnstructuredContent()), "failed to resolve references of paved") | ||
| } | ||
|
|
||
| func (eg *ExampleGenerator) resolveReferences(params map[string]interface{}) error { // nolint:gocyclo | ||
| for k, v := range params { | ||
| switch t := v.(type) { | ||
| case map[string]interface{}: | ||
| if err := eg.resolveReferences(t); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| case []interface{}: | ||
| for _, e := range t { | ||
| eM, ok := e.(map[string]interface{}) | ||
| if !ok { | ||
| continue | ||
| } | ||
| if err := eg.resolveReferences(eM); err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| case string: | ||
| g := reRef.FindStringSubmatch(t) | ||
| if len(g) != 2 { | ||
| continue | ||
| } | ||
| path := strings.Split(g[1], ".") | ||
| // expected reference format is <resource type>.<resource name>.<field name> | ||
| if len(path) < 3 { | ||
| continue | ||
| } | ||
| pm := eg.resources[path[0]] | ||
| if pm == nil || pm.paved == nil { | ||
| continue | ||
| } | ||
| if err := eg.resolveReferencesOfPaved(pm); err != nil { | ||
| return errors.Wrapf(err, "cannot recursively resolve references for %q", path[0]) | ||
| } | ||
| pathStr := strings.Join(append([]string{"spec", "forProvider"}, path[2:]...), ".") | ||
| s, err := pm.paved.GetString(pathStr) | ||
| if fieldpath.IsNotFound(err) { | ||
| continue | ||
| } | ||
| if err != nil { | ||
| return errors.Wrapf(err, "cannot get string value from paved: %s", pathStr) | ||
| } | ||
| params[k] = s | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // Generate generates an example manifest for the specified Terraform resource. | ||
| func (eg *ExampleGenerator) Generate(group, version string, r *config.Resource, fieldTransformations map[string]tjtypes.Transformation) error { | ||
|
There was a problem hiding this comment. Just to confirm, we have There was a problem hiding this comment. Yes, currently we resolve any references globally so we store example manifests with their references resolved after we have generated CRDs for all resources. For test automation, when we switch to an implementation in which we use example manifests for the dependencies specified together with the target resource, we may start resolving references locally ("local resolution"). But this may not hold for all resources, i.e., for some resources, Terraform registry may not contain example manifests for the dependencies locally (specified together with the target resource's manifest). Thus, first doing the resolution locally and then globally (prioritizing local resolution over global resolution but also keeping global resolution) would provide better coverage potentially requiring less manual configuration. Let's think about this in a future PR implementing local resolution. |
||
| rm := eg.resourceMeta[r.Name] | ||
| if rm == nil || len(rm.Examples) == 0 { | ||
| return nil | ||
| } | ||
| var exampleParams map[string]interface{} | ||
| if err := json.TFParser.Unmarshal([]byte(rm.Examples[0].Manifest), &exampleParams); err != nil { | ||
| return errors.Wrapf(err, "cannot unmarshal example manifest for resource: %s", r.Name) | ||
| } | ||
| transformFields(exampleParams, r.ExternalName.OmittedFields, fieldTransformations, "") | ||
|
|
||
| example := map[string]interface{}{ | ||
| "apiVersion": fmt.Sprintf("%s/%s", group, version), | ||
| "kind": r.Kind, | ||
| "metadata": map[string]interface{}{ | ||
| "name": "example", | ||
| }, | ||
| "spec": map[string]interface{}{ | ||
| "forProvider": exampleParams, | ||
| "providerConfigRef": map[string]interface{}{ | ||
| "name": "example", | ||
| }, | ||
| }, | ||
| } | ||
| manifestDir := filepath.Join(eg.rootDir, "examples-generated", strings.ToLower(strings.Split(group, ".")[0])) | ||
| eg.resources[r.Name] = &pavedWithManifest{ | ||
| manifestPath: filepath.Join(manifestDir, fmt.Sprintf("%s.yaml", strings.ToLower(r.Kind))), | ||
| paved: fieldpath.Pave(example), | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| func getHierarchicalName(prefix, name string) string { | ||
| if prefix == "" { | ||
| return name | ||
| } | ||
| return fmt.Sprintf("%s.%s", prefix, name) | ||
| } | ||
|
|
||
| func transformFields(params map[string]interface{}, omittedFields []string, t map[string]tjtypes.Transformation, namePrefix string) { // nolint:gocyclo | ||
| for _, hn := range omittedFields { | ||
| for n := range params { | ||
| if hn == getHierarchicalName(namePrefix, n) { | ||
| delete(params, n) | ||
| break | ||
| } | ||
| } | ||
| } | ||
|
|
||
| for n, v := range params { | ||
| switch pT := v.(type) { | ||
| case map[string]interface{}: | ||
| transformFields(pT, omittedFields, t, getHierarchicalName(namePrefix, n)) | ||
|
|
||
| case []interface{}: | ||
| for _, e := range pT { | ||
| eM, ok := e.(map[string]interface{}) | ||
| if !ok { | ||
| continue | ||
| } | ||
| transformFields(eM, omittedFields, t, getHierarchicalName(namePrefix, n)) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| for hn, transform := range t { | ||
| for n, v := range params { | ||
| if hn == getHierarchicalName(namePrefix, n) { | ||
| delete(params, n) | ||
| if transform.IsRef { | ||
| if !transform.IsSensitive { | ||
| params[transform.TransformedName] = getRefField(v, | ||
| map[string]interface{}{ | ||
| "name": "example", | ||
| }) | ||
| } else { | ||
| secretName, secretKey := getSecretRef(v) | ||
| params[transform.TransformedName] = getRefField(v, | ||
| map[string]interface{}{ | ||
| "name": secretName, | ||
| "namespace": "crossplane-system", | ||
| "key": secretKey, | ||
| }) | ||
| } | ||
| } else { | ||
| params[transform.TransformedName] = v | ||
| } | ||
| break | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| func getRefField(v interface{}, ref map[string]interface{}) interface{} { | ||
| switch v.(type) { | ||
| case []interface{}: | ||
| return []interface{}{ | ||
| ref, | ||
| } | ||
|
|
||
| default: | ||
| return ref | ||
| } | ||
| } | ||
|
|
||
| func getSecretRef(v interface{}) (string, string) { | ||
| secretName := "example-secret" | ||
| secretKey := "example-key" | ||
| s, ok := v.(string) | ||
| if !ok { | ||
| return secretName, secretKey | ||
| } | ||
| g := reRef.FindStringSubmatch(s) | ||
| if len(g) != 2 { | ||
| return secretName, secretKey | ||
| } | ||
| f := reFile.FindStringSubmatch(g[1]) | ||
| switch { | ||
| case len(f) == 2: // then a file reference | ||
| _, file := filepath.Split(f[1]) | ||
| secretKey = fmt.Sprintf("attribute.%s", file) | ||
| default: | ||
| parts := strings.Split(g[1], ".") | ||
| if len(parts) < 3 { | ||
| return secretName, secretKey | ||
| } | ||
| secretName = fmt.Sprintf("example-%s", strings.Join(strings.Split(parts[0], "_")[1:], "-")) | ||
| secretKey = fmt.Sprintf("attribute.%s", strings.Join(parts[2:], ".")) | ||
| } | ||
| return secretName, secretKey | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There are two classes of references in the examples from what I've seen:
locationfromResourceGroupIDofResourceGroupWe're resolving references only for the static values here, right? There isn't any code here that will infer actual object reference that could potentially populate
Referencesconfiguration field?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes, the context here is to generate the example manifests and the sort of reference-resolving we do here result in segments like:
where in the last fragment,
serverNameRefis a reference to a PostgreSQL server resource.For automatically populating referencer fields from the provided example HCL configurations, the proposed metadata document format contains the
examples[].referencesarray.In this implementation, we do not use this metadata extracted from example HCL configurations to generate cross-resource referencer fields. But as an example we have something like:
for the
azurerm_data_protection_backup_instance_postgresqlresource extracted as metadata.