Skip to content

Commit 88ead91

Browse files
donch1989donch1989
authored
feat(recipe): rename format to type in variable section (#971)
Because - We are going to rename Instill Format to Instill Type. This commit - Renames `format` to `type` in variable section.
1 parent 11f8f5c commit 88ead91

File tree

16 files changed

+168
-25
lines changed

16 files changed

+168
-25
lines changed

‎config/config.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ database:
2626
host: pg-sql
2727
port: 5432
2828
name: pipeline
29-
version: 39
29+
version: 40
3030
timezone: Etc/UTC
3131
pool:
3232
idleconnections: 5

‎integration-test/pipeline/const.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ version: v1beta
8888
variable:
8989
input:
9090
title: Input
91-
format: string
91+
type: string
9292
9393
output:
9494
answer:

‎integration-test/pipeline/rest-integration.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -335,7 +335,7 @@ export function CheckConnections(data) {
335335
version: v1beta
336336
variable:
337337
recipients:
338-
format: array:string
338+
type: array:string
339339
output:
340340
resp:
341341
title: Response

‎integration-test/pipeline/rest-trigger.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ const recipeWithoutSetup = `
1111
version: v1beta
1212
variable:
1313
recipients:
14-
format: array:string
14+
type: array:string
1515
output:
1616
resp:
1717
title: Response

‎pkg/component/CONTRIBUTING.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -584,7 +584,7 @@ variable:
584584
who:
585585
title: Who
586586
description: Who should be greeted?
587-
format: string
587+
type: string
588588
component:
589589
hello-0:
590590
type: hello

‎pkg/component/operator/document/v0/.compogen/bottom.mdx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ component:
4646
variable:
4747
contract_pdf_file:
4848
title: Contract PDF file
49-
type: "*/*"
49+
type: document
5050
question:
5151
title: Question
5252
type: string

‎pkg/component/operator/document/v0/README.mdx

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
title: "Document"
33
lang: "en-US"
44
draft: false
5-
description: "Learn about how to set up a VDP Document component https://github.com/instill-ai/instill-core"
5+
description: "Learn about how to set up a Document component https://github.com/instill-ai/instill-core"
66
---
77

88
The Document component is an operator component that allows users to manipulate Document files.
@@ -173,7 +173,7 @@ component:
173173
variable:
174174
contract_pdf_file:
175175
title: Contract PDF file
176-
type: "*/*"
176+
type: document
177177
question:
178178
title: Question
179179
type: string

‎pkg/component/operator/json/v0/.compogen/bottom.mdx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ variable:
8787
resume:
8888
title: resume
8989
description: The PDF file of the candidates resume
90-
type: "*/*"
90+
type: document
9191
output:
9292
output:
9393
title: output

‎pkg/component/operator/json/v0/README.mdx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -262,7 +262,7 @@ variable:
262262
resume:
263263
title: resume
264264
description: The PDF file of the candidates resume
265-
type: "*/*"
265+
type: document
266266
output:
267267
output:
268268
title: output

‎pkg/datamodel/datamodel.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -272,9 +272,12 @@ func (p *PipelineRelease) AfterFind(tx *gorm.DB) (err error) {
272272
}
273273

274274
type Variable struct {
275-
Title string `json:"title,omitempty" yaml:"title,omitempty"`
276-
Description string `json:"description,omitempty" yaml:"description,omitempty"`
277-
Format string `json:"instillFormat,omitempty" yaml:"format,omitempty"`
275+
Title string `json:"title,omitempty" yaml:"title,omitempty"`
276+
Description string `json:"description,omitempty" yaml:"description,omitempty"`
277+
278+
// The `instillFormat` field has been renamed to `type`. The old field name is kept
279+
// in the JSON tag to preserve backward compatibility for Console.
280+
Type string `json:"instillFormat,omitempty" yaml:"type,omitempty"`
278281
Listen []string `json:"listen,omitempty" yaml:"listen,omitempty"`
279282
Default any `json:"default,omitempty" yaml:"default,omitempty"`
280283
InstillUIOrder int32 `json:"instillUiOrder,omitempty" yaml:"instill-ui-order,omitempty"`

‎pkg/db/migration/000040_rename_format_to_type.down.sql

Whitespace-only changes.

‎pkg/db/migration/000040_rename_format_to_type.up.sql

Whitespace-only changes.

‎pkg/db/migration/convert/convert000040/convert.go

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
package convert000040
2+
3+
import (
4+
"bytes"
5+
"fmt"
6+
7+
"go.uber.org/zap"
8+
"gopkg.in/yaml.v3"
9+
"gorm.io/gorm"
10+
11+
"github.com/instill-ai/pipeline-backend/pkg/datamodel"
12+
"github.com/instill-ai/pipeline-backend/pkg/db/migration/convert"
13+
)
14+
15+
const batchSize = 100
16+
17+
type RenameFormatToType struct {
18+
convert.Basic
19+
}
20+
21+
func (c *RenameFormatToType) Migrate() error {
22+
if err := c.migratePipeline(); err != nil {
23+
return err
24+
}
25+
26+
return c.migratePipelineRelease()
27+
}
28+
29+
func (c *RenameFormatToType) migratePipeline() error {
30+
pipelines := make([]*datamodel.Pipeline, 0, batchSize)
31+
return c.DB.Select("uid", "recipe_yaml").FindInBatches(&pipelines, batchSize, func(tx *gorm.DB, _ int) error {
32+
for _, p := range pipelines {
33+
isRecipeUpdated := false
34+
l := c.Logger.With(zap.String("pipelineUID", p.UID.String()))
35+
36+
var node yaml.Node
37+
if p.Recipe != nil {
38+
// Update recipe_yaml content
39+
if err := yaml.Unmarshal([]byte(p.RecipeYAML), &node); err != nil {
40+
return fmt.Errorf("unmarshaling recipe yaml: %w", err)
41+
}
42+
43+
// Find and update the variable section
44+
if len(node.Content) > 0 && len(node.Content[0].Content) > 0 {
45+
for i := 0; i < len(node.Content[0].Content); i += 2 {
46+
if node.Content[0].Content[i].Value == "variable" {
47+
variableNode := node.Content[0].Content[i+1]
48+
for j := 0; j < len(variableNode.Content); j += 2 {
49+
varContent := variableNode.Content[j+1]
50+
for k := 0; k < len(varContent.Content); k += 2 {
51+
if varContent.Content[k].Value == "format" {
52+
varContent.Content[k].Value = "type"
53+
isRecipeUpdated = true
54+
}
55+
}
56+
}
57+
break
58+
}
59+
}
60+
}
61+
}
62+
63+
if isRecipeUpdated {
64+
buf := bytes.Buffer{}
65+
encoder := yaml.NewEncoder(&buf)
66+
encoder.SetIndent(2)
67+
err := encoder.Encode(&node.Content[0])
68+
if err != nil {
69+
return fmt.Errorf("marshalling recipe: %w", err)
70+
}
71+
recipeYAML := buf.String()
72+
result := tx.Model(p).Where("uid = ?", p.UID).Update("recipe_yaml", recipeYAML)
73+
if result.Error != nil {
74+
l.Error("Failed to update pipeline.")
75+
return fmt.Errorf("updating pipeline recipe: %w", result.Error)
76+
}
77+
}
78+
}
79+
80+
return nil
81+
}).Error
82+
}
83+
84+
func (c *RenameFormatToType) migratePipelineRelease() error {
85+
releases := make([]*datamodel.PipelineRelease, 0, batchSize)
86+
return c.DB.Select("uid", "recipe_yaml").FindInBatches(&releases, batchSize, func(tx *gorm.DB, _ int) error {
87+
for _, r := range releases {
88+
isRecipeUpdated := false
89+
l := c.Logger.With(zap.String("pipelineReleaseUID", r.UID.String()))
90+
91+
var node yaml.Node
92+
if r.Recipe != nil {
93+
// Update recipe_yaml content
94+
if err := yaml.Unmarshal([]byte(r.RecipeYAML), &node); err != nil {
95+
return fmt.Errorf("unmarshaling recipe yaml: %w", err)
96+
}
97+
98+
// Find and update the variable section
99+
if len(node.Content) > 0 && len(node.Content[0].Content) > 0 {
100+
for i := 0; i < len(node.Content[0].Content); i += 2 {
101+
if node.Content[0].Content[i].Value == "variable" {
102+
variableNode := node.Content[0].Content[i+1]
103+
for j := 0; j < len(variableNode.Content); j += 2 {
104+
varContent := variableNode.Content[j+1]
105+
for k := 0; k < len(varContent.Content); k += 2 {
106+
if varContent.Content[k].Value == "format" {
107+
varContent.Content[k].Value = "type"
108+
isRecipeUpdated = true
109+
}
110+
}
111+
}
112+
break
113+
}
114+
}
115+
}
116+
}
117+
118+
if isRecipeUpdated {
119+
buf := bytes.Buffer{}
120+
encoder := yaml.NewEncoder(&buf)
121+
encoder.SetIndent(2)
122+
err := encoder.Encode(&node.Content[0])
123+
if err != nil {
124+
return fmt.Errorf("marshalling recipe: %w", err)
125+
}
126+
recipeYAML := buf.String()
127+
result := tx.Model(r).Where("uid = ?", r.UID).Update("recipe_yaml", recipeYAML)
128+
if result.Error != nil {
129+
l.Error("Failed to update pipeline release.")
130+
return fmt.Errorf("updating pipeline recipe: %w", result.Error)
131+
}
132+
}
133+
}
134+
135+
return nil
136+
}).Error
137+
}

‎pkg/db/migration/migration.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"github.com/instill-ai/pipeline-backend/pkg/db/migration/convert/convert000034"
2121
"github.com/instill-ai/pipeline-backend/pkg/db/migration/convert/convert000036"
2222
"github.com/instill-ai/pipeline-backend/pkg/db/migration/convert/convert000039"
23+
"github.com/instill-ai/pipeline-backend/pkg/db/migration/convert/convert000040"
2324
"github.com/instill-ai/pipeline-backend/pkg/service"
2425

2526
mgmtpb "github.com/instill-ai/protogen-go/core/mgmt/v1beta"
@@ -91,6 +92,8 @@ func (cm *CodeMigrator) Migrate(version uint) error {
9192
Basic: bc,
9293
RetentionHandler: cm.RetentionHandler,
9394
}
95+
case 40:
96+
m = &convert000040.RenameFormatToType{Basic: bc}
9497
default:
9598
return nil
9699
}

‎pkg/service/pipeline.go

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -860,11 +860,11 @@ func (s *service) preTriggerPipeline(
860860
return ErrExceedMaxBatchSize
861861
}
862862

863-
formatMap := map[string]string{}
863+
typeMap := map[string]string{}
864864
defaultValueMap := map[string]any{}
865865

866866
for k, v := range r.Variable {
867-
formatMap[k] = v.Format
867+
typeMap[k] = v.Type
868868
defaultValueMap[k] = v.Default
869869
}
870870

@@ -887,7 +887,7 @@ func (s *service) preTriggerPipeline(
887887
for k := range m {
888888
switch str := m[k].(type) {
889889
case string:
890-
if isUnstructuredFormat(formatMap[k]) {
890+
if isUnstructuredType(typeMap[k]) {
891891
// Skip the base64 decoding if the string is a URL.
892892
if strings.HasPrefix(str, "http://") || strings.HasPrefix(str, "https://") {
893893
continue
@@ -899,7 +899,7 @@ func (s *service) preTriggerPipeline(
899899
vars.Fields[k] = structpb.NewStringValue(downloadURL)
900900
}
901901
case []string:
902-
if isUnstructuredFormat(formatMap[k]) {
902+
if isUnstructuredType(typeMap[k]) {
903903
for idx := range str {
904904
// Skip the base64 decoding if the string is a URL
905905
if strings.HasPrefix(str[idx], "http://") || strings.HasPrefix(str[idx], "https://") {
@@ -927,9 +927,9 @@ func (s *service) preTriggerPipeline(
927927
return err
928928
}
929929

930-
formats := map[string][]string{}
931-
for k, v := range formatMap {
932-
formats[k] = []string{v}
930+
types := map[string][]string{}
931+
for k, v := range typeMap {
932+
types[k] = []string{v}
933933
}
934934

935935
uploadingPipelineData := make([]map[string]any, len(pipelineData))
@@ -941,9 +941,9 @@ func (s *service) preTriggerPipeline(
941941
for idx, d := range pipelineData {
942942

943943
variable := data.Map{}
944-
for k := range formatMap {
944+
for k := range typeMap {
945945
v := d.Variable.Fields[k]
946-
if _, ok := formatMap[k]; !ok {
946+
if _, ok := typeMap[k]; !ok {
947947
continue
948948
}
949949

@@ -963,7 +963,7 @@ func (s *service) preTriggerPipeline(
963963
}
964964
}
965965

966-
switch formatMap[k] {
966+
switch typeMap[k] {
967967
case "boolean":
968968
if v == nil {
969969
variable[k] = data.NewBoolean(defaultValueMap[k].(bool))
@@ -1097,7 +1097,7 @@ func (s *service) preTriggerPipeline(
10971097

10981098
// TODO: this is a temporary solution to handle the default
10991099
// value of integer type, we will implement a better
1100-
// solution for conversion JSON betweeen instill format
1100+
// solution for conversion JSON betweeen instill type
11011101
for i, val := range defaultValueMap[k].([]any) {
11021102
switch num := val.(type) {
11031103
case int:

‎pkg/service/utils.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -235,7 +235,7 @@ func parseRecipeMetadata(ctx context.Context, metadataMap map[string][]byte, con
235235
return pbStruct, dataSpec, nil
236236
}
237237

238-
func isUnstructuredFormat(format string) bool {
238+
func isUnstructuredType(format string) bool {
239239
if strings.HasPrefix(format, "array:") {
240240
return format != "array:string" &&
241241
format != "array:number" &&

0 commit comments

Comments
 (0)