From 5406f81a01d73b8c38ce16a9bfa4b682d7bbd4d9 Mon Sep 17 00:00:00 2001 From: Douglas Lee Date: Wed, 20 Sep 2023 20:21:04 +0800 Subject: [PATCH 1/3] feat(changelog-tool): add changelog command line tool --- .gitignore | 1 + Makefile | 9 + changelog-markdown.tmpl | 28 +++ go.mod | 14 ++ main.go | 403 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 455 insertions(+) create mode 100644 .gitignore create mode 100644 Makefile create mode 100644 changelog-markdown.tmpl create mode 100644 go.mod create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..485dee6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.idea diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..7d88c1b --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ +test: + rm -rf kong + git clone https://github.com/Kong/kong.git + rm -f 1.0.0.md + echo "# 1.0.0" > 1.0.0.md + # use CHANGELOG/unreleased/kong to generate Kong system changelog + go build && ./changelog generate --repo Kong/kong --repo_path kong --changelog_path CHANGELOG/unreleased/kong --system Kong >> 1.0.0.md + # use CHANGELOG/unreleased/kong to generate Kong-enterprise system changelog + go build && ./changelog generate --repo Kong/kong --repo_path kong --changelog_path CHANGELOG/unreleased/kong --system Kong-enterprise >> 1.0.0.md diff --git a/changelog-markdown.tmpl b/changelog-markdown.tmpl new file mode 100644 index 0000000..3b8f04f --- /dev/null +++ b/changelog-markdown.tmpl @@ -0,0 +1,28 @@ +{{- /* ===== entry template ==== */ -}} +{{ define "entry" }} +- {{ $.Message }} +{{ range $i, $github := $.ParsedGithubs }} [{{ $github.Name }}]({{ $github.Link }}) {{ end }} +{{ range $i, $jira := $.ParsedJiras }} [{{ $jira.ID }}]({{ $jira.Link }}) {{ end }} +{{- end }} +{{- /* ===== entry template ==== */ -}} +{{- /* ==== section template ==== */ -}} +{{ define "section" }} +{{- if .scopes }} +{{- $length := len .scopes }} +{{- if gt $length 0 }} +### {{ .sectionName }} +{{ range $i, $scope := .scopes }} +#### {{ $scope.ScopeName }} +{{ range $j, $entry := $scope.Entries }} {{ template "entry" $entry }} {{ end }} +{{ end }} +{{- end }} +{{- end }} +{{- end }} +{{- /* ==== section template ==== */ -}} +## {{ .System }} +{{ template "section" (dict "sectionName" "Performance" "scopes" .Type.performance) }} +{{ template "section" (dict "sectionName" "Breaking Changes" "scopes" .Type.breaking_change ) }} +{{ template "section" (dict "sectionName" "Deprecations" "scopes" .Type.deprecation ) }} +{{ template "section" (dict "sectionName" "Dependencies" "scopes" .Type.dependency ) }} +{{ template "section" (dict "sectionName" "Features" "scopes" .Type.feature ) }} +{{ template "section" (dict "sectionName" "Fixes" "scopes" .Type.bugfix ) }} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..5653d35 --- /dev/null +++ b/go.mod @@ -0,0 +1,14 @@ +module konghq.com/changelog + +go 1.20 + +require ( + github.com/urfave/cli/v2 v2.25.7 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect +) diff --git a/main.go b/main.go new file mode 100644 index 0000000..274e742 --- /dev/null +++ b/main.go @@ -0,0 +1,403 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "github.com/urfave/cli/v2" + "gopkg.in/yaml.v3" + "io" + "log" + "net/http" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "text/template" +) + +const ( + JiraBaseUrl = "https://konghq.atlassian.net/browse/" +) + +var ( + ScopePriority = map[string]int{ + "Performance": 10, + "Configuration": 20, + "Core": 30, + "PDK": 40, + "Plugin": 50, + "Admin API": 60, + "Clustering": 70, + "Default": 100, // default priority + } + repoPath string + changelogPath string + system string + repo string + token string +) + +type CommitCtx struct { + SHA string + Message string +} + +type PullCtx struct { + Number int + Title string + Body string +} + +type CommitContext struct { + Commit CommitCtx + PullCtx PullCtx +} + +func isYAML(filename string) bool { + return strings.HasSuffix(filename, ".yaml") || strings.HasSuffix(filename, ".yml") +} + +func fetchCommitContext(filename string) (*CommitContext, error) { + ctx := &CommitContext{} + //if true { + // return ctx, nil + //} + filename = filepath.Join(changelogPath, filename) + + client := &http.Client{} + + req, err := http.NewRequest("GET", fmt.Sprintf("https://api.github.com/repos/%s/commits?path=%s", repo, filename), nil) + if len(token) > 0 { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + } + response, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch commits: %v", err) + } + defer response.Body.Close() + if response.StatusCode != 200 { + return nil, fmt.Errorf("failed to fetch commits: %d %s", response.StatusCode, response.Status) + } + + bytes, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + + var res []map[string]interface{} + err = json.Unmarshal(bytes, &res) + if err != nil { + return nil, fmt.Errorf("failed unmarshal: %v", err) + } + + ctx.Commit = CommitCtx{ + SHA: res[0]["sha"].(string), + Message: res[0]["commit"].(map[string]interface{})["message"].(string), + } + + req, err = http.NewRequest("GET", fmt.Sprintf("https://api.github.com/repos/%s/commits/%s/pulls", repo, ctx.Commit.SHA), nil) + if len(token) > 0 { + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token)) + } + response, err = client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch pulls: %v", err) + } + defer response.Body.Close() + + if response.StatusCode != 200 { + return nil, fmt.Errorf("failed to fetch pulls: %d %s", response.StatusCode, response.Status) + } + + bytes, err = io.ReadAll(response.Body) + if err != nil { + return nil, err + } + err = json.Unmarshal(bytes, &res) + if err != nil { + return nil, fmt.Errorf("failed unmarshal: %v", err) + } + ctx.PullCtx = PullCtx{ + Number: int(res[len(res)-1]["number"].(float64)), + Title: res[len(res)-1]["title"].(string), + Body: res[len(res)-1]["body"].(string), + } + + return ctx, nil +} + +func matchPattern(text, pattern string, t *[]string) { + re := regexp.MustCompile(pattern) + matches := re.FindAllString(text, -1) + *t = append(*t, matches...) +} + +type ScopeEntries struct { + ScopeName string + Entries []*ChangelogEntry +} + +type Data struct { + System string + Type map[string][]ScopeEntries +} + +type Jira struct { + ID string + Link string +} + +type Github struct { + Name string + Link string +} + +type ChangelogEntry struct { + Message string `yaml:"message"` + Type string `yaml:"type"` + Scope string `yaml:"scope"` + Prs []int `yaml:"prs"` + Githubs []int `yaml:"githubs"` + Jiras []string `yaml:"jiras"` + ParsedJiras []*Jira + ParsedGithubs []*Github +} + +func parseGithub(githubNos []int) []*Github { + list := make([]*Github, 0) + for _, no := range githubNos { + github := &Github{ + Name: fmt.Sprintf("#%d", no), + Link: fmt.Sprintf("https://github.com/%s/issues/%d", repo, no), + } + list = append(list, github) + } + return list +} + +func processEntry(filename string, entry *ChangelogEntry) error { + if entry.Scope == "" { + entry.Scope = "Default" + } + + ctx, err := fetchCommitContext(filename) + if err != nil { + return fmt.Errorf("faield to fetch commit ctx: %v", err) + } + + // jiras + if len(entry.Jiras) == 0 { + jiraMap := make(map[string]bool) + r := regexp.MustCompile(`[a-zA-Z]+-\d+`) + jiras := r.FindAllString(ctx.PullCtx.Body, -1) + for _, jira := range jiras { + if !jiraMap[jira] { + entry.Jiras = append(entry.Jiras, jira) + jiraMap[jira] = true + } + } + } + for _, jiraId := range entry.Jiras { + jira := Jira{ + ID: jiraId, + Link: JiraBaseUrl + jiraId, + } + entry.ParsedJiras = append(entry.ParsedJiras, &jira) + } + + // githubs + if len(entry.Githubs) == 0 { + entry.Githubs = entry.Prs + } + if len(entry.Githubs) == 0 { + entry.Githubs = append(entry.Githubs, ctx.PullCtx.Number) + } + + entry.ParsedGithubs = parseGithub(entry.Githubs) + + return nil +} + +func mapKeys(m map[string][]*ChangelogEntry) []string { + keys := make([]string, 0) + for k := range m { + keys = append(keys, k) + } + return keys +} + +func collect() (*Data, error) { + path := filepath.Join(repoPath, changelogPath) + files, err := os.ReadDir(path) + if err != nil { + return nil, err + } + + data := &Data{ + System: system, + Type: make(map[string][]ScopeEntries), + } + + maps := make(map[string]map[string][]*ChangelogEntry) + + for _, file := range files { + if file.IsDir() || !isYAML(file.Name()) { + continue + } + content, err := os.ReadFile(filepath.Join(path, file.Name())) + if err != nil { + return nil, err + } + + // parse entry + entry := &ChangelogEntry{} + err = yaml.Unmarshal(content, entry) + + if err != nil { + return nil, fmt.Errorf("failed to unmarshal YAML from %s: %v", file.Name(), err) + } + + err = processEntry(file.Name(), entry) + if err != nil { + return nil, fmt.Errorf("fialed to process entry: %v", err) + } + + if maps[entry.Type] == nil { + maps[entry.Type] = make(map[string][]*ChangelogEntry) + } + maps[entry.Type][entry.Scope] = append(maps[entry.Type][entry.Scope], entry) + } + + data.Type = make(map[string][]ScopeEntries) + for t, scopeEntries := range maps { + scopes := mapKeys(scopeEntries) + sort.Slice(scopes, func(i, j int) bool { + scopei := scopes[i] + scopej := scopes[j] + return ScopePriority[scopei] < ScopePriority[scopej] + }) + + list := make([]ScopeEntries, 0) + for _, scope := range scopes { + entries := ScopeEntries{ + ScopeName: scope, + Entries: scopeEntries[scope], + } + list = append(list, entries) + } + data.Type[t] = list + } + + //bytes, _ := json.Marshal(data) + //fmt.Println(string(bytes)) + + return data, nil +} + +func generate(data *Data) (string, error) { + tmpl, err := template.New("changelog-markdown.tmpl").Funcs(template.FuncMap{ + "arr": func(values ...any) []any { return values }, + "dict": func(values ...any) (map[string]any, error) { + if len(values)%2 != 0 { + return nil, errors.New("invalid dictionary call") + } + + root := make(map[string]any) + + for i := 0; i < len(values); i += 2 { + dict := root + var key string + switch v := values[i].(type) { + case string: + key = v + case []string: + for i := 0; i < len(v)-1; i++ { + key = v[i] + var m map[string]any + v, found := dict[key] + if found { + m = v.(map[string]any) + } else { + m = make(map[string]any) + dict[key] = m + } + dict = m + } + key = v[len(v)-1] + default: + return nil, errors.New("invalid dictionary key") + } + dict[key] = values[i+1] + } + + return root, nil + }, + }).ParseFiles("changelog-markdown.tmpl") + if err != nil { + panic(err) + } + err = tmpl.Execute(os.Stdout, data) + if err != nil { + panic(err) + } + + return "", nil +} + +func main() { + token = os.Getenv("GITHUB_TOKEN") + + var app = cli.App{ + Name: "changelog", + Version: "1.0.0", + Commands: []*cli.Command{ + // generate command + { + Name: "generate", + Usage: "Generate changelog", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "changelog_path", + Usage: "The changelog path. (e.g. CHANGELOG/unreleased)", + Required: true, + }, + &cli.StringFlag{ + Name: "system", + Usage: "The system name. (e.g. Kong)", + Required: true, + }, + &cli.StringFlag{ + Name: "repo_path", + Usage: "The repository path. (e.g. /path/to/your/repository)", + Required: true, + }, + &cli.StringFlag{ + Name: "repo", + Usage: "The repository name. (e.g. Kong/kong)", + Required: true, + }, + }, + Action: func(c *cli.Context) error { + repoPath = c.String("repo_path") + changelogPath = c.String("changelog_path") + system = c.String("system") + repo = c.String("repo") + + data, err := collect() + if err != nil { + return err + } + data.System = system + changelog, err := generate(data) + _ = changelog + return err + }, + }, + }, + } + if err := app.Run(os.Args); err != nil { + log.Fatal(err) + } +} From fe6abc035aa2bf9e706bbf679161bc2756f14068 Mon Sep 17 00:00:00 2001 From: Datong Sun Date: Wed, 27 Sep 2023 02:09:06 -0700 Subject: [PATCH 2/3] remove unused Makefile --- Makefile | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 7d88c1b..0000000 --- a/Makefile +++ /dev/null @@ -1,9 +0,0 @@ -test: - rm -rf kong - git clone https://github.com/Kong/kong.git - rm -f 1.0.0.md - echo "# 1.0.0" > 1.0.0.md - # use CHANGELOG/unreleased/kong to generate Kong system changelog - go build && ./changelog generate --repo Kong/kong --repo_path kong --changelog_path CHANGELOG/unreleased/kong --system Kong >> 1.0.0.md - # use CHANGELOG/unreleased/kong to generate Kong-enterprise system changelog - go build && ./changelog generate --repo Kong/kong --repo_path kong --changelog_path CHANGELOG/unreleased/kong --system Kong-enterprise >> 1.0.0.md From 47f0358e865e6f92c9bf80133a3bcb14ceb727ae Mon Sep 17 00:00:00 2001 From: Datong Sun Date: Wed, 27 Sep 2023 02:36:31 -0700 Subject: [PATCH 3/3] style fixes and logging improvements --- changelog-markdown.tmpl | 18 +++++++++++------- main.go | 25 ++++++++++++------------- 2 files changed, 23 insertions(+), 20 deletions(-) diff --git a/changelog-markdown.tmpl b/changelog-markdown.tmpl index 3b8f04f..a939a84 100644 --- a/changelog-markdown.tmpl +++ b/changelog-markdown.tmpl @@ -1,25 +1,29 @@ {{- /* ===== entry template ==== */ -}} {{ define "entry" }} -- {{ $.Message }} -{{ range $i, $github := $.ParsedGithubs }} [{{ $github.Name }}]({{ $github.Link }}) {{ end }} -{{ range $i, $jira := $.ParsedJiras }} [{{ $jira.ID }}]({{ $jira.Link }}) {{ end }} +- {{ trim $.Message }} +{{ range $i, $github := $.ParsedGithubs }} [{{ $github.Name }}]({{ $github.Link }}) {{- end }} +{{ range $i, $jira := $.ParsedJiras }} [{{ $jira.ID }}]({{ $jira.Link }}) {{- end }} {{- end }} {{- /* ===== entry template ==== */ -}} {{- /* ==== section template ==== */ -}} -{{ define "section" }} +{{- define "section" }} {{- if .scopes }} {{- $length := len .scopes }} {{- if gt $length 0 }} ### {{ .sectionName }} -{{ range $i, $scope := .scopes }} +{{- range $i, $scope := .scopes }} #### {{ $scope.ScopeName }} -{{ range $j, $entry := $scope.Entries }} {{ template "entry" $entry }} {{ end }} -{{ end }} +{{- range $j, $entry := $scope.Entries }} +{{ template "entry" $entry }} +{{- end }} +{{- end }} {{- end }} {{- end }} {{- end }} {{- /* ==== section template ==== */ -}} + ## {{ .System }} + {{ template "section" (dict "sectionName" "Performance" "scopes" .Type.performance) }} {{ template "section" (dict "sectionName" "Breaking Changes" "scopes" .Type.breaking_change ) }} {{ template "section" (dict "sectionName" "Deprecations" "scopes" .Type.deprecation ) }} diff --git a/main.go b/main.go index 274e742..267cacd 100644 --- a/main.go +++ b/main.go @@ -56,14 +56,11 @@ type CommitContext struct { } func isYAML(filename string) bool { - return strings.HasSuffix(filename, ".yaml") || strings.HasSuffix(filename, ".yml") + return strings.HasSuffix(filename, ".yml") } func fetchCommitContext(filename string) (*CommitContext, error) { ctx := &CommitContext{} - //if true { - // return ctx, nil - //} filename = filepath.Join(changelogPath, filename) client := &http.Client{} @@ -242,15 +239,19 @@ func collect() (*Data, error) { maps := make(map[string]map[string][]*ChangelogEntry) - for _, file := range files { + for i, file := range files { if file.IsDir() || !isYAML(file.Name()) { + log.Printf("Skipping file: %s (%d/%d)", file.Name(), i + 1, len(files)) continue } + content, err := os.ReadFile(filepath.Join(path, file.Name())) if err != nil { return nil, err } + log.Printf("Processing file: %s (%d/%d)", file.Name(), i + 1, len(files)) + // parse entry entry := &ChangelogEntry{} err = yaml.Unmarshal(content, entry) @@ -290,9 +291,6 @@ func collect() (*Data, error) { data.Type[t] = list } - //bytes, _ := json.Marshal(data) - //fmt.Println(string(bytes)) - return data, nil } @@ -334,6 +332,7 @@ func generate(data *Data) (string, error) { return root, nil }, + "trim": strings.TrimSpace, }).ParseFiles("changelog-markdown.tmpl") if err != nil { panic(err) @@ -350,7 +349,7 @@ func main() { token = os.Getenv("GITHUB_TOKEN") var app = cli.App{ - Name: "changelog", + Name: "Kong changelog generator", Version: "1.0.0", Commands: []*cli.Command{ // generate command @@ -360,22 +359,22 @@ func main() { Flags: []cli.Flag{ &cli.StringFlag{ Name: "changelog_path", - Usage: "The changelog path. (e.g. CHANGELOG/unreleased)", + Usage: "The changelog path under repo_path (relative). (e.g. changelog/unreleased/kong)", Required: true, }, &cli.StringFlag{ Name: "system", - Usage: "The system name. (e.g. Kong)", + Usage: "The software name. (e.g. Kong)", Required: true, }, &cli.StringFlag{ Name: "repo_path", - Usage: "The repository path. (e.g. /path/to/your/repository)", + Usage: "The repository path (full). (e.g. /path/to/your/repository)", Required: true, }, &cli.StringFlag{ Name: "repo", - Usage: "The repository name. (e.g. Kong/kong)", + Usage: "The repository ORG/NAME under GitHub. (e.g. Kong/kong)", Required: true, }, },