Merge pull request #1 from hyperonecom/fix-go

Minor tweaks for Go & Python build

Author: ad-m

.github/workflows/generate.yaml

@@ -7,7 +7,29 @@ on:
 
 jobs:
   generate:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - language: ts
+            copy: "/bin/cp -rf ./clients/ts/. h1-client-ts"
+            directory: "h1-client-ts"
+            repository: "hyperonecom/h1-client-js"
+            build: "yarn"
+          - language: go
+            copy: "/bin/cp -rf ./clients/go/. h1-client-go"
+            directory: "h1-client-go"
+            repository: "hyperonecom/h1-client-go"
+            build: "go build -o temporary_build && rm temporary_build"
+          - language: python
+            copy: "/bin/cp -rf ./clients/python/. h1-client-python"
+            directory: "h1-client-python"
+            repository: "hyperonecom/h1-client-python"
+            build: "pip install -e ."
     runs-on: ubuntu-latest
+    env:
+      GO_POST_PROCESS_FILE: "/usr/local/bin/gofmt -w"
+      LANGUAGE: "${{ matrix.language }}"
     steps:
       - name: Checkout generator repository
         uses: actions/checkout@v1
@@ -23,6 +45,10 @@ jobs:
         uses: actions/setup-go@v2
         with:
           go-version: 1.14.x
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
       - name: Configure github credentials
         run: |
           git config --global user.email "action@github.com"
@@ -35,76 +61,29 @@ jobs:
           PASSPORT_FILE: ${{ secrets.PASSPORT_FILE }}
       - name: Install dependencies
         run: yarn
-      - name: Generate and test TS client
-        run: yarn start ts
-      - name: Checkout TS Client repo
-        uses: actions/checkout@v2
-        with:
-          persist-credentials: false
-          fetch-depth: 0
-          repository: hyperonecom/h1-client-js
-          path: h1-client-ts
-      - name: Copy TS client
-        run: /bin/cp -rf ./clients/ts/. h1-client-ts
-      - name: Commit TS client
-        run: |
-          git add .
-          git diff-index --quiet HEAD || git commit -m "Update $(date)"
-        working-directory: h1-client-ts
-      - name: Push TS client
-        uses: ad-m/github-push-action@master
-        with:
-          github_token: ${{ secrets.PUSH_TOKEN }}
-          directory: h1-client-ts
-          repository: hyperonecom/h1-client-js
-      - name: Generate and test Go client
-        run: yarn start go
-      - name: Test building Go client
-        run: go build -o temporary_build && rm temporary_build
-        working-directory: ./clients/go/
-      - name: Checkout Go Client repo
-        uses: actions/checkout@v2
-        with:
-          persist-credentials: false
-          fetch-depth: 0
-          repository: hyperonecom/h1-client-go
-          path: h1-client-go
-      - name: Copy Go client
-        run: /bin/cp -rf ./clients/go/. h1-client-go
-      - name: Commit Go client
-        run: |
-          git add .
-          git diff-index --quiet HEAD || git commit -m "Update $(date)"
-        working-directory: h1-client-go
-      - name: Push Go client
-        uses: ad-m/github-push-action@master
-        with:
-          github_token: ${{ secrets.PUSH_TOKEN }}
-          directory: h1-client-go
-          repository: hyperonecom/h1-client-go
-      - name: Set up Python 3.6
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.6
-      - name: Generate and test Python client
-        run: yarn start python
-      - name: Checkout Python Client repo
+      - name: Generate and test client
+        run: yarn start "$LANGUAGE"
+      - name: Test building client
+        run: "${{ matrix.build }}"
+        working-directory: "./clients/${{ matrix.language }}"
+      - name: Checkout Client repo
         uses: actions/checkout@v2
         with:
           persist-credentials: false
           fetch-depth: 0
-          repository: hyperonecom/h1-client-python
-          path: h1-client-python
-      - name: Copy Python client
-        run: /bin/cp -rf ./clients/python/. h1-client-python
-      - name: Commit Python client
+          repository: "${{ matrix.repository }}"
+          path: "${{ matrix.directory }}"
+      - name: Copy client
+        run: "${{ matrix.copy }}"
+      - name: Commit client
         run: |
           git add .
           git diff-index --quiet HEAD || git commit -m "Update $(date)"
-        working-directory: h1-client-python
-      - name: Push Python client
+        working-directory: "${{ matrix.directory }}"
+      - name: Push client
         uses: ad-m/github-push-action@master
+        if: "${{ success() && github.ref == 'refs/heads/master' }}"
         with:
           github_token: ${{ secrets.PUSH_TOKEN }}
-          directory: h1-client-python
-          repository: hyperonecom/h1-client-python
+          directory: "${{ matrix.directory }}"
+          repository: "${{ matrix.repository }}"

src/generators/go/_README.md

@@ -24,7 +24,7 @@ func main() {
     if err != nil {
         // handle error with getting passport provider
     }
-    projects, response, err := c.IamProjectApi.IamProjectList(p.Ctx(), nil) // getting IamProjectList using IamProjectApi struct
+    projects, response, err := c.IamProjectApi.IamProjectList(p.Ctx()).Execute() // getting IamProjectList using IamProjectApi struct
 
     if err != nil {
         // handle error with request
@@ -70,7 +70,7 @@ Then use `Ctx`, or `CtxWithError` method.
 `Ctx` method usage:
 
 ```go
-projects, response, err := c.IamProjectApi.IamProjectList(p.Ctx(), nil)
+projects, response, err := c.IamProjectApi.IamProjectList(p.Ctx()).Execute()
 // [...]
 ```
 
@@ -82,7 +82,7 @@ if err != nil {
     log.panic(err)
 }
 
-projects, response, err := c.IamProjectApi.IamProjectList(p.Ctx(), nil)
+projects, response, err := c.IamProjectApi.IamProjectList(p.Ctx()).Execute()
 ```
 
 The difference between `Ctx` method and `CtxWithError` is that `Ctx` does not
@@ -105,7 +105,7 @@ func main() {
     for {
         time.Sleep(5 * time.Minute)
         // this will not work, since token will be invalid after 5 minutes
-        projects, _, err := client.IamProjectApi.IamProjectList(ctx, nil)
+        projects, _, err := client.IamProjectApi.IamProjectList(ctx).Execute()
         // [...]
     }
 }
@@ -115,7 +115,7 @@ func main() {
 
 ```go
 // [...]
-projects, _, err := client.IamProjectApi.IamProjectList(provider.Ctx(), nil)
+projects, _, err := client.IamProjectApi.IamProjectList(provider.Ctx()).Execute()
 ```
 
 ### "Prefer" header

src/generators/go/index.ts

@@ -15,7 +15,7 @@ export const generateGoClient = async (
   const generator = "go";
 
   await execute(
-    `yarn openapi-generator-cli generate -i ${openapiFile} -c ${config} -g ${generator} -o ${outputDir}`
+    `yarn openapi-generator-cli generate --git-user-id "hyperonecom" --git-repo-id "h1-client-go" -i ${openapiFile} -c ${config} -g ${generator} -o ${outputDir}`
   );
 
   const contextFileLocation = join(__dirname, "_passport_context.go");

src/generators/python/_README.md

@@ -25,8 +25,8 @@ library installation description
 Then import it and use it as a token provider in your code:
 
 ```python
-from <credentials library name> import get_passport_credentials_helper
-from <client library name> import ApiClient, Configuration
+from credentials import get_passport_credentials_helper
+from h1 import ApiClient, Configuration
 
 provider = get_passport_credentials_helper() # you can optionally pass passport file location
 cfg = Configuration()
@@ -42,8 +42,9 @@ Configuration object allows you to use choosen API client.
 Example:
 
 ```python
-from <credentials library name> import get_passport_credentials_helper
-from <client library name> import Configuration, ApiClient, IamProjectApi
+from credentials import get_passport_credentials_helper
+from h1 import Configuration, ApiClient
+from h1.api.iam_project_api import IamProjectApi
 
 provider = get_passport_credentials_helper()
 cfg = Configuration()
@@ -73,7 +74,8 @@ api_client = ApiClient(cfg, header_name="prefer", header_value="respond-async,wa
 Full example:
 
 ```python
-from <client library name> import Configuration, ApiClient, IamProjectApi
+from h1 import Configuration, ApiClient
+from h1.api.iam_project_api import IamProjectApi
 
 cfg = Configuration()
 api_client = ApiClient(cfg, header_name="prefer", header_value="respond-async,wait=86400")

src/generators/python/config.yaml

@@ -1,2 +1,2 @@
-packageName: h1-client
-projectName: h1
+packageName: h1
+projectName: h1-client

src/generators/python/index.ts

@@ -1,17 +1,52 @@
 import { join } from "path";
+import { readFile, writeFile } from "fs/promises";
+
 import { execute } from "../../utils/shellUtils";
 import { copyLicense } from "../../utils/licenseUtils";
 import { fixPathsInReplacedReadme } from "../../utils/fileUtils";
 
+const dropObjectPattern = (s: any) => {
+  if (!s.properties) return;
+  for (const property of Object.values(s.properties)) {
+    const p: any = property;
+    delete p.pattern;
+    if (p.items) {
+      dropObjectPattern(p.items)
+    }
+  }
+}
+
 export const generatePythonClient = async (
   openapiFile: string,
   outputDir: string
 ): Promise<void> => {
   const config = join(__dirname, "config.yaml");
   const generator = "python";
-
+  const specification = JSON.parse(await readFile(openapiFile, {
+    encoding: 'utf-8'
+  }));
+  // python does not pass validation our regexp
+  // example value: '.components.schemas.iam_project_policy_create.properties.resource'
+  for (const schema of Object.values(specification.components.schemas)) {
+    const s: any = schema;
+    dropObjectPattern(s);
+  }
+  // example value: .paths["/iam/project/{projectId}/policy"].get.parameters
+  for (const endpoint of Object.values(specification.paths)) {
+    const e: any = endpoint;
+    for (const operation of Object.values(e)) {
+      const o: any = operation;
+      if (!o.parameters) continue;
+      for (const parameter of o.parameters) {
+        const p: any = parameter;
+        if (!p.schema) continue;
+        delete p.schema.pattern;
+      }
+    }
+  }
+  await writeFile(openapiFile, JSON.stringify(specification, null, 4));
   await execute(
-    `yarn openapi-generator-cli generate -i ${openapiFile} -c ${config} -g ${generator} -o ${outputDir}`
+    `yarn openapi-generator-cli generate --git-user-id "hyperonecom" --git-repo-id "h1-client-python" -i ${openapiFile} -c ${config} -g ${generator} -o ${outputDir}`
   );
 
   await execute("mv README.md docs/README.md", outputDir);
@@ -32,7 +67,7 @@ export const generatePythonClient = async (
   const replacementReadmeLocation = join(__dirname, "_README.md");
   await execute(`cp ${replacementReadmeLocation} README.md`, outputDir);
 
-  await execute("pip install .", outputDir);
+  await execute("python -m pip install .", outputDir);
 
   await copyLicense("MIT", outputDir);
 };

src/generators/ts/index.ts

@@ -11,7 +11,7 @@ export const generateTSClient = async (
   const generator = "typescript-axios";
 
   await execute(
-    `yarn openapi-generator-cli generate -i ${openapiFile} -c ${config} -g ${generator} -o ${outputDir}`
+    `yarn openapi-generator-cli generate --git-user-id "hyperonecom" --git-repo-id "h1-client-js" -i ${openapiFile} -c ${config} -g ${generator} -o ${outputDir}`
   );
   await execute(
     "yarn add -D typedoc@0.19.2 typedoc-plugin-markdown@3.0.3",

src/utils/shellUtils.ts

@@ -1,12 +1,16 @@
 import { exec, ExecOptions } from "child_process";
 
+const TEN_MEGA_BYTE = 1024 * 1024 * 10;
+
 export const execute = async (cmd: string, cwd?: string) => {
   const options: ExecOptions = {};
   options.cwd = cwd;
+  options.maxBuffer = TEN_MEGA_BYTE;
 
   return new Promise((resolve, reject) => {
     exec(cmd, options, (err, stdout, stderr) => {
       if (err) {
+        console.log(`Command '${cmd}' execution failed`)
         reject(err);
       } else {
         console.log(stdout);

tests/go/_api_test.go

@@ -13,7 +13,7 @@ func TestPassportContext(t *testing.T) {
 	p, err := GetPassportContextProvider("", context.Background())
 	require.NoError(t, err)
 
-	_, response, err := c.IamProjectApi.IamProjectList(p.Ctx(), nil)
+	_, response, err := c.IamProjectApi.IamProjectList(p.Ctx()).Execute()
 	require.NoError(t, err)
 
 	if response.StatusCode != 200 {
@@ -30,7 +30,7 @@ func TestPassportContextWithError(t *testing.T) {
 	ctx, err := p.CtxWithError()
 	require.NoError(t, err)
 
-	_, response, err := c.IamProjectApi.IamProjectList(ctx, nil)
+	_, response, err := c.IamProjectApi.IamProjectList(ctx).Execute()
 	require.NoError(t, err)
 
 	if response.StatusCode != 200 {

tests/python/execute.sh

@@ -1,4 +1,6 @@
-pip install -r ../../clients/python/requirements.txt
-pip install git+git://github.com/kuskoman/h1-credentials-helper-python.git
+#!/bin/sh
+set -eux
+python -m pip install -r ../../clients/python/requirements.txt
+python -m pip install git+git://github.com/hyperonecom/h1-credentials-helper-python.git
 
 python test.py