Skip to content

Commit

Permalink
fix: GitHub PAT for Argo Workflows Pull Requests TDE-940 (#768)
Browse files Browse the repository at this point in the history
#### Motivation

To be consistent with Basemaps and so bulk processing for imagery and
elevation is more reliable.

#### Modification

Move imagery and elevation GitHub Pull Requests from secrets to a PAT.
Use existing Basemaps code, moving the core function into `utils` for
common use.

#### Checklist

_If not applicable, provide explanation of why._

Tests still run successfully.

- [ ] Tests updated
- [X] Docs updated
- [X] Issue linked in Title

---------

Co-authored-by: paulfouquet <86932794+paulfouquet@users.noreply.github.com>
  • Loading branch information
amfage and paulfouquet authored Nov 29, 2023
1 parent 9d4563f commit 392371f
Show file tree
Hide file tree
Showing 4 changed files with 27 additions and 46 deletions.
4 changes: 0 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -196,10 +196,6 @@ Format and push a STAC collection.json file to a GitHub repository. Used by the

#### Example

Environment variables:
GIT_AUTHOR_NAME="Example User"
GIT_AUTHOR_EMAIL="<example@example.com>"

```bash
stac github-import --source s3://path/to/collection/ --target s3://linz-imagery/path/to/dataset/ --repo-name "linz/imagery-test)" (`--repo-name` is optional and defaults to linz/imagery).
```
Expand Down
2 changes: 1 addition & 1 deletion src/commands/basemaps-github/make.cog.github.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ import { fsa } from '@chunkd/fs';

import { logger } from '../../log.js';
import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.js';
import { createPR, GithubApi } from '../../utils/github.js';
import { prettyPrint } from '../format/pretty.print.js';
import { createPR, GithubApi } from './github.js';

export enum Category {
Urban = 'Urban Aerial Photos',
Expand Down
63 changes: 24 additions & 39 deletions src/commands/stac-github-import/stac.github.import.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
import path from 'node:path';

import { fsa } from '@chunkd/fs';
import { execFileSync } from 'child_process';
import { command, option, string, Type } from 'cmd-ts';
import * as st from 'stac-ts';

import { CliInfo } from '../../cli.info.js';
import { logger } from '../../log.js';
import { DEFAULT_PRETTIER_FORMAT } from '../../utils/config.js';
import { createPR, GithubApi } from '../../utils/github.js';
import { config, registerCli, verbose } from '../common.js';
import { prettyPrint } from '../format/pretty.print.js';

Expand Down Expand Up @@ -48,57 +46,44 @@ export const commandStacGithubImport = command({
async handler(args) {
registerCli(this, args);

const gitName = process.env['GIT_AUTHOR_NAME'] ?? 'imagery[bot]';
const gitEmail = process.env['GIT_AUTHOR_EMAIL'] ?? 'imagery@linz.govt.nz';

const sourceCollection = new URL('collection.json', args.source);
const targetCollection = new URL('collection.json', args.target);

const collection = await fsa.readJson<st.StacCollection>(sourceCollection.href);
const gh = new GithubApi(args.repoName);

const gitRepo = '/tmp/gitrepo/';
const collectionPath = path.join(gitRepo, 'stac', targetCollection.pathname);

// Clone the GitHub repo
logger.info({ repo: args.repoName }, 'Git:clone');
execFileSync('git', ['clone', `git@github.com:${args.repoName}`, gitRepo]);
execFileSync('git', ['config', 'user.email', gitEmail], { cwd: gitRepo });
execFileSync('git', ['config', 'user.name', gitName], { cwd: gitRepo });

logger.info({ template: path.join(gitRepo, 'template', 'catalog.json') }, 'Stac:ReadTemplate');
// Load information from the template inside the repo
const catalog = await fsa.readJson<st.StacCatalog>(path.join(gitRepo, 'template', 'catalog.json'));
// Catalog template should have a absolute link to its self
const selfLink = catalog.links.find((f) => f.rel === 'self');
if (selfLink == null) throw new Error('unable to find self link in catalog');
logger.info({ template: fsa.joinAll('template', 'catalog.json') }, 'Stac:ReadTemplate');
const catalogPath = fsa.joinAll('template', 'catalog.json');
const catalog = await gh.getContent(catalogPath);
const catalogJson = JSON.parse(catalog) as st.StacCatalog;

// Catalog template should have a absolute link to itself
const selfLink = catalogJson.links.find((f) => f.rel === 'self');
if (selfLink == null) throw new Error('unable to find self link in catalog');
logger.info({ href: selfLink.href }, 'Stac:SetRoot');
// Update the root link in the collection to the one defined in the repo template

sortLinks(collection.links);
const sourceCollection = new URL('collection.json', args.source);
const targetCollection = new URL('collection.json', args.target);
const targetCollectionPath = fsa.joinAll('stac', targetCollection.pathname);

const collection = await fsa.readJson<st.StacCollection>(sourceCollection.href);

// Update the root link in the collection to the one defined in the repo template
const rootLink = collection.links.find((f) => f.rel === 'root');
if (rootLink) {
rootLink.href = selfLink.href;
rootLink.type = 'application/json';
} else {
collection.links.unshift({ rel: 'root', href: selfLink.href, type: 'application/json' });
}

// Write the file to targetCollection
await fsa.write(collectionPath, await prettyPrint(JSON.stringify(collection), DEFAULT_PRETTIER_FORMAT));

execFileSync('git', ['add', collectionPath], { cwd: gitRepo });
logger.info({ path: collectionPath }, 'git:add');
sortLinks(collection.links);

// branch: "feat/bot-01GYXCC7823GVKF6BJA6K354TR"
execFileSync('git', ['checkout', '-B', `feat/bot-${collection.id}`], { cwd: gitRepo });
// commit: "feat: import Manawatū-Whanganui 0.4m Rural Aerial Photos (2010-2011)"
execFileSync('git', ['commit', '-am', `feat: import ${collection.title}`], { cwd: gitRepo });
logger.info({ commit: `feat: import ${collection.title}`, branch: `feat/bot-${collection.id}` }, 'git:commit');

// Push branch
execFileSync('git', ['push', 'origin', 'HEAD', '--force'], { cwd: gitRepo });
const branch = `feat/bot-${collection.id}`;
// commit and pull request title: "feat: import Manawatū-Whanganui 0.4m Rural Aerial Photos (2010-2011)"
const title = `feat: import ${collection.title}`;
const collectionFileContent = await prettyPrint(JSON.stringify(collection), DEFAULT_PRETTIER_FORMAT);
const collectionFile = { path: targetCollectionPath, content: collectionFileContent };
logger.info({ commit: `feat: import ${collection.title}`, branch: `feat/bot-${collection.id}` }, 'Git:Commit');
// create pull request
await createPR(gh, branch, title, [collectionFile]);
},
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { Octokit } from '@octokit/core';
import { restEndpointMethods } from '@octokit/plugin-rest-endpoint-methods';
import { Api } from '@octokit/plugin-rest-endpoint-methods/dist-types/types.js';

import { logger } from '../../log.js';
import { logger } from '../log.js';

export interface Job {
imagery: string;
Expand Down Expand Up @@ -205,7 +205,7 @@ export async function createPR(gh: GithubApi, branch: string, title: string, fil
const commitSha = await gh.createCommit(blobs, title, sha);

// git push
logger.info({ branch }, 'GitHub: Push commit to Brach');
logger.info({ branch }, 'GitHub: Push commit to Branch');
await gh.updateBranch(branch, commitSha);

// git pr create
Expand Down

0 comments on commit 392371f

Please sign in to comment.