From e08141365942ec6f54084a1734294fa38b0ef5bb Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Wed, 2 Aug 2023 01:35:36 -0700 Subject: [PATCH 1/5] shell.nix - Import key definitions from buildkit --- shell.nix | 48 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/shell.nix b/shell.nix index c39ac84f..d7fa37c3 100644 --- a/shell.nix +++ b/shell.nix @@ -1,24 +1,42 @@ /** * This shell is suitable for compiling civix.phar.... and not much else. * - * Ex: `nix-shell --run ./build.sh` + * Ex: `nix-shell --run ./scripts/build.sh` */ -# { pkgs ? import {} }: + +{ pkgs ? import {} }: + let - pkgSrc = fetchTarball { - url = "https://github.com/nixos/nixpkgs/archive/ce6aa13369b667ac2542593170993504932eb836.tar.gz"; - sha256 = "0d643wp3l77hv2pmg2fi7vyxn4rwy0iyr8djcw1h5x72315ck9ik"; - }; - pkgs = import pkgSrc {}; - myphp = pkgs.php81.buildEnv { - extraConfig = '' - memory_limit=-1 - ''; - }; + + buildkit = import (pkgs.fetchFromGitHub { + owner = "totten"; + repo = "civicrm-buildkit"; + rev = "153371e9bdcb22392b878cca545df0888fb61925"; + sha256 = "sha256-rdwmA4uqIqfqXu2f+ewVH0Gs/BzcB13p8oRbbTdUsAs="; + }); + + ## If you're trying to patch buildkit at the sametime, then use a local copy: + #buildkit = import ((builtins.getEnv "HOME") + "/bknix/default.nix"); in pkgs.mkShell { - # nativeBuildInputs is usually what you want -- tools you need to run - nativeBuildInputs = [ myphp pkgs.php81Packages.composer ]; -} + nativeBuildInputs = buildkit.profiles.base ++ [ + + (buildkit.pins.v2305.php81.buildEnv { + extraConfig = '' + memory_limit=-1 + ''; + }) + + buildkit.pkgs.box + buildkit.pkgs.composer + buildkit.pkgs.pogo + buildkit.pkgs.phpunit8 + + pkgs.bash-completion + ]; + shellHook = '' + source ${pkgs.bash-completion}/etc/profile.d/bash_completion.sh + ''; + } From 371b61ac332bc427eff594c35263147b608a5d86 Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Thu, 3 Aug 2023 15:50:06 -0700 Subject: [PATCH 2/5] Simplify build.sh --- build.sh | 24 +----------------------- 1 file changed, 1 insertion(+), 23 deletions(-) diff --git a/build.sh b/build.sh index cdef888f..5858e532 100755 --- a/build.sh +++ b/build.sh @@ -12,34 +12,12 @@ PRJDIR=$(absdirname "$0") OUTFILE="$PRJDIR/bin/civix.phar" set -ex -BOX_VERSION=4.3.8 -BOX_URL="https://github.com/box-project/box/releases/download/${BOX_VERSION}/box.phar" -BOX_DIR="$PRJDIR/extern/box-$BOX_VERSION" -BOX_BIN="$BOX_DIR/box" -[ ! -f "$BOX_BIN" ] && ( mkdir -p "$BOX_DIR" ; curl -L "$BOX_URL" -o "$BOX_BIN" ) - ## Box's temp file convention is not multi-user aware. Prone to permission error when second user tries to write. export TMPDIR="/tmp/box-$USER" if [ ! -d "$TMPDIR" ]; then mkdir "$TMPDIR" ; fi pushd "$PRJDIR" >> /dev/null composer install --prefer-dist --no-progress --no-suggest --no-dev - BOX_ALLOW_XDEBUG=1 php -d phar.readonly=0 "$BOX_BIN" compile -v - - ## Box needs the PHP INI to specify `phar.readonly=0`. We've being doing this with `php -d` since forever. - ## It appears that newer versions of Box try to do this automatically (yah!), but the implementation is buggy (arg!). - ## Setting BOX_ALLOW_XDEBUG=1 opts-out of the buggy implementation. - - ## The specific bug - it shows a bazillion warnings like this (observed on bknix with php74 or php80) - ## Ex: `Warning: Module "memcached" is already loaded in Unknown on line 0` - ## In some cases, these warnings appear as errors. (I suspect the extra output provokes the error.) - ## Ex: When `box compile` calls down to `composer dumpautoload`, esp on php80 - - ## How to opt-out of the buggy implementation? One needs to see that Box has borrowed half of the implementation from - ## `composer/xdebug-handler`. (Both have a need to manipulate PHP INI.) The flag `BOX_ALLOW_XDEBUG` is defined by their - ## upstream. Setting the flag doesn't actually configure xdebug -- rather, it disables PHP INI automanipulations, so that you - ## are _allowed_ to set PHP INI options (`xdebug.*`, `phar.*`, etc) on your own. - + box compile -v php scripts/check-phar.php "$OUTFILE" - popd >> /dev/null From 86204af8da96db21abdf6746bc684c50f9e9e2a5 Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Thu, 3 Aug 2023 16:01:03 -0700 Subject: [PATCH 3/5] Move build.sh into ./scripts/ --- build.sh => scripts/build.sh | 3 ++- scripts/make-snapshots.sh | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) rename build.sh => scripts/build.sh (91%) diff --git a/build.sh b/scripts/build.sh similarity index 91% rename from build.sh rename to scripts/build.sh index 5858e532..f5e4c95a 100755 --- a/build.sh +++ b/scripts/build.sh @@ -8,7 +8,8 @@ function absdirname() { popd >> /dev/null } -PRJDIR=$(absdirname "$0") +SCRDIR=$(absdirname "$0") +PRJDIR=$(dirname "$SCRDIR") OUTFILE="$PRJDIR/bin/civix.phar" set -ex diff --git a/scripts/make-snapshots.sh b/scripts/make-snapshots.sh index 65b4cb74..5676e006 100755 --- a/scripts/make-snapshots.sh +++ b/scripts/make-snapshots.sh @@ -165,11 +165,11 @@ set -ex # Main if [ "$CIVIX_BUILD_TYPE" = "--phar" ]; then - if [ ! -f "box.json" -o ! -f "build.sh" ]; then + if [ ! -f "box.json" -o ! -f "scripts/build.sh" ]; then echo "Must call from civix root dir" exit 1 fi - ./build.sh + ./scripts/build.sh CIVIX="$PWD"/bin/civix.phar else composer install From 588e4d853b0f02ef1102b311b040ab6cc715b44c Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Thu, 3 Aug 2023 15:48:01 -0700 Subject: [PATCH 4/5] README.md - Recommend nix-shell first --- README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index df66f20d..5352ff39 100644 --- a/README.md +++ b/README.md @@ -80,26 +80,26 @@ civix generate:page --help ### Development: Custom Build -If you are developing new changes to `civix` and want to create custom build of -`civix.phar` from source, you must have [`git`](https://git-scm.com), [`composer`](https://getcomposer.org/), and -[`box`](http://box-project.github.io/box2/) installed. Then run: +`civix.phar` is usually compiled inside a [nix](https://nixos.org/download.html) shell, i.e. -``` -$ git clone https://github.com/totten/civix -... -$ cd civix -$ composer install -... -$ which box -/usr/local/bin/box -$ php -dphar.readonly=0 /usr/local/bin/box build +```bash +nix-shell --run ./scripts/build.sh ``` -If you want to run with the same versions of PHP+box that are used for official builds, then run: +You may also compile it manually in another environment -- if you have [`git`](https://git-scm.com), +[composer](https://getcomposer.org/), and [box](http://box-project.github.io/box2/): +```bash +git clone https://github.com/totten/civix +cd civix +composer install +box compile ``` -nix-shell --run ./build.sh -``` + +> __Tips__ +> +> * To match exact versions of the toolchain, consult [shell.nix](shell.nix) and the corresponding release of [buildkit pkgs](https://github.com/civicrm/civicrm-buildkit/blob/master/nix/pkgs/default.nix). +> * `box` may require updating `php.ini`. ### Development: Testing From 5ef4606873cf7a86695aa5d1969a322cfc6ce4fe Mon Sep 17 00:00:00 2001 From: Tim Otten Date: Thu, 3 Aug 2023 22:20:49 -0700 Subject: [PATCH 5/5] Add scripts/releaser.php --- scripts/releaser.php | 179 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100755 scripts/releaser.php diff --git a/scripts/releaser.php b/scripts/releaser.php new file mode 100755 index 00000000..fec09970 --- /dev/null +++ b/scripts/releaser.php @@ -0,0 +1,179 @@ +#!/usr/bin/env pogo +register(plugins()); + +############################################################################### +## Configuration + +$c['ghRepo'] = 'totten/civix'; +$c['srcDir'] = fn() => realpath(dirname(pogo_script_dir())); +$c['buildDir'] = fn($srcDir) => autodir("$srcDir/build"); +$c['distDir'] = fn($buildDir) => autodir("$buildDir/dist"); +$c['toolName'] = fn($boxOutputPhar) => preg_replace(';\.phar$;', '', basename($boxOutputPhar)); +$c['gcloudUrl'] = fn($toolName) => joinUrl('gs://civicrm', $toolName); + +// Ex: "v1.2.3" ==> publishedTagName="v1.2.3", publishedPharName="mytool-1.2.3.phar" +// Ex: "1.2.3" ==> publishedTagName="v1.2.3", publishedPharName="mytool-1.2.3.phar" +$c['publishedTagName'] = fn($input) => preg_replace(';^v?([\d\.]+);', 'v\1', $input->getArgument('new-version')); +$c['publishedPharName'] = fn($toolName, $publishedTagName) => $toolName . "-" . preg_replace(';^v;', '', $publishedTagName) . '.phar'; + +############################################################################### +## Services and other helpers + +$c['gpg'] = function(Credentials $cred): \Crypt_GPG { + // It's easier to sign multiple files if we use Crypt_GPG wrapper API. + #!require pear/crypt_gpg: ~1.6.4 + $gpg = new \Crypt_GPG(['binary' => trim(`which gpg`)]); + $gpg->addSignKey($cred->get('GPG_KEY'), $cred->get('GPG_PASSPHRASE')); + return $gpg; +}; + +$c['boxJson'] = function(string $srcDir): array { + $file = $srcDir . '/box.json'; + assertThat(file_exists($file), "File not found: $file"); + return fromJSON(file_get_contents($file)); +}; + +// Ex: /home/me/src/mytool/bin/mytool.phar +$c['boxOutputPhar'] = function($srcDir, $boxJson) { + assertThat(!empty($boxJson['output']), 'box.json must declare output file'); + return $srcDir . '/' . $boxJson['output']; +}; + +/** + * Make a directory (if needed). Return the name. + * @param string $path + * @return string + */ +function autodir(string $path): string { + if (!file_exists($path)) { + mkdir($path); + } + return $path; +} + +############################################################################### +## Commands +$globalOptions = '[-N|--dry-run] [-S|--step]'; +$commonOptions = '[-N|--dry-run] [-S|--step] new-version'; + +$c['app']->command("release $commonOptions", function (string $publishedTagName, SymfonyStyle $io, Taskr $taskr) use ($c) { + if ($vars = $io->askHidden('(Optional) Paste a batch list of secrets (KEY1=VALUE1 KEY2=VALUE2...)')) { + assertThat(!preg_match(';[\'\\"];', $vars), "Sorry, not clever enough to handle meta-characters."); + foreach (explode(' ', $vars) as $keyValue) { + [$key, $value] = explode('=', $keyValue, 2); + putenv($keyValue); + $_ENV[$key] = $_SERVER[$key] = $value; + } + } + + $taskr->subcommand('tag {{0|s}}', [$publishedTagName]); + $taskr->subcommand('build {{0|s}}', [$publishedTagName]); + $taskr->subcommand('sign {{0|s}}', [$publishedTagName]); + $taskr->subcommand('upload {{0|s}}', [$publishedTagName]); + $taskr->subcommand('tips {{0|s}}', [$publishedTagName]); + // TODO: $taskr->subcommand('clean {{0|s}}', [$publishedTagName]); +}); + +$c['app']->command("tag $commonOptions", function ($publishedTagName, SymfonyStyle $io, Taskr $taskr) use ($c) { + $io->title("Create tag ($publishedTagName)"); + chdir($c['srcDir']); + $taskr->passthru('git tag -f {{0|s}}', [$publishedTagName]); +}); + +$c['app']->command("build $commonOptions", function (SymfonyStyle $io, Taskr $taskr) use ($c) { + $io->title('Build PHAR'); + chdir($c['srcDir']); + $taskr->passthru('bash ./scripts/build.sh'); +}); + +$c['app']->command("sign $commonOptions", function (SymfonyStyle $io, Taskr $taskr, \Crypt_GPG $gpg, $input) use ($c) { + $io->title('Generate checksum and GPG signature'); + ['Init', $c['srcDir'], $c['distDir'], $c['publishedPharName']]; + chdir($c['distDir']); + + $pharFile = $c['publishedPharName']; + $sha256File = preg_replace(';\.phar$;', '.SHA256SUMS', $pharFile); + + $taskr->passthru('cp {{0|s}} {{1|s}}', [$c['boxOutputPhar'], $pharFile]); + $taskr->passthru('sha256sum {{0|s}} > {{1|s}}', [$pharFile, $sha256File]); + + $io->writeln("Sign $pharFile ($pharFile.asc)"); + if (!$input->getOption('dry-run')) { + $gpg->signFile($pharFile, "$pharFile.asc", \Crypt_GPG::SIGN_MODE_DETACHED); + assertThat(!empty($gpg->verifyFile($pharFile, file_get_contents("$pharFile.asc"))), "$pharFile should have valid signature"); + } +}); + +$c['app']->command("upload $commonOptions", function ($publishedTagName, SymfonyStyle $io, Taskr $taskr, Credentials $cred) use ($c) { + $io->title("Upload code and build artifacts"); + ['Init', $c['srcDir'], $c['ghRepo'], $c['distDir'], $c['publishedPharName']]; + chdir($c['srcDir']); + + $vars = [ + 'GCLOUD' => $c['gcloudUrl'], + 'GH_TOKEN' => 'GH_TOKEN=' . $cred->get('GH_TOKEN', $c['ghRepo']), + 'VER' => $publishedTagName, + 'REPO' => $c['ghRepo'], + 'DIST' => $c['distDir'], + 'PHAR' => $c['distDir'] . '/' . $c['publishedPharName'], + 'PHAR_NAME' => $c['publishedPharName'], + 'TOOL_NAME' => basename($c['boxOutputPhar']), + ]; + + $io->section('Check connections'); + $taskr->run('gsutil ls {{GCLOUD|s}}', $vars); + $taskr->run('{{GH_TOKEN|s}} gh release list', $vars); + + $io->section('Send source-code to Github'); + $taskr->passthru('git push -f origin {{VER|s}}', $vars); + + $io->section('Send binaries to Github'); + $taskr->passthru('{{GH_TOKEN|s}} gh release create {{VER|s}} --repo {{REPO|s}} --generate-notes', $vars); + $taskr->passthru('{{GH_TOKEN|s}} gh release upload {{VER|s}} --repo {{REPO|s}} --clobber {{DIST|s}}/*', $vars); + + $io->section('Send binaries to Google Cloud Storage'); + $taskr->passthru('gsutil cp {{DIST|s}}/* {{GCLOUD|s}}/', $vars); + if (preg_match(';^v\d;', $publishedTagName)) { + // Finalize: "mytool-1.2.3.phar" will be the default "mytool.phar" + $suffixes = ['.phar', '.phar.asc', '.SHA256SUMS']; + foreach ($suffixes as $suffix) { + $taskr->passthru('gsutil cp {{GCLOUD|s}}/{{OLD_NAME}} {{GCLOUD|s}}/{{NEW_NAME}}', $vars + [ + 'OLD_NAME' => preg_replace(';\.phar$;', $suffix, $c['publishedPharName']), + 'NEW_NAME' => preg_replace(';\.phar$;', $suffix, basename($c['boxOutputPhar'])), + ]); + } + } +}); + +$c['app']->command("tips $commonOptions", function (SymfonyStyle $io) use ($c) { + $io->title('Tips'); + $cleanup = sprintf('%s clean', basename(__FILE__)); + $io->writeln("Cleanup temp files: $cleanup"); + $url = sprintf('https://github.com/%s/releases/edit/%s', $c['ghRepo'], $c['publishedTagName']); + $io->writeln("Update release notes: $url"); +}); + +$c['app']->command("clean $globalOptions", function (SymfonyStyle $io, Taskr $taskr) use ($c) { + $io->title('Clean build directory'); + ['Init', $c['srcDir'], $c['buildDir'], $c['boxOutputPhar']]; + chdir($c['srcDir']); + + $taskr->passthru('rm -rf {{0|@s}}', [[$c['buildDir'], $c['boxOutputPhar']]]); +}); + +############################################################################### +## Go! + +$c['app']->run();