Merge pull request #305 from totten/master-releaser

Multiple toolchain updates

README.md

@@ -80,26 +80,26 @@ civix generate:page --help
 
 ### Development: Custom Build
 
-If you are developing new changes to `civix` and want to create custom build of
-`civix.phar` from source, you must have [`git`](https://git-scm.com), [`composer`](https://getcomposer.org/), and
-[`box`](http://box-project.github.io/box2/) installed. Then run:
+`civix.phar` is usually compiled inside a [nix](https://nixos.org/download.html) shell, i.e.
 
-```
-$ git clone https://github.com/totten/civix
-...
-$ cd civix
-$ composer install
-...
-$ which box
-/usr/local/bin/box
-$ php -dphar.readonly=0 /usr/local/bin/box build
+```bash
+nix-shell --run ./scripts/build.sh
 ```
 
-If you want to run with the same versions of PHP+box that are used for official builds, then run:
+You may also compile it manually in another environment -- if you have [`git`](https://git-scm.com),
+[composer](https://getcomposer.org/), and [box](http://box-project.github.io/box2/):
 
+```bash
+git clone https://github.com/totten/civix
+cd civix
+composer install
+box compile
 ```
-nix-shell --run ./build.sh
-```
+
+> __Tips__
+>
+> * To match exact versions of the toolchain, consult [shell.nix](shell.nix) and the corresponding release of [buildkit pkgs](https://github.com/civicrm/civicrm-buildkit/blob/master/nix/pkgs/default.nix).
+> * `box` may require updating `php.ini`.
 
 ### Development: Testing
 

scripts/build.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+## Determine the absolute path of the directory with the file
+## usage: absdirname <file-path>
+function absdirname() {
+  pushd $(dirname $0) >> /dev/null
+    pwd
+  popd >> /dev/null
+}
+
+SCRDIR=$(absdirname "$0")
+PRJDIR=$(dirname "$SCRDIR")
+OUTFILE="$PRJDIR/bin/civix.phar"
+set -ex
+
+## Box's temp file convention is not multi-user aware. Prone to permission error when second user tries to write.
+export TMPDIR="/tmp/box-$USER"
+if [ ! -d "$TMPDIR" ]; then mkdir "$TMPDIR"  ; fi
+
+pushd "$PRJDIR" >> /dev/null
+  composer install --prefer-dist --no-progress --no-suggest --no-dev
+  box compile -v
+  php scripts/check-phar.php "$OUTFILE"
+popd >> /dev/null

scripts/make-snapshots.sh

@@ -165,11 +165,11 @@ set -ex
 # Main
 
 if [ "$CIVIX_BUILD_TYPE" = "--phar" ]; then
-  if [ ! -f "box.json" -o ! -f "build.sh" ]; then
+  if [ ! -f "box.json" -o ! -f "scripts/build.sh" ]; then
     echo "Must call from civix root dir"
     exit 1
   fi
-  ./build.sh
+  ./scripts/build.sh
   CIVIX="$PWD"/bin/civix.phar
 else
   composer install

scripts/releaser.php

@@ -0,0 +1,179 @@
+#!/usr/bin/env pogo
+<?php
+#!depdir ../extern/releaser-deps
+#!require clippy/std: ~0.4.6
+#!require clippy/container: '~1.2'
+
+###############################################################################
+## Bootstrap
+namespace Clippy;
+
+use Symfony\Component\Console\Style\SymfonyStyle;
+
+assertThat(PHP_SAPI === 'cli', "Releaser may only run via CLI");
+$c = clippy()->register(plugins());
+
+###############################################################################
+## Configuration
+
+$c['ghRepo'] = 'totten/civix';
+$c['srcDir'] = fn() => realpath(dirname(pogo_script_dir()));
+$c['buildDir'] = fn($srcDir) => autodir("$srcDir/build");
+$c['distDir'] = fn($buildDir) => autodir("$buildDir/dist");
+$c['toolName'] = fn($boxOutputPhar) => preg_replace(';\.phar$;', '', basename($boxOutputPhar));
+$c['gcloudUrl'] = fn($toolName) => joinUrl('gs://civicrm', $toolName);
+
+// Ex: "v1.2.3" ==> publishedTagName="v1.2.3", publishedPharName="mytool-1.2.3.phar"
+// Ex: "1.2.3"  ==> publishedTagName="v1.2.3", publishedPharName="mytool-1.2.3.phar"
+$c['publishedTagName'] = fn($input) => preg_replace(';^v?([\d\.]+);', 'v\1', $input->getArgument('new-version'));
+$c['publishedPharName'] = fn($toolName, $publishedTagName) => $toolName . "-" . preg_replace(';^v;', '', $publishedTagName) . '.phar';
+
+###############################################################################
+## Services and other helpers
+
+$c['gpg'] = function(Credentials $cred): \Crypt_GPG {
+  // It's easier to sign multiple files if we use Crypt_GPG wrapper API.
+  #!require pear/crypt_gpg: ~1.6.4
+  $gpg = new \Crypt_GPG(['binary' => trim(`which gpg`)]);
+  $gpg->addSignKey($cred->get('GPG_KEY'), $cred->get('GPG_PASSPHRASE'));
+  return $gpg;
+};
+
+$c['boxJson'] = function(string $srcDir): array {
+  $file = $srcDir . '/box.json';
+  assertThat(file_exists($file), "File not found: $file");
+  return fromJSON(file_get_contents($file));
+};
+
+// Ex: /home/me/src/mytool/bin/mytool.phar
+$c['boxOutputPhar'] = function($srcDir, $boxJson) {
+  assertThat(!empty($boxJson['output']), 'box.json must declare output file');
+  return $srcDir . '/' . $boxJson['output'];
+};
+
+/**
+ * Make a directory (if needed). Return the name.
+ * @param string $path
+ * @return string
+ */
+function autodir(string $path): string {
+  if (!file_exists($path)) {
+    mkdir($path);
+  }
+  return $path;
+}
+
+###############################################################################
+## Commands
+$globalOptions = '[-N|--dry-run] [-S|--step]';
+$commonOptions = '[-N|--dry-run] [-S|--step] new-version';
+
+$c['app']->command("release $commonOptions", function (string $publishedTagName, SymfonyStyle $io, Taskr $taskr) use ($c) {
+  if ($vars = $io->askHidden('(Optional) Paste a batch list of secrets (KEY1=VALUE1 KEY2=VALUE2...)')) {
+    assertThat(!preg_match(';[\'\\"];', $vars), "Sorry, not clever enough to handle meta-characters.");
+    foreach (explode(' ', $vars) as $keyValue) {
+      [$key, $value] = explode('=', $keyValue, 2);
+      putenv($keyValue);
+      $_ENV[$key] = $_SERVER[$key] = $value;
+    }
+  }
+
+  $taskr->subcommand('tag {{0|s}}', [$publishedTagName]);
+  $taskr->subcommand('build {{0|s}}', [$publishedTagName]);
+  $taskr->subcommand('sign {{0|s}}', [$publishedTagName]);
+  $taskr->subcommand('upload {{0|s}}', [$publishedTagName]);
+  $taskr->subcommand('tips {{0|s}}', [$publishedTagName]);
+  // TODO: $taskr->subcommand('clean {{0|s}}', [$publishedTagName]);
+});
+
+$c['app']->command("tag $commonOptions", function ($publishedTagName, SymfonyStyle $io, Taskr $taskr) use ($c) {
+  $io->title("Create tag ($publishedTagName)");
+  chdir($c['srcDir']);
+  $taskr->passthru('git tag -f {{0|s}}', [$publishedTagName]);
+});
+
+$c['app']->command("build $commonOptions", function (SymfonyStyle $io, Taskr $taskr) use ($c) {
+  $io->title('Build PHAR');
+  chdir($c['srcDir']);
+  $taskr->passthru('bash ./scripts/build.sh');
+});
+
+$c['app']->command("sign $commonOptions", function (SymfonyStyle $io, Taskr $taskr, \Crypt_GPG $gpg, $input) use ($c) {
+  $io->title('Generate checksum and GPG signature');
+  ['Init', $c['srcDir'], $c['distDir'], $c['publishedPharName']];
+  chdir($c['distDir']);
+
+  $pharFile = $c['publishedPharName'];
+  $sha256File = preg_replace(';\.phar$;', '.SHA256SUMS', $pharFile);
+
+  $taskr->passthru('cp {{0|s}} {{1|s}}', [$c['boxOutputPhar'], $pharFile]);
+  $taskr->passthru('sha256sum {{0|s}} > {{1|s}}', [$pharFile, $sha256File]);
+
+  $io->writeln("Sign $pharFile ($pharFile.asc)");
+  if (!$input->getOption('dry-run')) {
+    $gpg->signFile($pharFile, "$pharFile.asc", \Crypt_GPG::SIGN_MODE_DETACHED);
+    assertThat(!empty($gpg->verifyFile($pharFile, file_get_contents("$pharFile.asc"))), "$pharFile should have valid signature");
+  }
+});
+
+$c['app']->command("upload $commonOptions", function ($publishedTagName, SymfonyStyle $io, Taskr $taskr, Credentials $cred) use ($c) {
+  $io->title("Upload code and build artifacts");
+  ['Init', $c['srcDir'], $c['ghRepo'], $c['distDir'], $c['publishedPharName']];
+  chdir($c['srcDir']);
+
+  $vars = [
+    'GCLOUD' => $c['gcloudUrl'],
+    'GH_TOKEN' => 'GH_TOKEN=' . $cred->get('GH_TOKEN', $c['ghRepo']),
+    'VER' => $publishedTagName,
+    'REPO' => $c['ghRepo'],
+    'DIST' => $c['distDir'],
+    'PHAR' => $c['distDir'] . '/' . $c['publishedPharName'],
+    'PHAR_NAME' => $c['publishedPharName'],
+    'TOOL_NAME' => basename($c['boxOutputPhar']),
+  ];
+
+  $io->section('Check connections');
+  $taskr->run('gsutil ls {{GCLOUD|s}}', $vars);
+  $taskr->run('{{GH_TOKEN|s}} gh release list', $vars);
+
+  $io->section('Send source-code to Github');
+  $taskr->passthru('git push -f origin {{VER|s}}', $vars);
+
+  $io->section('Send binaries to Github');
+  $taskr->passthru('{{GH_TOKEN|s}} gh release create {{VER|s}} --repo {{REPO|s}} --generate-notes', $vars);
+  $taskr->passthru('{{GH_TOKEN|s}} gh release upload {{VER|s}} --repo {{REPO|s}} --clobber {{DIST|s}}/*', $vars);
+
+  $io->section('Send binaries to Google Cloud Storage');
+  $taskr->passthru('gsutil cp {{DIST|s}}/* {{GCLOUD|s}}/', $vars);
+  if (preg_match(';^v\d;', $publishedTagName)) {
+    // Finalize: "mytool-1.2.3.phar" will be the default "mytool.phar"
+    $suffixes = ['.phar', '.phar.asc', '.SHA256SUMS'];
+    foreach ($suffixes as $suffix) {
+      $taskr->passthru('gsutil cp {{GCLOUD|s}}/{{OLD_NAME}} {{GCLOUD|s}}/{{NEW_NAME}}', $vars + [
+        'OLD_NAME' => preg_replace(';\.phar$;', $suffix, $c['publishedPharName']),
+        'NEW_NAME' => preg_replace(';\.phar$;', $suffix, basename($c['boxOutputPhar'])),
+      ]);
+    }
+  }
+});
+
+$c['app']->command("tips $commonOptions", function (SymfonyStyle $io) use ($c) {
+  $io->title('Tips');
+  $cleanup = sprintf('%s clean', basename(__FILE__));
+  $io->writeln("Cleanup temp files: <comment>$cleanup</comment>");
+  $url = sprintf('https://github.com/%s/releases/edit/%s', $c['ghRepo'], $c['publishedTagName']);
+  $io->writeln("Update release notes: <comment>$url</comment>");
+});
+
+$c['app']->command("clean $globalOptions", function (SymfonyStyle $io, Taskr $taskr) use ($c) {
+  $io->title('Clean build directory');
+  ['Init', $c['srcDir'], $c['buildDir'], $c['boxOutputPhar']];
+  chdir($c['srcDir']);
+
+  $taskr->passthru('rm -rf {{0|@s}}', [[$c['buildDir'], $c['boxOutputPhar']]]);
+});
+
+###############################################################################
+## Go!
+
+$c['app']->run();

shell.nix

@@ -1,24 +1,42 @@
 /**
  * This shell is suitable for compiling civix.phar.... and not much else.
  *
- * Ex: `nix-shell --run ./build.sh`
+ * Ex: `nix-shell --run ./scripts/build.sh`
  */
-# { pkgs ? import <nixpkgs> {} }:
+
+{ pkgs ? import <nixpkgs> {} }:
+
 let
-  pkgSrc = fetchTarball {
-    url = "https://github.com/nixos/nixpkgs/archive/ce6aa13369b667ac2542593170993504932eb836.tar.gz";
-    sha256 = "0d643wp3l77hv2pmg2fi7vyxn4rwy0iyr8djcw1h5x72315ck9ik";
-  };
-  pkgs = import pkgSrc {};
-  myphp = pkgs.php81.buildEnv {
-    extraConfig = ''
-      memory_limit=-1
-    '';
-  };
+
+  buildkit = import (pkgs.fetchFromGitHub {
+    owner = "totten";
+    repo = "civicrm-buildkit";
+    rev = "153371e9bdcb22392b878cca545df0888fb61925";
+    sha256 = "sha256-rdwmA4uqIqfqXu2f+ewVH0Gs/BzcB13p8oRbbTdUsAs=";
+  });
+
+  ## If you're trying to patch buildkit at the sametime, then use a local copy:
+  #buildkit = import ((builtins.getEnv "HOME") + "/bknix/default.nix");
 
 in
 
   pkgs.mkShell {
-    # nativeBuildInputs is usually what you want -- tools you need to run
-    nativeBuildInputs = [ myphp pkgs.php81Packages.composer ];
-}
+    nativeBuildInputs = buildkit.profiles.base ++ [
+
+      (buildkit.pins.v2305.php81.buildEnv {
+        extraConfig = ''
+          memory_limit=-1
+        '';
+      })
+
+      buildkit.pkgs.box
+      buildkit.pkgs.composer
+      buildkit.pkgs.pogo
+      buildkit.pkgs.phpunit8
+
+      pkgs.bash-completion
+    ];
+    shellHook = ''
+      source ${pkgs.bash-completion}/etc/profile.d/bash_completion.sh
+    '';
+  }