Refresh and side tag

README.md

@@ -6,7 +6,7 @@ Scripts to ease the update of Ruby on Rails core packages for Fedora Project.
 ## Usage
 
 ```
-./railsbuild FEDORA_VERSION OLD_RAILS_VERSION NEW_RAILS_VERSION
+./railsbuild FEDORA_BRANCH OLD_RAILS_VERSION NEW_RAILS_VERSION
 ```
 
 To update Rails from 4.1.4 to 4.1.5 for rawhide:
@@ -29,41 +29,33 @@ build, but just let railsbuild to prepare the update.
 ### railsbuild-prepare
 
 ```
-./railsbuild-prepare FEDORA_VERSION
+./railsbuild-prepare FEDORA_BRANCH
 ```
 Prepares directories for the railsbuild scripts including fetching Fedora
 and upstream repositories at ~/.railsbuild/upstream/rails and
-~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem.
-
-### railsbuild-fetch-tests
-
-```
-railsbuild-fetch-tests FEDORA_VERSION RAILS_VERSION
-```
-Gets the upstream tests from the repository at ~/.railsbuild/upstream/rails
-and move the gzipped test suites to Fedora git repositories at
-~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem.
+~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem.
 
 ### railsbuild-update-pkgs
 
 ```
-./railsbuild-update-pkgs FEDORA_VERSION OLD_RAILS_VERSION NEW_RAILS_VERSION
+./railsbuild-update-pkgs FEDORA_BRANCH OLD_RAILS_VERSION NEW_RAILS_VERSION
 ```
 Prepares the update of all Rails gems at once in Fedora git repositories at:
-~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem.
+~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem.
 
-### railsbuild-update-bootstrapped
+### railsbuild-fetch-sources
 
 ```
-./railsbuild-update-bootstrapped FEDORA_VERSION OLD_RAILS_VERSION NEW_RAILS_VERSION
+railsbuild-fetch-sources FEDORA_BRANCH
 ```
-Prepare the update of previously bootstrapped Rails gems at once in Fedora git
-repositories at: ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem.
+Get the information about sources (test suites, js, ...) for Rails gems from
+their spec files, prepare the `git archive` tarballs to Fedora git repositories
+at ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem
 
 ### railsbuild-build
 
 ```
-./railsbuild-build FEDORA_VERSION RAILS_VERSION
+./railsbuild-build FEDORA_BRANCH RAILS_VERSION
 ```
 Does the builds of gems. It always tries to do a scratch-build first and
 if that goes fine, does a final build. Afterwards it waits for the build

railsbuild

@@ -6,15 +6,15 @@
 # Automatic builds of Ruby on Rails core packages for Fedora Project
 #
 # Usage:
-#   ./railsbuild [-n] FEDORA_VERSION OLD_RAILS_VERSION NEW_RAILS_VERSION
+#   ./railsbuild [-n] FEDORA_BRANCH OLD_RAILS_VERSION NEW_RAILS_VERSION
 #
 #     -n  Do not run builds.  Useful for major version upgrades.
 #
 # Configuration file:
 #   railsbuild-common
 
 print_help () {
-  echo "Usage: ./railsbuild [-n] FEDORA_VERSION OLD_RAILS_VERSION NEW_RAILS_VERSION"
+  echo "Usage: ./railsbuild [-n] FEDORA_BRANCH OLD_RAILS_VERSION NEW_RAILS_VERSION"
 }
 
 if [ "$1" = "--help" ]; then
@@ -34,20 +34,20 @@ if [ "$3" = "" ] || [ "$2" = "" ] || [ "$1" = "" ]; then
   exit 1
 fi
 
-FEDORA_VERSION=$1
+FEDORA_BRANCH=$1
 OLD_RAILS_VERSION=$2
 NEW_RAILS_VERSION=$3
 
 # Create directory structure and get the upstream repositories
-$(dirname $(readlink -f "$0"))/railsbuild-prepare $FEDORA_VERSION
-
-# Get the test suites
-$(dirname $(readlink -f "$0"))/railsbuild-fetch-tests $FEDORA_VERSION $NEW_RAILS_VERSION
+$(dirname $(readlink -f "$0"))/railsbuild-prepare $FEDORA_BRANCH
 
 # Update specs
-$(dirname $(readlink -f "$0"))/railsbuild-update-pkgs $FEDORA_VERSION $OLD_RAILS_VERSION $NEW_RAILS_VERSION
+$(dirname $(readlink -f "$0"))/railsbuild-update-pkgs $FEDORA_BRANCH $OLD_RAILS_VERSION $NEW_RAILS_VERSION
+
+# Fetch sources
+$(dirname $(readlink -f "$0"))/railsbuild-fetch-sources $FEDORA_BRANCH $OLD_RAILS_VERSION $NEW_RAILS_VERSION
 
 # Run the build if not opted-out
 [[ "$NO_BUILD" ]] || {
-  $(dirname $(readlink -f "$0"))/railsbuild-build $FEDORA_VERSION $NEW_RAILS_VERSION
+  $(dirname $(readlink -f "$0"))/railsbuild-build $FEDORA_BRANCH $NEW_RAILS_VERSION
 }

railsbuild-build

@@ -6,42 +6,69 @@
 # this script after it failed will pick up the work from the last run.
 #
 # Usage:
-#   ./railsbuild-build FEDORA_VERSION VERSION
+#   ./railsbuild-build FEDORA_BRANCH VERSION
 
 . $(dirname $(readlink -f "$0"))/railsbuild-common
 
-FEDORA_VERSION=$1
+FEDORA_BRANCH=$1
 VERSION=$2
 
+# Check changelog in spec file for same EVR
 for gem in "${GEMS[@]}"
 do
-  pushd ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem
-    # Skip already done builds
-    if [ -f .built ]; then
-      echo "$gem: build for $gem already done"
-      continue
-    fi
-
-    # Check changelog in spec file for same EVR
+  pushd ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem
     if [ -n "$(
       sed -n '/^%changelog/,$p' rubygem-$gem.spec \
-        | grep -E " [0-9]+\:[0-9]+\.[0-9]+\.[0-9]+\-[0-9]+$" \
+        | grep -E " ([0-9]+:)?[0-9]+\.[0-9]+\.[0-9]+-[0-9]+$" \
         | head -n 2 | rev | cut -d' ' -f1 \
         | sort | uniq -c | grep -vE "^[ ]*1 "
       )" ]; then
         echo "= FAIL ======"
         echo "$gem: same EVR in changelog already exists"
         exit 1
     fi
+  popd
+done
+
+
+# Create side tag
+if [[ -f ~/.railsbuild/$FEDORA_BRANCH/.side-tag ]]; then
+  echo -n "Reusing side tag: "
+else
+  echo -n "Requesting new side tag: "
+  # Run this under the dist-gi repo to let fedpkg to figure the right base
+  # tag.
+  fedpkg --release $FEDORA_BRANCH request-side-tag \
+    | sed -nr "/Side tag/ s/.*'(f[[:digit:]]+-build-side-[[:digit:]]+)'.*/\1/ p" > ~/.railsbuild/$FEDORA_BRANCH/.side-tag
+fi
+SIDE_TAG=`cat ~/.railsbuild/$FEDORA_BRANCH/.side-tag`
+echo $SIDE_TAG
+
+
+koji edit-sidetag --rpm-macro _with_bootstrap=1 $SIDE_TAG
+
+
+# Build all gems in first round
+for gem in "${GEMS[@]}"
+do
+  pushd ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem
+    # Skip already done builds
+    if [ -f .built ]; then
+      echo "$gem: build for $gem already done"
+      popd
+      continue
+    fi
 
+    rm .scratch-built
     rm *.src.rpm
-    fedpkg scratch-build --srpm
+    fedpkg scratch-build --target=$SIDE_TAG --srpm
 
     if [ "$?" = "0" ]; then
       touch .scratch-built
-      git stash
       echo "= SUCCESS ======"
       echo "$gem: scratch build of rubygem-$gem succeeded"
+
+      git stash
       fedpkg import rubygem-$gem-*.src.rpm > /dev/null
 
       if [ ! "$?" = "0" ]; then
@@ -50,15 +77,21 @@ do
         exit 1
       fi
 
-      git commit -am "Update to $gem $VERSION"
+      fedpkg clog
+      sed -i 's/^\s*//' clog
+
+      git commit -F clog
       git push
-      fedpkg build
+
+      PACKAGE_NVR=$(fedpkg verrel)$(test -f .bootstrapped && cat .bootstrapped)
+
+      fedpkg build --target=$SIDE_TAG
 
       if [ "$?" = "0" ]; then
         echo "= SUCCESS ======"
         echo "$gem: build of rubygem-$gem succeeded"
         echo "$gem: waiting until the build becomes available..."
-        koji wait-repo f$FEDORA_VERSION-build --build=rubygem-$gem-$VERSION-1.fc$FEDORA_VERSION
+        koji wait-repo $SIDE_TAG --request --build=$PACKAGE_NVR
         touch .built
       else
         echo "= FAIL ======"
@@ -74,45 +107,39 @@ do
   popd
 done
 
-# Update bootsrapped gems
-$(dirname $(readlink -f "$0"))/railsbuild-update-bootstrapped $FEDORA_VERSION $VERSION
 
-# Build bootsrapped gems again
+koji edit-sidetag --remove-rpm-macro _with_bootstrap $SIDE_TAG
+
+
+# Build bootstrapped gems again
 for gem in "${BOOTSTRAPPED_GEMS[@]}"
 do
-  pushd ~/.railsbuild/f$FEDORA_VERSION/rubygem-$gem
+  pushd ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem
     # Skip already done builds
     if [ ! -f .bootstrapped ]; then
-      echo "$gem: final build for bootsrapped $gem already done"
+      echo "$gem: final build for bootstrapped $gem already done"
+      popd
       continue
     fi
+
     rm .scratch-built
     rm *.src.rpm
-    fedpkg scratch-build --srpm
+    fedpkg scratch-build --target=$SIDE_TAG --srpm
 
     if [ "$?" = "0" ]; then
       touch .scratch-built
-      git stash
       echo "= SUCCESS ======"
       echo "$gem: scratch build of rubygem-$gem succeeded"
-      fedpkg import rubygem-$gem-*.src.rpm > /dev/null
 
-      if [ ! "$?" = "0" ]; then
-        echo "= FAIL ======"
-        echo "$gem: fedpkg import failed"
-        exit 1
-      fi
+      PACKAGE_NVR=$(fedpkg verrel)
 
-      git commit -am "Enable tests"
-      git push
-      fedpkg build
+      fedpkg build --target=$SIDE_TAG
 
       if [ "$?" = "0" ]; then
         echo "= SUCCESS ======"
         echo "$gem: build of rubygem-$gem succeeded"
         echo "$gem: waiting until the build becomes available..."
-        koji wait-repo f$FEDORA_VERSION-build --build=rubygem-$gem-$VERSION-1.fc$FEDORA_VERSION
-        rm .bootsrapped
+        koji wait-repo $SIDE_TAG --request --build=$PACKAGE_NVR
       else
         echo "= FAIL ======"
         echo "$gem: build of rubygem-$gem failed"
@@ -130,7 +157,9 @@ done
 echo
 echo
 echo "= SUCCESSFULLY DONE ====="
-echo "Build of Ruby on Rails $VERSION for Fedora release $FEDORA_VERSION done."
+echo "Build of Ruby on Rails $VERSION for Fedora release $FEDORA_BRANCH done."
+echo
+echo "Please submit for $SIDE_TAG via Bodhi."
 echo
 echo "Run the following to delete the build metadata:"
-echo "  rm ~/.railsbuild/f$FEDORA_VERSION/rubygem-*/.*built"
+echo "  rm ~/.railsbuild/$FEDORA_BRANCH/rubygem-*/.*built"

railsbuild-common

@@ -3,20 +3,23 @@
 
 RAILSBUILD_VERSION=0.4
 
-# Fedora master branch
-FEDORA_RAWHIDE=24
-
 # Ruby on Rails gems
 GEMS=(activesupport
       activejob
       activemodel
-      activerecord
-      actionview
       actionpack
+      actionview
+      activerecord
       actionmailer
+      actioncable
       railties
+      activestorage
+      actionmailbox
+      actiontext
       rails)
 
 # Gems that needs to be rebuild without bootstrapping
-BOOTSTRAPPED_GEMS=(actionview
-                   actionpack)
+BOOTSTRAPPED_GEMS=(activemodel
+                   actionpack
+                   actionview
+                   railties)

railsbuild-fetch-sources

@@ -0,0 +1,59 @@
+#!/bin/bash
+# Get the information about sources (test suites, js, ...) for Rails gems from
+# their spec files, prepare the `git archive` tarballs from
+# ~/.railsbuild/upstream/rails (upstream git repo) and place them in
+# ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem
+#
+# Usage:
+#  ./railsbuild-fetch-tests FEDORA_BRANCH
+
+. $(dirname $(readlink -f "$0"))/railsbuild-common
+
+FEDORA_BRANCH=$1
+
+cd ~/.railsbuild/upstream/rails
+git pull
+
+for gem in "${GEMS[@]}"
+do
+  echo "Getting sources for $gem..."
+
+  VERSION=`cat ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/rubygem-$gem.spec | sed -rn "/^Version: / s/^Version: (.*)$/\1/ p"`
+  GEM_FILE="$gem-$VERSION.gem"
+
+  # Is .gem already in place?
+  if [[ -f ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/$GEM_FILE ]]; then
+    echo "  ✔ $GEM_FILE already exists"
+  else
+    pushd ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/ >> /dev/null
+    gem fetch $gem --version $VERSION
+    popd >> /dev/null
+  fi
+
+  GIT_ARCHIVE_CMDS=`grep -R -B1 '^Source[[:digit:]]*: ' ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/rubygem-$gem.spec | \
+    sed -nr "/^#/ s/.*(git archive.*$gem.*\.tar\.gz.*)/\1/ p"`
+
+  if [[ -z "$GIT_ARCHIVE_CMDS" ]]; then
+    echo "  ⚠  No additional source identified"
+    continue
+  fi
+
+  echo "$GIT_ARCHIVE_CMDS" | while read GIT_ARCHIVE_CMD; do
+    TARBALL=`echo $GIT_ARCHIVE_CMD | sed -r "s/.*($gem.*\.tar.\gz).*/\1/"`
+
+    # Is source already in place?
+    if [[ -f ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/$TARBALL ]]; then
+      echo "  ✔ $TARBALL already exists"
+      continue
+    fi
+
+    cd ~/.railsbuild/upstream/rails/$gem
+
+    `$GIT_ARCHIVE_CMD`
+
+    mv $TARBALL ~/.railsbuild/$FEDORA_BRANCH/rubygem-$gem/.
+
+    echo "  $TARBALL created"
+  done
+done
+