Merge pull request #87 from jonludlam/buildroot2

Buildroot merge attempt 2

Makefile

@@ -0,0 +1,90 @@
+DIST := .el6
+
+.PHONY: all rpms srpms srpm_repo
+
+all: rpms
+
+.PHONY: clean
+clean:
+	rm -rf deps RPMS SRPMS
+
+
+
+############################################################################
+# RPM build rules
+############################################################################
+
+# Build a source RPM from a Spec file and a tarball.   We define %dist
+# to ensure that the names of the source RPMs, which are built outside the
+# mock chroot, match the names of the binary RPMs, which are built inside
+# the chroot.	Without this we might generate foo-1.0.fc20.src.rpm
+# (Fedora host) and foo-1.0.el6.x86_64.rpm (CentOS chroot).
+%.src.rpm:
+	@echo [RPMBUILD] $@ 
+	@rpmbuild --quiet --define "_topdir ." \
+		--define "%dist $(DIST)" -bs $<
+
+# Phony target to create repository metadata for the SRPMs.   This makes
+# it possible to add the SRPMS directory to yum.conf and use yumdownloader
+# to install source RPMs.
+srpm_repo: srpms
+	@echo [CREATEREPO] SRPMS
+	@flock --timeout 30 ./SRPMS createrepo --quiet --update ./SRPMS
+
+# Build one or more binary RPMs from a source RPM.   A typical source RPM
+# might produce a base binary RPM, a -devel binary RPM containing library
+# and header files and a -debuginfo binary RPM containing debug symbols.
+# The repository metadata is updated after building a binary package so that
+# a subsequent mock build for a package which depend on this one is able
+# to find and install it.
+%.rpm:
+	@echo [MOCK] $@
+	@mock --configdir=mock --quiet \
+		--resultdir=$(dir $@) --uniqueext=$(notdir $@) --rebuild $<
+	@echo [CREATEREPO] $@
+	@flock --timeout 30 ./RPMS createrepo --quiet --update ./RPMS
+
+
+############################################################################
+# Deb build rules
+############################################################################
+
+# Build a Debian source package from a Spec file and a tarball.
+# makedeb.py loads the Spec file, generates an equivalent Debian source
+# directory structure, then runs 'dpkg-source' to create the .dsc file.
+# The conversion is basic, but works fairly well for straightforward Spec
+# files.
+%.dsc: 
+	@echo [MAKEDEB] $@
+	@scripts/deb/makedeb.py $<
+	@echo [UPDATEREPO] $@
+	@flock --timeout 30 ./SRPMS scripts/deb/updaterepo sources SRPMS
+
+# Build one or more binary Debian packages from from a source package.
+# As with the RPM build, a typical source package might produce several
+# binary packages.  The repository metadata is updated after building a
+# binary package so that a subsequent build for a package which depends
+# on this one is able to find and install it.
+%.deb:
+	@echo [COWBUILDER] $@
+	@touch RPMS/Packages
+	@sudo cowbuilder --build \
+		--configfile pbuilder/pbuilderrc \
+		--buildresult RPMS $<
+	@echo [UPDATEREPO] $@
+	@flock --timeout 30 ./RPMS scripts/deb/updaterepo packages RPMS
+
+
+############################################################################
+# Dependency build rules
+############################################################################
+
+# Generate dependency rules linking spec files to tarballs, source
+# packages and binary packages.   specdep.py generates rules suitable
+# for RPM or Debian builds depending on the host distribution.
+deps: SPECS/*.spec specdep.py scripts/lib/mappkgname.py
+	@echo Updating dependencies...
+	@./specdep.py -d $(DIST) --ignore-from ignore SPECS/*.spec > $@
+
+-include deps
+

pkg.py

@@ -0,0 +1,206 @@
+"""Classes for handling RPM spec files.   The classes defined here
+   are mostly just wrappers around rpm.rpm, adding information which
+   the rpm library does not currently provide."""
+
+
+import os
+import re
+import rpm
+import urlparse
+from scripts.lib import debianmisc
+
+# Could have a decorator / context manager to set and unset all the RPM macros
+# around methods such as 'provides'
+
+
+# for debugging, make all paths relative to PWD
+rpm.addMacro('_topdir', '.')
+
+# Directories where rpmbuild/mock expects to find inputs
+# and writes outputs
+RPMDIR  = rpm.expandMacro('%_rpmdir')
+SRPMDIR = rpm.expandMacro('%_srcrpmdir')
+SPECDIR = rpm.expandMacro('%_specdir')
+SRCDIR  = rpm.expandMacro('%_sourcedir')
+
+
+def flatten(lst):
+    """Flatten a list of lists"""
+    return sum(lst, [])
+
+
+def identity(name):
+    """Identity mapping"""
+    return name
+
+
+def identity_list(name):
+    """Identity mapping, injected into a list"""
+    return [name]
+
+
+def map_arch_deb(arch):
+    """Map RPM package architecture to equivalent Deb architecture"""
+    if arch == "x86_64":
+        return "amd64"
+    elif arch == "armv7l":
+        return "armhf"
+    elif arch == "noarch":
+        return "all"
+    else:
+        return arch
+
+
+class SpecNameMismatch(Exception):
+    """Exception raised when a spec file's name does not match the name
+       of the package defined within it"""
+    pass
+
+
+class Spec(object):
+    """Represents an RPM spec file"""
+
+    def __init__(self, path, target="rpm", map_name=None, dist=""):
+        if map_name:
+            self.map_package_name = map_name
+        else:
+            self.map_package_name = identity_list
+
+        self.path = os.path.join(SPECDIR, os.path.basename(path))
+
+        with open(path) as spec:
+            self.spectext = spec.readlines()
+
+        # '%dist' in the host (where we build the source package)
+        # might not match '%dist' in the chroot (where we build
+        # the binary package).   We must override it on the host,
+        # otherwise the names of packages in the dependencies won't
+        # match the files actually produced by mock.
+	self.dist = ""
+        if target == "rpm":
+            self.dist = dist
+
+        rpm.addMacro('dist', self.dist)
+        self.spec = rpm.ts().parseSpec(path)
+
+        if os.path.basename(path).split(".")[0] != self.name():
+            raise SpecNameMismatch(
+                "spec file name '%s' does not match package name '%s'" %
+                (path, self.name()))
+
+        if target == "rpm":
+            self.rpmfilenamepat = rpm.expandMacro('%_build_name_fmt')
+            self.srpmfilenamepat = rpm.expandMacro('%_build_name_fmt')
+            self.map_arch = identity
+
+        else:
+            sep = '.' if debianmisc.is_native(self.spec) else '-'
+            if debianmisc.is_native(self.spec):
+                self.rpmfilenamepat = "%{NAME}_%{VERSION}.%{RELEASE}_%{ARCH}.deb"
+                self.srpmfilenamepat = "%{NAME}_%{VERSION}.%{RELEASE}.dsc"
+            else:
+                self.rpmfilenamepat = "%{NAME}_%{VERSION}-%{RELEASE}_%{ARCH}.deb"
+                self.srpmfilenamepat = "%{NAME}_%{VERSION}-%{RELEASE}.dsc"
+            self.map_arch = map_arch_deb
+
+    def specpath(self):
+        """Return the path to the spec file"""
+        return self.path
+
+
+    def provides(self):
+        """Return a list of package names provided by this spec"""
+        provides = flatten([pkg.header['provides'] + [pkg.header['name']]
+                          for pkg in self.spec.packages])
+
+        # RPM 4.6 adds architecture constraints to dependencies.  Drop them.
+        provides = [re.sub(r'\(x86-64\)$', '', pkg) for pkg in provides]
+        return set(flatten([self.map_package_name(p) for p in provides]))
+
+
+    def name(self):
+        """Return the package name"""
+        return self.spec.sourceHeader['name']
+
+
+    def version(self):
+        """Return the package version"""
+        return self.spec.sourceHeader['version']
+
+
+    def source_urls(self):
+        """Return the URLs from which the sources can be downloaded"""
+        return [source for (source, _, _) in self.spec.sources]
+
+
+    def source_paths(self):
+        """Return the filesystem paths to source files"""
+        sources = []
+        for source in self.source_urls():
+            url = urlparse.urlparse(source)
+
+            # Source comes from a remote HTTP server
+            if url.scheme in ["http", "https"]:
+                sources.append(os.path.join(SRCDIR, os.path.basename(url.path)))
+
+            # Source comes from a local file or directory
+            if url.scheme == "file":
+                sources.append(
+                    os.path.join(SRCDIR, os.path.basename(url.fragment)))
+
+            # Source is an otherwise unqualified file, probably a patch
+            if url.scheme == "":
+                sources.append(os.path.join(SRCDIR, url.path))
+
+        return sources
+
+
+    # RPM build dependencies.   The 'requires' key for the *source* RPM is
+    # actually the 'buildrequires' key from the spec
+    def buildrequires(self):
+        """Return the set of packages needed to build this spec
+           (BuildRequires)"""
+        return set(flatten([self.map_package_name(r) for r
+                           in self.spec.sourceHeader['requires']]))
+
+
+    def source_package_path(self):
+        """Return the path of the source package which building this
+           spec will produce"""
+        hdr = self.spec.sourceHeader
+        rpm.addMacro('NAME', self.map_package_name(hdr['name'])[0])
+        rpm.addMacro('VERSION', hdr['version'])
+        rpm.addMacro('RELEASE', hdr['release'])
+        rpm.addMacro('ARCH', 'src')
+
+        # There doesn't seem to be a macro for the name of the source
+        # rpm, but the name appears to be the same as the rpm name format.
+        # Unfortunately expanding that macro gives us a leading 'src' that we
+        # don't want, so we strip that off
+
+        srpmname = os.path.basename(rpm.expandMacro(self.srpmfilenamepat))
+
+        rpm.delMacro('NAME')
+        rpm.delMacro('VERSION')
+        rpm.delMacro('RELEASE')
+        rpm.delMacro('ARCH')
+
+        return os.path.join(SRPMDIR, srpmname)
+
+
+    def binary_package_paths(self):
+        """Return a list of binary packages built by this spec"""
+        def rpm_name_from_header(hdr):
+            """Return the name of the binary package file which
+               will be built from hdr"""
+            rpm.addMacro('NAME', self.map_package_name(hdr['name'])[0])
+            rpm.addMacro('VERSION', hdr['version'])
+            rpm.addMacro('RELEASE', hdr['release'])
+            rpm.addMacro('ARCH', self.map_arch(hdr['arch']))
+            rpmname = rpm.expandMacro(self.rpmfilenamepat)
+            rpm.delMacro('NAME')
+            rpm.delMacro('VERSION')
+            rpm.delMacro('RELEASE')
+            rpm.delMacro('ARCH')
+            return os.path.join(RPMDIR, rpmname)
+        return [rpm_name_from_header(pkg.header) for pkg in self.spec.packages]

planex/build.py

@@ -194,7 +194,7 @@ def do_build(srpm, target, build_number, use_mock, xs_build_sys):
     if xs_build_sys:
 	mock = "/usr/bin/mock"
     else:
-	mock = "mock"
+	mock = "planex-cache"
     if use_mock:
         cmd = [mock, "--configdir=mock", 
                "--resultdir=%s" % TMP_RPM_PATH, "--rebuild",
@@ -203,7 +203,7 @@ def do_build(srpm, target, build_number, use_mock, xs_build_sys):
                "--define", "extrarelease .%d" % build_number,
                "-v", srpm]
         if not xs_build_sys:
-            cmd = ["sudo"] + cmd + ["--disable-plugin=package_state"]
+            cmd = cmd
     else:
         cmd = ["rpmbuild", "--rebuild", "-v", "%s" % srpm,
                "--target", target, "--define",

planex/cache.py

@@ -177,8 +177,10 @@ def get_srpm_hash(srpm, yumbase, mock_config):
     pkg_hash.update(PLANEX_CACHE_SALT)
     pkg_hash.update(mock_config)
 
-    log_debug("Hashes of SRPM contents (%s):" %
-        RFC4880_HASHES[srpm.filedigestalgo])
+    if srpm.filedigestalgo:
+        log_debug("Hashes of SRPM contents (%s):" %
+            RFC4880_HASHES[srpm.filedigestalgo])
+
     for name, digest in zip(srpm.filenames, srpm.filedigests):
         log_debug("  %s: %s" % (name, digest))
         pkg_hash.update(digest)
@@ -215,7 +217,7 @@ def build_package(configdir, root, passthrough_args):
     working_directory = tempfile.mkdtemp(prefix="planex-cache")
     log_debug("Mock working directory: %s" % working_directory)
 
-    cmd = ["mock", "--configdir=%s" % configdir,
+    cmd = ["sudo", "mock", "--configdir=%s" % configdir,
            "--root=%s" % root,
            "--resultdir=%s" % working_directory] + passthrough_args
 

planex/configure.py

@@ -304,7 +304,36 @@ def parse_cmdline(argv=None):
     """
     Parse command line options
     """
-    parser = argparse.ArgumentParser()
+    parser = argparse.ArgumentParser(description="""
+    Configure the planex build directory.
+
+    This command will generate the directory structure planex requires
+    to build RPMs. The following directories will be created in the
+    curent directory:
+
+        planex-build-root/{RPMS,SRPMS,SPECS}
+        mock
+
+    The configuration directory should contain a template mock
+    configuration directory, a set of SPEC files and/or SPEC file
+    templates. The files in the mock template will be processed and
+    the following substitions made:
+
+        @PLANEX_BUILD_ROOT@ -> the full path of the planex-build-root 
+                               directory.
+
+    The SPEC file templates (.spec.in) are processed in the following way.
+    Any Source directive that references a git or mercurial repository will
+    be extended with a SCM hash and an archive filename. The filename contains
+    a version derived from the SCM repository. Additionally, the following
+    definitions are also rewritten if they were present in the template:
+
+        %source{n}_version -> version derived from the nth repository
+        %source{n}_hash    -> SCM hash from the nth repository
+        %planex_version    -> combined version
+        %planex_release    -> 1%{?extrarelease}
+    """,formatter_class=argparse.RawDescriptionHelpFormatter)
+
     parser.add_argument(
         '--mirror_path', help='Rewrite URLs to point to this directory', 
         default="")