diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..5e5244d6 --- /dev/null +++ b/Makefile @@ -0,0 +1,90 @@ +DIST := .el6 + +.PHONY: all rpms srpms srpm_repo + +all: rpms + +.PHONY: clean +clean: + rm -rf deps RPMS SRPMS + + + +############################################################################ +# RPM build rules +############################################################################ + +# Build a source RPM from a Spec file and a tarball. We define %dist +# to ensure that the names of the source RPMs, which are built outside the +# mock chroot, match the names of the binary RPMs, which are built inside +# the chroot. Without this we might generate foo-1.0.fc20.src.rpm +# (Fedora host) and foo-1.0.el6.x86_64.rpm (CentOS chroot). +%.src.rpm: + @echo [RPMBUILD] $@ + @rpmbuild --quiet --define "_topdir ." \ + --define "%dist $(DIST)" -bs $< + +# Phony target to create repository metadata for the SRPMs. This makes +# it possible to add the SRPMS directory to yum.conf and use yumdownloader +# to install source RPMs. +srpm_repo: srpms + @echo [CREATEREPO] SRPMS + @flock --timeout 30 ./SRPMS createrepo --quiet --update ./SRPMS + +# Build one or more binary RPMs from a source RPM. A typical source RPM +# might produce a base binary RPM, a -devel binary RPM containing library +# and header files and a -debuginfo binary RPM containing debug symbols. +# The repository metadata is updated after building a binary package so that +# a subsequent mock build for a package which depend on this one is able +# to find and install it. +%.rpm: + @echo [MOCK] $@ + @mock --configdir=mock --quiet \ + --resultdir=$(dir $@) --uniqueext=$(notdir $@) --rebuild $< + @echo [CREATEREPO] $@ + @flock --timeout 30 ./RPMS createrepo --quiet --update ./RPMS + + +############################################################################ +# Deb build rules +############################################################################ + +# Build a Debian source package from a Spec file and a tarball. +# makedeb.py loads the Spec file, generates an equivalent Debian source +# directory structure, then runs 'dpkg-source' to create the .dsc file. +# The conversion is basic, but works fairly well for straightforward Spec +# files. +%.dsc: + @echo [MAKEDEB] $@ + @scripts/deb/makedeb.py $< + @echo [UPDATEREPO] $@ + @flock --timeout 30 ./SRPMS scripts/deb/updaterepo sources SRPMS + +# Build one or more binary Debian packages from from a source package. +# As with the RPM build, a typical source package might produce several +# binary packages. The repository metadata is updated after building a +# binary package so that a subsequent build for a package which depends +# on this one is able to find and install it. +%.deb: + @echo [COWBUILDER] $@ + @touch RPMS/Packages + @sudo cowbuilder --build \ + --configfile pbuilder/pbuilderrc \ + --buildresult RPMS $< + @echo [UPDATEREPO] $@ + @flock --timeout 30 ./RPMS scripts/deb/updaterepo packages RPMS + + +############################################################################ +# Dependency build rules +############################################################################ + +# Generate dependency rules linking spec files to tarballs, source +# packages and binary packages. specdep.py generates rules suitable +# for RPM or Debian builds depending on the host distribution. +deps: SPECS/*.spec specdep.py scripts/lib/mappkgname.py + @echo Updating dependencies... + @./specdep.py -d $(DIST) --ignore-from ignore SPECS/*.spec > $@ + +-include deps + diff --git a/pkg.py b/pkg.py new file mode 100755 index 00000000..61cd44ec --- /dev/null +++ b/pkg.py @@ -0,0 +1,206 @@ +"""Classes for handling RPM spec files. The classes defined here + are mostly just wrappers around rpm.rpm, adding information which + the rpm library does not currently provide.""" + + +import os +import re +import rpm +import urlparse +from scripts.lib import debianmisc + +# Could have a decorator / context manager to set and unset all the RPM macros +# around methods such as 'provides' + + +# for debugging, make all paths relative to PWD +rpm.addMacro('_topdir', '.') + +# Directories where rpmbuild/mock expects to find inputs +# and writes outputs +RPMDIR = rpm.expandMacro('%_rpmdir') +SRPMDIR = rpm.expandMacro('%_srcrpmdir') +SPECDIR = rpm.expandMacro('%_specdir') +SRCDIR = rpm.expandMacro('%_sourcedir') + + +def flatten(lst): + """Flatten a list of lists""" + return sum(lst, []) + + +def identity(name): + """Identity mapping""" + return name + + +def identity_list(name): + """Identity mapping, injected into a list""" + return [name] + + +def map_arch_deb(arch): + """Map RPM package architecture to equivalent Deb architecture""" + if arch == "x86_64": + return "amd64" + elif arch == "armv7l": + return "armhf" + elif arch == "noarch": + return "all" + else: + return arch + + +class SpecNameMismatch(Exception): + """Exception raised when a spec file's name does not match the name + of the package defined within it""" + pass + + +class Spec(object): + """Represents an RPM spec file""" + + def __init__(self, path, target="rpm", map_name=None, dist=""): + if map_name: + self.map_package_name = map_name + else: + self.map_package_name = identity_list + + self.path = os.path.join(SPECDIR, os.path.basename(path)) + + with open(path) as spec: + self.spectext = spec.readlines() + + # '%dist' in the host (where we build the source package) + # might not match '%dist' in the chroot (where we build + # the binary package). We must override it on the host, + # otherwise the names of packages in the dependencies won't + # match the files actually produced by mock. + self.dist = "" + if target == "rpm": + self.dist = dist + + rpm.addMacro('dist', self.dist) + self.spec = rpm.ts().parseSpec(path) + + if os.path.basename(path).split(".")[0] != self.name(): + raise SpecNameMismatch( + "spec file name '%s' does not match package name '%s'" % + (path, self.name())) + + if target == "rpm": + self.rpmfilenamepat = rpm.expandMacro('%_build_name_fmt') + self.srpmfilenamepat = rpm.expandMacro('%_build_name_fmt') + self.map_arch = identity + + else: + sep = '.' if debianmisc.is_native(self.spec) else '-' + if debianmisc.is_native(self.spec): + self.rpmfilenamepat = "%{NAME}_%{VERSION}.%{RELEASE}_%{ARCH}.deb" + self.srpmfilenamepat = "%{NAME}_%{VERSION}.%{RELEASE}.dsc" + else: + self.rpmfilenamepat = "%{NAME}_%{VERSION}-%{RELEASE}_%{ARCH}.deb" + self.srpmfilenamepat = "%{NAME}_%{VERSION}-%{RELEASE}.dsc" + self.map_arch = map_arch_deb + + def specpath(self): + """Return the path to the spec file""" + return self.path + + + def provides(self): + """Return a list of package names provided by this spec""" + provides = flatten([pkg.header['provides'] + [pkg.header['name']] + for pkg in self.spec.packages]) + + # RPM 4.6 adds architecture constraints to dependencies. Drop them. + provides = [re.sub(r'\(x86-64\)$', '', pkg) for pkg in provides] + return set(flatten([self.map_package_name(p) for p in provides])) + + + def name(self): + """Return the package name""" + return self.spec.sourceHeader['name'] + + + def version(self): + """Return the package version""" + return self.spec.sourceHeader['version'] + + + def source_urls(self): + """Return the URLs from which the sources can be downloaded""" + return [source for (source, _, _) in self.spec.sources] + + + def source_paths(self): + """Return the filesystem paths to source files""" + sources = [] + for source in self.source_urls(): + url = urlparse.urlparse(source) + + # Source comes from a remote HTTP server + if url.scheme in ["http", "https"]: + sources.append(os.path.join(SRCDIR, os.path.basename(url.path))) + + # Source comes from a local file or directory + if url.scheme == "file": + sources.append( + os.path.join(SRCDIR, os.path.basename(url.fragment))) + + # Source is an otherwise unqualified file, probably a patch + if url.scheme == "": + sources.append(os.path.join(SRCDIR, url.path)) + + return sources + + + # RPM build dependencies. The 'requires' key for the *source* RPM is + # actually the 'buildrequires' key from the spec + def buildrequires(self): + """Return the set of packages needed to build this spec + (BuildRequires)""" + return set(flatten([self.map_package_name(r) for r + in self.spec.sourceHeader['requires']])) + + + def source_package_path(self): + """Return the path of the source package which building this + spec will produce""" + hdr = self.spec.sourceHeader + rpm.addMacro('NAME', self.map_package_name(hdr['name'])[0]) + rpm.addMacro('VERSION', hdr['version']) + rpm.addMacro('RELEASE', hdr['release']) + rpm.addMacro('ARCH', 'src') + + # There doesn't seem to be a macro for the name of the source + # rpm, but the name appears to be the same as the rpm name format. + # Unfortunately expanding that macro gives us a leading 'src' that we + # don't want, so we strip that off + + srpmname = os.path.basename(rpm.expandMacro(self.srpmfilenamepat)) + + rpm.delMacro('NAME') + rpm.delMacro('VERSION') + rpm.delMacro('RELEASE') + rpm.delMacro('ARCH') + + return os.path.join(SRPMDIR, srpmname) + + + def binary_package_paths(self): + """Return a list of binary packages built by this spec""" + def rpm_name_from_header(hdr): + """Return the name of the binary package file which + will be built from hdr""" + rpm.addMacro('NAME', self.map_package_name(hdr['name'])[0]) + rpm.addMacro('VERSION', hdr['version']) + rpm.addMacro('RELEASE', hdr['release']) + rpm.addMacro('ARCH', self.map_arch(hdr['arch'])) + rpmname = rpm.expandMacro(self.rpmfilenamepat) + rpm.delMacro('NAME') + rpm.delMacro('VERSION') + rpm.delMacro('RELEASE') + rpm.delMacro('ARCH') + return os.path.join(RPMDIR, rpmname) + return [rpm_name_from_header(pkg.header) for pkg in self.spec.packages] diff --git a/planex/build.py b/planex/build.py index 994379b6..5b6e0173 100755 --- a/planex/build.py +++ b/planex/build.py @@ -194,7 +194,7 @@ def do_build(srpm, target, build_number, use_mock, xs_build_sys): if xs_build_sys: mock = "/usr/bin/mock" else: - mock = "mock" + mock = "planex-cache" if use_mock: cmd = [mock, "--configdir=mock", "--resultdir=%s" % TMP_RPM_PATH, "--rebuild", @@ -203,7 +203,7 @@ def do_build(srpm, target, build_number, use_mock, xs_build_sys): "--define", "extrarelease .%d" % build_number, "-v", srpm] if not xs_build_sys: - cmd = ["sudo"] + cmd + ["--disable-plugin=package_state"] + cmd = cmd else: cmd = ["rpmbuild", "--rebuild", "-v", "%s" % srpm, "--target", target, "--define", diff --git a/planex/cache.py b/planex/cache.py index 28b20910..20cbab58 100755 --- a/planex/cache.py +++ b/planex/cache.py @@ -177,8 +177,10 @@ def get_srpm_hash(srpm, yumbase, mock_config): pkg_hash.update(PLANEX_CACHE_SALT) pkg_hash.update(mock_config) - log_debug("Hashes of SRPM contents (%s):" % - RFC4880_HASHES[srpm.filedigestalgo]) + if srpm.filedigestalgo: + log_debug("Hashes of SRPM contents (%s):" % + RFC4880_HASHES[srpm.filedigestalgo]) + for name, digest in zip(srpm.filenames, srpm.filedigests): log_debug(" %s: %s" % (name, digest)) pkg_hash.update(digest) @@ -215,7 +217,7 @@ def build_package(configdir, root, passthrough_args): working_directory = tempfile.mkdtemp(prefix="planex-cache") log_debug("Mock working directory: %s" % working_directory) - cmd = ["mock", "--configdir=%s" % configdir, + cmd = ["sudo", "mock", "--configdir=%s" % configdir, "--root=%s" % root, "--resultdir=%s" % working_directory] + passthrough_args diff --git a/planex/configure.py b/planex/configure.py index a1a79c96..00fa50d4 100755 --- a/planex/configure.py +++ b/planex/configure.py @@ -304,7 +304,36 @@ def parse_cmdline(argv=None): """ Parse command line options """ - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser(description=""" + Configure the planex build directory. + + This command will generate the directory structure planex requires + to build RPMs. The following directories will be created in the + curent directory: + + planex-build-root/{RPMS,SRPMS,SPECS} + mock + + The configuration directory should contain a template mock + configuration directory, a set of SPEC files and/or SPEC file + templates. The files in the mock template will be processed and + the following substitions made: + + @PLANEX_BUILD_ROOT@ -> the full path of the planex-build-root + directory. + + The SPEC file templates (.spec.in) are processed in the following way. + Any Source directive that references a git or mercurial repository will + be extended with a SCM hash and an archive filename. The filename contains + a version derived from the SCM repository. Additionally, the following + definitions are also rewritten if they were present in the template: + + %source{n}_version -> version derived from the nth repository + %source{n}_hash -> SCM hash from the nth repository + %planex_version -> combined version + %planex_release -> 1%{?extrarelease} + """,formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument( '--mirror_path', help='Rewrite URLs to point to this directory', default="") diff --git a/planex/debianchangelog.py b/planex/debianchangelog.py new file mode 100644 index 00000000..1830eba3 --- /dev/null +++ b/planex/debianchangelog.py @@ -0,0 +1,52 @@ +from tree import Tree +import mappkgname +import re +import time + +def changelog_from_spec(spec, isnative): + res = Tree() + + hdr = spec.sourceHeader + log = "" + for (name, timestamp, text) in zip(hdr['changelogname'], + hdr['changelogtime'], + hdr['changelogtext']): + + # A Debian package's version is defined by the version of the + # first entry in the changelog, so we must get this right. + # Most spec files have changelog entries starting "First Last + # - version" - this seems to be the standard + # for Red Hat spec files. + # Some of our changelos only have "First Last ". + # For these, we use the version from the spec. + match = re.match( "^(.+) - (\S+)$", name ) + if match: + author = match.group(1) + version = match.group(2) + if isnative: + version = re.sub('-', '.', version) + else: + author = name + sep = '.' if isnative else '-' + version = "%s%s%s" % (spec.sourceHeader['version'], + sep, + spec.sourceHeader['release']) + print version + + package_name = mappkgname.map_package(hdr['name'])[0] + log += "%s (%s) UNRELEASED; urgency=low\n" % (package_name, version) + log += "\n" + + text = re.sub( "^-", "*", text, flags=re.MULTILINE ) + text = re.sub( "^", " ", text, flags=re.MULTILINE ) + log += "%s\n" % text + log += "\n" + + date_string = time.strftime("%a, %d %b %Y %H:%M:%S %z", + time.gmtime(int(timestamp))) + log += " -- %s %s\n" % (author, date_string) + log += "\n" + + res.append('debian/changelog', log) + return res + diff --git a/planex/debiancontrol.py b/planex/debiancontrol.py new file mode 100644 index 00000000..ea40baf2 --- /dev/null +++ b/planex/debiancontrol.py @@ -0,0 +1,81 @@ +from tree import Tree +import mappkgname +import textwrap + + +STANDARDS_VERSION = "3.9.3" + + +def control_from_spec(spec): + res = Tree() + source_deb_from_spec(spec, res) + for pkg in spec.packages: + binary_deb_from_spec(pkg, res) + return res + + +def source_deb_from_spec(spec, tree): + res = "" + res += "Source: %s\n" % mappkgname.map_package(spec.sourceHeader['name'])[0] + res += "Priority: %s\n" % "optional" + res += "Maintainer: %s\n" % "Euan Harris " + res += "Section: %s\n" % mappkgname.map_section(spec.sourceHeader['group']) + res += "Standards-Version: %s\n" % STANDARDS_VERSION + + res += "Build-Depends:\n" + build_depends = ["debhelper (>= 8)", "dh-ocaml (>= 0.9)", "ocaml-nox", "python"] + for pkg, version in zip(spec.sourceHeader['requires'], + spec.sourceHeader['requireVersion']): + deps = mappkgname.map_package(pkg) + for dep in deps: + if version: + dep += " (>= %s)" % version + build_depends.append(dep) + + res += ",\n".join(set([" %s" % d for d in build_depends])) + res += "\n\n" + + tree.append('debian/control', res) + + +def binary_deb_from_spec(spec, tree): + res = "" + res += "Package: %s\n" % mappkgname.map_package_name(spec.header) + if spec.header['arch'] in ["x86_64", "i686", "armhf", "armv7l"]: + res += "Architecture: any\n" + else: + res += "Architecture: all\n" + + res += "Depends:\n" + depends = ["${ocaml:Depends}", "${shlibs:Depends}", "${misc:Depends}"] + for pkg, version in zip(spec.header['requires'], + spec.header['requireVersion']): + deps = mappkgname.map_package(pkg) + for dep in deps: + if version: + dep += " (>= %s)" % version + depends.append(dep) + res += ",\n".join([" %s" % d for d in depends]) + res += "\n" + + # XXX These lines should only be added for ocaml packages + res += "Provides: ${ocaml:Provides}\n" + res += "Recommends: ocaml-findlib\n" + + res += "Description: %s\n" % spec.header['summary'] + res += format_description(spec.header['description']) + res += "\n\n" + + tree.append('debian/control', res) + + +def format_description(description): + """need to format this - correct line length, initial one space indent, + and blank lines must be replaced by dots""" + + paragraphs = "".join(description).split("\n\n") + wrapped = ["\n".join(textwrap.wrap(p, initial_indent=" ", + subsequent_indent=" ")) + for p in paragraphs] + return "\n .\n".join(wrapped) + diff --git a/planex/debianmisc.py b/planex/debianmisc.py new file mode 100644 index 00000000..bb87e2e4 --- /dev/null +++ b/planex/debianmisc.py @@ -0,0 +1,107 @@ +import rpm +from tree import Tree +import mappkgname +import os +import re +import rpmextra + +def conffiles_from_spec(spec, specpath): + # Configuration files, not to be overwritten on upgrade. + # Files in /etc are automatically marked as config files, + # so we only need to list files here if they are in a + # different place. + res = Tree() + pkgname = mappkgname.map_package_name(spec.sourceHeader) + files = rpmextra.files_from_spec(pkgname, specpath) + if files.has_key( pkgname + "-%config" ): + for filename in files[pkgname + "-%config"]: + res.append('debian/conffiles', "%s\n" % filename) + return res + + +def filelists_from_spec(spec, specpath): + res = Tree() + for pkg in spec.packages: + name = "%s.install.in" % mappkgname.map_package_name(pkg.header) + res.append("debian/%s" % name, + files_from_pkg(spec.sourceHeader['name'], pkg, specpath)) + return res + + +def files_from_pkg(basename, pkg, specpath): + # should be able to build this from the files sections - can't find how + # to get at them from the spec object + res = "" + files = rpmextra.files_from_spec(basename, specpath) + for filename in files.get(pkg.header['name'], []): + # Debian packages must not contain compiled Python files. + # Instead, the python2 helper arranges to compile these + # files when they are installed. + if os.path.splitext(filename)[1] in [".pyc", ".pyo"]: + continue + + rpm.addMacro("_libdir", "usr/lib") + rpm.addMacro("_bindir", "usr/bin") + + # deb just wants relative paths + src = rpm.expandMacro(filename).lstrip("/") + rpm.delMacro("_bindir") + rpm.delMacro("_libdir") + rpm.addMacro("_libdir", "/usr/lib") + rpm.addMacro("_bindir", "/usr/bin") + dst = rpm.expandMacro(filename) + + # destination paths should be directories, not files. + # if the file is foo and the path is /usr/bin/foo, the + # package will end up install /usr/bin/foo/foo + if not dst.endswith("/"): + dst = os.path.dirname(dst) + rpm.delMacro("_bindir") + rpm.delMacro("_libdir") + res += "%s %s\n" % (src, dst) + return res + + +# Patches can be added to debian/patches, with a series file +# We use dpkg-source -b --auto-commit + +def patches_from_spec(spec, src_dir): + res = Tree() + patches = [(seq, name) for (name, seq, typ) in spec.sources + if typ == 2] + patches = [name for (seq, name) in sorted(patches)] + for patch in patches: + with open(os.path.join(src_dir, patch)) as patchfile: + contents = patchfile.read() + permissions = os.fstat(patchfile.fileno()).st_mode + res.append(os.path.join("debian/patches", patch), + contents, permissions) + res.append("debian/patches/series", "%s\n" % patch) + return res + + +def compat_from_spec(_spec): + res = Tree() + res.append("debian/compat", "8") + return res + +def format_from_spec(_spec, isnative): + res = Tree() + fmt = "native" if isnative else "quilt" + res.append("debian/source/format", "3.0 (%s)\n" % fmt) + return res + +def copyright_from_spec(_spec): + res = Tree() + res.append("debian/copyright", "FIXME") + return res + +def principal_source_file(spec): + return os.path.basename([name for (name, seq, filetype) + in spec.sources + if seq == 0 and filetype == 1][0]) + +def is_native(_spec): + tarball = principal_source_file(_spec) + match = re.match("^(.+)((\.tar\.(gz|bz2|lzma|xz)|\.tbz)$)", tarball) + return match == None diff --git a/planex/debianrules.py b/planex/debianrules.py new file mode 100755 index 00000000..351ce3f9 --- /dev/null +++ b/planex/debianrules.py @@ -0,0 +1,137 @@ +#!/usr/bin/python + +import rpm +import rpmextra +import os +import re +import mappkgname +from tree import Tree + + +def rules_from_spec(spec, specpath): + res = Tree() + ocaml_rules_preamble(spec, res) + rules_configure_from_spec(spec, res) + rules_build_from_spec(spec, res) + rules_install_from_spec(spec, res) + rules_dh_install_from_spec(spec, res, specpath) + rules_clean_from_spec(spec, res) + rules_test_from_spec(spec, res) + python_setuptools_cfg(spec, res) + return res + + +def ocaml_rules_preamble(_spec, tree): + # TODO: should only include if we have packed up ocaml files + rule = "#!/usr/bin/make -f\n" + rule += "\n" + rule += "#include /usr/share/cdbs/1/rules/debhelper.mk\n" + rule += "#include /usr/share/cdbs/1/class/makefile.mk\n" + rule += "#include /usr/share/cdbs/1/rules/ocaml.mk\n" + rule += "\n" + rule += "export DH_VERBOSE=1\n" + rule += "export DH_OPTIONS\n" + rule += "export DESTDIR=$(CURDIR)/debian/tmp\n" + rule += "%:\n" + rule += "\tdh $@ --with ocaml --with python2\n" + rule += "\n" + + tree.append('debian/rules', rule) + + +def rules_configure_from_spec(_spec, tree): + # RPM doesn't have a configure target - everything happens in the + # build target. Nevertheless we must override the auto_configure target + # because some OASIS packages have configure scripts. If debhelper + # sees a configure script it will assume it's from autoconf and will + # run it with arguments that an OASIS configure script won't understand. + + rule = ".PHONY: override_dh_auto_configure\n" + rule += "override_dh_auto_configure:\n" + rule += "\n" + + tree.append('debian/rules', rule) + + +def rules_build_from_spec(spec, tree): + # RPM's build rule is just a script which is run at the appropriate time. + # debian/rules is a Makefile. Makefile recipes aren't shell scripts - each + # line is run independently, so exports don't survive from line to line and + # multi-line constructions such as if statements don't work. + # To work around this, we put these recipes in helper scripts in the debian/ + # directory. + + if not spec.build: + return {} + + rule = ".PHONY: override_dh_auto_build\n" + rule += "override_dh_auto_build:\n" + rule += "\tdebian/build.sh\n" + rule += "\n" + + helper = "#!/bin/sh\n" + helper += "unset CFLAGS\n" #XXX HACK for ocaml-oclock + helper += spec.build.replace("$RPM_BUILD_ROOT", "${DESTDIR}") + + tree.append('debian/rules', rule) + tree.append('debian/build.sh', helper, permissions=0o755) + + +def rules_install_from_spec(spec, tree): + rule = ".PHONY: override_dh_auto_install\n" + rule += "override_dh_auto_install:\n" + rule += "\tdebian/install.sh\n" + rule += "\n" + + helper = "#!/bin/sh\n" + helper += spec.install.replace("$RPM_BUILD_ROOT", "${DESTDIR}") + + tree.append('debian/rules', rule) + tree.append('debian/install.sh', helper, permissions=0o755) + +def rules_dh_install_from_spec(spec, tree, specpath): + rule = ".PHONY: override_dh_install\n" + rule += "override_dh_install:\n" + rule += "\tdh_install\n" + + pkgname = mappkgname.map_package_name(spec.sourceHeader) + files = rpmextra.files_from_spec(pkgname, specpath) + if files.has_key( pkgname + "-%exclude" ): + for pat in files[pkgname + "-%exclude"]: + path = "\trm -f debian/%s/%s\n" % (pkgname, rpm.expandMacro(pat)) + rule += os.path.normpath(path) + rule += "\n" + + tree.append('debian/rules', rule) + + + +def rules_clean_from_spec(spec, tree): + rule = ".PHONY: override_dh_auto_clean\n" + rule += "override_dh_auto_clean:\n" + rule += "\tdebian/clean.sh\n" + rule += re.sub("^", "\t", spec.clean.strip(), flags=re.MULTILINE) + rule += "\n\n" + + helper = "#!/bin/sh\n" + spec.clean.replace("$RPM_BUILD_ROOT", "${DESTDIR}") + + tree.append('debian/rules', rule) + tree.append('debian/clean.sh', helper, permissions=0o755) + + +def rules_test_from_spec(_spec, tree): + # XXX HACK for ocaml-oclock - don't try to run the tests when building + rule = ".PHONY: override_dh_auto_test\n" + rule += "override_dh_auto_test:\n" + + tree.append('debian/rules', rule) + + +def python_setuptools_cfg(_spec, tree): + # Configuration file for python setuptools, which defaults to installing + # in /usr/local/lib instead of /usr/lib + content = "[install]\n" + content += "install-layout=deb\n" + + tree.append('setup.cfg', content) + diff --git a/planex/downloader.py b/planex/downloader.py new file mode 100755 index 00000000..a964fcac --- /dev/null +++ b/planex/downloader.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python + +import sys +import urlparse +import os +import subprocess +import time + +def guess_file_type(path): + # If we have python2.7: + #line = subprocess.check_output(["file", path]) + p = subprocess.Popen(["file", path], stdout = subprocess.PIPE) + line = p.stdout.read() + p.communicate() + if p.returncode <> 0: + print >>sys.stderr, "file %s: failed with exit code %d" % (path, p.returncode) + exit(1) + # file: description + if len(line) < (len(path) + 2): + print >>sys.stderr, "Malformed output from 'file %s': '%s'" % (path, line) + exit(1) + return line[len(path) + 2:].strip() + +def looks_like_an_archive(path): + ty = guess_file_type(path) + if ty.startswith("ASCII") or ty.startswith("HTML"): + return False + if ty.startswith("gzip") or ty.startswith("bzip"): + return True + print >>sys.stderr, "%s has an unrecognised file type: %s" % (path, ty) + print >>sys.stderr, "Please extend %s:looks_like_an_archive to include this case." + exit(1) + +def download(url, destination): + args = ["curl", "--silent", "--show-error", "-L", "-o", destination, url] + if "forge.ocamlcore.org" in url: + args.append("--insecure") + print >>sys.stderr, "Running %s" % (" ".join(args)) + returncode = subprocess.call(args) + if returncode <> 0: + print >>sys.stderr, "Downloading %s failed: sleeping 5s" % url + time.sleep(5) + +def look_for_it(url, destination): + for attempt in [5, 4, 3, 2, 1, 0]: + if os.path.exists(destination): + if looks_like_an_archive(destination): + print >>sys.stderr, "%s exists and looks like an archive" % destination + # Refresh the archive's last access time. + # If the spec file which depends on this archive has been modified, it will have a later access time than the archive and make will continually rebuild it. 'Touch' the archive, to prevent this happening + # There is a potential race here: http://stackoverflow.com/questions/1158076/implement-touch-using-python + with open(destination, 'a'): + os.utime(destination, None) + return + else: + print >>sys.stderr, "%s is not an archive, deleting it" % destination + os.unlink(destination) + if attempt == 0: + print >>sys.stderr, "That was our last attempt so giving up." + exit(1) + else: + print >>sys.stderr, "%s does not exist: downloading" % destination + download(url, destination) + +def main(): + if len(sys.argv) <> 3: + print >>sys.stderr, "Wrong number of arguments. Use %s " % sys.argv[0] + exit(1) + url_string = sys.argv[1] + destination = sys.argv[2] + url = urlparse.urlparse(url_string) + path = url.path.split('/') + # Avoid relying on github to set the destination filename + if url.netloc == "github.com" and path[-3] == "archive": + ext = None + possible_exts = [ ".tar", ".tar.gz", ".zip", ".tbz", "tar.bz2" ] + for e in possible_exts: + if url.path.endswith(e): + ext = e + break + if not ext: + print >>sys.stderr, "I did not recognise extension of %s. I know about: %s" % (url.path, ", ".join(possible_exts)) + exit(1) + url_path = "/".join(path[0:-2] + [ path[-2] + ext ]) + url_string = str(urlparse.urlunsplit((url.scheme, url.netloc, url_path, url.query, url.fragment),)) + look_for_it(url_string, destination) + +if __name__ == "__main__": + main() diff --git a/planex/makedeb.py b/planex/makedeb.py new file mode 100755 index 00000000..78db683a --- /dev/null +++ b/planex/makedeb.py @@ -0,0 +1,186 @@ +#!/usr/bin/python + +import glob +import os +import re +import rpm +import shutil +import subprocess +import sys +import tempfile + +SCRIPTDIR=os.path.dirname(os.path.abspath(__file__)) +LIBDIR=os.path.normpath(os.path.join(SCRIPTDIR, "../lib")) +sys.path.append(LIBDIR) + +import debianchangelog +import debiancontrol +import debianmisc +import debianrules +import mappkgname +import rpmextra + +# BUGS: +# Hack to disable CFLAGS for ocaml-oclock +# Hack to disable tests for ocaml-oclock +# Hard coded install files only install ocaml dir +# Should be building signed debs + + +# By default, RPM expects everything to be in $HOME/rpmbuild. +# We want it to run in the current directory. +rpm.addMacro('_topdir', os.getcwd()) + + +# Directories where rpmbuild expects to find inputs +# and writes outputs +TMPDIR = tempfile.mkdtemp(prefix="makedeb") + +SRPM_DIR = rpm.expandMacro('%_srcrpmdir') +SRC_DIR = rpm.expandMacro('%_sourcedir') +rpm.addMacro("_builddir", TMPDIR) +BUILD_DIR = rpm.expandMacro('%_builddir') + + +# Fedora puts executables run by other programs in /usr/libexec, but +# Debian puts them in /usr/lib, which apparently follows the FHS: +# http://www.debian.org/doc/manuals/maint-guide/advanced.en.html#ftn.idp2018768 +rpm.addMacro('_libexecdir', "/usr/lib") + + +# Override some macros interpolated into build rules, so +# paths are appropriate for debuild +# (Actually, using {}, not (), because these identifiers +# end up in helper scripts, not in the makefile +rpm.addMacro("buildroot", "${DESTDIR}") +rpm.addMacro("_libdir", "/usr/lib") + + +def debian_dir_from_spec(spec, path, specpath, isnative): + os.makedirs(os.path.join(path, "debian/source")) + + control = debiancontrol.control_from_spec(spec) + control.apply(path) + + rules = debianrules.rules_from_spec(spec, specpath) + rules.apply(path) + + compat = debianmisc.compat_from_spec(spec) + compat.apply(path) + + fmt = debianmisc.format_from_spec(spec, isnative) + fmt.apply(path) + + copyright_file = debianmisc.copyright_from_spec(spec) + copyright_file.apply(path) + + changelog = debianchangelog.changelog_from_spec(spec, isnative) + changelog.apply(path) + + filelists = debianmisc.filelists_from_spec(spec, specpath) + filelists.apply(path) + + patches = debianmisc.patches_from_spec(spec, SRC_DIR) + patches.apply(path) + + conffiles = debianmisc.conffiles_from_spec(spec, specpath) + conffiles.apply(path) + + +def prepare_build_dir(spec, build_subdir): + # To prepare the build dir, RPM cds into $TOPDIR/BUILD + # and expands all paths in the prep script with $TOPDIR. + # It unpacks the tarball and then cds into the directory it + # creates before applying patches. + # $TOPDIR should be an absolute path to the top RPM build + # directory, not a relative path, so that references to SOURCES + # expand to reachable paths inside the source tree (getting the + # tarball from ../SOURCES works in the outer BUILD dir, but getting + # patches from ../SOURCES doesn't work when we have cd'ed into the + # source tree. + + unpack_dir = os.path.join(BUILD_DIR, build_subdir) + subprocess.call(spec.prep.replace("$RPM_BUILD_ROOT", unpack_dir), + shell=True) + # could also just do: RPMBUILD_PREP = 1<<0; spec._doBuild() + + +def rename_source(spec, pkgname, pkgversion): + # Debian source package name should probably match the tarball name + origfilename = debianmisc.principal_source_file(spec) + if origfilename.endswith(".tbz"): + filename = origfilename[:-len(".tbz")] + ".tar.bz2" + else: + filename = origfilename + match = re.match("^(.+)(\.tar\.(gz|bz2|lzma|xz))", filename) + if not match: + print "error: could not parse filename %s" % filename + _, ext = match.groups()[:2] + base_filename = "%s_%s.orig%s" % (mappkgname.map_package(pkgname)[0], + pkgversion, ext) + shutil.copy(os.path.join(SRC_DIR, origfilename), + os.path.join(BUILD_DIR, base_filename)) + + +def main(): + shutil.rmtree(BUILD_DIR) + os.mkdir(BUILD_DIR) + spec = rpmextra.spec_from_file(sys.argv[1]) + clean = True + if "-noclean" in sys.argv: + clean = False + + # subdirectory of builddir in which the tarball is unpacked; + # set by RPM after processing the spec file + # if the source file isn't a tarball this won't be set! + build_subdir = rpm.expandMacro("%buildsubdir") + prepare_build_dir(spec, build_subdir) + + if os.path.isdir(os.path.join(BUILD_DIR, build_subdir, "debian")): + shutil.rmtree(os.path.join(BUILD_DIR, build_subdir, "debian")) + + # a package with no original tarball is built as a 'native debian package' + native = debianmisc.is_native(spec) + + if not native: + # copy over the source, run the prep rule to unpack it, then + # rename it as deb expects this should be based on the rewritten + # (or not) source name in the debian package - build the debian + # dir first and then rename the tarball as needed + rename_source(spec, spec.sourceHeader['name'], + spec.sourceHeader['version']) + + debian_dir_from_spec(spec, os.path.join(BUILD_DIR, build_subdir), + sys.argv[1], native) + + cmd = "cd %s\ndpkg-source -b --auto-commit %s" % (BUILD_DIR, build_subdir) + print cmd + res = subprocess.call(cmd, shell=True) + assert res == 0 + + for i in glob.glob(os.path.join(BUILD_DIR, "*")): + if build_subdir in i: + continue + shutil.copy2(i, SRPM_DIR) + if clean: + os.unlink(i) + if clean: + shutil.rmtree(TMPDIR) + else: + print "makedeb: dpkg input files in %s" % TMPDIR + + # At this point we have a debian source package (at least 3 files) in SRPMS. + # To build: + # pbuilder --create --distribution raring --architecture amd64 \ + # --debootstrap qemu-debootstrap --mirror http://ports.ubuntu.com \ + # --basetgz /var/cache/pbuilder/qemu-raring-armhf-base.tar.gz + # + # To build for ARM: + # pbuilder --create --distribution raring --architecture armhf \ + # --debootstrap qemu-debootstrap --mirror http://ports.ubuntu.com \ + # --basetgz /var/cache/pbuilder/qemu-raring-armhf-base.tar.gz + + +if __name__ == '__main__': + main() + diff --git a/planex/mappkgname.py b/planex/mappkgname.py new file mode 100755 index 00000000..48550ebc --- /dev/null +++ b/planex/mappkgname.py @@ -0,0 +1,209 @@ +#!/usr/bin/python + +import platform + +"""Maps an RPM package name to the equivalent DEB. + The MAPPING is static, but in future will be + made dynamically by querying the package databases.""" + +TARGET_SPECIFIC_MAPPING = { + 'debian:jessie/sid': { + 'kernel': ['linux-image-amd64'], + 'kernel-firmware': ['firmware-linux-free'], + "xen-libs": ["libxen-4.4"], + }, + 'ubuntu:14.04': { + "xen-libs": ["libxen-4.4"], + }, + 'linaro:14.04': { + "xen-libs": ["libxen-4.4"], + }, + } + +MAPPING = { + # Our packages + "ocaml-biniou": ["libbiniou-ocaml"], + "ocaml-cmdliner": ["libcmdliner-ocaml"], + "deriving-ocsigen": ["libderiving-ocsigen-ocaml"], + "ocaml-easy-format": ["libeasy-format-ocaml"], + "linux-guest-loader": ["linux-guest-loader"], + "iscsi-initiator-utils": ["open-iscsi"], + "js_of_ocaml": ["libjs-of-ocaml"], + "libnl3-cli": ["libnl-cli-3-200"], + "libnl3-doc": ["libnl-doc"], + "libnl3": ["libnl-3-200", "libnl-route-3-200"], + "libffi": ["libffi6"], + "ocaml-bitstring": ["libbitstring-ocaml"], + "ocaml-camomile-data": ["libcamomile-data"], + "ocaml-camomile": ["libcamomile-ocaml"], + "ocaml-cdrom": ["libcdrom-ocaml"], + "ocaml-cohttp": ["libcohttp-ocaml"], + "ocaml-cstruct": ["libcstruct-ocaml"], + "ocaml-ctypes": ["libctypes-ocaml"], + "ocaml-crc": ["libcrc-ocaml"], + "ocaml-fd-send-recv": ["libfd-send-recv-ocaml"], + "ocaml-gnt": ["libgnt-ocaml"], + "ocaml-lambda-term": ["liblambda-term-ocaml"], + "ocaml-libvhd": ["libvhd-ocaml"], + "ocaml-libvirt": ["libvirt-ocaml"], + "ocaml-lwt": ["liblwt-ocaml"], + "ocaml-nbd": ["libnbd-ocaml"], + "ocaml-netdev": ["libnetdev-ocaml"], + "ocaml-obuild": ["ocaml-obuild"], + "ocaml-oclock": ["liboclock-ocaml"], + "ocaml-ocplib-endian": ["ocplib-endian-ocaml"], + "ocaml-ounit": ["libounit-ocaml"], + "ocaml-opasswd": ["libopasswd-ocaml"], + "ocaml-qmp": ["libqmp-ocaml"], + "ocaml-react": ["libreact-ocaml"], + "ocaml-re": ["libre-ocaml"], + "ocaml-rpc": ["librpc-ocaml"], + "ocaml-rrd-transport": ["librrd-transport-ocaml"], + "ocaml-sexplib": ["libsexplib-camlp4"], + "ocaml-ssl": ["libssl-ocaml"], + "ocaml-stdext": ["libstdext-ocaml"], + "ocaml-tapctl": ["libtapctl-ocaml"], + "ocaml-text": ["libtext-ocaml"], + "ocaml-type-conv": ["libtype-conv-camlp4"], + "ocaml-uri": ["liburi-ocaml"], + "ocaml-uuidm": ["libuuidm-ocaml"], + "ocaml-xcp-idl": ["libxcp-idl-ocaml"], + "ocaml-xcp-inventory": ["libxcp-inventory-ocaml"], + "ocaml-xcp-rrd": ["libxcp-rrd-ocaml"], + "ocaml-xen-api-client": ["libxen-api-client-ocaml"], + "ocaml-xen-api-libs-transitional": ["ocaml-xen-api-libs-transitional"], + "ocaml-xen-lowlevel-libs": ["ocaml-xen-lowlevel-libs"], + "ocaml-xenops": ["libxenops-ocaml"], + "ocaml-xenstore-clients": ["libxenstore-clients-ocaml"], + "ocaml-xenstore": ["libxenstore-ocaml"], + "ocaml-yojson": ["libyojson-ocaml"], + "ocaml-zed": ["libzed-ocaml"], + "ocaml-vhd": ["vhd-ocaml"], + "ocaml-tar": ["tar-ocaml"], + "ocaml-uutf": ["uutf-ocaml"], + "ocaml-odn": ["libodn-ocaml"], + "ocaml-fileutils": ["libfileutils-ocaml"], + "ocaml-io-page": ["libio-page-ocaml"], + "ocaml-sha": ["libsha-ocaml"], + "ocaml-ipaddr": ["libipaddr-ocaml"], + "ocaml-mirage-types": ["libmirage-types-ocaml"], + "openstack-xapi-plugins": ["openstack-xapi-plugins"], + "optcomp": ["optcomp-ocaml"], + "xapi-libvirt-storage": ["libxapi-libvirt-storage-ocaml"], + "ocaml-xmlm": ["libxmlm-ocaml"], + "xsconsole0": ["xsconsole"], + "xenserver-core-latest-snapshot": ["xenserver-core-latest-snapshot"], + "python-setuptools": ["python-setuptools", "python-setuptools-git"], + + # Distribution packages + "ocaml": ["ocaml-nox", "ocaml-native-compilers"], + "ocaml-findlib": ["ocaml-findlib"], + "ocaml-ocamldoc": ["ocaml-nox"], + "ocaml-compiler-libs": ["ocaml-compiler-libs"], + "ocaml-camlp4": ["camlp4", "camlp4-extra"], + "openssl": ["libssl1.0.0"], + "xen": ["xen-hypervisor", "qemu-system-x86", "blktap-utils"], + "libuuid": ["uuid"], + "libvirt": ["libvirt0", "libvirt-bin"], + "xen-libs": ["libxen-4.2"], + "ncurses": ["libncurses5"], + "chkconfig": [], + "initscripts": [], + "PyPAM": ["python-pam"], + "pam": ["libpam0g"], + "tetex-latex": ["texlive-base"], + "zlib": ["zlib1g"], + "stunnel": ["stunnel"], + "bash-completion": ["bash-completion"], + "python2": ["python"], + "newt": ["libnewt0.52"], + "/sbin/ldconfig": ["/sbin/ldconfig"], + "kernel-headers": ["linux-headers-3.2.0-51-generic"], + "libvirt-docs": ["libvirt-doc"], + "kernel": ["linux-image"], + "kernel-firmware": ["linux-firmware"], + "/bin/sh": [], + "xen-runtime": ["xen-utils"], + "nfs-utils": ["nfs-common"], + "redhat-lsb-core": ["lsb-base"], + "sg3_utils": ["sg3-utils"], + "python-argparse": ["libpython2.7-stdlib"], + "util-linux-ng": ["uuid-runtime"], + "autoconf": ["autoconf"], + "automake": ["automake"], +} + +SECONDARY_MAPPING = { + "camlp4-dev": ["camlp4"], + "camlp4-extra-dev": ["camlp4-extra"], + # packages with 'ocaml' or 'camlp4' in the name must have a -dev... + "libssl1.0.0-dev": ["libssl-dev"], + "libtype-conv-camlp4": ["libtype-conv-camlp4-dev"], + "libxapi-libvirt-storage-ocaml": ["libxapi-libvirt-storage-ocaml-dev"], + "ocaml-findlib-dev": ["ocaml-findlib", "libfindlib-ocaml-dev"], + "xen-hypervisor-dev": ["libxen-dev", "blktap-dev"], + "libvirt0-dev": ["libvirt-dev"], + "libxen-4.2-dev": ["libxen-dev"], + "libffi6-dev": ["libffi-dev"], + "libvirt-bin-dev": ["libvirt-bin"], + "blktap-utils-dev": ["blktap-utils"], + "qemu-system-x86-dev": ["qemu-system-x86"], +} + +def map_package(name, target=None): + """map an rpm to a corresponding deb, based on file contents""" + is_devel = False + + if target is None: + dist = platform.linux_distribution(full_distribution_name=False) + target = "%s:%s" % (dist[0].lower(), dist[1].lower()) + + # RPM 4.6 adds architecture constraints to dependencies. Drop them. + if name.endswith( "(x86-64)" ): + name = name[ :-len("(x86-64)") ] + if name.endswith( "-devel" ): + is_devel = True + name = name[ :-len("-devel") ] + + default = [name] + mapped = MAPPING.get(name, default) + + if target in TARGET_SPECIFIC_MAPPING: + mapped = TARGET_SPECIFIC_MAPPING[target].get(name, mapped) + + res = [] + for debname in mapped: + if is_devel: + debname += "-dev" + res += SECONDARY_MAPPING.get(debname, [debname]) + return res + + +def map_package_name(hdr, target=None): + """rewrite an rpm name to fit with debian standards""" + name = hdr['name'] + + # Debian adds a -dev suffix to development packages, + # whereas Fedora uses -devel + is_devel = False + if name.endswith( "-devel" ): + is_devel = True + name = name[ :-len("-devel") ] + + # Debian prefixes library packag names with 'lib' + #if "Libraries" in hdr['group'] or "library" in hdr['summary'].lower(): + # name = "lib" + name + name = name.replace( name, map_package(name, target)[0] ) + + if is_devel: + name += "-dev" + + # hack for type-conv. dh_ocaml insists that there must be a + # -dev package for anything with ocaml or camlp4 in the name... + if name == "libtype-conv-camlp4": + name = "libtype-conv-camlp4-dev" + return name + +def map_section(_rpm_name): + return "ocaml" # XXXXX + diff --git a/planex/rpmextra.py b/planex/rpmextra.py new file mode 100644 index 00000000..05267130 --- /dev/null +++ b/planex/rpmextra.py @@ -0,0 +1,95 @@ +import rpm + + +def spec_from_file(spec): + return rpm.ts().parseSpec(spec) + + +def files_from_spec(basename, specpath): + """The RPM library doesn't seem to give us access to the files section, + so we need to go and get it ourselves. This parsing algorithm is + based on build/parseFiles.c in RPM. The list of section titles + comes from build/parseSpec.c. We should get this by using ctypes + to load the rpm library.""" + # XXX shouldn't be parsing this by hand. will need to handle conditionals + # within and surrounding files and packages sections.""" + + otherparts = [ + "%package", + "%prep", + "%build", + "%install", + "%check", + "%clean", + "%preun", + "%postun", + "%pretrans", + "%posttrans", + "%pre", + "%post", + "%changelog", + "%description", + "%triggerpostun", + "%triggerprein", + "%triggerun", + "%triggerin", + "%trigger", + "%verifyscript", + "%sepolicy", + ] + + files = {} + with open(specpath) as spec: + in_files = False + section = "" + for line in spec: + tokens = line.strip().split(" ") + if tokens and tokens[0].lower() == "%files": + section = basename + in_files = True + if len(tokens) > 1: + section = basename + "-" + tokens[1] + continue + + if tokens and tokens[0] in otherparts: + in_files = False + + if in_files: + if tokens[0].lower().startswith("%defattr"): + continue + if tokens[0].lower().startswith("%attr"): + files[section] = files.get(section, []) + tokens[1:] + continue + if tokens[0].lower() == "%doc": + docsection = section + "-doc" + files[docsection] = files.get(docsection, []) + tokens[1:] + continue + if tokens[0].lower() == "%if" or tokens[0].lower() == "%endif": + # XXX evaluate the if condition and do the right thing here + continue + if tokens[0].lower() == "%exclude": + excludesection = section + "-%exclude" + files[excludesection] = \ + files.get(excludesection, []) + tokens[1:] + continue + if tokens[0].lower().startswith("%config"): + # dh_install automatically considers files in /etc + # to be config files so we don't have to do anything + # special for them. The Debian packaging policy says + # that all configuration files must be installed in /etc, + # so we can rewrite _sysconfigdir to /etc. The spec file + # documentation says that # a %config directive can only + # apply to a single file, so there should only be one filename + # to consider. + configsection = section + "-%config" + tokens[1] = tokens[1].replace("%{_sysconfdir}", "/etc") + if tokens[1].startswith("/etc"): + files[section] = files.get(section, []) + tokens[1:] + else: + files[configsection] = \ + files.get(configsection, []) + tokens[1:] + continue + if line.strip(): + files[section] = files.get(section, []) + [line.strip()] + return files + diff --git a/planex/spec.py b/planex/spec.py index 0d91876d..96581204 100755 --- a/planex/spec.py +++ b/planex/spec.py @@ -7,6 +7,7 @@ import re import rpm import urlparse +import debianmisc # Could have a decorator / context manager to set and unset all the RPM macros # around methods such as 'provides' @@ -42,6 +43,8 @@ def map_arch_deb(arch): """Map RPM package architecture to equivalent Deb architecture""" if arch == "x86_64": return "amd64" + elif arch == "armv7l": + return "armhf" elif arch == "noarch": return "all" else: @@ -58,37 +61,26 @@ class Spec(object): """Represents an RPM spec file""" def __init__(self, path, target="rpm", map_name=None, dist=""): - if target == "rpm": - self.rpmfilenamepat = rpm.expandMacro('%_build_name_fmt') - self.srpmfilenamepat = rpm.expandMacro('%_build_name_fmt') - self.map_arch = identity - - # '%dist' in the host (where we build the source package) - # might not match '%dist' in the chroot (where we build - # the binary package). We must override it on the host, - # otherwise the names of packages in the dependencies won't - # match the files actually produced by mock. - self.dist = dist - - else: - self.rpmfilenamepat = "%{NAME}_%{VERSION}-%{RELEASE}_%{ARCH}.deb" - self.srpmfilenamepat = "%{NAME}_%{VERSION}-%{RELEASE}.dsc" - self.map_arch = map_arch_deb - self.dist = "" - - rpm.addMacro('dist', self.dist) - if map_name: self.map_package_name = map_name else: self.map_package_name = identity_list - self.path = os.path.join(SPECDIR, os.path.basename(path)) with open(path) as spec: self.spectext = spec.readlines() + # '%dist' in the host (where we build the source package) + # might not match '%dist' in the chroot (where we build + # the binary package). We must override it on the host, + # otherwise the names of packages in the dependencies won't + # match the files actually produced by mock. + self.dist = "" + if target == "rpm": + self.dist = dist + + rpm.addMacro('dist', self.dist) self.spec = rpm.ts().parseSpec(path) if os.path.basename(path).split(".")[0] != self.name(): @@ -96,6 +88,20 @@ def __init__(self, path, target="rpm", map_name=None, dist=""): "spec file name '%s' does not match package name '%s'" % (path, self.name())) + if target == "rpm": + self.rpmfilenamepat = rpm.expandMacro('%_build_name_fmt') + self.srpmfilenamepat = rpm.expandMacro('%_build_name_fmt') + self.map_arch = identity + + else: + sep = '.' if debianmisc.is_native(self.spec) else '-' + if debianmisc.is_native(self.spec): + self.rpmfilenamepat = "%{NAME}_%{VERSION}.%{RELEASE}_%{ARCH}.deb" + self.srpmfilenamepat = "%{NAME}_%{VERSION}.%{RELEASE}.dsc" + else: + self.rpmfilenamepat = "%{NAME}_%{VERSION}-%{RELEASE}_%{ARCH}.deb" + self.srpmfilenamepat = "%{NAME}_%{VERSION}-%{RELEASE}.dsc" + self.map_arch = map_arch_deb def specpath(self): """Return the path to the spec file""" diff --git a/planex/specdep.py b/planex/specdep.py new file mode 100755 index 00000000..6356363d --- /dev/null +++ b/planex/specdep.py @@ -0,0 +1,174 @@ +#!/usr/bin/python + +# see http://docs.fedoraproject.org/en-US/Fedora_Draft_Documentation/0.1/html/RPM_Guide/ch16s04.html + +import argparse +import os +import spec as pkg +import platform +import sys +import urlparse +import mappkgname + + +def build_type(): + debian_like = ["ubuntu", "debian", "linaro"] + rhel_like = ["fedora", "redhat", "centos"] + + dist = platform.linux_distribution(full_distribution_name=False)[0].lower() + assert dist in debian_like + rhel_like + + if dist in debian_like: + return "deb" + elif dist in rhel_like: + return "rpm" + + +# Rules to build SRPM from SPEC +def build_srpm_from_spec(spec): + srpmpath = spec.source_package_path() + print '%s: %s %s' % (srpmpath, spec.specpath(), + " ".join(spec.source_paths())) + + + +# Rules to download sources + +# Assumes each RPM only needs one download - we have some multi-source +# packages but in all cases the additional sources are patches provided +# in the Git repository +def download_rpm_sources(spec): + for (url, path) in zip(spec.source_urls(), spec.source_paths()): + source = urlparse.urlparse(url) + + # Source comes from a remote HTTP server + if source.scheme in ["http", "https"]: + print '%s: %s' % (path, spec.specpath()) + print '\t@echo [DOWNLOADER] $@' + print '\t@planex-downloader %s %s' % (url, path) + + # Source comes from a local file or directory + if source.scheme == "file": + print '%s: %s $(shell find %s)' % ( + path, spec.specpath(), source.path) + + # Assume that the directory name is already what's expected by the + # spec file, and prefix it with the version number in the tarball + print '\t@echo [GIT] $@' + dirname = "%s-%s" % (os.path.basename(source.path), spec.version()) + print '\t@git --git-dir=%s/.git '\ + 'archive --prefix %s/ -o $@ HEAD' % (source.path, dirname) + + +# Rules to build RPMS from SRPMS (uses information from the SPECs to +# get packages) +def build_rpm_from_srpm(spec): + # We only generate a rule for the first binary RPM produced by the + # specfile. If we generate multiple rules (one for the base package, + # one for -devel and so on), make will interpret these as completely + # separate targets which must be built separately. At best, this means + # that the same package will be built more than once; at worst, in a + # concurrent build, there is a risk that the targets might not be rebuilt + # correctly. + # + # Make does understand the concept of multiple targets being built by + # a single rule invocation, but only for pattern rules (e.g. %.h %.c: %.y). + # It is tricky to generate correct pattern rules for RPM builds. + + rpm_path = spec.binary_package_paths()[0] + srpm_path = spec.source_package_path() + print '%s: %s' % (rpm_path, srpm_path) + + +def package_to_rpm_map(specs): + provides_to_rpm = {} + for spec in specs: + for provided in spec.provides(): + provides_to_rpm[provided] = spec.binary_package_paths()[0] + return provides_to_rpm + + +def buildrequires_for_rpm(spec, provides_to_rpm): + rpmpath = spec.binary_package_paths()[0] + for buildreq in spec.buildrequires(): + # Some buildrequires come from the system repository + if provides_to_rpm.has_key(buildreq): + buildreqrpm = provides_to_rpm[buildreq] + print "%s: %s" % (rpmpath, buildreqrpm) + + +def parse_cmdline(): + """ + Parse command line options + """ + parser = argparse.ArgumentParser(description= + "Generate Makefile dependencies from RPM Spec files") + parser.add_argument("specs", metavar="SPEC", nargs="+", help="spec file") + parser.add_argument("-i", "--ignore", metavar="PKG", action="append", + default=[], help="package name to ignore") + parser.add_argument("-I", "--ignore-from", metavar="FILE", action="append", + default=[], help="file of package names to be ignored") + parser.add_argument("-d", "--dist", metavar="DIST", + default="", help="distribution tag (used in RPM filenames)") + return parser.parse_args() + + +def main(): + args = parse_cmdline() + specs = {} + + pkgs_to_ignore = args.ignore + for ignore_from in args.ignore_from: + with open(ignore_from) as f: + for name in f.readlines(): + pkgs_to_ignore.append(name.strip()) + for i in pkgs_to_ignore: + print "# Will ignore: %s" % i + + for spec_path in args.specs: + try: + if build_type() == "deb": + os_type = platform.linux_distribution(full_distribution_name=False)[1].lower() + map_name_fn=lambda name: mappkgname.map_package(name, os_type) + spec = pkg.Spec(spec_path, target="deb", map_name=map_name_fn) + else: + spec = pkg.Spec(spec_path, target="rpm", dist=args.dist) + pkg_name = spec.name() + if pkg_name in pkgs_to_ignore: + continue + + specs[os.path.basename(spec_path)] = spec + + except pkg.SpecNameMismatch as exn: + sys.stderr.write("error: %s\n" % exn.message) + sys.exit(1) + + provides_to_rpm = package_to_rpm_map(specs.values()) + + for spec in specs.itervalues(): + build_srpm_from_spec(spec) + download_rpm_sources(spec) + build_rpm_from_srpm(spec) + buildrequires_for_rpm(spec, provides_to_rpm) + print "" + + # Generate targets to build all srpms and all rpms + all_rpms = [] + all_srpms = [] + for spec in specs.itervalues(): + rpm_path = spec.binary_package_paths()[0] + all_rpms.append(rpm_path) + all_srpms.append(spec.source_package_path()) + print "%s: %s" % (spec.name(), rpm_path) + print "" + + print "rpms: " + " \\\n\t".join(all_rpms) + print "" + print "srpms: " + " \\\n\t".join(all_srpms) + print "" + print "install: all" + print "\t. scripts/%s/install.sh" % build_type() + + +if __name__ == "__main__": + main() diff --git a/planex/tree.py b/planex/tree.py new file mode 100644 index 00000000..92788be7 --- /dev/null +++ b/planex/tree.py @@ -0,0 +1,45 @@ +import os + +class Tree(object): + def __init__(self): + self.tree = {} + + def append(self, filename, contents=None, permissions=None): + node = self.tree.get(filename, {}) + if contents: + node['contents'] = node.get('contents', '') + contents + if permissions: + if node.has_key('permissions') and \ + node['permissions'] != permissions: + raise Exception("Trying to change permissions for " % filename) + + if permissions: + node['permissions'] = permissions + else: + node['permissions'] = 0o644 + self.tree[filename] = node + + def apply(self, basepath): + for subpath, node in self.tree.items(): + permissions = node.get("permissions", 0o644) + contents = node.get("contents", "") + fullpath = os.path.join(basepath, subpath) + + if not os.path.isdir(os.path.dirname(fullpath)): + os.makedirs(os.path.dirname(fullpath)) + + out = os.open(os.path.join(basepath, subpath), + os.O_WRONLY | os.O_CREAT, permissions) + os.write(out, contents) + os.close(out) + + def __repr__(self): + res = "" + for subpath, node in self.tree.items(): + permissions = node.get("permissions", 0o644) + contents = node.get("contents", "") + res += "%s (0o%o):\n" % (subpath, permissions) + res += contents + res += "\n\n" + return res + diff --git a/tests/test_pkg.py b/tests/test_pkg.py new file mode 100644 index 00000000..8db71b57 --- /dev/null +++ b/tests/test_pkg.py @@ -0,0 +1,136 @@ +# Run these tests with 'nosetests': +# install the 'python-nose' package (Fedora/CentOS or Ubuntu) +# run 'nosetests' in the root of the repository + +import unittest + +import pkg + +class RpmTests(unittest.TestCase): + def setUp(self): + # 'setUp' breaks Pylint's naming rules + # pylint: disable=C0103 + self.spec = pkg.Spec("tests/data/ocaml-cohttp.spec", dist=".el6") + + def test_good_filename_preprocessor(self): + pkg.Spec("tests/data/ocaml-cohttp.spec.in") + + def test_bad_filename(self): + self.assertRaises(pkg.SpecNameMismatch, pkg.Spec, + "tests/data/bad-name.spec") + + def test_bad_filename_preprocessor(self): + self.assertRaises(pkg.SpecNameMismatch, pkg.Spec, + "tests/data/bad-name.spec.in") + + def test_name(self): + self.assertEqual(self.spec.name(), "ocaml-cohttp") + + def test_specpath(self): + self.assertEqual(self.spec.specpath(), "./SPECS/ocaml-cohttp.spec") + + def test_version(self): + self.assertEqual(self.spec.version(), "0.9.8") + + def test_provides(self): + self.assertEqual(self.spec.provides(), + set(["ocaml-cohttp", "ocaml-cohttp-devel"])) + + def test_source_urls(self): + self.assertEqual(self.spec.source_urls(), + ["ocaml-cohttp-init", + "file:///code/ocaml-cohttp-extra#ocaml-cohttp-extra-0.9.8.tar.gz", + "https://github.com/mirage/ocaml-cohttp/archive/" + "ocaml-cohttp-0.9.8/ocaml-cohttp-0.9.8.tar.gz"]) + + def test_source_paths(self): + self.assertEqual(self.spec.source_paths(), + ["./SOURCES/ocaml-cohttp-init", + "./SOURCES/ocaml-cohttp-extra-0.9.8.tar.gz", + "./SOURCES/ocaml-cohttp-0.9.8.tar.gz"]) + + def test_buildrequires(self): + self.assertEqual(self.spec.buildrequires(), + set(["ocaml", "ocaml-findlib", "ocaml-re-devel", + "ocaml-uri-devel", "ocaml-cstruct-devel", + "ocaml-lwt-devel", "ocaml-ounit-devel", + "ocaml-ocamldoc", "ocaml-camlp4-devel", + "openssl", "openssl-devel"])) + + def test_source_package_path(self): + self.assertEqual(self.spec.source_package_path(), + "./SRPMS/ocaml-cohttp-0.9.8-1.el6.src.rpm") + + def test_binary_package_paths(self): + self.assertEqual(sorted(self.spec.binary_package_paths()), + sorted(["./RPMS/x86_64/ocaml-cohttp-0.9.8-1.el6.x86_64.rpm", + "./RPMS/x86_64/ocaml-cohttp-devel-0.9.8-1.el6.x86_64.rpm"])) + + +class DebTests(unittest.TestCase): + def setUp(self): + # 'setUp' breaks Pylint's naming rules + # pylint: disable=C0103 + def map_rpm_to_deb(name): + mapping = {"ocaml-cohttp": ["libcohttp-ocaml"], + "ocaml-cohttp-devel": ["libcohttp-ocaml-dev"], + "ocaml": ["ocaml-nox", "ocaml-native-compilers"], + "ocaml-findlib": ["ocaml-findlib"], + "ocaml-re-devel": ["libre-ocaml-dev"], + "ocaml-uri-devel": ["liburi-ocaml-dev"], + "ocaml-cstruct-devel": ["libcstruct-ocaml-dev"], + "ocaml-lwt-devel": ["liblwt-ocaml-dev"], + "ocaml-ounit-devel": ["libounit-ocaml-dev"], + "ocaml-ocamldoc": ["ocaml-nox"], + "ocaml-camlp4-devel": ["camlp4", "camlp4-extra"], + "openssl": ["libssl1.0.0"], + "openssl-devel": ["libssl-dev"]} + return mapping[name] + + self.spec = pkg.Spec("./tests/data/ocaml-cohttp.spec", target="deb", + map_name=map_rpm_to_deb) + + def test_name(self): + self.assertEqual(self.spec.name(), "ocaml-cohttp") + + def test_specpath(self): + self.assertEqual(self.spec.specpath(), "./SPECS/ocaml-cohttp.spec") + + def test_version(self): + self.assertEqual(self.spec.version(), "0.9.8") + + def test_provides(self): + self.assertEqual(self.spec.provides(), + set(["libcohttp-ocaml", "libcohttp-ocaml-dev"])) + + def test_source_urls(self): + self.assertEqual(self.spec.source_urls(), + ["ocaml-cohttp-init", + "file:///code/ocaml-cohttp-extra#ocaml-cohttp-extra-0.9.8.tar.gz", + "https://github.com/mirage/ocaml-cohttp/archive/" + "ocaml-cohttp-0.9.8/ocaml-cohttp-0.9.8.tar.gz"]) + + def test_source_paths(self): + self.assertEqual(self.spec.source_paths(), + ["./SOURCES/ocaml-cohttp-init", + "./SOURCES/ocaml-cohttp-extra-0.9.8.tar.gz", + "./SOURCES/ocaml-cohttp-0.9.8.tar.gz"]) + + def test_buildrequires(self): + self.assertEqual(self.spec.buildrequires(), + set(["ocaml-nox", "ocaml-native-compilers", + "ocaml-findlib", "libre-ocaml-dev", + "liburi-ocaml-dev", "libcstruct-ocaml-dev", + "liblwt-ocaml-dev", "libounit-ocaml-dev", + "camlp4", "camlp4-extra", "libssl1.0.0", + "libssl-dev"])) + + def test_source_package_path(self): + self.assertEqual(self.spec.source_package_path(), + "./SRPMS/libcohttp-ocaml_0.9.8-1.dsc") + + def test_binary_package_paths(self): + self.assertEqual(sorted(self.spec.binary_package_paths()), + sorted(["./RPMS/libcohttp-ocaml_0.9.8-1_amd64.deb", + "./RPMS/libcohttp-ocaml-dev_0.9.8-1_amd64.deb"])) + diff --git a/tests/test_specdep.py b/tests/test_specdep.py new file mode 100644 index 00000000..3053c9aa --- /dev/null +++ b/tests/test_specdep.py @@ -0,0 +1,72 @@ +# Run these tests with 'nosetests': +# install the 'python-nose' package (Fedora/CentOS or Ubuntu) +# run 'nosetests' in the root of the repository + +import glob +import os +import sys +import unittest + +import specdep +import pkg + +class BasicTests(unittest.TestCase): + def setUp(self): + self.spec = pkg.Spec("SPECS/ocaml-cohttp.spec", dist=".el6") + + def test_build_srpm_from_spec(self): + specdep.build_srpm_from_spec(self.spec) + + self.assertEqual(sys.stdout.getvalue(), + "./SRPMS/ocaml-cohttp-0.9.8-1.el6.src.rpm: " + "./SPECS/ocaml-cohttp.spec " + "./SOURCES/ocaml-cohttp-0.9.8.tar.gz\n") + + def test_download_rpm_sources(self): + specdep.download_rpm_sources(self.spec) + + self.assertEqual(sys.stdout.getvalue(), + "./SOURCES/ocaml-cohttp-0.9.8.tar.gz: ./SPECS/ocaml-cohttp.spec\n" + " @echo [CURL] $@\n" + " @curl --silent --show-error -L -o $@ https://github.com/" + "mirage/ocaml-cohttp/archive/ocaml-cohttp-0.9.8/" + "ocaml-cohttp-0.9.8.tar.gz\n") + + def test_build_rpm_from_srpm(self): + specdep.build_rpm_from_srpm(self.spec) + + self.assertEqual(sys.stdout.getvalue(), + "./RPMS/x86_64/ocaml-cohttp-0.9.8-1.el6.x86_64.rpm: " + "./SRPMS/ocaml-cohttp-0.9.8-1.el6.src.rpm\n" + "./RPMS/x86_64/ocaml-cohttp-devel-0.9.8-1.el6.x86_64.rpm: " + "./SRPMS/ocaml-cohttp-0.9.8-1.el6.src.rpm\n") + + + def test_buildrequires_for_rpm(self): + spec_paths = glob.glob(os.path.join("./SPECS", "*.spec")) + specs = [pkg.Spec(spec_path, dist='.el6') for spec_path in spec_paths] + + specdep.buildrequires_for_rpm(self.spec, + specdep.package_to_rpm_map(specs)) + + self.assertEqual(sys.stdout.getvalue(), + "./RPMS/x86_64/ocaml-cohttp-0.9.8-1.el6.x86_64.rpm: " + "./RPMS/x86_64/ocaml-uri-devel-1.3.8-1.el6.x86_64.rpm\n" + "./RPMS/x86_64/ocaml-cohttp-0.9.8-1.el6.x86_64.rpm: " + "./RPMS/x86_64/ocaml-cstruct-devel-0.7.1-2.el6.x86_64.rpm\n" + "./RPMS/x86_64/ocaml-cohttp-0.9.8-1.el6.x86_64.rpm: " + "./RPMS/x86_64/ocaml-ounit-devel-1.1.2-3.el6.x86_64.rpm\n" + "./RPMS/x86_64/ocaml-cohttp-0.9.8-1.el6.x86_64.rpm: " + "./RPMS/x86_64/ocaml-re-devel-1.2.1-1.el6.x86_64.rpm\n" + "./RPMS/x86_64/ocaml-cohttp-0.9.8-1.el6.x86_64.rpm: " + "./RPMS/x86_64/ocaml-lwt-devel-2.4.3-1.el6.x86_64.rpm\n" + "./RPMS/x86_64/ocaml-cohttp-devel-0.9.8-1.el6.x86_64.rpm: " + "./RPMS/x86_64/ocaml-uri-devel-1.3.8-1.el6.x86_64.rpm\n" + "./RPMS/x86_64/ocaml-cohttp-devel-0.9.8-1.el6.x86_64.rpm: " + "./RPMS/x86_64/ocaml-cstruct-devel-0.7.1-2.el6.x86_64.rpm\n" + "./RPMS/x86_64/ocaml-cohttp-devel-0.9.8-1.el6.x86_64.rpm: " + "./RPMS/x86_64/ocaml-ounit-devel-1.1.2-3.el6.x86_64.rpm\n" + "./RPMS/x86_64/ocaml-cohttp-devel-0.9.8-1.el6.x86_64.rpm: " + "./RPMS/x86_64/ocaml-re-devel-1.2.1-1.el6.x86_64.rpm\n" + "./RPMS/x86_64/ocaml-cohttp-devel-0.9.8-1.el6.x86_64.rpm: " + "./RPMS/x86_64/ocaml-lwt-devel-2.4.3-1.el6.x86_64.rpm\n")