Improve and refactor aliDoctor (#295)

ktf · dberzano · commit 9f2c86656817 · 2016-06-01T18:46:03.000+02:00
The package list creation logic has been abstracted out of aliBuild and now powers aliDoctor. This allows us to limit the check to the actual packages being built and not all of them. Moreover it also allows for checking the system requirements. Output is also prettified and human readable. Fixes #267.
diff --git a/aliBuild b/aliBuild
@@ -12,7 +12,7 @@ import json
 import logging
 from alibuild_helpers.log import debug, error, warning, banner, info
 from alibuild_helpers.log import logger_handler, logger, LogFormatter, ProgressPrint, riemannStream
-from alibuild_helpers.utilities import doDetectArch, format, getVersion
+from alibuild_helpers.utilities import format, getVersion, detectArch
 from alibuild_helpers.analytics import decideAnalytics, askForAnalytics, report_screenview, report_exception, report_event
 from alibuild_helpers.analytics import enable_analytics, disable_analytics
 
@@ -149,26 +149,6 @@ VALID_ARCHS_RE = ["slc[5-9]+_(x86-64|ppc64)",
                   "(ubuntu|ubt|osx)[0-9]*_x86-64",
                  ]
 
-# Try to guess a good platform. This does not try to cover all the
-# possibly compatible linux distributions, but tries to get right the
-# common one, obvious one. If you use a Unknownbuntu which is compatible
-# with Ubuntu 15.10 you will still have to give an explicit platform
-# string. 
-#
-# FIXME: we should have a fallback for lsb_release, since platform.dist
-# is going away.
-def detectArch():
-  hasOsRelease = exists("/etc/os-release")
-  osReleaseLines = open("/etc/os-release").readlines() if hasOsRelease else []
-  try:
-    import platform
-    platformTuple = platform.dist()
-    platformSystem = platform.system()
-    platformProcessor = platform.processor()
-    return doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, platformProcessor)
-  except:
-    return None
-
 # Detect number of available CPUs. Fallback to 1.
 def detectJobs():
   # Python 2.6+
diff --git a/aliDoctor b/aliDoctor
@@ -5,18 +5,20 @@ from os.path import exists, abspath
 from glob import glob
 import yaml
 from commands import getstatusoutput
+import logging
+from alibuild_helpers.log import debug, error, banner, info, success, warning
+from alibuild_helpers.log import logger_handler, logger, LogFormatter, ProgressPrint
+from alibuild_helpers.utilities import getPackageList, format, detectArch
 import subprocess
 
-def execute(command, prefix):
-  print prefix + ":" + command.strip("\n")
+def execute(command):
   popen = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
   lines_iterator = iter(popen.stdout.readline, "")
+  txt = ""
   for line in lines_iterator:
-    print prefix + ":" +line.strip("\n")  # yield line
-  output = popen.communicate()[0]
-  for x in output.split("\n"):
-    print prefix + ":" + output
-  print prefix + ": exit code " + str(popen.returncode)
+    txt += line # yield line
+  txt += popen.communicate()[0]
+  return (popen.returncode, txt)
 
 def prunePaths(workDir):
   for x in ["PATH", "LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH"]:
@@ -25,10 +27,64 @@ def prunePaths(workDir):
     workDirEscaped = re.escape("%s" % workDir) + "[^:]*:?"
     os.environ[x] = re.sub(workDirEscaped, "", os.environ[x])
 
+def checkPreferSystem(spec, cmd, homebrew_replacement):
+    if cmd == "false":
+      debug("Package %s can only be managed via alibuild." % spec["package"])
+      return (1, "")
+    cmd = homebrew_replacement + cmd
+    err, out = execute(cmd)
+    if not err:
+      success("Package %s will be picked up from the system." % spec["package"])
+      for x in out.split("\n"):
+        debug(spec["package"] + ": " + x)
+      return (err, "")
+
+    warning(format("Package %(p)s cannot be picked up from the system and will be built by aliBuild.\n"
+                   "This is due to the fact the following script fails:\n\n"
+                   "%(cmd)s\n\n"
+                   "with the following output:\n\n"
+                   "%(error)s\n",
+                   p=spec["package"],
+                   cmd=cmd,
+                   error="\n".join(["%s: %s" % (spec["package"],x) for x in out.split("\n")])))
+    return (err, "")
+
+def checkRequirements(spec, cmd, homebrew_replacement):
+    if cmd == "false":
+      debug("Package %s is not a system requirement." % spec["package"])
+      return (0, "")
+    cmd = homebrew_replacement + cmd
+    err, out = execute(cmd)
+    if not err:
+      success("Required package %s will be picked up from the system." % spec["package"])
+      debug(cmd)
+      for x in out.split("\n"):
+        debug(spec["package"] + ": " + x)
+      return (0, "")
+    error(format("Package %(p)s is a system requirement and cannot be found.\n"
+                 "This is due to the fact that the following script fails:\n\n"
+                 "%(cmd)s\n"
+                 "with the following output:\n\n"
+                 "%(error)s\n"
+                 "%(help)s\n",
+                 p=spec["package"],
+                 cmd=cmd,
+                 error="\n".join(["%s: %s" % (spec["package"],x) for x in out.split("\n")]),
+                 help=spec.get("system_requirement_missing")))
+    return (err, "")
+
 if __name__ == "__main__":
   parser = argparse.ArgumentParser()
-  parser.add_argument("-c", "--config", help="path to alidist", dest="config", default="alidist")
-  parser.add_argument("-w", "--work-dir", help="path to work dir", dest="workDir", default="workDir")
+  parser.add_argument("-a", "--architecture", help="force architecture",
+                      dest="architecture", default=detectArch())
+  parser.add_argument("-c", "--config", help="path to alidist",
+                      dest="config", default="alidist")
+  parser.add_argument("-w", "--work-dir", help="path to work dir",
+                      dest="workDir", default="workDir")
+  parser.add_argument("-d", "--debug", help="Show also successful tests.",
+                      dest="debug", action="store_true", default=False)
+  parser.add_argument("packages", nargs="+", help="Package to test",
+                      default=[])
   args = parser.parse_args()
   if not exists(args.config):
     parser.error("Wrong path to alidist specified: %s" % args.config)
@@ -37,16 +93,49 @@ if __name__ == "__main__":
 
   # Decide if we can use homebrew. If not, we replace it with "true" so
   # that we do not get spurious messages on linux
-  homebrew_replacement = "brew"
+  homebrew_replacement = ""
   err, output = getstatusoutput("which brew")
   if err:
-    homebrew_replacement = "true"
-
-  for x in glob("%s/*.sh" % args.config):
-    print "Reading", x
-    txt = file(x).read()
-    header, body = txt.split("---", 1)
-    spec = yaml.safe_load(header)
-    if "prefer_system_check" in spec:
-      execute(spec["prefer_system_check"].replace("brew", homebrew_replacement),
-              spec["package"])
+    homebrew_replacement = "brew() { true; }; "
+
+  logger.setLevel(logging.BANNER)
+  if args.debug:
+    logger.setLevel(logging.DEBUG)
+
+  specs = {}
+  packages = []
+  for p in args.packages:
+    path = "%s/%s.sh" % (args.config, p.lower())
+    if not exists(path):
+      error("Cannot find recipe %s for package %s." % (path, p))
+      continue
+    packages.append(p)
+
+  specs = {}
+  def unreachable():
+    assert(False)
+  (fromSystem, own, failed) = getPackageList(packages=packages,
+                                         specs=specs,
+                                         configDir=args.config,
+                                         preferSystem=False,
+                                         noSystem=False,
+                                         architecture=args.architecture,
+                                         disable=[],
+                                         defaults="release",
+                                         dieOnError=lambda x, y : unreachable,
+                                         performPreferCheck=lambda pkg, cmd : checkPreferSystem(pkg, cmd, homebrew_replacement),
+                                         performRequirementCheck=lambda pkg, cmd : checkRequirements(pkg, cmd, homebrew_replacement))
+
+  if fromSystem:
+    banner("The following packages will be picked up from the system:\n\n- " +
+           "\n- ".join(fromSystem) +
+           "\n\nIf this is not you want, you have to uninstall / unload them.")
+  if own:
+    banner("The following packages will be build by aliBuild because they couldn't be picked up from the system:\n\n- " +
+           "\n- ".join(own) +
+           "\n\nThis is not a real issue, but it might take longer the first time you invoke aliBuild.")
+  if failed:
+    banner("The following packages are system dependencies and could not be found:\n\n- "+
+          "\n- ".join(failed)
+         )
+    exit(1)
diff --git a/alibuild_helpers/log.py b/alibuild_helpers/log.py
@@ -4,7 +4,7 @@
 import socket, time
 from alibuild_helpers.utilities import format
 
-debug, error, warning, info, riemannStream = (None, None, None, None, None)
+debug, error, warning, info, success, riemannStream = (None, None, None, None, None, None)
 
 # A stream object which will end up pushing data to a riemann server
 class RiemannStream(object):
@@ -66,7 +66,8 @@ def __init__(self, fmtstr):
     self.COLOR_RESET = "\033[m" if sys.stdout.isatty() else ""
     self.LEVEL_COLORS = { logging.WARNING:  "\033[4;33m",
                           logging.ERROR:    "\033[4;31m",
-                          logging.CRITICAL: "\033[1;37;41m" } if sys.stdout.isatty() else {}
+                          logging.CRITICAL: "\033[1;37;41m",
+                          logging.SUCCESS:  "\033[1;32m" } if sys.stdout.isatty() else {}
   def format(self, record):
     if record.levelno == logging.BANNER and sys.stdout.isatty():
       lines = str(record.msg).split("\n")
@@ -115,6 +116,14 @@ def log_banner(self, message, *args, **kws):
     self._log(logging.BANNER, message, args, **kws)
 logging.Logger.banner = log_banner
 
+# Add loglevel SUCCESS (same as ERROR, but green)
+logging.SUCCESS = 45
+logging.addLevelName(logging.SUCCESS, "SUCCESS")
+def log_success(self, message, *args, **kws):
+  if self.isEnabledFor(logging.SUCCESS):
+    self._log(logging.SUCCESS, message, args, **kws)
+logging.Logger.success = log_success
+
 logger = logging.getLogger('alibuild')
 logger_handler = logging.StreamHandler()
 logger.addHandler(logger_handler)
@@ -125,6 +134,7 @@ def log_banner(self, message, *args, **kws):
 warning = logger.warning
 info = logger.info
 banner = logger.banner
+success = logger.success
 
 riemannStream = RiemannStream(host=getenv("RIEMANN_HOST"),
                               port=getenv("RIEMANN_PORT", "5555"))
diff --git a/alibuild_helpers/utilities.py b/alibuild_helpers/utilities.py
@@ -1,8 +1,9 @@
 #!/usr/bin/env python
-import subprocess
+import subprocess, re, yaml
 import pkg_resources  # part of setuptools
 from commands import getstatusoutput
-from os.path import dirname
+from os.path import dirname, exists
+import platform
 
 def format(s, **kwds):
   return s % kwds
@@ -48,10 +49,106 @@ def doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, pl
                 v=version.split(".")[0],
                 c=processor.replace("_", "-"))
 
+# Try to guess a good platform. This does not try to cover all the
+# possibly compatible linux distributions, but tries to get right the
+# common one, obvious one. If you use a Unknownbuntu which is compatible
+# with Ubuntu 15.10 you will still have to give an explicit platform
+# string. 
+#
+# FIXME: we should have a fallback for lsb_release, since platform.dist
+# is going away.
+def detectArch():
+  hasOsRelease = exists("/etc/os-release")
+  osReleaseLines = open("/etc/os-release").readlines() if hasOsRelease else []
+  try:
+    import platform
+    platformTuple = platform.dist()
+    platformSystem = platform.system()
+    platformProcessor = platform.processor()
+    return doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, platformProcessor)
+  except:
+    return None
+
 def getVersion():
   try:
     return pkg_resources.require("alibuild")[0].version
   except:
     cmd = "GIT_DIR=\'%s/.git\' git describe --tags" % dirname(dirname(__file__))
     err, version = getstatusoutput(cmd)
     return version if not err else "Unknown version."
+
+def filterByArchitecture(arch, requires):
+  for r in requires:
+    require, matcher = ":" in r and r.split(":", 1) or (r, ".*")
+    if re.match(matcher, arch):
+      yield require
+
+def getPackageList(packages, specs, configDir, preferSystem, noSystem,
+                   architecture, disable, defaults, dieOnError, performPreferCheck, performRequirementCheck):
+  systemPackages = set()
+  ownPackages = set()
+  failedRequirements = set()
+  testCache = {}
+  requirementsCache = {}
+  while packages:
+    p = packages.pop(0)
+    if p in specs:
+      continue
+    try:
+      d = open("%s/%s.sh" % (configDir, p.lower())).read()
+    except IOError,e:
+      dieOnError(True, str(e))
+    header, recipe = d.split("---", 1)
+    spec = yaml.safe_load(header)
+    dieOnError(spec["package"].lower() != p.lower(),
+               "%s.sh has different package field: %s" % (p, spec["package"]))
+    # If --always-prefer-system is passed or if prefer_system is set to true
+    # inside the recipe, use the script specified in the prefer_system_check
+    # stanza to see if we can use the system version of the package.
+    if not noSystem and (preferSystem or re.match(spec.get("prefer_system", "(?!.*)"), architecture)):
+      cmd = spec.get("prefer_system_check", "false")
+      if not spec["package"] in testCache:
+        testCache[spec["package"]] = performPreferCheck(spec, cmd.strip())
+
+      err, output = testCache[spec["package"]]
+
+      if not err:
+        systemPackages.update([spec["package"]])
+        disable.append(spec["package"])
+      else:
+        ownPackages.update([spec["package"]])
+
+    dieOnError(("system_requirement" in spec) and recipe.strip("\n\t "),
+               "System requirements %s cannot have a recipe" % spec["package"])
+    if re.match(spec.get("system_requirement", "(?!.*)"), architecture):
+      cmd = spec.get("system_requirement_check", "false")
+      if not spec["package"] in requirementsCache:
+        requirementsCache[spec["package"]] = performRequirementCheck(spec, cmd.strip())
+
+      err, output = requirementsCache[spec["package"]]
+      if err:
+        failedRequirements.update([spec["package"]])
+        spec["version"] = "failed"
+      else:
+        disable.append(spec["package"])
+
+    if spec["package"] in disable:
+      continue
+
+    # For the moment we treat build_requires just as requires.
+    fn = lambda what: filterByArchitecture(architecture, spec.get(what, []))
+    spec["requires"] = [x for x in fn("requires") if not x in disable]
+    spec["build_requires"] = [x for x in fn("build_requires") if not x in disable]
+    if spec["package"] != "defaults-" + defaults:
+      spec["build_requires"].append("defaults-" + defaults)
+    spec["runtime_requires"] = spec["requires"]
+    spec["requires"] = spec["runtime_requires"] + spec["build_requires"]
+    # Check that version is a string
+    dieOnError(not isinstance(spec["version"], basestring),
+               "In recipe \"%s\": version must be a string" % p)
+    spec["tag"] = spec.get("tag", spec["version"])
+    spec["version"] = spec["version"].replace("/", "_")
+    spec["recipe"] = recipe.strip("\n")
+    specs[spec["package"]] = spec
+    packages += spec["requires"]
+  return (systemPackages, ownPackages, failedRequirements)
