From 028c59615f05ffcea653d65ef80c9f659e5d0931 Mon Sep 17 00:00:00 2001
From: Alfred Bratterud <alfred@includeos.org>
Date: Tue, 24 Jan 2017 16:33:08 +0000
Subject: [PATCH 01/27] install: added missing python deps for Linux. #1087

---
 etc/install_build_requirements.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/etc/install_build_requirements.sh b/etc/install_build_requirements.sh
index 206c1b35c7..ffdc1d35ed 100755
--- a/etc/install_build_requirements.sh
+++ b/etc/install_build_requirements.sh
@@ -28,7 +28,7 @@ case $SYSTEM in
                     clang_version="3.8"
                 fi
 
-                DEPENDENCIES="curl make clang-$clang_version nasm bridge-utils qemu jq cmake $DEPENDENCIES"
+                DEPENDENCIES="curl make clang-$clang_version nasm bridge-utils qemu jq python-jsonschema python-psutil cmake $DEPENDENCIES"
                 echo ">>> Installing dependencies (requires sudo):"
                 echo "    Packages: $DEPENDENCIES"
                 sudo apt-get update || exit 1
@@ -36,14 +36,14 @@ case $SYSTEM in
                 exit 0;
                 ;;
             "fedora")
-                DEPENDENCIES="curl make clang nasm bridge-utils qemu jq python-jsonschema cmake"
+                DEPENDENCIES="curl make clang nasm bridge-utils qemu jq python-jsonschema python-psutil cmake"
                 echo ">>> Installing dependencies (requires sudo):"
                 echo "    Packages: $DEPENDENCIES"
                 sudo dnf install $DEPENDENCIES || exit 1
                 exit 0;
                 ;;
             "arch")
-                DEPENDENCIES="curl make clang nasm bridge-utils qemu jq cmake"
+                DEPENDENCIES="curl make clang nasm bridge-utils qemu jq python-jsonschema python-psutil cmake"
                 echo ">>> Installing dependencies (requires sudo):"
                 echo "    Packages: $DEPENDENCIES"
                 sudo pacman -Syyu

From 1319e58bafa4041c6db6c35eefa6f0cf1934e6ae Mon Sep 17 00:00:00 2001
From: Gonzo <fwsgonzo@hotmail.com>
Date: Tue, 24 Jan 2017 18:00:00 +0100
Subject: [PATCH 02/27] pci: Enable missing command flags

---
 src/hw/pci_device.cpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/hw/pci_device.cpp b/src/hw/pci_device.cpp
index 6cec875c65..d022e78493 100644
--- a/src/hw/pci_device.cpp
+++ b/src/hw/pci_device.cpp
@@ -38,6 +38,10 @@
 #define PCI_HEADER_REG		0x0e
 #define PCI_BIST_REG		0x0f
 
+#define PCI_COMMAND_IO			0x01
+#define PCI_COMMAND_MEM			0x02
+#define PCI_COMMAND_MASTER	0x04
+
 namespace hw {
 
   static const char* classcodes[] {
@@ -150,6 +154,11 @@ namespace hw {
   PCI_Device::PCI_Device(const uint16_t pci_addr, const uint32_t device_id)
       : pci_addr_{pci_addr}, device_id_{device_id}
   {
+    // set master, mem and io flags
+    uint32_t cmd = read_dword(PCI_CMD_REG);
+    cmd |= PCI_COMMAND_MASTER | PCI_COMMAND_MEM | PCI_COMMAND_IO;
+    write_dword(PCI_CMD_REG, cmd);
+
     //We have device, so probe for details
     devtype_.reg = read_dword(pci_addr, PCI::CONFIG_CLASS_REV);
 

From 4b34758b2be90abb812b532f88719ec7e2e882ac Mon Sep 17 00:00:00 2001
From: Alfred Bratterud <alfred@includeos.org>
Date: Tue, 24 Jan 2017 18:04:48 +0000
Subject: [PATCH 03/27] vmrunner: make mac optional, use virtio/tap by default

---
 vmrunner/vmrunner.py | 41 ++++++++++++++++++++++++++++-------------
 1 file changed, 28 insertions(+), 13 deletions(-)

diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py
index d728605b8b..b9d3182b3e 100644
--- a/vmrunner/vmrunner.py
+++ b/vmrunner/vmrunner.py
@@ -38,7 +38,9 @@
               "CALLBACK_FAILED" : 68,
               "BUILD_FAIL" : 69,
               "ABORT" : 70,
-              "VM_EOT" : 71 }
+              "VM_EOT" : 71,
+              "BOOT_FAILED": 72
+}
 
 def get_exit_code_name (exit_code):
     for name, code in exit_codes.iteritems():
@@ -124,15 +126,25 @@ def __init__(self, config):
     def name(self):
         return "Qemu"
 
-    def drive_arg(self, filename, drive_type="virtio", drive_format="raw", media_type="disk"):
-        return ["-drive","file="+filename+",format="+drive_format+",if="+drive_type+",media="+media_type]
+    def drive_arg(self, filename, drive_type = "virtio", drive_format = "raw", media_type = "disk"):
+        return ["-drive","file=" + filename
+                + ",format=" + drive_format
+                + ",if=" + drive_type
+                + ",media=" + media_type]
 
-    def net_arg(self, backend, device, if_name = "net0", mac="c0:01:0a:00:00:2a"):
-        qemu_ifup = INCLUDEOS_HOME+"/includeos/scripts/qemu-ifup"
-        # FIXME: this needs to get removed
+    def net_arg(self, backend, device, if_name = "net0", mac = None):
+        qemu_ifup = INCLUDEOS_HOME + "/includeos/scripts/qemu-ifup"
+
+        # FIXME: this needs to get removed, e.g. fetched from the schema
         names = {"virtio" : "virtio-net", "vmxnet" : "vmxnet3", "vmxnet3" : "vmxnet3"}
-        return ["-device", names[device]+",netdev="+if_name+",mac="+mac,
-                "-netdev", backend+",id="+if_name+",script="+qemu_ifup]
+
+        device = names[device] + ",netdev=" + if_name
+
+        # Add mac-address if specified
+        if mac: device += ",mac=" + mac
+
+        return ["-device", device,
+                "-netdev", backend + ",id=" + if_name + ",script=" + qemu_ifup]
 
     def kvm_present(self):
         command = "egrep -m 1 '^flags.*(vmx|svm)' /proc/cpuinfo"
@@ -203,7 +215,8 @@ def boot(self, multiboot, kernel_args = "", image_name = None):
         i = 0
         if "net" in self._config:
             for net in self._config["net"]:
-                net_args += self.net_arg(net["backend"], net["device"], "net"+str(i), net["mac"])
+                mac = net["mac"] if "mac" in net else None
+                net_args += self.net_arg(net["backend"], net["device"], "net"+str(i), mac)
                 i+=1
 
         mem_arg = []
@@ -213,7 +226,7 @@ def boot(self, multiboot, kernel_args = "", image_name = None):
         vga_arg = ["-nographic" ]
         if "vga" in self._config:
             vga_arg = ["-vga", str(self._config["vga"])]
-        
+
         # TODO: sudo is only required for tap networking and kvm. Check for those.
         command = ["sudo", "qemu-system-x86_64"]
         if self.kvm_present(): command.append("--enable-kvm")
@@ -459,9 +472,10 @@ def boot(self, timeout = 60, multiboot = True, kernel_args = "booted with vmrunn
         try:
             self._hyper.boot(multiboot, kernel_args, image_name)
         except Exception as err:
-            print color.WARNING("Exception raised while booting ")
+            print color.WARNING("Exception raised while booting: ")
+            print_exception()
             if (timeout): self._timer.cancel()
-            self.exit(exit_codes["CALLBACK_FAILED"], str(err))
+            self.exit(exit_codes["BOOT_FAILED"], str(err))
 
         # Start analyzing output
         while self._hyper.poll() == None and not self._exit_status:
@@ -526,7 +540,8 @@ def boot(self, timeout = 60, multiboot = True, kernel_args = "booted with vmrunn
 validate_vm.load_schema(schema_path)
 validate_vm.has_required_stuff(".")
 
-default_spec = {"image" : "service.img"}
+default_spec = {"image" : "service.img",
+                "net" : [{"device" : "virtio", "backend" : "tap" }] }
 
 # Provide a list of VM's with validated specs
 vms = []

From e51db87894575419756d95eea197f17f7032d7d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20=C3=85kesson?= <andreas.jm.akesson@gmail.com>
Date: Wed, 25 Jan 2017 11:32:49 +0100
Subject: [PATCH 04/27] install: Added pip installation & modules to macOS
 install #1087

---
 etc/install_osx.sh | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/etc/install_osx.sh b/etc/install_osx.sh
index bbcfcb68d6..27c1261fca 100755
--- a/etc/install_osx.sh
+++ b/etc/install_osx.sh
@@ -67,6 +67,19 @@ function install_binutils {
   source ./etc/install_binutils.sh
 }
 
+## python packages (pip) ##
+PIP_INSTALLED=false
+PIP_MODS=(jsonschema psutil)
+echo -e "\n python pip\t - for installing necessary python modules used when booting services: ${PIP_MODS[*]}"
+
+## Check if pip is installed ##
+if command -v pip >/dev/null 2>&1; then
+  PIP_INSTALLED=true
+  echo -e " > Found"
+else
+  echo -e " > Not found"
+fi
+
 ## WARN ABOUT XCODE CLT ##
 
 echo -e "\n NOTE: Cannot tell if Xcode Command Line Tools is installed - installation MAY fail if not installed."
@@ -85,6 +98,15 @@ then
         install_binutils
     fi
 
+    if (! $PIP_INSTALLED); then
+      echo -e "\n>> Installing pip (with sudo)"
+      curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+      sudo python get-pip.py
+      rm get-pip.py
+    fi
+
+    pip install ${PIP_MODS[*]}
+
     echo -e "\n>>> Done installing dependencies."
 fi
 

From 80326dc1b7000741320c054031279b88b6bfb346 Mon Sep 17 00:00:00 2001
From: fwsgonzo <fwsgonzo@hotmail.com>
Date: Wed, 25 Jan 2017 18:18:01 +0100
Subject: [PATCH 05/27] kernel: Rearrange ELF symbol header, simplify some
 things

---
 src/crt/c_abi.c      |  43 ++++++++--------
 src/kernel/elf.cpp   | 116 +++++++++++++++++++++++--------------------
 vmbuild/elf_syms.cpp |  27 ++++------
 3 files changed, 93 insertions(+), 93 deletions(-)

diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c
index 24a5276d9d..3343b04e56 100644
--- a/src/crt/c_abi.c
+++ b/src/crt/c_abi.c
@@ -24,7 +24,7 @@
 #include <string.h>
 #include <kprint>
 
-#define HEAP_ALIGNMENT   16
+#define HEAP_ALIGNMENT   63
 caddr_t heap_begin;
 caddr_t heap_end;
 
@@ -47,28 +47,26 @@ void _init_c_runtime()
   extern char _end;
 
   /// init backtrace functionality
-  extern void _move_elf_symbols(void*, void*);
-  extern void _apply_parser_data(void*);
-  extern int  _get_elf_section_size(const void*);
-  // first measure the size of symbols
-  int symsize = _get_elf_section_size(&_ELF_SYM_START_);
-  // estimate somewhere in heap its safe to move them
-  char* SYM_LOCATION = &_end + 2 * (4096 + symsize);
-  _move_elf_symbols(&_ELF_SYM_START_, SYM_LOCATION);
+  extern int  _get_elf_section_datasize(const void*);
+  extern void _move_elf_syms_location(const void*, void*);
+  // store symbols temporarily in a safe location
+  char* sym_temp = &_end + 
+        2*4096 + _get_elf_section_datasize(&_ELF_SYM_START_);
+  _move_elf_syms_location(&_ELF_SYM_START_, sym_temp);
 
   // Initialize .bss section
   extern char _BSS_START_, _BSS_END_;
   streamset8(&_BSS_START_, 0, &_BSS_END_ - &_BSS_START_);
 
   // Initialize the heap before exceptions
-  heap_begin = &_end + 0xfff;
-  // page-align heap, because its not aligned
-  heap_begin = (char*) ((uintptr_t)heap_begin & ~(uintptr_t) 0xfff);
+  // cache-align heap, because its not aligned
+  heap_begin = &_end + 64 + HEAP_ALIGNMENT;
+  heap_begin = (char*) ((uintptr_t)heap_begin & ~(uintptr_t) HEAP_ALIGNMENT);
   // heap end tracking, used with sbrk
   heap_end   = heap_begin;
-  // validate that heap is page aligned
+  // validate that heap is aligned
   int validate_heap_alignment =
-      ((uintptr_t)heap_begin & (uintptr_t) 0xfff) == 0;
+      ((uintptr_t)heap_begin & (uintptr_t) HEAP_ALIGNMENT) == 0;
 
   /// initialize newlib I/O
   _REENT_INIT_PTR(_REENT);
@@ -77,9 +75,13 @@ void _init_c_runtime()
   stdout = _REENT->_stdout; // stdout == 2
   stderr = _REENT->_stderr; // stderr == 3
 
-  // move symbols (again) to heap, before calling global constructors
-  extern void* _relocate_to_heap(void*);
-  void* symheap = _relocate_to_heap(SYM_LOCATION);
+  /// init ELF / backtrace functionality
+  extern void _elf_relocate_to_heap();
+  extern void _init_elf_parser();
+  // move ELF symbols into heap
+  _elf_relocate_to_heap();
+  // enable ELF symbols here (before global constructors)
+  _init_elf_parser();
 
   /// initialize exceptions before we can run constructors
   extern char __eh_frame_start[];
@@ -91,23 +93,18 @@ void _init_c_runtime()
   extern void _init();
   _init();
 
-  // set ELF symbols location here (after initializing everything else)
-  _apply_parser_data(symheap);
-
   // sanity checks
   assert(heap_begin >= &_end);
   assert(heap_end >= heap_begin);
   assert(validate_heap_alignment);
 }
 
-// global/static objects should never be destructed here, so ignore this
-void* __dso_handle;
-
 // stack-protector
 __attribute__((noreturn))
 void __stack_chk_fail(void)
 {
   panic("Stack protector: Canary modified");
+  __builtin_unreachable();
 }
 
 // old function result system
diff --git a/src/kernel/elf.cpp b/src/kernel/elf.cpp
index 2f217d2a42..b252645129 100644
--- a/src/kernel/elf.cpp
+++ b/src/kernel/elf.cpp
@@ -74,10 +74,10 @@ class ElfTables
   func_offset getsym(Elf32_Addr addr)
   {
     // probably just a null pointer with ofs=addr
-    if (UNLIKELY(addr < 0x7c00))
+    if (UNLIKELY(addr < 0x1000))
         return {null_stringz, 0, addr};
     // definitely in the bootloader
-    if (UNLIKELY(addr < 0x7e00))
+    if (UNLIKELY(addr >= 0x7c00 && addr < 0x7e00))
         return {boot_stringz, 0x7c00, addr - 0x7c00};
     // resolve manually from symtab
     auto* sym = getaddr(addr);
@@ -87,7 +87,11 @@ class ElfTables
       auto base   = sym->st_value;
       auto offset = addr - base;
       // return string name for symbol
-      return {demangle( sym_name(sym) ), base, offset};
+      const char* name = sym_name(sym);
+      if (name)
+          return {demangle(name), base, offset};
+      else
+          return {to_hex_string(base), base, offset};
     }
     // function or space not found
     return {to_hex_string(addr), addr, 0};
@@ -95,9 +99,11 @@ class ElfTables
   safe_func_offset getsym_safe(Elf32_Addr addr, char* buffer, size_t length)
   {
     // probably just a null pointer with ofs=addr
-    if (addr < 0x7c00) return {null_stringz, 0, addr};
+    if (UNLIKELY(addr < 0x1000))
+        return {null_stringz, 0, addr};
     // definitely in the bootloader
-    if (addr < 0x7e00) return {boot_stringz, 0x7c00, addr - 0x7c00};
+    if (UNLIKELY(addr >= 0x7c00 && addr < 0x7e00))
+        return {boot_stringz, 0x7c00, addr - 0x7c00};
     // resolve manually from symtab
     auto* sym = getaddr(addr);
     if (LIKELY(sym)) {
@@ -153,17 +159,13 @@ class ElfTables
   }
   std::string demangle(const char* name) const
   {
-    if (name[0] == '_') {
-      int status;
-      // internally, demangle just returns buf when status is ok
-      auto* res = __cxa_demangle(name, nullptr, 0, &status);
-      if (status == 0) {
-        std::string result(res);
-        std::free(res);
-        return result;
-      }
-    }
-    return std::string(name);
+    char buffer[2048];
+    const char* res = demangle_safe(name, buffer, sizeof(buffer));
+    assert(strlen(res));
+    if (res)
+        return std::string(res);
+    else
+        return std::string(name);
   }
   const char* demangle_safe(const char* name, char* buffer, size_t buflen) const
   {
@@ -256,8 +258,8 @@ std::vector<func_offset> Elf::get_functions()
 
 void print_backtrace()
 {
-  char _symbol_buffer[512];
-  char _btrace_buffer[512];
+  char _symbol_buffer[1024];
+  char _btrace_buffer[1024];
 
   if (Elf::get_strtab() == NULL) {
     int len = snprintf(_btrace_buffer, sizeof(_btrace_buffer),
@@ -343,59 +345,67 @@ void _validate_elf_symbols()
   }
 }
 
-struct relocate_header {
-  SymTab symtab;
-  StrTab strtab;
+static struct relocated_header {
+  uint32_t   entries = 0xFFFF;
+  uint32_t   strsize = 0xFFFF;
+  Elf32_Sym* syms = nullptr;
+
+  const char* strings() const {
+    return (char*) &syms[entries];
+  }
+} relocs;
+struct elfsyms_header {
+  uint32_t  symtab_entries;
+  uint32_t  strtab_size;
+  Elf32_Sym syms[0];
 };
 
+#include <kprint>
 extern "C"
-int _get_elf_section_size(const void* location)
+int _get_elf_section_datasize(const void* location)
 {
-  auto& hdr = *(relocate_header*) location;
-  return sizeof(relocate_header) + hdr.symtab.entries * sizeof(Elf32_Sym) + hdr.strtab.size;
+  auto& hdr = *(elfsyms_header*) location;
+  // special case for missing call to elf_syms
+  if (hdr.symtab_entries == 0) return 0;
+  return hdr.symtab_entries * sizeof(Elf32_Sym) + hdr.strtab_size;
 }
 
-#include <kprint>
 extern "C"
-void _move_elf_symbols(void* old_location, void* new_location)
+void _move_elf_syms_location(const void* location, void* new_location)
 {
-  int size = _get_elf_section_size(old_location);
-  // validate locations
-  if ((char*) new_location >= (char*) old_location &&
-      (char*) new_location + size < (char*) old_location)
-  {
-    kprintf("ELF symbol sections are inside each other!\n");
-    kprintf("Moving %d from %p -> %p (%p)\n",
-        size, old_location, new_location, (char*) new_location + size);
-    assert(0);
+  int size = _get_elf_section_datasize(location);
+  // stripped variant
+  if (size == 0) {
+    relocs.entries = 0;
+    relocs.strsize = 0;
+    return;
   }
-  // copy over
-  memcpy(new_location, old_location, size);
+  // old header
+  auto* hdr = (elfsyms_header*) location;
   // update header
-  auto* newhdr = (relocate_header*) new_location;
-  const char* base = ((char*) newhdr) + sizeof(relocate_header);
-  newhdr->symtab.base = (Elf32_Sym*) base;
-  newhdr->strtab.base = &base[newhdr->symtab.entries * sizeof(Elf32_Sym)];
+  relocs.entries = hdr->symtab_entries;
+  relocs.strsize = hdr->strtab_size;
+  relocs.syms    = (Elf32_Sym*) new_location;
+  // copy symbol and string data
+  memcpy(relocs.syms, hdr->syms, size);
 }
-#if !defined(__MACH__)
-#include <malloc.h>
-#endif
 extern "C"
-void* _relocate_to_heap(void* old_location)
+void _elf_relocate_to_heap()
 {
-  int total = _get_elf_section_size(old_location);
-  void* heap_location = malloc(total);
-  _move_elf_symbols(old_location, heap_location);
-  return heap_location;
+  int size = relocs.entries * sizeof(Elf32_Sym) + relocs.strsize;
+  char* newloc = new char[size];
+  // copy syms & update location
+  memcpy(newloc, relocs.syms, size);
+  relocs.syms = (Elf32_Sym*) newloc;
 }
 
+#include <service>
 extern "C"
-void _apply_parser_data(char* location)
+void _init_elf_parser()
 {
-  if (location) {
-    auto& hdr = *(relocate_header*) location;
+  if (relocs.entries) {
     // apply changes to the symbol parser from custom location
-    parser.set(hdr.symtab.base, hdr.symtab.entries, hdr.strtab.base);
+    parser.set(relocs.syms, relocs.entries, relocs.strings());
   }
   else {
     // symbols and strings are stripped out
diff --git a/vmbuild/elf_syms.cpp b/vmbuild/elf_syms.cpp
index bda13fc2f4..6cd5d08f92 100644
--- a/vmbuild/elf_syms.cpp
+++ b/vmbuild/elf_syms.cpp
@@ -15,9 +15,8 @@ static const char* elf_offset(int o) noexcept {
   return (char*)elf_header_location + o;
 }
 
-static void prune_elf_symbols();
-static char* pruned_location;
-static int   pruned_size = 0;
+static int prune_elf_symbols();
+static char* pruned_location = nullptr;
 
 int main(int argc, const char** args)
 {
@@ -38,7 +37,7 @@ int main(int argc, const char** args)
 
   // validate symbols
   elf_header_location = (decltype(elf_header_location)) fdata;
-  prune_elf_symbols();
+  int pruned_size = prune_elf_symbols();
 
   // write symbols to binary file
   f = fopen("_elf_symbols.bin", "w");
@@ -58,16 +57,11 @@ struct StrTab {
 
   StrTab(char* base, uint32_t size) : base(base), size(size) {}
 };
-struct relocate_header {
-  SymTab symtab;
-  StrTab strtab;
-};
 
 struct relocate_header32 {
-  uint32_t symtab_base;
-  uint32_t symtab_entries;
-  uint32_t strtab_base;
-  uint32_t strtab_size;
+  uint32_t  symtab_entries;
+  uint32_t  strtab_size;
+  Elf32_Sym syms[0];
 };
 
 static int relocate_pruned_sections(char* new_location, SymTab& symtab, StrTab& strtab)
@@ -75,7 +69,7 @@ static int relocate_pruned_sections(char* new_location, SymTab& symtab, StrTab&
   auto& hdr = *(relocate_header32*) new_location;
 
   // first prune symbols
-  auto*  symloc = (Elf32_Sym*) (new_location + sizeof(relocate_header32));
+  auto*  symloc = hdr.syms;
   size_t symidx = 0;
   for (size_t i = 0; i < symtab.entries; i++)
   {
@@ -110,7 +104,7 @@ static int relocate_pruned_sections(char* new_location, SymTab& symtab, StrTab&
          hdr.strtab_size * sizeof(char);
 }
 
-static void prune_elf_symbols()
+static int prune_elf_symbols()
 {
   SymTab symtab { nullptr, 0 };
   std::vector<StrTab> strtabs;
@@ -144,10 +138,9 @@ static void prune_elf_symbols()
     // allocate worst case, guaranteeing we have enough space
     pruned_location =
         new char[sizeof(relocate_header32) + symtab.entries * sizeof(Elf32_Sym) + strtab.size];
-    pruned_size = relocate_pruned_sections(pruned_location, symtab, strtab);
-    return;
+    return relocate_pruned_sections(pruned_location, symtab, strtab);
   }
   // stripped variant
   pruned_location = new char[sizeof(relocate_header32)];
-  pruned_size = sizeof(relocate_header32);
+  return sizeof(relocate_header32);
 }

From a359cb3d825ca3ac104f4af91ad303759c46e244 Mon Sep 17 00:00:00 2001
From: fwsgonzo <fwsgonzo@hotmail.com>
Date: Wed, 25 Jan 2017 20:55:24 +0100
Subject: [PATCH 06/27] posix: Add various stubs to signal, mman and a few
 linux stubs

---
 api/posix/sys/mman.h   | 24 ++++++++++++++++++++----
 api/sys/dlfcn.h        | 25 +++++++++++++++++++++++++
 api/sys/signal.h       | 42 ++++++++++++++++++++++++++++++++++++++++++
 src/crt/c_abi.c        | 11 +++++++++++
 src/posix/sys/mman.cpp |  6 ++++++
 5 files changed, 104 insertions(+), 4 deletions(-)
 create mode 100644 api/sys/dlfcn.h
 create mode 100644 api/sys/signal.h

diff --git a/api/posix/sys/mman.h b/api/posix/sys/mman.h
index 23faf6b7ef..25c55d8d81 100644
--- a/api/posix/sys/mman.h
+++ b/api/posix/sys/mman.h
@@ -24,13 +24,29 @@ extern "C" {
 #endif
 
 #include <string.h>
+#include <sys/types.h>
 typedef _off_t off_t;
 
-void *mmap(void* addr, size_t length,
-            int prot,  int flags,
-            int fd,    off_t offset);
-int munmap(void* addr, size_t length);
+struct posix_typed_mem_info
+{
+  size_t  posix_tmi_length;
+};
 
+int    mlock(const void *, size_t);
+int    mlockall(int);
+void  *mmap(void *, size_t, int, int, int, off_t);
+int    mprotect(void *, size_t, int);
+int    msync(void *, size_t, int);
+int    munlock(const void *, size_t);
+int    munlockall(void);
+int    munmap(void *, size_t);
+int    posix_madvise(void *, size_t, int);
+int    posix_mem_offset(const void *__restrict__, size_t, off_t *__restrict__,
+           size_t *__restrict__, int *__restrict__);
+int    posix_typed_mem_get_info(int, struct posix_typed_mem_info *);
+int    posix_typed_mem_open(const char *, int, int);
+int    shm_open(const char *, int, mode_t);
+int    shm_unlink(const char *);
 
 // Page can be executed.
 #define PROT_EXEC     0x1
diff --git a/api/sys/dlfcn.h b/api/sys/dlfcn.h
new file mode 100644
index 0000000000..49e7ff83a5
--- /dev/null
+++ b/api/sys/dlfcn.h
@@ -0,0 +1,25 @@
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void *dlopen(const char *filename, int flag);
+char *dlerror(void);
+void *dlsym(void *handle, const char *symbol);
+int dlclose(void *handle); 
+
+
+#define RTLD_LAZY      1
+#define RTLD_NOW       2
+#define RTLD_GLOBAL    3
+#define RTLD_LOCAL     4
+#define RTLD_NODELETE  6
+#define RTLD_NOLOAD    7
+#define RTLD_DEEPBIND  8
+
+#define RTLD_DEFAULT   1
+
+#ifdef __cplusplus
+}
+#endif
diff --git a/api/sys/signal.h b/api/sys/signal.h
new file mode 100644
index 0000000000..0753636fef
--- /dev/null
+++ b/api/sys/signal.h
@@ -0,0 +1,42 @@
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2016 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#ifndef SYS_SIGNAL_H
+#define SYS_SIGNAL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct siginfo_t
+{
+  int    si_signo; // Signal number.
+  int    si_code;  // Signal code.
+
+  int    sa_sigaction;
+};
+
+#define SA_RESETHAND  666
+
+#ifdef __cplusplus
+}
+#endif
+
+#include_next <signal.h>
+
+#endif
diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c
index 3343b04e56..b1c9b92faa 100644
--- a/src/crt/c_abi.c
+++ b/src/crt/c_abi.c
@@ -114,6 +114,17 @@ int* __errno_location(void)
   return &errno;
 }
 
+#include <setjmp.h>
+int _setjmp(jmp_buf env)
+{
+  return setjmp(env);
+}
+// linux strchr variant (NOTE: not completely the same!)
+void *__rawmemchr (const void *s, int c)
+{
+  return strchr((const char*) s, c);
+}
+
 /// assert() interface of ISO POSIX (2003)
 void __assert_fail(const char * assertion, const char * file, unsigned int line, const char * function)
 {
diff --git a/src/posix/sys/mman.cpp b/src/posix/sys/mman.cpp
index 12170358f8..5c448fdf06 100644
--- a/src/posix/sys/mman.cpp
+++ b/src/posix/sys/mman.cpp
@@ -70,3 +70,9 @@ int munmap(void* addr, size_t length)
   errno = EINVAL;
   return -1;
 }
+
+int mprotect(void *addr, size_t len, int prot)
+{
+  printf("mprotect %p:%u with flags %#x\n", addr, len, prot);
+  return 0;
+}

From f0e2c835499b419f0171a1ae281c627e280a1aee Mon Sep 17 00:00:00 2001
From: Alfred Bratterud <alfred.bratterud@hioa.no>
Date: Wed, 25 Jan 2017 23:09:46 +0100
Subject: [PATCH 07/27] boot: no cmake required for binaries, use verbose
 option, added json option, better defaults

---
 etc/boot                |  75 +++++++++++++-------
 vmrunner/validate_vm.py |  63 +++++++++--------
 vmrunner/vm.schema.json |   6 ++
 vmrunner/vmrunner.py    | 153 ++++++++++++++++++++++++++++------------
 4 files changed, 197 insertions(+), 100 deletions(-)

diff --git a/etc/boot b/etc/boot
index aa15728adc..19486f045f 100755
--- a/etc/boot
+++ b/etc/boot
@@ -18,6 +18,12 @@ if not os.path.isdir(INCLUDEOS_PREFIX + "/includeos"):
     print "Couldn't find IncludeOS installation. If you installed to a non-default location (e.g. not " + default_loc + ") please set the environment variable INCLUDEOS_PREFIX to point to this location."
     sys.exit(0)
 
+# Location of vmrunner
+sys.path.append(INCLUDEOS_PREFIX + "/includeos")
+
+from vmrunner.prettify import color
+
+
 # Argparse
 parser = argparse.ArgumentParser(
   description="IncludeOS vmrunner. Builds and runs an IncludeOS service")
@@ -37,51 +43,68 @@ parser.add_argument("--create-bridge", dest="bridge", action="store_true",
 parser.add_argument("-g", "--grub", dest="grub", action="store_true",
                     help="Create image with GRUB bootloader that will boot provided binary")
 
-parser.add_argument('binary', action="store", type=str,
-                    help="Filename of the IncludeOS service binary")
+parser.add_argument("-j", "--config", dest="config", type = str,
+                    help="Location of VM JSON config")
+
+parser.add_argument('vm_location', action="store", type = str,
+                    help="Location of the IncludeOS service binary, image or source")
 
 parser.add_argument('vmargs', nargs='*', help="Arguments to pass on to the VM start / main")
+
 args = parser.parse_args()
 
-# Location of vmrunner
-sys.path.append(INCLUDEOS_PREFIX + "/includeos")
 
-from vmrunner.prettify import color
+# Pretty printing from this command
+nametag = "<boot>"
+INFO = color.INFO(nametag)
 
 # Override VM output prepension
 color.VM_PREPEND = ""
 
-# Pretty printing from this command
-nametag = "<boot>"
-INFO = color.INFO(nametag)
+# in verbose mode we will set VERBOSE=1 for this environment
+VERB = False
+if (args.verbose):
+    os.environ["VERBOSE"] = "1"
+    VERB = True
+    print INFO, "VERBOSE mode set for environment"
+# Avoid the verbose var to hang around next run
+elif "VERBOSE" in os.environ:
+    del os.environ["VERBOSE"]
+
+# Note: importing vmrunner will make it start looking for VM's
+# vmrunner also relies on the verbose env var to be set on initialization
+from vmrunner import vmrunner
+
 
 # We can boot either a binary without bootloader, or an image with bootloader allready attached
 has_bootloader = False
 
-print INFO , "Args to pass to VM: ", args.vmargs
+if VERB: print INFO , "Args to pass to VM: ", args.vmargs
 # path w/name of VM image to run
-image_name = args.binary
+image_name = args.vm_location
 
 # if the binary argument is a directory, go there immediately and
 # then initialize stuff ...
-if (os.path.isdir(args.binary)):
-    image_name = os.path.abspath(args.binary)
-    print "Changing directory to: " + image_name
-    os.chdir(os.path.abspath(args.binary))
+if (os.path.isdir(args.vm_location)):
+    image_name = os.path.abspath(args.vm_location)
+    if VERB: print INFO, "Changing directory to  " + image_name
+    os.chdir(os.path.abspath(args.vm_location))
 
-# in verbose mode we will just set VERBOSE=1 for this environment
-if (args.verbose):
-    os.environ['VERBOSE'] = "1"
 
-# Note: importing vmrunner will make it start looking for VM's
-from vmrunner import vmrunner
 if len(vmrunner.vms) < 1:
-    print color.FAIL("No vm description files found - nothing to boot")
+    # This should never happen - there should always be a default
+    print color.FAIL("No vm object found in vmrunner - nothing to boot")
     exit(-1)
 
-print color.INFO(nametag), len(vmrunner.vms), "VM initialized. Commencing build- and boot..."
+if VERB:
+    print INFO, len(vmrunner.vms), "VM initialized. Commencing build- and boot..."
+
+config = None
+if args.config:
+    config = os.path.abspath(args.config)
+    if VERB: print INFO, "Using config file", config
 
-vm = vmrunner.vms[0]
+vm = vmrunner.vm(config = config)
 
 # For multible JSON configs, find the one matching the provided image name
 for vm_ in vmrunner.vms:
@@ -99,15 +122,15 @@ if (args.bridge):
 # If the binary name is a folder, such as ".", build the service
 if (os.path.isdir(image_name)):
     vm.cmake()
-    # also, set image name to 
+    # also, set image name to
     image_name = open("binary.txt", 'r').read()
 # If the binary name has an extension (e.g. a '.'), assume it has a bootloader attached
 # NOTE: the idea is to support e.g. .vdi and virtualbox etc. in the future
-elif ("." in args.binary):
+elif ("." in args.vm_location):
     has_bootloader = True
 
 if (args.grub):
-  print INFO, "Creating GRUB image from ", args.binary
+  print INFO, "Creating GRUB image from ", args.vm_location
   subprocess.call(INCLUDEOS_PREFIX + "/includeos/scripts/grubify.sh " + image_name, shell=True)
   image_name = image_name + ".grub.img"
   has_bootloader = True
@@ -116,6 +139,6 @@ if (args.grub):
 if (args.build): exit(0);
 
 if (not has_bootloader):
-    vm.boot(timeout = None, kernel_args=" ".join(args.vmargs), image_name = image_name)
+    vm.boot(timeout = None, multiboot = True, kernel_args = " ".join(args.vmargs), image_name = image_name)
 else:
     vm.boot(timeout = None, multiboot = False, kernel_args = None, image_name = image_name)
diff --git a/vmrunner/validate_vm.py b/vmrunner/validate_vm.py
index 5ea0f113c6..097e35a9f2 100755
--- a/vmrunner/validate_vm.py
+++ b/vmrunner/validate_vm.py
@@ -35,50 +35,53 @@ def set_defaults(validator, properties, instance, schema):
 validator = extend_with_default(Draft4Validator)
 
 package_path = os.path.dirname(os.path.realpath(__file__))
+default_schema = package_path + "/vm.schema.json"
 
-
-def load_schema(filename):
+def load_schema(filename = default_schema):
   global vm_schema
   vm_schema = json.loads(open(filename).read());
 
+
+#def load_from_json(
+
 def validate_vm_spec(filename):
+    vm_spec = None
 
-  global valid_vms
-  vm_spec = None
+    # Load and parse as JSON
+    try:
+        vm_spec = json.loads(open(filename).read())
+    except:
+        raise Exception("JSON load / parse Error for " + filename)
 
-  # Load and parse as JSON
-  try:
-    vm_spec = json.loads(open(filename).read())
-  except:
-    raise Exception("JSON load / parse Error for " + filename)
+    if (not vm_schema): load_schema()
 
-  # Validate JSON according to schema
-  try:
+    # Validate JSON according to schema
     validator(vm_schema).validate(vm_spec)
-  except Exception as err:
-    raise Exception("JSON schema validation failed: " + err.message)
 
-  valid_vms.append(vm_spec)
+    return vm_spec, filename
+
 
+def load_config(path):
+    global valid_vms
+    global jsons
 
-def has_required_stuff(path):
+    if (os.path.isfile(path)):
+        jsons = [path]
 
-  global jsons
+    if (os.path.isdir(path)):
+        jsons = glob.glob(path + "/*.json")
+        jsons.sort()
 
-  # Certain files are mandatory
-  required_files = [ "CMakeLists.txt"]
-  for file in required_files:
-      if not glob.glob(file):
-          raise Exception("missing " + file)
+    # JSON-files must conform to VM-schema
+    for json in jsons:
+        spec = validate_vm_spec(json)
+        try:
+            spec = validate_vm_spec(json)
+            valid_vms.append(spec)
+        except Exception as e:
+            pass
+    return valid_vms
 
-  # JSON-files must conform to VM-schema
-  jsons = glob.glob("*.json")
-  jsons.sort()
-  for json in jsons:
-      try:
-          validate_vm_spec(json)
-      except Exception as e:
-          pass
 
 def validate_path(path, verb = False):
   global verbose
@@ -88,7 +91,7 @@ def validate_path(path, verb = False):
     load_schema(package_path + "/vm.schema.json")
   os.chdir(path)
   try:
-    has_required_stuff(path)
+    load_config(path)
     if verbose:
       print "<validate_test> \tPASS: ",os.getcwd()
     return True
diff --git a/vmrunner/vm.schema.json b/vmrunner/vm.schema.json
index 8471b8988e..f6f6ad3dfa 100644
--- a/vmrunner/vm.schema.json
+++ b/vmrunner/vm.schema.json
@@ -4,11 +4,17 @@
   "type" : "object",
   "properties" : {
 
+    "description" : {
+      "description" : "A human-readable description of this config",
+      "type" : "string"
+    },
+
     "image" : {
       "description" : "A bootable virtual machine image",
       "type" : "string",
       "default" : "service.img"
     },
+
     "bios" : {
       "description" : "64k BIOS image",
       "type" : "string"
diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py
index b9d3182b3e..7e049034ad 100644
--- a/vmrunner/vmrunner.py
+++ b/vmrunner/vmrunner.py
@@ -15,8 +15,6 @@
 
 INCLUDEOS_HOME = None
 
-nametag = "<VMRunner>"
-
 if "INCLUDEOS_PREFIX" not in os.environ:
     def_home = "/usr/local"
     print color.WARNING("WARNING:"), "Environment varialble INCLUDEOS_PREFIX is not set. Trying default", def_home
@@ -27,6 +25,40 @@
 
 package_path = os.path.dirname(os.path.realpath(__file__))
 
+default_config = {"description" : "Single virtio nic, otherwise hypervisor defaults",
+                  "net" : [{"device" : "virtio", "backend" : "tap" }] }
+
+
+nametag = "<VMRunner>"
+INFO = color.INFO(nametag)
+VERB = bool(os.environ["VERBOSE"]) if "VERBOSE" in os.environ else False
+
+class Logger:
+    def __init__(self, tag):
+        self.tag = tag
+        if (VERB):
+            self.info = self.info_verb
+        else:
+            self.info = self.info_silent
+
+    def __call__(self, *args):
+        self.info(args)
+
+    def info_verb(self, args):
+        print self.tag,
+        for arg in args:
+            print arg,
+        print
+
+    def info_silent(self, args):
+        pass
+
+# Define verbose printing function "info", with multiple args
+default_logger = Logger(INFO)
+def info(*args):
+    default_logger.info(args)
+
+
 # The end-of-transmission character
 EOT = chr(4)
 
@@ -119,9 +151,7 @@ def __init__(self, config):
         self._sudo = False
 
         # Pretty printing
-        self._nametag = "<" + type(self).__name__ + ">"
-        self.INFO = color.INFO(self._nametag)
-
+        self.info = Logger(color.INFO("<" + type(self).__name__ + ">"))
 
     def name(self):
         return "Qemu"
@@ -150,11 +180,11 @@ def kvm_present(self):
         command = "egrep -m 1 '^flags.*(vmx|svm)' /proc/cpuinfo"
         try:
             subprocess.check_output(command, shell = True)
-            print self.INFO, "KVM ON"
+            self.info("KVM ON")
             return True
 
         except Exception as err:
-            print self.INFO, "KVM OFF"
+            self.info("KVM OFF")
             return False
 
     # Start a process and preserve in- and output pipes
@@ -172,7 +202,7 @@ def start_process(self, cmdlist):
                                       stdout = subprocess.PIPE,
                                       stderr = subprocess.PIPE,
                                       stdin = subprocess.PIPE)
-        print self.INFO, "Started process PID ",self._proc.pid
+        self.info("Started process PID ",self._proc.pid)
 
         return self._proc
 
@@ -198,11 +228,11 @@ def boot(self, multiboot, kernel_args = "", image_name = None):
 
             kernel_args = ["-kernel", image_name, "-append", kernel_args]
             disk_args = []
-            print self.INFO, "Booting", image_name, "directly without bootloader (multiboot / -kernel args)"
+            info ( "Booting", image_name, "directly without bootloader (multiboot / -kernel args)")
         else:
             kernel_args = []
             disk_args = self.drive_arg(image_name, "ide")
-            print self.INFO, "Booting", image_name, "with a bootable disk image"
+            info ("Booting", image_name, "with a bootable disk image")
 
         if "bios" in self._config:
             kernel_args.extend(["-bios", self._config["bios"]])
@@ -235,8 +265,7 @@ def boot(self, multiboot, kernel_args = "", image_name = None):
 
         command += disk_args + net_args + mem_arg + vga_arg
 
-        print self.INFO, "command:"
-        print color.DATA(" ".join(command))
+        info("Command:", " ".join(command))
 
         try:
             self.start_process(command)
@@ -258,17 +287,17 @@ def stop(self):
         if self._proc and self._proc.poll() == None :
 
             if not self._sudo:
-                print self.INFO,"Stopping child process (no sudo required)"
+                info ("Stopping child process (no sudo required)")
                 self._proc.terminate()
             else:
                 # Find and terminate all child processes, since parent is "sudo"
                 parent = psutil.Process(self._proc.pid)
                 children = parent.children()
 
-                print self.INFO, "Stopping", self._config["image"], "PID",self._proc.pid, "with", signal
+                info ("Stopping", self._config["image"], "PID",self._proc.pid, "with", signal)
 
                 for child in children:
-                    print self.INFO," + child process ", child.pid
+                    info (" + child process ", child.pid)
 
                     # The process might have gotten an exit status by now so check again to avoid negative exit
                     if (not self._proc.poll()):
@@ -312,18 +341,21 @@ def poll(self):
 # VM class
 class vm:
 
-    def __init__(self, config, hyper = qemu):
+    def __init__(self, config = None, hyper = qemu):
+
         self._exit_status = 0
         self._exit_msg = ""
-        self._config = config
+        self._config = load_single_config(config)
         self._on_success = lambda(line) : self.exit(exit_codes["SUCCESS"], nametag + " All tests passed")
         self._on_panic =  self.panic
         self._on_timeout = self.timeout
         self._on_output = {
             "PANIC" : self._on_panic,
             "SUCCESS" : self._on_success }
+
+        # Initialize hypervisor with config
         assert(issubclass(hyper, hypervisor))
-        self._hyper  = hyper(config)
+        self._hyper  = hyper(self._config)
         self._timeout_after = None
         self._timer = None
         self._on_exit_success = lambda : None
@@ -348,15 +380,15 @@ def poll(self):
     def exit(self, status, msg):
         self._exit_status = status
         self.stop()
-        print color.INFO(nametag),"Exit called with status", self._exit_status, "(",get_exit_code_name(self._exit_status),")"
-        print color.INFO(nametag),"Calling on_exit"
+        info("Exit called with status", self._exit_status, "(",get_exit_code_name(self._exit_status),")")
+        info("Calling on_exit")
         # Change back to test source
         os.chdir(self._root)
         self._on_exit()
         if status == 0:
             # Print success message and return to caller
             print color.SUCCESS(msg)
-            print color.INFO(nametag),"Calling on_exit_success"
+            info("Calling on_exit_success")
             return self._on_exit_success()
 
         # Print fail message and exit with appropriate code
@@ -365,7 +397,7 @@ def exit(self, status, msg):
 
     # Default timeout event
     def timeout(self):
-        print color.INFO("<timeout>"), "VM timed out"
+        if VERB: print color.INFO("<timeout>"), "VM timed out"
 
         # Note: we have to stop the VM since the main thread is blocking on vm.readline
         #self.exit(exit_codes["TIMEOUT"], nametag + " Test timed out")
@@ -376,7 +408,7 @@ def timeout(self):
     # Default panic event
     def panic(self, panic_line):
         panic_reason = self._hyper.readline()
-        print color.INFO(nametag), "VM signalled PANIC. Reading until EOT (", hex(ord(EOT)), ")"
+        info("VM signalled PANIC. Reading until EOT (", hex(ord(EOT)), ")")
         print color.VM(panic_reason),
         remaining_output = self._hyper.read_until_EOT()
         for line in remaining_output.split("\n"):
@@ -414,7 +446,7 @@ def writeline(self, line):
 
     # Make using GNU Make
     def make(self, params = []):
-        print color.INFO(nametag), "Building with 'make' (params=" + str(params) + ")"
+        print INFO, "Building with 'make' (params=" + str(params) + ")"
         make = ["make"]
         make.extend(params)
         cmd(make)
@@ -422,7 +454,7 @@ def make(self, params = []):
 
     # Call cmake
     def cmake(self, args = []):
-        print color.INFO(nametag), "Building with cmake (%s)" % args
+        print INFO, "Building with cmake (%s)" % args
         # install dir:
         INSTDIR = os.getcwd()
 
@@ -452,7 +484,7 @@ def cmake(self, args = []):
 
     # Clean cmake build folder
     def clean(self):
-        print color.INFO(nametag), "Cleaning cmake build folder"
+        print INFO, "Cleaning cmake build folder"
         subprocess.call(["rm","-rf","build"])
 
     # Boot the VM and start reading output. This is the main event loop.
@@ -464,7 +496,7 @@ def boot(self, timeout = 60, multiboot = True, kernel_args = "booted with vmrunn
 
         # Start the timeout thread
         if (timeout):
-            print color.INFO(nametag),"setting timeout to",timeout,"seconds"
+            info("setting timeout to",timeout,"seconds")
             self._timer = threading.Timer(timeout, self._on_timeout)
             self._timer.start()
 
@@ -532,32 +564,65 @@ def boot(self, timeout = 60, multiboot = True, kernel_args = "booted with vmrunn
         # If everything went well we can return
         return self
 
+# Load a single vm config. Fallback to default
+def load_single_config(path = "."):
+
+    config = default_config
+    description = None
+
+    # If path is explicitly "None", try current dir
+    if not path: path = "."
+
+    info("Loading config from", path)
+    try:
+        # Try loading the first valid config
+        config, path = validate_vm.load_config(path)[0]
+        info ("vm config loaded from", path, ":",)
+    except Exception as e:
+        info ("No JSON config found - using default:",)
+
+    if config.has_key("description"):
+        description = config["description"]
+    else:
+        description = str(config)
 
-print color.HEADER("IncludeOS vmrunner loading VM configs")
+    info('"',description,'"')
 
-schema_path = package_path + "/vm.schema.json"
-print color.INFO(nametag), "Validating JSON according to schema ",schema_path
-validate_vm.load_schema(schema_path)
-validate_vm.has_required_stuff(".")
+    return config
 
-default_spec = {"image" : "service.img",
-                "net" : [{"device" : "virtio", "backend" : "tap" }] }
 
 # Provide a list of VM's with validated specs
-vms = []
+# One unconfigured vm is created by default, which will try to load a config if booted
+vms = [vm()]
 
-if validate_vm.valid_vms:
-    print color.INFO(nametag), "Loaded VM specification(s) from JSON"
-    for spec in validate_vm.valid_vms:
-        print color.INFO(nametag), "Found VM spec: "
-        print color.DATA(spec.__str__())
-        vms.append(vm(spec))
+def load_configs(config_path = "."):
+    global vms
 
-else:
-    print color.WARNING(nametag), "No VM specification JSON found, trying default config"
-    vms.append(vm(default_spec))
+    # Clear out the default unconfigured vm
+    if (not vms[0]._config):
+        vms = []
+
+    print color.HEADER("IncludeOS vmrunner loading VM configs")
+
+    schema_path = package_path + "/vm.schema.json"
+
+    print INFO, "Validating JSON according to schema ",schema_path
+
+    validate_vm.load_schema(schema_path)
+    validate_vm.load_configs(config_path)
+
+    if validate_vm.valid_vms:
+        print INFO, "Loaded VM specification(s) from JSON"
+        for spec in validate_vm.valid_vms:
+            print INFO, "Found VM spec: "
+            print color.DATA(spec.__str__())
+            vms.append(vm(spec))
 
+    else:
+        print color.WARNING(nametag), "No VM specification JSON found, trying default config"
+        vms.append(vm(default_config))
 
+    return vms
 
 # Handler for SIGINT
 def handler(signum, frame):

From c8cb2b4ce19bea63fe65a19d73ea1f077c3d37bf Mon Sep 17 00:00:00 2001
From: Alfred Bratterud <alfred.bratterud@hioa.no>
Date: Wed, 25 Jan 2017 23:29:25 +0100
Subject: [PATCH 08/27] boot: added metavar to config arg

---
 etc/boot | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/etc/boot b/etc/boot
index 19486f045f..1cfd322b97 100755
--- a/etc/boot
+++ b/etc/boot
@@ -43,8 +43,8 @@ parser.add_argument("--create-bridge", dest="bridge", action="store_true",
 parser.add_argument("-g", "--grub", dest="grub", action="store_true",
                     help="Create image with GRUB bootloader that will boot provided binary")
 
-parser.add_argument("-j", "--config", dest="config", type = str,
-                    help="Location of VM JSON config")
+parser.add_argument("-j", "--config", dest="config", type = str, metavar = "PATH",
+                    help="Location of VM config file - JSON validated against a schema")
 
 parser.add_argument('vm_location', action="store", type = str,
                     help="Location of the IncludeOS service binary, image or source")

From 93c2161f732724b6090d197d290256ab510e27db Mon Sep 17 00:00:00 2001
From: fwsgonzo <fwsgonzo@hotmail.com>
Date: Wed, 25 Jan 2017 23:29:36 +0100
Subject: [PATCH 09/27] posix: Add dlfcn stubs

---
 src/CMakeLists.txt  |  3 +--
 src/posix/dlfcn.cpp | 40 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+), 2 deletions(-)
 create mode 100644 src/posix/dlfcn.cpp

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index f72abb44fa..0ce7f05af4 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -53,8 +53,7 @@ set(OS_OBJECTS
   posix/fd.cpp posix/tcp_fd.cpp posix/udp_fd.cpp posix/unistd.cpp posix/fcntl.cpp posix/syslog.cpp
   posix/sys/socket.cpp posix/sys/select.cpp posix/sys/utsname.cpp posix/sys/mman.cpp posix/arpa/inet.cpp
   posix/ucontext.cpp posix/ucontext_asm.asm
-  posix/sys/stat.cpp posix/ftw.cpp
-  posix/file_fd.cpp
+  posix/sys/stat.cpp posix/ftw.cpp posix/file_fd.cpp posix/dlfcn.cpp
 )
 
 add_library(os STATIC ${OS_OBJECTS} apic_boot.o)
diff --git a/src/posix/dlfcn.cpp b/src/posix/dlfcn.cpp
new file mode 100644
index 0000000000..cb5f1a7b61
--- /dev/null
+++ b/src/posix/dlfcn.cpp
@@ -0,0 +1,40 @@
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <dlfcn.h>
+#include <cstdio>
+
+void* dlopen(const char *filename, int flag)
+{
+  printf("dlopen called for %s with flag %#x\n", filename, flag);
+  return nullptr;
+}
+char* dlerror(void)
+{
+  printf("dlerror\n");
+  return nullptr;
+}
+void* dlsym(void*, const char* symbol)
+{
+  printf("dlsym on %s\n", symbol);
+  return nullptr;
+}
+int   dlclose(void*)
+{
+  printf("dlclose\n");
+  return 0;
+}

From 762d4de1544ec4dbd365554a32a2d941b919f969 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20=C3=85kesson?= <andreas.jm.akesson@gmail.com>
Date: Thu, 26 Jan 2017 13:56:48 +0100
Subject: [PATCH 10/27] seed: Restructured seed and added a template for
 building libraries

---
 README.md                         |  6 ++---
 etc/library.cmake                 | 38 +++++++++++++-------------
 seed/.gitignore                   |  4 ---
 seed/debug/service.gdb            |  5 ----
 seed/library/.gitignore           |  2 ++
 seed/library/CMakeLists.txt       | 44 +++++++++++++++++++++++++++++++
 seed/{ => library}/cmake_build.sh |  0
 seed/run.sh                       |  8 ------
 seed/service/.gitignore           |  1 +
 seed/{ => service}/CMakeLists.txt |  0
 seed/service/cmake_build.sh       |  7 +++++
 seed/{ => service}/docker_run.sh  |  0
 seed/{ => service}/service.cpp    |  0
 13 files changed, 77 insertions(+), 38 deletions(-)
 delete mode 100644 seed/.gitignore
 delete mode 100644 seed/debug/service.gdb
 create mode 100644 seed/library/.gitignore
 create mode 100644 seed/library/CMakeLists.txt
 rename seed/{ => library}/cmake_build.sh (100%)
 delete mode 100755 seed/run.sh
 create mode 100644 seed/service/.gitignore
 rename seed/{ => service}/CMakeLists.txt (100%)
 create mode 100755 seed/service/cmake_build.sh
 rename seed/{ => service}/docker_run.sh (100%)
 rename seed/{ => service}/service.cpp (100%)

diff --git a/README.md b/README.md
index 9834823862..bbd1d82041 100644
--- a/README.md
+++ b/README.md
@@ -97,13 +97,13 @@ More information is [available on the wiki](https://github.com/hioa-cs/IncludeOS
 
 ### Writing your first service
 
-1. Copy the [./seed](./seed) directory to a convenient location like `~/your_service`. Then, just start implementing the `Service::start` function in the `Service` class, located in [your_service/service.cpp](./seed/service.cpp) (Very simple example provided). This function will be called once the OS is up and running.
-2. Update the [CMakeLists.txt](./seed/CMakeLists.txt) to specify the name of your project, enable any needed drivers or plugins, etc.
+1. Copy the [./seed/service](./seed/service) directory to a convenient location like `~/your_service`. Then, just start implementing the `Service::start` function in the `Service` class, located in [your_service/service.cpp](./seed/service.cpp) (Very simple example provided). This function will be called once the OS is up and running.
+2. Update the [CMakeLists.txt](./seed/service/CMakeLists.txt) to specify the name of your project, enable any needed drivers or plugins, etc.
 
 **Example:**
 
 ```
-    $ cp -r seed ~/my_service
+    $ cp -r seed/service ~/my_service
     $ cd ~/my_service
     $ emacs service.cpp
     ... add your code
diff --git a/etc/library.cmake b/etc/library.cmake
index 2452a87354..d22ea8b582 100644
--- a/etc/library.cmake
+++ b/etc/library.cmake
@@ -1,7 +1,12 @@
 #
-# 
+# CMakeList for IncludeOS library
 #
 
+# IncludeOS install location
+if (NOT DEFINED ENV{INCLUDEOS_PREFIX})
+  set(ENV{INCLUDEOS_PREFIX} /usr/local)
+endif()
+
 # test compiler
 if(CMAKE_COMPILER_IS_GNUCC)
 	# currently gcc is not supported due to problems cross-compiling a unikernel
@@ -12,14 +17,10 @@ endif(CMAKE_COMPILER_IS_GNUCC)
 set(CMAKE_ASM_NASM_OBJECT_FORMAT "elf")
 enable_language(ASM_NASM)
 
-# stackrealign is needed to guarantee 16-byte stack alignment for SSE
-# the compiler seems to be really dumb in this regard, creating a misaligned stack left and right
-set(CAPABS "-mstackrealign -msse3 -fstack-protector-strong")
-
 # Various global defines
 # * OS_TERMINATE_ON_CONTRACT_VIOLATION provides classic assert-like output from Expects / Ensures
 # * _GNU_SOURCE enables POSIX-extensions in newlib, such as strnlen. ("everything newlib has", ref. cdefs.h)
-set(CAPABS "${CAPABS} -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_GNU_SOURCE -DSERVICE=\"\\\"${BINARY}\\\"\" -DSERVICE_NAME=\"\\\"${SERVICE_NAME}\\\"\"")
+set(CAPABS "-msse3 -fstack-protector-strong -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_GNU_SOURCE")
 set(WARNS  "-Wall -Wextra") #-pedantic
 
 # configure options
@@ -32,23 +33,24 @@ if (minimal)
   set(OPTIMIZE "-Os")
 endif()
 if (debug)
-  set(CAPABS "${CAPABS} -g"
+  set(CAPABS "${CAPABS} -g")
 endif()
 
 # these kinda work with llvm
 set(CMAKE_CXX_FLAGS "-MMD -target i686-elf ${CAPABS} ${OPTIMIZE} ${WARNS} -c -m32 -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1")
 set(CMAKE_C_FLAGS "-MMD -target i686-elf ${CAPABS} ${OPTIMIZE} ${WARNS} -c -m32")
 
-
 # includes
 include_directories(${LOCAL_INCLUDES})
-include_directories(${INCLUDEOS_ROOT}/include/libcxx)
-include_directories(${INCLUDEOS_ROOT}/include/api/sys)
-include_directories(${INCLUDEOS_ROOT}/include/newlib)
-include_directories(${INCLUDEOS_ROOT}/include/api/posix)
-include_directories(${INCLUDEOS_ROOT}/include/api)
-include_directories(${INCLUDEOS_ROOT}/include/gsl)
-
-# output <- input
-add_library(library STATIC ${SOURCES})
-set_target_properties(library PROPERTIES OUTPUT_NAME ${LIBRARY_NAME})
+include_directories($ENV{INCLUDEOS_PREFIX}/includeos/include/libcxx)
+include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api/sys)
+include_directories($ENV{INCLUDEOS_PREFIX}/includeos/include/newlib)
+include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api/posix)
+include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api)
+include_directories($ENV{INCLUDEOS_PREFIX}/includeos/include)
+include_directories($ENV{INCLUDEOS_PREFIX}/include)
+
+add_library(${LIBRARY_NAME} STATIC ${SOURCES})
+
+#install(TARGETS ${LIBRARY_NAME} DESTINATION includeos/lib)
+#install(DIRECTORY ${LIBRARY_HEADERS} DESTINATION includeos/include)
diff --git a/seed/.gitignore b/seed/.gitignore
deleted file mode 100644
index c6079b4532..0000000000
--- a/seed/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-*.o
-*.d
-*.img
-_elf_symbols.bin
diff --git a/seed/debug/service.gdb b/seed/debug/service.gdb
deleted file mode 100644
index f5acb4ff3c..0000000000
--- a/seed/debug/service.gdb
+++ /dev/null
@@ -1,5 +0,0 @@
-file service
-break _start
-break OS::start
-set non-stop off
-target remote localhost:1234
\ No newline at end of file
diff --git a/seed/library/.gitignore b/seed/library/.gitignore
new file mode 100644
index 0000000000..d2e6ee39e8
--- /dev/null
+++ b/seed/library/.gitignore
@@ -0,0 +1,2 @@
+build/*
+lib/
diff --git a/seed/library/CMakeLists.txt b/seed/library/CMakeLists.txt
new file mode 100644
index 0000000000..4cd6ffde4f
--- /dev/null
+++ b/seed/library/CMakeLists.txt
@@ -0,0 +1,44 @@
+cmake_minimum_required(VERSION 2.8.9)
+
+# IncludeOS install location
+if (NOT DEFINED ENV{INCLUDEOS_PREFIX})
+  set(ENV{INCLUDEOS_PREFIX} /usr/local)
+endif()
+
+# Use toolchain (if needed)
+set(CMAKE_TOOLCHAIN_FILE $ENV{INCLUDEOS_PREFIX}/includeos/i686-elf-toolchain.cmake)
+
+# Name of your project
+project (libseed)
+
+# Name of your IncludeOS library
+set(LIBRARY_NAME "seed") # => libseed.a
+
+# Source files to be built into your IncludeOS library
+set(SOURCES
+  # seed.cpp # ...add more here
+  )
+
+# Necessary includes to build your library
+set(LOCAL_INCLUDES
+  # "include"
+  )
+
+# include library build script
+include($ENV{INCLUDEOS_PREFIX}/includeos/library.cmake)
+
+
+# INSTALLATION (OPTIONAL):
+
+# If CMAKE_INSTALL_PREFIX is not set, install to source directory
+if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
+  set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}") # $ENV{INCLUDEOS_PREFIX}/includeos
+endif()
+
+# Where to install library
+install(TARGETS ${LIBRARY_NAME} DESTINATION ${CMAKE_INSTALL_PREFIX}/lib)
+
+# Where to install library headers
+# NOTE: There is a difference between installing a list of files and a directory
+# set(LIBRARY_HEADERS "include/seed")
+# install(DIRECTORY ${LIBRARY_HEADERS} DESTINATION ${CMAKE_INSTALL_PREFIX}/include)
diff --git a/seed/cmake_build.sh b/seed/library/cmake_build.sh
similarity index 100%
rename from seed/cmake_build.sh
rename to seed/library/cmake_build.sh
diff --git a/seed/run.sh b/seed/run.sh
deleted file mode 100755
index 8b5ad911d8..0000000000
--- a/seed/run.sh
+++ /dev/null
@@ -1,8 +0,0 @@
-#! /bin/bash
-set -
-INCLUDEOS_PREFIX=${INCLUDEOS_PREFIX-/usr/local}
-sudo $INCLUDEOS_PREFIX/includeos/scripts/create_bridge.sh
-
-FILE=$1
-shift
-source $INCLUDEOS_PREFIX/includeos/scripts/run.sh $FILE ${*}
diff --git a/seed/service/.gitignore b/seed/service/.gitignore
new file mode 100644
index 0000000000..a007feab07
--- /dev/null
+++ b/seed/service/.gitignore
@@ -0,0 +1 @@
+build/*
diff --git a/seed/CMakeLists.txt b/seed/service/CMakeLists.txt
similarity index 100%
rename from seed/CMakeLists.txt
rename to seed/service/CMakeLists.txt
diff --git a/seed/service/cmake_build.sh b/seed/service/cmake_build.sh
new file mode 100755
index 0000000000..9fdd05a8a0
--- /dev/null
+++ b/seed/service/cmake_build.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+INSTALL=`pwd`
+mkdir -p build
+pushd build
+cmake .. -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL
+make install
+popd
diff --git a/seed/docker_run.sh b/seed/service/docker_run.sh
similarity index 100%
rename from seed/docker_run.sh
rename to seed/service/docker_run.sh
diff --git a/seed/service.cpp b/seed/service/service.cpp
similarity index 100%
rename from seed/service.cpp
rename to seed/service/service.cpp

From 5ce11186ad2e903c4477353c21e0d8617775edce Mon Sep 17 00:00:00 2001
From: Ingve Vormestrand <ingvev@gmail.com>
Date: Thu, 26 Jan 2017 13:58:15 +0100
Subject: [PATCH 11/27] posix: add uio header, make utsname.h includable from c
 files

---
 api/posix/sys/uio.h     | 38 ++++++++++++++++++++++++++++++++++++++
 api/posix/sys/utsname.h |  4 ++--
 2 files changed, 40 insertions(+), 2 deletions(-)
 create mode 100644 api/posix/sys/uio.h

diff --git a/api/posix/sys/uio.h b/api/posix/sys/uio.h
new file mode 100644
index 0000000000..7dd8e4948d
--- /dev/null
+++ b/api/posix/sys/uio.h
@@ -0,0 +1,38 @@
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#ifndef POSIX_SYS_UIO_H
+#define POSIX_SYS_UIO_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct iovec {
+  void   *iov_base; // Base address of a memory region for input or output
+  size_t  iov_len; // Size of memory pointed to by iov_base
+};
+
+ssize_t readv(int, const struct iovec *, int);
+ssize_t writev(int, const struct iovec *, int);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // POSIX_SYS_UIO_H
diff --git a/api/posix/sys/utsname.h b/api/posix/sys/utsname.h
index 6d5a9b67c4..0d64e5a51b 100644
--- a/api/posix/sys/utsname.h
+++ b/api/posix/sys/utsname.h
@@ -23,9 +23,9 @@
 extern "C" {
 #endif
 
-struct utsname {
-  static const int LENGTH = 256;
+static const int LENGTH = 256;
 
+struct utsname {
   /* Name of this implementation of the operating system */
   char sysname[LENGTH];
 

From 3739659e11331d71d7fa941907a22b0330b4ff61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20=C3=85kesson?= <andreas.jm.akesson@gmail.com>
Date: Thu, 26 Jan 2017 14:22:57 +0100
Subject: [PATCH 12/27] readme: A link

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index bbd1d82041..336d35b60c 100644
--- a/README.md
+++ b/README.md
@@ -97,7 +97,7 @@ More information is [available on the wiki](https://github.com/hioa-cs/IncludeOS
 
 ### Writing your first service
 
-1. Copy the [./seed/service](./seed/service) directory to a convenient location like `~/your_service`. Then, just start implementing the `Service::start` function in the `Service` class, located in [your_service/service.cpp](./seed/service.cpp) (Very simple example provided). This function will be called once the OS is up and running.
+1. Copy the [./seed/service](./seed/service) directory to a convenient location like `~/your_service`. Then, just start implementing the `Service::start` function in the `Service` class, located in [your_service/service.cpp](./seed/service/service.cpp) (Very simple example provided). This function will be called once the OS is up and running.
 2. Update the [CMakeLists.txt](./seed/service/CMakeLists.txt) to specify the name of your project, enable any needed drivers or plugins, etc.
 
 **Example:**

From 8e4194250699fd3a345645bc549f6a4bd910b97f Mon Sep 17 00:00:00 2001
From: Alfred Bratterud <alfred.bratterud@hioa.no>
Date: Thu, 26 Jan 2017 17:05:03 +0100
Subject: [PATCH 13/27] test: separated validation of tests from vm config
 validation

---
 test/testrunner.py      | 16 ++++++------
 test/validate_all.py    | 30 ----------------------
 test/validate_tests.py  | 56 +++++++++++++++++++++++++++++++++++++++++
 vmrunner/validate_vm.py | 26 +++----------------
 4 files changed, 67 insertions(+), 61 deletions(-)
 delete mode 100755 test/validate_all.py
 create mode 100755 test/validate_tests.py

diff --git a/test/testrunner.py b/test/testrunner.py
index 663070a39c..4bcc383832 100755
--- a/test/testrunner.py
+++ b/test/testrunner.py
@@ -14,7 +14,7 @@
 
 from vmrunner.prettify import color as pretty
 from vmrunner import validate_vm
-import validate_all
+import validate_tests
 
 startdir = os.getcwd()
 
@@ -54,10 +54,7 @@ def print_skipped(tests):
     for test in tests:
         if test.skip_:
             print pretty.WARNING("* Skipping " + test.name_)
-            if "validate_vm" in test.skip_reason_:
-                validate_vm.validate_path(test.path_, verb = True)
-            else:
-                print "  Reason: {0:40}".format(test.skip_reason_)
+            print "Reason: {0:40}".format(test.skip_reason_)
 
 
 class Test:
@@ -179,9 +176,10 @@ def check_valid(self):
         self: Class function
         """
         # Test 1
-        if not validate_vm.validate_path(self.path_, verb = False):
+        valid, err = validate_tests.validate_test(self.path_, verb = False)
+        if not valid:
             self.skip_ = True
-            self.skip_reason_ = 'Failed validate_vm, missing files'
+            self.skip_reason_ = err
             return
 
         # Test 2
@@ -212,7 +210,7 @@ def stress_test():
         print pretty.WARNING("Stress test skipped")
         return 0
 
-    if (not validate_vm.validate_path("stress")):
+    if (not validate_tests.validate_test("stress")):
         raise Exception("Stress test failed validation")
 
     print pretty.HEADER("Starting stress test")
@@ -383,7 +381,7 @@ def filter_tests(all_tests, arguments):
                         or x.name_ in add_args ]
 
     # 2) Remove tests defined by the skip argument
-    print pretty.INFO("Tests to skip"), ", ".join(skip_args)
+    print pretty.INFO("Tests marked skip on command line"), ", ".join(skip_args)
     skipped_tests = [ x for x in tests_added
                   if x.type_ in skip_args
                   or x.category_ in skip_args
diff --git a/test/validate_all.py b/test/validate_all.py
deleted file mode 100755
index cb1cf1168e..0000000000
--- a/test/validate_all.py
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env python
-import sys
-sys.path.insert(0, ".")
-sys.path.insert(0, "..")
-
-import subprocess
-import os
-from vmrunner import validate_vm
-from vmrunner.prettify import color
-
-def valid_tests(verb = False):
-  tests = []
-
-  dirs = os.walk('.').next()[1]
-  for directory in  dirs:
-    subdirs = os.walk(directory).next()[1]
-    if "integration" in subdirs:
-      subdirs = os.walk(directory + "/integration").next()[1]
-      if subdirs:
-        for d in subdirs:
-          path = directory + "/integration/" + d
-          if validate_test.validate_path(path, verb):
-            tests.append(path)
-          else:
-            print color.WARNING("Validator: " + path + " failed validation")
-
-  return tests
-
-if __name__ == "__main__":
-  valid_tests(verb = True)
diff --git a/test/validate_tests.py b/test/validate_tests.py
new file mode 100755
index 0000000000..c6f4248968
--- /dev/null
+++ b/test/validate_tests.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+import sys
+sys.path.insert(0, ".")
+sys.path.insert(0, "..")
+
+import subprocess
+import os
+from vmrunner import validate_vm
+from vmrunner.prettify import color
+
+# Verify if a given path is a valid integration test
+def validate_test(path, verb = False):
+
+    try:
+        # Load any valid configs in path
+        valid_configs = validate_vm.load_config(path)
+        if not valid_configs:
+            raise Exception("No valid JSON config in path")
+
+        # Additional requirements for tests
+        required_files = ["CMakeLists.txt", "test.py"]
+
+        for f in required_files:
+            if not os.path.isfile(path + "/" + f):
+                raise Exception("Required file " + f + " missing")
+
+        if verb:
+            print "<validate_test> \tPASS: ",path
+        return True, "PASS"
+
+    except Exception as err:
+        if verb:
+            print "<validate_test> \tFAIL: ", path, ": " , err.message
+        
+        return False, err.message
+
+
+def valid_tests(verb = False):
+  tests = []
+
+  dirs = os.walk('.').next()[1]
+  for directory in  dirs:
+    subdirs = os.walk(directory).next()[1]
+    if "integration" in subdirs:
+      subdirs = os.walk(directory + "/integration").next()[1]
+      if subdirs:
+        for d in subdirs:
+          path = directory + "/integration/" + d
+          passed, err = validate_test(path, verb)
+          if passed:
+            tests.append(path)
+
+  return tests
+
+if __name__ == "__main__":
+  valid_tests(verb = True)
diff --git a/vmrunner/validate_vm.py b/vmrunner/validate_vm.py
index 097e35a9f2..96616bd043 100755
--- a/vmrunner/validate_vm.py
+++ b/vmrunner/validate_vm.py
@@ -42,8 +42,6 @@ def load_schema(filename = default_schema):
   vm_schema = json.loads(open(filename).read());
 
 
-#def load_from_json(
-
 def validate_vm_spec(filename):
     vm_spec = None
 
@@ -83,24 +81,8 @@ def load_config(path):
     return valid_vms
 
 
-def validate_path(path, verb = False):
-  global verbose
-  verbose = verb
-  current_dir = os.getcwd()
-  if not vm_schema:
-    load_schema(package_path + "/vm.schema.json")
-  os.chdir(path)
-  try:
-    load_config(path)
-    if verbose:
-      print "<validate_test> \tPASS: ",os.getcwd()
-    return True
-  except Exception as err:
-    if verbose:
-      print "<validate_test> \tFAIL: unmet requirements in " + path, ": " , err.message
-  finally:
-    os.chdir(current_dir)
-
 if __name__ == "__main__":
-  path = sys.argv[1] if len(sys.argv) > 1 else "."
-  validate_path(path)
+    path = sys.argv[1] if len(sys.argv) > 1 else "."
+    if not load_config(path):
+        print "No valid config found"
+        exit(-1)

From 06c39ad18666e79984a49f99315fc502ed2f532b Mon Sep 17 00:00:00 2001
From: Alfred Bratterud <alfred.bratterud@hioa.no>
Date: Thu, 26 Jan 2017 17:06:20 +0100
Subject: [PATCH 14/27] test: http test now runs server in separate thread

---
 test/net/integration/http/service.cpp | 3 +++
 test/net/integration/http/test.py     | 9 +++++----
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/test/net/integration/http/service.cpp b/test/net/integration/http/service.cpp
index 75f2d12db0..7c6c9371fe 100644
--- a/test/net/integration/http/service.cpp
+++ b/test/net/integration/http/service.cpp
@@ -48,6 +48,9 @@ void Service::ready()
   client_->send(std::move(req), {inet.gateway(), 9011},
   [] (auto err, auto res)
   {
+    if (err)
+      printf("Error: %s \n", err.to_string().c_str());
+
     CHECKSERT(!err, "No error");
     CHECKSERT(res->body() == "/testing", "Received body: \"/testing\"");
     printf("Response:\n%s\n", res->to_string().c_str());
diff --git a/test/net/integration/http/test.py b/test/net/integration/http/test.py
index f83b095155..9cf35ac4f6 100755
--- a/test/net/integration/http/test.py
+++ b/test/net/integration/http/test.py
@@ -2,6 +2,7 @@
 
 import sys
 import os
+import thread
 
 includeos_src = os.environ.get('INCLUDEOS_SRC',
                                os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0])
@@ -23,7 +24,7 @@ def do_GET(s):
         s.wfile.write("%s" % s.path)
 
 
-def Client_test(trigger_line):
+def Client_test():
     server_class = BaseHTTPServer.HTTPServer
     httpd = server_class((HOST, PORT), RequestHandler)
     global DO_SERVE
@@ -32,11 +33,11 @@ def Client_test(trigger_line):
         DO_SERVE = False
     httpd.server_close()
 
+# Start web server in a separate thread
+thread.start_new_thread(Client_test, ())
+
 # Get an auto-created VM from the vmrunner
 vm = vmrunner.vms[0]
 
-# Add custom event-handler
-vm.on_output("Testing against local server", Client_test)
-
 # Boot the VM, taking a timeout as parameter
 vm.cmake().boot(20).clean()

From ad1c61f74d3474bc048c8419f89a6457f9d3a606 Mon Sep 17 00:00:00 2001
From: Alfred Bratterud <alfred.bratterud@hioa.no>
Date: Thu, 26 Jan 2017 17:07:02 +0100
Subject: [PATCH 15/27] prettify: removed extra newline for warnings

---
 vmrunner/prettify.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/vmrunner/prettify.py b/vmrunner/prettify.py
index 5d34a61eb6..b53bf5333a 100644
--- a/vmrunner/prettify.py
+++ b/vmrunner/prettify.py
@@ -47,7 +47,7 @@ def code(fg = WHITE, bg = BLACK, style = NORMAL, fg_intensity = FG_NORMAL, bg_in
 
     @staticmethod
     def WARNING(string):
-        return color.C_WARNING + "[ WARNING ] " + string.rstrip() + color.C_ENDC + "\n"
+        return color.C_WARNING + "[ WARNING ] " + string.rstrip() + color.C_ENDC
 
     @staticmethod
     def FAIL(string):

From 46f8ac72e5f52d4d498007e2db15109694cc23b5 Mon Sep 17 00:00:00 2001
From: Annika Hammervoll <annika_limfe@hotmail.com>
Date: Fri, 27 Jan 2017 13:46:44 +0100
Subject: [PATCH 16/27] Posix socket: RW -> WR

---
 api/posix/sys/socket.h | 4 ++--
 src/posix/tcp_fd.cpp   | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/api/posix/sys/socket.h b/api/posix/sys/socket.h
index 56c4afea3a..9023b45ba5 100644
--- a/api/posix/sys/socket.h
+++ b/api/posix/sys/socket.h
@@ -137,8 +137,8 @@ int     socketpair(int domain, int type, int protocol,
 enum
 {
   SHUT_RD = 0,
-  SHUT_RW,
-  SHUT_RDRW
+  SHUT_WR,
+  SHUT_RDWR
 };
 
 #define INVALID_SOCKET          ((SOCKET)(~0))
diff --git a/src/posix/tcp_fd.cpp b/src/posix/tcp_fd.cpp
index 656177581d..6254fe5923 100644
--- a/src/posix/tcp_fd.cpp
+++ b/src/posix/tcp_fd.cpp
@@ -249,7 +249,7 @@ ssize_t TCP_FD_Conn::send(const void* data, size_t len, int)
     len,
     [&written] (bool) { written = true; }
   );
-  
+
   // sometimes we can just write and forget
   if (written) return len;
   while (!written) {
@@ -305,14 +305,14 @@ int TCP_FD_Conn::shutdown(int mode)
     return -1;
   }
   switch (mode) {
-  case SHUT_RDRW:
+  case SHUT_RDWR:
     conn->close();
     return 0;
   case SHUT_RD:
     printf("Ignoring shutdown(SHUT_RD)\n");
     return 0;
-  case SHUT_RW:
-    printf("Ignoring shutdown(SHUT_RW)\n");
+  case SHUT_WR:
+    printf("Ignoring shutdown(SHUT_WR)\n");
     return 0;
   default:
     errno = EINVAL;

From 8da5ef507a189459d8bed14d739467179913344a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20=C3=85kesson?= <andreas.jm.akesson@gmail.com>
Date: Fri, 27 Jan 2017 14:37:45 +0100
Subject: [PATCH 17/27] http: Simplified Connection to be more general, client
 now uses specialized connection

---
 api/net/http/client.hpp                       | 31 ++++----
 api/net/http/client_connection.hpp            | 69 ++++++++++++++++
 api/net/http/connection.hpp                   | 79 ++++++++-----------
 src/CMakeLists.txt                            |  2 +-
 src/net/http/client.cpp                       | 10 +--
 .../{connection.cpp => client_connection.cpp} | 42 ++++------
 6 files changed, 139 insertions(+), 94 deletions(-)
 create mode 100644 api/net/http/client_connection.hpp
 rename src/net/http/{connection.cpp => client_connection.cpp} (76%)

diff --git a/api/net/http/client.hpp b/api/net/http/client.hpp
index 5f8964ae39..8580f2fcdf 100644
--- a/api/net/http/client.hpp
+++ b/api/net/http/client.hpp
@@ -1,6 +1,6 @@
 // This file is a part of the IncludeOS unikernel - www.includeos.org
 //
-// Copyright 2016 Oslo and Akershus University College of Applied Sciences
+// Copyright 2016-2017 Oslo and Akershus University College of Applied Sciences
 // and Alfred Bratterud
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,12 +19,10 @@
 #ifndef HTTP_CLIENT_HPP
 #define HTTP_CLIENT_HPP
 
-#include "common.hpp"
-#include "request.hpp"
-#include "response.hpp"
+// http
+#include "client_connection.hpp"
+
 #include <net/tcp/tcp.hpp>
-#include "connection.hpp"
-#include "error.hpp"
 #include <vector>
 #include <map>
 
@@ -32,13 +30,13 @@ namespace http {
 
   class Client {
   public:
-    using TCP               = net::TCP;
-    using Host              = net::tcp::Socket;
+    using TCP                 = net::TCP;
+    using Host                = net::tcp::Socket;
 
-    using Connection_set     = std::vector<std::unique_ptr<Connection>>;
-    using Connection_mapset  = std::map<Host, Connection_set>;
+    using Connection_set      = std::vector<std::unique_ptr<Client_connection>>;
+    using Connection_mapset   = std::map<Host, Connection_set>;
 
-    using timeout_duration  = Connection::timeout_duration;
+    using timeout_duration    = Client_connection::timeout_duration;
 
     const static timeout_duration     DEFAULT_TIMEOUT; // client.cpp, 5s
     constexpr static size_t           DEFAULT_BUFSIZE = 2048;
@@ -153,10 +151,11 @@ namespace http {
     inline void post(Host host, std::string path, Header_set hfields, const std::string& data, Response_handler cb, Options options = {});
 
   private:
-    TCP& tcp_;
-    Connection_mapset conns_;
+    friend class Client_connection;
 
-    bool keep_alive_ = false;
+    TCP&              tcp_;
+    Connection_mapset conns_;
+    bool              keep_alive_ = false;
 
     void resolve(const std::string& host, ResolveCallback);
 
@@ -172,9 +171,9 @@ namespace http {
     /** Add data and content length */
     void add_data(Request&, const std::string& data);
 
-    Connection& get_connection(const Host host);
+    Client_connection& get_connection(const Host host);
 
-    void close(Connection&);
+    void close(Client_connection&);
 
   }; // < class Client
 
diff --git a/api/net/http/client_connection.hpp b/api/net/http/client_connection.hpp
new file mode 100644
index 0000000000..28d3f4f86e
--- /dev/null
+++ b/api/net/http/client_connection.hpp
@@ -0,0 +1,69 @@
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2016-2017 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+#ifndef HTTP_CLIENT_CONNECTION_HPP
+#define HTTP_CLIENT_CONNECTION_HPP
+
+// http
+#include "common.hpp"
+#include "connection.hpp"
+#include "error.hpp"
+
+#include <util/timer.hpp>
+
+namespace http {
+
+  class Client;
+
+  class Client_connection : public Connection {
+  public:
+    using timeout_duration  = std::chrono::milliseconds;
+
+  public:
+    explicit Client_connection(Client&, TCP_conn);
+
+    bool available() const
+    { return on_response_ == nullptr && keep_alive_; }
+
+    bool occupied() const
+    { return !available(); }
+
+    void send(Request_ptr, Response_handler, const size_t bufsize, timeout_duration = timeout_duration::zero());
+
+  private:
+    Client&           client_;
+    Response_handler  on_response_;
+    Timer             timer_;
+    timeout_duration  timeout_dur_;
+
+    void send_request(const size_t bufsize);
+
+    void recv_response(buffer_t buf, size_t len);
+
+    void end_response(Error err = Error::NONE);
+
+    void timeout_request()
+    { end_response(Error::TIMEOUT); }
+
+    void close();
+
+  }; // < class Client_connection
+
+} // < namespace http
+
+#endif // < HTTP_CLIENT_CONNECTION_HPP
diff --git a/api/net/http/connection.hpp b/api/net/http/connection.hpp
index ae6656d955..49139c0047 100644
--- a/api/net/http/connection.hpp
+++ b/api/net/http/connection.hpp
@@ -1,6 +1,6 @@
 // This file is a part of the IncludeOS unikernel - www.includeos.org
 //
-// Copyright 2016 Oslo and Akershus University College of Applied Sciences
+// Copyright 2016-2017 Oslo and Akershus University College of Applied Sciences
 // and Alfred Bratterud
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -19,74 +19,59 @@
 #ifndef HTTP_CONNECTION_HPP
 #define HTTP_CONNECTION_HPP
 
-#include "common.hpp"
+// http
 #include "request.hpp"
 #include "response.hpp"
-#include "error.hpp"
+
 #include <net/tcp/connection.hpp>
-#include <map>
-#include <vector>
-#include <delegate>
-#include <util/timer.hpp>
 
 namespace http {
 
   class Connection {
   public:
-    using TCP_conn_ptr      = net::tcp::Connection_ptr;
-    using Peer              = net::tcp::Socket;
-    using buffer_t          = net::tcp::buffer_t;
-    using Close_handler     = delegate<void(Connection&)>;
-    using timeout_duration  = std::chrono::milliseconds;
+    using TCP_conn      = net::tcp::Connection_ptr;
+    using Peer          = net::tcp::Socket;
+    using buffer_t      = net::tcp::buffer_t;
 
   public:
-
-    explicit Connection(TCP_conn_ptr, Close_handler);
+    explicit Connection(TCP_conn tcpconn, bool keep_alive = true);
 
     template <typename TCP>
-    explicit Connection(TCP&, Peer, Close_handler);
-
-    bool available() const
-    { return on_response_ == nullptr && keep_alive_; }
-
-    bool occupied() const
-    { return !available(); }
+    explicit Connection(TCP&, Peer);
 
-    void send(Request_ptr, Response_handler, const size_t bufsize, timeout_duration = timeout_duration::zero());
-
-    net::tcp::port_t local_port() const
+    net::tcp::port_t local_port() const noexcept
     { return (tcpconn_) ? tcpconn_->local_port() : 0; }
 
-    Peer peer() const
+    Peer peer() const noexcept
     { return (tcpconn_) ? tcpconn_->remote() : Peer(); }
-    //bool operator==(const Connection& other)
-    //{ return this == &other; }
-    //{ return tcpconn_->local_port() == other.tcpconn_->local_port(); }
 
-  private:
-    TCP_conn_ptr      tcpconn_;
+    void timeout()
+    { tcpconn_->is_closing() ? tcpconn_->abort() : tcpconn_->close(); }
+
+  protected:
+    TCP_conn          tcpconn_;
     Request_ptr       req_;
     Response_ptr      res_;
-    Close_handler     on_close_;
-    Response_handler  on_response_;
-    Timer             timer_;
-
-    timeout_duration  timeout_dur_;
-    bool keep_alive_;
-
-    void send_request(const size_t bufsize);
-
-    void recv_response(buffer_t buf, size_t len);
-
-    void end_response(Error err = Error::NONE);
-
-    void timeout_request()
-    { end_response(Error::TIMEOUT); }
-
-    void close();
+    bool              keep_alive_;
 
   }; // < class Connection
 
+  Connection::Connection(TCP_conn tcpconn, bool keep_alive)
+    : tcpconn_{std::move(tcpconn)},
+      req_{nullptr},
+      res_{nullptr},
+      keep_alive_{keep_alive}
+  {
+    Ensures(tcpconn_ != nullptr);
+    debug("<http::Connection> Created %u -> %s %p\n", local_port(), peer().to_string().c_str(), this);
+  }
+
+  template <typename TCP>
+  Connection::Connection(TCP& tcp, Peer addr)
+    : Connection(tcp.connect(addr))
+  {
+  }
+
 } // < namespace http
 
 #endif // < HTTP_CONNECTION_HPP
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 0ce7f05af4..5366e9c7e4 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -47,7 +47,7 @@ set(OS_OBJECTS
   net/super_stack.cpp
   net/http/header.cpp net/http/header_fields.cpp net/http/message.cpp net/http/request.cpp
   net/http/response.cpp net/http/status_codes.cpp net/http/time.cpp net/http/version.cpp
-  net/http/mime_types.cpp net/http/client.cpp net/http/connection.cpp net/http/cookie.cpp
+  net/http/mime_types.cpp net/http/client.cpp net/http/client_connection.cpp net/http/cookie.cpp
   fs/disk.cpp fs/filesystem.cpp fs/mbr.cpp fs/path.cpp
   fs/fat.cpp fs/fat_async.cpp fs/fat_sync.cpp fs/memdisk.cpp
   posix/fd.cpp posix/tcp_fd.cpp posix/udp_fd.cpp posix/unistd.cpp posix/fcntl.cpp posix/syslog.cpp
diff --git a/src/net/http/client.cpp b/src/net/http/client.cpp
index 785a4e3f6d..90b93a008e 100644
--- a/src/net/http/client.cpp
+++ b/src/net/http/client.cpp
@@ -1,6 +1,6 @@
 // This file is a part of the IncludeOS unikernel - www.includeos.org
 //
-// Copyright 2016 Oslo and Akershus University College of Applied Sciences
+// Copyright 2016-2017 Oslo and Akershus University College of Applied Sciences
 // and Alfred Bratterud
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -197,7 +197,7 @@ namespace http {
     stack.resolve(host, cb);
   }
 
-  Connection& Client::get_connection(const Host host)
+  Client_connection& Client::get_connection(const Host host)
   {
     // return/create a set for the given host
     auto& cset = conns_[host];
@@ -210,17 +210,17 @@ namespace http {
     }
 
     // no non-occupied connections, emplace a new one
-    cset.push_back(std::make_unique<Connection>(tcp_.connect(host), Connection::Close_handler{this, &Client::close}));
+    cset.push_back(std::make_unique<Client_connection>(*this, tcp_.connect(host)));
     return *cset.back();
   }
 
-  void Client::close(Connection& c)
+  void Client::close(Client_connection& c)
   {
     debug("<http::Client> Closing %u:%s %p\n", c.local_port(), c.peer().to_string().c_str(), &c);
     auto& cset = conns_.at(c.peer());
 
     cset.erase(std::remove_if(cset.begin(), cset.end(),
-    [port = c.local_port()] (const std::unique_ptr<Connection>& conn)->bool
+    [port = c.local_port()] (const std::unique_ptr<Client_connection>& conn)->bool
     {
       return conn->local_port() == port;
     }));
diff --git a/src/net/http/connection.cpp b/src/net/http/client_connection.cpp
similarity index 76%
rename from src/net/http/connection.cpp
rename to src/net/http/client_connection.cpp
index 61efc9bcdb..de6bb172e4 100644
--- a/src/net/http/connection.cpp
+++ b/src/net/http/client_connection.cpp
@@ -1,6 +1,6 @@
 // This file is a part of the IncludeOS unikernel - www.includeos.org
 //
-// Copyright 2016 Oslo and Akershus University College of Applied Sciences
+// Copyright 2017 Oslo and Akershus University College of Applied Sciences
 // and Alfred Bratterud
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,33 +15,25 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#include <net/http/connection.hpp>
-#include <gsl/gsl_assert>
+#include <net/http/client_connection.hpp>
+#include <net/http/client.hpp>
+
 #include <debug>
 
 namespace http {
 
-  Connection::Connection(TCP_conn_ptr tcpconn, Close_handler on_close)
-    : tcpconn_{std::move(tcpconn)},
-      req_{nullptr},
-      res_{nullptr},
-      on_close_{std::move(on_close)},
+  Client_connection::Client_connection(Client& client, TCP_conn tcpconn)
+    : Connection{std::move(tcpconn)},
+      client_(client),
       on_response_{nullptr},
-      timer_({this, &Connection::timeout_request}),
-      timeout_dur_{timeout_duration::zero()},
-      keep_alive_{true}
+      timer_({this, &Client_connection::timeout_request}),
+      timeout_dur_{timeout_duration::zero()}
   {
-    debug("<http::Connection> Created %u -> %s %p\n", local_port(), peer().to_string().c_str(), this);
     // setup close event
-    tcpconn_->on_close({this, &Connection::close});
-  }
-  template <typename TCP>
-  Connection::Connection(TCP& tcp, Peer addr, Close_handler on_close)
-    : Connection(tcp.connect(addr), std::move(on_close))
-  {
+    tcpconn_->on_close({this, &Client_connection::close});
   }
 
-  void Connection::send(Request_ptr req, Response_handler on_res, const size_t bufsize, timeout_duration timeout)
+  void Client_connection::send(Request_ptr req, Response_handler on_res, const size_t bufsize, timeout_duration timeout)
   {
     Expects(available());
     req_ = std::move(req);
@@ -56,16 +48,16 @@ namespace http {
     send_request(bufsize);
   }
 
-  void Connection::send_request(const size_t bufsize)
+  void Client_connection::send_request(const size_t bufsize)
   {
     keep_alive_ = (req_->header().value(header::Connection) != "close");
 
-    tcpconn_->on_read(bufsize, {this, &Connection::recv_response});
+    tcpconn_->on_read(bufsize, {this, &Client_connection::recv_response});
 
     tcpconn_->write(req_->to_string());
   }
 
-  void Connection::recv_response(buffer_t buf, size_t len)
+  void Client_connection::recv_response(buffer_t buf, size_t len)
   {
     if(len == 0) {
       end_response({Error::NO_REPLY});
@@ -138,7 +130,7 @@ namespace http {
     }
   }
 
-  void Connection::end_response(Error err)
+  void Client_connection::end_response(Error err)
   {
     // move response to a copy in case of callback result in new request
     Ensures(on_response_);
@@ -158,7 +150,7 @@ namespace http {
       tcpconn_->close();
   }
 
-  void Connection::close()
+  void Client_connection::close()
   {
     // if the user already
     if(on_response_ != nullptr)
@@ -169,7 +161,7 @@ namespace http {
       callback(Error::CLOSING, std::move(res_));
     }
 
-    on_close_(*this);
+    client_.close(*this);
   }
 
 }

From 59cd09d06fb7cb9d6a6c451c89a8416aa0b583ca Mon Sep 17 00:00:00 2001
From: Alfred Bratterud <alfred.bratterud@hioa.no>
Date: Fri, 27 Jan 2017 14:59:28 +0100
Subject: [PATCH 18/27] boot/vmrunner: don't assume image name in config and
 don't look for SUCCESS in boot

---
 etc/boot             |  3 +++
 vmrunner/vmrunner.py | 63 ++++++++++++++++++++++++++------------------
 2 files changed, 40 insertions(+), 26 deletions(-)

diff --git a/etc/boot b/etc/boot
index 1cfd322b97..fcee55c285 100755
--- a/etc/boot
+++ b/etc/boot
@@ -111,6 +111,9 @@ for vm_ in vmrunner.vms:
     if vm_._config.has_key("image") and vm_._config["image"].startswith(image_name):
         vm = vm_
 
+# Don't listen to events needed by testrunner
+vm.on_success(lambda: None, do_exit = False)
+
 if (args.clean):
     print INFO, "Cleaning build"
     vm.clean()
diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py
index 7e049034ad..5d3099104a 100644
--- a/vmrunner/vmrunner.py
+++ b/vmrunner/vmrunner.py
@@ -109,36 +109,39 @@ def abstract():
 # (It seems to be recommended for "new style classes" to inherit object)
 class hypervisor(object):
 
-  def __init__(self, config):
-    self._config = config;
+    def __init__(self, config):
+        self._config = config;
 
-  # Boot a VM, returning a hypervisor handle for reuse
-  def boot(self):
-    abstract()
+    # Boot a VM, returning a hypervisor handle for reuse
+    def boot(self):
+        abstract()
 
-  # Stop the VM booted by boot
-  def stop(self):
-    abstract()
+    # Stop the VM booted by boot
+    def stop(self):
+        abstract()
 
-  # Read a line of output from vm
-  def readline(self):
-    abstract()
+    # Read a line of output from vm
+    def readline(self):
+        abstract()
 
-  # Verify that the hypervisor is available
-  def available(self, config_data = None):
-    abstract()
+    # Verify that the hypervisor is available
+    def available(self, config_data = None):
+        abstract()
 
-  # Wait for this VM to exit
-  def wait(self):
-    abstract()
+    # Wait for this VM to exit
+    def wait(self):
+        abstract()
 
-  # Wait for this VM to exit
-  def poll(self):
-    abstract()
+    # Wait for this VM to exit
+    def poll(self):
+        abstract()
 
-  # A descriptive name
-  def name(self):
-    abstract()
+    # A descriptive name
+    def name(self):
+        abstract()
+
+    def image_name(self):
+        abstract()
 
 
 # Qemu Hypervisor interface
@@ -149,6 +152,7 @@ def __init__(self, config):
         self._proc = None
         self._stopped = False
         self._sudo = False
+        self._image_name = self._config if "image" in self._config else self.name() + " vm"
 
         # Pretty printing
         self.info = Logger(color.INFO("<" + type(self).__name__ + ">"))
@@ -156,6 +160,9 @@ def __init__(self, config):
     def name(self):
         return "Qemu"
 
+    def image_name(serlf):
+        return self._image_name
+
     def drive_arg(self, filename, drive_type = "virtio", drive_format = "raw", media_type = "disk"):
         return ["-drive","file=" + filename
                 + ",format=" + drive_format
@@ -219,6 +226,8 @@ def boot(self, multiboot, kernel_args = "", image_name = None):
         if not image_name:
             image_name = self._config["image"]
 
+        self._image_name = image_name
+
         # multiboot - e.g. boot with '-kernel' and no bootloader
         if multiboot:
 
@@ -294,7 +303,7 @@ def stop(self):
                 parent = psutil.Process(self._proc.pid)
                 children = parent.children()
 
-                info ("Stopping", self._config["image"], "PID",self._proc.pid, "with", signal)
+                info ("Stopping", self._image_name, "PID",self._proc.pid, "with", signal)
 
                 for child in children:
                     info (" + child process ", child.pid)
@@ -421,8 +430,10 @@ def panic(self, panic_line):
     def on_output(self, output, callback):
         self._on_output[ output ] = callback
 
-    def on_success(self, callback):
-        self._on_output["SUCCESS"] = lambda(line) : [callback(line), self._on_success(line)]
+    def on_success(self, callback, do_exit = True):
+        if do_exit:
+            self._on_output["SUCCESS"] = lambda(line) : [callback(line), self._on_success(line)]
+        else: self._on_output["SUCCESS"] = callback
 
     def on_panic(self, callback):
         self._on_output["PANIC"] = lambda(line) : [callback(line), self._on_panic(line)]

From 936b776b1a5593e9f0df5996c350a6b1271d386a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20=C3=85kesson?= <andreas.jm.akesson@gmail.com>
Date: Fri, 27 Jan 2017 17:01:40 +0100
Subject: [PATCH 19/27] vmbuild: Avoid using IncludeOS sys headers

---
 vmbuild/CMakeLists.txt | 2 +-
 vmbuild/vmbuild.cpp    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/vmbuild/CMakeLists.txt b/vmbuild/CMakeLists.txt
index 554b8d2375..0c43ca8f38 100644
--- a/vmbuild/CMakeLists.txt
+++ b/vmbuild/CMakeLists.txt
@@ -9,7 +9,7 @@ set(ELF_SYMS_SOURCES elf_syms.cpp)
 set(CMAKE_CXX_FLAGS "-std=c++14 -Wall -Wextra -O3")
 
 # TODO: write scripts that automatically find include directories
-include_directories(. ./../api ./../mod/GSL/)
+include_directories(. ./../mod/GSL/)
 
 add_executable(vmbuild ${SOURCES})
 add_executable(elf_syms ${ELF_SYMS_SOURCES})
diff --git a/vmbuild/vmbuild.cpp b/vmbuild/vmbuild.cpp
index 4453c94f4f..dc21e27aa0 100644
--- a/vmbuild/vmbuild.cpp
+++ b/vmbuild/vmbuild.cpp
@@ -25,7 +25,7 @@
 #include <stdlib.h>
 #include <sys/stat.h>
 
-#include <boot/multiboot.h>
+#include "../api/boot/multiboot.h"
 #include <gsl/gsl>
 #include "elf.h"
 #include "elf_binary.hpp"

From bb1d351e9713ef82bb33f6496efe92af4b3c0998 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= <fwsgonzo@hotmail.com>
Date: Fri, 27 Jan 2017 17:04:00 +0100
Subject: [PATCH 20/27] diskbuild: Avoid using IncludeOS sys headers by copy
 FAT structs

---
 diskimagebuild/CMakeLists.txt   |  1 -
 diskimagebuild/fat_internal.hpp | 62 +++++++++++++++++++++++++++++++++
 diskimagebuild/writer.cpp       |  2 +-
 3 files changed, 63 insertions(+), 2 deletions(-)
 create mode 100644 diskimagebuild/fat_internal.hpp

diff --git a/diskimagebuild/CMakeLists.txt b/diskimagebuild/CMakeLists.txt
index b48ef7e516..45de2934d6 100644
--- a/diskimagebuild/CMakeLists.txt
+++ b/diskimagebuild/CMakeLists.txt
@@ -5,7 +5,6 @@ set (CMAKE_CXX_STANDARD 14)
 
 set(SOURCES main.cpp filetree.cpp writer.cpp)
 
-include_directories(../api)
 include_directories(../mod/GSL)
 add_executable(diskbuilder ${SOURCES})
 
diff --git a/diskimagebuild/fat_internal.hpp b/diskimagebuild/fat_internal.hpp
new file mode 100644
index 0000000000..4861a7e53d
--- /dev/null
+++ b/diskimagebuild/fat_internal.hpp
@@ -0,0 +1,62 @@
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2015 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Slim copy of <api/fs/fat_internal.hpp>
+
+#pragma once
+#ifndef FS_FAT_INTERNAL_HPP
+#define FS_FAT_INTERNAL_HPP
+
+#include <cstdint>
+
+// Attribute masks
+static const uint8_t ATTR_READ_ONLY = 0x01;
+static const uint8_t ATTR_HIDDEN    = 0x02;
+static const uint8_t ATTR_SYSTEM    = 0x04;
+static const uint8_t ATTR_VOLUME_ID = 0x08;
+static const uint8_t ATTR_DIRECTORY = 0x10;
+static const uint8_t ATTR_ARCHIVE   = 0x20;
+
+// Mask for the last longname entry
+static const uint8_t LAST_LONG_ENTRY = 0x40;
+
+struct cl_dir
+{
+  uint8_t  shortname[11];
+  uint8_t  attrib;
+  uint8_t  pad1[8];
+  uint16_t cluster_hi;
+  uint32_t modified;
+  uint16_t cluster_lo;
+  uint32_t filesize;
+
+} __attribute__((packed));
+
+struct cl_long
+{
+  uint8_t  index;
+  uint16_t first[5];
+  uint8_t  attrib;
+  uint8_t  entry_type;
+  uint8_t  checksum;
+  uint16_t second[6];
+  uint16_t zero;
+  uint16_t third[2];
+
+} __attribute__((packed));
+
+#endif
diff --git a/diskimagebuild/writer.cpp b/diskimagebuild/writer.cpp
index 2d6ace5a9a..f8d10e2b26 100644
--- a/diskimagebuild/writer.cpp
+++ b/diskimagebuild/writer.cpp
@@ -1,7 +1,7 @@
 #include "filetree.hpp"
 
 #include "../api/fs/mbr.hpp"
-#include "../api/fs/fat_internal.hpp"
+#include "fat_internal.hpp"
 #include <cassert>
 #include <cstring>
 

From 1b1bcb1149f7811dd1fb23222c082341769b8f5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20=C3=85kesson?= <andreas.jm.akesson@gmail.com>
Date: Fri, 27 Jan 2017 17:14:48 +0100
Subject: [PATCH 21/27] test: Update to reflect connection rename in 8da5ef5

---
 test/CMakeLists.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index c6fccf929d..efc564731f 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -119,7 +119,7 @@ set(OS_SOURCES
   ${SRC}/net/dns/dns.cpp
   ${SRC}/net/ethernet/ethernet.cpp
   ${SRC}/net/http/client.cpp
-  ${SRC}/net/http/connection.cpp
+  ${SRC}/net/http/client_connection.cpp
   ${SRC}/net/http/cookie.cpp
   ${SRC}/net/http/header.cpp
   ${SRC}/net/http/header_fields.cpp

From 1e930d1d211740261cefaa2b3a423fde0af59668 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20=C3=85kesson?= <andreas.jm.akesson@gmail.com>
Date: Fri, 27 Jan 2017 17:29:40 +0100
Subject: [PATCH 22/27] http: Added forgotten inline on Connection constructor

---
 api/net/http/connection.hpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/api/net/http/connection.hpp b/api/net/http/connection.hpp
index 49139c0047..6e6feb092b 100644
--- a/api/net/http/connection.hpp
+++ b/api/net/http/connection.hpp
@@ -34,7 +34,7 @@ namespace http {
     using buffer_t      = net::tcp::buffer_t;
 
   public:
-    explicit Connection(TCP_conn tcpconn, bool keep_alive = true);
+    inline explicit Connection(TCP_conn tcpconn, bool keep_alive = true);
 
     template <typename TCP>
     explicit Connection(TCP&, Peer);
@@ -56,7 +56,7 @@ namespace http {
 
   }; // < class Connection
 
-  Connection::Connection(TCP_conn tcpconn, bool keep_alive)
+  inline Connection::Connection(TCP_conn tcpconn, bool keep_alive)
     : tcpconn_{std::move(tcpconn)},
       req_{nullptr},
       res_{nullptr},

From b15912d9a04fc2f7c17fda29cabc70802bb8ec95 Mon Sep 17 00:00:00 2001
From: Ingve Vormestrand <ingvev@gmail.com>
Date: Mon, 30 Jan 2017 12:17:27 +0100
Subject: [PATCH 23/27] posix: pwd-related functions

---
 src/CMakeLists.txt                         |   1 +
 src/posix/pwd.cpp                          | 143 +++++++++++++++++++++
 test/posix/integration/conf/CMakeLists.txt |   1 +
 test/posix/integration/conf/disk/passwd    |   4 +
 test/posix/integration/conf/service.cpp    |   4 +
 test/posix/integration/conf/test_pwd.c     |  53 ++++++++
 6 files changed, 206 insertions(+)
 create mode 100644 src/posix/pwd.cpp
 create mode 100644 test/posix/integration/conf/test_pwd.c

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 0ce7f05af4..e5b2d53359 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -54,6 +54,7 @@ set(OS_OBJECTS
   posix/sys/socket.cpp posix/sys/select.cpp posix/sys/utsname.cpp posix/sys/mman.cpp posix/arpa/inet.cpp
   posix/ucontext.cpp posix/ucontext_asm.asm
   posix/sys/stat.cpp posix/ftw.cpp posix/file_fd.cpp posix/dlfcn.cpp
+  posix/pwd.cpp
 )
 
 add_library(os STATIC ${OS_OBJECTS} apic_boot.o)
diff --git a/src/posix/pwd.cpp b/src/posix/pwd.cpp
new file mode 100644
index 0000000000..d64f97f261
--- /dev/null
+++ b/src/posix/pwd.cpp
@@ -0,0 +1,143 @@
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+#include <vector>
+#include <fstream>
+#include <pwd.h>
+#include <info>
+#include <gsl/gsl_assert>
+
+const char* pwd_path = "/etc/passwd";
+
+/* Applications are not allowed to modify the structure to which the return
+   values point, nor any storage areas pointed to by pointers within the
+   structure. Returned pointers might be invalidated or the structure/
+   storage areas might be overwritten by subsequent calls. */
+static struct passwd ret;
+
+class Pwd {
+public:
+  static Pwd& instance() {
+    static Pwd pwd;
+    return pwd;
+  }
+
+  void rewind() {pos_ = 0;}
+
+  void close() {open_ = false;}
+
+  struct passwd* find(const char* name) {
+    Expects(name != nullptr);
+    if (!open_) {
+      read();
+    }
+    const auto it = std::find_if(begin(entries_), end(entries_), [name](const auto& entry){
+      return (strcmp(name, entry.c_str()) == 0);
+    });
+    if (it == end(entries_)) {
+      return nullptr;
+    }
+    extract(*it);
+    return &ret;
+  }
+
+  struct passwd* next() noexcept {
+    if (!open_) {
+      read();
+    }
+    if (pos_ >= entries_.size()) {
+      return nullptr;
+    }
+    try {
+      const auto& ent = entries_[pos_];
+      int field_seps = std::count(std::begin(ent), std::end(ent), '\0');
+      if (field_seps != 6) {
+        INFO("pwd", "not a valid passwd file entry");
+        return nullptr;
+      }
+      extract(ent);
+      ++pos_;
+      return &ret;
+    }
+    catch (...) {
+      INFO("pwd", "error parsing pwd entry");
+      return nullptr;
+    }
+  }
+
+  void extract(const std::string& ent) {
+    ret.pw_name = const_cast<char*>(ent.c_str());
+    size_t pw_pos = ent.find('\0') + 1;
+    size_t uid_pos = ent.find('\0', pw_pos) + 1;
+    size_t gid_pos = ent.find('\0', uid_pos) + 1;
+    size_t info_pos = ent.find('\0', gid_pos) + 1;
+    size_t dir_pos = ent.find('\0', info_pos) + 1;
+    size_t shell_pos = ent.find('\0', dir_pos) + 1;
+    ret.pw_uid = std::stol(ret.pw_name + uid_pos);
+    ret.pw_gid = std::stol(ret.pw_name + gid_pos);
+    ret.pw_dir = ret.pw_name + dir_pos;
+    ret.pw_shell = ret.pw_name + shell_pos;
+  }
+
+  void print() {
+    printf("  instance @%p\n", this);
+    printf("  open: %d\n", open_);
+    printf("  pos: %d\n", pos_);
+    printf("  entries: %d\n", entries_.size());
+  }
+
+private:
+  Pwd() : open_ {false}, pos_ {0} {}
+
+  void read() {
+    std::ifstream is(pwd_path);
+    std::string line;
+    while (std::getline(is, line)) {
+      std::replace(std::begin(line), std::end(line), ':', '\0');
+      entries_.push_back(line);
+    }
+    rewind();
+    open_ = true;
+  }
+  bool open_;
+  size_t pos_;
+  std::vector<std::string> entries_;
+};
+
+void endpwent(void) {
+  Pwd& pwd = Pwd::instance();
+  pwd.close();
+}
+
+void setpwent(void) {
+  Pwd& pwd = Pwd::instance();
+  pwd.rewind();
+}
+
+struct passwd *getpwent(void) {
+  Pwd& pwd = Pwd::instance();
+  return pwd.next();
+}
+
+struct passwd *getpwnam(const char *name) {
+  if (name == nullptr) {
+    return nullptr;
+  }
+  Pwd& pwd = Pwd::instance();
+  return pwd.find(name);
+}
diff --git a/test/posix/integration/conf/CMakeLists.txt b/test/posix/integration/conf/CMakeLists.txt
index aba6a2501d..6f194393e6 100644
--- a/test/posix/integration/conf/CMakeLists.txt
+++ b/test/posix/integration/conf/CMakeLists.txt
@@ -24,6 +24,7 @@ set(SOURCES
   service.cpp
   test_sysconf.c
   test_pathconf.c
+  test_pwd.c
   )
 
 # DRIVERS / PLUGINS:
diff --git a/test/posix/integration/conf/disk/passwd b/test/posix/integration/conf/disk/passwd
index 81e656055f..94dd03764a 100644
--- a/test/posix/integration/conf/disk/passwd
+++ b/test/posix/integration/conf/disk/passwd
@@ -1 +1,5 @@
+nobody:*:-2:-2:Unprivileged User:/var/empty:/usr/bin/false
 root:*:0:0:System Administrator:/var/root:/bin/sh
+daemon:*:1:1:System Services:/var/root:/usr/bin/false
+_ftp:*:98:-2:FTP Daemon:/var/empty:/usr/bin/false
+_launchservicesd:*:239:239:_launchservicesd:/var/empty:/usr/bin/false
diff --git a/test/posix/integration/conf/service.cpp b/test/posix/integration/conf/service.cpp
index 425409bff4..749f72bd42 100644
--- a/test/posix/integration/conf/service.cpp
+++ b/test/posix/integration/conf/service.cpp
@@ -27,6 +27,7 @@
 
 extern "C" void test_sysconf();
 extern "C" void test_pathconf();
+extern "C" void test_pwd();
 
 fs::Disk_ptr& memdisk() {
   static auto disk = fs::new_shared_memdisk();
@@ -74,5 +75,8 @@ int main(int argc, char *argv[]) {
   value = getenv("INCLUDEOS_CORE_DUMP");
   printf("INCLUDEOS_CORE_DUMP: %s\n", value);
 
+  // test pwd-related functions
+  test_pwd();
+
   INFO("Conf", "SUCCESS");
 }
diff --git a/test/posix/integration/conf/test_pwd.c b/test/posix/integration/conf/test_pwd.c
new file mode 100644
index 0000000000..228fb095cb
--- /dev/null
+++ b/test/posix/integration/conf/test_pwd.c
@@ -0,0 +1,53 @@
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <pwd.h>
+#include <stdio.h>
+
+void print_pwd_entry(struct passwd* pwd_ent);
+
+void test_pwd() {
+  struct passwd* pwd_ent;
+
+  while((pwd_ent = getpwent()) != NULL) {
+    print_pwd_entry(pwd_ent);
+  }
+
+  printf("Back to start!\n");
+  setpwent();
+  pwd_ent = getpwent();
+  print_pwd_entry(pwd_ent);
+  endpwent();
+
+  pwd_ent = getpwnam("root");
+  if (pwd_ent != NULL) {
+    print_pwd_entry(pwd_ent);
+  }
+
+  pwd_ent = getpwnam("alfred");
+  if (pwd_ent != NULL) {
+    print_pwd_entry(pwd_ent);
+  }
+}
+
+void print_pwd_entry(struct passwd* pwd_ent) {
+  printf("Name: %s\n", pwd_ent->pw_name);
+  printf("Uid: %d\n", pwd_ent->pw_uid);
+  printf("Gid: %d\n", pwd_ent->pw_gid);
+  printf("Path: %s\n", pwd_ent->pw_dir);
+  printf("Shell: %s\n\n", pwd_ent->pw_shell);
+}

From 9008ba295215883650dfc0ae5a02b97a2060a1ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20=C3=85kesson?= <andreas.jm.akesson@gmail.com>
Date: Mon, 30 Jan 2017 13:44:15 +0100
Subject: [PATCH 24/27] posix: Clean up header mess

---
 api/kernel/mman.hpp            |  6 ------
 api/{sys => posix}/math.h      |  0
 api/{sys => posix}/signal.h    |  4 ++--
 api/{ => posix}/sys/__config   |  0
 api/{ => posix}/sys/ctype.h    |  0
 api/{ => posix}/sys/dlfcn.h    |  0
 api/{ => posix}/sys/features.h |  0
 api/{ => posix}/sys/stddef.h   |  0
 api/{ => posix}/sys/stdlib.h   | 12 ++++++------
 api/{ => posix}/sys/time.h     |  0
 api/{ => posix}/sys/wchar.h    |  0
 api/{sys => posix}/unistd.h    |  0
 api/{sys => }/quick_exit       |  0
 src/CMakeLists.txt             |  1 -
 src/posix/sys/mman.cpp         |  5 +++--
 15 files changed, 11 insertions(+), 17 deletions(-)
 delete mode 100644 api/kernel/mman.hpp
 rename api/{sys => posix}/math.h (100%)
 rename api/{sys => posix}/signal.h (100%)
 rename api/{ => posix}/sys/__config (100%)
 rename api/{ => posix}/sys/ctype.h (100%)
 rename api/{ => posix}/sys/dlfcn.h (100%)
 rename api/{ => posix}/sys/features.h (100%)
 rename api/{ => posix}/sys/stddef.h (100%)
 rename api/{ => posix}/sys/stdlib.h (96%)
 rename api/{ => posix}/sys/time.h (100%)
 rename api/{ => posix}/sys/wchar.h (100%)
 rename api/{sys => posix}/unistd.h (100%)
 rename api/{sys => }/quick_exit (100%)

diff --git a/api/kernel/mman.hpp b/api/kernel/mman.hpp
deleted file mode 100644
index e87f3dc5a1..0000000000
--- a/api/kernel/mman.hpp
+++ /dev/null
@@ -1,6 +0,0 @@
-#ifndef KERNEL_MMAN_HPP
-#define KERNEL_MMAN_HPP
-
-#include <sys/mman.h>
-
-#endif
diff --git a/api/sys/math.h b/api/posix/math.h
similarity index 100%
rename from api/sys/math.h
rename to api/posix/math.h
diff --git a/api/sys/signal.h b/api/posix/signal.h
similarity index 100%
rename from api/sys/signal.h
rename to api/posix/signal.h
index 0753636fef..42c677b159 100644
--- a/api/sys/signal.h
+++ b/api/posix/signal.h
@@ -19,6 +19,8 @@
 #ifndef SYS_SIGNAL_H
 #define SYS_SIGNAL_H
 
+#include_next <signal.h>
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -37,6 +39,4 @@ struct siginfo_t
 }
 #endif
 
-#include_next <signal.h>
-
 #endif
diff --git a/api/sys/__config b/api/posix/sys/__config
similarity index 100%
rename from api/sys/__config
rename to api/posix/sys/__config
diff --git a/api/sys/ctype.h b/api/posix/sys/ctype.h
similarity index 100%
rename from api/sys/ctype.h
rename to api/posix/sys/ctype.h
diff --git a/api/sys/dlfcn.h b/api/posix/sys/dlfcn.h
similarity index 100%
rename from api/sys/dlfcn.h
rename to api/posix/sys/dlfcn.h
diff --git a/api/sys/features.h b/api/posix/sys/features.h
similarity index 100%
rename from api/sys/features.h
rename to api/posix/sys/features.h
diff --git a/api/sys/stddef.h b/api/posix/sys/stddef.h
similarity index 100%
rename from api/sys/stddef.h
rename to api/posix/sys/stddef.h
diff --git a/api/sys/stdlib.h b/api/posix/sys/stdlib.h
similarity index 96%
rename from api/sys/stdlib.h
rename to api/posix/sys/stdlib.h
index 9344556f62..7e1e37389b 100644
--- a/api/sys/stdlib.h
+++ b/api/posix/sys/stdlib.h
@@ -6,9 +6,9 @@
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
-// 
+//
 //     http://www.apache.org/licenses/LICENSE-2.0
-// 
+//
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -22,19 +22,19 @@
 #include_next <stdlib.h>
 
 // More C11 requirements here
-#include "quick_exit"
+#include <quick_exit>
 
 #ifdef __cplusplus
 extern "C" {
 #endif
-  
+
   //New stuff in C11, required by libunwind, compiler-rt etc. in llvm
-  
+
 void *aligned_alloc( size_t alignment, size_t size );
 
 #ifdef __cplusplus
 }
-#endif 
+#endif
 #endif //SYS_STDLIB_H
 
 
diff --git a/api/sys/time.h b/api/posix/sys/time.h
similarity index 100%
rename from api/sys/time.h
rename to api/posix/sys/time.h
diff --git a/api/sys/wchar.h b/api/posix/sys/wchar.h
similarity index 100%
rename from api/sys/wchar.h
rename to api/posix/sys/wchar.h
diff --git a/api/sys/unistd.h b/api/posix/unistd.h
similarity index 100%
rename from api/sys/unistd.h
rename to api/posix/unistd.h
diff --git a/api/sys/quick_exit b/api/quick_exit
similarity index 100%
rename from api/sys/quick_exit
rename to api/quick_exit
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 5366e9c7e4..47e0d04bca 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,7 +1,6 @@
 set(CMAKE_ASM_NASM_OBJECT_FORMAT "elf")
 enable_language(ASM_NASM)
 
-include_directories(${INCLUDEOS_ROOT}/api/sys)
 include_directories(${INCLUDEOS_ROOT}/api/posix)
 include_directories(${LIBCXX_INCLUDE_DIR})
 include_directories(${NEWLIB_INCLUDE_DIR})
diff --git a/src/posix/sys/mman.cpp b/src/posix/sys/mman.cpp
index 5c448fdf06..6293ae5021 100644
--- a/src/posix/sys/mman.cpp
+++ b/src/posix/sys/mman.cpp
@@ -1,5 +1,6 @@
 #include <sys/mman.h>
 
+#include <cstdio>
 #include <cstdint>
 #include <cstdlib>
 #include <cassert>
@@ -30,7 +31,7 @@ void* mmap(void* addr, size_t length,
     return MAP_FAILED;
   }
 
-  // TODO: 
+  // TODO:
   // validate fd is open file
 
   // associate some VA space with open file @fd
@@ -48,7 +49,7 @@ void* mmap(void* addr, size_t length,
   // TODO:
   // read entire file into entry space
   // deallocate entry space on failure
-  
+
   // create the mapping
   _mmap_entries[addr] = entry;
   return entry.addr;

From b11b042b52076d4bdb890a3a8c0805ce44fdc44e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20=C3=85kesson?= <andreas.jm.akesson@gmail.com>
Date: Mon, 30 Jan 2017 14:12:19 +0100
Subject: [PATCH 25/27] posix: Made unittests work on Mac

---
 api/posix/math.h              | 8 +++++++-
 api/posix/syslog.h            | 1 -
 api/util/syslog_facility.hpp  | 2 ++
 test/CMakeLists.txt           | 1 -
 test/posix/unit/inet_test.cpp | 6 +++---
 5 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/api/posix/math.h b/api/posix/math.h
index fca5d9b13f..50a204a0cc 100644
--- a/api/posix/math.h
+++ b/api/posix/math.h
@@ -18,6 +18,9 @@
 #ifndef SYS_MATH_H
 #define SYS_MATH_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 // Long double math stubs
 long double cosl(long double);
@@ -85,7 +88,10 @@ long double tgammal(long double);
 long double truncl(long double);
 long double nanl(const char*);
 
-
+#ifdef __cplusplus
+}
 #endif
 
 #include_next <math.h>
+
+#endif
diff --git a/api/posix/syslog.h b/api/posix/syslog.h
index bfd430260d..c15e8d8664 100644
--- a/api/posix/syslog.h
+++ b/api/posix/syslog.h
@@ -48,7 +48,6 @@ extern "C" {
 #define LOG_MAIL			2 	/* Mail system */
 #define LOG_DAEMON		3 	/* System daemons */
 #define LOG_AUTH			4 	/* Security/authorization messages */
-#define LOG_INTERNAL	5		/* Messages generated internally by syslogd */
 #define LOG_LPR				6 	/* Line printer subsystem */
 #define LOG_NEWS			7 	/* Network news subsystem */
 #define LOG_UUCP			8 	/* UUCP subsystem */
diff --git a/api/util/syslog_facility.hpp b/api/util/syslog_facility.hpp
index 8597c17de1..700d869327 100644
--- a/api/util/syslog_facility.hpp
+++ b/api/util/syslog_facility.hpp
@@ -21,6 +21,8 @@
 #ifndef UTIL_SYSLOG_FACILITY_HPP
 #define UTIL_SYSLOG_FACILITY_HPP
 
+#define LOG_INTERNAL  5   /* Messages generated internally by syslogd */
+
 #include <cstdio>
 #include <iostream>
 #include <string>
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index efc564731f..3633c509f3 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -14,7 +14,6 @@ set(TEST ${INCLUDEOS_ROOT}/test)
 
 include_directories(
   ${INCLUDEOS_ROOT}/api
-  ${INCLUDEOS_ROOT}/api/posix
   ${INCLUDEOS_ROOT}/src/
   ${INCLUDEOS_ROOT}/src/include
   ${INCLUDEOS_ROOT}/mod/
diff --git a/test/posix/unit/inet_test.cpp b/test/posix/unit/inet_test.cpp
index 55146eb1e6..80d25d4155 100644
--- a/test/posix/unit/inet_test.cpp
+++ b/test/posix/unit/inet_test.cpp
@@ -27,7 +27,7 @@ CASE("IPv4 address manipulation - str to addr")
   in_addr_t addr;
 
   addr = inet_addr(TEST_ADDRSTR);
-  EXPECT(::ntohl(addr) == TEST_ADDR);
+  EXPECT(ntohl(addr) == TEST_ADDR);
 
   addr = inet_addr("Invalid.Address");
   EXPECT(addr == -1);
@@ -35,7 +35,7 @@ CASE("IPv4 address manipulation - str to addr")
 CASE("IPv4 address manipulation - addr to str")
 {
   struct in_addr addr;
-  addr.s_addr = ::htonl(TEST_ADDR);
+  addr.s_addr = htonl(TEST_ADDR);
 
   char* addrstr = inet_ntoa(addr);
   EXPECT(strcmp(addrstr, TEST_ADDRSTR) == 0);
@@ -43,7 +43,7 @@ CASE("IPv4 address manipulation - addr to str")
 CASE("IPv4 address manipulation - addr to pointer")
 {
   struct in_addr addr;
-  addr.s_addr = ::htonl(TEST_ADDR);
+  addr.s_addr = htonl(TEST_ADDR);
 
   char addrstr[INET_ADDRSTRLEN];
 

From 488e04c254bbc8c5e3cafbe979289238e2156717 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Andreas=20=C3=85kesson?= <andreas.jm.akesson@gmail.com>
Date: Mon, 30 Jan 2017 15:09:57 +0100
Subject: [PATCH 26/27] build: Now building with -nostdlibinc (only clang
 builtin headers)

---
 CMakeLists.txt               |  4 ++--
 api/posix/dlfcn.h            | 39 ++++++++++++++++++++++++++++++++++++
 api/posix/nl_types.h         | 39 ++++++++++++++++++++++++++++++++++++
 api/posix/{sys => }/stddef.h |  0
 etc/service.cmake            |  7 +++----
 lib/mana/CMakeLists.txt      |  1 -
 mod/CMakeLists.txt           |  1 -
 7 files changed, 83 insertions(+), 8 deletions(-)
 create mode 100644 api/posix/dlfcn.h
 create mode 100644 api/posix/nl_types.h
 rename api/posix/{sys => }/stddef.h (100%)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 21f5b29b06..695be07f65 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -82,8 +82,8 @@ endif(silent)
 set(CAPABS "${CAPABS} ${OPTIMIZE}")
 
 # these kinda work with llvm
-set(CMAKE_CXX_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -c -m32 -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1 -DOS_VERSION=\\\"${OS_VERSION}\\\"")
-set(CMAKE_C_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -c -m32 -DOS_VERSION=\"\"${OS_VERSION}\"\"")
+set(CMAKE_CXX_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -c -m32 -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1 -DOS_VERSION=\\\"${OS_VERSION}\\\"")
+set(CMAKE_C_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -c -m32 -DOS_VERSION=\"\"${OS_VERSION}\"\"")
 
 # either download or cross-compile needed libraries
 option(from_bundle "Download and use pre-compiled libraries for cross-comilation" ON)
diff --git a/api/posix/dlfcn.h b/api/posix/dlfcn.h
new file mode 100644
index 0000000000..50e5a37ab5
--- /dev/null
+++ b/api/posix/dlfcn.h
@@ -0,0 +1,39 @@
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2017 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef POSIX_DLFCN_H
+#define POSIX_DLFCN_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define RTLD_LAZY   1   // Relocations are performed at an implementation-dependent time.
+#define RTLD_NOW    2   // Relocations are performed when the object is loaded.
+#define RTLD_GLOBAL 3   // All symbols are available for relocation processing of other modules.
+#define RTLD_LOCAL  4   // All symbols are not made available for relocation processing by other modules.
+
+void  *dlopen(const char *, int);
+void  *dlsym(void *, const char *);
+int    dlclose(void *);
+char  *dlerror(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // < POSIX_DLFCN_H
diff --git a/api/posix/nl_types.h b/api/posix/nl_types.h
new file mode 100644
index 0000000000..5660e4291c
--- /dev/null
+++ b/api/posix/nl_types.h
@@ -0,0 +1,39 @@
+// This file is a part of the IncludeOS unikernel - www.includeos.org
+//
+// Copyright 2017 Oslo and Akershus University College of Applied Sciences
+// and Alfred Bratterud
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef POSIX_NL_TYPES_H
+#define POSIX_NL_TYPES_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define NL_SETD       1
+#define NL_CAT_LOCALE 1
+
+typedef void *nl_catd;
+typedef int   nl_item;
+
+int       catclose(nl_catd);
+char     *catgets(nl_catd, int, int, const char *);
+nl_catd   catopen(const char *, int);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // < POSIX_NL_TYPES_H
diff --git a/api/posix/sys/stddef.h b/api/posix/stddef.h
similarity index 100%
rename from api/posix/sys/stddef.h
rename to api/posix/stddef.h
diff --git a/etc/service.cmake b/etc/service.cmake
index bf5432b103..6562a358d4 100644
--- a/etc/service.cmake
+++ b/etc/service.cmake
@@ -48,8 +48,8 @@ if (debug)
 endif()
 
 # these kinda work with llvm
-set(CMAKE_CXX_FLAGS "-MMD -target i686-elf ${CAPABS} ${OPTIMIZE} ${WARNS} -c -m32 -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1")
-set(CMAKE_C_FLAGS "-MMD -target i686-elf ${CAPABS} ${OPTIMIZE} ${WARNS} -c -m32")
+set(CMAKE_CXX_FLAGS "-MMD -target i686-elf ${CAPABS} ${OPTIMIZE} ${WARNS} -nostdlib -nostdlibinc -c -m32 -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1")
+set(CMAKE_C_FLAGS "-MMD -target i686-elf ${CAPABS} ${OPTIMIZE} ${WARNS} -nostdlib -nostdlibinc -c -m32")
 
 # executable
 set(SERVICE_STUB "$ENV{INCLUDEOS_PREFIX}/includeos/src/service_name.cpp")
@@ -145,10 +145,9 @@ endforeach()
 
 # includes
 include_directories(${LOCAL_INCLUDES})
+include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api/posix)
 include_directories($ENV{INCLUDEOS_PREFIX}/includeos/include/libcxx)
-include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api/sys)
 include_directories($ENV{INCLUDEOS_PREFIX}/includeos/include/newlib)
-include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api/posix)
 include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api)
 include_directories($ENV{INCLUDEOS_PREFIX}/includeos/include)
 include_directories($ENV{INCLUDEOS_PREFIX}/include)
diff --git a/lib/mana/CMakeLists.txt b/lib/mana/CMakeLists.txt
index b0e252c3cb..daa440a028 100644
--- a/lib/mana/CMakeLists.txt
+++ b/lib/mana/CMakeLists.txt
@@ -4,7 +4,6 @@ set(LIB_MANA ${CMAKE_CURRENT_SOURCE_DIR})
 include_directories(${LIB_MANA}/include)
 include_directories(${INCLUDEOS_ROOT}/mod/rapidjson/include)
 
-include_directories(${INCLUDEOS_ROOT}/api/sys)
 include_directories(${INCLUDEOS_ROOT}/api/posix)
 include_directories(${LIBCXX_INCLUDE_DIR})
 include_directories(${NEWLIB_INCLUDE_DIR})
diff --git a/mod/CMakeLists.txt b/mod/CMakeLists.txt
index eb67887a5c..5bfe384629 100644
--- a/mod/CMakeLists.txt
+++ b/mod/CMakeLists.txt
@@ -1,7 +1,6 @@
 set(CMAKE_ASM_NASM_OBJECT_FORMAT "elf")
 enable_language(ASM_NASM)
 
-include_directories(${INCLUDEOS_ROOT}/api/sys)
 include_directories(${INCLUDEOS_ROOT}/api/posix)
 include_directories(${LIBCXX_INCLUDE_DIR})
 include_directories(${NEWLIB_INCLUDE_DIR})

From 19a65ca49e405c020a6bb02664328dbd5374fa36 Mon Sep 17 00:00:00 2001
From: Alfred Bratterud <alfred.bratterud@hioa.no>
Date: Mon, 30 Jan 2017 14:24:17 +0000
Subject: [PATCH 27/27] some posix documentation / guidelines

---
 api/posix/README.md | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 api/posix/README.md

diff --git a/api/posix/README.md b/api/posix/README.md
new file mode 100644
index 0000000000..ef0216d24b
--- /dev/null
+++ b/api/posix/README.md
@@ -0,0 +1,11 @@
+# IncludeOS POSIX extensions
+
+IncludeOS intends to provide a POSIX interface suffucient for linking and running many conventional C libraries and programs. A lot of the POSIX functionality will be header-only stubs and some of it is provided externally by e.g. the compiler or standard library.
+
+### Other providers of POSIX content
+* *newlib*: is our current C library which also provides many POSIX features (indeed the C standard itself overlaps with POSIX). Newlib provides most of the C standard library, including `stdlib.h`, `stdio.h`, `math.h` etc., but is mising some C11 extensions. Those are rarely used and  provided here as stubs.
+* *clang*: Clang provides a few POSIX headers such as `stddef.h`, `stdarg.h` and `limits.h`. It also provides compiler intrinsics such as `x86intrin.h`. When building IncludeOS we use the `-nostdlibinc` flag to allow inclusion of these headers, without including the standard library headers from the host.
+
+### Guidelines for this folder
+* Only actually standardized POSIX content should live here, and only content not allready provided by alternative sources above.
+* Extensions to POSIX headers that IncludeOS needs, but which isn't present on one of the supportet platforms (e.g. macOS or Linux) should not live here, since we'd like to be able to build directly on those platforms with their respective POSIX implementations. As an example, our syslog implementation defines `LOG_INTERNAL` in addition to `LOG_MAIL` etc. While defining this symbol in the `syslog.h` POSIX header is allowed by the standard it introduces an implicit expectation in IncludeOS application code making it less portable. Such extensions can be placed in the IncludeOS API instead.