diff --git a/CMakeLists.txt b/CMakeLists.txt index 21f5b29b06..695be07f65 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -82,8 +82,8 @@ endif(silent) set(CAPABS "${CAPABS} ${OPTIMIZE}") # these kinda work with llvm -set(CMAKE_CXX_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -c -m32 -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1 -DOS_VERSION=\\\"${OS_VERSION}\\\"") -set(CMAKE_C_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -c -m32 -DOS_VERSION=\"\"${OS_VERSION}\"\"") +set(CMAKE_CXX_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -c -m32 -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1 -DOS_VERSION=\\\"${OS_VERSION}\\\"") +set(CMAKE_C_FLAGS "-target i686 -MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -c -m32 -DOS_VERSION=\"\"${OS_VERSION}\"\"") # either download or cross-compile needed libraries option(from_bundle "Download and use pre-compiled libraries for cross-comilation" ON) diff --git a/README.md b/README.md index cf0554ffe4..63efc80ac1 100644 --- a/README.md +++ b/README.md @@ -97,13 +97,13 @@ More information is [available on the wiki](https://github.com/hioa-cs/IncludeOS ### Writing your first service -1. Copy the [./seed](./seed) directory to a convenient location like `~/your_service`. Then, just start implementing the `Service::start` function in the `Service` class, located in [your_service/service.cpp](./seed/service.cpp) (Very simple example provided). This function will be called once the OS is up and running. -2. Update the [CMakeLists.txt](./seed/CMakeLists.txt) to specify the name of your project, enable any needed drivers or plugins, etc. +1. Copy the [./seed/service](./seed/service) directory to a convenient location like `~/your_service`. Then, just start implementing the `Service::start` function in the `Service` class, located in [your_service/service.cpp](./seed/service/service.cpp) (Very simple example provided). This function will be called once the OS is up and running. +2. Update the [CMakeLists.txt](./seed/service/CMakeLists.txt) to specify the name of your project, enable any needed drivers or plugins, etc. **Example:** ``` - $ cp -r seed ~/my_service + $ cp -r seed/service ~/my_service $ cd ~/my_service $ emacs service.cpp ... add your code diff --git a/api/kernel/mman.hpp b/api/kernel/mman.hpp deleted file mode 100644 index e87f3dc5a1..0000000000 --- a/api/kernel/mman.hpp +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef KERNEL_MMAN_HPP -#define KERNEL_MMAN_HPP - -#include - -#endif diff --git a/api/net/http/client.hpp b/api/net/http/client.hpp index 5f8964ae39..8580f2fcdf 100644 --- a/api/net/http/client.hpp +++ b/api/net/http/client.hpp @@ -1,6 +1,6 @@ // This file is a part of the IncludeOS unikernel - www.includeos.org // -// Copyright 2016 Oslo and Akershus University College of Applied Sciences +// Copyright 2016-2017 Oslo and Akershus University College of Applied Sciences // and Alfred Bratterud // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,12 +19,10 @@ #ifndef HTTP_CLIENT_HPP #define HTTP_CLIENT_HPP -#include "common.hpp" -#include "request.hpp" -#include "response.hpp" +// http +#include "client_connection.hpp" + #include -#include "connection.hpp" -#include "error.hpp" #include #include @@ -32,13 +30,13 @@ namespace http { class Client { public: - using TCP = net::TCP; - using Host = net::tcp::Socket; + using TCP = net::TCP; + using Host = net::tcp::Socket; - using Connection_set = std::vector>; - using Connection_mapset = std::map; + using Connection_set = std::vector>; + using Connection_mapset = std::map; - using timeout_duration = Connection::timeout_duration; + using timeout_duration = Client_connection::timeout_duration; const static timeout_duration DEFAULT_TIMEOUT; // client.cpp, 5s constexpr static size_t DEFAULT_BUFSIZE = 2048; @@ -153,10 +151,11 @@ namespace http { inline void post(Host host, std::string path, Header_set hfields, const std::string& data, Response_handler cb, Options options = {}); private: - TCP& tcp_; - Connection_mapset conns_; + friend class Client_connection; - bool keep_alive_ = false; + TCP& tcp_; + Connection_mapset conns_; + bool keep_alive_ = false; void resolve(const std::string& host, ResolveCallback); @@ -172,9 +171,9 @@ namespace http { /** Add data and content length */ void add_data(Request&, const std::string& data); - Connection& get_connection(const Host host); + Client_connection& get_connection(const Host host); - void close(Connection&); + void close(Client_connection&); }; // < class Client diff --git a/api/net/http/client_connection.hpp b/api/net/http/client_connection.hpp new file mode 100644 index 0000000000..28d3f4f86e --- /dev/null +++ b/api/net/http/client_connection.hpp @@ -0,0 +1,69 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2016-2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef HTTP_CLIENT_CONNECTION_HPP +#define HTTP_CLIENT_CONNECTION_HPP + +// http +#include "common.hpp" +#include "connection.hpp" +#include "error.hpp" + +#include + +namespace http { + + class Client; + + class Client_connection : public Connection { + public: + using timeout_duration = std::chrono::milliseconds; + + public: + explicit Client_connection(Client&, TCP_conn); + + bool available() const + { return on_response_ == nullptr && keep_alive_; } + + bool occupied() const + { return !available(); } + + void send(Request_ptr, Response_handler, const size_t bufsize, timeout_duration = timeout_duration::zero()); + + private: + Client& client_; + Response_handler on_response_; + Timer timer_; + timeout_duration timeout_dur_; + + void send_request(const size_t bufsize); + + void recv_response(buffer_t buf, size_t len); + + void end_response(Error err = Error::NONE); + + void timeout_request() + { end_response(Error::TIMEOUT); } + + void close(); + + }; // < class Client_connection + +} // < namespace http + +#endif // < HTTP_CLIENT_CONNECTION_HPP diff --git a/api/net/http/connection.hpp b/api/net/http/connection.hpp index ae6656d955..6e6feb092b 100644 --- a/api/net/http/connection.hpp +++ b/api/net/http/connection.hpp @@ -1,6 +1,6 @@ // This file is a part of the IncludeOS unikernel - www.includeos.org // -// Copyright 2016 Oslo and Akershus University College of Applied Sciences +// Copyright 2016-2017 Oslo and Akershus University College of Applied Sciences // and Alfred Bratterud // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,74 +19,59 @@ #ifndef HTTP_CONNECTION_HPP #define HTTP_CONNECTION_HPP -#include "common.hpp" +// http #include "request.hpp" #include "response.hpp" -#include "error.hpp" + #include -#include -#include -#include -#include namespace http { class Connection { public: - using TCP_conn_ptr = net::tcp::Connection_ptr; - using Peer = net::tcp::Socket; - using buffer_t = net::tcp::buffer_t; - using Close_handler = delegate; - using timeout_duration = std::chrono::milliseconds; + using TCP_conn = net::tcp::Connection_ptr; + using Peer = net::tcp::Socket; + using buffer_t = net::tcp::buffer_t; public: - - explicit Connection(TCP_conn_ptr, Close_handler); + inline explicit Connection(TCP_conn tcpconn, bool keep_alive = true); template - explicit Connection(TCP&, Peer, Close_handler); - - bool available() const - { return on_response_ == nullptr && keep_alive_; } - - bool occupied() const - { return !available(); } + explicit Connection(TCP&, Peer); - void send(Request_ptr, Response_handler, const size_t bufsize, timeout_duration = timeout_duration::zero()); - - net::tcp::port_t local_port() const + net::tcp::port_t local_port() const noexcept { return (tcpconn_) ? tcpconn_->local_port() : 0; } - Peer peer() const + Peer peer() const noexcept { return (tcpconn_) ? tcpconn_->remote() : Peer(); } - //bool operator==(const Connection& other) - //{ return this == &other; } - //{ return tcpconn_->local_port() == other.tcpconn_->local_port(); } - private: - TCP_conn_ptr tcpconn_; + void timeout() + { tcpconn_->is_closing() ? tcpconn_->abort() : tcpconn_->close(); } + + protected: + TCP_conn tcpconn_; Request_ptr req_; Response_ptr res_; - Close_handler on_close_; - Response_handler on_response_; - Timer timer_; - - timeout_duration timeout_dur_; - bool keep_alive_; - - void send_request(const size_t bufsize); - - void recv_response(buffer_t buf, size_t len); - - void end_response(Error err = Error::NONE); - - void timeout_request() - { end_response(Error::TIMEOUT); } - - void close(); + bool keep_alive_; }; // < class Connection + inline Connection::Connection(TCP_conn tcpconn, bool keep_alive) + : tcpconn_{std::move(tcpconn)}, + req_{nullptr}, + res_{nullptr}, + keep_alive_{keep_alive} + { + Ensures(tcpconn_ != nullptr); + debug(" Created %u -> %s %p\n", local_port(), peer().to_string().c_str(), this); + } + + template + Connection::Connection(TCP& tcp, Peer addr) + : Connection(tcp.connect(addr)) + { + } + } // < namespace http #endif // < HTTP_CONNECTION_HPP diff --git a/api/posix/README.md b/api/posix/README.md new file mode 100644 index 0000000000..ef0216d24b --- /dev/null +++ b/api/posix/README.md @@ -0,0 +1,11 @@ +# IncludeOS POSIX extensions + +IncludeOS intends to provide a POSIX interface suffucient for linking and running many conventional C libraries and programs. A lot of the POSIX functionality will be header-only stubs and some of it is provided externally by e.g. the compiler or standard library. + +### Other providers of POSIX content +* *newlib*: is our current C library which also provides many POSIX features (indeed the C standard itself overlaps with POSIX). Newlib provides most of the C standard library, including `stdlib.h`, `stdio.h`, `math.h` etc., but is mising some C11 extensions. Those are rarely used and provided here as stubs. +* *clang*: Clang provides a few POSIX headers such as `stddef.h`, `stdarg.h` and `limits.h`. It also provides compiler intrinsics such as `x86intrin.h`. When building IncludeOS we use the `-nostdlibinc` flag to allow inclusion of these headers, without including the standard library headers from the host. + +### Guidelines for this folder +* Only actually standardized POSIX content should live here, and only content not allready provided by alternative sources above. +* Extensions to POSIX headers that IncludeOS needs, but which isn't present on one of the supportet platforms (e.g. macOS or Linux) should not live here, since we'd like to be able to build directly on those platforms with their respective POSIX implementations. As an example, our syslog implementation defines `LOG_INTERNAL` in addition to `LOG_MAIL` etc. While defining this symbol in the `syslog.h` POSIX header is allowed by the standard it introduces an implicit expectation in IncludeOS application code making it less portable. Such extensions can be placed in the IncludeOS API instead. diff --git a/api/posix/dlfcn.h b/api/posix/dlfcn.h new file mode 100644 index 0000000000..50e5a37ab5 --- /dev/null +++ b/api/posix/dlfcn.h @@ -0,0 +1,39 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef POSIX_DLFCN_H +#define POSIX_DLFCN_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define RTLD_LAZY 1 // Relocations are performed at an implementation-dependent time. +#define RTLD_NOW 2 // Relocations are performed when the object is loaded. +#define RTLD_GLOBAL 3 // All symbols are available for relocation processing of other modules. +#define RTLD_LOCAL 4 // All symbols are not made available for relocation processing by other modules. + +void *dlopen(const char *, int); +void *dlsym(void *, const char *); +int dlclose(void *); +char *dlerror(void); + +#ifdef __cplusplus +} +#endif + +#endif // < POSIX_DLFCN_H diff --git a/api/sys/math.h b/api/posix/math.h similarity index 97% rename from api/sys/math.h rename to api/posix/math.h index fca5d9b13f..50a204a0cc 100644 --- a/api/sys/math.h +++ b/api/posix/math.h @@ -18,6 +18,9 @@ #ifndef SYS_MATH_H #define SYS_MATH_H +#ifdef __cplusplus +extern "C" { +#endif // Long double math stubs long double cosl(long double); @@ -85,7 +88,10 @@ long double tgammal(long double); long double truncl(long double); long double nanl(const char*); - +#ifdef __cplusplus +} #endif #include_next + +#endif diff --git a/api/posix/nl_types.h b/api/posix/nl_types.h new file mode 100644 index 0000000000..5660e4291c --- /dev/null +++ b/api/posix/nl_types.h @@ -0,0 +1,39 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef POSIX_NL_TYPES_H +#define POSIX_NL_TYPES_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define NL_SETD 1 +#define NL_CAT_LOCALE 1 + +typedef void *nl_catd; +typedef int nl_item; + +int catclose(nl_catd); +char *catgets(nl_catd, int, int, const char *); +nl_catd catopen(const char *, int); + +#ifdef __cplusplus +} +#endif + +#endif // < POSIX_NL_TYPES_H diff --git a/api/posix/signal.h b/api/posix/signal.h new file mode 100644 index 0000000000..42c677b159 --- /dev/null +++ b/api/posix/signal.h @@ -0,0 +1,42 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2016 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef SYS_SIGNAL_H +#define SYS_SIGNAL_H + +#include_next + +#ifdef __cplusplus +extern "C" { +#endif + +struct siginfo_t +{ + int si_signo; // Signal number. + int si_code; // Signal code. + + int sa_sigaction; +}; + +#define SA_RESETHAND 666 + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/api/sys/stddef.h b/api/posix/stddef.h similarity index 100% rename from api/sys/stddef.h rename to api/posix/stddef.h diff --git a/api/sys/__config b/api/posix/sys/__config similarity index 100% rename from api/sys/__config rename to api/posix/sys/__config diff --git a/api/sys/ctype.h b/api/posix/sys/ctype.h similarity index 100% rename from api/sys/ctype.h rename to api/posix/sys/ctype.h diff --git a/api/posix/sys/dlfcn.h b/api/posix/sys/dlfcn.h new file mode 100644 index 0000000000..49e7ff83a5 --- /dev/null +++ b/api/posix/sys/dlfcn.h @@ -0,0 +1,25 @@ +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +void *dlopen(const char *filename, int flag); +char *dlerror(void); +void *dlsym(void *handle, const char *symbol); +int dlclose(void *handle); + + +#define RTLD_LAZY 1 +#define RTLD_NOW 2 +#define RTLD_GLOBAL 3 +#define RTLD_LOCAL 4 +#define RTLD_NODELETE 6 +#define RTLD_NOLOAD 7 +#define RTLD_DEEPBIND 8 + +#define RTLD_DEFAULT 1 + +#ifdef __cplusplus +} +#endif diff --git a/api/sys/features.h b/api/posix/sys/features.h similarity index 100% rename from api/sys/features.h rename to api/posix/sys/features.h diff --git a/api/posix/sys/mman.h b/api/posix/sys/mman.h index 23faf6b7ef..25c55d8d81 100644 --- a/api/posix/sys/mman.h +++ b/api/posix/sys/mman.h @@ -24,13 +24,29 @@ extern "C" { #endif #include +#include typedef _off_t off_t; -void *mmap(void* addr, size_t length, - int prot, int flags, - int fd, off_t offset); -int munmap(void* addr, size_t length); +struct posix_typed_mem_info +{ + size_t posix_tmi_length; +}; +int mlock(const void *, size_t); +int mlockall(int); +void *mmap(void *, size_t, int, int, int, off_t); +int mprotect(void *, size_t, int); +int msync(void *, size_t, int); +int munlock(const void *, size_t); +int munlockall(void); +int munmap(void *, size_t); +int posix_madvise(void *, size_t, int); +int posix_mem_offset(const void *__restrict__, size_t, off_t *__restrict__, + size_t *__restrict__, int *__restrict__); +int posix_typed_mem_get_info(int, struct posix_typed_mem_info *); +int posix_typed_mem_open(const char *, int, int); +int shm_open(const char *, int, mode_t); +int shm_unlink(const char *); // Page can be executed. #define PROT_EXEC 0x1 diff --git a/api/posix/sys/socket.h b/api/posix/sys/socket.h index 56c4afea3a..9023b45ba5 100644 --- a/api/posix/sys/socket.h +++ b/api/posix/sys/socket.h @@ -137,8 +137,8 @@ int socketpair(int domain, int type, int protocol, enum { SHUT_RD = 0, - SHUT_RW, - SHUT_RDRW + SHUT_WR, + SHUT_RDWR }; #define INVALID_SOCKET ((SOCKET)(~0)) diff --git a/api/sys/stdlib.h b/api/posix/sys/stdlib.h similarity index 96% rename from api/sys/stdlib.h rename to api/posix/sys/stdlib.h index 9344556f62..7e1e37389b 100644 --- a/api/sys/stdlib.h +++ b/api/posix/sys/stdlib.h @@ -6,9 +6,9 @@ // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,19 +22,19 @@ #include_next // More C11 requirements here -#include "quick_exit" +#include #ifdef __cplusplus extern "C" { #endif - + //New stuff in C11, required by libunwind, compiler-rt etc. in llvm - + void *aligned_alloc( size_t alignment, size_t size ); #ifdef __cplusplus } -#endif +#endif #endif //SYS_STDLIB_H diff --git a/api/sys/time.h b/api/posix/sys/time.h similarity index 100% rename from api/sys/time.h rename to api/posix/sys/time.h diff --git a/api/posix/sys/uio.h b/api/posix/sys/uio.h new file mode 100644 index 0000000000..7dd8e4948d --- /dev/null +++ b/api/posix/sys/uio.h @@ -0,0 +1,38 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef POSIX_SYS_UIO_H +#define POSIX_SYS_UIO_H + +#ifdef __cplusplus +extern "C" { +#endif + +struct iovec { + void *iov_base; // Base address of a memory region for input or output + size_t iov_len; // Size of memory pointed to by iov_base +}; + +ssize_t readv(int, const struct iovec *, int); +ssize_t writev(int, const struct iovec *, int); + +#ifdef __cplusplus +} +#endif + +#endif // POSIX_SYS_UIO_H diff --git a/api/posix/sys/utsname.h b/api/posix/sys/utsname.h index 6d5a9b67c4..0d64e5a51b 100644 --- a/api/posix/sys/utsname.h +++ b/api/posix/sys/utsname.h @@ -23,9 +23,9 @@ extern "C" { #endif -struct utsname { - static const int LENGTH = 256; +static const int LENGTH = 256; +struct utsname { /* Name of this implementation of the operating system */ char sysname[LENGTH]; diff --git a/api/sys/wchar.h b/api/posix/sys/wchar.h similarity index 100% rename from api/sys/wchar.h rename to api/posix/sys/wchar.h diff --git a/api/posix/syslog.h b/api/posix/syslog.h index bfd430260d..c15e8d8664 100644 --- a/api/posix/syslog.h +++ b/api/posix/syslog.h @@ -48,7 +48,6 @@ extern "C" { #define LOG_MAIL 2 /* Mail system */ #define LOG_DAEMON 3 /* System daemons */ #define LOG_AUTH 4 /* Security/authorization messages */ -#define LOG_INTERNAL 5 /* Messages generated internally by syslogd */ #define LOG_LPR 6 /* Line printer subsystem */ #define LOG_NEWS 7 /* Network news subsystem */ #define LOG_UUCP 8 /* UUCP subsystem */ diff --git a/api/sys/unistd.h b/api/posix/unistd.h similarity index 100% rename from api/sys/unistd.h rename to api/posix/unistd.h diff --git a/api/sys/quick_exit b/api/quick_exit similarity index 100% rename from api/sys/quick_exit rename to api/quick_exit diff --git a/api/util/syslog_facility.hpp b/api/util/syslog_facility.hpp index 8597c17de1..700d869327 100644 --- a/api/util/syslog_facility.hpp +++ b/api/util/syslog_facility.hpp @@ -21,6 +21,8 @@ #ifndef UTIL_SYSLOG_FACILITY_HPP #define UTIL_SYSLOG_FACILITY_HPP +#define LOG_INTERNAL 5 /* Messages generated internally by syslogd */ + #include #include #include diff --git a/diskimagebuild/CMakeLists.txt b/diskimagebuild/CMakeLists.txt index b48ef7e516..45de2934d6 100644 --- a/diskimagebuild/CMakeLists.txt +++ b/diskimagebuild/CMakeLists.txt @@ -5,7 +5,6 @@ set (CMAKE_CXX_STANDARD 14) set(SOURCES main.cpp filetree.cpp writer.cpp) -include_directories(../api) include_directories(../mod/GSL) add_executable(diskbuilder ${SOURCES}) diff --git a/diskimagebuild/fat_internal.hpp b/diskimagebuild/fat_internal.hpp new file mode 100644 index 0000000000..4861a7e53d --- /dev/null +++ b/diskimagebuild/fat_internal.hpp @@ -0,0 +1,62 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Slim copy of + +#pragma once +#ifndef FS_FAT_INTERNAL_HPP +#define FS_FAT_INTERNAL_HPP + +#include + +// Attribute masks +static const uint8_t ATTR_READ_ONLY = 0x01; +static const uint8_t ATTR_HIDDEN = 0x02; +static const uint8_t ATTR_SYSTEM = 0x04; +static const uint8_t ATTR_VOLUME_ID = 0x08; +static const uint8_t ATTR_DIRECTORY = 0x10; +static const uint8_t ATTR_ARCHIVE = 0x20; + +// Mask for the last longname entry +static const uint8_t LAST_LONG_ENTRY = 0x40; + +struct cl_dir +{ + uint8_t shortname[11]; + uint8_t attrib; + uint8_t pad1[8]; + uint16_t cluster_hi; + uint32_t modified; + uint16_t cluster_lo; + uint32_t filesize; + +} __attribute__((packed)); + +struct cl_long +{ + uint8_t index; + uint16_t first[5]; + uint8_t attrib; + uint8_t entry_type; + uint8_t checksum; + uint16_t second[6]; + uint16_t zero; + uint16_t third[2]; + +} __attribute__((packed)); + +#endif diff --git a/diskimagebuild/writer.cpp b/diskimagebuild/writer.cpp index 2d6ace5a9a..f8d10e2b26 100644 --- a/diskimagebuild/writer.cpp +++ b/diskimagebuild/writer.cpp @@ -1,7 +1,7 @@ #include "filetree.hpp" #include "../api/fs/mbr.hpp" -#include "../api/fs/fat_internal.hpp" +#include "fat_internal.hpp" #include #include diff --git a/etc/boot b/etc/boot index aa15728adc..fcee55c285 100755 --- a/etc/boot +++ b/etc/boot @@ -18,6 +18,12 @@ if not os.path.isdir(INCLUDEOS_PREFIX + "/includeos"): print "Couldn't find IncludeOS installation. If you installed to a non-default location (e.g. not " + default_loc + ") please set the environment variable INCLUDEOS_PREFIX to point to this location." sys.exit(0) +# Location of vmrunner +sys.path.append(INCLUDEOS_PREFIX + "/includeos") + +from vmrunner.prettify import color + + # Argparse parser = argparse.ArgumentParser( description="IncludeOS vmrunner. Builds and runs an IncludeOS service") @@ -37,57 +43,77 @@ parser.add_argument("--create-bridge", dest="bridge", action="store_true", parser.add_argument("-g", "--grub", dest="grub", action="store_true", help="Create image with GRUB bootloader that will boot provided binary") -parser.add_argument('binary', action="store", type=str, - help="Filename of the IncludeOS service binary") +parser.add_argument("-j", "--config", dest="config", type = str, metavar = "PATH", + help="Location of VM config file - JSON validated against a schema") + +parser.add_argument('vm_location', action="store", type = str, + help="Location of the IncludeOS service binary, image or source") parser.add_argument('vmargs', nargs='*', help="Arguments to pass on to the VM start / main") + args = parser.parse_args() -# Location of vmrunner -sys.path.append(INCLUDEOS_PREFIX + "/includeos") -from vmrunner.prettify import color +# Pretty printing from this command +nametag = "" +INFO = color.INFO(nametag) # Override VM output prepension color.VM_PREPEND = "" -# Pretty printing from this command -nametag = "" -INFO = color.INFO(nametag) +# in verbose mode we will set VERBOSE=1 for this environment +VERB = False +if (args.verbose): + os.environ["VERBOSE"] = "1" + VERB = True + print INFO, "VERBOSE mode set for environment" +# Avoid the verbose var to hang around next run +elif "VERBOSE" in os.environ: + del os.environ["VERBOSE"] + +# Note: importing vmrunner will make it start looking for VM's +# vmrunner also relies on the verbose env var to be set on initialization +from vmrunner import vmrunner + # We can boot either a binary without bootloader, or an image with bootloader allready attached has_bootloader = False -print INFO , "Args to pass to VM: ", args.vmargs +if VERB: print INFO , "Args to pass to VM: ", args.vmargs # path w/name of VM image to run -image_name = args.binary +image_name = args.vm_location # if the binary argument is a directory, go there immediately and # then initialize stuff ... -if (os.path.isdir(args.binary)): - image_name = os.path.abspath(args.binary) - print "Changing directory to: " + image_name - os.chdir(os.path.abspath(args.binary)) +if (os.path.isdir(args.vm_location)): + image_name = os.path.abspath(args.vm_location) + if VERB: print INFO, "Changing directory to " + image_name + os.chdir(os.path.abspath(args.vm_location)) -# in verbose mode we will just set VERBOSE=1 for this environment -if (args.verbose): - os.environ['VERBOSE'] = "1" -# Note: importing vmrunner will make it start looking for VM's -from vmrunner import vmrunner if len(vmrunner.vms) < 1: - print color.FAIL("No vm description files found - nothing to boot") + # This should never happen - there should always be a default + print color.FAIL("No vm object found in vmrunner - nothing to boot") exit(-1) -print color.INFO(nametag), len(vmrunner.vms), "VM initialized. Commencing build- and boot..." +if VERB: + print INFO, len(vmrunner.vms), "VM initialized. Commencing build- and boot..." + +config = None +if args.config: + config = os.path.abspath(args.config) + if VERB: print INFO, "Using config file", config -vm = vmrunner.vms[0] +vm = vmrunner.vm(config = config) # For multible JSON configs, find the one matching the provided image name for vm_ in vmrunner.vms: if vm_._config.has_key("image") and vm_._config["image"].startswith(image_name): vm = vm_ +# Don't listen to events needed by testrunner +vm.on_success(lambda: None, do_exit = False) + if (args.clean): print INFO, "Cleaning build" vm.clean() @@ -99,15 +125,15 @@ if (args.bridge): # If the binary name is a folder, such as ".", build the service if (os.path.isdir(image_name)): vm.cmake() - # also, set image name to + # also, set image name to image_name = open("binary.txt", 'r').read() # If the binary name has an extension (e.g. a '.'), assume it has a bootloader attached # NOTE: the idea is to support e.g. .vdi and virtualbox etc. in the future -elif ("." in args.binary): +elif ("." in args.vm_location): has_bootloader = True if (args.grub): - print INFO, "Creating GRUB image from ", args.binary + print INFO, "Creating GRUB image from ", args.vm_location subprocess.call(INCLUDEOS_PREFIX + "/includeos/scripts/grubify.sh " + image_name, shell=True) image_name = image_name + ".grub.img" has_bootloader = True @@ -116,6 +142,6 @@ if (args.grub): if (args.build): exit(0); if (not has_bootloader): - vm.boot(timeout = None, kernel_args=" ".join(args.vmargs), image_name = image_name) + vm.boot(timeout = None, multiboot = True, kernel_args = " ".join(args.vmargs), image_name = image_name) else: vm.boot(timeout = None, multiboot = False, kernel_args = None, image_name = image_name) diff --git a/etc/install_build_requirements.sh b/etc/install_build_requirements.sh index 206c1b35c7..ffdc1d35ed 100755 --- a/etc/install_build_requirements.sh +++ b/etc/install_build_requirements.sh @@ -28,7 +28,7 @@ case $SYSTEM in clang_version="3.8" fi - DEPENDENCIES="curl make clang-$clang_version nasm bridge-utils qemu jq cmake $DEPENDENCIES" + DEPENDENCIES="curl make clang-$clang_version nasm bridge-utils qemu jq python-jsonschema python-psutil cmake $DEPENDENCIES" echo ">>> Installing dependencies (requires sudo):" echo " Packages: $DEPENDENCIES" sudo apt-get update || exit 1 @@ -36,14 +36,14 @@ case $SYSTEM in exit 0; ;; "fedora") - DEPENDENCIES="curl make clang nasm bridge-utils qemu jq python-jsonschema cmake" + DEPENDENCIES="curl make clang nasm bridge-utils qemu jq python-jsonschema python-psutil cmake" echo ">>> Installing dependencies (requires sudo):" echo " Packages: $DEPENDENCIES" sudo dnf install $DEPENDENCIES || exit 1 exit 0; ;; "arch") - DEPENDENCIES="curl make clang nasm bridge-utils qemu jq cmake" + DEPENDENCIES="curl make clang nasm bridge-utils qemu jq python-jsonschema python-psutil cmake" echo ">>> Installing dependencies (requires sudo):" echo " Packages: $DEPENDENCIES" sudo pacman -Syyu diff --git a/etc/install_osx.sh b/etc/install_osx.sh index bbcfcb68d6..27c1261fca 100755 --- a/etc/install_osx.sh +++ b/etc/install_osx.sh @@ -67,6 +67,19 @@ function install_binutils { source ./etc/install_binutils.sh } +## python packages (pip) ## +PIP_INSTALLED=false +PIP_MODS=(jsonschema psutil) +echo -e "\n python pip\t - for installing necessary python modules used when booting services: ${PIP_MODS[*]}" + +## Check if pip is installed ## +if command -v pip >/dev/null 2>&1; then + PIP_INSTALLED=true + echo -e " > Found" +else + echo -e " > Not found" +fi + ## WARN ABOUT XCODE CLT ## echo -e "\n NOTE: Cannot tell if Xcode Command Line Tools is installed - installation MAY fail if not installed." @@ -85,6 +98,15 @@ then install_binutils fi + if (! $PIP_INSTALLED); then + echo -e "\n>> Installing pip (with sudo)" + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + sudo python get-pip.py + rm get-pip.py + fi + + pip install ${PIP_MODS[*]} + echo -e "\n>>> Done installing dependencies." fi diff --git a/etc/library.cmake b/etc/library.cmake index 2452a87354..d22ea8b582 100644 --- a/etc/library.cmake +++ b/etc/library.cmake @@ -1,7 +1,12 @@ # -# +# CMakeList for IncludeOS library # +# IncludeOS install location +if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) + set(ENV{INCLUDEOS_PREFIX} /usr/local) +endif() + # test compiler if(CMAKE_COMPILER_IS_GNUCC) # currently gcc is not supported due to problems cross-compiling a unikernel @@ -12,14 +17,10 @@ endif(CMAKE_COMPILER_IS_GNUCC) set(CMAKE_ASM_NASM_OBJECT_FORMAT "elf") enable_language(ASM_NASM) -# stackrealign is needed to guarantee 16-byte stack alignment for SSE -# the compiler seems to be really dumb in this regard, creating a misaligned stack left and right -set(CAPABS "-mstackrealign -msse3 -fstack-protector-strong") - # Various global defines # * OS_TERMINATE_ON_CONTRACT_VIOLATION provides classic assert-like output from Expects / Ensures # * _GNU_SOURCE enables POSIX-extensions in newlib, such as strnlen. ("everything newlib has", ref. cdefs.h) -set(CAPABS "${CAPABS} -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_GNU_SOURCE -DSERVICE=\"\\\"${BINARY}\\\"\" -DSERVICE_NAME=\"\\\"${SERVICE_NAME}\\\"\"") +set(CAPABS "-msse3 -fstack-protector-strong -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_GNU_SOURCE") set(WARNS "-Wall -Wextra") #-pedantic # configure options @@ -32,23 +33,24 @@ if (minimal) set(OPTIMIZE "-Os") endif() if (debug) - set(CAPABS "${CAPABS} -g" + set(CAPABS "${CAPABS} -g") endif() # these kinda work with llvm set(CMAKE_CXX_FLAGS "-MMD -target i686-elf ${CAPABS} ${OPTIMIZE} ${WARNS} -c -m32 -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1") set(CMAKE_C_FLAGS "-MMD -target i686-elf ${CAPABS} ${OPTIMIZE} ${WARNS} -c -m32") - # includes include_directories(${LOCAL_INCLUDES}) -include_directories(${INCLUDEOS_ROOT}/include/libcxx) -include_directories(${INCLUDEOS_ROOT}/include/api/sys) -include_directories(${INCLUDEOS_ROOT}/include/newlib) -include_directories(${INCLUDEOS_ROOT}/include/api/posix) -include_directories(${INCLUDEOS_ROOT}/include/api) -include_directories(${INCLUDEOS_ROOT}/include/gsl) - -# output <- input -add_library(library STATIC ${SOURCES}) -set_target_properties(library PROPERTIES OUTPUT_NAME ${LIBRARY_NAME}) +include_directories($ENV{INCLUDEOS_PREFIX}/includeos/include/libcxx) +include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api/sys) +include_directories($ENV{INCLUDEOS_PREFIX}/includeos/include/newlib) +include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api/posix) +include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api) +include_directories($ENV{INCLUDEOS_PREFIX}/includeos/include) +include_directories($ENV{INCLUDEOS_PREFIX}/include) + +add_library(${LIBRARY_NAME} STATIC ${SOURCES}) + +#install(TARGETS ${LIBRARY_NAME} DESTINATION includeos/lib) +#install(DIRECTORY ${LIBRARY_HEADERS} DESTINATION includeos/include) diff --git a/etc/service.cmake b/etc/service.cmake index bf5432b103..6562a358d4 100644 --- a/etc/service.cmake +++ b/etc/service.cmake @@ -48,8 +48,8 @@ if (debug) endif() # these kinda work with llvm -set(CMAKE_CXX_FLAGS "-MMD -target i686-elf ${CAPABS} ${OPTIMIZE} ${WARNS} -c -m32 -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1") -set(CMAKE_C_FLAGS "-MMD -target i686-elf ${CAPABS} ${OPTIMIZE} ${WARNS} -c -m32") +set(CMAKE_CXX_FLAGS "-MMD -target i686-elf ${CAPABS} ${OPTIMIZE} ${WARNS} -nostdlib -nostdlibinc -c -m32 -std=c++14 -D_LIBCPP_HAS_NO_THREADS=1") +set(CMAKE_C_FLAGS "-MMD -target i686-elf ${CAPABS} ${OPTIMIZE} ${WARNS} -nostdlib -nostdlibinc -c -m32") # executable set(SERVICE_STUB "$ENV{INCLUDEOS_PREFIX}/includeos/src/service_name.cpp") @@ -145,10 +145,9 @@ endforeach() # includes include_directories(${LOCAL_INCLUDES}) +include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api/posix) include_directories($ENV{INCLUDEOS_PREFIX}/includeos/include/libcxx) -include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api/sys) include_directories($ENV{INCLUDEOS_PREFIX}/includeos/include/newlib) -include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api/posix) include_directories($ENV{INCLUDEOS_PREFIX}/includeos/api) include_directories($ENV{INCLUDEOS_PREFIX}/includeos/include) include_directories($ENV{INCLUDEOS_PREFIX}/include) diff --git a/lib/mana/CMakeLists.txt b/lib/mana/CMakeLists.txt index b0e252c3cb..daa440a028 100644 --- a/lib/mana/CMakeLists.txt +++ b/lib/mana/CMakeLists.txt @@ -4,7 +4,6 @@ set(LIB_MANA ${CMAKE_CURRENT_SOURCE_DIR}) include_directories(${LIB_MANA}/include) include_directories(${INCLUDEOS_ROOT}/mod/rapidjson/include) -include_directories(${INCLUDEOS_ROOT}/api/sys) include_directories(${INCLUDEOS_ROOT}/api/posix) include_directories(${LIBCXX_INCLUDE_DIR}) include_directories(${NEWLIB_INCLUDE_DIR}) diff --git a/mod/CMakeLists.txt b/mod/CMakeLists.txt index eb67887a5c..5bfe384629 100644 --- a/mod/CMakeLists.txt +++ b/mod/CMakeLists.txt @@ -1,7 +1,6 @@ set(CMAKE_ASM_NASM_OBJECT_FORMAT "elf") enable_language(ASM_NASM) -include_directories(${INCLUDEOS_ROOT}/api/sys) include_directories(${INCLUDEOS_ROOT}/api/posix) include_directories(${LIBCXX_INCLUDE_DIR}) include_directories(${NEWLIB_INCLUDE_DIR}) diff --git a/seed/.gitignore b/seed/.gitignore deleted file mode 100644 index c6079b4532..0000000000 --- a/seed/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -*.o -*.d -*.img -_elf_symbols.bin diff --git a/seed/debug/service.gdb b/seed/debug/service.gdb deleted file mode 100644 index f5acb4ff3c..0000000000 --- a/seed/debug/service.gdb +++ /dev/null @@ -1,5 +0,0 @@ -file service -break _start -break OS::start -set non-stop off -target remote localhost:1234 \ No newline at end of file diff --git a/seed/library/.gitignore b/seed/library/.gitignore new file mode 100644 index 0000000000..d2e6ee39e8 --- /dev/null +++ b/seed/library/.gitignore @@ -0,0 +1,2 @@ +build/* +lib/ diff --git a/seed/library/CMakeLists.txt b/seed/library/CMakeLists.txt new file mode 100644 index 0000000000..4cd6ffde4f --- /dev/null +++ b/seed/library/CMakeLists.txt @@ -0,0 +1,44 @@ +cmake_minimum_required(VERSION 2.8.9) + +# IncludeOS install location +if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) + set(ENV{INCLUDEOS_PREFIX} /usr/local) +endif() + +# Use toolchain (if needed) +set(CMAKE_TOOLCHAIN_FILE $ENV{INCLUDEOS_PREFIX}/includeos/i686-elf-toolchain.cmake) + +# Name of your project +project (libseed) + +# Name of your IncludeOS library +set(LIBRARY_NAME "seed") # => libseed.a + +# Source files to be built into your IncludeOS library +set(SOURCES + # seed.cpp # ...add more here + ) + +# Necessary includes to build your library +set(LOCAL_INCLUDES + # "include" + ) + +# include library build script +include($ENV{INCLUDEOS_PREFIX}/includeos/library.cmake) + + +# INSTALLATION (OPTIONAL): + +# If CMAKE_INSTALL_PREFIX is not set, install to source directory +if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) + set(CMAKE_INSTALL_PREFIX "${CMAKE_CURRENT_SOURCE_DIR}") # $ENV{INCLUDEOS_PREFIX}/includeos +endif() + +# Where to install library +install(TARGETS ${LIBRARY_NAME} DESTINATION ${CMAKE_INSTALL_PREFIX}/lib) + +# Where to install library headers +# NOTE: There is a difference between installing a list of files and a directory +# set(LIBRARY_HEADERS "include/seed") +# install(DIRECTORY ${LIBRARY_HEADERS} DESTINATION ${CMAKE_INSTALL_PREFIX}/include) diff --git a/seed/cmake_build.sh b/seed/library/cmake_build.sh similarity index 100% rename from seed/cmake_build.sh rename to seed/library/cmake_build.sh diff --git a/seed/run.sh b/seed/run.sh deleted file mode 100755 index 8b5ad911d8..0000000000 --- a/seed/run.sh +++ /dev/null @@ -1,8 +0,0 @@ -#! /bin/bash -set - -INCLUDEOS_PREFIX=${INCLUDEOS_PREFIX-/usr/local} -sudo $INCLUDEOS_PREFIX/includeos/scripts/create_bridge.sh - -FILE=$1 -shift -source $INCLUDEOS_PREFIX/includeos/scripts/run.sh $FILE ${*} diff --git a/seed/service/.gitignore b/seed/service/.gitignore new file mode 100644 index 0000000000..a007feab07 --- /dev/null +++ b/seed/service/.gitignore @@ -0,0 +1 @@ +build/* diff --git a/seed/CMakeLists.txt b/seed/service/CMakeLists.txt similarity index 100% rename from seed/CMakeLists.txt rename to seed/service/CMakeLists.txt diff --git a/seed/service/cmake_build.sh b/seed/service/cmake_build.sh new file mode 100755 index 0000000000..9fdd05a8a0 --- /dev/null +++ b/seed/service/cmake_build.sh @@ -0,0 +1,7 @@ +#!/bin/bash +INSTALL=`pwd` +mkdir -p build +pushd build +cmake .. -DCMAKE_INSTALL_PREFIX:PATH=$INSTALL +make install +popd diff --git a/seed/docker_run.sh b/seed/service/docker_run.sh similarity index 100% rename from seed/docker_run.sh rename to seed/service/docker_run.sh diff --git a/seed/service.cpp b/seed/service/service.cpp similarity index 100% rename from seed/service.cpp rename to seed/service/service.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f72abb44fa..d2825d9a93 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,7 +1,6 @@ set(CMAKE_ASM_NASM_OBJECT_FORMAT "elf") enable_language(ASM_NASM) -include_directories(${INCLUDEOS_ROOT}/api/sys) include_directories(${INCLUDEOS_ROOT}/api/posix) include_directories(${LIBCXX_INCLUDE_DIR}) include_directories(${NEWLIB_INCLUDE_DIR}) @@ -47,14 +46,14 @@ set(OS_OBJECTS net/super_stack.cpp net/http/header.cpp net/http/header_fields.cpp net/http/message.cpp net/http/request.cpp net/http/response.cpp net/http/status_codes.cpp net/http/time.cpp net/http/version.cpp - net/http/mime_types.cpp net/http/client.cpp net/http/connection.cpp net/http/cookie.cpp + net/http/mime_types.cpp net/http/client.cpp net/http/client_connection.cpp net/http/cookie.cpp fs/disk.cpp fs/filesystem.cpp fs/mbr.cpp fs/path.cpp fs/fat.cpp fs/fat_async.cpp fs/fat_sync.cpp fs/memdisk.cpp posix/fd.cpp posix/tcp_fd.cpp posix/udp_fd.cpp posix/unistd.cpp posix/fcntl.cpp posix/syslog.cpp posix/sys/socket.cpp posix/sys/select.cpp posix/sys/utsname.cpp posix/sys/mman.cpp posix/arpa/inet.cpp posix/ucontext.cpp posix/ucontext_asm.asm - posix/sys/stat.cpp posix/ftw.cpp - posix/file_fd.cpp + posix/sys/stat.cpp posix/ftw.cpp posix/file_fd.cpp posix/dlfcn.cpp + posix/pwd.cpp ) add_library(os STATIC ${OS_OBJECTS} apic_boot.o) diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c index 24a5276d9d..b1c9b92faa 100644 --- a/src/crt/c_abi.c +++ b/src/crt/c_abi.c @@ -24,7 +24,7 @@ #include #include -#define HEAP_ALIGNMENT 16 +#define HEAP_ALIGNMENT 63 caddr_t heap_begin; caddr_t heap_end; @@ -47,28 +47,26 @@ void _init_c_runtime() extern char _end; /// init backtrace functionality - extern void _move_elf_symbols(void*, void*); - extern void _apply_parser_data(void*); - extern int _get_elf_section_size(const void*); - // first measure the size of symbols - int symsize = _get_elf_section_size(&_ELF_SYM_START_); - // estimate somewhere in heap its safe to move them - char* SYM_LOCATION = &_end + 2 * (4096 + symsize); - _move_elf_symbols(&_ELF_SYM_START_, SYM_LOCATION); + extern int _get_elf_section_datasize(const void*); + extern void _move_elf_syms_location(const void*, void*); + // store symbols temporarily in a safe location + char* sym_temp = &_end + + 2*4096 + _get_elf_section_datasize(&_ELF_SYM_START_); + _move_elf_syms_location(&_ELF_SYM_START_, sym_temp); // Initialize .bss section extern char _BSS_START_, _BSS_END_; streamset8(&_BSS_START_, 0, &_BSS_END_ - &_BSS_START_); // Initialize the heap before exceptions - heap_begin = &_end + 0xfff; - // page-align heap, because its not aligned - heap_begin = (char*) ((uintptr_t)heap_begin & ~(uintptr_t) 0xfff); + // cache-align heap, because its not aligned + heap_begin = &_end + 64 + HEAP_ALIGNMENT; + heap_begin = (char*) ((uintptr_t)heap_begin & ~(uintptr_t) HEAP_ALIGNMENT); // heap end tracking, used with sbrk heap_end = heap_begin; - // validate that heap is page aligned + // validate that heap is aligned int validate_heap_alignment = - ((uintptr_t)heap_begin & (uintptr_t) 0xfff) == 0; + ((uintptr_t)heap_begin & (uintptr_t) HEAP_ALIGNMENT) == 0; /// initialize newlib I/O _REENT_INIT_PTR(_REENT); @@ -77,9 +75,13 @@ void _init_c_runtime() stdout = _REENT->_stdout; // stdout == 2 stderr = _REENT->_stderr; // stderr == 3 - // move symbols (again) to heap, before calling global constructors - extern void* _relocate_to_heap(void*); - void* symheap = _relocate_to_heap(SYM_LOCATION); + /// init ELF / backtrace functionality + extern void _elf_relocate_to_heap(); + extern void _init_elf_parser(); + // move ELF symbols into heap + _elf_relocate_to_heap(); + // enable ELF symbols here (before global constructors) + _init_elf_parser(); /// initialize exceptions before we can run constructors extern char __eh_frame_start[]; @@ -91,23 +93,18 @@ void _init_c_runtime() extern void _init(); _init(); - // set ELF symbols location here (after initializing everything else) - _apply_parser_data(symheap); - // sanity checks assert(heap_begin >= &_end); assert(heap_end >= heap_begin); assert(validate_heap_alignment); } -// global/static objects should never be destructed here, so ignore this -void* __dso_handle; - // stack-protector __attribute__((noreturn)) void __stack_chk_fail(void) { panic("Stack protector: Canary modified"); + __builtin_unreachable(); } // old function result system @@ -117,6 +114,17 @@ int* __errno_location(void) return &errno; } +#include +int _setjmp(jmp_buf env) +{ + return setjmp(env); +} +// linux strchr variant (NOTE: not completely the same!) +void *__rawmemchr (const void *s, int c) +{ + return strchr((const char*) s, c); +} + /// assert() interface of ISO POSIX (2003) void __assert_fail(const char * assertion, const char * file, unsigned int line, const char * function) { diff --git a/src/hw/pci_device.cpp b/src/hw/pci_device.cpp index 6cec875c65..d022e78493 100644 --- a/src/hw/pci_device.cpp +++ b/src/hw/pci_device.cpp @@ -38,6 +38,10 @@ #define PCI_HEADER_REG 0x0e #define PCI_BIST_REG 0x0f +#define PCI_COMMAND_IO 0x01 +#define PCI_COMMAND_MEM 0x02 +#define PCI_COMMAND_MASTER 0x04 + namespace hw { static const char* classcodes[] { @@ -150,6 +154,11 @@ namespace hw { PCI_Device::PCI_Device(const uint16_t pci_addr, const uint32_t device_id) : pci_addr_{pci_addr}, device_id_{device_id} { + // set master, mem and io flags + uint32_t cmd = read_dword(PCI_CMD_REG); + cmd |= PCI_COMMAND_MASTER | PCI_COMMAND_MEM | PCI_COMMAND_IO; + write_dword(PCI_CMD_REG, cmd); + //We have device, so probe for details devtype_.reg = read_dword(pci_addr, PCI::CONFIG_CLASS_REV); diff --git a/src/kernel/elf.cpp b/src/kernel/elf.cpp index 2f217d2a42..b252645129 100644 --- a/src/kernel/elf.cpp +++ b/src/kernel/elf.cpp @@ -74,10 +74,10 @@ class ElfTables func_offset getsym(Elf32_Addr addr) { // probably just a null pointer with ofs=addr - if (UNLIKELY(addr < 0x7c00)) + if (UNLIKELY(addr < 0x1000)) return {null_stringz, 0, addr}; // definitely in the bootloader - if (UNLIKELY(addr < 0x7e00)) + if (UNLIKELY(addr >= 0x7c00 && addr < 0x7e00)) return {boot_stringz, 0x7c00, addr - 0x7c00}; // resolve manually from symtab auto* sym = getaddr(addr); @@ -87,7 +87,11 @@ class ElfTables auto base = sym->st_value; auto offset = addr - base; // return string name for symbol - return {demangle( sym_name(sym) ), base, offset}; + const char* name = sym_name(sym); + if (name) + return {demangle(name), base, offset}; + else + return {to_hex_string(base), base, offset}; } // function or space not found return {to_hex_string(addr), addr, 0}; @@ -95,9 +99,11 @@ class ElfTables safe_func_offset getsym_safe(Elf32_Addr addr, char* buffer, size_t length) { // probably just a null pointer with ofs=addr - if (addr < 0x7c00) return {null_stringz, 0, addr}; + if (UNLIKELY(addr < 0x1000)) + return {null_stringz, 0, addr}; // definitely in the bootloader - if (addr < 0x7e00) return {boot_stringz, 0x7c00, addr - 0x7c00}; + if (UNLIKELY(addr >= 0x7c00 && addr < 0x7e00)) + return {boot_stringz, 0x7c00, addr - 0x7c00}; // resolve manually from symtab auto* sym = getaddr(addr); if (LIKELY(sym)) { @@ -153,17 +159,13 @@ class ElfTables } std::string demangle(const char* name) const { - if (name[0] == '_') { - int status; - // internally, demangle just returns buf when status is ok - auto* res = __cxa_demangle(name, nullptr, 0, &status); - if (status == 0) { - std::string result(res); - std::free(res); - return result; - } - } - return std::string(name); + char buffer[2048]; + const char* res = demangle_safe(name, buffer, sizeof(buffer)); + assert(strlen(res)); + if (res) + return std::string(res); + else + return std::string(name); } const char* demangle_safe(const char* name, char* buffer, size_t buflen) const { @@ -256,8 +258,8 @@ std::vector Elf::get_functions() void print_backtrace() { - char _symbol_buffer[512]; - char _btrace_buffer[512]; + char _symbol_buffer[1024]; + char _btrace_buffer[1024]; if (Elf::get_strtab() == NULL) { int len = snprintf(_btrace_buffer, sizeof(_btrace_buffer), @@ -343,59 +345,67 @@ void _validate_elf_symbols() } } -struct relocate_header { - SymTab symtab; - StrTab strtab; +static struct relocated_header { + uint32_t entries = 0xFFFF; + uint32_t strsize = 0xFFFF; + Elf32_Sym* syms = nullptr; + + const char* strings() const { + return (char*) &syms[entries]; + } +} relocs; +struct elfsyms_header { + uint32_t symtab_entries; + uint32_t strtab_size; + Elf32_Sym syms[0]; }; +#include extern "C" -int _get_elf_section_size(const void* location) +int _get_elf_section_datasize(const void* location) { - auto& hdr = *(relocate_header*) location; - return sizeof(relocate_header) + hdr.symtab.entries * sizeof(Elf32_Sym) + hdr.strtab.size; + auto& hdr = *(elfsyms_header*) location; + // special case for missing call to elf_syms + if (hdr.symtab_entries == 0) return 0; + return hdr.symtab_entries * sizeof(Elf32_Sym) + hdr.strtab_size; } -#include extern "C" -void _move_elf_symbols(void* old_location, void* new_location) +void _move_elf_syms_location(const void* location, void* new_location) { - int size = _get_elf_section_size(old_location); - // validate locations - if ((char*) new_location >= (char*) old_location && - (char*) new_location + size < (char*) old_location) - { - kprintf("ELF symbol sections are inside each other!\n"); - kprintf("Moving %d from %p -> %p (%p)\n", - size, old_location, new_location, (char*) new_location + size); - assert(0); + int size = _get_elf_section_datasize(location); + // stripped variant + if (size == 0) { + relocs.entries = 0; + relocs.strsize = 0; + return; } - // copy over - memcpy(new_location, old_location, size); + // old header + auto* hdr = (elfsyms_header*) location; // update header - auto* newhdr = (relocate_header*) new_location; - const char* base = ((char*) newhdr) + sizeof(relocate_header); - newhdr->symtab.base = (Elf32_Sym*) base; - newhdr->strtab.base = &base[newhdr->symtab.entries * sizeof(Elf32_Sym)]; + relocs.entries = hdr->symtab_entries; + relocs.strsize = hdr->strtab_size; + relocs.syms = (Elf32_Sym*) new_location; + // copy symbol and string data + memcpy(relocs.syms, hdr->syms, size); } -#if !defined(__MACH__) -#include -#endif extern "C" -void* _relocate_to_heap(void* old_location) +void _elf_relocate_to_heap() { - int total = _get_elf_section_size(old_location); - void* heap_location = malloc(total); - _move_elf_symbols(old_location, heap_location); - return heap_location; + int size = relocs.entries * sizeof(Elf32_Sym) + relocs.strsize; + char* newloc = new char[size]; + // copy syms & update location + memcpy(newloc, relocs.syms, size); + relocs.syms = (Elf32_Sym*) newloc; } +#include extern "C" -void _apply_parser_data(char* location) +void _init_elf_parser() { - if (location) { - auto& hdr = *(relocate_header*) location; + if (relocs.entries) { // apply changes to the symbol parser from custom location - parser.set(hdr.symtab.base, hdr.symtab.entries, hdr.strtab.base); + parser.set(relocs.syms, relocs.entries, relocs.strings()); } else { // symbols and strings are stripped out diff --git a/src/net/http/client.cpp b/src/net/http/client.cpp index 785a4e3f6d..90b93a008e 100644 --- a/src/net/http/client.cpp +++ b/src/net/http/client.cpp @@ -1,6 +1,6 @@ // This file is a part of the IncludeOS unikernel - www.includeos.org // -// Copyright 2016 Oslo and Akershus University College of Applied Sciences +// Copyright 2016-2017 Oslo and Akershus University College of Applied Sciences // and Alfred Bratterud // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -197,7 +197,7 @@ namespace http { stack.resolve(host, cb); } - Connection& Client::get_connection(const Host host) + Client_connection& Client::get_connection(const Host host) { // return/create a set for the given host auto& cset = conns_[host]; @@ -210,17 +210,17 @@ namespace http { } // no non-occupied connections, emplace a new one - cset.push_back(std::make_unique(tcp_.connect(host), Connection::Close_handler{this, &Client::close})); + cset.push_back(std::make_unique(*this, tcp_.connect(host))); return *cset.back(); } - void Client::close(Connection& c) + void Client::close(Client_connection& c) { debug(" Closing %u:%s %p\n", c.local_port(), c.peer().to_string().c_str(), &c); auto& cset = conns_.at(c.peer()); cset.erase(std::remove_if(cset.begin(), cset.end(), - [port = c.local_port()] (const std::unique_ptr& conn)->bool + [port = c.local_port()] (const std::unique_ptr& conn)->bool { return conn->local_port() == port; })); diff --git a/src/net/http/connection.cpp b/src/net/http/client_connection.cpp similarity index 76% rename from src/net/http/connection.cpp rename to src/net/http/client_connection.cpp index 61efc9bcdb..de6bb172e4 100644 --- a/src/net/http/connection.cpp +++ b/src/net/http/client_connection.cpp @@ -1,6 +1,6 @@ // This file is a part of the IncludeOS unikernel - www.includeos.org // -// Copyright 2016 Oslo and Akershus University College of Applied Sciences +// Copyright 2017 Oslo and Akershus University College of Applied Sciences // and Alfred Bratterud // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,33 +15,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include -#include +#include +#include + #include namespace http { - Connection::Connection(TCP_conn_ptr tcpconn, Close_handler on_close) - : tcpconn_{std::move(tcpconn)}, - req_{nullptr}, - res_{nullptr}, - on_close_{std::move(on_close)}, + Client_connection::Client_connection(Client& client, TCP_conn tcpconn) + : Connection{std::move(tcpconn)}, + client_(client), on_response_{nullptr}, - timer_({this, &Connection::timeout_request}), - timeout_dur_{timeout_duration::zero()}, - keep_alive_{true} + timer_({this, &Client_connection::timeout_request}), + timeout_dur_{timeout_duration::zero()} { - debug(" Created %u -> %s %p\n", local_port(), peer().to_string().c_str(), this); // setup close event - tcpconn_->on_close({this, &Connection::close}); - } - template - Connection::Connection(TCP& tcp, Peer addr, Close_handler on_close) - : Connection(tcp.connect(addr), std::move(on_close)) - { + tcpconn_->on_close({this, &Client_connection::close}); } - void Connection::send(Request_ptr req, Response_handler on_res, const size_t bufsize, timeout_duration timeout) + void Client_connection::send(Request_ptr req, Response_handler on_res, const size_t bufsize, timeout_duration timeout) { Expects(available()); req_ = std::move(req); @@ -56,16 +48,16 @@ namespace http { send_request(bufsize); } - void Connection::send_request(const size_t bufsize) + void Client_connection::send_request(const size_t bufsize) { keep_alive_ = (req_->header().value(header::Connection) != "close"); - tcpconn_->on_read(bufsize, {this, &Connection::recv_response}); + tcpconn_->on_read(bufsize, {this, &Client_connection::recv_response}); tcpconn_->write(req_->to_string()); } - void Connection::recv_response(buffer_t buf, size_t len) + void Client_connection::recv_response(buffer_t buf, size_t len) { if(len == 0) { end_response({Error::NO_REPLY}); @@ -138,7 +130,7 @@ namespace http { } } - void Connection::end_response(Error err) + void Client_connection::end_response(Error err) { // move response to a copy in case of callback result in new request Ensures(on_response_); @@ -158,7 +150,7 @@ namespace http { tcpconn_->close(); } - void Connection::close() + void Client_connection::close() { // if the user already if(on_response_ != nullptr) @@ -169,7 +161,7 @@ namespace http { callback(Error::CLOSING, std::move(res_)); } - on_close_(*this); + client_.close(*this); } } diff --git a/src/posix/dlfcn.cpp b/src/posix/dlfcn.cpp new file mode 100644 index 0000000000..cb5f1a7b61 --- /dev/null +++ b/src/posix/dlfcn.cpp @@ -0,0 +1,40 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +void* dlopen(const char *filename, int flag) +{ + printf("dlopen called for %s with flag %#x\n", filename, flag); + return nullptr; +} +char* dlerror(void) +{ + printf("dlerror\n"); + return nullptr; +} +void* dlsym(void*, const char* symbol) +{ + printf("dlsym on %s\n", symbol); + return nullptr; +} +int dlclose(void*) +{ + printf("dlclose\n"); + return 0; +} diff --git a/src/posix/pwd.cpp b/src/posix/pwd.cpp new file mode 100644 index 0000000000..d64f97f261 --- /dev/null +++ b/src/posix/pwd.cpp @@ -0,0 +1,143 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include + +const char* pwd_path = "/etc/passwd"; + +/* Applications are not allowed to modify the structure to which the return + values point, nor any storage areas pointed to by pointers within the + structure. Returned pointers might be invalidated or the structure/ + storage areas might be overwritten by subsequent calls. */ +static struct passwd ret; + +class Pwd { +public: + static Pwd& instance() { + static Pwd pwd; + return pwd; + } + + void rewind() {pos_ = 0;} + + void close() {open_ = false;} + + struct passwd* find(const char* name) { + Expects(name != nullptr); + if (!open_) { + read(); + } + const auto it = std::find_if(begin(entries_), end(entries_), [name](const auto& entry){ + return (strcmp(name, entry.c_str()) == 0); + }); + if (it == end(entries_)) { + return nullptr; + } + extract(*it); + return &ret; + } + + struct passwd* next() noexcept { + if (!open_) { + read(); + } + if (pos_ >= entries_.size()) { + return nullptr; + } + try { + const auto& ent = entries_[pos_]; + int field_seps = std::count(std::begin(ent), std::end(ent), '\0'); + if (field_seps != 6) { + INFO("pwd", "not a valid passwd file entry"); + return nullptr; + } + extract(ent); + ++pos_; + return &ret; + } + catch (...) { + INFO("pwd", "error parsing pwd entry"); + return nullptr; + } + } + + void extract(const std::string& ent) { + ret.pw_name = const_cast(ent.c_str()); + size_t pw_pos = ent.find('\0') + 1; + size_t uid_pos = ent.find('\0', pw_pos) + 1; + size_t gid_pos = ent.find('\0', uid_pos) + 1; + size_t info_pos = ent.find('\0', gid_pos) + 1; + size_t dir_pos = ent.find('\0', info_pos) + 1; + size_t shell_pos = ent.find('\0', dir_pos) + 1; + ret.pw_uid = std::stol(ret.pw_name + uid_pos); + ret.pw_gid = std::stol(ret.pw_name + gid_pos); + ret.pw_dir = ret.pw_name + dir_pos; + ret.pw_shell = ret.pw_name + shell_pos; + } + + void print() { + printf(" instance @%p\n", this); + printf(" open: %d\n", open_); + printf(" pos: %d\n", pos_); + printf(" entries: %d\n", entries_.size()); + } + +private: + Pwd() : open_ {false}, pos_ {0} {} + + void read() { + std::ifstream is(pwd_path); + std::string line; + while (std::getline(is, line)) { + std::replace(std::begin(line), std::end(line), ':', '\0'); + entries_.push_back(line); + } + rewind(); + open_ = true; + } + bool open_; + size_t pos_; + std::vector entries_; +}; + +void endpwent(void) { + Pwd& pwd = Pwd::instance(); + pwd.close(); +} + +void setpwent(void) { + Pwd& pwd = Pwd::instance(); + pwd.rewind(); +} + +struct passwd *getpwent(void) { + Pwd& pwd = Pwd::instance(); + return pwd.next(); +} + +struct passwd *getpwnam(const char *name) { + if (name == nullptr) { + return nullptr; + } + Pwd& pwd = Pwd::instance(); + return pwd.find(name); +} diff --git a/src/posix/sys/mman.cpp b/src/posix/sys/mman.cpp index 12170358f8..6293ae5021 100644 --- a/src/posix/sys/mman.cpp +++ b/src/posix/sys/mman.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -30,7 +31,7 @@ void* mmap(void* addr, size_t length, return MAP_FAILED; } - // TODO: + // TODO: // validate fd is open file // associate some VA space with open file @fd @@ -48,7 +49,7 @@ void* mmap(void* addr, size_t length, // TODO: // read entire file into entry space // deallocate entry space on failure - + // create the mapping _mmap_entries[addr] = entry; return entry.addr; @@ -70,3 +71,9 @@ int munmap(void* addr, size_t length) errno = EINVAL; return -1; } + +int mprotect(void *addr, size_t len, int prot) +{ + printf("mprotect %p:%u with flags %#x\n", addr, len, prot); + return 0; +} diff --git a/src/posix/tcp_fd.cpp b/src/posix/tcp_fd.cpp index 656177581d..6254fe5923 100644 --- a/src/posix/tcp_fd.cpp +++ b/src/posix/tcp_fd.cpp @@ -249,7 +249,7 @@ ssize_t TCP_FD_Conn::send(const void* data, size_t len, int) len, [&written] (bool) { written = true; } ); - + // sometimes we can just write and forget if (written) return len; while (!written) { @@ -305,14 +305,14 @@ int TCP_FD_Conn::shutdown(int mode) return -1; } switch (mode) { - case SHUT_RDRW: + case SHUT_RDWR: conn->close(); return 0; case SHUT_RD: printf("Ignoring shutdown(SHUT_RD)\n"); return 0; - case SHUT_RW: - printf("Ignoring shutdown(SHUT_RW)\n"); + case SHUT_WR: + printf("Ignoring shutdown(SHUT_WR)\n"); return 0; default: errno = EINVAL; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c6fccf929d..3633c509f3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -14,7 +14,6 @@ set(TEST ${INCLUDEOS_ROOT}/test) include_directories( ${INCLUDEOS_ROOT}/api - ${INCLUDEOS_ROOT}/api/posix ${INCLUDEOS_ROOT}/src/ ${INCLUDEOS_ROOT}/src/include ${INCLUDEOS_ROOT}/mod/ @@ -119,7 +118,7 @@ set(OS_SOURCES ${SRC}/net/dns/dns.cpp ${SRC}/net/ethernet/ethernet.cpp ${SRC}/net/http/client.cpp - ${SRC}/net/http/connection.cpp + ${SRC}/net/http/client_connection.cpp ${SRC}/net/http/cookie.cpp ${SRC}/net/http/header.cpp ${SRC}/net/http/header_fields.cpp diff --git a/test/net/integration/http/service.cpp b/test/net/integration/http/service.cpp index 75f2d12db0..7c6c9371fe 100644 --- a/test/net/integration/http/service.cpp +++ b/test/net/integration/http/service.cpp @@ -48,6 +48,9 @@ void Service::ready() client_->send(std::move(req), {inet.gateway(), 9011}, [] (auto err, auto res) { + if (err) + printf("Error: %s \n", err.to_string().c_str()); + CHECKSERT(!err, "No error"); CHECKSERT(res->body() == "/testing", "Received body: \"/testing\""); printf("Response:\n%s\n", res->to_string().c_str()); diff --git a/test/net/integration/http/test.py b/test/net/integration/http/test.py index f83b095155..9cf35ac4f6 100755 --- a/test/net/integration/http/test.py +++ b/test/net/integration/http/test.py @@ -2,6 +2,7 @@ import sys import os +import thread includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) @@ -23,7 +24,7 @@ def do_GET(s): s.wfile.write("%s" % s.path) -def Client_test(trigger_line): +def Client_test(): server_class = BaseHTTPServer.HTTPServer httpd = server_class((HOST, PORT), RequestHandler) global DO_SERVE @@ -32,11 +33,11 @@ def Client_test(trigger_line): DO_SERVE = False httpd.server_close() +# Start web server in a separate thread +thread.start_new_thread(Client_test, ()) + # Get an auto-created VM from the vmrunner vm = vmrunner.vms[0] -# Add custom event-handler -vm.on_output("Testing against local server", Client_test) - # Boot the VM, taking a timeout as parameter vm.cmake().boot(20).clean() diff --git a/test/posix/integration/conf/CMakeLists.txt b/test/posix/integration/conf/CMakeLists.txt index aba6a2501d..6f194393e6 100644 --- a/test/posix/integration/conf/CMakeLists.txt +++ b/test/posix/integration/conf/CMakeLists.txt @@ -24,6 +24,7 @@ set(SOURCES service.cpp test_sysconf.c test_pathconf.c + test_pwd.c ) # DRIVERS / PLUGINS: diff --git a/test/posix/integration/conf/disk/passwd b/test/posix/integration/conf/disk/passwd index 81e656055f..94dd03764a 100644 --- a/test/posix/integration/conf/disk/passwd +++ b/test/posix/integration/conf/disk/passwd @@ -1 +1,5 @@ +nobody:*:-2:-2:Unprivileged User:/var/empty:/usr/bin/false root:*:0:0:System Administrator:/var/root:/bin/sh +daemon:*:1:1:System Services:/var/root:/usr/bin/false +_ftp:*:98:-2:FTP Daemon:/var/empty:/usr/bin/false +_launchservicesd:*:239:239:_launchservicesd:/var/empty:/usr/bin/false diff --git a/test/posix/integration/conf/service.cpp b/test/posix/integration/conf/service.cpp index 425409bff4..749f72bd42 100644 --- a/test/posix/integration/conf/service.cpp +++ b/test/posix/integration/conf/service.cpp @@ -27,6 +27,7 @@ extern "C" void test_sysconf(); extern "C" void test_pathconf(); +extern "C" void test_pwd(); fs::Disk_ptr& memdisk() { static auto disk = fs::new_shared_memdisk(); @@ -74,5 +75,8 @@ int main(int argc, char *argv[]) { value = getenv("INCLUDEOS_CORE_DUMP"); printf("INCLUDEOS_CORE_DUMP: %s\n", value); + // test pwd-related functions + test_pwd(); + INFO("Conf", "SUCCESS"); } diff --git a/test/posix/integration/conf/test_pwd.c b/test/posix/integration/conf/test_pwd.c new file mode 100644 index 0000000000..228fb095cb --- /dev/null +++ b/test/posix/integration/conf/test_pwd.c @@ -0,0 +1,53 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +void print_pwd_entry(struct passwd* pwd_ent); + +void test_pwd() { + struct passwd* pwd_ent; + + while((pwd_ent = getpwent()) != NULL) { + print_pwd_entry(pwd_ent); + } + + printf("Back to start!\n"); + setpwent(); + pwd_ent = getpwent(); + print_pwd_entry(pwd_ent); + endpwent(); + + pwd_ent = getpwnam("root"); + if (pwd_ent != NULL) { + print_pwd_entry(pwd_ent); + } + + pwd_ent = getpwnam("alfred"); + if (pwd_ent != NULL) { + print_pwd_entry(pwd_ent); + } +} + +void print_pwd_entry(struct passwd* pwd_ent) { + printf("Name: %s\n", pwd_ent->pw_name); + printf("Uid: %d\n", pwd_ent->pw_uid); + printf("Gid: %d\n", pwd_ent->pw_gid); + printf("Path: %s\n", pwd_ent->pw_dir); + printf("Shell: %s\n\n", pwd_ent->pw_shell); +} diff --git a/test/posix/unit/inet_test.cpp b/test/posix/unit/inet_test.cpp index 55146eb1e6..80d25d4155 100644 --- a/test/posix/unit/inet_test.cpp +++ b/test/posix/unit/inet_test.cpp @@ -27,7 +27,7 @@ CASE("IPv4 address manipulation - str to addr") in_addr_t addr; addr = inet_addr(TEST_ADDRSTR); - EXPECT(::ntohl(addr) == TEST_ADDR); + EXPECT(ntohl(addr) == TEST_ADDR); addr = inet_addr("Invalid.Address"); EXPECT(addr == -1); @@ -35,7 +35,7 @@ CASE("IPv4 address manipulation - str to addr") CASE("IPv4 address manipulation - addr to str") { struct in_addr addr; - addr.s_addr = ::htonl(TEST_ADDR); + addr.s_addr = htonl(TEST_ADDR); char* addrstr = inet_ntoa(addr); EXPECT(strcmp(addrstr, TEST_ADDRSTR) == 0); @@ -43,7 +43,7 @@ CASE("IPv4 address manipulation - addr to str") CASE("IPv4 address manipulation - addr to pointer") { struct in_addr addr; - addr.s_addr = ::htonl(TEST_ADDR); + addr.s_addr = htonl(TEST_ADDR); char addrstr[INET_ADDRSTRLEN]; diff --git a/test/testrunner.py b/test/testrunner.py index 663070a39c..4bcc383832 100755 --- a/test/testrunner.py +++ b/test/testrunner.py @@ -14,7 +14,7 @@ from vmrunner.prettify import color as pretty from vmrunner import validate_vm -import validate_all +import validate_tests startdir = os.getcwd() @@ -54,10 +54,7 @@ def print_skipped(tests): for test in tests: if test.skip_: print pretty.WARNING("* Skipping " + test.name_) - if "validate_vm" in test.skip_reason_: - validate_vm.validate_path(test.path_, verb = True) - else: - print " Reason: {0:40}".format(test.skip_reason_) + print "Reason: {0:40}".format(test.skip_reason_) class Test: @@ -179,9 +176,10 @@ def check_valid(self): self: Class function """ # Test 1 - if not validate_vm.validate_path(self.path_, verb = False): + valid, err = validate_tests.validate_test(self.path_, verb = False) + if not valid: self.skip_ = True - self.skip_reason_ = 'Failed validate_vm, missing files' + self.skip_reason_ = err return # Test 2 @@ -212,7 +210,7 @@ def stress_test(): print pretty.WARNING("Stress test skipped") return 0 - if (not validate_vm.validate_path("stress")): + if (not validate_tests.validate_test("stress")): raise Exception("Stress test failed validation") print pretty.HEADER("Starting stress test") @@ -383,7 +381,7 @@ def filter_tests(all_tests, arguments): or x.name_ in add_args ] # 2) Remove tests defined by the skip argument - print pretty.INFO("Tests to skip"), ", ".join(skip_args) + print pretty.INFO("Tests marked skip on command line"), ", ".join(skip_args) skipped_tests = [ x for x in tests_added if x.type_ in skip_args or x.category_ in skip_args diff --git a/test/validate_all.py b/test/validate_all.py deleted file mode 100755 index cb1cf1168e..0000000000 --- a/test/validate_all.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python -import sys -sys.path.insert(0, ".") -sys.path.insert(0, "..") - -import subprocess -import os -from vmrunner import validate_vm -from vmrunner.prettify import color - -def valid_tests(verb = False): - tests = [] - - dirs = os.walk('.').next()[1] - for directory in dirs: - subdirs = os.walk(directory).next()[1] - if "integration" in subdirs: - subdirs = os.walk(directory + "/integration").next()[1] - if subdirs: - for d in subdirs: - path = directory + "/integration/" + d - if validate_test.validate_path(path, verb): - tests.append(path) - else: - print color.WARNING("Validator: " + path + " failed validation") - - return tests - -if __name__ == "__main__": - valid_tests(verb = True) diff --git a/test/validate_tests.py b/test/validate_tests.py new file mode 100755 index 0000000000..c6f4248968 --- /dev/null +++ b/test/validate_tests.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +import sys +sys.path.insert(0, ".") +sys.path.insert(0, "..") + +import subprocess +import os +from vmrunner import validate_vm +from vmrunner.prettify import color + +# Verify if a given path is a valid integration test +def validate_test(path, verb = False): + + try: + # Load any valid configs in path + valid_configs = validate_vm.load_config(path) + if not valid_configs: + raise Exception("No valid JSON config in path") + + # Additional requirements for tests + required_files = ["CMakeLists.txt", "test.py"] + + for f in required_files: + if not os.path.isfile(path + "/" + f): + raise Exception("Required file " + f + " missing") + + if verb: + print " \tPASS: ",path + return True, "PASS" + + except Exception as err: + if verb: + print " \tFAIL: ", path, ": " , err.message + + return False, err.message + + +def valid_tests(verb = False): + tests = [] + + dirs = os.walk('.').next()[1] + for directory in dirs: + subdirs = os.walk(directory).next()[1] + if "integration" in subdirs: + subdirs = os.walk(directory + "/integration").next()[1] + if subdirs: + for d in subdirs: + path = directory + "/integration/" + d + passed, err = validate_test(path, verb) + if passed: + tests.append(path) + + return tests + +if __name__ == "__main__": + valid_tests(verb = True) diff --git a/vmbuild/CMakeLists.txt b/vmbuild/CMakeLists.txt index 554b8d2375..0c43ca8f38 100644 --- a/vmbuild/CMakeLists.txt +++ b/vmbuild/CMakeLists.txt @@ -9,7 +9,7 @@ set(ELF_SYMS_SOURCES elf_syms.cpp) set(CMAKE_CXX_FLAGS "-std=c++14 -Wall -Wextra -O3") # TODO: write scripts that automatically find include directories -include_directories(. ./../api ./../mod/GSL/) +include_directories(. ./../mod/GSL/) add_executable(vmbuild ${SOURCES}) add_executable(elf_syms ${ELF_SYMS_SOURCES}) diff --git a/vmbuild/elf_syms.cpp b/vmbuild/elf_syms.cpp index bda13fc2f4..6cd5d08f92 100644 --- a/vmbuild/elf_syms.cpp +++ b/vmbuild/elf_syms.cpp @@ -15,9 +15,8 @@ static const char* elf_offset(int o) noexcept { return (char*)elf_header_location + o; } -static void prune_elf_symbols(); -static char* pruned_location; -static int pruned_size = 0; +static int prune_elf_symbols(); +static char* pruned_location = nullptr; int main(int argc, const char** args) { @@ -38,7 +37,7 @@ int main(int argc, const char** args) // validate symbols elf_header_location = (decltype(elf_header_location)) fdata; - prune_elf_symbols(); + int pruned_size = prune_elf_symbols(); // write symbols to binary file f = fopen("_elf_symbols.bin", "w"); @@ -58,16 +57,11 @@ struct StrTab { StrTab(char* base, uint32_t size) : base(base), size(size) {} }; -struct relocate_header { - SymTab symtab; - StrTab strtab; -}; struct relocate_header32 { - uint32_t symtab_base; - uint32_t symtab_entries; - uint32_t strtab_base; - uint32_t strtab_size; + uint32_t symtab_entries; + uint32_t strtab_size; + Elf32_Sym syms[0]; }; static int relocate_pruned_sections(char* new_location, SymTab& symtab, StrTab& strtab) @@ -75,7 +69,7 @@ static int relocate_pruned_sections(char* new_location, SymTab& symtab, StrTab& auto& hdr = *(relocate_header32*) new_location; // first prune symbols - auto* symloc = (Elf32_Sym*) (new_location + sizeof(relocate_header32)); + auto* symloc = hdr.syms; size_t symidx = 0; for (size_t i = 0; i < symtab.entries; i++) { @@ -110,7 +104,7 @@ static int relocate_pruned_sections(char* new_location, SymTab& symtab, StrTab& hdr.strtab_size * sizeof(char); } -static void prune_elf_symbols() +static int prune_elf_symbols() { SymTab symtab { nullptr, 0 }; std::vector strtabs; @@ -144,10 +138,9 @@ static void prune_elf_symbols() // allocate worst case, guaranteeing we have enough space pruned_location = new char[sizeof(relocate_header32) + symtab.entries * sizeof(Elf32_Sym) + strtab.size]; - pruned_size = relocate_pruned_sections(pruned_location, symtab, strtab); - return; + return relocate_pruned_sections(pruned_location, symtab, strtab); } // stripped variant pruned_location = new char[sizeof(relocate_header32)]; - pruned_size = sizeof(relocate_header32); + return sizeof(relocate_header32); } diff --git a/vmbuild/vmbuild.cpp b/vmbuild/vmbuild.cpp index 4453c94f4f..dc21e27aa0 100644 --- a/vmbuild/vmbuild.cpp +++ b/vmbuild/vmbuild.cpp @@ -25,7 +25,7 @@ #include #include -#include +#include "../api/boot/multiboot.h" #include #include "elf.h" #include "elf_binary.hpp" diff --git a/vmrunner/prettify.py b/vmrunner/prettify.py index 5d34a61eb6..b53bf5333a 100644 --- a/vmrunner/prettify.py +++ b/vmrunner/prettify.py @@ -47,7 +47,7 @@ def code(fg = WHITE, bg = BLACK, style = NORMAL, fg_intensity = FG_NORMAL, bg_in @staticmethod def WARNING(string): - return color.C_WARNING + "[ WARNING ] " + string.rstrip() + color.C_ENDC + "\n" + return color.C_WARNING + "[ WARNING ] " + string.rstrip() + color.C_ENDC @staticmethod def FAIL(string): diff --git a/vmrunner/validate_vm.py b/vmrunner/validate_vm.py index 5ea0f113c6..96616bd043 100755 --- a/vmrunner/validate_vm.py +++ b/vmrunner/validate_vm.py @@ -35,69 +35,54 @@ def set_defaults(validator, properties, instance, schema): validator = extend_with_default(Draft4Validator) package_path = os.path.dirname(os.path.realpath(__file__)) +default_schema = package_path + "/vm.schema.json" - -def load_schema(filename): +def load_schema(filename = default_schema): global vm_schema vm_schema = json.loads(open(filename).read()); + def validate_vm_spec(filename): + vm_spec = None - global valid_vms - vm_spec = None + # Load and parse as JSON + try: + vm_spec = json.loads(open(filename).read()) + except: + raise Exception("JSON load / parse Error for " + filename) - # Load and parse as JSON - try: - vm_spec = json.loads(open(filename).read()) - except: - raise Exception("JSON load / parse Error for " + filename) + if (not vm_schema): load_schema() - # Validate JSON according to schema - try: + # Validate JSON according to schema validator(vm_schema).validate(vm_spec) - except Exception as err: - raise Exception("JSON schema validation failed: " + err.message) - - valid_vms.append(vm_spec) - - -def has_required_stuff(path): - - global jsons - - # Certain files are mandatory - required_files = [ "CMakeLists.txt"] - for file in required_files: - if not glob.glob(file): - raise Exception("missing " + file) - - # JSON-files must conform to VM-schema - jsons = glob.glob("*.json") - jsons.sort() - for json in jsons: - try: - validate_vm_spec(json) - except Exception as e: - pass - -def validate_path(path, verb = False): - global verbose - verbose = verb - current_dir = os.getcwd() - if not vm_schema: - load_schema(package_path + "/vm.schema.json") - os.chdir(path) - try: - has_required_stuff(path) - if verbose: - print " \tPASS: ",os.getcwd() - return True - except Exception as err: - if verbose: - print " \tFAIL: unmet requirements in " + path, ": " , err.message - finally: - os.chdir(current_dir) + + return vm_spec, filename + + +def load_config(path): + global valid_vms + global jsons + + if (os.path.isfile(path)): + jsons = [path] + + if (os.path.isdir(path)): + jsons = glob.glob(path + "/*.json") + jsons.sort() + + # JSON-files must conform to VM-schema + for json in jsons: + spec = validate_vm_spec(json) + try: + spec = validate_vm_spec(json) + valid_vms.append(spec) + except Exception as e: + pass + return valid_vms + if __name__ == "__main__": - path = sys.argv[1] if len(sys.argv) > 1 else "." - validate_path(path) + path = sys.argv[1] if len(sys.argv) > 1 else "." + if not load_config(path): + print "No valid config found" + exit(-1) diff --git a/vmrunner/vm.schema.json b/vmrunner/vm.schema.json index 8471b8988e..f6f6ad3dfa 100644 --- a/vmrunner/vm.schema.json +++ b/vmrunner/vm.schema.json @@ -4,11 +4,17 @@ "type" : "object", "properties" : { + "description" : { + "description" : "A human-readable description of this config", + "type" : "string" + }, + "image" : { "description" : "A bootable virtual machine image", "type" : "string", "default" : "service.img" }, + "bios" : { "description" : "64k BIOS image", "type" : "string" diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index d728605b8b..5d3099104a 100644 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -15,8 +15,6 @@ INCLUDEOS_HOME = None -nametag = "" - if "INCLUDEOS_PREFIX" not in os.environ: def_home = "/usr/local" print color.WARNING("WARNING:"), "Environment varialble INCLUDEOS_PREFIX is not set. Trying default", def_home @@ -27,6 +25,40 @@ package_path = os.path.dirname(os.path.realpath(__file__)) +default_config = {"description" : "Single virtio nic, otherwise hypervisor defaults", + "net" : [{"device" : "virtio", "backend" : "tap" }] } + + +nametag = "" +INFO = color.INFO(nametag) +VERB = bool(os.environ["VERBOSE"]) if "VERBOSE" in os.environ else False + +class Logger: + def __init__(self, tag): + self.tag = tag + if (VERB): + self.info = self.info_verb + else: + self.info = self.info_silent + + def __call__(self, *args): + self.info(args) + + def info_verb(self, args): + print self.tag, + for arg in args: + print arg, + print + + def info_silent(self, args): + pass + +# Define verbose printing function "info", with multiple args +default_logger = Logger(INFO) +def info(*args): + default_logger.info(args) + + # The end-of-transmission character EOT = chr(4) @@ -38,7 +70,9 @@ "CALLBACK_FAILED" : 68, "BUILD_FAIL" : 69, "ABORT" : 70, - "VM_EOT" : 71 } + "VM_EOT" : 71, + "BOOT_FAILED": 72 +} def get_exit_code_name (exit_code): for name, code in exit_codes.iteritems(): @@ -75,36 +109,39 @@ def abstract(): # (It seems to be recommended for "new style classes" to inherit object) class hypervisor(object): - def __init__(self, config): - self._config = config; + def __init__(self, config): + self._config = config; - # Boot a VM, returning a hypervisor handle for reuse - def boot(self): - abstract() + # Boot a VM, returning a hypervisor handle for reuse + def boot(self): + abstract() - # Stop the VM booted by boot - def stop(self): - abstract() + # Stop the VM booted by boot + def stop(self): + abstract() + + # Read a line of output from vm + def readline(self): + abstract() - # Read a line of output from vm - def readline(self): - abstract() + # Verify that the hypervisor is available + def available(self, config_data = None): + abstract() - # Verify that the hypervisor is available - def available(self, config_data = None): - abstract() + # Wait for this VM to exit + def wait(self): + abstract() - # Wait for this VM to exit - def wait(self): - abstract() + # Wait for this VM to exit + def poll(self): + abstract() - # Wait for this VM to exit - def poll(self): - abstract() + # A descriptive name + def name(self): + abstract() - # A descriptive name - def name(self): - abstract() + def image_name(self): + abstract() # Qemu Hypervisor interface @@ -115,34 +152,46 @@ def __init__(self, config): self._proc = None self._stopped = False self._sudo = False + self._image_name = self._config if "image" in self._config else self.name() + " vm" # Pretty printing - self._nametag = "<" + type(self).__name__ + ">" - self.INFO = color.INFO(self._nametag) - + self.info = Logger(color.INFO("<" + type(self).__name__ + ">")) def name(self): return "Qemu" - def drive_arg(self, filename, drive_type="virtio", drive_format="raw", media_type="disk"): - return ["-drive","file="+filename+",format="+drive_format+",if="+drive_type+",media="+media_type] + def image_name(serlf): + return self._image_name + + def drive_arg(self, filename, drive_type = "virtio", drive_format = "raw", media_type = "disk"): + return ["-drive","file=" + filename + + ",format=" + drive_format + + ",if=" + drive_type + + ",media=" + media_type] - def net_arg(self, backend, device, if_name = "net0", mac="c0:01:0a:00:00:2a"): - qemu_ifup = INCLUDEOS_HOME+"/includeos/scripts/qemu-ifup" - # FIXME: this needs to get removed + def net_arg(self, backend, device, if_name = "net0", mac = None): + qemu_ifup = INCLUDEOS_HOME + "/includeos/scripts/qemu-ifup" + + # FIXME: this needs to get removed, e.g. fetched from the schema names = {"virtio" : "virtio-net", "vmxnet" : "vmxnet3", "vmxnet3" : "vmxnet3"} - return ["-device", names[device]+",netdev="+if_name+",mac="+mac, - "-netdev", backend+",id="+if_name+",script="+qemu_ifup] + + device = names[device] + ",netdev=" + if_name + + # Add mac-address if specified + if mac: device += ",mac=" + mac + + return ["-device", device, + "-netdev", backend + ",id=" + if_name + ",script=" + qemu_ifup] def kvm_present(self): command = "egrep -m 1 '^flags.*(vmx|svm)' /proc/cpuinfo" try: subprocess.check_output(command, shell = True) - print self.INFO, "KVM ON" + self.info("KVM ON") return True except Exception as err: - print self.INFO, "KVM OFF" + self.info("KVM OFF") return False # Start a process and preserve in- and output pipes @@ -160,7 +209,7 @@ def start_process(self, cmdlist): stdout = subprocess.PIPE, stderr = subprocess.PIPE, stdin = subprocess.PIPE) - print self.INFO, "Started process PID ",self._proc.pid + self.info("Started process PID ",self._proc.pid) return self._proc @@ -177,6 +226,8 @@ def boot(self, multiboot, kernel_args = "", image_name = None): if not image_name: image_name = self._config["image"] + self._image_name = image_name + # multiboot - e.g. boot with '-kernel' and no bootloader if multiboot: @@ -186,11 +237,11 @@ def boot(self, multiboot, kernel_args = "", image_name = None): kernel_args = ["-kernel", image_name, "-append", kernel_args] disk_args = [] - print self.INFO, "Booting", image_name, "directly without bootloader (multiboot / -kernel args)" + info ( "Booting", image_name, "directly without bootloader (multiboot / -kernel args)") else: kernel_args = [] disk_args = self.drive_arg(image_name, "ide") - print self.INFO, "Booting", image_name, "with a bootable disk image" + info ("Booting", image_name, "with a bootable disk image") if "bios" in self._config: kernel_args.extend(["-bios", self._config["bios"]]) @@ -203,7 +254,8 @@ def boot(self, multiboot, kernel_args = "", image_name = None): i = 0 if "net" in self._config: for net in self._config["net"]: - net_args += self.net_arg(net["backend"], net["device"], "net"+str(i), net["mac"]) + mac = net["mac"] if "mac" in net else None + net_args += self.net_arg(net["backend"], net["device"], "net"+str(i), mac) i+=1 mem_arg = [] @@ -213,7 +265,7 @@ def boot(self, multiboot, kernel_args = "", image_name = None): vga_arg = ["-nographic" ] if "vga" in self._config: vga_arg = ["-vga", str(self._config["vga"])] - + # TODO: sudo is only required for tap networking and kvm. Check for those. command = ["sudo", "qemu-system-x86_64"] if self.kvm_present(): command.append("--enable-kvm") @@ -222,8 +274,7 @@ def boot(self, multiboot, kernel_args = "", image_name = None): command += disk_args + net_args + mem_arg + vga_arg - print self.INFO, "command:" - print color.DATA(" ".join(command)) + info("Command:", " ".join(command)) try: self.start_process(command) @@ -245,17 +296,17 @@ def stop(self): if self._proc and self._proc.poll() == None : if not self._sudo: - print self.INFO,"Stopping child process (no sudo required)" + info ("Stopping child process (no sudo required)") self._proc.terminate() else: # Find and terminate all child processes, since parent is "sudo" parent = psutil.Process(self._proc.pid) children = parent.children() - print self.INFO, "Stopping", self._config["image"], "PID",self._proc.pid, "with", signal + info ("Stopping", self._image_name, "PID",self._proc.pid, "with", signal) for child in children: - print self.INFO," + child process ", child.pid + info (" + child process ", child.pid) # The process might have gotten an exit status by now so check again to avoid negative exit if (not self._proc.poll()): @@ -299,18 +350,21 @@ def poll(self): # VM class class vm: - def __init__(self, config, hyper = qemu): + def __init__(self, config = None, hyper = qemu): + self._exit_status = 0 self._exit_msg = "" - self._config = config + self._config = load_single_config(config) self._on_success = lambda(line) : self.exit(exit_codes["SUCCESS"], nametag + " All tests passed") self._on_panic = self.panic self._on_timeout = self.timeout self._on_output = { "PANIC" : self._on_panic, "SUCCESS" : self._on_success } + + # Initialize hypervisor with config assert(issubclass(hyper, hypervisor)) - self._hyper = hyper(config) + self._hyper = hyper(self._config) self._timeout_after = None self._timer = None self._on_exit_success = lambda : None @@ -335,15 +389,15 @@ def poll(self): def exit(self, status, msg): self._exit_status = status self.stop() - print color.INFO(nametag),"Exit called with status", self._exit_status, "(",get_exit_code_name(self._exit_status),")" - print color.INFO(nametag),"Calling on_exit" + info("Exit called with status", self._exit_status, "(",get_exit_code_name(self._exit_status),")") + info("Calling on_exit") # Change back to test source os.chdir(self._root) self._on_exit() if status == 0: # Print success message and return to caller print color.SUCCESS(msg) - print color.INFO(nametag),"Calling on_exit_success" + info("Calling on_exit_success") return self._on_exit_success() # Print fail message and exit with appropriate code @@ -352,7 +406,7 @@ def exit(self, status, msg): # Default timeout event def timeout(self): - print color.INFO(""), "VM timed out" + if VERB: print color.INFO(""), "VM timed out" # Note: we have to stop the VM since the main thread is blocking on vm.readline #self.exit(exit_codes["TIMEOUT"], nametag + " Test timed out") @@ -363,7 +417,7 @@ def timeout(self): # Default panic event def panic(self, panic_line): panic_reason = self._hyper.readline() - print color.INFO(nametag), "VM signalled PANIC. Reading until EOT (", hex(ord(EOT)), ")" + info("VM signalled PANIC. Reading until EOT (", hex(ord(EOT)), ")") print color.VM(panic_reason), remaining_output = self._hyper.read_until_EOT() for line in remaining_output.split("\n"): @@ -376,8 +430,10 @@ def panic(self, panic_line): def on_output(self, output, callback): self._on_output[ output ] = callback - def on_success(self, callback): - self._on_output["SUCCESS"] = lambda(line) : [callback(line), self._on_success(line)] + def on_success(self, callback, do_exit = True): + if do_exit: + self._on_output["SUCCESS"] = lambda(line) : [callback(line), self._on_success(line)] + else: self._on_output["SUCCESS"] = callback def on_panic(self, callback): self._on_output["PANIC"] = lambda(line) : [callback(line), self._on_panic(line)] @@ -401,7 +457,7 @@ def writeline(self, line): # Make using GNU Make def make(self, params = []): - print color.INFO(nametag), "Building with 'make' (params=" + str(params) + ")" + print INFO, "Building with 'make' (params=" + str(params) + ")" make = ["make"] make.extend(params) cmd(make) @@ -409,7 +465,7 @@ def make(self, params = []): # Call cmake def cmake(self, args = []): - print color.INFO(nametag), "Building with cmake (%s)" % args + print INFO, "Building with cmake (%s)" % args # install dir: INSTDIR = os.getcwd() @@ -439,7 +495,7 @@ def cmake(self, args = []): # Clean cmake build folder def clean(self): - print color.INFO(nametag), "Cleaning cmake build folder" + print INFO, "Cleaning cmake build folder" subprocess.call(["rm","-rf","build"]) # Boot the VM and start reading output. This is the main event loop. @@ -451,7 +507,7 @@ def boot(self, timeout = 60, multiboot = True, kernel_args = "booted with vmrunn # Start the timeout thread if (timeout): - print color.INFO(nametag),"setting timeout to",timeout,"seconds" + info("setting timeout to",timeout,"seconds") self._timer = threading.Timer(timeout, self._on_timeout) self._timer.start() @@ -459,9 +515,10 @@ def boot(self, timeout = 60, multiboot = True, kernel_args = "booted with vmrunn try: self._hyper.boot(multiboot, kernel_args, image_name) except Exception as err: - print color.WARNING("Exception raised while booting ") + print color.WARNING("Exception raised while booting: ") + print_exception() if (timeout): self._timer.cancel() - self.exit(exit_codes["CALLBACK_FAILED"], str(err)) + self.exit(exit_codes["BOOT_FAILED"], str(err)) # Start analyzing output while self._hyper.poll() == None and not self._exit_status: @@ -518,31 +575,65 @@ def boot(self, timeout = 60, multiboot = True, kernel_args = "booted with vmrunn # If everything went well we can return return self +# Load a single vm config. Fallback to default +def load_single_config(path = "."): + + config = default_config + description = None + + # If path is explicitly "None", try current dir + if not path: path = "." -print color.HEADER("IncludeOS vmrunner loading VM configs") + info("Loading config from", path) + try: + # Try loading the first valid config + config, path = validate_vm.load_config(path)[0] + info ("vm config loaded from", path, ":",) + except Exception as e: + info ("No JSON config found - using default:",) + + if config.has_key("description"): + description = config["description"] + else: + description = str(config) -schema_path = package_path + "/vm.schema.json" -print color.INFO(nametag), "Validating JSON according to schema ",schema_path -validate_vm.load_schema(schema_path) -validate_vm.has_required_stuff(".") + info('"',description,'"') + + return config -default_spec = {"image" : "service.img"} # Provide a list of VM's with validated specs -vms = [] +# One unconfigured vm is created by default, which will try to load a config if booted +vms = [vm()] -if validate_vm.valid_vms: - print color.INFO(nametag), "Loaded VM specification(s) from JSON" - for spec in validate_vm.valid_vms: - print color.INFO(nametag), "Found VM spec: " - print color.DATA(spec.__str__()) - vms.append(vm(spec)) +def load_configs(config_path = "."): + global vms -else: - print color.WARNING(nametag), "No VM specification JSON found, trying default config" - vms.append(vm(default_spec)) + # Clear out the default unconfigured vm + if (not vms[0]._config): + vms = [] + + print color.HEADER("IncludeOS vmrunner loading VM configs") + + schema_path = package_path + "/vm.schema.json" + + print INFO, "Validating JSON according to schema ",schema_path + + validate_vm.load_schema(schema_path) + validate_vm.load_configs(config_path) + + if validate_vm.valid_vms: + print INFO, "Loaded VM specification(s) from JSON" + for spec in validate_vm.valid_vms: + print INFO, "Found VM spec: " + print color.DATA(spec.__str__()) + vms.append(vm(spec)) + else: + print color.WARNING(nametag), "No VM specification JSON found, trying default config" + vms.append(vm(default_config)) + return vms # Handler for SIGINT def handler(signum, frame):