Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

support for files without external meta data #12

Merged
merged 11 commits into from
Oct 16, 2024
Merged

support for files without external meta data #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
f8c0bc7
version 1 works
bcumming Oct 4, 2024
f9c6a9a
simplify and improve cleanup of temporary paths
bcumming Oct 4, 2024
96dd758
use the unsquash feature to use in-image meta data
bcumming Oct 14, 2024
6ad89ea
fix meta path finding
bcumming Oct 15, 2024
6640086
delete temporary paths before execve to ensure they are always deleted
bcumming Oct 15, 2024
e5e2391
tunable logging in unit tests - disable verbose logging by default
bcumming Oct 16, 2024
8b8919a
generate sqfs images for testing that have no external meta data
bcumming Oct 16, 2024
da95476
use deque instead of vector for persistance of temp paths
bcumming Oct 16, 2024
03effa7
only use SQFSMNT_FWD_ prefix for setuid vars - so that PATH etc are s…
bcumming Oct 16, 2024
c5538c4
clean up comments and formatting
bcumming Oct 16, 2024
0d674be
fix missing header in tests; clean up output
bcumming Oct 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion meson.build
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,11 @@ lib_src = [
'src/uenv/meta.cpp',
'src/uenv/parse.cpp',
'src/uenv/repository.cpp',
'src/uenv/uenv.cpp',
'src/util/fs.cpp',
'src/util/shell.cpp',
'src/util/strings.cpp',
'src/util/subprocess.cpp',
'src/uenv/uenv.cpp',
]
lib_inc = include_directories('src')

Expand Down Expand Up @@ -76,6 +77,7 @@ endif
unit_src = [
'test/unit/env.cpp',
'test/unit/envvars.cpp',
'test/unit/fs.cpp',
'test/unit/lex.cpp',
'test/unit/main.cpp',
'test/unit/parse.cpp',
Expand Down
171 changes: 123 additions & 48 deletions src/uenv/env.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,62 @@
#include <uenv/meta.h>
#include <uenv/parse.h>
#include <uenv/repository.h>
#include <util/fs.h>
#include <util/subprocess.h>

namespace uenv {

using util::unexpected;

struct meta_info {
std::optional<std::filesystem::path> path;
std::optional<std::filesystem::path> env;
};

meta_info find_meta_path(const std::filesystem::path& sqfs_path) {
namespace fs = std::filesystem;

meta_info meta;
if (fs::is_directory(sqfs_path.parent_path() / "meta")) {
meta.path = sqfs_path.parent_path() / "meta";
spdlog::debug("find_meta_path: {} found external meta path {}",
sqfs_path.string(), meta.path->string());
} else if (auto p = util::unsquashfs_tmp(sqfs_path, "meta")) {
meta.path = *p / "meta";
spdlog::debug("find_meta_path: {} found internal meta path {}",
sqfs_path.string(), meta.path->string());
} else {
spdlog::debug("find_meta_path: {} no meta path found",
sqfs_path.string());
}

if (meta.path) {
auto env_meta = meta.path.value() / "env.json";

if (fs::is_regular_file(env_meta)) {
meta.env = env_meta;
spdlog::debug("find_meta_path: {} found env meta {}",
sqfs_path.string(), meta.env->string());
} else {
spdlog::debug("find_meta_path: {} no env meta", sqfs_path.string());
}
}

return meta;
}

util::expected<env, std::string>
concretise_env(const std::string& uenv_args,
std::optional<std::string> view_args,
std::optional<std::filesystem::path> repo_arg) {
namespace fs = std::filesystem;

// parse the uenv description that was provided as a command line argument.
// the command line argument is a comma-separated list of uenvs, where each
// uenv is either
// parse the uenv description that was provided as a command line
// argument. the command line argument is a comma-separated list of
// uenvs, where each uenv is either
// - the path of a squashfs image; or
// - a uenv description of the form name[/version][:tag][@system][%uarch]
// with an optional mount point.
// - a uenv description of the form
// name[/version][:tag][@system][%uarch] with an optional mount point.
const auto uenv_descriptions = uenv::parse_uenv_args(uenv_args);
if (!uenv_descriptions) {
return unexpected(fmt::format("invalid uenv description: {}",
Expand All @@ -38,8 +77,8 @@ concretise_env(const std::string& uenv_args,

// concretise the uenv descriptions by looking for the squashfs file, or
// looking up the uenv descrition in a registry.
// after this loop, we have fully validated list of uenvs, mount points and
// meta data (if they have meta data).
// after this loop, we have fully validated list of uenvs, mount points
// and meta data (if they have meta data).

std::unordered_map<std::string, uenv::concrete_uenv> uenvs;
std::set<fs::path> used_mounts;
Expand All @@ -52,9 +91,10 @@ concretise_env(const std::string& uenv_args,
// it has to be looked up in a repo.
if (auto label = desc.label()) {
if (!repo_arg) {
return unexpected(
"a repo needs to be provided either using the --repo flag "
"or by setting the UENV_REPO_PATH environment variable");
return unexpected("a repo needs to be provided either "
"using the --repo flag "
"or by setting the UENV_REPO_PATH "
"environment variable");
}
auto store = uenv::open_repository(*repo_arg);
if (!store) {
Expand Down Expand Up @@ -119,22 +159,22 @@ concretise_env(const std::string& uenv_args,
}
spdlog::info("{} squashfs image {}", desc, sqfs_path.string());

// set the meta data path and env.json path if they exist
const auto meta_p = sqfs_path.parent_path() / "meta";
const auto env_meta_p = meta_p / "env.json";
const std::optional<fs::path> meta_path =
fs::is_directory(meta_p) ? meta_p : std::optional<fs::path>{};
const std::optional<fs::path> env_meta_path =
fs::is_regular_file(env_meta_p) ? env_meta_p
: std::optional<fs::path>{};

// if meta/env.json exists, parse the json therein
std::string name;
std::string description;
// create a default "anonymous" name for the uenv that will be
// overwritten if meta data is provided.
std::string name = "anonymous";
unsigned name_idx = 0;
while (uenvs.count(name)) {
name = fmt::format("anonymous{}", name_idx);
++name_idx;
}
std::string description = "";
std::optional<std::string> mount_meta;
std::unordered_map<std::string, concrete_view> views;
if (env_meta_path) {
if (const auto result = uenv::load_meta(*env_meta_path)) {

// if meta/env.json exists, parse the json therein
auto meta = find_meta_path(sqfs_path);
if (meta.env) {
if (const auto result = uenv::load_meta(*(meta.env))) {
name = std::move(result->name);
description = result->description.value_or("");
mount_meta = result->mount;
Expand All @@ -143,27 +183,19 @@ concretise_env(const std::string& uenv_args,
mount_meta);
} else {
spdlog::warn("{} opening the uenv meta data {}: {}", desc,
*env_meta_path, result.error());
meta.env->string(), result.error());
}
} else {
spdlog::debug("{} no meta file found at expected location {}", desc,
meta_path);
description = "";
// generate a unique name for the uenv
name = "anonymous";
unsigned i = 1;
while (uenvs.count(name)) {
name = fmt::format("anonymous{}", i);
++i;
}
spdlog::warn("{} no meta file available for {}", desc,
sqfs_path.string());
}

// if an explicit mount point was provided, use that
// otherwise use the mount point provided in the meta data
auto mount_string = desc.mount() ? desc.mount() : mount_meta;

// handle the case where no mount point was provided by the CLI or meta
// data
// handle the case where no mount point was provided by the CLI or
// meta data
if (!mount_string) {
return unexpected(
fmt::format("no mount point provided for {}", desc));
Expand Down Expand Up @@ -212,7 +244,7 @@ concretise_env(const std::string& uenv_args,
}

uenvs[name] = concrete_uenv{name, mount, sqfs_path,
meta_path, description, std::move(views)};
meta.path, description, std::move(views)};
}

// A dictionary with view name as a key, and a list of uenv that provide
Expand Down Expand Up @@ -249,7 +281,8 @@ concretise_env(const std::string& uenv_args,
// a list of uenv that have a view with name v.name
const auto& matching_uenvs = view2uenv[view.name];

// handle the case where no uenv name was provided, e.g. develop
// handle the case where no uenv name was provided, e.g.
// develop
if (!view.uenv) {
// it is ambiguous if more than one option is available
if (matching_uenvs.size() > 1) {
Expand All @@ -263,8 +296,8 @@ concretise_env(const std::string& uenv_args,
}
views.push_back({matching_uenvs[0], view.name});
}
// handle the case where both uenv and view name are provided,
// e.g. prgenv-gnu:develop
// handle the case where both uenv and view name are
// provided, e.g. prgenv-gnu:develop
else {
auto it = std::find_if(
matching_uenvs.begin(), matching_uenvs.end(),
Expand Down Expand Up @@ -298,19 +331,18 @@ std::unordered_map<std::string, std::string> getenv(const env& environment) {

// returns the value of an environment variable.
// if the variable has been recorded in env_vars, that value is returned
// else the cstdlib getenv function is called to get the currently set value
// returns nullptr if the variable is not set anywhere
// else the cstdlib getenv function is called to get the currently set
// value returns nullptr if the variable is not set anywhere
auto ge = [&env_vars](const std::string& name) -> const char* {
if (env_vars.count(name)) {
return env_vars[name].c_str();
}
return ::getenv(name.c_str());
};

// iterate over each view in order, and set the environment variables that
// each view configures.
// the variables are not set directly, instead they are accumulated in
// env_vars.
// iterate over each view in order, and set the environment variables
// that each view configures. the variables are not set directly,
// instead they are accumulated in env_vars.
for (auto& view : environment.views) {
auto result = environment.uenvs.at(view.uenv)
.views.at(view.name)
Expand All @@ -323,11 +355,53 @@ std::unordered_map<std::string, std::string> getenv(const env& environment) {
return env_vars;
}

// list of environment variables that are ignored in setuid applications
// the full list is defined here:
// https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/generic/unsecvars.h
std::set<std::string_view> unsecure_envvars__{
"GCONV_PATH",
"GETCONF_DIR",
"GLIBC_TUNABLES",
"HOSTALIASES",
"LD_AUDIT",
"LD_BIND_NOT",
"LD_BIND_NOW",
"LD_DEBUG",
"LD_DEBUG_OUTPUT",
"LD_DYNAMIC_WEAK",
"LD_LIBRARY_PATH",
"LD_ORIGIN_PATH",
"LD_PRELOAD",
"LD_PROFILE",
"LD_SHOW_AUXV",
"LD_VERBOSE",
"LD_WARN",
"LOCALDOMAIN",
"LOCPATH",
"MALLOC_ARENA_MAX",
"MALLOC_ARENA_TEST",
"MALLOC_MMAP_MAX_",
"MALLOC_MMAP_THRESHOLD_",
"MALLOC_PERTURB_",
"MALLOC_TOP_PAD_",
"MALLOC_TRACE",
"MALLOC_TRIM_THRESHOLD_",
"NIS_PATH",
"NLSPATH",
"RESOLV_HOST_CONF",
"RES_OPTIONS",
"TMPDIR",
};

util::expected<int, std::string>
setenv(const std::unordered_map<std::string, std::string>& variables,
const std::string& prefix) {
for (auto var : variables) {
std::string fwd_name = prefix + var.first;
// prepend prefix to unsecure environment variables
std::string fwd_name = unsecure_envvars__.contains(var.first)
? prefix + var.first
: var.first;
fmt::println("setting {} to {}", fwd_name, var.second);
if (auto rcode = ::setenv(fwd_name.c_str(), var.second.c_str(), true)) {
switch (rcode) {
case EINVAL:
Expand All @@ -340,6 +414,7 @@ setenv(const std::unordered_map<std::string, std::string>& variables,
fmt::format("unknown error setting {}", fwd_name));
}
}
fmt::println("set!");
}
return 0;
}
Expand Down
Loading