From 63a67a435a516af2578b3d7f890beb34668d8b53 Mon Sep 17 00:00:00 2001 From: lassulus Date: Fri, 3 Jan 2025 02:44:38 +0100 Subject: [PATCH 1/2] WIP: implement a secret vars store in nixpkgs This allows to create secrets (and public files) outside or inside the nix store in a more delclarative way. This is shipped with an example (working) implementation of an on-machine storage. The vars options can easily be used to implement custom backends and extend the behaviour to integrate with already existing solutions, like sops-nix or agenix. --- nixos/modules/module-list.nix | 2 + .../system/vars/on-machine-backend.nix | 140 +++++++++++++ nixos/modules/system/vars/options.nix | 184 ++++++++++++++++++ nixos/tests/all-tests.nix | 1 + nixos/tests/vars.nix | 115 +++++++++++ 5 files changed, 442 insertions(+) create mode 100644 nixos/modules/system/vars/on-machine-backend.nix create mode 100644 nixos/modules/system/vars/options.nix create mode 100644 nixos/tests/vars.nix diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 06979a4df508f..f475cb92e2fc9 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1709,6 +1709,8 @@ ./system/boot/timesyncd.nix ./system/boot/tmp.nix ./system/boot/uvesafb.nix + ./system/vars/options.nix + ./system/vars/on-machine-backend.nix ./system/etc/etc-activation.nix ./tasks/auto-upgrade.nix ./tasks/bcache.nix diff --git a/nixos/modules/system/vars/on-machine-backend.nix b/nixos/modules/system/vars/on-machine-backend.nix new file mode 100644 index 0000000000000..d01025f05ca5b --- /dev/null +++ b/nixos/modules/system/vars/on-machine-backend.nix @@ -0,0 +1,140 @@ +# we use this vars backend as an example backend. +# this generates a script which creates the values at the expected path. +# this script has to be run manually (I guess after updating the system) to generate the required vars. +{ + pkgs, + lib, + config, + ... +}: +let + cfg = config.vars.settings.on-machine; + sortedGenerators = + (lib.toposort (a: b: builtins.elem a.name b.dependencies) (lib.attrValues config.vars.generators)) + .result; + + promptCmd = { + hidden = "read -sr prompt_value"; + line = "read -r prompt_value"; + multiline = '' + echo 'press control-d to finish' + prompt_value=$(cat) + ''; + }; + generate-vars = pkgs.writeShellApplication { + name = "generate-vars"; + text = '' + set -efuo pipefail + + PATH=${lib.makeBinPath [ pkgs.coreutils ]} + + # make the output directory overridable + OUT_DIR=''${OUT_DIR:-${cfg.fileLocation}} + + # check if all files are present or all files are missing + # if not, they are in an incosistent state and we bail out + ${lib.concatMapStringsSep "\n" (gen: '' + all_files_missing=true + all_files_present=true + ${lib.concatMapStringsSep "\n" (file: '' + if test -e ${file.path} ; then + all_files_missing=false + else + all_files_present=false + fi + '') (lib.attrValues gen.files)} + + if [ $all_files_missing = false ] && [ $all_files_present = false ] ; then + echo "Inconsistent state for generator ${gen.name}" + exit 1 + fi + if [ $all_files_present = true ] ; then + echo "All secrets for ${gen.name} are present" + elif [ $all_files_missing = true ] ; then + + # prompts + prompts=$(mktemp -d) + trap 'rm -rf $prompts' EXIT + export prompts + mkdir -p "$prompts" + ${lib.concatMapStringsSep "\n" (prompt: '' + echo ${lib.escapeShellArg prompt.description} + ${promptCmd.${prompt.type}} + echo -n "$prompt_value" > "$prompts"/${prompt.name} + '') (lib.attrValues gen.prompts)} + echo "Generating vars for ${gen.name}" + + # dependencies + in=$(mktemp -d) + trap 'rm -rf $in' EXIT + export in + mkdir -p "$in" + ${lib.concatMapStringsSep "\n" (input: '' + mkdir -p "$in"/${input} + ${lib.concatMapStringsSep "\n" (file: '' + cp "$OUT_DIR"/${ + if file.secret then "secret" else "public" + }/${input}/${file.name} "$in"/${input}/${file.name} + '') (lib.attrValues config.vars.generators.${input}.files)} + '') gen.dependencies} + + # outputs + out=$(mktemp -d) + trap 'rm -rf $out' EXIT + export out + mkdir -p "$out" + + ( + # prepare PATH + unset PATH + ${lib.optionalString (gen.runtimeInputs != [ ]) '' + PATH=${lib.makeBinPath gen.runtimeInputs} + export PATH + ''} + + # actually run the generator + ${gen.script} + ) + + # check if all files got generated + ${lib.concatMapStringsSep "\n" (file: '' + if ! test -e "$out"/${file.name} ; then + echo 'generator ${gen.name} failed to generate ${file.name}' + exit 1 + fi + '') (lib.attrValues gen.files)} + + # move the files to the correct location + ${lib.concatMapStringsSep "\n" (file: '' + OUT_FILE="$OUT_DIR"/${if file.secret then "secret" else "public"}/${file.generator}/${file.name} + mkdir -p "$(dirname "$OUT_FILE")" + mv "$out"/'${file.name}' "$OUT_FILE" + '') (lib.attrValues gen.files)} + rm -rf "$out" + fi + '') sortedGenerators} + ''; + }; +in +{ + options.vars.settings.on-machine = { + enable = lib.mkEnableOption "Enable on-machine vars backend"; + fileLocation = lib.mkOption { + type = lib.types.str; + default = "/etc/vars"; + }; + }; + config = lib.mkIf cfg.enable { + vars.settings.fileModule = file: { + path = + if file.config.secret then + "${cfg.fileLocation}/secret/${file.config.generator}/${file.config.name}" + else + "${cfg.fileLocation}/public/${file.config.generator}/${file.config.name}"; + }; + environment.systemPackages = [ + generate-vars + ]; + system.build.generate-vars = generate-vars; + }; +} diff --git a/nixos/modules/system/vars/options.nix b/nixos/modules/system/vars/options.nix new file mode 100644 index 0000000000000..eb52e18bdf15d --- /dev/null +++ b/nixos/modules/system/vars/options.nix @@ -0,0 +1,184 @@ +{ + lib, + config, + pkgs, + ... +}: +{ + options.vars = { + settings = { + fileModule = lib.mkOption { + type = lib.types.deferredModule; + internal = true; + description = '' + A module to be imported in every vars.files. submodule. + Used by backends to define the `path` attribute. + + Takes the file as an arument and returns maybe an attrset with should at least contain the `path` attribute. + Can be used to set other file attributes as well, like `value`. + ''; + default = { }; + }; + }; + generators = lib.mkOption { + description = '' + A set of generators that can be used to generate files. + Generators are scripts that produce files based on the values of other generators and user input. + Each generator is expected to produce a set of files under a directory. + ''; + default = { }; + type = lib.types.attrsOf ( + lib.types.submodule (generator: { + options = { + name = lib.mkOption { + type = lib.types.str; + description = '' + The name of the generator. + This name will be used to refer to the generator in other generators. + ''; + readOnly = true; + default = generator.config._module.args.name; + defaultText = "Name of the generator"; + }; + + dependencies = lib.mkOption { + description = '' + A list of other generators that this generator depends on. + The output values of these generators will be available to the generator script as files. + For example, the file 'file1' of a dependency named 'dep1' will be available via $in/dep1/file1. + ''; + type = lib.types.listOf lib.types.str; + default = [ ]; + }; + files = lib.mkOption { + description = '' + A set of files to generate. + The generator 'script' is expected to produce exactly these files under $out. + ''; + defaultText = "attrs of files"; + type = lib.types.attrsOf ( + lib.types.submodule (file: { + imports = [ + config.vars.settings.fileModule + ]; + options = { + name = lib.mkOption { + type = lib.types.str; + description = '' + name of the public fact + ''; + readOnly = true; + default = file.config._module.args.name; + defaultText = "Name of the file"; + }; + generator = lib.mkOption { + description = '' + The generator that produces the file. + This is the name of another generator. + ''; + type = lib.types.str; + readOnly = true; + internal = true; + default = generator.config.name; + defaultText = "Name of the generator"; + }; + deploy = lib.mkOption { + description = '' + Whether the file should be deployed to the target machine. + + Enable this if the generated file is only used as an input to other generators. + ''; + type = lib.types.bool; + default = true; + }; + secret = lib.mkOption { + description = '' + Whether the file should be treated as a secret. + ''; + type = lib.types.bool; + default = true; + }; + path = lib.mkOption { + description = '' + The path to the file containing the content of the generated value. + This will be set automatically + ''; + type = lib.types.str; + }; + }; + }) + ); + }; + prompts = lib.mkOption { + description = '' + A set of prompts to ask the user for values. + Prompts are available to the generator script as files. + For example, a prompt named 'prompt1' will be available via $prompts/prompt1 + ''; + default = { }; + type = lib.types.attrsOf ( + lib.types.submodule (prompt: { + options = { + name = lib.mkOption { + description = '' + The name of the prompt. + This name will be used to refer to the prompt in the generator script. + ''; + type = lib.types.str; + default = prompt.config._module.args.name; + defaultText = "Name of the prompt"; + }; + description = lib.mkOption { + description = '' + The description of the prompted value + ''; + type = lib.types.str; + example = "SSH private key"; + default = prompt.config._module.args.name; + defaultText = "Name of the prompt"; + }; + type = lib.mkOption { + description = '' + The input type of the prompt. + The following types are available: + - hidden: A hidden text (e.g. password) + - line: A single line of text + - multiline: A multiline text + ''; + type = lib.types.enum [ + "hidden" + "line" + "multiline" + ]; + default = "line"; + }; + }; + }) + ); + }; + runtimeInputs = lib.mkOption { + description = '' + A list of packages that the generator script requires. + These packages will be available in the PATH when the script is run. + ''; + type = lib.types.listOf lib.types.package; + default = [ pkgs.coreutils ]; + }; + script = lib.mkOption { + description = '' + The script to run to generate the files. + The script will be run with the following environment variables: + - $in: The directory containing the output values of all declared dependencies + - $out: The output directory to put the generated files + - $prompts: The directory containing the prompted values as files + The script should produce the files specified in the 'files' attribute under $out. + ''; + type = lib.types.either lib.types.str lib.types.path; + default = ""; + }; + }; + }) + ); + }; + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index cf06168863017..60a0afb73ca95 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1132,6 +1132,7 @@ in { user-home-mode = handleTest ./user-home-mode.nix {}; ustreamer = handleTest ./ustreamer.nix {}; uwsgi = handleTest ./uwsgi.nix {}; + vars = handleTest ./vars.nix {}; v2ray = handleTest ./v2ray.nix {}; varnish60 = handleTest ./varnish.nix { package = pkgs.varnish60; }; varnish75 = handleTest ./varnish.nix { package = pkgs.varnish75; }; diff --git a/nixos/tests/vars.nix b/nixos/tests/vars.nix new file mode 100644 index 0000000000000..2e765bcf776a1 --- /dev/null +++ b/nixos/tests/vars.nix @@ -0,0 +1,115 @@ +import ./make-test-python.nix ( + { lib, pkgs, ... }: + + { + name = "vars"; + meta.maintainers = with lib.maintainers; [ lassulus ]; + + nodes.machine = + { ... }: + { + vars.settings.on-machine.enable = true; + vars.generators = { + simple = { + files.simple = { }; + script = '' + echo simple > "$out"/simple + ''; + }; + + a = { + files.a = { }; + script = '' + echo a > "$out"/a + ''; + }; + b = { + dependencies = [ "a" ]; + files.b = { }; + script = '' + cat "$in"/a/a > "$out"/b + echo b >> "$out"/b + ''; + }; + + prompts = { + files.prompt_line = { }; + files.prompt_hidden = { }; + files.prompt_multiline = { }; + prompts.line = { + description = '' + a simple line prompt + ''; + }; + prompts.hidden = { + type = "hidden"; + description = '' + a prompt that doesn't show the input + ''; + }; + prompts.aamulti = { + type = "multiline"; + description = '' + a prompt with multiple lines + ''; + }; + script = '' + cp "$prompts"/line "$out"/prompt_line + cp "$prompts"/hidden "$out"/prompt_hidden + cp "$prompts"/aamulti "$out"/prompt_multiline + ''; + }; + }; + }; + + testScript = + { nodes, ... }: + '' + import subprocess + from pathlib import Path + + process = subprocess.Popen( + ["${nodes.machine.config.system.build.generate-vars}/bin/generate-vars"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + text=True, + env={ + "OUT_DIR": "./vars", + }, + ) + + # Function to check for expected outputs and send corresponding texts + + def interact_with_process(process, interactions): + while interactions: + output = process.stdout.readline() + if output: + print(output.strip()) # Print the output for debugging + for expected_output, text_to_send in interactions: + if expected_output in output: + print("sending", text_to_send) + process.stdin.write(text_to_send + '\n') + process.stdin.flush() + interactions.remove((expected_output, text_to_send)) + break + + interactions = [ + ("a simple line prompt", "simple prompt content"), + ("a prompt that doesn't show the input", "hidden prompt content"), + ("press control-d to finish", f"multi line content1\nmulti line content2\n\n{chr(4)}\n"), + ("another prompt after EOF", "another prompt content"), + ] + + # Interact with the process + interact_with_process(process, interactions) + + # Wait for the process to complete + process.wait() + + vars_folder = Path("vars") + print(list(vars_folder.glob("*"))) + assert((vars_folder / "secret" / "a" / "a").exists()) + + ''; + } +) From fc64ee40eba0715341023fb0c0e3a394dba93d83 Mon Sep 17 00:00:00 2001 From: lassulus Date: Tue, 7 Jan 2025 21:37:47 +0100 Subject: [PATCH 2/2] WIP: nixos/syncthing: manage secrets with vars --- nixos/modules/services/networking/syncthing.nix | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/nixos/modules/services/networking/syncthing.nix b/nixos/modules/services/networking/syncthing.nix index 2d32cf4517062..f3f30f706b7f4 100644 --- a/nixos/modules/services/networking/syncthing.nix +++ b/nixos/modules/services/networking/syncthing.nix @@ -621,6 +621,21 @@ in { config = mkIf cfg.enable { + vars.generators.syncthing = { + files."cert.pem" = {}; + files."key.pem" = {}; + files."syncthing.pub".secret = false; + runtimeInputs = [ + pkgs.coreutils + pkgs.gnugrep + pkgs.syncthing + ]; + script = '' + syncthing generate --config "$out" + < "$out"/config.xml grep -oP '(?<= "$out"/syncthing.pub + ''; + }; + networking.firewall = mkIf cfg.openDefaultPorts { allowedTCPPorts = [ 22000 ]; allowedUDPPorts = [ 21027 22000 ];