Skip to content

Commit

Permalink
Speed up nix build by splitting built into multiple nix packages
Browse files Browse the repository at this point in the history
  • Loading branch information
@mpscholten
mpscholten committed Sep 10, 2024
1 parent b5e5cda commit c50d6af
Showing 1 changed file with 83 additions and 51 deletions.
134 changes: 83 additions & 51 deletions NixSupport/default.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,16 +25,6 @@ let
(otherDeps pkgs)
];

appBinary = if optimized
then "build/bin/RunOptimizedProdServer"
else "build/bin/RunUnoptimizedProdServer";

jobsBinary = if optimized
then "build/bin/RunJobsOptimized"
else "build/bin/RunJobs";

odir = if optimized then "RunOptimizedProdServer" else "RunUnoptimizedProdServer";

schemaObjectFiles =
let
self = projectPath;
Expand All @@ -46,7 +36,7 @@ let
build-generated-code
export IHP=${ihp}/lib/IHP
ghc -O${if optimized then optimizationLevel else "0"} $(make print-ghc-options) --make build/Generated/Types.hs -odir build/${odir} -hidir build/${odir}
ghc -O${if optimized then optimizationLevel else "0"} $(make print-ghc-options) --make build/Generated/Types.hs -odir build/RunProdServer -hidir build/RunProdServer
cp -r build $out
'';
Expand All @@ -58,18 +48,85 @@ let
dontInstall = true;
dontFixup = false;
};

prodGhcOptions = "-funbox-strict-fields -fconstraint-solver-iterations=100 -fdicts-strict -with-rtsopts=\"${rtsFlags}\"";

binaries =
pkgs.stdenv.mkDerivation {
name = appName + "-binaries";
buildPhase = ''
mkdir -p build/Generated build/RunProdServer
cp -r ${schemaObjectFiles}/RunProdServer build/
cp -r ${schemaObjectFiles}/Generated build/
chmod -R +w build/RunProdServer/*
export IHP_LIB=${ihp}/lib/IHP
export IHP=${ihp}/lib/IHP
mkdir -p build/bin build/RunUnoptimizedProdServer
echo ghc -O${if optimized then optimizationLevel else "0"} $(make print-ghc-options) ${if optimized then prodGhcOptions else ""} Main.hs -o build/bin/RunProdServer -odir build/RunProdServer -hidir build/RunProdServer
ghc -O${if optimized then optimizationLevel else "0"} $(make print-ghc-options) ${if optimized then prodGhcOptions else ""} Main.hs -o build/bin/RunProdServer -odir build/RunProdServer -hidir build/RunProdServer
# Build job runner if there are any jobs
if find -type d -iwholename \*/Job|grep .; then
echo "module RunJobs (main) where" > build/RunJobs.hs
echo "import Application.Script.Prelude" >> build/RunJobs.hs
echo "import IHP.ScriptSupport" >> build/RunJobs.hs
echo "import IHP.Job.Runner" >> build/RunJobs.hs
echo "import qualified Config" >> build/RunJobs.hs
echo "import Main ()" >> build/RunJobs.hs
echo "main :: IO ()" >> build/RunJobs.hs
echo "main = runScript Config.config (runJobWorkers (workers RootApplication))" >> build/RunJobs.hs
ghc -O${if optimized then optimizationLevel else "0"} -main-is 'RunJobs.main' $(make print-ghc-options) ${if optimized then prodGhcOptions else ""} build/RunJobs.hs -o build/bin/RunJobs -odir build/RunProdServer -hidir build/RunProdServer
fi;
# Build all scripts if there are any
mkdir -p Application/Script
SCRIPT_TARGETS=`find Application/Script -type f -iwholename '*.hs' -not -name 'Prelude.hs' -exec basename {} .hs ';' | sed 's#^#build/bin/Script/#' | tr "\n" " "`
if [[ ! -z "$SCRIPT_TARGETS" ]]; then
# Need to use -j1 here to avoid race conditions of temp files created by GHC.
#
# These errors look like:
#
# <no location info>: error:
# build/RunUnoptimizedProdServer/Application/Script/Prelude.o.tmp: renameFile:renamePath:rename: does not exist (No such file or directory)
#
make -j1 $SCRIPT_TARGETS;
fi;
'';
installPhase = ''
mkdir -p "$out"
mkdir -p $out/bin $out/lib
mv build/bin/RunProdServer $out/bin/RunProdServer
# Copy job runner binary to bin/ if we built it
if [ -f build/bin/RunJobs ]; then
mv build/bin/RunJobs $out/bin/RunJobs;
fi;
# Copy IHP Script binaries to bin/
mkdir -p build/bin/Script
find build/bin/Script/ -type f -print0 |
while read -d $'\0' script; do
script_basename=$(basename "$script")
mv "build/bin/Script/$script_basename" "$out/bin/$script_basename";
done
'';
dontFixup = true;
src = filter { root = pkgs.nix-gitignore.gitignoreSource [] projectPath; include = [filter.isDirectory "Makefile" (filter.matchExt "hs")]; exclude = ["static" "Frontend"]; };
buildInputs = [allHaskellPackages];
nativeBuildInputs = [ pkgs.makeWrapper ];
enableParallelBuilding = true;
};
in
pkgs.stdenv.mkDerivation {
name = appName;
buildPhase = ''
runHook preBuild
mkdir -p build/Generated build/${odir}
cp -r ${schemaObjectFiles}/${odir} build/
cp -r ${schemaObjectFiles}/Generated build/
chmod -R +w build/${odir}/*
# When npm install is executed by the project's makefile it will fail with:
#
# EACCES: permission denied, mkdir '/homeless-shelter'
Expand All @@ -81,29 +138,8 @@ in
export IHP_LIB=${ihp}/lib/IHP
export IHP=${ihp}/lib/IHP
export APP_RTS_FLAGS="${rtsFlags}"
export OPTIMIZATION_LEVEL="${optimizationLevel}"
make -j ${appBinary}
# Build job runner if there are any jobs
if find -type d -iwholename \*/Job|grep .; then
make -j ${jobsBinary};
fi;
# Build all scripts if there are any
mkdir -p Application/Script
SCRIPT_TARGETS=`find Application/Script -type f -iwholename '*.hs' -not -name 'Prelude.hs' -exec basename {} .hs ';' | sed 's#^#build/bin/Script/#' | tr "\n" " "`
if [[ ! -z "$SCRIPT_TARGETS" ]]; then
# Need to use -j1 here to avoid race conditions of temp files created by GHC.
#
# These errors look like:
#
# <no location info>: error:
# build/RunUnoptimizedProdServer/Application/Script/Prelude.o.tmp: renameFile:renamePath:rename: does not exist (No such file or directory)
#
make -j1 $SCRIPT_TARGETS;
fi;
make -j static/app.css static/app.js
runHook postBuild
'';
Expand All @@ -113,23 +149,19 @@ in
mkdir -p "$out"
mkdir -p $out/bin $out/lib
mv ${appBinary} $out/bin/RunProdServerWithoutOptions
INPUT_HASH="$((basename $out) | cut -d - -f 1)"
makeWrapper $out/bin/RunProdServerWithoutOptions $out/bin/RunProdServer --set-default IHP_ASSET_VERSION $INPUT_HASH --set-default IHP_LIB ${ihp}/lib/IHP --run "cd $out/lib" --prefix PATH : ${pkgs.lib.makeBinPath (otherDeps pkgs)}
makeWrapper ${binaries}/bin/RunProdServer $out/bin/RunProdServer --set-default IHP_ASSET_VERSION $INPUT_HASH --set-default IHP_LIB ${ihp}/lib/IHP --run "cd $out/lib" --prefix PATH : ${pkgs.lib.makeBinPath (otherDeps pkgs)}
# Copy job runner binary to bin/ if we built it
if [ -f ${jobsBinary} ]; then
mv ${jobsBinary} $out/bin/RunJobsWithoutOptions;
makeWrapper $out/bin/RunJobsWithoutOptions $out/bin/RunJobs --set-default IHP_ASSET_VERSION $INPUT_HASH --set-default IHP_LIB ${ihp}/lib/IHP --run "cd $out/lib" --prefix PATH : ${pkgs.lib.makeBinPath (otherDeps pkgs)}
if [ -f ${binaries}/bin/RunJobs ]; then
makeWrapper ${binaries}/bin/RunJobs $out/bin/RunJobs --set-default IHP_ASSET_VERSION $INPUT_HASH --set-default IHP_LIB ${ihp}/lib/IHP --run "cd $out/lib" --prefix PATH : ${pkgs.lib.makeBinPath (otherDeps pkgs)}
fi;
# Copy IHP Script binaries to bin/
mkdir -p build/bin/Script
find build/bin/Script/ -type f -print0 |
while read -d $'\0' script; do
script_basename=$(basename "$script")
mv "build/bin/Script/$script_basename" "$out/bin/$script_basename";
# Copy other binaries, excluding RunProdServer and RunJobs
find ${binaries}/bin/ -type f -not -name 'RunProdServer' -not -name 'RunJobs' -print0 |
while read -d $'\0' binary; do
binary_basename=$(basename "$binary")
cp "$binary" "$out/bin/$binary_basename";
done
mv static "$out/lib/static"
Expand Down

0 comments on commit c50d6af

Please sign in to comment.