diff --git a/.devcontainer.json b/.devcontainer.json new file mode 100644 index 000000000..2f6920618 --- /dev/null +++ b/.devcontainer.json @@ -0,0 +1,12 @@ +{ + "customizations": { + "vscode": { + "extensions": [ + "mkhl.direnv" + ] + } + }, + "image": "ghcr.io/cachix/devenv:latest", + "overrideCommand": false, + "updateContentCommand": "devenv test" +} diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..a7cb2dfe9 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake . --impure --show-trace diff --git a/.github/workflows/nix-cache-build-deps.yaml b/.github/workflows/nix-cache-build-deps.yaml new file mode 100644 index 000000000..7c29d2157 --- /dev/null +++ b/.github/workflows/nix-cache-build-deps.yaml @@ -0,0 +1,27 @@ +name: Cache Nix CI Dependencies + +on: + workflow_dispatch: + pull-request: + paths: ['.nix/*', 'flake.nix', 'flake.lock'] + push: + branches: ['master', 'v*'] + paths: ['.nix/*', 'flake.nix', 'flake.lock'] + +jobs: + build-and-cache-dependencies: + name: Build and Cache Dependencies + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + with: + determinate: true + extra-conf: | + experimental-features = nix-command flakes auto-allocate-uids + extra-substituters = https://nix-community.cachix.org https://cuda-maintainers.cachix.org + extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E= + - uses: DeterminateSystems/magic-nix-cache-action@main + - name: Build and Cache Dependencies + run: | + export NIXPKGS_ALLOW_UNFREE=1 + nix build --impure -L '.#checks.x86_64-linux.deps' diff --git a/.github/workflows/nix.yaml b/.github/workflows/nix.yaml new file mode 100644 index 000000000..9fe06cfd5 --- /dev/null +++ b/.github/workflows/nix.yaml @@ -0,0 +1,40 @@ +name: Quick CI Builds + +on: + workflow_dispatch: + pull_request: + paths: + - "configure.ac" + - "Makefile.am" + - "autogen.sh" + - "include/**" + - "m4/**" + - "src/**" + - "tests/**" + - ".github/workflows/nix*.yaml" + - "flake.nix" + - "flake.lock" + - ".nix/**" + +jobs: + nix-build: + name: nix build smoke test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + with: + determinate: true + extra-conf: | + experimental-features = nix-command flakes auto-allocate-uids + extra-substituters = https://nix-community.cachix.org https://cuda-maintainers.cachix.org + extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E= + - uses: DeterminateSystems/magic-nix-cache-action@main + - name: Build and Cache Dependencies + run: | + export NIXPKGS_ALLOW_UNFREE=1 + nix build --impure -L '.#checks.x86_64-linux.default" + # We want to use the cache here, but we don't want to cache this + # specific build, so purge the cache now to prevent it from being + # pushed on cleanup. + nix-collect-garbage -d diff --git a/.gitignore b/.gitignore index 7f8ecadfc..82bc262f5 100644 --- a/.gitignore +++ b/.gitignore @@ -73,5 +73,10 @@ m4/ltversion.m4 m4/lt~obsolete.m4 .idea/ -.devenv/ +*.src.rpm +dockerbld +.devenv* +devenv.local.nix .direnv +.pre-commit-config.yaml +result diff --git a/.nix/combos.nix b/.nix/combos.nix new file mode 100644 index 000000000..1f94af8c7 --- /dev/null +++ b/.nix/combos.nix @@ -0,0 +1,10 @@ +{ + accelerator = [ "-cuda" "-neuron" ]; + platform = [ "-aws" "" ]; + tracing = [ [ "" ] [ "-nvtx" ] [ "-lttng" ] [ "-nvtx" "-lttng" ] ]; + debug = [ "-debug" "" ]; + memory = [ "-valgrind" "" ]; + traceprints = [ "-trace" "" ]; + cpp = [ "-cpp" "" ]; + stdenv = [ (pkgs: pkgs.gcc7Stdenv) (pkgs: pkgs.clangStdenv) (pkgs: pkgs.gcc14Stdenv) ]; +} diff --git a/.nix/default.nix b/.nix/default.nix new file mode 100644 index 000000000..0ee9776e1 --- /dev/null +++ b/.nix/default.nix @@ -0,0 +1,143 @@ +{ lib +, fetchFromGitHub +, symlinkJoin +, gitUpdater +, stdenv +, config +, libfabric +, hwloc +, autoreconfHook +, lttng-ust +, valgrind +, mpi +, cudaPackages ? { } +, enableTests ? true +, enableTracePrints ? (enableTests) +, neuronSupport ? (!config.cudaSupport) +, cudaSupport ? (config.cudaSupport && !neuronSupport) +, enableLTTNGTracing ? false +, enableNVTXTracing ? false +, enableValgrind ? false +, enableAwsTuning ? false +, enableCPPMode ? false +}: + +assert neuronSupport != cudaSupport; +#assert !enableNVTXTracing || (enableNVTXTracing && !neuronSupport); + +let + basename = "lib${if neuronSupport then "nccom" else "nccl"}-net-ofi"; + pname = "${basename}${if enableAwsTuning == true then "-aws" else ""}"; + version = "1.11.0"; + src = fetchFromGitHub { + owner = "aws"; + repo = "aws-ofi-nccl"; + rev = "v${version}-aws"; + sha256 = "sha256-y3yVPqak+36UXI6L/ddQIfBBwpeiciW571noc8LNefU="; + }; + cuda_build_deps_joined = symlinkJoin { + name = "cuda-build-deps-joined"; + paths = lib.optionals (cudaSupport) [ + (lib.getOutput "static" cudaPackages.cuda_cudart) + (lib.getDev cudaPackages.cuda_cudart) + (lib.getDev cudaPackages.cuda_nvcc) + ]; + }; +in +stdenv.mkDerivation { + inherit pname version src; + + enableParallelBuilding = true; + separateDebugInfo = true; + strictDeps = true; + + nativeBuildInputs = [ autoreconfHook ]; + configureFlags = + [ + "--enable-picky-compiler" + "--enable-werror" + "--with-hwloc=${lib.getDev hwloc}" + "--with-libfabric=${lib.getDev libfabric}" + ] + ++ lib.optionals enableCPPMode [ + "--enable-cpp=yes" + ] + ++ lib.optionals (!enableTests) [ + "--disable-tests" + ] + ++ lib.optionals enableTests [ + "--enable-tests" + "--with-mpi=${lib.getDev mpi}" + ] + ++ lib.optionals enableTracePrints [ + "--enable-trace" + ] + ++ lib.optionals cudaSupport [ + "--with-cuda=${cuda_build_deps_joined}" + ] + ++ lib.optionals enableLTTNGTracing [ + "--with-lttng=${lib.getDev lttng-ust}" + ] + ++ lib.optionals enableValgrind [ + "--with-valgrind=${lib.getDev valgrind}" + ] + ++ lib.optionals (enableNVTXTracing && cudaSupport) [ + "--with-nvtx=${lib.getDev cudaPackages.cuda_nvtx}" + ] + ++ lib.optionals enableAwsTuning [ + "--enable-platform-aws" + ] + ++ lib.optionals neuronSupport [ + "--enable-neuron" + ]; + + buildInputs = + [ + libfabric + hwloc + ] + ++ lib.optionals cudaSupport [ + cuda_build_deps_joined + ] + ++ lib.optionals enableValgrind [ + valgrind + ] + ++ lib.optionals enableTests [ + mpi + ] + ++ lib.optionals enableLTTNGTracing [ + lttng-ust + ]; + postInstall = ''find $out/lib | grep -E \.la$ | xargs rm''; + + doCheck = enableTests; + checkPhase = '' + set -euo pipefail + for test in $(find tests/unit/ -type f -executable -print | xargs) ; do + echo "======================================================================" + echo "Running $test" + ./$test + test $? -eq 0 && (echo "✅ Passed" || (echo "❌ Failed!" && exit 1)) + done + echo "All unit tests passed successfully." + set +u + ''; + + passthru = { + inherit cudaSupport; + updateScript = gitUpdater { + inherit pname version; + rev-prefix = "v"; + }; + }; + meta = with lib; { + homepage = "https://github.com/aws/aws-ofi-nccl"; + license = licenses.asl20; + broken = (cudaSupport && !config.cudaSupport); + maintainers = with maintainers; [ sielicki ]; + platforms = [ + "x86_64-linux" + "aarch64-linux" + ]; + }; +} diff --git a/.nix/lib.nix b/.nix/lib.nix new file mode 100644 index 000000000..cce7b312a --- /dev/null +++ b/.nix/lib.nix @@ -0,0 +1,25 @@ +{ inputs, lib }: +rec { + mkGitVersion = i: "git${if (i ? rev) then (builtins.substring 0 7 "${i.rev}") else "dirty" + (builtins.substring 0 7 "${i.dirtyRev}")}"; + compilerName = s: "-${if s.cc.isGNU then "gcc" else "clang"}${s.cc.version}"; + genComboName = pkgs: combo: prevname: "${prevname}${(lib.strings.concatStrings combo.tracing)}${combo.debug}${combo.memory}${combo.traceprints}${combo.cpp}${compilerName (combo.stdenv pkgs)}"; + genPkgFromCombo = pkgs: combo: (pkgs.callPackage ./default.nix { + stdenv = (combo.stdenv pkgs); + cudaSupport = (combo.accelerator == "-cuda"); + neuronSupport = (combo.accelerator == "-neuron"); + enableAwsTuning = (combo.platform == "-aws"); + enableNVTXTracing = (combo.accelerator == "-cuda" && (builtins.elem "-nvtx" combo.tracing)); + enableLTTNGTracing = (builtins.elem "-lttng" combo.tracing); + enableValgrind = (combo.memory == "-valgrind"); + enableTracePrints = (combo.traceprints == "-trace"); + enableCPPMode = (combo.cpp == "-cpp"); + }).overrideAttrs (pprev: { + src = inputs.self; + version = mkGitVersion inputs.self; + }); + + genAttrsFromCombo = { pkgs }: combo: let + value = (genPkgFromCombo pkgs combo); + name = (genComboName pkgs combo value.pname); + in { inherit name; value = value.overrideAttrs { inherit name; }; }; +} diff --git a/.nix/overlay.nix b/.nix/overlay.nix new file mode 100644 index 000000000..632ee3ed2 --- /dev/null +++ b/.nix/overlay.nix @@ -0,0 +1,96 @@ +{ inputs }: +(final: prev: +let + lib = prev.lib // (import ./lib.nix { inherit inputs; lib = prev.lib; }); +in +{ + + libgdrcopy = final.cudaPackages.backendStdenv.mkDerivation rec { + pname = "libgdrcopy"; + src = inputs.gdrcopy; + version = lib.mkGitVersion inputs.gdrcopy; + makeFlags = [ + "LIB_MAJOR_VER=2" + "LIB_MINOR_VER=5" + "DESTLIB=$out/lib" + "DESTINC=$out/include" + "GDRAPI_ARCH=X86" + ]; + patchPhase = "chmod +x config_arch"; + buildPhase = "make -C src all"; + depsTargetTarget = with final.cudaPackages; [ cuda_cudart ]; + installPhase = "mkdir -p $out/lib && make ${lib.strings.concatStringsSep " " makeFlags} lib_install"; + }; + + rdma-core = prev.rdma-core.overrideAttrs (pprev: { + src = inputs.rdma-core; + version = lib.mkGitVersion inputs.rdma-core; + }); + + hwloc = prev.hwloc.overrideAttrs (pprev: { + src = inputs.hwloc; + version = lib.mkGitVersion inputs.hwloc; + nativeBuildInputs = (pprev.nativeBuildInputs or [ ]) ++ [ prev.autoreconfHook ]; + }); + + # pmix/prrte/openmpi cannot support new hwloc + pmix = prev.pmix.override { hwloc = final.hwloc; }; + prrte = prev.prrte.override { hwloc = final.hwloc; }; + openmpi = (prev.openmpi.override { + cudaSupport = true; + libfabric = final.libfabric; + rdma-core = final.rdma-core; + hwloc = final.hwloc; + }).overrideAttrs (pprev: { + src = inputs.openmpi; + version = lib.mkGitVersion inputs.openmpi; + nativeBuildInputs = (pprev.nativeBuildInputs or [ ]) ++ [ + prev.autoconf + prev.automake + prev.libtool + prev.perl + prev.git + prev.flex + ]; + prePatch = '' + patchShebangs . + ./autogen.pl + ''; + outputs = final.lib.lists.remove "man" pprev.outputs; + NIX_CFLAGS_COMPILE = "-Wno-deprecated-declarations"; + }); + + libfabric = (prev.libfabric.override { + enableOpx = false; + enablePsm2 = false; + }).overrideAttrs (pprev: { + src = inputs.libfabric; + version = lib.mkGitVersion inputs.libfabric; + configureFlags = (prev.configureFlags or [ ]) ++ [ + "--enable-efa=yes" + "--with-cuda=${prev.lib.getDev final.cudaPackages.cudatoolkit}" + "--enable-cuda-dlopen" + "--with-gdrcopy=${prev.lib.getDev final.libgdrcopy}" + "--enable-gdrcopy-dlopen" + ]; + buildInputs = (pprev.buildInputs or [ ]) ++ [ + final.rdma-core + ]; + }); + + cudaPackages = prev.cudaPackages.overrideScope (ffinal: pprev: rec { + nccl = pprev.nccl.overrideAttrs { + src = inputs.nccl; + version = lib.mkGitVersion inputs.nccl; + }; + nccl-tests = (pprev.nccl-tests.overrideAttrs { + src = inputs.nccl-tests; + version = lib.mkGitVersion inputs.nccl-tests; + }).override { + mpiSupport = true; + mpi = final.openmpi; + cudaPackages = pprev.cudaPackages // { inherit nccl; }; + config.cudaSupport = true; + }; + }); +}) diff --git a/.nix/shell.nix b/.nix/shell.nix new file mode 100644 index 000000000..2b2fc8fa5 --- /dev/null +++ b/.nix/shell.nix @@ -0,0 +1,28 @@ +{ self, system, inputs, pkgs }: +pkgs.mkShell { + inputsFrom = [ self.packages.${system}.default ]; + packages = [ + pkgs.clang-analyzer + pkgs.clang-tools + pkgs.clang + pkgs.gcc + pkgs.gdb + + pkgs.ccache + pkgs.cppcheck + pkgs.universal-ctags + pkgs.rtags + pkgs.ripgrep + pkgs.irony-server + pkgs.bear + pkgs.compdb + pkgs.act + + pkgs.gh + pkgs.git + pkgs.eksctl + pkgs.awscli2 + + pkgs.nixpkgs-fmt + ]; +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..0caf7dd1d --- /dev/null +++ b/flake.lock @@ -0,0 +1,208 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": [ + "systems" + ] + }, + "locked": { + "lastModified": 1710146030, + "narHash": "sha256-SZ5L6eA7HJ/nmkzGG7/ISclqe6oZdOZTNoesiInkXPQ=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "b1d9ab70662946ef0850d488da1c9019f3a9752a", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "gdrcopy": { + "flake": false, + "locked": { + "lastModified": 1720403983, + "narHash": "sha256-vK13tpyVHEQuI2x717XtgY/ByvF1Sx3LPcLsXnmcuDU=", + "owner": "NVIDIA", + "repo": "gdrcopy", + "rev": "1366e20d140c5638fcaa6c72b373ac69f7ab2532", + "type": "github" + }, + "original": { + "owner": "NVIDIA", + "repo": "gdrcopy", + "type": "github" + } + }, + "hwloc": { + "flake": false, + "locked": { + "lastModified": 1725633129, + "narHash": "sha256-feXQIEDEsTlbwV17/qAOUMjh4d1b5oXeEZoE2B+Q6w8=", + "owner": "open-mpi", + "repo": "hwloc", + "rev": "abd749327f3f27567034f19e7bd3e3a77f4e4296", + "type": "github" + }, + "original": { + "owner": "open-mpi", + "ref": "v2.x", + "repo": "hwloc", + "type": "github" + } + }, + "libfabric": { + "flake": false, + "locked": { + "lastModified": 1726518975, + "narHash": "sha256-cNMvPrKcKtjv4C53sqg70+55PLp717eu2gPdMr4wDPI=", + "owner": "ofiwg", + "repo": "libfabric", + "rev": "3d7bc021c8b4d7b7adfca7c17bb71e8cfa6463db", + "type": "github" + }, + "original": { + "owner": "ofiwg", + "ref": "main", + "repo": "libfabric", + "type": "github" + } + }, + "nccl": { + "flake": false, + "locked": { + "lastModified": 1726523257, + "narHash": "sha256-DlMxlLO2F079fBkhORNPVN/ASYiVIRfLJw7bDoiClHw=", + "owner": "NVIDIA", + "repo": "nccl", + "rev": "1a16f427507cb985123e7565874f57b77a1ba2df", + "type": "github" + }, + "original": { + "owner": "NVIDIA", + "repo": "nccl", + "type": "github" + } + }, + "nccl-tests": { + "flake": false, + "locked": { + "lastModified": 1722376734, + "narHash": "sha256-36Xk/CNr8uwPoqop3zXm2SzWpZUjqXy6M9uYRKIOu3Y=", + "owner": "NVIDIA", + "repo": "nccl-tests", + "rev": "9d26b8422ba76c098df996b96e13b8ddf3a71165", + "type": "github" + }, + "original": { + "owner": "NVIDIA", + "repo": "nccl-tests", + "type": "github" + } + }, + "nix-github-actions": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1720066371, + "narHash": "sha256-uPlLYH2S0ACj0IcgaK9Lsf4spmJoGejR9DotXiXSBZQ=", + "owner": "nix-community", + "repo": "nix-github-actions", + "rev": "622f829f5fe69310a866c8a6cd07e747c44ef820", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nix-github-actions", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1726481836, + "narHash": "sha256-MWTBH4dd5zIz2iatDb8IkqSjIeFum9jAqkFxgHLdzO4=", + "rev": "20f9370d5f588fb8c72e844c54511cab054b5f40", + "revCount": 682060, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/nixpkgs-weekly/0.1.682060%2Brev-20f9370d5f588fb8c72e844c54511cab054b5f40/0191fe7a-9a92-742d-80d8-41f210baba7c/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/DeterminateSystems/nixpkgs-weekly/0.1.678339.tar.gz" + } + }, + "openmpi": { + "flake": false, + "locked": { + "lastModified": 1725999768, + "narHash": "sha256-63YPwKc/8a+0aomlQPYnUpO6uB3VNPnxQ2omPwveGYY=", + "ref": "main", + "rev": "adee1a4bba9de414002b777df8b466b0f6133cad", + "revCount": 34498, + "submodules": true, + "type": "git", + "url": "https://github.com/open-mpi/ompi.git" + }, + "original": { + "ref": "main", + "submodules": true, + "type": "git", + "url": "https://github.com/open-mpi/ompi.git" + } + }, + "rdma-core": { + "flake": false, + "locked": { + "lastModified": 1726471809, + "narHash": "sha256-uHerTWawF38LX4Dr8+2NH4mPpTcihmbV1fAQRy1EP+Q=", + "owner": "linux-rdma", + "repo": "rdma-core", + "rev": "27b9ef34cd9272c656dca4d0d5d65a34b17a0150", + "type": "github" + }, + "original": { + "owner": "linux-rdma", + "ref": "master", + "repo": "rdma-core", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "gdrcopy": "gdrcopy", + "hwloc": "hwloc", + "libfabric": "libfabric", + "nccl": "nccl", + "nccl-tests": "nccl-tests", + "nix-github-actions": "nix-github-actions", + "nixpkgs": "nixpkgs", + "openmpi": "openmpi", + "rdma-core": "rdma-core", + "systems": "systems" + } + }, + "systems": { + "locked": { + "lastModified": 1689347949, + "narHash": "sha256-12tWmuL2zgBgZkdoB6qXZsgJEH9LR3oUgpaQq2RbI80=", + "owner": "nix-systems", + "repo": "default-linux", + "rev": "31732fcf5e8fea42e59c2488ad31a0e651500f68", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default-linux", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..e45ea1d08 --- /dev/null +++ b/flake.nix @@ -0,0 +1,109 @@ +# Copyright (c) 2024, Amazon.com, Inc. or its affiliates. All rights reserved. +# +# See LICENSE.txt for license information +# +{ + description = "libnccl-net-ofi flake. (UNSUPPORTED; INTENDED FOR DEVELOPERS ONLY)"; + + outputs = inputs: with inputs; + flake-utils.lib.eachDefaultSystem + (system: + let + pkgs = import inputs.nixpkgs { + inherit system; + overlays = [ (import ./.nix/overlay.nix { inherit inputs; }) ]; + config = { + cudaSupport = true; + allowUnfree = true; + }; + }; + lib = pkgs.lib // (import ./.nix/lib.nix { inherit inputs; inherit (pkgs) lib; }); + base-packages = builtins.listToAttrs (lib.attrsets.mapCartesianProduct (lib.genAttrsFromCombo { inherit pkgs; }) (import ./.nix/combos.nix)); + all-dep-packages = (pkgs.linkFarmFromDrvs "all_deps" (with pkgs; [ + cudaPackages.nccl-tests + cudaPackages.nccl + openmpi + libfabric + rdma-core + libgdrcopy + hwloc + ])); + filterPackages = s: (lib.attrsets.filterAttrs (k: v: lib.strings.hasInfix s k) base-packages); + makeCheck = name: filterString: pkgs.linkFarmFromDrvs name (builtins.attrValues (filterPackages filterString)); + in + { + inherit _base-packages; + checks = rec { + all = (makeCheck "all" ""); + default = all; + deps = all-dep-packages; + neuron = (makeCheck "neuron" "neuron"); + cuda = (makeCheck "cuda" "cuda"); + lttng = (makeCheck "lttng" "lttng"); + nvtx = (makeCheck "nvtx" "nvtx"); + clang = (makeCheck "clang" "clang"); + gcc7 = (makeCheck "gcc7" "gcc7"); + gcc14 = (makeCheck "gcc14" "gcc14"); + cpp = (makeCheck "cpp" "cpp"); + valgrind = (makeCheck "valgrind" "valgrind"); + debug = (makeCheck "debug" "debug"); + aws = (makeCheck "aws" "aws"); + }; + packages = { + inherit (pkgs) + libfabric + openmpi + rdma-core + libgdrcopy + hwloc + ; + inherit (pkgs.cudaPackages) + nccl + nccl-tests + ; + default = base-packages."libnccl-net-ofi-aws-lttng-clang18.1.8"; + }; + devShells.default = pkgs.callPackage ./.nix/shell.nix { + inherit self inputs pkgs system; + }; + } + ) // { + githubActionChecks = inputs.nix-github-actions.lib.mkGithubMatrix { + checks = nixpkgs.lib.getAttrs [ "x86_64-linux" ] self.outputs.base-packages; + }; + }; + + inputs = { + nixpkgs.url = "https://flakehub.com/f/DeterminateSystems/nixpkgs-weekly/0.1.678339.tar.gz"; + flake-utils.url = "github:numtide/flake-utils"; + systems.url = "github:nix-systems/default-linux"; + flake-utils.inputs.systems.follows = "systems"; + nix-github-actions.url = "github:nix-community/nix-github-actions"; + nix-github-actions.inputs.nixpkgs.follows = "nixpkgs"; + + nccl = { url = "github:NVIDIA/nccl"; flake = false; }; + nccl-tests = { url = "github:NVIDIA/nccl-tests"; flake = false; }; + gdrcopy = { url = "github:NVIDIA/gdrcopy"; flake = false; }; + rdma-core = { url = "github:linux-rdma/rdma-core/master"; flake = false; }; + libfabric = { url = "github:ofiwg/libfabric/main"; flake = false; }; + openmpi = { url = "git+https://github.com/open-mpi/ompi.git?ref=main&submodules=1"; flake = false; }; + hwloc = { url = "github:open-mpi/hwloc/v2.x"; flake = false; }; + }; + + nixConfig = { + allowUnfree = true; + cudaSupport = true; + extra-substituters = [ + "https://numtide.cachix.org" + "https://nix-community.cachix.org" + "https://devenv.cachix.org" + "https://cuda-maintainers.cachix.org" + ]; + extra-trusted-public-keys = [ + "numtide.cachix.org-1:2ps1kLBUWjxIneOy1Ik6cQjb41X0iXVXeHigGmycPPE=" + "nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" + "devenv.cachix.org-1:w1cLUi8dv3hnoSPGAuibQv+f9TZLr6cv/Hm9XgU50cw=" + "cuda-maintainers.cachix.org-1:0dq3bujKpuEPMCX6U4WylrUDZ9JyUG0VpVZa7CNfq5E=" + ]; + }; +}