refactor(pops/packages): add python support

tao3k · Nov 22, 2023 · 6ec2c0e · 6ec2c0e
1 parent 216e428
commit 6ec2c0e
Show file tree

Hide file tree

Showing 14 changed files with 280 additions and 75 deletions.
diff --git a/docs/org/pops-packages.org b/docs/org/pops-packages.org
@@ -5,7 +5,7 @@
 :EXP-DIR: ../../examples
 :CODE-DIR: ../../src/pops
 :END:
-#+TITLE: pops-packages
+#+TITLE: pops.packages
 #+AUTHOR: GuangTao Zhang
 #+EMAIL: gtrunsec@hardenedlinux.org
 #+DATE: 2023-11-19 Sun 16:25
@@ -16,51 +16,94 @@ function: ~omnibus.pops.pacakges~ is responsible for loading packages in a tree
 - code
 
 #+begin_src nix :tangle (concat (org-entry-get nil "CODE-DIR" t) "/packages.nix")
-{ super, root }:
+{
+  super,
+  root,
+  POP,
+  flops,
+  lib,
+}:
 load:
 let
-  inherit
-    (root.errors.requiredInputs load.inputs "omnibus.pops.packages" [ "nixpkgs" ])
-    nixpkgs
-  ;
-  inherit (nixpkgs) newScope;
-  inherit (nixpkgs.lib) makeScope;
+  inherit (flops) recursiveMerge';
 in
-makeScope newScope (
-  selfScope:
-  ((super.load load).addLoadExtender {
-    load = {
+(super.load (
+  recursiveMerge' [
+    {
       loader =
         __inputs__: path:
-        (selfScope.overrideScope (_: _: { inherit __inputs__; })).callPackage path { };
+        #  without the scope loader
+        (__inputs__.inputs.nixpkgs.extend (_: _: { inherit __inputs__; })).callPackage path { };
       transformer = [ (_cursor: dir: if dir ? default then dir.default else dir) ];
-    };
-  }).addExporters
-    [
-      (POP.extendPop flops.haumea.pops.exporter (
-        self: _super: {
-          exports = {
-            overlay =
-              final: prev:
+    }
+    load
+  ]
+)).addExporters
+  [
+    (POP.extendPop flops.haumea.pops.exporter (
+      self: _super: {
+        exports =
+          let
+            inherit
+              (root.errors.requiredInputs self.layouts.self.load.inputs "omnibus.pops.packages" [ "nixpkgs" ])
+              nixpkgs
+            ;
+            inherit (nixpkgs) newScope;
+            inherit (nixpkgs.lib) makeScope;
+          in
+          {
+            derivations = lib.attrsets.filterDerivations self.exports.packages;
+
+            scopePackagesPop =
+              selfScope:
               (self.layouts.self.addLoadExtender {
                 load = {
-                  inputs.inputs.nixpkgs = final;
                   loader =
-                    __inputs__: path:
-                    (__inputs__.inputs.nixpkgs.extend (_: _: { inherit __inputs__; })).callPackage
-                      path
-                      { };
+                    __inputs__: path: (selfScope.overrideScope (_: _: { inherit __inputs__; })).callPackage path { };
+                  inputs = {
+                    callPackagesWithOmnibus =
+                      selfScope: src:
+                      (super.load {
+                        loader =
+                          _: path:
+                          (selfScope.overrideScope (_: _: { __inputs__ = self.layouts.self.load.inputs; })).callPackage path
+                            { };
+                        src = src;
+                      }).exports.default;
+                  };
                 };
-              }).exports.default;
+              });
+
+            packages = makeScope newScope (
+              selfScope: (self.exports.scopePackagesPop selfScope).exports.default
+            );
+
+            overlays = {
+              default =
+                final: prev:
+                ((self.exports.packages.overrideScope (
+                  _: scopeSuper: {
+                    python3Packages = prev.python3Packages.override (
+                      old: {
+                        overrides = prev.lib.composeExtensions (old.packageOverrides or (_: _: { })) (
+                          pythonSelf: _: if scopeSuper ? py then scopeSuper.py.packages pythonSelf else { }
+                        );
+                      }
+                    );
+                  }
+                )).packages
+                  (final // { overrideScope = self.exports.packages.overrideScope; })
+                );
+            };
           };
-        }
-      ))
-    ]
-)
+      }
+    ))
+  ]
 #+end_src
 
 - *pops.packages* have an arg named by anti-namespace of *__inputs__* in the scopePackages, that *__inptus__* can pass all args same as haumea.inputs, such as {super,self, ...} => {__inputs__.self, __inputs__.super, inputs.{...} }
 
+- callPythonPackages: creating a named ~py~ directory for loading the python packages.
 
 ** Example
 
@@ -103,6 +146,6 @@ nix-repl> :p Flake.examples.packages.exports.overlay
 { super, omnibus }:
 let
   nixos-23_05 = omnibus.flake.inputs.nixos-23_05;
-in
-nixos-23_05.legacyPackages.x86_64-linux.extend super.packages.exports.overlay
+In
+nixos-23_05.legacyPackages.x86_64-linux.extend super.packages.exports.overlays.default
     #+end_src
diff --git a/docs/org/pops.org b/docs/org/pops.org
@@ -14,5 +14,8 @@ Please note that in Pops, the outputs/results should not be directly printed, bu
 #+end_quote
 
 * Loaders
+:PROPERTIES:
+:ID:       ad6e2605-7cf2-46e4-b828-493f950b58d8
+:END:
 
 - [[id:f7adb0ad-2cc1-4723-a796-bf608682456a][pops-packages]] (loading the packages into nixpkgs from a directory)
diff --git a/examples/packages/__fixture/py/btest_1.nix b/examples/packages/__fixture/py/btest_1.nix
@@ -0,0 +1,37 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  pythonOlder,
+  pkgs,
+}:
+
+buildPythonPackage rec {
+  pname = "btest";
+  version = "1.1-test";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.7";
+
+  src = fetchFromGitHub {
+    owner = "zeek";
+    repo = "btest";
+    rev = "refs/tags/v${version}";
+    hash = "sha256-D01hAKcE52eKJRUh1/x5DGxRQpWgA2J0nutshpKrtRU=";
+  };
+
+  # No tests available and no module to import
+  doCheck = false;
+
+  passthru = {
+    __inputs__ = pkgs.__inputs__;
+  };
+
+  meta = with lib; {
+    description = "A Generic Driver for Powerful System Tests";
+    homepage = "https://github.com/zeek/btest";
+    changelog = "https://github.com/zeek/btest/blob/${version}/CHANGES";
+    license = licenses.bsd3;
+    maintainers = with maintainers; [ fab ];
+  };
+}
diff --git a/examples/packages/__fixture/py/btest_2.nix b/examples/packages/__fixture/py/btest_2.nix
@@ -0,0 +1,37 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  pythonOlder,
+  btest_1,
+}:
+
+buildPythonPackage rec {
+  pname = "btest";
+  version = "1.2-test";
+  format = "setuptools";
+
+  disabled = pythonOlder "3.7";
+
+  src = fetchFromGitHub {
+    owner = "zeek";
+    repo = "btest";
+    rev = "refs/tags/v${version}";
+    hash = "sha256-D01hAKcE52eKJRUh1/x5DGxRQpWgA2J0nutshpKrtRU=";
+  };
+
+  # No tests available and no module to import
+  doCheck = false;
+
+  passthru = {
+    btest = btest_1;
+  };
+
+  meta = with lib; {
+    description = "A Generic Driver for Powerful System Tests";
+    homepage = "https://github.com/zeek/btest";
+    changelog = "https://github.com/zeek/btest/blob/${version}/CHANGES";
+    license = licenses.bsd3;
+    maintainers = with maintainers; [ fab ];
+  };
+}
diff --git a/examples/packages/__fixture/py/default.nix b/examples/packages/__fixture/py/default.nix
@@ -0,0 +1,12 @@
+{
+  python3Packages,
+  python3,
+  lib,
+  __inputs__,
+}:
+let
+  inherit (lib) makeScope;
+  inherit (python3Packages) newScope;
+  inherit (__inputs__) callPackagesWithOmnibus;
+in
+makeScope newScope (selfScope: callPackagesWithOmnibus selfScope ./.)
diff --git a/examples/packages/default.nix b/examples/packages/default.nix
@@ -4,11 +4,14 @@
 
 # [[file:../../docs/org/pops-packages.org::*Example][Example:1]]
 { omnibus, inputs }:
-omnibus.pops.packages {
+let
+  nixpkgs = inputs.nixpkgs.legacyPackages.x86_64-linux;
+in
+(omnibus.pops.packages {
   src = ./__fixture;
   inputs = {
-    nixpkgs = inputs.nixpkgs.legacyPackages.x86_64-linux;
+    inherit nixpkgs;
   };
-}
+})
 # => out.exports { default = {...}, packages = {...}; }
 # Example:1 ends here
diff --git a/examples/packagesOverlay.nix b/examples/packagesOverlay.nix
@@ -7,5 +7,5 @@
 let
   nixos-23_05 = omnibus.flake.inputs.nixos-23_05;
 in
-nixos-23_05.legacyPackages.x86_64-linux.extend super.packages.exports.overlay
+nixos-23_05.legacyPackages.x86_64-linux.extend super.packages.exports.overlays.default
 # Example:3 ends here
diff --git a/flake.nix b/flake.nix
@@ -22,6 +22,10 @@
       inherit src;
       inherit (src) lib ops errors;
 
+      nixci.default = {
+        dir1.dir = "./local";
+      };
+
       templates = {
         nixos = {
           path = ./templates/nixos;

diff --git a/local/cells/repo/shells.nix b/local/cells/repo/shells.nix
@@ -34,6 +34,7 @@ in
       nixpkgs.tree
 
       nixpkgs.reuse
+      nixpkgs.nixci
     ];
 
     commands = [ ];

diff --git a/local/flake.nix b/local/flake.nix
@@ -32,7 +32,6 @@
       inputs.nixpkgs.follows = "nixpkgs";
       inputs.nixago-exts.follows = "";
     };
-
     devshell = {
       url = "github:numtide/devshell";
       inputs.nixpkgs.follows = "nixpkgs";