feat(linstor): parallelize SR.scan/cleanup (vhd-util calls)

.github/workflows/main.yml

@@ -3,15 +3,34 @@ name: Test SM
 on: [push, pull_request]
 
 jobs:
+  static-analysis:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python 3
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev_requirements_static_analysis.txt
+
+      - name: mypy
+        run: |
+          mypy .
+
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
 
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python 3
         uses: actions/setup-python@v4
         with:
-          python-version: '3.6'
+          python-version: '3.8' # This version is the closest available to the one currently used in production (3.6).
 
       - name: Install dependencies
         run: |

.mypy.ini

@@ -0,0 +1,3 @@
+[mypy]
+enable_error_code = explicit-override
+

Makefile

@@ -8,12 +8,19 @@ SM_DRIVERS += Dummy
 SM_DRIVERS += udev
 SM_DRIVERS += ISO
 SM_DRIVERS += HBA
+SM_DRIVERS += Linstor
 SM_DRIVERS += LVHD
 SM_DRIVERS += LVHDoISCSI
 SM_DRIVERS += LVHDoHBA
 SM_DRIVERS += SHM
 SM_DRIVERS += SMB
 SM_DRIVERS += LVHDoFCoE
+SM_DRIVERS += CephFS
+SM_DRIVERS += GlusterFS
+SM_DRIVERS += XFS
+SM_DRIVERS += ZFS
+SM_DRIVERS += MooseFS
+SM_DRIVERS += LargeBlock
 
 SM_LIBS := SR
 SM_LIBS += SRCommand
@@ -27,6 +34,9 @@ SM_LIBS += verifyVHDsOnSR
 SM_LIBS += scsiutil
 SM_LIBS += scsi_host_rescan
 SM_LIBS += vhdutil
+SM_LIBS += linstorjournaler
+SM_LIBS += linstorvhdutil
+SM_LIBS += linstorvolumemanager
 SM_LIBS += lvhdutil
 SM_LIBS += cifutils
 SM_LIBS += xs_errors
@@ -77,6 +87,7 @@ PLUGIN_SCRIPT_DEST := /etc/xapi.d/plugins/
 LIBEXEC := /opt/xensource/libexec/
 UDEV_RULES_DIR := /etc/udev/rules.d/
 UDEV_SCRIPTS_DIR := /etc/udev/scripts/
+SYSTEMD_CONF_DIR := /etc/systemd/system/
 SYSTEMD_SERVICE_DIR := /usr/lib/systemd/system/
 INIT_DIR := /etc/rc.d/init.d/
 MPATH_CONF_DIR := /etc/multipath.xenserver/
@@ -92,7 +103,8 @@ SM_PY_FILES = $(foreach LIB, $(SM_LIBS), drivers/$(LIB).py) $(foreach DRIVER, $(
 
 .PHONY: build
 build:
-	make -C dcopy 
+	make -C dcopy
+	make -C linstor
 
 .PHONY: precommit
 precommit: build
@@ -115,14 +127,17 @@ precheck: build
 	echo "Precheck succeeded with no outstanding issues found."
 
 .PHONY: install
-install: precheck
+install: build
 	mkdir -p $(SM_STAGING)
 	$(call mkdir_clean,$(SM_STAGING))
 	mkdir -p $(SM_STAGING)$(SM_DEST)
 	mkdir -p $(SM_STAGING)$(SM_DEST)/plugins
 	mkdir -p $(SM_STAGING)$(UDEV_RULES_DIR)
 	mkdir -p $(SM_STAGING)$(UDEV_SCRIPTS_DIR)
 	mkdir -p $(SM_STAGING)$(INIT_DIR)
+	mkdir -p $(SM_STAGING)$(SYSTEMD_CONF_DIR)
+	mkdir -p $(SM_STAGING)$(SYSTEMD_CONF_DIR)/drbd-reactor.service.d
+	mkdir -p $(SM_STAGING)$(SYSTEMD_CONF_DIR)/linstor-satellite.service.d
 	mkdir -p $(SM_STAGING)$(SYSTEMD_SERVICE_DIR)
 	mkdir -p $(SM_STAGING)$(MPATH_CONF_DIR)
 	mkdir -p $(SM_STAGING)$(MPATH_CUSTOM_CONF_DIR)
@@ -151,6 +166,12 @@ install: precheck
 	  $(SM_STAGING)/$(SM_DEST)
 	install -m 644 etc/logrotate.d/$(SMLOG_CONF) \
 	  $(SM_STAGING)/$(LOGROTATE_DIR)
+	install -m 644 etc/systemd/system/drbd-reactor.service.d/override.conf \
+	  $(SM_STAGING)/$(SYSTEMD_CONF_DIR)/drbd-reactor.service.d/
+	install -m 644 etc/systemd/system/linstor-satellite.service.d/override.conf \
+	  $(SM_STAGING)/$(SYSTEMD_CONF_DIR)/linstor-satellite.service.d/
+	install -m 644 etc/systemd/system/var-lib-linstor.service \
+	  $(SM_STAGING)/$(SYSTEMD_CONF_DIR)
 	install -m 644 etc/make-dummy-sr.service \
 	  $(SM_STAGING)/$(SYSTEMD_SERVICE_DIR)
 	install -m 644 systemd/xs-sm.service \
@@ -169,6 +190,8 @@ install: precheck
 	  $(SM_STAGING)/$(SYSTEMD_SERVICE_DIR)
 	install -m 644 systemd/SMGC@.service \
 	  $(SM_STAGING)/$(SYSTEMD_SERVICE_DIR)
+	install -m 644 systemd/linstor-monitor.service \
+	  $(SM_STAGING)/$(SYSTEMD_SERVICE_DIR)
 	for i in $(UDEV_RULES); do \
 	  install -m 644 udev/$$i.rules \
 	    $(SM_STAGING)$(UDEV_RULES_DIR); done
@@ -186,6 +209,7 @@ install: precheck
 	cd $(SM_STAGING)$(SM_DEST) && rm -f LVHDoFCoESR && ln -sf LVHDoFCoESR.py LVMoFCoESR
 	ln -sf $(SM_DEST)mpathutil.py $(SM_STAGING)/sbin/mpathutil
 	install -m 755 drivers/02-vhdcleanup $(SM_STAGING)$(MASTER_SCRIPT_DEST)
+	install -m 755 drivers/linstor-manager $(SM_STAGING)$(PLUGIN_SCRIPT_DEST)
 	install -m 755 drivers/lvhd-thin $(SM_STAGING)$(PLUGIN_SCRIPT_DEST)
 	install -m 755 drivers/on_slave.py $(SM_STAGING)$(PLUGIN_SCRIPT_DEST)/on-slave
 	install -m 755 drivers/testing-hooks $(SM_STAGING)$(PLUGIN_SCRIPT_DEST)
@@ -197,6 +221,9 @@ install: precheck
 	install -m 755 drivers/iscsilib.py $(SM_STAGING)$(SM_DEST)
 	install -m 755 drivers/fcoelib.py $(SM_STAGING)$(SM_DEST)
 	mkdir -p $(SM_STAGING)$(LIBEXEC)
+	install -m 755 scripts/fork-log-daemon $(SM_STAGING)$(LIBEXEC)
+	install -m 755 scripts/linstor-kv-tool $(SM_STAGING)$(BIN_DEST)
+	install -m 755 scripts/safe-umount $(SM_STAGING)$(LIBEXEC)
 	install -m 755 scripts/local-device-change $(SM_STAGING)$(LIBEXEC)
 	install -m 755 scripts/check-device-sharing $(SM_STAGING)$(LIBEXEC)
 	install -m 755 scripts/usb_change $(SM_STAGING)$(LIBEXEC)
@@ -205,6 +232,8 @@ install: precheck
 	mkdir -p $(SM_STAGING)/etc/xapi.d/xapi-pre-shutdown/
 	install -m 755 scripts/stop_all_gc $(SM_STAGING)/etc/xapi.d/xapi-pre-shutdown/
 	$(MAKE) -C dcopy install DESTDIR=$(SM_STAGING)
+	$(MAKE) -C linstor install DESTDIR=$(SM_STAGING)
+	$(MAKE) -C sm_typing install DESTDIR=$(SM_STAGING)
 	ln -sf $(SM_DEST)blktap2.py $(SM_STAGING)$(BIN_DEST)/blktap2
 	ln -sf $(SM_DEST)lcache.py $(SM_STAGING)$(BIN_DEST)tapdisk-cache-stats
 	ln -sf /dev/null $(SM_STAGING)$(UDEV_RULES_DIR)/69-dm-lvm-metad.rules
@@ -215,4 +244,3 @@ install: precheck
 .PHONY: clean
 clean:
 	rm -rf $(SM_STAGING)
-

dev_requirements.txt

@@ -1,4 +1,5 @@
 coverage
-astroid==2.3.3
-pylint==2.4.4
+astroid
+pylint
 bitarray
+python-linstor

dev_requirements_static_analysis.txt

@@ -0,0 +1,3 @@
+bitarray
+mypy
+python-linstor

drivers/BaseISCSI.py

@@ -18,7 +18,10 @@
 # ISCSISR: ISCSI software initiator SR driver
 #
 
+from sm_typing import override
+
 import SR
+import VDI
 import util
 import time
 import LUNperVDI
@@ -100,11 +103,12 @@ def address(self):
             self._initPaths()
         return self._address
 
-    def handles(type):
+    @override
+    @staticmethod
+    def handles(type) -> bool:
         return False
-    handles = staticmethod(handles)
 
-    def _synchroniseAddrList(self, addrlist):
+    def _synchroniseAddrList(self, addrlist) -> None:
         if not self.multihomed:
             return
         change = False
@@ -133,7 +137,8 @@ def _synchroniseAddrList(self, addrlist):
             except:
                 pass
 
-    def load(self, sr_uuid):
+    @override
+    def load(self, sr_uuid) -> None:
         if self.force_tapdisk:
             self.sr_vditype = 'aio'
         else:
@@ -175,23 +180,23 @@ def load(self, sr_uuid):
                 and ('chappassword' in self.dconf or 'chappassword_secret' in self.dconf):
             self.chapuser = self.dconf['chapuser'].encode('utf-8')
             if 'chappassword_secret' in self.dconf:
-                self.chappassword = util.get_secret(self.session, self.dconf['chappassword_secret'])
+                chappassword = util.get_secret(self.session, self.dconf['chappassword_secret'])
             else:
-                self.chappassword = self.dconf['chappassword']
+                chappassword = self.dconf['chappassword']
 
-            self.chappassword = self.chappassword.encode('utf-8')
+            self.chappassword = chappassword.encode('utf-8')
 
         self.incoming_chapuser = ""
         self.incoming_chappassword = ""
         if 'incoming_chapuser' in self.dconf \
                 and ('incoming_chappassword' in self.dconf or 'incoming_chappassword_secret' in self.dconf):
             self.incoming_chapuser = self.dconf['incoming_chapuser'].encode('utf-8')
             if 'incoming_chappassword_secret' in self.dconf:
-                self.incoming_chappassword = util.get_secret(self.session, self.dconf['incoming_chappassword_secret'])
+                incoming_chappassword = util.get_secret(self.session, self.dconf['incoming_chappassword_secret'])
             else:
-                self.incoming_chappassword = self.dconf['incoming_chappassword']
+                incoming_chappassword = self.dconf['incoming_chappassword']
 
-            self.incoming_chappassword = self.incoming_chappassword.encode('utf-8')
+            self.incoming_chappassword = incoming_chappassword.encode('utf-8')
 
         self.port = DEFAULT_PORT
         if 'port' in self.dconf and self.dconf['port']:
@@ -276,7 +281,7 @@ def _initPaths(self):
             self._address = self.tgtidx
         self._synchroniseAddrList(addrlist)
 
-    def _init_adapters(self):
+    def _init_adapters(self) -> None:
         # Generate a list of active adapters
         ids = scsiutil._genHostList(ISCSI_PROCNAME)
         util.SMlog(ids)
@@ -293,7 +298,8 @@ def _init_adapters(self):
                 pass
         self._devs = scsiutil.cacheSCSIidentifiers()
 
-    def attach(self, sr_uuid):
+    @override
+    def attach(self, sr_uuid) -> None:
         self._mpathHandle()
 
         multiTargets = False
@@ -391,10 +397,9 @@ def attach(self, sr_uuid):
         util._incr_iscsiSR_refcount(self.targetIQN, sr_uuid)
         IQNs = []
         if "multiSession" in self.dconf:
-            IQNs = ""
             for iqn in self.dconf['multiSession'].split("|"):
                 if len(iqn):
-                    IQNs += iqn.split(',')[2]
+                    IQNs.append(iqn.split(',')[2])
         else:
             IQNs.append(self.targetIQN)
 
@@ -433,7 +438,11 @@ def attach(self, sr_uuid):
                 realdev = os.path.realpath(os.path.join(dev_path, dev))
                 util.set_scheduler(os.path.basename(realdev))
 
-    def detach(self, sr_uuid, delete=False):
+    @override
+    def detach(self, sr_uuid) -> None:
+        self.detach_and_delete(sr_uuid, delete=False)
+
+    def detach_and_delete(self, sr_uuid, delete=True) -> None:
         keys = []
         pbdref = None
         try:
@@ -474,7 +483,8 @@ def detach(self, sr_uuid, delete=False):
 
         self.attached = False
 
-    def create(self, sr_uuid, size):
+    @override
+    def create(self, sr_uuid, size) -> None:
         # Check whether an SR already exists
         SRs = self.session.xenapi.SR.get_all_records()
         for sr in SRs:
@@ -503,11 +513,13 @@ def create(self, sr_uuid, size):
         self.session.xenapi.SR.set_sm_config(self.sr_ref, self.sm_config)
         return
 
-    def delete(self, sr_uuid):
+    @override
+    def delete(self, sr_uuid) -> None:
         self.detach(sr_uuid)
         return
 
-    def probe(self):
+    @override
+    def probe(self) -> str:
         SRs = self.session.xenapi.SR.get_all_records()
         Recs = {}
         for sr in SRs:
@@ -517,8 +529,9 @@ def probe(self):
                sm_config['targetIQN'] == self.targetIQN:
                 Recs[record["uuid"]] = sm_config
         return self.srlist_toxml(Recs)
-
-    def scan(self, sr_uuid):
+
+    @override
+    def scan(self, sr_uuid) -> None:
         if not self.passthrough:
             if not self.attached:
                 raise xs_errors.XenError('SRUnavailable')
@@ -530,9 +543,10 @@ def scan(self, sr_uuid):
                 if vdi.managed:
                     self.physical_utilisation += vdi.size
             self.virtual_allocation = self.physical_utilisation
-        return super(BaseISCSISR, self).scan(sr_uuid)
+        super(BaseISCSISR, self).scan(sr_uuid)
 
-    def vdi(self, uuid):
+    @override
+    def vdi(self, uuid) -> VDI.VDI:
         return LUNperVDI.RAWVDI(self, uuid)
 
     def _scan_IQNs(self):