Merge pull request #717 from zacikpa/affinity-error-logs

scheduler: Log process info when its affinity cannot be changed and re-check blacklisted cgroups

Author: yarda

tuned/plugins/plugin_scheduler.py

@@ -657,15 +657,24 @@ def _set_rt(self, pid, sched, prio):
 	def _is_kthread(self, process):
 		return process["stat"]["flags"] & procfs.pidstat.PF_KTHREAD != 0
 
+	def _process_in_blacklisted_cgroup(self, process):
+		if self._cgroup_ps_blacklist_re == "":
+			return False
+		return re.search(self._cgroup_ps_blacklist_re, self._get_stat_cgroup(process)) is not None
+
 	# Returns True if we can ignore a failed affinity change of
 	# a process with the given PID and therefore not report it as an error.
-	def _ignore_set_affinity_error(self, pid):
+	def _ignore_set_affinity_error(self, process):
+		pid = process.pid
 		try:
-			process = procfs.process(pid)
 			if process["stat"]["state"] == "Z":
 				log.debug("Affinity of zombie task with PID %d could not be changed."
 						% pid)
 				return True
+			if self._process_in_blacklisted_cgroup(process):
+				log.debug("Affinity of task with PID %d could not be changed, the task was moved into a blacklisted cgroup."
+						% pid)
+				return True
 			if process["stat"].is_bound_to_cpu():
 				if self._is_kthread(process):
 					log.debug("Affinity of kernel thread with PID %d cannot be changed, the task's affinity mask is fixed."
@@ -674,6 +683,9 @@ def _ignore_set_affinity_error(self, pid):
 					log.warning("Affinity of task with PID %d cannot be changed, the task's affinity mask is fixed."
 							% pid)
 				return True
+			log.info("Task %d cmdline: %s" % (pid, self._get_cmdline(process)))
+			log.info("Task %d cgroup: %s" % (pid, self._get_stat_cgroup(process)))
+			log.info("Task %d affinity: %s" % (pid, list(self._scheduler_utils.get_affinity(pid))))
 		except (OSError, IOError) as e:
 			if e.errno == errno.ENOENT or e.errno == errno.ESRCH:
 				log.debug("Failed to get task info for PID %d, the task vanished."
@@ -1181,13 +1193,17 @@ def _get_affinity(self, pid):
 		return res
 
 	def _set_affinity(self, pid, affinity):
+		process = procfs.process(pid)
+		if self._process_in_blacklisted_cgroup(process):
+			log.debug("Not setting CPU affinity of PID %d, the task belongs to a blacklisted cgroup." % pid)
+			return
 		log.debug("Setting CPU affinity of PID %d to '%s'." % (pid, affinity))
 		try:
 			self._scheduler_utils.set_affinity(pid, affinity)
 		# Workaround for old python-schedutils (pre-0.4) which
 		# incorrectly raised SystemError instead of OSError
 		except (SystemError, OSError) as e:
-			if not self._ignore_set_affinity_error(pid):
+			if not self._ignore_set_affinity_error(process):
 				log.error("Failed to set affinity of PID %d to '%s': %s"
 						% (pid, affinity, e))
 
@@ -1204,9 +1220,6 @@ def _set_all_obj_affinity(self, objs, affinity, threads = False):
 		if self._ps_blacklist != "":
 			psl = [v for v in psl if re.search(self._ps_blacklist,
 					self._get_stat_comm(v)) is None]
-		if self._cgroup_ps_blacklist_re != "":
-			psl = [v for v in psl if re.search(self._cgroup_ps_blacklist_re,
-					self._get_stat_cgroup(v)) is None]
 		psd = dict([(v.pid, v) for v in psl])
 		for pid in psd:
 			try: