From 2271a4543e3afd317c635055a410c5c06b8c1312 Mon Sep 17 00:00:00 2001 From: raylliu Date: Sat, 14 Sep 2024 11:29:35 +0800 Subject: [PATCH 1/2] [jb] Fix task log SSL: CERTIFICATE_VERIFY_FAILED error in yarn https-only env. --- desktop/libs/hadoop/src/hadoop/yarn/clients.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/desktop/libs/hadoop/src/hadoop/yarn/clients.py b/desktop/libs/hadoop/src/hadoop/yarn/clients.py index d70f4809f7f..94f9f3f1b0d 100644 --- a/desktop/libs/hadoop/src/hadoop/yarn/clients.py +++ b/desktop/libs/hadoop/src/hadoop/yarn/clients.py @@ -22,7 +22,6 @@ import sys import threading import time -import urllib.parse import heapq from desktop.lib.rest.http_client import HttpClient @@ -61,6 +60,7 @@ def get_log_client(log_link): if client_tuple is None: client = HttpClient(base_url, logger=LOG) yarn_cluster = cluster.get_cluster_conf_for_job_submission() + client.set_verify(yarn_cluster.SSL_CERT_CA_VERIFY.get()) if yarn_cluster.SECURITY_ENABLED.get(): client.set_kerberos_auth() else: From 19ca94c7f860a17bcf26023c9261f39743a50918 Mon Sep 17 00:00:00 2001 From: raylliu Date: Sat, 14 Sep 2024 15:04:00 +0800 Subject: [PATCH 2/2] [jb]The log is blank and the interface is empty in the SparkSQL task --- desktop/libs/hadoop/src/hadoop/yarn/spark_history_server_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/desktop/libs/hadoop/src/hadoop/yarn/spark_history_server_api.py b/desktop/libs/hadoop/src/hadoop/yarn/spark_history_server_api.py index 6b7fcb23be3..1a9dc66698e 100644 --- a/desktop/libs/hadoop/src/hadoop/yarn/spark_history_server_api.py +++ b/desktop/libs/hadoop/src/hadoop/yarn/spark_history_server_api.py @@ -171,7 +171,7 @@ def retrieve_log_content(self, log_links, log_name, username, offset): def get_executors_loglinks(self, job): executor = None if job.metrics and 'executors' in job.metrics and job.metrics['executors']: - executors = [executor for executor in job.metrics['executors'] if executor[0] == 'driver'] # look up driver executor + executors = [executor for executor in job.metrics['executors'] if executor[0] != 'driver'] # look up driver executor if not executors: executor = job.metrics['executors'][0] else: