From 5ec331d40219f22a37e91aa868db4d60d67cf39c Mon Sep 17 00:00:00 2001
From: Ananya_Agarwal <68558847+ananya-agarwal@users.noreply.github.com>
Date: Thu, 5 Sep 2024 14:16:20 +0530
Subject: [PATCH] [Threads] Remove Threads tab from Administrator Server Tab
(#3815)
* Basic frontend file found
* [Threads]Remove Threads tab from Admin Pages
* Removing the frontend file
* Removing the desktop.views.threads test
* Removing the COLLECT_THREADS instance from hue_collect_data.sh file
* Removing the hue_collect_data.sh file
* Adding test function back
* Passing the errors on GitHub
* Fixes based on runruff linting
* nit
* Fixes based on review comments
* Final changes based on review comments
---
.../core/src/desktop/js/onePageViewModel.js | 9 -
.../src/desktop/templates/about_layout.mako | 3 -
.../templates/global_js_constants.mako | 1 -
.../core/src/desktop/templates/threads.mako | 61 ---
desktop/core/src/desktop/tests.py | 11 +-
desktop/core/src/desktop/urls.py | 1 -
desktop/core/src/desktop/views.py | 131 ++---
tools/ops/hue_collect_data.sh | 464 ------------------
8 files changed, 73 insertions(+), 608 deletions(-)
delete mode 100644 desktop/core/src/desktop/templates/threads.mako
delete mode 100755 tools/ops/hue_collect_data.sh
diff --git a/desktop/core/src/desktop/js/onePageViewModel.js b/desktop/core/src/desktop/js/onePageViewModel.js
index 6cc5db2a3fa..3955ef45850 100644
--- a/desktop/core/src/desktop/js/onePageViewModel.js
+++ b/desktop/core/src/desktop/js/onePageViewModel.js
@@ -533,15 +533,6 @@ class OnePageViewModel {
}
},
{ url: '/desktop/dump_config', app: 'dump_config' },
- {
- url: '/desktop/debug/threads',
- app: function () {
- self.loadApp('threads');
- self.getActiveAppViewModel(viewModel => {
- viewModel.fetchThreads();
- });
- }
- },
{
url: '/gist',
app: function () {
diff --git a/desktop/core/src/desktop/templates/about_layout.mako b/desktop/core/src/desktop/templates/about_layout.mako
index 7fa132e395f..3c15eb6a81a 100644
--- a/desktop/core/src/desktop/templates/about_layout.mako
+++ b/desktop/core/src/desktop/templates/about_layout.mako
@@ -67,9 +67,6 @@ def is_selected(section, matcher):
${_('Server Logs')}
-
- ${_('Threads')}
-
% if METRICS.ENABLE_WEB_METRICS.get():
${_('Metrics')}
diff --git a/desktop/core/src/desktop/templates/global_js_constants.mako b/desktop/core/src/desktop/templates/global_js_constants.mako
index 1aed9aa7d62..246b2ab7162 100644
--- a/desktop/core/src/desktop/templates/global_js_constants.mako
+++ b/desktop/core/src/desktop/templates/global_js_constants.mako
@@ -102,7 +102,6 @@
admin_wizard: { url: '/about/admin_wizard', title: '${_('Admin Wizard')}' },
logs: { url: '/logs', title: '${_('Logs')}' },
dump_config: { url: '/desktop/dump_config', title: '${_('Dump Configuration')}' },
- threads: { url: '/desktop/debug/threads', title: '${_('Threads')}' },
metrics: { url: '/desktop/metrics', title: '${_('Metrics')}' },
taskserver: { url: '/task_server', title: '${_('Task Server')}' },
connectors: { url: '/desktop/connectors', title: '${_('Connectors')}' },
diff --git a/desktop/core/src/desktop/templates/threads.mako b/desktop/core/src/desktop/templates/threads.mako
deleted file mode 100644
index cb27175d44b..00000000000
--- a/desktop/core/src/desktop/templates/threads.mako
+++ /dev/null
@@ -1,61 +0,0 @@
-## Licensed to Cloudera, Inc. under one
-## or more contributor license agreements. See the NOTICE file
-## distributed with this work for additional information
-## regarding copyright ownership. Cloudera, Inc. licenses this file
-## to you under the Apache License, Version 2.0 (the
-## "License"); you may not use this file except in compliance
-## with the License. You may obtain a copy of the License at
-##
-## http://www.apache.org/licenses/LICENSE-2.0
-##
-## Unless required by applicable law or agreed to in writing, software
-## distributed under the License is distributed on an "AS IS" BASIS,
-## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-## See the License for the specific language governing permissions and
-## limitations under the License.
-<%!
-import sys
-from desktop.views import commonheader, commonfooter
-if sys.version_info[0] > 2:
- from django.utils.translation import gettext as _
-else:
- from django.utils.translation import ugettext as _
-%>
-
-<%namespace name="actionbar" file="actionbar.mako" />
-<%namespace name="layout" file="about_layout.mako" />
-
-%if not is_embeddable:
-${ commonheader(_('Threads'), "about", user, request) | n,unicode }
-%endif
-
-
-
-
-${layout.menubar(section='threads')}
-
-
-
-%if not is_embeddable:
-${ commonfooter(request, messages) | n,unicode }
-%endif
diff --git a/desktop/core/src/desktop/tests.py b/desktop/core/src/desktop/tests.py
index 27d5a18f4c8..52b3a634105 100644
--- a/desktop/core/src/desktop/tests.py
+++ b/desktop/core/src/desktop/tests.py
@@ -357,13 +357,6 @@ def assert_page(page, data, start, end):
assert_page(pgn.page(2), list(range(20, 25)), 21, 25)
-@pytest.mark.django_db
-def test_thread_dump():
- c = make_logged_in_client()
- response = c.get("/desktop/debug/threads", HTTP_X_REQUESTED_WITH='XMLHttpRequest')
- assert b"test_thread_dump" in response.content
-
-
def test_truncating_model():
class TinyModel(TruncatingModel):
short_field = CharField(max_length=10)
@@ -674,7 +667,7 @@ def check_app(status_code, app_name):
@pytest.mark.django_db
def test_error_handling_failure():
# Change rewrite_user to call has_hue_permission
- # Try to get filebrowser page
+ # Try to get logs page
# test for default 500 page
# Restore rewrite_user
import desktop.auth.backend
@@ -698,7 +691,7 @@ def rewrite_user(user):
# Make sure we are showing default 500.html page.
# See django.test.client#L246
with pytest.raises(AttributeError):
- c.get(reverse('desktop.views.threads'))
+ c.get(reverse('desktop.views.log_view'))
finally:
# Restore the world
restore_django_debug()
diff --git a/desktop/core/src/desktop/urls.py b/desktop/core/src/desktop/urls.py
index 47e904571d4..94088853258 100644
--- a/desktop/core/src/desktop/urls.py
+++ b/desktop/core/src/desktop/urls.py
@@ -87,7 +87,6 @@
re_path(r'^desktop/status_bar/?$', desktop_views.status_bar),
re_path(r'^desktop/debug/is_alive$', desktop_views.is_alive),
re_path(r'^desktop/debug/is_idle$', desktop_views.is_idle),
- re_path(r'^desktop/debug/threads$', desktop_views.threads, name="desktop.views.threads"),
re_path(r'^desktop/debug/memory$', desktop_views.memory),
re_path(r'^desktop/debug/check_config$', desktop_views.check_config, name="desktop.views.check_config"),
re_path(r'^desktop/debug/check_config_ajax$', desktop_views.check_config_ajax),
diff --git a/desktop/core/src/desktop/views.py b/desktop/core/src/desktop/views.py
index 72c4cef8401..c033cacd2cb 100644
--- a/desktop/core/src/desktop/views.py
+++ b/desktop/core/src/desktop/views.py
@@ -15,44 +15,40 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from future import standard_library
-
-standard_library.install_aliases()
-import json
-import logging
import os
import re
-import six
-import socket
import sys
-import tempfile
+import json
import time
-import traceback
+import socket
+import logging
import zipfile
-import validate
+import tempfile
+import traceback
+from io import StringIO as string_io
+from wsgiref.util import FileWrapper
-from django.http import HttpResponseRedirect
+import six
+import validate
+import django.views.debug
+from configobj import ConfigObj, ConfigObjError, get_extra_values
from django.conf import settings
from django.contrib.staticfiles.storage import staticfiles_storage
-from django.shortcuts import render as django_render
-from django.http import HttpResponse
+from django.http import HttpResponse, HttpResponseRedirect
from django.http.response import StreamingHttpResponse
+from django.shortcuts import redirect, render as django_render
from django.urls import reverse
-from django.shortcuts import redirect
+from django.utils.translation import gettext as _
from django.views.decorators.http import require_POST
-from configobj import ConfigObj, get_extra_values, ConfigObjError
-from wsgiref.util import FileWrapper
from webpack_loader.utils import get_static
-import django.views.debug
import desktop.conf
import desktop.log.log_buffer
-
from desktop import appmanager
-from desktop.api import massaged_tags_for_json, massaged_documents_for_json, _get_docs
+from desktop.api import _get_docs, massaged_documents_for_json, massaged_tags_for_json
from desktop.auth.backend import is_admin
from desktop.auth.decorators import admin_required, hue_admin_required
-from desktop.conf import USE_NEW_EDITOR, HUE_LOAD_BALANCER, get_clusters, ENABLE_CONNECTORS
+from desktop.conf import ENABLE_CONNECTORS, HUE_LOAD_BALANCER, USE_NEW_EDITOR, get_clusters
from desktop.lib import django_mako, fsmanager
from desktop.lib.conf import GLOBAL_CONFIG, BoundConfig, _configs_from_dir
from desktop.lib.config_spec_dump import ConfigSpec
@@ -61,21 +57,16 @@
from desktop.lib.paths import get_desktop_root
from desktop.lib.thread_util import dump_traceback
from desktop.lib.view_util import is_ajax
-from desktop.log.access import access_log_level, access_warn, AccessInfo
-from desktop.log import set_all_debug as _set_all_debug, reset_all_debug as _reset_all_debug, \
- get_all_debug as _get_all_debug, DEFAULT_LOG_DIR
-from desktop.models import Settings, hue_version, _get_apps, UserPreferences
+from desktop.log import (
+ DEFAULT_LOG_DIR,
+ get_all_debug as _get_all_debug,
+ reset_all_debug as _reset_all_debug,
+ set_all_debug as _set_all_debug,
+)
+from desktop.log.access import AccessInfo, access_log_level, access_warn
+from desktop.models import Settings, UserPreferences, _get_apps, hue_version
from libsaml.conf import REQUIRED_GROUPS, REQUIRED_GROUPS_ATTRIBUTE
-from useradmin.models import get_profile
-from useradmin.models import User
-
-if sys.version_info[0] > 2:
- from io import StringIO as string_io
- from django.utils.translation import gettext as _
-else:
- from StringIO import StringIO as string_io
- from django.utils.translation import ugettext as _
-
+from useradmin.models import User, get_profile
LOG = logging.getLogger()
@@ -83,12 +74,13 @@
def is_alive(request):
return HttpResponse('')
+
def samlgroup_check(request):
if 'SAML2Backend' in desktop.auth.forms.get_backend_names():
if REQUIRED_GROUPS.get():
try:
userprofile = get_profile(request.user)
- except:
+ except Exception:
return False
json_data = json.loads(userprofile.json_data)
@@ -115,6 +107,7 @@ def samlgroup_check(request):
LOG.info("User %s found in the required SAML groups %s" % (request.user.username, ",".join(saml_group_found)))
return True
+
def saml_login_headers(request):
userprofile = get_profile(request.user)
try:
@@ -125,9 +118,10 @@ def saml_login_headers(request):
try:
userprofile.update_data({'X-CSRF-TOKEN': request.META['CSRF_COOKIE']})
userprofile.save()
- except:
+ except Exception:
LOG.error("X-CSRF-TOKEN header not found")
+
def hue(request):
current_app, other_apps, apps_list = _get_apps(request.user, '')
clusters = list(get_clusters(request.user).values())
@@ -270,15 +264,18 @@ def log_view(request):
prev_log_file_size = os.path.getsize(prev_log_file)
with open(prev_log_file, 'rb') as fh1:
fh1.seek(prev_log_file_size - BUF_SIZE - log_file_size)
- for l in fh1.readlines(): buffer.append(l)
+ for line in fh1.readlines():
+ buffer.append(line)
# read the current log file
with open(log_file, 'rb') as fh:
fh.seek(0)
- for l in fh.readlines(): buffer.append(l)
+ for line in fh.readlines():
+ buffer.append(line)
else:
with open(log_file, 'rb') as fh:
fh.seek(log_file_size - BUF_SIZE)
- for l in fh.readlines(): buffer.append(l)
+ for line in fh.readlines():
+ buffer.append(line)
return render('logs.mako', request, dict(
log=buffer,
query=request.GET.get("q", ""),
@@ -290,6 +287,7 @@ def log_view(request):
)
)
+
def task_server_view(request):
"""
This view renders the Task Server page with basic functionality.
@@ -303,6 +301,7 @@ def task_server_view(request):
'users_json': json.dumps(list(User.objects.values_list('id', flat=True)))
})
+
@hue_admin_required
@access_log_level(logging.WARN)
def download_log_view(request):
@@ -327,22 +326,25 @@ def download_log_view(request):
prev_log_file_size = os.path.getsize(prev_log_file)
with open(prev_log_file, 'rb') as fh1:
fh1.seek(prev_log_file_size - BUF_SIZE - log_file_size)
- for l in fh1.readlines(): buffer.append(l)
+ for line in fh1.readlines():
+ buffer.append(line)
# read the current log file
with open(log_file, 'rb') as fh:
fh.seek(0)
- for l in fh.readlines(): buffer.append(l)
+ for line in fh.readlines():
+ buffer.append(line)
else:
with open(log_file, 'rb') as fh:
fh.seek(log_file_size - BUF_SIZE)
- for l in fh.readlines(): buffer.append(l)
+ for line in fh.readlines():
+ buffer.append(line)
try:
# We want to avoid doing a '\n'.join of the entire log in memory
# in case it is rather big. So we write it to a file line by line
# and pass that file to zipfile, which might follow a more efficient path.
tmp = tempfile.NamedTemporaryFile()
log_tmp = tempfile.NamedTemporaryFile("w+t") if sys.version_info[0] == 2 else tempfile.NamedTemporaryFile("w+t", encoding='utf-8')
- for l in buffer:
+ for line in buffer:
log_tmp.write(smart_str(l, errors='replace'))
# This is not just for show - w/out flush, we often get truncated logs
log_tmp.flush()
@@ -360,7 +362,7 @@ def download_log_view(request):
response['Content-Disposition'] = 'attachment; filename=hue-logs-%s.zip' % t
response['Content-Length'] = length
return response
- except Exception as e:
+ except Exception:
LOG.exception("Couldn't construct zip file to write logs")
return log_view(request)
@@ -384,6 +386,8 @@ def bootstrap(request):
_status_bar_views = []
+
+
def register_status_bar_view(view):
global _status_bar_views
_status_bar_views.append(view)
@@ -403,7 +407,7 @@ def status_bar(request):
resp += r.content
else:
LOG.warning("Failed to execute status_bar view %s" % (view,))
- except:
+ except Exception:
LOG.exception("Failed to execute status_bar view %s" % (view,))
return HttpResponse(resp)
@@ -416,19 +420,6 @@ def dump_config(request):
return render("dump_config.mako", request, {})
-@hue_admin_required
-@access_log_level(logging.WARN)
-def threads(request):
- """Dumps out server threads. Useful for debugging."""
- out = string_io()
- dump_traceback(file=out)
-
- if is_ajax(request):
- return HttpResponse(out.getvalue(), content_type="text/plain")
- else:
- return render("threads.mako", request, {'text': out.getvalue(), 'is_embeddable': request.GET.get('is_embeddable', False)})
-
-
@hue_admin_required
@access_log_level(logging.WARN)
def memory(request):
@@ -483,10 +474,12 @@ def global_js_constants(request):
def ace_sql_location_worker(request):
return HttpResponse(render('ace_sql_location_worker.mako', request, None), content_type="application/javascript")
+
def ace_sql_syntax_worker(request):
return HttpResponse(render('ace_sql_syntax_worker.mako', request, None), content_type="application/javascript")
-#Redirect to static resources no need for auth. Fails with 401 with Knox.
+
+# Redirect to static resources no need for auth. Fails with 401 with Knox.
@login_notrequired
def dynamic_bundle(request, config, bundle_name):
try:
@@ -498,28 +491,34 @@ def dynamic_bundle(request, config, bundle_name):
LOG.exception("Failed loading dynamic bundle %s: %s" % (bundle_name, ex))
return render("404.mako", request, dict(uri=request.build_absolute_uri()), status=404)
+
def assist_m(request):
return render('assist_m.mako', request, None)
+
def index(request):
return redirect('desktop_views_hue')
+
def csrf_failure(request, reason=None):
"""Registered handler for CSRF."""
access_warn(request, reason)
return render("403_csrf.mako", request, dict(uri=request.build_absolute_uri()), status=403)
+
@login_notrequired
def serve_403_error(request, *args, **kwargs):
"""Registered handler for 403. We just return a simple error"""
access_warn(request, "403 access forbidden")
return render("403.mako", request, dict(uri=request.build_absolute_uri()), status=403)
+
def serve_404_error(request, *args, **kwargs):
"""Registered handler for 404. We just return a simple error"""
access_warn(request, "404 not found")
return render("404.mako", request, dict(uri=request.build_absolute_uri()), status=404)
+
def serve_500_error(request, *args, **kwargs):
"""Registered handler for 500. We use the debug view to make debugging easier."""
try:
@@ -544,6 +543,7 @@ def serve_500_error(request, *args, **kwargs):
# - Packaging and install issues
pass
+
_LOG_LEVELS = {
"critical": logging.CRITICAL,
"error": logging.ERROR,
@@ -616,6 +616,7 @@ def commonheader(title, section, user, request=None, padding="90px", skip_topbar
'banner_message': get_banner_message(request)
})
+
def get_banner_message(request):
banner_message = None
forwarded_host = request.get_host()
@@ -636,24 +637,31 @@ def get_banner_message(request):
return banner_message
+
def commonshare():
return django_mako.render_to_string("common_share.mako", {})
+
def commonshare2():
return django_mako.render_to_string("common_share2.mako", {})
+
def commonimportexport(request):
return django_mako.render_to_string("common_import_export.mako", {'request': request})
+
def login_modal(request):
return desktop.auth.views.dt_login(request, True)
+
def is_idle(request):
return HttpResponse("no!")
+
def commonfooter_m(request, messages=None):
return commonfooter(request, messages, True)
+
def commonfooter(request, messages=None, is_mobile=False):
"""
Returns the rendered common footer
@@ -691,6 +699,7 @@ def collect_usage():
#
_CONFIG_ERROR_LIST = None
+
def _get_config_errors(request, cache=True):
"""Returns a list of (confvar, err_msg) tuples."""
global _CONFIG_ERROR_LIST
@@ -746,6 +755,7 @@ def validate_by_spec(error_list):
if configspec:
os.remove(configspec.name)
+
def load_confs(configspecpath, conf_source=None):
"""Loads and merges all of the configurations passed in,
returning a ConfigObj for the result.
@@ -880,6 +890,7 @@ def reset_all_debug(request):
def _ko(str=""):
return _(str).replace("'", "\\'")
+
# This global Mako filtering option, use it with ${ yourvalue | n,antixss }
def antixss(value):
xss_regex = re.compile(r'<[^>]+>')
@@ -889,5 +900,5 @@ def antixss(value):
def topo(request, location='world'):
file_path = os.path.join('desktop', 'ext', 'topo', location + '.topo.json')
response = StreamingHttpResponse(streaming_content=staticfiles_storage.open(file_path))
- #//return settings.STATIC_URL + path
+ # return settings.STATIC_URL + path
return response
diff --git a/tools/ops/hue_collect_data.sh b/tools/ops/hue_collect_data.sh
deleted file mode 100755
index 0cecb1248dc..00000000000
--- a/tools/ops/hue_collect_data.sh
+++ /dev/null
@@ -1,464 +0,0 @@
-#!/bin/bash
-# Licensed to Cloudera, Inc. under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. Cloudera, Inc. licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# NOTE: This script requires curl, strace and lsof to be installed. It
-# must be run on the Hue server. Set HUE_USER to a user in Hue with
-# Superuser access to get thread dumps. Set HUE_PASSWORD to HUE_USER's
-# password.
-
-# Please change USER to contain the user to login
-HUE_USER="admin"
-
-# Please change PASSWORD to contain the password for the above user
-HUE_PASSWORD="admin"
-
-#parse command line arguments
-parse_arguments()
-{
- # Test that we're using compatible getopt version.
- getopt -T > /dev/null
- if [[ $? -ne 4 ]]; then
- echo "Incompatible getopt version."
- exit 1
- fi
-
- # Parse short and long option parameters.
- STRACE_WAIT=15
- RUNS=4
- RUN_WAIT=30
- OUTPUT_DIR_BASE=/tmp/hue_collect_data
- COLLECT_STRACE=true
- COLLECT_LSOF=true
- COLLECT_NETSTAT=true
- COLLECT_THREADS=
- #This is necessary to handle AD auth, doesn't seem to hurt non-ad auth
- #if they have multiple ldap servers or for some reason the drop down
- #at login says something other than "LDAP", then this must match the drop
- #down
- HUE_AUTH_SERVER="LDAP"
- COLLECT_INSTANCE=true
- GETOPT=`getopt -n $0 -o s,l,n,w:,r:,t,u:,p:,i,o:,d:,y:,a:,h \
- -l strace,,lsof,netstat,wait:,runs:,threads,hueuser:,huepass:,instance,outdir:,huelog:,swait:,authserver:,help \
- -- "$@"`
- eval set -- "$GETOPT"
- while true;
- do
- case "$1" in
- -s|--strace)
- COLLECT_STRACE=
- shift
- ;;
- -y|--swait)
- STRACE_WAIT=$2
- shift 2
- ;;
- -l|--lsof)
- COLLECT_LSOF=
- shift
- ;;
- -n|--netstat)
- COLLECT_NETSTAT=
- shift
- ;;
- -t|--threads)
- COLLECT_THREADS=$1
- shift
- ;;
- -u|--hueuser)
- HUE_USER=$2
- shift 2
- ;;
- -p|--huepass)
- HUE_PASSWORD=$2
- shift 2
- ;;
- -a|--authserver)
- HUE_AUTH_SERVER=$2
- shift 2
- ;;
- -i|--instance)
- COLLECT_INSTANCE=
- shift
- ;;
- -w|--wait)
- RUN_WAIT=$2
- shift 2
- ;;
- -r|--runs)
- RUNS=$2
- shift 2
- ;;
- -o|--outdir)
- OUTPUT_DIR_BASE=$2
- shift 2
- ;;
- -d|--huelog)
- HUE_LOG_DIR=$2
- shift 2
- ;;
- --)
- shift
- break
- ;;
- *)
- usage
- exit 1
- ;;
- esac
- done
- #
-}
-
-usage()
-{
-cat << EOF
-usage: $0 [options]
-
-Collects troubleshooting data from Hue server:
-
-OPTIONS
- -s|--strace Setting this will disable collecting strace, default
- strace will be collected
- -y|--swait Interval to wait before stopping strace - default 15.
- -l|--lsof Setting this will disable collecting lsof, default
- lsof will be collected
- -n|--netstat Setting this will disable collecting netstat, default
- netstat will be collected
- -t|--threads This will enable thread collection, default threads
- will NOT be collected
- -u|--hueuser Hue username for collecting threads - must be admin user - default admin.
- -p|--huepass Hue password for collecting threads - default admin.
- -a|--authserver This is the Ldap auth server name in the hue.ini if using
- multiple ldap servers for auth. Must be set to the auth
- server that "hueuser" belongs to.
- -i|--instance Setting this will disable collecting CM Hue configs, default
- CM Hue configs will be collected
- -w|--wait Seconds to wait between runs of collecting data - default 30.
- -r|--runs Number of times to collect data - default 4.
- -o|--outdir Location to dump collected data - default /tmp/hue_collect_data.
- -h|--help Show this message.
-EOF
-}
-
-main()
-{
-
- parse_arguments "$@"
-
- if [[ ! ${USER} =~ .*root* ]]
- then
- echo "Script must be run as root: exiting"
- exit 1
- fi
-
- AGENT_PROCESS_DIR="/var/run/cloudera-scm-agent/process"
- START_DATE=$(date '+%Y%m%d-%H%M')
- MKDIR="mkdir -p"
-
- for PID in `ps aux | grep [r]uncherrypyserver | awk '{print $2}'`
- do
- if [[ ! ${PID} == ?(-)+([0-9]) ]]
- then
- echo "Unable to get PID from Process, either Hue is not running on this host or Hue is not using CherryPy server"
- exit 1
- fi
-
- OUTPUT_DIR_DATE=${OUTPUT_DIR_BASE}/${START_DATE}
- OUTPUT_DIR=${OUTPUT_DIR_DATE}/${PID}
- HUE_USAGE_FILE=${OUTPUT_DIR}/cpu_mem_usage/cpu_mem_usage
- HUE_THREADS_FILE=${OUTPUT_DIR}/threads/threads
- HUE_STRACE_FILE=${OUTPUT_DIR}/strace/strace
- HUE_LSOF_FILE=${OUTPUT_DIR}/lsof/lsof
- HUE_ENVIRON_FILE=${OUTPUT_DIR}/environ/environ
- HUE_CMDLINE_FILE=${OUTPUT_DIR}/cmdline/cmdline
- HUE_LIMITS_FILE=${OUTPUT_DIR}/limits/limits
- HUE_NETSTAT_FILE=${OUTPUT_DIR}/netstat/netstat
- HUE_SUDO=${OUTPUT_DIR}/sudo
- HUE_CONFS=${OUTPUT_DIR}/confs
- COOKIE_JAR=${OUTPUT_DIR}/${USER}_cookie.jar
-
- HOSTNAME=$(hostname)
- HUE_SERVER=${HOSTNAME}
-
- echo "Making ${OUTPUT_DIR} if it does not exist"
- ${MKDIR} ${OUTPUT_DIR}
-
- get_cm_process_dir ${PID}
-
- if [[ -f ${CM_PROPS} ]]
- then
- HUE_LOG_DIR=`grep log_dir ${CM_PROPS} | awk '{print $3}'`
- else
- HUE_LOG_DIR=/var/log/hue
- fi
- HUE_PORT=`grep http_port ${HUE_INI} | awk -F= '{print $2}'`
- if [[ -z ${HUE_PORT} ]]
- then
- HUE_PORT=8888
- fi
- SSL_CERT=`grep ssl_certificate ${HUE_INI} | awk -F= '{print $2}'`
- if [[ ! -z ${SSL_CERT} ]]
- then
- HUE_HTTP="https"
- else
- HUE_HTTP="http"
- fi
- HUE_PASS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/accounts/login/"
- HUE_THREADS_URL="${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/desktop/debug/threads"
-
- if [[ ! -z ${COLLECT_THREADS} ]]
- then
- hue_login
- fi
-
- echo "Gathering info:"
- for (( x=1; x<=${RUNS}; x++ ))
- do
- DATE=$(date '+%Y%m%d-%H%M%S')
- echo "DATE: ${DATE}"
-
- echo "Getting CPU and Memory usage"
- do_ps ${PID}
- ${MKDIR} `dirname ${HUE_USAGE_FILE}`
- echo "PID CPU MEM MEM_MB" >> ${HUE_USAGE_FILE}_${DATE}
- echo "${PID} ${CPU} ${MEM} ${MEM_MB}" >> ${HUE_USAGE_FILE}_${DATE}
- echo "free -m results" >> ${HUE_USAGE_FILE}_${DATE}
- free -m >> ${HUE_USAGE_FILE}_${DATE}
-
- if [[ ${COLLECT_NETSTAT} ]]
- then
- echo "Gathering netstat info"
- ${MKDIR} `dirname ${HUE_NETSTAT_FILE}`
- netstat -anp | grep ${PID} >> ${HUE_NETSTAT_FILE}_${DATE}
- fi
-
- if [[ ${COLLECT_STRACE} ]]
- then
- echo "Getting strace"
- ${MKDIR} `dirname ${HUE_STRACE_FILE}`
- do_strace \
- ${PID} \
- ${STRACE_WAIT} \
- -o ${HUE_STRACE_FILE}_${DATE} -T -t
- fi
-
- if [[ ${COLLECT_LSOF} ]]
- then
- echo "Getting open connections"
- ${MKDIR} `dirname ${HUE_LSOF_FILE}`
- do_lsof \
- ${PID} \
- ${HUE_LSOF_FILE}_${DATE}
- fi
-
- if [[ ! -z ${COLLECT_THREADS} ]]
- then
- echo "Getting a thread dump:"
- ${MKDIR} `dirname ${HUE_THREADS_FILE}`
- do_curl \
- GET \
- "${HUE_THREADS_URL}" \
- -L -o ${HUE_THREADS_FILE}_${DATE}
- fi
-
- sleep ${RUN_WAIT}
- done
-
- if [[ ! -z ${COLLECT_INSTANCE} ]]
- then
- echo "Gathering CM config instances"
- do_instances ${HUE_PORT}
- fi
-
- echo "Gathering process info"
- ${MKDIR} `dirname ${HUE_ENVIRON_FILE}`
- strings /proc/${PID}/environ >> ${HUE_ENVIRON_FILE}_${DATE}
- ${MKDIR} `dirname ${HUE_CMDLINE_FILE}`
- strings /proc/${PID}/cmdline >> ${HUE_CMDLINE_FILE}_${DATE}
- ${MKDIR} `dirname ${HUE_LIMITS_FILE}`
- strings /proc/${PID}/limits >> ${HUE_LIMITS_FILE}_${DATE}
-
- ${MKDIR} ${OUTPUT_DIR}/logs
- cp -pr ${HUE_LOG_DIR}/* ${OUTPUT_DIR}/logs
-
- done
-
- echo "Gathering environment info"
- do_sudo \
- ${HUE_SUDO} \
- ${DATE}
-
- if [[ ! -z ${STRACE_PID} ]]
- then
- echo "strace still running, waiting for it to complete: ${STRACE_PID}"
- wait ${STRACE_PID}
- fi
-
- echo "Collecting done, please zip ${OUTPUT_DIR_DATE} and upload to the ticket"
-}
-
-function do_curl() {
-
- METHOD=$1
- shift
- URL=$1
- shift
- ARGS=$@
-
- CURL=$(which curl)
- if [ -z ${COOKIE_JAR} ]
- then
- COOKIE_JAR=/tmp/cookie.jar
- fi
- if [ -f ${COOKIE_JAR} ]
- then
- CSRF_TOKEN=`grep ${HOSTNAME} ${COOKIE_JAR} | grep csrftoken | cut -f 7`
- fi
- if [ ! -f ${CURL} ]
- then
- echo "curl not found, unable to run any curl commands"
- else
- ${CURL} \
- ${CURL_OPTS} \
- -k \
- -e "${HUE_HTTP}://${HUE_SERVER}:${HUE_PORT}/" \
- -b @${COOKIE_JAR} \
- -c ${COOKIE_JAR} \
- -H "X-CSRFToken: ${CSRF_TOKEN}" \
- -X ${METHOD} \
- -s \
- -f \
- ${URL} \
- ${ARGS}
- fi
-
-}
-
-function hue_login() {
- echo "Login to Hue to get Cookie:"
- do_curl \
- GET \
- "${HUE_PASS_URL}" \
- -L 2>&1 > /dev/null
-
- do_curl \
- POST \
- "${HUE_PASS_URL}" \
- -F username=${HUE_USER} -F password="${HUE_PASSWORD}" -F server="${HUE_AUTH_SERVER}" 2>&1 > /dev/null
-}
-
-function do_strace()
-{
- SPID=$1
- shift
- WAIT=$1
- shift
- ARGS=$@
-
- STRACE=$(which strace)
- if [ ! -f ${STRACE} ]
- then
- echo "strace not found, unable to collect strace info"
- else
- if [[ ! -z ${STRACE_PID} ]]
- then
- echo "strace still running, waiting for it to complete: ${STRACE_PID}"
- wait ${STRACE_PID}
- fi
- timeout ${WAIT}s ${STRACE} -f -v -p ${SPID} ${ARGS} &
- STRACE_PID=$!
- fi
-}
-
-function do_lsof()
-{
- LPID=$1
- shift
- LOG_FILE=$1
- shift
- ARGS=$@
-
- if [ -z ${LOG_FILE} ]
- then
- LOG_FILE=/tmp/lsof.log
- fi
-
- LSOF=$(which lsof)
- if [ ! -f ${LSOF} ]
- then
- echo "lsof not found, unable to determine number of connections"
- else
- ${LSOF} -P -p ${LPID} ${ARGS} > ${LOG_FILE}
- fi
-}
-
-function do_ps()
-{
- PID=$1
- shift
- PS_COMMAND=$(ps aux | grep ${PID} | grep [r]uncherrypyserver | tail -1 | awk '{print $6" "$2" "$3" "$12}')
- MEM=$(echo ${PS_COMMAND} | awk '{print $1}')
- CPU=$(echo ${PS_COMMAND} | awk '{print $3}')
- PROC=$(echo ${PS_COMMAND} | awk '{print $4}')
- MEM_MB=$(expr ${MEM} / 1024)
-}
-
-function do_sudo()
-{
- HUE_SUDO=$1
- shift
- DATE=$1
- shift
- ARGS=$@
- sudo -u hue /bin/bash -c "/usr/bin/env" >> ${HUE_SUDO}_env_${DATE}
- sudo -u hue /bin/bash -c "/usr/bin/env which python2.6" >> ${HUE_SUDO}_python_${DATE}
-}
-
-function get_cm_process_dir()
-{
- PID=$1
- shift
- if [[ -d ${AGENT_PROCESS_DIR} ]]
- then
- HUE_CONF_DIR=`strings /proc/${PID}/environ | grep HUE_CONF_DIR | awk -F\= '{print $2}'`
- else
- HUE_CONF_DIR=/etc/hue/conf
- fi
- HUE_INI=${HUE_CONF_DIR}/hue.ini
- CM_PROPS=${HUE_CONF_DIR}/cloudera-monitor.properties
-}
-
-function do_instances()
-{
- HUE_PORT=$1
- ${MKDIR} ${HUE_CONFS}
- if [[ -d ${AGENT_PROCESS_DIR} ]]
- then
- for x in `find ${AGENT_PROCESS_DIR}/*hue-HUE_SERVER -name "hue.ini" -exec grep -H ${HUE_PORT} {} \; | awk -F\/ '{print $6}' | sort -n | tail -3`
- do
- cp -pr ${AGENT_PROCESS_DIR}/$x ${HUE_CONFS}
- done
- else
- cp -pr /etc/hue/conf ${HUE_CONFS}/0-hue-HUE_SERVER
- fi
- for x in `find ${HUE_CONFS} -name "*" -type f`
- do
- sed -i "/password/Id" $x
- done
-}
-
-main "$@"