From c24fd7cff71917b8534a8881cf09766555087a18 Mon Sep 17 00:00:00 2001 From: rzuo Date: Fri, 11 Aug 2023 13:59:11 +0800 Subject: [PATCH 1/2] add logsearch support to ambari --- .../AMBARI_LOGSEARCH/3.0.0/alerts.json | 77 + .../configuration/logfeeder-ambari-config.xml | 38 + .../3.0.0/configuration/logfeeder-env.xml | 187 ++ .../3.0.0/configuration/logfeeder-grok.xml | 49 + .../logfeeder-kafka-output-config.xml | 61 + .../3.0.0/configuration/logfeeder-log4j.xml | 78 + .../configuration/logfeeder-output-config.xml | 37 + .../configuration/logfeeder-properties.xml | 145 ++ .../logfeeder-system_log-env.xml | 61 + .../configuration/logsearch-admin-json.xml | 54 + .../logsearch-audit_logs-solrconfig.xml | 62 + .../configuration/logsearch-common-env.xml | 124 ++ .../logsearch-common-properties.xml | 23 + .../3.0.0/configuration/logsearch-env.xml | 201 ++ .../3.0.0/configuration/logsearch-log4j.xml | 121 ++ .../configuration/logsearch-properties.xml | 317 +++ .../logsearch-service_logs-solrconfig.xml | 62 + .../AMBARI_LOGSEARCH/3.0.0/kerberos.json | 72 + .../AMBARI_LOGSEARCH/3.0.0/metainfo.xml | 204 ++ .../AMBARI_LOGSEARCH/3.0.0/metrics.json | 53 + .../3.0.0/package/alerts/alert_logfeeder.py | 85 + .../3.0.0/package/scripts/logfeeder.py | 70 + .../3.0.0/package/scripts/logsearch.py | 71 + .../3.0.0/package/scripts/params.py | 429 ++++ .../3.0.0/package/scripts/service_check.py | 41 + .../3.0.0/package/scripts/setup_logfeeder.py | 136 ++ .../3.0.0/package/scripts/setup_logsearch.py | 162 ++ .../3.0.0/package/scripts/status_params.py | 38 + .../templates/HadoopServiceConfig.json.j2 | 505 +++++ .../package/templates/global.config.json.j2 | 27 + .../templates/input.config-logsearch.json.j2 | 55 + .../package/templates/logfeeder_jaas.conf.j2 | 26 + .../package/templates/logsearch_jaas.conf.j2 | 26 + .../properties/audit_logs-solrconfig.xml.j2 | 1889 +++++++++++++++++ .../properties/input.config-ambari.json.j2 | 858 ++++++++ .../3.0.0/properties/kafka-output.json.j2 | 34 + .../logfeeder-default_grok_patterns.j2 | 148 ++ .../3.0.0/properties/logfeeder-env.sh.j2 | 47 + .../3.0.0/properties/logfeeder-log4j.xml.j2 | 72 + .../3.0.0/properties/logsearch-admin.json.j2 | 25 + .../3.0.0/properties/logsearch-env.sh.j2 | 51 + .../3.0.0/properties/logsearch-log4j.xml.j2 | 82 + .../3.0.0/properties/output.config.json.j2 | 59 + .../3.0.0/properties/secure_log.json | 31 + .../properties/service_logs-solrconfig.xml.j2 | 1889 +++++++++++++++++ .../3.0.0/properties/system_messages.json | 31 + .../3.0.0/quicklinks/quicklinks.json | 35 + .../3.0.0/role_command_order.json | 9 + .../AMBARI_LOGSEARCH/3.0.0/service_advisor.py | 176 ++ .../3.0.0/themes/credentials.json | 42 + .../3.0.0/themes/directories.json | 137 ++ .../AMBARI_LOGSEARCH/3.0.0/themes/theme.json | 480 +++++ 52 files changed, 9792 insertions(+) create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/alerts.json create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-ambari-config.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-env.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-grok.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-kafka-output-config.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-log4j.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-output-config.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-properties.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-system_log-env.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-admin-json.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-audit_logs-solrconfig.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-common-env.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-common-properties.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-env.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-log4j.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-properties.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-service_logs-solrconfig.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/kerberos.json create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/metainfo.xml create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/metrics.json create mode 100755 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/alerts/alert_logfeeder.py create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/logfeeder.py create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/logsearch.py create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/params.py create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/service_check.py create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/setup_logfeeder.py create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/setup_logsearch.py create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/status_params.py create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/HadoopServiceConfig.json.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/global.config.json.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/input.config-logsearch.json.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/logfeeder_jaas.conf.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/logsearch_jaas.conf.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/audit_logs-solrconfig.xml.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/input.config-ambari.json.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/kafka-output.json.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logfeeder-default_grok_patterns.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logfeeder-env.sh.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logfeeder-log4j.xml.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logsearch-admin.json.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logsearch-env.sh.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logsearch-log4j.xml.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/output.config.json.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/secure_log.json create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/service_logs-solrconfig.xml.j2 create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/system_messages.json create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/quicklinks/quicklinks.json create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/role_command_order.json create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/service_advisor.py create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/themes/credentials.json create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/themes/directories.json create mode 100644 ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/themes/theme.json diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/alerts.json b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/alerts.json new file mode 100644 index 00000000000..9f9a38d94ec --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/alerts.json @@ -0,0 +1,77 @@ +{ + "AMBARI_LOGSEARCH": { + "service": [ + { + "name": "logfeeder_process_percent", + "label": "Percent LogFeeders Available", + "description": "This service-level alert is triggered if the configured percentage of LogFeeders processes cannot be determined to be up and listening on the network for the configured warning and critical thresholds. It aggregates the results of LogFeeders process down checks.", + "interval": 1, + "scope": "SERVICE", + "enabled": true, + "source": { + "type": "AGGREGATE", + "alert_name": "logfeeder_process", + "reporting": { + "ok": { + "text": "affected: [{1}], total: [{0}]" + }, + "warning": { + "text": "affected: [{1}], total: [{0}]", + "value": 10 + }, + "critical": { + "text": "affected: [{1}], total: [{0}]", + "value": 30 + }, + "units" : "%", + "type": "PERCENT" + } + } + } + ], + "LOGSEARCH_SERVER": [ + { + "name": "logsearch_ui", + "label": "Log Search Web UI", + "description": "This host-level alert is triggered if the Log Search UI is unreachable.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "WEB", + "uri": { + "http": "{{logsearch-properties/logsearch.protocol}}://0.0.0.0:{{logsearch-properties/logsearch.http.port}}/api/v1/info", + "https": "{{logsearch-properties/logsearch.protocol}}://0.0.0.0:{{logsearch-properties/logsearch.https.port}}/api/v1/info", + "https_property": "{{logsearch-properties/logsearch.protocol}}", + "https_property_value": "https", + "default_port": 61888, + "connection_timeout": 5.0 + }, + "reporting": { + "ok": { + "text": "HTTP {0} response in {2:.3f}s" + }, + "warning": { + "text": "HTTP {0} response from {1} in {2:.3f}s ({3})" + }, + "critical": { + "text": "Connection failed to {1} ({3})" + } + } + } + } + ], + "LOGSEARCH_LOGFEEDER": [ + { + "name": "logfeeder_process", + "label": "LogFeeder Status", + "description": "This alert indicates the status of the LogFeeder process as determined by the logfeeder status script.", + "interval": 1, + "scope": "ANY", + "source": { + "type": "SCRIPT", + "path": "AMBARI_LOGSEARCH/3.0.0/package/alerts/alert_logfeeder.py" + } + } + ] + } +} diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-ambari-config.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-ambari-config.xml new file mode 100644 index 00000000000..868e32c5281 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-ambari-config.xml @@ -0,0 +1,38 @@ + + + + + + content + Ambari Log Feeder config + Grok filters and input file locations for ambari related log files + + VALUE_FROM_PROPERTY_FILE + + content + false + input.config-ambari.json.j2 + text + false + + + + \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-env.xml new file mode 100644 index 00000000000..3076a06e562 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-env.xml @@ -0,0 +1,187 @@ + + + + + + + logfeeder_pid_dir + /var/run/ambari-logsearch-logfeeder + Log Feeder Process ID Directory + Log Feeder pid dir + + directory + + + + + logfeeder_log_dir + /var/log/ambari-logsearch-logfeeder + Log dir for Log Feeder + Log Feeder log dir + + directory + + + + + logfeeder_max_mem + 512m + Max Memory for Logfeeder + Log Feeder max memory + + + + logfeeder_solr_service_logs_enable + true + Service Logs enabled + Enable service logs to Solr + + boolean + + + + + logfeeder_solr_audit_logs_enable + true + Audit Logs enabled + Enable audit logs to Solr + + boolean + + + + + logfeeder_truststore_location + /etc/security/serverKeys/logsearch.trustStore.jks + Log Feeder trust store location + Location of the trust store file. + + + + logfeeder_truststore_type + jks + Log Feeder trust store type + Type of the trust store file. + + + + logfeeder_truststore_password + bigdata + Log Feeder trust store password + PASSWORD + Password to open the trust store file. + + password + true + + + + + logfeeder_keystore_location + /etc/security/serverKeys/logsearch.keyStore.jks + Log Feeder key store location + Location of the key store file. + + + + logfeeder_keystore_type + jks + Log Feeder store type + Type of the key store file. + + + + logfeeder_keystore_password + bigdata + Log Feeder key store password + PASSWORD + Password to open the key store file. + + password + true + + + + + logfeeder_external_solr_kerberos_keytab + + Log Feeder External Solr keytab + The path to the Kerberos Keytab file containing service principal of Log Feeder e.g. /etc/security/keytabs/logfeeder.service.keytab + + + + logfeeder_external_solr_kerberos_principal + + Log Feeder External Solr principal + The service principal for Log Feeder which will be used to access SolrClient e.g. logfeeder/_HOST@REALM + KERBEROS_PRINCIPAL + + + + logfeeder_kerberos_keytab + /etc/security/keytabs/logfeeder.service.keytab + Log Feeder Solr keytab + The path to the Kerberos Keytab file containing service principal of the Log Feeder. + + + + logfeeder_kerberos_principal + logfeeder + Log Feeder Solr principal + The service principal for Log Feeder which will be used to access Log Search Solr. + KERBEROS_PRINCIPAL + + + + ambari_server_log_dir + /var/log/ambari-server + Log dir of Ambari Server. Use this property when bootstrapping Logsearch only. + Ambari Server log dir + + directory + true + + + + + ambari_agent_log_dir + /var/log/ambari-agent + Log dir of Ambari Agent. Use this property when bootstrapping Logsearch only. + Ambari Agent log dir + + directory + true + + + + + content + logfeeder-env template + This is the jinja template for logfeeder-env.sh file + + VALUE_FROM_PROPERTY_FILE + + logfeeder-env.sh.j2 + text + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-grok.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-grok.xml new file mode 100644 index 00000000000..e280bb9ae27 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-grok.xml @@ -0,0 +1,49 @@ + + + + + + + default_grok_patterns + Default Grok Patterns + This is the jinja template for default grok patterns + + VALUE_FROM_PROPERTY_FILE + + true + logfeeder-default_grok_patterns.j2 + text + + + + + custom_grok_patterns + Custom Grok Patterns + This is the jinja template for custom grok patterns + + TEXT + + multiLine + true + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-kafka-output-config.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-kafka-output-config.xml new file mode 100644 index 00000000000..1f711097c92 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-kafka-output-config.xml @@ -0,0 +1,61 @@ + + + + + + logfeeder_kafka_output_enabled + false + Enable Kafka output for Log Feeder + Log Feeder Kafka output enabled + + boolean + + + + + logfeeder_kafka_broker_list + localhost:6667 + Log Feeder Kafka broker list + Broker list for Kafka output (define the right value, as it can be external, also you can add PLAINTEXT or PLAINTEXTSASL protocol as well to the addresses) + + + + logfeeder_kafka_topic + log-streaming + Log Feeder Kafka topic + Kafka topic which is the destination of Log Feeder outputs + + + + content + Log Feeder Kafka output config + Kafka output configuration for Log Feeder (data shipping) + + VALUE_FROM_PROPERTY_FILE + + content + false + kafka-output.json.j2 + text + + + + \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-log4j.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-log4j.xml new file mode 100644 index 00000000000..ba33a007bed --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-log4j.xml @@ -0,0 +1,78 @@ + + + + + + + logfeeder_log_maxfilesize + 10 + The maximum size of backup file before the log is rotated + Log Feeder Log: backup file size + + MB + + + + + logfeeder_log_maxbackupindex + 10 + The number of backup files + Log Feeder Log: # of backup files + + int + 0 + + + + + logfeeder_json_log_maxfilesize + 10 + The maximum size of backup file before the log is rotated + Log Feeder JSON Log: backup file size + + MB + + + + + logfeeder_json_log_maxbackupindex + 10 + The number of backup files + Log Feeder JSON Log: # of backup files + + int + 0 + + + + + content + logfeeder-log4j template + This is the jinja template for log4j.xml file for Log Feeder + + VALUE_FROM_PROPERTY_FILE + + logfeeder-log4j.xml.j2 + xml + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-output-config.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-output-config.xml new file mode 100644 index 00000000000..29999ba25f6 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-output-config.xml @@ -0,0 +1,37 @@ + + + + + + content + Log Feeder output config + Output configuration for Log Feeder (data shipping) + + VALUE_FROM_PROPERTY_FILE + + content + false + output.config.json.j2 + text + + + + \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-properties.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-properties.xml new file mode 100644 index 00000000000..1b7643f2a16 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-properties.xml @@ -0,0 +1,145 @@ + + + + + + logfeeder.config.files + {default_config_files} + Comma separated config files in grok format + Log Feeder config files + + + + logfeeder.checkpoint.folder + /usr/lib/ambari-logsearch-logfeeder/conf/checkpoints + Checkpoint folder for Log Feeder + Log Feeder checkpoint dir + + directory + + + + + logfeeder.metrics.collector.hosts + {metrics_collector_hosts} + Metrics collector hosts for pushing metrics by Log Feeder + Metrics collector hosts + + + + logfeeder.log.filter.enable + true + Enable Log filtering based on log level (INFO,ERROR,WARN etc.) + Log Filter enable + + boolean + + + + + logfeeder.solr.config.interval + 5 + Interval to fetch filter config from Solr in sec + Fetch Solr config interval + + int + + + + + logfeeder.solr.implicit.routing + false + + Use implicit routing for Solr Collections. + + Log Feeder Solr implicit routing + + boolean + + + + + logfeeder.cache.enabled + false + + Enable input cache for every monitored input file. The cache stores log lines, based on the data, duplications can be dropped. + + Input cache enabled + + boolean + + + + + logfeeder.cache.size + 100 + Size of the input caches + Input cache size + + int + + + + + logfeeder.cache.dedup.interval + 1000 + + If input cache is enabled, Log Feeder can drop any duplicated line during log processing, + but only if the duplicated lines/messages are in the same interval (in milliseconds) with the original message/line. + + Input cache dedup interval + + int + + + + + logfeeder.cache.last.dedup.enabled + false + + If last dedup is enabled for input cache, Log Feeder will drop every new line (message), which is the same as the last line. + + Input cache last dedup + + boolean + + + + + logfeeder.cache.key.field + log_message + + Key field, which will be used as keys in the Input cache. (by defalt, log_message represets the message part of processed data) + + Input cache key field + + + + logfeeder.include.default.level + FATAL,ERROR,WARN + Include default Log Feeder Log Levels for Log Search. Used for bootstrapping the configuration only. (levels: FATAL,ERROR,WARN,INFO,DEBUG,TRACE) + Log Feeder Log Levels + + true + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-system_log-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-system_log-env.xml new file mode 100644 index 00000000000..0461409c531 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logfeeder-system_log-env.xml @@ -0,0 +1,61 @@ + + + + + + + logfeeder_system_messages_content + System messages grok filter + This is the jinja template for input.config.system_messages.json file + + VALUE_FROM_PROPERTY_FILE + + system_messages.json + text + + + + + + logfeeder_secure_log_content + Secure log grok filter + This is the jinja template for input.config.secure_log.json file + + VALUE_FROM_PROPERTY_FILE + + secure_log.json + text + + + + + + logfeeder_system_log_enabled + Enable system logs + Enable Log Feeders to load system logs to the Log Search solr + false + + boolean + + + + + \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-admin-json.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-admin-json.xml new file mode 100644 index 00000000000..34f64abd9d8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-admin-json.xml @@ -0,0 +1,54 @@ + + + + + + logsearch_admin_username + ambari_logsearch_admin + Admin username + Log Search admin username + + + + logsearch_admin_password + + PASSWORD + Admin password + Log Search admin password + + false + password + + + + + content + logsearch-admin-json template + This is the jinja template for logsearch-admin.json file + + VALUE_FROM_PROPERTY_FILE + + logsearch-admin.json.j2 + text + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-audit_logs-solrconfig.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-audit_logs-solrconfig.xml new file mode 100644 index 00000000000..5a07b6946fb --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-audit_logs-solrconfig.xml @@ -0,0 +1,62 @@ + + + + + + + logsearch_audit_logs_max_retention + 7 + Max retention + Days to retain the audit logs in Solr + + int + false + + + + + logsearch_audit_logs_merge_factor + 5 + Merge factor + The mergeFactor value tells Lucene how many segments of equal size to build before merging them into a + single segment. High value merge factor (e.g. 25) improves indexing speed, but slows down searching. Low value + (e.g. 5) improves searching, but slows down indexing. + + + int + false + + + + + + content + Solrconfig template + This is the jinja template for solrconfig.xml file for audit logs + + VALUE_FROM_PROPERTY_FILE + + audit_logs-solrconfig.xml.j2 + xml + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-common-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-common-env.xml new file mode 100644 index 00000000000..0d2f029f3ee --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-common-env.xml @@ -0,0 +1,124 @@ + + + + + + + logsearch_use_external_solr + false + Use External Solr + Use External Solr to store logs + + value-list + false + + + true + + + + false + + + + 1 + + + + + logsearch_external_solr_zk_znode + + External Solr Logs Znode + Only needed if using custom solr cloud. E.g. /logsearch_solr + + + + logsearch_external_solr_zk_quorum + + External Solr Logs ZK Quorum + Only needed if using custom solr cloud. E.g. zk1:2181,zk2:2182 + + + + logsearch_external_solr_ssl_enabled + false + Log Search External Solr SSL Enabled + Use SSL to connect to the external solr + + value-list + false + + + true + + + + false + + + + 1 + + + + + logsearch_external_solr_kerberos_enabled + false + Log Search External Solr Kerberos Enabled + Use Kerberos to connect to the external solr + + value-list + false + + + true + + + + false + + + + 1 + + + + + + logsearch_zookeeper_external_principal + zookeeper/_HOST@EXAMPLE.COM + External ZK principal + The kerberos service principal name for external ZooKeeper. + + + + + logsearch_zookeeper_external_enabled + false + Enable External ZK + Enable external ZooKeeper. If the Solr is secure, the external ZK should be secure as well. + + boolean + + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-common-properties.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-common-properties.xml new file mode 100644 index 00000000000..00870281471 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-common-properties.xml @@ -0,0 +1,23 @@ + + + + + \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-env.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-env.xml new file mode 100644 index 00000000000..1f32655fc1f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-env.xml @@ -0,0 +1,201 @@ + + + + + + + logsearch_pid_dir + /var/run/ambari-logsearch-portal + Log Search Process ID Directory + Log Search pid dir + + directory + + + + + logsearch_log_dir + /var/log/ambari-logsearch-portal + Log directory for Log Search + Log Search log dir + + directory + + + + + logsearch_user + logsearch + USER + Log Search user + Log Search User + + user + false + + + cluster-env + user_group + + + + + + + logsearch_app_max_memory + 1024 + Max memory for Log Search Portal + Log Search Maximum Memory + + int + 512 + 2048 + MB + 256 + + + + + logsearch_debug_enabled + false + Log Search Debug Enabled + Enable debug mode for Log Search Server + + boolean + + + + + logsearch_debug_port + 5005 + Log Search Debug Port + Debug port for Log Search Server + + + + logsearch_external_solr_kerberos_keytab + + Log Search External Solr keytab + The path to the Kerberos Keytab file containing service principal of Log Search e.g. /etc/security/keytabs/logsearch.service.keytab + + + + logsearch_external_solr_kerberos_principal + + Log Search External Solr principal + The service principal for Log Search which will be used to access SolrClient e.g. logsearch/_HOST@REALM + KERBEROS_PRINCIPAL + + + + logsearch_kerberos_service_users + Log Search service users + + + true + + + List of comma separated kerberos service users who can read from logsearch collections if the cluster is + secure. (logsearch user supported by default) + + + + + logsearch_truststore_location + /usr/lib/ambari-logsearch-portal/conf/keys/logsearch.jks + Log Search trust store location + Location of the trust store file. + + + + logsearch_truststore_type + jks + Log Search trust store type + Type of the trust store file. + + + + logsearch_truststore_password + bigdata + Log Search trust store password + PASSWORD + Password to open the trust store file. + + password + true + + + + + logsearch_keystore_location + /usr/lib/ambari-logsearch-portal/conf/keys/logsearch.jks + Log Search key store location + Location of the key store file. + + + + logsearch_keystore_type + jks + Log Search key store type + Type of the key store file. + + + + logsearch_keystore_password + bigdata + Log Search key store password + PASSWORD + Password to open the key store file. + + password + true + + + + + logsearch_kerberos_keytab + /etc/security/keytabs/logsearch.service.keytab + Log Search keytab + The path to the Kerberos Keytab file containing service principal of Log Search. + + + + logsearch_kerberos_principal + logsearch + Log Search principal + The service principal for Log Search which will be used to access SolrClient. + KERBEROS_PRINCIPAL + + + + + content + logsearch-env template + This is the jinja template for logsearch-env.sh file + + VALUE_FROM_PROPERTY_FILE + + logsearch-env.sh.j2 + text + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-log4j.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-log4j.xml new file mode 100644 index 00000000000..31f7d57fbea --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-log4j.xml @@ -0,0 +1,121 @@ + + + + + + + + logsearch_log_maxfilesize + 10 + The maximum size of backup file before the log is rotated + Log Search Log: backup file size + + MB + + + + + logsearch_log_maxbackupindex + 10 + The number of backup files + Log Search Log: # of backup files + + int + 0 + + + + + logsearch_json_log_maxfilesize + 10 + The maximum size of backup file before the log is rotated + Log Search JSON Log: backup file size + + MB + + + + + logsearch_json_log_maxbackupindex + 10 + The number of backup files + Log Search JSON Log: # of backup files + + int + 0 + + + + + logsearch_audit_log_maxfilesize + 10 + The maximum size of backup file before the log is rotated + Log Search Audit Log: backup file size + + MB + + + + + logsearch_audit_log_maxbackupindex + 10 + The number of backup files + Log Search Audit Log: # of backup files + + int + 0 + + + + + logsearch_perf_log_maxfilesize + 10 + The maximum size of backup file before the log is rotated + Log Search Performance Log: backup file size + + MB + + + + + logsearch_perf_log_maxbackupindex + 10 + The number of backup files + Log Search Performance Log: # of backup files + + int + 0 + + + + + content + logsearch-log4j template + This is the jinja template for log4j.xml file for Log Search server + + VALUE_FROM_PROPERTY_FILE + + logsearch-log4j.xml.j2 + xml + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-properties.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-properties.xml new file mode 100644 index 00000000000..853fa023ab9 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-properties.xml @@ -0,0 +1,317 @@ + + + + + + logsearch.http.port + 61888 + Default port for Log Search UI (HTTP) + Log Search UI Http Port + + + + logsearch.https.port + 61888 + Default port for Log Search UI (HTTPS) + Log Search UI Https Port + + + + logsearch.protocol + http + Protocol (http or https) + Log Search UI Protocol + + + + logsearch.collection.service.logs.numshards + 4 + Log Search Solr Service Logs Shards + Number of shards for Service Logs collection + + int + 1 + 20 + int + 1 + + + + + logsearch.collection.service.logs.replication.factor + 2 + Log Search Solr Service Logs Replication Factor + Replication factor for Service Logs Solr collection + + int + 1 + 10 + int + 1 + + + + + logsearch.collection.audit.logs.numshards + 4 + Log Search Solr Audit Logs Shards + Number of shards for Audit Logs collection + + int + 1 + 20 + int + 1 + + + + + logsearch.collection.audit.logs.replication.factor + 2 + Log Search Solr Audit Logs Replication Factor + Replication factor for Audit Logs Solr collection + + int + 1 + 10 + int + 1 + + + + + logsearch.solr.collection.service.logs + hadoop_logs + Log Search Solr Service Logs Collection + Name for the service logs collection + + + + logsearch.solr.collection.audit.logs + audit_logs + Log Search Solr Audit Logs Collection + Name for the audit logs collection + + + + logsearch.service.logs.fields + logtime,level,event_count,ip,type,seq_num,path,file,line_number,host,log_message,id + Log Search Solr Service Logs Fields + Solr fields for service logs + + + + logsearch.web.service_logs.component.labels + hst_agent:HST Agent,hst_server:HST Server,activity_analyser:Activity Analyzer,ams_hbase_master:AMS HBase Master,ams_hbase_regionserver:AMS HBase RegionServer,ams_collector:AMS Metrics Collector,ams_monitor:AMS Metric Monitor,ams_grafana:AMS Grafana,ranger_kms:Ranger KMS,mapred_historyserver:MapReduce History Server,zookeeper:ZooKeeper Server,logsearch_app:Log Search Server,logsearch_feeder:Log Feeder,logsearch_perf:Log Search Performance,zeppelin:Zeppelin Notebook,spark2_jobhistory_server:Spark History Server,spark2_thriftserver:Spark Thrift Server,livy2_server:Spark Livy Server,ranger_dbpatch:Ranger DB Patch,ambari_config_changes:Ambari Config Change,ambari_eclipselink:Ambari EclipseLink,ambari_server_check_database:Ambari Check DB,storm_drpc:Storm DRPC,storm_ui:Storm UI,oozie_app:Oozie Server,kafka_logcleaner:Kafka Log Cleaner,kafka_server:Kafka Server,kafka_statechange:Kafka State Change,hbase_master:HBase Master,hbase_regionserver:HBase RegionServer,hbase_phoenix_server:HBase Phoenix Query Server,atlas_app:Atlas Metadata Server,yarn_nodemanager:YARN NodeManager,yarn_resourcemanager:YARN ResourceManager,yarn_timelineserver:YARN Timeline Server,yarn_historyserver:YARN History Server,yarn_jobsummary:YARN Job Summary,knox_cli:Knox CLI,knox_ldap:Knox LDAP,hive_server:HiveServer2,hive_server_interactive:HiveServer2 Interactive,hdfs_datanode:HDFS DataNode,hdfs_namenode:HDFS NameNode,hdfs_journalnode:HDFS JournalNode,hdfs_secondary_namenode:HDFS Secondary NameNode,hdfs_zkfc:HDFS ZKFailoverController,hdfs_nfs3:HDFS NFSGateway + Log Search Component Names + Component name mapping for the Log Search UI (mycomponent1:My Label1,mycomponent2:My Label2), by default _ are replaced with spaces and first letters and latters after undersocre transformed to uppercase latters + + + + logsearch.service.logs.split.interval.mins + 15 + Log Search Service Logs split interval + Will create multiple collections and use alias. Valid values are single,hour_week + + + + logsearch.audit.logs.split.interval.mins + 15 + Log Search Audit Logs split interval + Will switch the shard after the interval specified. Valid values are none and greater than 1 + + + + + logsearch.solr.implicit.routing + false + + Use implicit routing for Solr Collections. + + Log Search Solr implicit routing + + boolean + + + + + logsearch.solr.audit.logs.use.ranger + false + Ranger Audit Logs Enabled + Use Ranger Audit collection. This is supported only if Ranger Solr is installed in SolrCloud mode + + boolean + + + + + logsearch.ranger.audit.logs.collection.name + ranger_audits + Log Search Solr Ranger Audit Logs Collection + Name for the ranger audit logs collection + + + + logsearch.solr.metrics.collector.hosts + {metrics_collector_hosts} + Metrics collector hosts for pushing metrics by Log Search Solr + Metrics collector hosts + + true + + + + + logsearch.auth.external_auth.enabled + true + External authentication + Enable external authentication + + boolean + + + + + logsearch.auth.external_auth.host_url + {ambari_server_auth_host_url} + External authentication url - host and port + The host and port of the external server used to authenticate + + + + logsearch.auth.external_auth.login_url + /api/v1/users/$USERNAME/privileges?fields=* + External authentication url - path and query + The path and query of the external server used to authenticate + + + + logsearch.roles.allowed + AMBARI.ADMINISTRATOR,CLUSTER.ADMINISTRATOR + Roles allowed + Comma separated roles for external authentication + + + + logsearch.auth.jwt.enabled + false + JWT authentication enabled + Enable JWT based authentication for Log Search portal + + value-list + false + + + true + + + + false + + + + 1 + + + + + logsearch.auth.jwt.provider_url + + JWT authentication provider url + URL to the JWT authentication server + + + + logsearch.auth.jwt.query.param.original_url + originalUrl + JWT original url query param + Name of the original request URL which is used to redirect to Log Search Portal + + + + logsearch.auth.jwt.cookie.name + hadoop-jwt + JWT cookie name + The name of the cookie that contains the JWT token + + + + logsearch.auth.jwt.audiances + + JWT expected audiances + Comma separated list of acceptable audiences for the JWT token + + true + + + + + logsearch.auth.jwt.public_key + + JWT authentication signing key + PEM formatted public key for JWT token without the header and the footer + + content + + + + + logsearch.spnego.kerberos.enabled + false + Http Spnego enabled + Enable SPNEGO based authentication on the Log Search UI + + boolean + + + + + logsearch.spnego.kerberos.principal + + Http Spnego principal + Principal for SPNEGO authentication for Http requests + KERBEROS_PRINCIPAL + + true + + + + + logsearch.spnego.kerberos.keytab + + Http Spnego keytab + Keytab for SPNEGO authentication for Http requests + + true + + + + + logsearch.spnego.kerberos.host + _HOST + Http Spnego hostname + Fully-qualified host name in the Kerberos service principal name (SPN) that is used by SPNEGO + + true + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-service_logs-solrconfig.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-service_logs-solrconfig.xml new file mode 100644 index 00000000000..81f5482dbf8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/configuration/logsearch-service_logs-solrconfig.xml @@ -0,0 +1,62 @@ + + + + + + + logsearch_service_logs_max_retention + 7 + Max retention + Days to retain the service logs in Solr + + int + false + + + + + logsearch_service_logs_merge_factor + 5 + Merge factor + The mergeFactor value tells Lucene how many segments of equal size to build before merging them into a + single segment. High value merge factor (e.g. 25) improves indexing speed, but slows down searching. Low value + (e.g. 5) improves searching, but slows down indexing. + + + int + false + + + + + + content + solrconfig template + This is the jinja template for solrconfig.xml file for service logs + + VALUE_FROM_PROPERTY_FILE + + service_logs-solrconfig.xml.j2 + xml + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/kerberos.json b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/kerberos.json new file mode 100644 index 00000000000..ad1f56f807f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/kerberos.json @@ -0,0 +1,72 @@ +{ + "services": [ + { + "name": "AMBARI_LOGSEARCH", + "identities": [ + { + "name": "logsearch_smokeuser", + "reference": "/smokeuser" + } + ], + "components": [ + { + "name": "LOGSEARCH_SERVER", + "identities": [ + { + "name": "logsearch", + "principal": { + "value": "logsearch/_HOST@${realm}", + "type": "service", + "configuration": "logsearch-env/logsearch_kerberos_principal" + }, + "keytab": { + "file": "${keytab_dir}/logsearch.service.keytab", + "owner": { + "name": "${logsearch-env/logsearch_user}", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "logsearch-env/logsearch_kerberos_keytab" + } + }, + { + "name": "logsearch_logsearch_server_infra-solr", + "reference": "/AMBARI_INFRA_SOLR/INFRA_SOLR/infra-solr", + "when" : { + "contains" : ["services", "AMBARI_INFRA_SOLR"] + } + } + ] + }, + { + "name": "LOGSEARCH_LOGFEEDER", + "identities": [ + { + "name": "logfeeder", + "principal": { + "value": "logfeeder/_HOST@${realm}", + "type": "service", + "configuration": "logfeeder-env/logfeeder_kerberos_principal" + }, + "keytab": { + "file": "${keytab_dir}/logfeeder.service.keytab", + "owner": { + "name": "root", + "access": "r" + }, + "group": { + "name": "${cluster-env/user_group}", + "access": "" + }, + "configuration": "logfeeder-env/logfeeder_kerberos_keytab" + } + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/metainfo.xml b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/metainfo.xml new file mode 100644 index 00000000000..ec58163a922 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/metainfo.xml @@ -0,0 +1,204 @@ + + + + 2.0 + + + AMBARI_LOGSEARCH + Log Search + Log aggregation, analysis, and visualization for Ambari managed services. This service is <b>Technical Preview</b>. + 3.0.0 + TECH_PREVIEW + + + true + true + true + + + + + + LOGSEARCH_SERVER + logsearch + Log Search Server + MASTER + 0-1 + false + + + PYTHON + 1800 + + + + logsearch_app + true + + + logsearch_perf + + + + + UPGRADE_LOGSEARCH_PORTAL + true + + + PYTHON + 1200 + + + + + + AMBARI_INFRA_SOLR/INFRA_SOLR_CLIENT + host + + true + + + + + infra-solr-env + logsearch-properties + logsearch-env + logsearch-common-env + logsearch-log4j + logsearch-admin-json + logsearch-service_logs-solrconfig.xml + logsearch-audit_logs-solrconfig.xml + + + + + LOGSEARCH_LOGFEEDER + logfeeder + Log Feeder + SLAVE + ALL + false + + + PYTHON + + + + logsearch_feeder + true + + + + + + UPGRADE_LOGFEEDER + true + + + PYTHON + 1200 + + + + + + infra-solr-env + logsearch-common-env + logfeeder-properties + logfeeder-env + logfeeder-grok + logfeeder-log4j + logfeeder-system_log-env + logfeeder-output-config + + + + + + + + redhat7,amazonlinux2,redhat6,suse11,suse12 + + + ambari-logsearch-logfeeder + true + + + ambari-logsearch-portal + true + should_install_logsearch_portal + + + ambari-infra-solr-client + should_install_infra_solr_client + + + + + debian7,debian9,ubuntu12,ubuntu14,ubuntu16,ubuntu18 + + + ambari-logsearch-logfeeder + true + + + ambari-logsearch-portal + true + should_install_logsearch_portal + + + ambari-infra-solr-client + should_install_infra_solr_client + + + + + + + + PYTHON + 300 + + + + + + + + quicklinks.json + true + + + + + + theme.json + true + + + credentials.json + true + + + directories.json + true + + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/metrics.json b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/metrics.json new file mode 100644 index 00000000000..7ffb77ef14f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/metrics.json @@ -0,0 +1,53 @@ +{ + "LOGSEARCH_LOGFEEDER": { + "Component": [ + { + "type": "ganglia", + "metrics": { + "default": { + "metrics/filter/error/grok": { + "metric": "filter.error.grok", + "pointInTime": false, + "temporal": true + }, + "metrics/filter/error/truncate": { + "metric": "filter.error.truncate", + "pointInTime": false, + "temporal": true + }, + "metrics/input/files/read_lines": { + "metric": "input.files.read_lines", + "pointInTime": false, + "temporal": true + }, + "metrics/input/files/read_bytes": { + "metric": "input.files.read_bytes", + "pointInTime": false, + "temporal": true + }, + "metrics/output/kafka/write_logs": { + "metric": "output.kafka.write_logs", + "pointInTime": false, + "temporal": true + }, + "metrics/output/kafka/write_bytes": { + "metric": "output.kafka.write_bytes", + "pointInTime": false, + "temporal": true + }, + "metrics/output/solr/write_logs": { + "metric": "output.solr.write_logs", + "pointInTime": false, + "temporal": true + }, + "metrics/output/solr/write_bytes": { + "metric": "output.solr.write_bytes", + "pointInTime": false, + "temporal": true + } + } + } + } + ] + } +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/alerts/alert_logfeeder.py b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/alerts/alert_logfeeder.py new file mode 100755 index 00000000000..022be07ba13 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/alerts/alert_logfeeder.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import os +import socket + +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.core.exceptions import ComponentIsNotRunning + +RESULT_CODE_OK = 'OK' +RESULT_CODE_CRITICAL = 'CRITICAL' +RESULT_CODE_UNKNOWN = 'UNKNOWN' + +LOGFEEDER_PID_DIR = '{{logfeeder-env/logfeeder_pid_dir}}' + +def get_tokens(): + """ + Returns a tuple of tokens in the format {{site/property}} that will be used + to build the dictionary passed into execute + """ + return (LOGFEEDER_PID_DIR,) + +def is_logfeeder_process_live(pid_file): + """ + Gets whether the LogSearch Logfeeder represented by the specified file is running. + :param pid_file: the PID file of the Logfeeder to check + :return: True if the Logfeeder is running, False otherwise + """ + live = False + + try: + check_process_status(pid_file) + live = True + except ComponentIsNotRunning: + pass + + return live + + +def execute(configurations={}, parameters={}, host_name=None): + """ + Returns a tuple containing the result code and a pre-formatted result label + + Keyword arguments: + configurations (dictionary): a mapping of configuration key to value + parameters (dictionary): a mapping of script parameter key to value + host_name (string): the name of this host where the alert is running + """ + + if configurations is None: + return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.']) + + if set([LOGFEEDER_PID_DIR]).issubset(configurations): + LOGFEEDER_PID_PATH = os.path.join(configurations[LOGFEEDER_PID_DIR], 'logfeeder.pid') + else: + return (RESULT_CODE_UNKNOWN, ['The logfeeder_pid_dir is a required parameter.']) + + if host_name is None: + host_name = socket.getfqdn() + + logfeeder_process_running = is_logfeeder_process_live(LOGFEEDER_PID_PATH) + + alert_state = RESULT_CODE_OK if logfeeder_process_running else RESULT_CODE_CRITICAL + + alert_label = 'LogFeeder is running on {0}' if logfeeder_process_running else 'LogFeeder is NOT running on {0}' + alert_label = alert_label.format(host_name) + + return (alert_state, [alert_label]) \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/logfeeder.py b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/logfeeder.py new file mode 100644 index 00000000000..db647ff736b --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/logfeeder.py @@ -0,0 +1,70 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from ambari_commons.repo_manager import ManagerFactory +from ambari_commons.shell import RepoCallContext +from resource_management.core.resources.system import Execute +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.script.script import Script +from setup_logfeeder import setup_logfeeder + +class LogFeeder(Script): + def install(self, env): + import params + env.set_params(params) + + self.install_packages(env) + + def configure(self, env, upgrade_type=None): + import params + env.set_params(params) + + setup_logfeeder() + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + + Execute((format('{logfeeder_dir}/bin/logfeeder.sh'), "start"), + sudo=True) + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + Execute((format('{logfeeder_dir}/bin/logfeeder.sh'), "stop"), + sudo=True) + + def status(self, env): + import status_params + env.set_params(status_params) + + check_process_status(status_params.logfeeder_pid_file) + + def upgrade_logfeeder(self, env): + pkg_provider = ManagerFactory.get() + context = RepoCallContext() + context.log_output = True + pkg_provider.remove_package('ambari-logsearch-logfeeder', context, ignore_dependencies=True) + pkg_provider.upgrade_package('ambari-logsearch-logfeeder', context) + +if __name__ == "__main__": + LogFeeder().execute() diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/logsearch.py b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/logsearch.py new file mode 100644 index 00000000000..fdafaaf73ce --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/logsearch.py @@ -0,0 +1,71 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from ambari_commons.repo_manager import ManagerFactory +from ambari_commons.shell import RepoCallContext +from resource_management.core.resources.system import Execute, File +from resource_management.libraries.functions.check_process_status import check_process_status +from resource_management.libraries.functions.format import format +from resource_management.libraries.script.script import Script +from setup_logsearch import setup_logsearch + +class LogSearch(Script): + def install(self, env): + import params + env.set_params(params) + self.install_packages(env) + + def configure(self, env, upgrade_type=None): + import params + env.set_params(params) + + setup_logsearch() + + def start(self, env, upgrade_type=None): + import params + env.set_params(params) + self.configure(env) + + Execute(format("{logsearch_dir}/bin/logsearch.sh start"), + user=params.logsearch_user + ) + + def stop(self, env, upgrade_type=None): + import params + env.set_params(params) + + Execute(format("{logsearch_dir}/bin/logsearch.sh stop"), + user=params.logsearch_user + ) + + def status(self, env): + import status_params + env.set_params(status_params) + + check_process_status(status_params.logsearch_pid_file) + + def upgrade_logsearch_portal(self, env): + pkg_provider = ManagerFactory.get() + context = RepoCallContext() + context.log_output = True + pkg_provider.remove_package('ambari-logsearch-portal', context, ignore_dependencies=True) + pkg_provider.upgrade_package('ambari-logsearch-portal', context) + +if __name__ == "__main__": + LogSearch().execute() diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/params.py new file mode 100644 index 00000000000..6f30144ec79 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/params.py @@ -0,0 +1,429 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import os +from ambari_commons.constants import AMBARI_SUDO_BINARY +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.is_empty import is_empty +from resource_management.libraries.script.script import Script + +import status_params + + +def get_port_from_url(address): + if not is_empty(address): + return address.split(':')[-1] + else: + return address + +def get_name_from_principal(principal): + if not principal: # return if empty + return principal + slash_split = principal.split('/') + if len(slash_split) == 2: + return slash_split[0] + else: + at_split = principal.split('@') + return at_split[0] + + +# config object that holds the configurations declared in the -site.xml file +config = Script.get_config() +tmp_dir = Script.get_tmp_dir() + +sudo = AMBARI_SUDO_BINARY +security_enabled = status_params.security_enabled + +credential_store_enabled = False +if 'credentialStoreEnabled' in config: + credential_store_enabled = config['credentialStoreEnabled'] + +logsearch_server_conf = "/usr/lib/ambari-logsearch-portal/conf" +logsearch_server_keys_folder = logsearch_server_conf + "/keys" +logsearch_logfeeder_conf = "/usr/lib/ambari-logsearch-logfeeder/conf" +logsearch_logfeeder_keys_folder = logsearch_logfeeder_conf + "/keys" + +logsearch_config_set_dir = format("{logsearch_server_conf}/solr_configsets") + +# logsearch pid file +logsearch_pid_dir = status_params.logsearch_pid_dir +logsearch_pid_file = status_params.logsearch_pid_file + +# logfeeder pid file +logfeeder_pid_dir = status_params.logfeeder_pid_dir +logfeeder_pid_file = status_params.logfeeder_pid_file + +user_group = config['configurations']['cluster-env']['user_group'] + +# shared configs +java_home = config['ambariLevelParams']['java_home'] +ambari_java_home = default("/commandParams/ambari_java_home", None) +java64_home = ambari_java_home if ambari_java_home is not None else java_home +cluster_name = str(config['clusterName']) + +configurations = config['configurations'] # need reference inside logfeeder jinja templates + +# for now just pick first collector +if 'metrics_collector_hosts' in config['clusterHostInfo']: + metrics_http_policy = config['configurations']['ams-site']['timeline.metrics.service.http.policy'] + metrics_collector_protocol = 'http' + if metrics_http_policy == 'HTTPS_ONLY': + metrics_collector_protocol = 'https' + + metrics_collector_hosts = ",".join(config['clusterHostInfo']['metrics_collector_hosts']) + metrics_collector_port = str(get_port_from_url(config['configurations']['ams-site']['timeline.metrics.service.webapp.address'])) + metrics_collector_path = '/ws/v1/timeline/metrics' +else: + metrics_collector_protocol = '' + metrics_collector_hosts = '' + metrics_collector_port = '' + metrics_collector_path = '' + +##################################### +# Infra Solr configs +##################################### +infra_solr_znode = '/infra-solr' +infra_solr_ssl_enabled = False +infra_solr_jmx_port = '' + +if 'infra-solr-env' in config['configurations']: + infra_solr_znode = default('/configurations/infra-solr-env/infra_solr_znode', '/infra-solr') + infra_solr_ssl_enabled = default('configurations/infra-solr-env/infra_solr_ssl_enabled', False) + infra_solr_jmx_port = config['configurations']['infra-solr-env']['infra_solr_jmx_port'] + +infra_solr_role_logsearch = default('configurations/infra-solr-security-json/infra_solr_role_logsearch', 'logsearch_user') +infra_solr_role_logfeeder = default('configurations/infra-solr-security-json/infra_solr_role_logfeeder', 'logfeeder_user') +infra_solr_role_dev = default('configurations/infra-solr-security-json/infra_solr_role_dev', 'dev') +infra_solr_role_ranger_admin = default('configurations/infra-solr-security-json/infra_solr_role_ranger_admin', 'ranger_user') + +_hostname_lowercase = config['agentLevelParams']['hostname'].lower() +logsearch_jaas_file = None +if security_enabled: + kinit_path_local = status_params.kinit_path_local + logsearch_jaas_file = logsearch_server_conf + '/logsearch_jaas.conf' + logfeeder_jaas_file = logsearch_logfeeder_conf + '/logfeeder_jaas.conf' + use_external_solr_with_kerberos = default('configurations/logsearch-common-env/logsearch_external_solr_kerberos_enabled', False) + if use_external_solr_with_kerberos: + logsearch_kerberos_keytab = config['configurations']['logsearch-env']['logsearch_external_solr_kerberos_keytab'] + logsearch_kerberos_principal = config['configurations']['logsearch-env']['logsearch_external_solr_kerberos_principal'].replace('_HOST',_hostname_lowercase) + logfeeder_kerberos_keytab = config['configurations']['logfeeder-env']['logfeeder_external_solr_kerberos_keytab'] + logfeeder_kerberos_principal = config['configurations']['logfeeder-env']['logfeeder_external_solr_kerberos_principal'].replace('_HOST',_hostname_lowercase) + else: + logsearch_kerberos_keytab = config['configurations']['logsearch-env']['logsearch_kerberos_keytab'] + logsearch_kerberos_principal = config['configurations']['logsearch-env']['logsearch_kerberos_principal'].replace('_HOST',_hostname_lowercase) + logfeeder_kerberos_keytab = config['configurations']['logfeeder-env']['logfeeder_kerberos_keytab'] + logfeeder_kerberos_principal = config['configurations']['logfeeder-env']['logfeeder_kerberos_principal'].replace('_HOST',_hostname_lowercase) + + zookeeper_principal_name = default("/configurations/zookeeper-env/zookeeper_principal_name", "zookeeper/_HOST@EXAMPLE.COM") + external_zk_principal_enabled = default("/configurations/logsearch-common-env/logsearch_zookeeper_external_principal", False) + external_zk_principal_name = default("/configurations/logsearch-common-env/logsearch_zookeeper_external_principal", "zookeeper/_HOST@EXAMPLE.COM") + zk_principal_name = external_zk_principal_name if external_zk_principal_enabled else zookeeper_principal_name + zk_principal_user = zk_principal_name.split('/')[0] + zk_security_opts = format('-Dzookeeper.sasl.client=true -Dzookeeper.sasl.client.username={zk_principal_user} -Dzookeeper.sasl.clientconfig=Client') + + logsearch_solr_service_users = [] + logsearch_kerberos_service_user = get_name_from_principal(logsearch_kerberos_principal) + logsearch_solr_service_users.append(logsearch_kerberos_service_user) + logsearch_kerberos_service_users_str = str(default('/configurations/logsearch-env/logsearch_kerberos_service_users', '')) + if logsearch_kerberos_service_users_str and logsearch_kerberos_service_users_str.strip(): + logsearch_kerberos_service_users = logsearch_kerberos_service_users_str.split(',') + logsearch_solr_service_users.extend(logsearch_kerberos_service_users) + +logsearch_spnego_host = config['configurations']['logsearch-properties']['logsearch.spnego.kerberos.host'].replace('_HOST', _hostname_lowercase) + +##################################### +# Logsearch configs +##################################### +logsearch_dir = '/usr/lib/ambari-logsearch-portal' + +logsearch_service_logs_max_retention = config['configurations']['logsearch-service_logs-solrconfig']['logsearch_service_logs_max_retention'] +logsearch_service_logs_merge_factor = config['configurations']['logsearch-service_logs-solrconfig']['logsearch_service_logs_merge_factor'] + +logsearch_audit_logs_max_retention = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_max_retention'] +logsearch_audit_logs_merge_factor = config['configurations']['logsearch-audit_logs-solrconfig']['logsearch_audit_logs_merge_factor'] + +logsearch_use_external_solr = default('/configurations/logsearch-common-env/logsearch_use_external_solr', False) + +if logsearch_use_external_solr: + logsearch_solr_zk_znode = config['configurations']['logsearch-common-env']['logsearch_external_solr_zk_znode'] + logsearch_solr_zk_quorum = config['configurations']['logsearch-common-env']['logsearch_external_solr_zk_quorum'] + logsearch_solr_ssl_enabled = default('configurations/logsearch-common-env/logsearch_external_solr_ssl_enabled', False) + logsearch_solr_kerberos_enabled = security_enabled and use_external_solr_with_kerberos +else: + logsearch_solr_zk_znode = infra_solr_znode + + logsearch_solr_zk_quorum = "" + zookeeper_port = default('/configurations/zoo.cfg/clientPort', None) + if 'zookeeper_server_hosts' in config['clusterHostInfo']: + for host in config['clusterHostInfo']['zookeeper_server_hosts']: + if logsearch_solr_zk_quorum: + logsearch_solr_zk_quorum += ',' + logsearch_solr_zk_quorum += host + ":" + str(zookeeper_port) + + logsearch_solr_ssl_enabled = infra_solr_ssl_enabled + logsearch_solr_kerberos_enabled = security_enabled + +zookeeper_quorum = logsearch_solr_zk_quorum + +# logsearch-env configs +logsearch_user = config['configurations']['logsearch-env']['logsearch_user'] +logsearch_log_dir = config['configurations']['logsearch-env']['logsearch_log_dir'] +logsearch_log = 'logsearch.out' +logsearch_debug_enabled = str(config['configurations']['logsearch-env']["logsearch_debug_enabled"]).lower() +logsearch_debug_port = config['configurations']['logsearch-env']["logsearch_debug_port"] +logsearch_app_max_memory = config['configurations']['logsearch-env']['logsearch_app_max_memory'] + +logsearch_keystore_location = config['configurations']['logsearch-env']['logsearch_keystore_location'] +logsearch_keystore_type = config['configurations']['logsearch-env']['logsearch_keystore_type'] +logsearch_keystore_password = config['configurations']['logsearch-env']['logsearch_keystore_password'] +logsearch_truststore_location = config['configurations']['logsearch-env']['logsearch_truststore_location'] +logsearch_truststore_type = config['configurations']['logsearch-env']['logsearch_truststore_type'] +logsearch_truststore_password = config['configurations']['logsearch-env']['logsearch_truststore_password'] + +logsearch_env_config = dict(config['configurations']['logsearch-env']) +logsearch_env_jceks_file = os.path.join(logsearch_server_conf, 'logsearch.jceks') + +#Logsearch log4j properties +logsearch_log_maxfilesize = default('/configurations/logsearch-log4j/logsearch_log_maxfilesize',10) +logsearch_log_maxbackupindex = default('/configurations/logsearch-log4j/logsearch_log_maxbackupindex',10) +logsearch_json_log_maxfilesize = default('/configurations/logsearch-log4j/logsearch_json_log_maxfilesize',10) +logsearch_json_log_maxbackupindex = default('/configurations/logsearch-log4j/logsearch_json_log_maxbackupindex',10) +logsearch_audit_log_maxfilesize = default('/configurations/logsearch-log4j/logsearch_audit_log_maxfilesize',10) +logsearch_audit_log_maxbackupindex =default('/configurations/logsearch-log4j/logsearch_audit_log_maxbackupindex',10) +logsearch_perf_log_maxfilesize =default('/configurations/logsearch-log4j/logsearch_perf_log_maxfilesize',10) +logsearch_perf_log_maxbackupindex =default('/configurations/logsearch-log4j/logsearch_perf_log_maxbackupindex',10) + +# store the log file for the service from the 'solr.log' property of the 'logsearch-env.xml' file +logsearch_env_content = config['configurations']['logsearch-env']['content'] +logsearch_service_logs_solrconfig_content = config['configurations']['logsearch-service_logs-solrconfig']['content'] +logsearch_audit_logs_solrconfig_content = config['configurations']['logsearch-audit_logs-solrconfig']['content'] +logsearch_app_log4j_content = config['configurations']['logsearch-log4j']['content'] + +# Log dirs +ambari_server_log_dir = default('/configurations/logfeeder-env/ambari_server_log_dir', "/var/log/ambari-server") +ambari_agent_log_dir = default('/configurations/logfeeder-env/ambari_agent_log_dir', "/var/log/ambari-agent") + +# System logs +logfeeder_system_messages_content = config['configurations']['logfeeder-system_log-env']['logfeeder_system_messages_content'] +logfeeder_secure_log_content = config['configurations']['logfeeder-system_log-env']['logfeeder_secure_log_content'] +logfeeder_system_log_enabled = default('/configurations/logfeeder-system_log-env/logfeeder_system_log_enabled', False) + +# Logsearch auth configs + +logsearch_admin_credential_file = 'logsearch-admin.json' +logsearch_admin_username = default('/configurations/logsearch-admin-json/logsearch_admin_username', "admin") +logsearch_admin_password = default('/configurations/logsearch-admin-json/logsearch_admin_password', "") +logsearch_admin_content = config['configurations']['logsearch-admin-json']['content'] + +# for now just pick first collector +if 'ambari_server_host' in config['ambariLevelParams']: + ambari_server_host = config['ambariLevelParams']['ambari_server_host'] + ambari_server_port = config['ambariLevelParams']['ambari_server_port'] + ambari_server_use_ssl = config['ambariLevelParams']['ambari_server_use_ssl'] + + ambari_server_protocol = 'https' if ambari_server_use_ssl else 'http' + + ambari_server_auth_host_url = format('{ambari_server_protocol}://{ambari_server_host}:{ambari_server_port}') +else: + ambari_server_auth_host_url = '' + +# Logsearch propreties + +logsearch_protocol = config['configurations']['logsearch-properties']["logsearch.protocol"] +logsearch_http_port = config['configurations']['logsearch-properties']["logsearch.http.port"] +logsearch_https_port = config['configurations']['logsearch-properties']["logsearch.https.port"] + +logsearch_properties = {} + +# default values + +logsearch_properties['logsearch.solr.zk_connect_string'] = logsearch_solr_zk_quorum + logsearch_solr_zk_znode +logsearch_properties['logsearch.solr.audit.logs.zk_connect_string'] = logsearch_solr_zk_quorum + logsearch_solr_zk_znode + +logsearch_properties['logsearch.solr.collection.history'] = 'history' +logsearch_properties['logsearch.solr.history.config.name'] = 'history' +logsearch_properties['logsearch.collection.history.replication.factor'] = '2' + +logsearch_properties['logsearch.solr.jmx.port'] = infra_solr_jmx_port + +logsearch_properties['logsearch.login.credentials.file'] = logsearch_admin_credential_file +logsearch_properties['logsearch.auth.file.enabled'] = 'true' +logsearch_properties['logsearch.auth.ldap.enabled'] = 'false' +logsearch_properties['logsearch.auth.simple.enabled'] = 'false' + +# load config values + +logsearch_properties = dict(logsearch_properties.items() +\ + dict(config['configurations']['logsearch-common-properties']).items() +\ + dict(config['configurations']['logsearch-properties']).items()) + +# load derivated values + +if logsearch_properties['logsearch.solr.audit.logs.use.ranger'] == 'false': + del logsearch_properties['logsearch.ranger.audit.logs.collection.name'] + +del logsearch_properties['logsearch.solr.audit.logs.use.ranger'] + +logsearch_properties['logsearch.solr.metrics.collector.hosts'] = format(logsearch_properties['logsearch.solr.metrics.collector.hosts']) +logsearch_properties['logsearch.auth.external_auth.host_url'] = format(logsearch_properties['logsearch.auth.external_auth.host_url']) +logsearch_properties['logsearch.spnego.kerberos.host'] = logsearch_spnego_host + +if not('logsearch.config.zk_connect_string' in logsearch_properties): + logsearch_properties['logsearch.config.zk_connect_string'] = logsearch_solr_zk_quorum + +if logsearch_solr_kerberos_enabled: + logsearch_properties['logsearch.solr.kerberos.enable'] = 'true' + logsearch_properties['logsearch.solr.jaas.file'] = logsearch_jaas_file + + +logsearch_solr_collection_service_logs = logsearch_properties['logsearch.solr.collection.service.logs'] +logsearch_service_logs_split_interval_mins = logsearch_properties['logsearch.service.logs.split.interval.mins'] +logsearch_collection_service_logs_numshards = logsearch_properties['logsearch.collection.service.logs.numshards'] + +logsearch_solr_collection_audit_logs = logsearch_properties['logsearch.solr.collection.audit.logs'] +logsearch_audit_logs_split_interval_mins = logsearch_properties['logsearch.audit.logs.split.interval.mins'] +logsearch_collection_audit_logs_numshards = logsearch_properties['logsearch.collection.audit.logs.numshards'] + +# check if logsearch uses ssl in any way + +logsearch_use_ssl = logsearch_solr_ssl_enabled or logsearch_protocol == 'https' or ambari_server_use_ssl + +##################################### +# Logfeeder configs +##################################### + +logfeeder_dir = "/usr/lib/ambari-logsearch-logfeeder" + +# logfeeder-log4j +logfeeder_log_maxfilesize = default('/configurations/logfeeder-log4j/logfeeder_log_maxfilesize',10) +logfeeder_log_maxbackupindex = default('/configurations/logfeeder-log4j/logfeeder_log_maxbackupindex',10) +logfeeder_json_log_maxfilesize = default('/configurations/logfeeder-log4j/logfeeder_json_log_maxfilesize',10) +logfeeder_json_log_maxbackupindex = default('/configurations/logfeeder-log4j/logfeeder_json_log_maxbackupindex',10) + +# logfeeder-env configs +logfeeder_log_dir = config['configurations']['logfeeder-env']['logfeeder_log_dir'] +logfeeder_log = 'logfeeder.out' +logfeeder_max_mem = config['configurations']['logfeeder-env']['logfeeder_max_mem'] +solr_service_logs_enable = default('/configurations/logfeeder-env/logfeeder_solr_service_logs_enable', True) +solr_audit_logs_enable = default('/configurations/logfeeder-env/logfeeder_solr_audit_logs_enable', True) +logfeeder_env_content = config['configurations']['logfeeder-env']['content'] +logfeeder_log4j_content = config['configurations']['logfeeder-log4j']['content'] + +logfeeder_keystore_location = config['configurations']['logfeeder-env']['logfeeder_keystore_location'] +logfeeder_keystore_type = config['configurations']['logfeeder-env']['logfeeder_keystore_type'] +logfeeder_keystore_password = config['configurations']['logfeeder-env']['logfeeder_keystore_password'] +logfeeder_truststore_location = config['configurations']['logfeeder-env']['logfeeder_truststore_location'] +logfeeder_truststore_type = config['configurations']['logfeeder-env']['logfeeder_truststore_type'] +logfeeder_truststore_password = config['configurations']['logfeeder-env']['logfeeder_truststore_password'] + +logfeeder_env_config = dict(config['configurations']['logfeeder-env']) +logfeeder_env_jceks_file = os.path.join(logsearch_logfeeder_conf, 'logfeeder.jceks') + +logfeeder_ambari_config_content = config['configurations']['logfeeder-ambari-config']['content'] +logfeeder_output_config_content = config['configurations']['logfeeder-output-config']['content'] +logfeeder_kafka_output_config_content = config['configurations']['logfeeder-kafka-output-config']['content'] + +logfeeder_kafka_output_enabled = False +if 'logfeeder-kafka-output-config' in config['configurations']: + logfeeder_kafka_output_enabled = default('/configurations/logfeeder-kafka-output-config/logfeeder_kafka_output_enabled', False) + logfeeder_kafka_broker_list = default('/configurations/logfeeder-kafka-output-config/logfeeder_kafka_broker_list', 'localhost:6667') + logfeeder_kafka_topic = default('/configurations/logfeeder-kafka-output-config/logfeeder_kafka_topic', 'log-streaming') + +if logfeeder_kafka_output_enabled: + default_config_files = ','.join(['output.config.json','global.config.json', 'kafka-output.json']) +else: + default_config_files = ','.join(['output.config.json','global.config.json']) + +logfeeder_grok_patterns = config['configurations']['logfeeder-grok']['default_grok_patterns'] +if config['configurations']['logfeeder-grok']['custom_grok_patterns'].strip(): + logfeeder_grok_patterns = \ + logfeeder_grok_patterns + '\n' + \ + '\n' + \ + '########################\n' +\ + '# Custom Grok Patterns #\n' +\ + '########################\n' +\ + '\n' + \ + config['configurations']['logfeeder-grok']['custom_grok_patterns'] + +# logfeeder properties + +# load default values + +logfeeder_properties = {} + +logfeeder_properties['logfeeder.solr.core.config.name'] = 'history' + +# load config values + +logfeeder_properties = dict(logfeeder_properties.items() +\ + dict(config['configurations']['logsearch-common-properties']).items() +\ + dict(config['configurations']['logfeeder-properties']).items()) + +# load derivated values + +logfeeder_properties['cluster.name'] = cluster_name +logfeeder_properties['logfeeder.config.dir'] = logsearch_logfeeder_conf +logfeeder_properties['logfeeder.config.files'] = format(logfeeder_properties['logfeeder.config.files']) +logfeeder_properties['logfeeder.solr.zk_connect_string'] = logsearch_solr_zk_quorum + logsearch_solr_zk_znode + +logfeeder_properties['logfeeder.metrics.collector.hosts'] = format(logfeeder_properties['logfeeder.metrics.collector.hosts']) +logfeeder_properties['logfeeder.metrics.collector.protocol'] = metrics_collector_protocol +logfeeder_properties['logfeeder.metrics.collector.port'] = metrics_collector_port +logfeeder_properties['logfeeder.metrics.collector.path'] = '/ws/v1/timeline/metrics' + +if not('logsearch.config.zk_connect_string' in logfeeder_properties): + logfeeder_properties['logsearch.config.zk_connect_string'] = logsearch_solr_zk_quorum + +if logsearch_solr_kerberos_enabled: + if 'logfeeder.solr.kerberos.enable' not in logfeeder_properties: + logfeeder_properties['logfeeder.solr.kerberos.enable'] = 'true' + if 'logfeeder.solr.jaas.file' not in logfeeder_properties: + logfeeder_properties['logfeeder.solr.jaas.file'] = logfeeder_jaas_file + +logfeeder_checkpoint_folder = logfeeder_properties['logfeeder.checkpoint.folder'] + +# check if logfeeder uses ssl in any way + +logfeeder_use_ssl = logsearch_solr_ssl_enabled or metrics_collector_protocol == 'https' + +logsearch_acls = '' +if 'infra-solr-env' in config['configurations'] and security_enabled and not logsearch_use_external_solr: + acl_infra_solr_principal = get_name_from_principal(config['configurations']['infra-solr-env']['infra_solr_kerberos_principal']) + acl_logsearch_principal = get_name_from_principal(config['configurations']['logsearch-env']['logsearch_kerberos_principal']) + logsearch_acls = format('world:anyone:r,sasl:{acl_infra_solr_principal}:cdrwa,sasl:{acl_logsearch_principal}:cdrwa') + logsearch_properties['logsearch.solr.zk.acls'] = logsearch_acls + logsearch_properties['logsearch.solr.audit.logs.zk.acls'] = logsearch_acls + if not('logsearch.config.zk_acls' in logsearch_properties): + logsearch_properties['logsearch.config.zk_acls'] = logsearch_acls + if not('logsearch.config.zk_acls' in logfeeder_properties): + logfeeder_properties['logsearch.config.zk_acls'] = logsearch_acls + +##################################### +# Smoke command +##################################### + +logsearch_server_hosts = default('/clusterHostInfo/logsearch_server_hosts', None) +logsearch_server_host = "" +logsearch_ui_port = logsearch_https_port if logsearch_protocol == 'https' else logsearch_http_port +if logsearch_server_hosts is not None and len(logsearch_server_hosts) > 0: + logsearch_server_host = logsearch_server_hosts[0] +smoke_logsearch_cmd = format('curl -k -s -o /dev/null -w "%{{http_code}}" {logsearch_protocol}://{logsearch_server_host}:{logsearch_ui_port}/api/v1/info | grep 200') diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/service_check.py new file mode 100644 index 00000000000..b7940365785 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/service_check.py @@ -0,0 +1,41 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.script.script import Script +from resource_management.core.logger import Logger +from resource_management.core.resources.system import Execute + +class LogSearchServiceCheck(Script): + def service_check(self, env): + import params + env.set_params(params) + + try: + if params.logsearch_server_host: + Execute(params.smoke_logsearch_cmd, user=params.logsearch_user, + tries=15, try_sleep=5, timeout=10) + Logger.info('Log Search Server up and running') + else: + Logger.info('No portal is installed on the cluster thus no service check is required') + except: + Logger.error('Log Search Server not running') + raise + +if __name__ == "__main__": + LogSearchServiceCheck().execute() \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/setup_logfeeder.py b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/setup_logfeeder.py new file mode 100644 index 00000000000..067da8c23a8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/setup_logfeeder.py @@ -0,0 +1,136 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.functions.default import default +from resource_management.core.resources.system import Directory, File +from resource_management.libraries.functions.format import format +from resource_management.core.source import InlineTemplate, Template +from resource_management.libraries.functions.generate_logfeeder_input_config import generate_logfeeder_input_config +from resource_management.libraries.resources.properties_file import PropertiesFile +from resource_management.libraries.functions.security_commons import update_credential_provider_path, HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME + +def setup_logfeeder(): + import params + + Directory([params.logfeeder_log_dir, params.logfeeder_pid_dir, params.logfeeder_checkpoint_folder], + mode=0755, + cd_access='a', + create_parents=True + ) + + Directory([params.logfeeder_dir, params.logsearch_logfeeder_conf], + mode=0755, + cd_access='a', + create_parents=True, + recursive_ownership=True + ) + + File(format("{logfeeder_log_dir}/{logfeeder_log}"), + mode=0644, + content='' + ) + + if params.credential_store_enabled: + params.logfeeder_env_config = update_credential_provider_path(params.logfeeder_env_config, + 'logfeeder-env', + params.logfeeder_env_jceks_file, + params.logsearch_user, + params.user_group + ) + params.logfeeder_properties[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file' + params.logfeeder_env_jceks_file + File(format("{logsearch_logfeeder_keys_folder}/ks_pass.txt"), + action="delete" + ) + File(format("{logsearch_logfeeder_keys_folder}/ts_pass.txt"), + action="delete" + ) + else: + Directory(params.logsearch_logfeeder_keys_folder, + cd_access='a', + mode=0755, + owner=params.logsearch_user, + group=params.user_group + ) + + File(format("{logsearch_logfeeder_keys_folder}/ks_pass.txt"), + content=params.logfeeder_keystore_password, + mode=0600, + owner=params.logsearch_user, + group=params.user_group + ) + + File(format("{logsearch_logfeeder_keys_folder}/ts_pass.txt"), + content=params.logfeeder_truststore_password, + mode=0600, + owner=params.logsearch_user, + group=params.user_group + ) + + PropertiesFile(format("{logsearch_logfeeder_conf}/logfeeder.properties"), + properties = params.logfeeder_properties + ) + + File(format("{logsearch_logfeeder_conf}/logfeeder-env.sh"), + content=InlineTemplate(params.logfeeder_env_content), + mode=0755 + ) + + File(format("{logsearch_logfeeder_conf}/log4j.xml"), + content=InlineTemplate(params.logfeeder_log4j_content) + ) + + File(format("{logsearch_logfeeder_conf}/grok-patterns"), + content=InlineTemplate(params.logfeeder_grok_patterns), + encoding="utf-8" + ) + + File(format("{logsearch_logfeeder_conf}/global.config.json"), + content=Template("global.config.json.j2") + ) + + File(format("{logsearch_logfeeder_conf}/input.config-ambari.json"), + content=InlineTemplate(params.logfeeder_ambari_config_content), + encoding="utf-8" + ) + + File(format("{logsearch_logfeeder_conf}/output.config.json"), + content=InlineTemplate(params.logfeeder_output_config_content), + encoding="utf-8" + ) + if params.logfeeder_kafka_output_enabled: + File(format("{logsearch_logfeeder_conf}/kafka-output.json"), + content=InlineTemplate(params.logfeeder_kafka_output_config_content), + encoding="utf-8" + ) + + if params.logfeeder_system_log_enabled: + File(format("{logsearch_logfeeder_conf}/input.config-system_messages.json"), + content=params.logfeeder_system_messages_content + ) + File(format("{logsearch_logfeeder_conf}/input.config-secure_log.json"), + content=params.logfeeder_secure_log_content + ) + + generate_logfeeder_input_config('logsearch', Template("input.config-logsearch.json.j2", extra_imports=[default])) + + if params.logsearch_solr_kerberos_enabled: + File(format("{logfeeder_jaas_file}"), + content=Template("logfeeder_jaas.conf.j2") + ) + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/setup_logsearch.py b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/setup_logsearch.py new file mode 100644 index 00000000000..e3e9ee113d8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/setup_logsearch.py @@ -0,0 +1,162 @@ +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.core.exceptions import Fail +from resource_management.core.resources.system import Directory, Execute, File +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.format import format +from resource_management.core.source import InlineTemplate, Template +from resource_management.libraries.functions import solr_cloud_util +from resource_management.libraries.functions.decorator import retry +from resource_management.libraries.functions.generate_logfeeder_input_config import generate_logfeeder_input_config +from resource_management.libraries.resources.properties_file import PropertiesFile +from resource_management.libraries.functions.security_commons import update_credential_provider_path, HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME + + +def setup_logsearch(): + import params + + Directory([params.logsearch_log_dir, params.logsearch_pid_dir], + mode=0755, + cd_access='a', + owner=params.logsearch_user, + group=params.user_group, + create_parents=True + ) + + Directory([params.logsearch_dir, params.logsearch_server_conf, params.logsearch_config_set_dir], + mode=0755, + cd_access='a', + owner=params.logsearch_user, + group=params.user_group, + create_parents=True, + recursive_ownership=True + ) + + Directory(params.logsearch_server_keys_folder, + cd_access='a', + mode=0755, + owner=params.logsearch_user, + group=params.user_group) + + File(format("{logsearch_log_dir}/{logsearch_log}"), + mode=0644, + owner=params.logsearch_user, + group=params.user_group, + content='' + ) + + if params.credential_store_enabled: + params.logsearch_env_config = update_credential_provider_path(params.logsearch_env_config, + 'logsearch-env', + params.logsearch_env_jceks_file, + params.logsearch_user, + params.user_group + ) + params.logsearch_properties[HADOOP_CREDENTIAL_PROVIDER_PROPERTY_NAME] = 'jceks://file' + params.logsearch_env_jceks_file + File(format("{logsearch_server_keys_folder}/ks_pass.txt"), + action="delete" + ) + File(format("{logsearch_server_keys_folder}/ts_pass.txt"), + action="delete" + ) + else: + File(format("{logsearch_server_keys_folder}/ks_pass.txt"), + content=params.logsearch_keystore_password, + mode=0600, + owner= params.logsearch_user, + group=params.user_group + ) + File(format("{logsearch_server_keys_folder}/ts_pass.txt"), + content=params.logsearch_truststore_password, + mode=0600, + owner= params.logsearch_user, + group=params.user_group + ) + + PropertiesFile(format("{logsearch_server_conf}/logsearch.properties"), + properties=params.logsearch_properties + ) + + File(format("{logsearch_server_conf}/HadoopServiceConfig.json"), + content=Template("HadoopServiceConfig.json.j2"), + owner=params.logsearch_user, + group=params.user_group + ) + + File(format("{logsearch_server_conf}/log4j.xml"), + content=InlineTemplate(params.logsearch_app_log4j_content), + owner=params.logsearch_user, + group=params.user_group + ) + + File(format("{logsearch_server_conf}/logsearch-env.sh"), + content=InlineTemplate(params.logsearch_env_content), + mode=0755, + owner=params.logsearch_user, + group=params.user_group + ) + + File(format("{logsearch_server_conf}/logsearch-admin.json"), + content=InlineTemplate(params.logsearch_admin_content), + owner=params.logsearch_user, + group=params.user_group + ) + + File(format("{logsearch_config_set_dir}/hadoop_logs/conf/solrconfig.xml"), + content=InlineTemplate(params.logsearch_service_logs_solrconfig_content), + owner=params.logsearch_user, + group=params.user_group + ) + + File(format("{logsearch_config_set_dir}/audit_logs/conf/solrconfig.xml"), + content=InlineTemplate(params.logsearch_audit_logs_solrconfig_content), + owner=params.logsearch_user, + group=params.user_group + ) + + if params.security_enabled: + File(format("{logsearch_jaas_file}"), + content=Template("logsearch_jaas.conf.j2"), + owner=params.logsearch_user + ) + Execute(("chmod", "-R", "ugo+r", format("{logsearch_server_conf}/solr_configsets")), + sudo=True + ) + generate_logfeeder_input_config('logsearch', Template("input.config-logsearch.json.j2", extra_imports=[default])) + check_znode() + + if params.security_enabled and not params.logsearch_use_external_solr: + solr_cloud_util.add_solr_roles(params.config, + roles = [params.infra_solr_role_logsearch, params.infra_solr_role_ranger_admin, params.infra_solr_role_dev], + new_service_principals = params.logsearch_solr_service_users) + solr_cloud_util.add_solr_roles(params.config, + roles = [params.infra_solr_role_logfeeder, params.infra_solr_role_dev], + new_service_principals = [params.logfeeder_kerberos_principal]) + +@retry(times=30, sleep_time=5, err_class=Fail) +def check_znode(): + import params + solr_cloud_util.check_znode( + zookeeper_quorum=params.logsearch_solr_zk_quorum, + solr_znode=params.logsearch_solr_zk_znode, + java64_home=params.java64_home, + retry=30, interval=5, + java_opts=params.zk_security_opts if params.security_enabled else None, + jaas_file=params.logsearch_jaas_file) diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/status_params.py new file mode 100644 index 00000000000..3046452c060 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/scripts/status_params.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +from resource_management.libraries.functions import get_kinit_path +from resource_management.libraries.functions.default import default +from resource_management.libraries.functions.format import format +from resource_management.libraries.script.script import Script + +config = Script.get_config() + +# logsearch pid file +logsearch_pid_dir = config['configurations']['logsearch-env']['logsearch_pid_dir'] +logsearch_pid_file = format("{logsearch_pid_dir}/logsearch.pid") + +# logfeeder pid file +logfeeder_pid_dir = config['configurations']['logfeeder-env']['logfeeder_pid_dir'] +logfeeder_pid_file = format("{logfeeder_pid_dir}/logfeeder.pid") + +security_enabled = config['configurations']['cluster-env']['security_enabled'] +kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/HadoopServiceConfig.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/HadoopServiceConfig.json.j2 new file mode 100644 index 00000000000..293a772cdc0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/HadoopServiceConfig.json.j2 @@ -0,0 +1,505 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "service": { + "accumulo": { + "label": "Accumulo", + "components": [ + { + "name": "accumulo_gc" + }, + { + "name": "accumulo_master" + }, + { + "name": "accumulo_monitor" + }, + { + "name": "accumulo_tracer" + }, + { + "name": "accumulo_tserver" + } + ], + "dependencies": [ + ] + }, + "atlas": { + "label": "Atlas", + "components": [ + { + "name": "atlas_app" + } + ], + "dependencies": [ + ] + }, + "ambari": { + "label": "Ambari", + "components": [ + { + "name": "ambari_agent" + }, + { + "name": "ambari_server" + }, + { + "name": "ambari_alerts" + }, + { + "name": "ambari_audit" + }, + { + "name": "ambari_config_changes" + }, + { + "name": "ambari_eclipselink" + }, + { + "name": "ambari_server_check_database" + } + ], + "dependencies": [ + ] + }, + "ams": { + "label": "AMS", + "components": [ + { + "name": "ams_hbase_master" + }, + { + "name": "ams_hbase_regionserver" + }, + { + "name": "ams_collector" + }, + { + "name": "ams_monitor" + }, + { + "name": "ams_grafana" + } + ], + "dependencies": [ + ] + }, + "falcon": { + "label": "Falcon", + "components": [ + { + "name": "falcon_app" + } + ], + "dependencies": [ + ] + }, + "flume": { + "label": "Flume", + "components": [ + { + "name": "flume_handler" + } + ], + "dependencies": [ + ] + }, + "hbase": { + "label": "HBase", + "components": [ + { + "name": "hbase_master" + }, + { + "name": "hbase_regionserver" + }, + { + "name": "hbase_phoenix_server" + } + ], + "dependencies": [ + { + "service": "hdfs", + "components": [ + "hdfs_namenode" + ] + } + ] + }, + "hdfs": { + "label": "HDFS", + "components": [ + { + "name": "hdfs_datanode" + }, + { + "name": "hdfs_namenode" + }, + { + "name": "hdfs_journalnode" + }, + { + "name": "hdfs_secondarynamenode" + }, + { + "name": "hdfs_zkfc" + }, + { + "name": "hdfs_nfs3" + }, + { + "name": "hdfs_audit", + "rowtype": "audit" + } + ], + "dependencies": [ + ] + }, + "hive": { + "label": "Hive", + "components": [ + { + "name": "hive_hiveserver2" + }, + { + "name": "hive_metastore" + }, + { + "name": "webhcat_server" + } + ], + "dependencies": [ + { + "service": "hdfs", + "components": [ + "hdfs_namenode" + ] + } + ] + }, + "infra" : { + "label" : "Infra", + "components": [ + { + "name": "infra_solr" + } + ], + "dependencies": [ + ] + }, + "kafka": { + "label": "Kafka", + "components": [ + { + "name": "kafka_controller" + }, + { + "name": "kafka_request" + }, + { + "name": "kafka_logcleaner" + }, + { + "name": "kafka_server" + }, + { + "name": "kafka_statechange" + } + ], + "dependencies": [ + { + "service": "zookeeper", + "components": [ + "zookeeper" + ] + } + ] + }, + "knox": { + "label": "Knox", + "components": [ + { + "name": "knox_gateway" + }, + { + "name": "knox_cli" + }, + { + "name": "knox_ldap" + } + ], + "dependencies": [ + ] + }, + "nifi": { + "label": "NiFi", + "components": [ + { + "name": "nifi_app" + }, + { + "name": "nifi_bootstrap" + }, + { + "name": "nifi_setup" + }, + { + "name": "nifi_user" + } + ], + "dependencies": [ + ] + }, + "mapred": { + "label": "MapReduce", + "components": [ + { + "name": "mapred_historyserver" + } + ], + "dependencies": [ + ] + }, + "logsearch": { + "label": "Logsearch", + "components": [ + { + "name": "logsearch_app" + }, + { + "name": "logsearch_feeder" + }, + { + "name": "logsearch_perf" + }, + { + "name": "logsearch_solr" + } + ], + "dependencies": [ + ] + }, + "ranger": { + "label": "Ranger", + "components": [ + { + "name": "ranger_admin" + }, + { + "name": "ranger_dbpatch" + }, + { + "name": "ranger_kms" + }, + { + "name": "ranger_usersync" + } + ], + "dependencies": [ + { + "service": "hdfs", + "required": "optional", + "components": [ + "hdfs_namenode" + ] + }, + { + "service": "hbase", + "required": "optional", + "components": [ + "hbase_master", + "hbase_regionserver" + ] + }, + { + "service": "hive", + "required": "optional", + "components": [ + "hive_hiveserver2" + ] + }, + { + "service": "kafka", + "required": "optional", + "components": [ + "kafka_ranger" + ] + }, + { + "service": "knox", + "required": "optional", + "components": [ + "knox_gateway" + ] + }, + { + "service": "storm", + "required": "optional", + "components": [ + "storm_supervisor" + ] + }, + { + "service": "yarn", + "required": "optional", + "components": [ + "yarn_resourcemanager" + ] + } + ] + }, + "oozie": { + "label": "Oozie", + "components": [ + { + "name": "oozie_app" + } + ], + "dependencies": [ + ] + }, + "hst": { + "label": "SmartSense", + "components": [ + { + "name": "hst_server" + }, + { + "name": "hst_agent" + }, + { + "name": "activity_analyzer" + }, + { + "name": "activity_explorer" + } + ], + "dependencies": [ + ] + }, + "spark": { + "label": "Spark", + "components": [ + { + "name": "spark_jobhistory_server" + }, + { + "name": "spark_thriftserver" + }, + { + "name": "livy_server" + } + ], + "dependencies": [ + ] + }, + "spark2": { + "label": "Spark 2", + "components": [ + { + "name": "spark2_jobhistory_server" + }, + { + "name": "spark2_thriftserver" + } + ], + "dependencies": [ + ] + }, + "storm": { + "label": "Storm", + "components": [ + { + "name": "storm_drpc" + }, + { + "name": "storm_logviewer" + }, + { + "name": "storm_nimbus" + }, + { + "name": "storm_supervisor" + }, + { + "name": "storm_ui" + }, + { + "name": "storm_worker" + } + ], + "dependencies": [ + ] + }, + "yarn": { + "label": "YARN", + "components": [ + { + "name": "yarn_nodemanager" + }, + { + "name": "yarn_resourcemanager" + }, + { + "name": "yarn_timelineserver" + }, + { + "name": "yarn_historyserver" + }, + { + "name": "yarn_jobsummary" + } + ], + "dependencies": [ + ] + }, + "zeppelin": { + "label": "Zeppelin", + "components": [ + { + "name": "zeppelin" + } + ], + "dependencies": [ + ] + }, + "zookeeper": { + "label": "ZooKeeper", + "components": [ + { + "name": "zookeeper" + } + ], + "dependencies": [ + ] + }, + "System": { + "label": "System", + "components": [ + { + "name": "system_message" + }, + { + "name": "secure_log" + } + ], + "dependencies": [ + ] + } + } +} diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/global.config.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/global.config.json.j2 new file mode 100644 index 00000000000..1b63e2aa6b9 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/global.config.json.j2 @@ -0,0 +1,27 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "global":{ + "add_fields":{ + "cluster":"{{cluster_name}}" + }, + "source":"file", + "tail":"true", + "gen_event_md5":"true" + } +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/input.config-logsearch.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/input.config-logsearch.json.j2 new file mode 100644 index 00000000000..dc014f81216 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/input.config-logsearch.json.j2 @@ -0,0 +1,55 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"logsearch_app", + "rowtype":"service", + "path":"{{default('/configurations/logsearch-env/logsearch_log_dir', '/var/log/ambari-logsearch-portal')}}/logsearch.json" + }, + { + "type":"logsearch_feeder", + "rowtype":"service", + "path":"{{default('/configurations/logfeeder-env/logfeeder_log_dir', '/var/log/ambari-logsearch-logfeeder')}}/logsearch-logfeeder.json" + }, + { + "type":"logsearch_perf", + "rowtype":"service", + "path":"{{default('/configurations/logsearch-env/logsearch_log_dir', '/var/log/ambari-logsearch-portal')}}/logsearch-performance.json" + } + ], + "filter":[ + { + "filter":"json", + "conditions":{ + "fields":{ + "type":[ + "logsearch_app", + "logsearch_feeder", + "logsearch_perf" + ] + + } + + } + + } + + ] + +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/logfeeder_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/logfeeder_jaas.conf.j2 new file mode 100644 index 00000000000..8c5974c9251 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/logfeeder_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + useTicketCache=false + keyTab="{{logfeeder_kerberos_keytab}}" + principal="{{logfeeder_kerberos_principal}}"; +}; \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/logsearch_jaas.conf.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/logsearch_jaas.conf.j2 new file mode 100644 index 00000000000..ff016e6c223 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/package/templates/logsearch_jaas.conf.j2 @@ -0,0 +1,26 @@ +{# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#} + +Client { + com.sun.security.auth.module.Krb5LoginModule required + useKeyTab=true + storeKey=true + useTicketCache=false + keyTab="{{logsearch_kerberos_keytab}}" + principal="{{logsearch_kerberos_principal}}"; +}; \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/audit_logs-solrconfig.xml.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/audit_logs-solrconfig.xml.j2 new file mode 100644 index 00000000000..f4b7d4a1372 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/audit_logs-solrconfig.xml.j2 @@ -0,0 +1,1889 @@ + + + + + + + + + 7.5.0 + + + + + + + + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + ${solr.hdfs.home:} + + ${solr.hdfs.confdir:} + + ${solr.hdfs.blockcache.enabled:true} + + ${solr.hdfs.blockcache.global:true} + + + + + + + + + + true + managed-schema + + + + + + + + + + 10000 + + + + + 50 + + + + + + + + + + + + + + + + {{logsearch_audit_logs_merge_factor}} + {{logsearch_audit_logs_merge_factor}} + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:5000} + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + static firstSearcher warming in solrconfig.xml + + + + + + true + + + 2 + + + + + + + + + + + + + + + + + + + + + + solr-data-config.xml + + + + + + + + explicit + 10 + text + + + + + + + + + + + + + + explicit + json + true + text + + + + + + + + true + json + true + + + + + + + + explicit + + + velocity + browse + layout + + + edismax + *:* + 10 + *,score + + + on + 1 + + + + + + + text + add-unknown-fields-to-the-schema + + + + + + + + + + + + + true + ignored_ + + + true + links + ignored_ + + + + + + + + + + + + + + + + + + + + + + + + solrpingquery + + + all + + + + + + + + + explicit + true + + + + + + + + + + + + + + + + key_lower_case + + + + + + default + text + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + wordbreak + solr.WordBreakSolrSpellChecker + name + true + true + 10 + + + + + + + + + + + + + + + + text + + default + wordbreak + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + mySuggester + FuzzyLookupFactory + DocumentDictionaryFactory + cat + price + string + + + + + + true + 10 + + + suggest + + + + + + + + + text + true + + + tvComponent + + + + + + + lingo + + + org.carrot2.clustering.lingo.LingoClusteringAlgorithm + + + clustering/carrot2 + + + + + stc + org.carrot2.clustering.stc.STCClusteringAlgorithm + + + + + kmeans + org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm + + + + + + + true + true + + name + + id + + features + + true + + + + false + + + edismax + + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + + *:* + 10 + *,score + + + clustering + + + + + + + + + + true + false + + + terms + + + + + + + + string + elevate.xml + + + + + + explicit + text + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\&quot;&apos;]{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? &#9;&#10;&#13; + + + + + + + WORD + + + en + US + + + + + + + + + + + + _ttl_ + +{{logsearch_audit_logs_max_retention}}DAYS + + + 30 + _ttl_ + _expire_at_ + + + _expire_at_ + + + + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSZ + yyyy-MM-dd'T'HH:mm:ss,SSSZ + yyyy-MM-dd'T'HH:mm:ss.SSS + yyyy-MM-dd'T'HH:mm:ss,SSS + yyyy-MM-dd'T'HH:mm:ssZ + yyyy-MM-dd'T'HH:mm:ss + yyyy-MM-dd'T'HH:mmZ + yyyy-MM-dd'T'HH:mm + yyyy-MM-dd HH:mm:ss.SSSZ + yyyy-MM-dd HH:mm:ss,SSSZ + yyyy-MM-dd HH:mm:ss.SSS + yyyy-MM-dd HH:mm:ss,SSS + yyyy-MM-dd HH:mm:ssZ + yyyy-MM-dd HH:mm:ss + yyyy-MM-dd HH:mmZ + yyyy-MM-dd HH:mm + yyyy-MM-dd + + + + key_lower_case + + java.lang.Boolean + boolean + + + java.util.Date + tdate + + + java.lang.Long + java.lang.Integer + tlong + + + java.lang.Number + tdouble + + + + + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + ${velocity.template.base.dir:} + + + + + 5 + + + + + + + + + + + + + + + + + + *:* + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/input.config-ambari.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/input.config-ambari.json.j2 new file mode 100644 index 00000000000..d231907adbc --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/input.config-ambari.json.j2 @@ -0,0 +1,858 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "input":[ + { + "type":"ambari_agent", + "rowtype":"service", + "path":"{{ambari_agent_log_dir}}/ambari-agent.log" + }, + { + "type":"ambari_server", + "rowtype":"service", + "path":"{{ambari_server_log_dir}}/ambari-server.log" + }, + { + "type":"ambari_alerts", + "rowtype":"service", + "add_fields":{ + "level":"UNKNOWN" + }, + "path":"{{ambari_server_log_dir}}/ambari-alerts.log" + }, + { + "type":"ambari_config_changes", + "rowtype":"service", + "path":"{{ambari_server_log_dir}}/ambari-config-changes.log" + }, + { + "type":"ambari_eclipselink", + "rowtype":"service", + "path":"{{ambari_server_log_dir}}/ambari-eclipselink.log" + }, + { + "type":"ambari_server_check_database", + "rowtype":"service", + "path":"{{ambari_server_log_dir}}/ambari-server-check-database.log" + }, + { + "type":"ambari_audit", + "rowtype":"audit", + "add_fields":{ + "logType":"AmbariAudit", + "enforcer":"ambari-acl", + "repoType":"1", + "repo":"ambari", + "level":"INFO" + }, + "path":"{{ambari_server_log_dir}}/ambari-audit.log" + } + + ], + "filter":[ + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_agent" + ] + + } + + }, + "log4j_format":"", + "multiline_pattern":"^(%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{LOGLEVEL:level} %{TIMESTAMP_ISO8601:logtime} %{JAVAFILE:file}:%{INT:line_number} - %{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + }, + "level":{ + "map_field_value":{ + "pre_value":"WARNING", + "post_value":"WARN" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_server" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_alerts" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_config_changes" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_eclipselink" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(\\[EL%{SPACE}%{DATA:level}\\])", + "message_pattern":"(?m)^\\[EL%{SPACE}%{DATA:level}\\]:%{SPACE}%{TIMESTAMP_ISO8601:logtime}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" + } + }, + "level":[ + { + "map_field_value":{ + "pre_value":"Severe", + "post_value":"ERROR" + } + }, + { + "map_field_value":{ + "pre_value":"Warning", + "post_value":"WARN" + } + }, + { + "map_field_value":{ + "pre_value":"Finer", + "post_value":"WARN" + } + }, + { + "map_field_value":{ + "pre_value":"Info", + "post_value":"INFO" + } + }, + { + "map_field_value":{ + "pre_value":"Config", + "post_value":"INFO" + } + }, + { + "map_field_value":{ + "pre_value":"Fine", + "post_value":"DEBUG" + } + }, + { + "map_field_value":{ + "pre_value":"Finest", + "post_value":"TRACE" + } + }, + { + "map_field_value":{ + "pre_value":"All", + "post_value":"TRACE" + } + } + ] + } + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_server_check_database" + ] + + } + + }, + "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" + } + + } + + } + + }, + { + "filter":"grok", + "conditions":{ + "fields":{ + "type":[ + "ambari_audit" + ] + + } + + }, + "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:evtTime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:evtTime},%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values":{ + "evtTime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd'T'HH:mm:ss.SSSXX" + } + + } + + } + + }, + { + "filter":"keyvalue", + "sort_order":1, + "conditions":{ + "fields":{ + "type":[ + "ambari_audit" + ] + + } + + }, + "source_field":"log_message", + "field_split":", ", + "value_borders":"()", + "post_map_values":{ + "User":{ + "map_field_value":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_field_name":{ + "new_field_name":"reqUser" + } + }, + "Hostname":{ + "map_field_name":{ + "new_field_name":"host" + } + }, + "Host name":{ + "map_field_name":{ + "new_field_name":"host" + } + }, + "RemoteIp":{ + "map_field_name":{ + "new_field_name":"cliIP" + } + }, + "RequestType":{ + "map_field_name":{ + "new_field_name":"cliType" + } + }, + "TaskId":{ + "map_field_name":{ + "new_field_name":"task_id" + } + }, + "Operation":{ + "map_field_name":{ + "new_field_name":"action" + } + }, + "Service":{ + "map_field_name":{ + "new_field_name":"ws_service" + } + }, + "url":{ + "map_field_name":{ + "new_field_name":"resource" + } + }, + "ResourcePath":{ + "map_field_name":{ + "new_field_name":"resource" + } + }, + "Cluster name":{ + "map_field_name":{ + "new_field_name":"cluster" + } + }, + "Old name":{ + "map_field_name":{ + "new_field_name":"ws_old_name" + } + }, + "New name":{ + "map_field_name":{ + "new_field_name":"ws_new_name" + } + }, + "Reason":{ + "map_field_name":{ + "new_field_name":"reason" + } + }, + "Base URL":{ + "map_field_name":{ + "new_field_name":"ws_base_url" + } + }, + "Base url":{ + "map_field_name":{ + "new_field_name":"ws_base_url" + } + }, + "Command":{ + "map_field_value":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_field_name":{ + "new_field_name":"ws_command" + } + }, + "Component":{ + "map_field_name":{ + "new_field_name":"ws_component" + } + }, + "Type":{ + "map_field_name":{ + "new_field_name":"ws_type" + } + }, + "Consecutive failures": { + "map_field_name":{ + "new_field_name":"ws_consecutive_failures" + } + }, + "Created Username": { + "map_field_name":{ + "new_field_name":"ws_username" + } + }, + "Affected username": { + "map_field_name":{ + "new_field_name":"ws_username" + } + }, + "Deleted Username": { + "map_field_name":{ + "new_field_name":"ws_username" + } + }, + "Alert group name": { + "map_field_name":{ + "new_field_name":"ws_alert_group_name" + } + }, + "Alert group ID": { + "map_field_name":{ + "new_field_name":"ws_alert_group_id" + } + }, + "Definition IDs": { + "map_field_name":{ + "new_field_name":"std_alert_definition_ids" + } + }, + "Notification ID": { + "map_field_name":{ + "new_field_name":"ws_alert_notification_id" + } + }, + "Notification IDs": { + "map_field_name":{ + "new_field_name":"std_alert_notification_ids" + } + }, + "Notification name": { + "map_field_name":{ + "new_field_name":"ws_alert_notification_name" + } + }, + "Notification type": { + "map_field_name":{ + "new_field_name":"ws_alert_notification_type" + } + }, + "Members": { + "map_field_name":{ + "new_field_name":"std_members" + } + }, + "Description": { + "map_field_name":{ + "new_field_name":"ws_description" + } + }, + "Email from": { + "map_field_name":{ + "new_field_name":"ws_alert_email_from" + } + }, + "Email to": { + "map_field_name":{ + "new_field_name":"ws_alert_email_to" + } + }, + "Group": { + "map_field_name":{ + "new_field_name":"ws_group" + } + }, + "Group IDs": { + "map_field_name":{ + "new_field_name":"std_alert_group_ids" + } + }, + "Alert states": { + "map_field_name":{ + "new_field_name":"std_alert_states" + } + }, + "Blueprint": { + "map_field_name":{ + "new_field_name":"ws_blueprint" + } + }, + "Blueprint name": { + "map_field_name":{ + "new_field_name":"ws_blueprint_name" + } + }, + "State": { + "map_field_name":{ + "new_field_name":"ws_state" + } + }, + "Principal": { + "map_field_name":{ + "new_field_name":"ws_principal" + } + }, + "Alias": { + "map_field_name":{ + "new_field_name":"ws_alias" + } + }, + "Keytab file": { + "map_field_name":{ + "new_field_name":"ws_keytab_file" + } + }, + "Upgrade type":{ + "map_field_name":{ + "new_field_name":"ws_upgrade_type" + } + }, + "Details":{ + "map_field_name":{ + "new_field_name":"ws_details" + } + }, + "Name":{ + "map_field_name":{ + "new_field_name":"ws_name" + } + }, + "Display name":{ + "map_field_value":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_field_name":{ + "new_field_name":"ws_display_name" + } + }, + "OS":{ + "map_field_name":{ + "new_field_name":"ws_os" + } + }, + "Repo id":{ + "map_field_name":{ + "new_field_name":"ws_repo_id" + } + }, + "Repo version":{ + "map_field_value":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_field_name":{ + "new_field_name":"ws_repo_version" + } + }, + "Repo version ID":{ + "map_field_name":{ + "new_field_name":"ws_repo_version_id" + } + }, + "Repositories":{ + "map_field_name":{ + "new_field_name":"ws_repositories" + } + }, + "RequestId":{ + "map_field_name":{ + "new_field_name":"ws_request_id" + } + }, + "Request id":{ + "map_field_name":{ + "new_field_name":"ws_request_id" + } + }, + "Repository ID":{ + "map_field_name":{ + "new_field_name":"ws_repo_id" + } + }, + "Repository name":{ + "map_field_name":{ + "new_field_name":"ws_repo_name" + } + }, + "Roles":{ + "map_field_name":{ + "new_field_name":"ws_roles" + } + }, + "Permissions":{ + "map_field_name":{ + "new_field_name":"std_permissions" + } + }, + "Stack":{ + "map_field_name":{ + "new_field_name":"ws_stack" + } + }, + "Stack version":{ + "map_field_name":{ + "new_field_name":"ws_stack_version" + } + }, + "Stage id":{ + "map_field_name":{ + "new_field_name":"ws_stage_id" + } + }, + "Administrator":{ + "map_field_value":{ + "pre_value":"yes", + "post_value":"1" + }, + "map_field_value":{ + "pre_value":"no", + "post_value":"0" + }, + "map_field_name":{ + "new_field_name":"ws_admin" + } + }, + "Active":{ + "map_field_value":{ + "pre_value":"y", + "post_value":"1" + }, + "map_field_value":{ + "pre_value":"n", + "post_value":"0" + }, + "map_field_name":{ + "new_field_name":"ws_active" + } + }, + "Version":{ + "map_field_name":{ + "new_field_name":"ws_version" + } + }, + "VersionNote":{ + "map_field_value":{ + "pre_value":"null", + "post_value":"unknown" + }, + "map_field_name":{ + "new_field_name":"ws_version_note" + } + }, + "VersionNumber":{ + "map_field_value":{ + "pre_value":"Vnull", + "post_value":"unknown" + }, + "map_field_name":{ + "new_field_name":"ws_version_number" + } + }, + "Status":[ + { + "map_field_copy":{ + "copy_name": "ws_status" + } + }, + { + "map_field_value":{ + "pre_value":"Success", + "post_value":"1" + } + }, + { + "map_field_value":{ + "pre_value":"Successfully queued", + "post_value":"1" + } + }, + { + "map_field_value":{ + "pre_value":"QUEUED", + "post_value":"1" + } + }, + { + "map_field_value":{ + "pre_value":"PENDING", + "post_value":"1" + } + }, + { + "map_field_value":{ + "pre_value":"COMPLETED", + "post_value":"1" + } + }, + { + "map_field_value":{ + "pre_value":"IN_PROGRESS", + "post_value":"1" + } + }, + { + "map_field_value":{ + "pre_value":"Failed", + "post_value":"0" + } + }, + { + "map_field_value":{ + "pre_value":"Failed to queue", + "post_value":"0" + } + }, + { + "map_field_value":{ + "pre_value":"HOLDING", + "post_value":"0" + } + }, + { + "map_field_value":{ + "pre_value":"HOLDING_FAILED", + "post_value":"0" + } + }, + { + "map_field_value":{ + "pre_value":"HOLDING_TIMEDOUT", + "post_value":"0" + } + }, + { + "map_field_value":{ + "pre_value":"FAILED", + "post_value":"0" + } + }, + { + "map_field_value":{ + "pre_value":"TIMEDOUT", + "post_value":"0" + } + }, + { + "map_field_value":{ + "pre_value":"ABORTED", + "post_value":"0" + } + }, + { + "map_field_value":{ + "pre_value":"SKIPPED_FAILED", + "post_value":"0" + } + }, + { + "map_field_name":{ + "new_field_name":"result" + } + } + ], + "ResultStatus":[ + { + "map_field_copy":{ + "copy_name": "ws_result_status" + } + }, + { + "map_field_value":{ + "pre_value":"200 OK", + "post_value":"1" + } + }, + { + "map_field_value":{ + "pre_value":"201 Created", + "post_value":"1" + } + }, + { + "map_field_value":{ + "pre_value":"202 Accepted", + "post_value":"1" + } + }, + { + "map_field_value":{ + "pre_value":"400 Bad Request", + "post_value":"0" + } + }, + { + "map_field_value":{ + "pre_value":"401 Unauthorized", + "post_value":"0" + } + }, + { + "map_field_value":{ + "pre_value":"403 Forbidden", + "post_value":"0" + } + }, + { + "map_field_value":{ + "pre_value":"404 Not Found", + "post_value":"0" + } + }, + { + "map_field_value":{ + "pre_value":"409 Resource Conflict", + "post_value":"0" + } + }, + { + "map_field_value":{ + "pre_value":"500 Internal Server Error", + "post_value":"0" + } + }, + { + "map_field_name":{ + "new_field_name":"result" + } + } + ] + + } + + } + + ] + +} diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/kafka-output.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/kafka-output.json.j2 new file mode 100644 index 00000000000..ceb1f94bdf9 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/kafka-output.json.j2 @@ -0,0 +1,34 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} + { + "output": [ + { + "is_enabled": "{{ logfeeder_kafka_output_enabled | lower }}", + "destination": "kafka", + "broker_list": "{{ logfeeder_kafka_broker_list }}", + "topic": "{{ logfeeder_kafka_topic }}", + "conditions": { + "fields": { + "rowtype": [ + "service" + ] + } + } + } + ] + } \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logfeeder-default_grok_patterns.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logfeeder-default_grok_patterns.j2 new file mode 100644 index 00000000000..78670061a15 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logfeeder-default_grok_patterns.j2 @@ -0,0 +1,148 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#Updated JAVACLASS to be same as JAVAFILE. Because if class doesn't have package, then it doesn't work. +JAVACLASS (?:[A-Za-z$0-9_. -]+) +#JAVACLASS (?:[a-zA-Z$_][a-zA-Z$_0-9]*\.)*[a-zA-Z$_][a-zA-Z$_0-9]* +#JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$]+ + +#Space is an allowed character to match special cases like 'Native Method' or 'Unknown Source' +JAVAFILE (?:[A-Za-z0-9_. -]+) +#Allow special or method +JAVAMETHOD (?:()|()|[a-zA-Z$_][a-zA-Z$_0-9]*) +#Line number is optional in special cases 'Native method' or 'Unknown source' +JAVASTACKTRACEPART %{SPACE}at %{JAVACLASS:class}\.%{JAVAMETHOD:method}\(%{JAVAFILE:file}(?::%{NUMBER:line})?\) +# Java Logs +JAVATHREAD (?:[A-Z]{2}-Processor[\d]+) + +JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\) +JAVALOGMESSAGE (.*) +# MMM dd, yyyy HH:mm:ss eg: Jan 9, 2014 7:13:13 AM +CATALINA_DATESTAMP %{MONTH} %{MONTHDAY}, 20%{YEAR} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) (?:AM|PM) +# yyyy-MM-dd HH:mm:ss,SSS ZZZ eg: 2014-01-09 17:32:25,527 -0800 +TOMCAT_DATESTAMP 20%{YEAR}-%{MONTHNUM}-%{MONTHDAY} %{HOUR}:?%{MINUTE}(?::?%{SECOND}) %{ISO8601_TIMEZONE} +CATALINALOG %{CATALINA_DATESTAMP:timestamp} %{JAVACLASS:class} %{JAVALOGMESSAGE:logmessage} +# 2014-01-09 20:03:28,269 -0800 | ERROR | com.example.service.ExampleService - something compeletely unexpected happened... +TOMCATLOG %{TOMCAT_DATESTAMP:timestamp} \| %{LOGLEVEL:level} \| %{JAVACLASS:class} - %{JAVALOGMESSAGE:logmessage} + +USERNAME [a-zA-Z0-9._-]+ +USER %{USERNAME} +EMAILLOCALPART [a-zA-Z][a-zA-Z0-9_.+-=:]+ +EMAILADDRESS %{EMAILLOCALPART}@%{HOSTNAME} +HTTPDUSER %{EMAILADDRESS}|%{USER} +INT (?:[+-]?(?:[0-9]+)) +BASE10NUM (?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) +NUMBER (?:%{BASE10NUM}) +BASE16NUM (?(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) +UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} + +# Networking +MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) +CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) +WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) +COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) +IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? +IPV4 (?[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ +URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? +URIHOST %{IPORHOST}(?::%{POSINT:port})? +# uripath comes loosely from RFC1738, but mostly from what Firefox +# doesn't turn into %XX +URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ +#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? +URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]* +URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? +URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? + +# Months: January, Feb, 3, 03, 12, December +MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May?|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b +MONTHNUM (?:0?[1-9]|1[0-2]) +MONTHNUM2 (?:0[1-9]|1[0-2]) +MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) + +# Days: Monday, Tue, Thu, etc... +DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) + +# Years? +YEAR (?>\d\d){1,2} +HOUR (?:2[0123]|[01]?[0-9]) +MINUTE (?:[0-5][0-9]) +# '60' is a leap second in most time standards and thus is valid. +SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) +TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) +# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) +DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} +DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} +DATE_EU2 %{YEAR}[./-]%{MONTHNUM}[./-]%{MONTHDAY} +ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) +ISO8601_SECOND (?:%{SECOND}|60) +TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? +DATE %{DATE_US}|%{DATE_EU}|%{DATE_EU2} +DATESTAMP %{DATE}[- ]%{TIME} +TZ (?:[PMCE][SD]T|UTC) +DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} +DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} +DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} +DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} +HTTPDERROR_DATE %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR} + +# Syslog Dates: Month Day HH:MM:SS +SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} +PROG [\x21-\x5a\x5c\x5e-\x7e]+ +SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])? +SYSLOGHOST %{IPORHOST} +SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}> +HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} + +# Shortcuts +QS %{QUOTEDSTRING} + +# Log formats +SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: +COMMONAPACHELOG %{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) +COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} +HTTPD20_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg} +HTTPD24_ERRORLOG \[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message} +HTTPD_ERRORLOG %{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG} + + +# Log Levels +LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) + + +# Log specific patterns +USER_SYNC_DATE %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} +SPARK_DATESTAMP %{YEAR}[/-]%{MONTHNUM2}[/-]%{MONTHDAY} %{TIME} +CUSTOM_DATESTAMP %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} +CUSTOM_SEPARATOR %{SPACE}\|%{SPACE} \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logfeeder-env.sh.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logfeeder-env.sh.j2 new file mode 100644 index 00000000000..b05c48c81ed --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logfeeder-env.sh.j2 @@ -0,0 +1,47 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +export LOGFEEDER_PATH={{logfeeder_dir}} + +export LOGFEEDER_CONF_DIR={{logsearch_logfeeder_conf}} + +#Logfile e.g. /var/log/logfeeder.log +export LOG_PATH={{logfeeder_log_dir}} +export LOG_FILE={{logfeeder_log}} + +#pid file e.g. /var/run/logfeeder.pid +export LOGFEEDER_PID_FILE={{logfeeder_pid_file}} + +export JAVA_HOME={{java64_home}} + +if [ "$LOGFEEDER_JAVA_MEM" = "" ]; then + export LOGFEEDER_JAVA_MEM=-Xmx{{logfeeder_max_mem}} +fi + +{% if logfeeder_use_ssl %} +export LOGFEEDER_SSL="true" +export LOGFEEDER_KEYSTORE_LOCATION={{logfeeder_keystore_location}} +export LOGFEEDER_KEYSTORE_TYPE={{logfeeder_keystore_type}} +export LOGFEEDER_TRUSTSTORE_LOCATION={{logfeeder_truststore_location}} +export LOGFEEDER_TRUSTSTORE_TYPE={{logfeeder_truststore_type}} +{% endif %} + +{% if security_enabled -%} +LOGFEEDER_KERBEROS_OPTS="-Djava.security.auth.login.config=/usr/lib/ambari-logsearch-logfeeder/conf/logfeeder_jaas.conf -Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder" +export LOGFEEDER_JAVA_OPTS="{{zk_security_opts}} $LOGFEEDER_KERBEROS_OPTS" +{% endif %} \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logfeeder-log4j.xml.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logfeeder-log4j.xml.j2 new file mode 100644 index 00000000000..0f535b24ece --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logfeeder-log4j.xml.j2 @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logsearch-admin.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logsearch-admin.json.j2 new file mode 100644 index 00000000000..abb0ddb447d --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logsearch-admin.json.j2 @@ -0,0 +1,25 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "users": [{ + "name": "Logsearch Admin", + "username": "{{logsearch_admin_username}}", + "password": "{{logsearch_admin_password}}", + "en_password": "" + }] +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logsearch-env.sh.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logsearch-env.sh.j2 new file mode 100644 index 00000000000..1aa628b6652 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logsearch-env.sh.j2 @@ -0,0 +1,51 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# path containing LogSearch.jar file +export LOGSEARCH_PATH={{logsearch_dir}} + +export LOGSEARCH_CONF_DIR={{logsearch_server_conf}} + +export LOG_PATH={{logsearch_log_dir}} +export LOG_FILE={{logsearch_log}} + +export LOGSEARCH_PID_FILE={{logsearch_pid_file}} + +export JAVA_HOME={{java64_home}} + +export LOGSEARCH_JAVA_MEM=-Xmx{{logsearch_app_max_memory}}m +if [ "$LOGSEARCH_JAVA_MEM" = "" ]; then + export LOGSEARCH_JAVA_MEM="-Xmx1g" +fi + +export LOGSEARCH_DEBUG={{logsearch_debug_enabled}} + +export LOGSEARCH_DEBUG_PORT={{logsearch_debug_port}} + +{% if logsearch_use_ssl %} +export LOGSEARCH_SSL="true" +export LOGSEARCH_KEYSTORE_LOCATION={{logsearch_keystore_location}} +export LOGSEARCH_KEYSTORE_TYPE={{logsearch_keystore_type}} +export LOGSEARCH_TRUSTSTORE_LOCATION={{logsearch_truststore_location}} +export LOGSEARCH_TRUSTSTORE_TYPE={{logsearch_truststore_type}} +{% endif %} + +{% if security_enabled -%} +LOGSEARCH_KERBEROS_OPTS="-Djava.security.auth.login.config=/usr/lib/ambari-logsearch-portal/conf/logsearch_jaas.conf -Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.Krb5HttpClientBuilder" +export LOGSEARCH_JAVA_OPTS="{{zk_security_opts}} $LOGSEARCH_KERBEROS_OPTS" +{% endif %} \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logsearch-log4j.xml.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logsearch-log4j.xml.j2 new file mode 100644 index 00000000000..98f58f29166 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/logsearch-log4j.xml.j2 @@ -0,0 +1,82 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/output.config.json.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/output.config.json.j2 new file mode 100644 index 00000000000..ffcfeaf6c5f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/output.config.json.j2 @@ -0,0 +1,59 @@ +{# + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + #} +{ + "output":[ + { + "comment":"Output to solr for service logs", + "is_enabled":"{{solr_service_logs_enable}}", + "destination":"solr", + "collection": "{{logsearch_solr_collection_service_logs}}", + "zk_connect_string":"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}", + "type": "service", + "conditions":{ + "fields":{ + "rowtype":[ + "service" + ] + + } + + } + + }, + { + "comment":"Output to solr for audit records", + "is_enabled":"{{solr_audit_logs_enable}}", + "destination":"solr", + "collection": "{{logsearch_solr_collection_audit_logs}}", + "zk_connect_string":"{{logsearch_solr_zk_quorum}}{{logsearch_solr_zk_znode}}", + "type": "audit", + "conditions":{ + "fields":{ + "rowtype":[ + "audit" + ] + + } + + } + + } + + ] + +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/secure_log.json b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/secure_log.json new file mode 100644 index 00000000000..9d9585ec7ea --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/secure_log.json @@ -0,0 +1,31 @@ +{ + "input": [ + { + "type": "secure_log", + "rowtype": "service", + "path": "/var/log/secure" + } + ], + "filter": [ + { + "filter": "grok", + "conditions": { + "fields": { + "type": [ + "secure_log" + ] + } + }, + "multiline_pattern": "^(%{SYSLOGTIMESTAMP:logtime})", + "message_pattern": "(?m)^%{SYSLOGTIMESTAMP:logtime}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values": { + "logtime": { + "map_date": { + "target_date_pattern": "yyyy-MM-dd HH:mm:ss,SSS", + "src_date_pattern" :"MMM dd HH:mm:ss" + } + } + } + } + ] +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/service_logs-solrconfig.xml.j2 b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/service_logs-solrconfig.xml.j2 new file mode 100644 index 00000000000..b631860741f --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/service_logs-solrconfig.xml.j2 @@ -0,0 +1,1889 @@ + + + + + + + + + 7.7.0 + + + + + + + + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + ${solr.hdfs.home:} + + ${solr.hdfs.confdir:} + + ${solr.hdfs.blockcache.enabled:true} + + ${solr.hdfs.blockcache.global:true} + + + + + + + + + + true + managed-schema + + + + + + + + + + 10000 + + + + + 50 + + + + + + + + + + + + + + + + {{logsearch_service_logs_merge_factor}} + {{logsearch_service_logs_merge_factor}} + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + true + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:5000} + + + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + static firstSearcher warming in solrconfig.xml + + + + + + true + + + 2 + + + + + + + + + + + + + + + + + + + + + + solr-data-config.xml + + + + + + + + explicit + 10 + text + + + + + + + + + + + + + + explicit + json + true + text + + + + + + + + true + json + true + + + + + + + + explicit + + + velocity + browse + layout + + + edismax + *:* + 10 + *,score + + + on + 1 + + + + + + + text + add-unknown-fields-to-the-schema + + + + + + + + + + + + + true + ignored_ + + + true + links + ignored_ + + + + + + + + + + + + + + + + + + + + + + + + solrpingquery + + + all + + + + + + + + + explicit + true + + + + + + + + + + + + + + + + lowercase + + + + + + default + text + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + wordbreak + solr.WordBreakSolrSpellChecker + name + true + true + 10 + + + + + + + + + + + + + + + + text + + default + wordbreak + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + mySuggester + FuzzyLookupFactory + DocumentDictionaryFactory + cat + price + string + + + + + + true + 10 + + + suggest + + + + + + + + + text + true + + + tvComponent + + + + + + + lingo + + + org.carrot2.clustering.lingo.LingoClusteringAlgorithm + + + clustering/carrot2 + + + + + stc + org.carrot2.clustering.stc.STCClusteringAlgorithm + + + + + kmeans + org.carrot2.clustering.kmeans.BisectingKMeansClusteringAlgorithm + + + + + + + true + true + + name + + id + + features + + true + + + + false + + + edismax + + text^0.5 features^1.0 name^1.2 sku^1.5 id^10.0 manu^1.1 cat^1.4 + + *:* + 10 + *,score + + + clustering + + + + + + + + + + true + false + + + terms + + + + + + + + string + elevate.xml + + + + + + explicit + text + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\&quot;&apos;]{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? &#9;&#10;&#13; + + + + + + + WORD + + + en + US + + + + + + + + + + + + _ttl_ + +{{logsearch_service_logs_max_retention}}DAYS + + + 86400 + _ttl_ + _expire_at_ + + + _expire_at_ + + + + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSZ + yyyy-MM-dd'T'HH:mm:ss,SSSZ + yyyy-MM-dd'T'HH:mm:ss.SSS + yyyy-MM-dd'T'HH:mm:ss,SSS + yyyy-MM-dd'T'HH:mm:ssZ + yyyy-MM-dd'T'HH:mm:ss + yyyy-MM-dd'T'HH:mmZ + yyyy-MM-dd'T'HH:mm + yyyy-MM-dd HH:mm:ss.SSSZ + yyyy-MM-dd HH:mm:ss,SSSZ + yyyy-MM-dd HH:mm:ss.SSS + yyyy-MM-dd HH:mm:ss,SSS + yyyy-MM-dd HH:mm:ssZ + yyyy-MM-dd HH:mm:ss + yyyy-MM-dd HH:mmZ + yyyy-MM-dd HH:mm + yyyy-MM-dd + + + + lowercase + + java.lang.Boolean + boolean + + + java.util.Date + tdate + + + java.lang.Long + java.lang.Integer + tlong + + + java.lang.Number + tdouble + + + + + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + ${velocity.template.base.dir:} + + + + + 5 + + + + + + + + + + + + + + + + + + *:* + + + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/system_messages.json b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/system_messages.json new file mode 100644 index 00000000000..e2ea3ebb3d0 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/properties/system_messages.json @@ -0,0 +1,31 @@ +{ + "input": [ + { + "type": "system_message", + "rowtype": "service", + "path": "/var/log/messages" + } + ], + "filter": [ + { + "filter": "grok", + "conditions": { + "fields": { + "type": [ + "system_message" + ] + } + }, + "multiline_pattern": "^(%{SYSLOGTIMESTAMP:logtime})", + "message_pattern": "(?m)^%{SYSLOGTIMESTAMP:logtime}%{SPACE}%{SYSLOGHOST:host}%{SPACE}%{GREEDYDATA:log_message}", + "post_map_values": { + "logtime": { + "map_date": { + "target_date_pattern": "yyyy-MM-dd HH:mm:ss,SSS", + "src_date_pattern" :"MMM dd HH:mm:ss" + } + } + } + } + ] +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/quicklinks/quicklinks.json b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/quicklinks/quicklinks.json new file mode 100644 index 00000000000..105796747dd --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/quicklinks/quicklinks.json @@ -0,0 +1,35 @@ +{ + "name": "default", + "description": "default quick links configuration", + "configuration": { + "protocol": { + "type":"https", + "checks":[ + { + "property":"logsearch.protocol", + "desired":"https", + "site":"logsearch-properties" + } + ] + }, + "links": [ + { + "name": "logsearch_ui", + "label": "Log Search UI", + "component_name": "LOGSEARCH_SERVER", + "url": "%@://%@:%@", + "requires_user_name": "false", + "attributes": ["authenticated", "sso"], + "port": { + "http_property": "logsearch.http.port", + "http_default_port": "8888", + "https_property": "logsearch.https.port", + "https_default_port": "8888", + "regex": "^(\\d+)$", + "site": "logsearch-properties" + } + } + ] + } +} + diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/role_command_order.json b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/role_command_order.json new file mode 100644 index 00000000000..4356bebbd23 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/role_command_order.json @@ -0,0 +1,9 @@ +{ + "general_deps" : { + "_comment" : "dependencies for AMBARI_LOGSEARCH", + "LOGSEARCH_SERVER-START": ["INFRA_SOLR-START", "ZOOKEEPER_SERVER-START"], + "LOGSEARCH_LOGFEEDER-START": ["INFRA_SOLR-START", "LOGSEARCH_SERVER-START"], + "AMBARI_LOGSEARCH_SERVICE_CHECK-SERVICE_CHECK": ["INFRA_SOLR-START", "LOGSEARCH_SERVER-START"], + "LOGSEARCH_SERVER-RESTART" : ["INFRA_SOLR-RESTART"] + } +} diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/service_advisor.py b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/service_advisor.py new file mode 100644 index 00000000000..97ff5cfaf37 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/service_advisor.py @@ -0,0 +1,176 @@ +#!/usr/bin/env ambari-python-wrap +""" +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +# Python imports +import imp +import os +import traceback + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/') +PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py') +if "BASE_SERVICE_ADVISOR" in os.environ: + PARENT_FILE = os.environ["BASE_SERVICE_ADVISOR"] + +try: + with open(PARENT_FILE, 'rb') as fp: + service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE)) +except Exception as e: + traceback.print_exc() + print "Failed to load parent" + +class AmbariLogSearchServiceAdvisor(service_advisor.ServiceAdvisor): + + def __init__(self, *args, **kwargs): + self.as_super = super(AmbariLogSearchServiceAdvisor, self) + self.as_super.__init__(*args, **kwargs) + + # Always call these methods + self.modifyMastersWithMultipleInstances() + self.modifyCardinalitiesDict() + self.modifyHeapSizeProperties() + self.modifyNotValuableComponents() + self.modifyComponentsNotPreferableOnServer() + self.modifyComponentLayoutSchemes() + + def modifyMastersWithMultipleInstances(self): + """ + Modify the set of masters with multiple instances. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyCardinalitiesDict(self): + """ + Modify the dictionary of cardinalities. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyHeapSizeProperties(self): + """ + Modify the dictionary of heap size properties. + Must be overriden in child class. + """ + pass + + def modifyNotValuableComponents(self): + """ + Modify the set of components whose host assignment is based on other services. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentsNotPreferableOnServer(self): + """ + Modify the set of components that are not preferable on the server. + Must be overriden in child class. + """ + # Nothing to do + pass + + def modifyComponentLayoutSchemes(self): + """ + Modify layout scheme dictionaries for components. + The scheme dictionary basically maps the number of hosts to + host index where component should exist. + Must be overriden in child class. + """ + # Nothing to do + pass + + def getServiceComponentLayoutValidations(self, services, hosts): + """ + Get a list of errors. + Must be overriden in child class. + """ + + return self.getServiceComponentCardinalityValidations(services, hosts, "AMBARI_LOGSEARCH") + + def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): + putLogSearchProperty = self.putProperty(configurations, "logsearch-properties", services) + putLogSearchAttribute = self.putPropertyAttribute(configurations, "logsearch-properties") + putLogSearchCommonEnvProperty = self.putProperty(configurations, "logsearch-common-env", services) + putLogSearchCommonEnvAttribute = self.putPropertyAttribute(configurations, "logsearch-common-env") + putLogSearchEnvAttribute = self.putPropertyAttribute(configurations, "logsearch-env") + putLogFeederEnvAttribute = self.putPropertyAttribute(configurations, "logfeeder-env") + + logSearchServerHosts = self.getComponentHostNames(services, "LOGSEARCH", "LOGSEARCH_SERVER") + # if there is no Log Search server on the cluster, i.e. there is an external server + if logSearchServerHosts is None or len(logSearchServerHosts) == 0: + # hide logsearch specific attributes + for key in services['configurations']['logsearch-env']['properties']: + putLogSearchEnvAttribute(key, 'visible', 'false') + for key in services['configurations']['logsearch-properties']['properties']: + putLogSearchAttribute(key, 'visible', 'false') + for key in services['configurations']['logsearch-audit_logs-solrconfig']['properties']: + self.putPropertyAttribute(configurations, "logsearch-audit_logs-solrconfig")(key, 'visible', 'false') + for key in services['configurations']['logsearch-service_logs-solrconfig']['properties']: + self.putPropertyAttribute(configurations, "logsearch-service_logs-solrconfig")(key, 'visible', 'false') + for key in services['configurations']['logsearch-log4j']['properties']: + self.putPropertyAttribute(configurations, "logsearch-log4j")(key, 'visible', 'false') + for key in services['configurations']['logsearch-admin-json']['properties']: + self.putPropertyAttribute(configurations, "logsearch-admin-json")(key, 'visible', 'false') + # if there is a Log Search server on the cluster + else: + infraSolrHosts = self.getComponentHostNames(services, "AMBARI_INFRA_SOLR", "INFRA_SOLR") + # if there is AMBARI_INFRA, calculate the min/max shards and recommendations based on the number of infra solr hosts + if infraSolrHosts is not None and len(infraSolrHosts) > 0 and "logsearch-properties" in services["configurations"]: + recommendedMinShards = len(infraSolrHosts) + recommendedShards = 2 * len(infraSolrHosts) + recommendedMaxShards = 10 * len(infraSolrHosts) + else: + # if there is no AMBARI_INFRA (i.e. external solr is used), use default values for min/max shards and recommendations + recommendedMinShards = 1 + recommendedShards = 1 + recommendedMaxShards = 100 + putLogSearchCommonEnvProperty('logsearch_use_external_solr', 'true') + + # recommend number of shard + putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'minimum', recommendedMinShards) + putLogSearchAttribute('logsearch.collection.service.logs.numshards', 'maximum', recommendedMaxShards) + putLogSearchProperty("logsearch.collection.service.logs.numshards", recommendedShards) + + putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'minimum', recommendedMinShards) + putLogSearchAttribute('logsearch.collection.audit.logs.numshards', 'maximum', recommendedMaxShards) + putLogSearchProperty("logsearch.collection.audit.logs.numshards", recommendedShards) + + kerberos_authentication_enabled = self.isSecurityEnabled(services) + # if there is no kerberos enabled hide kerberor related properties + if not kerberos_authentication_enabled: + putLogSearchCommonEnvProperty('logsearch_external_solr_kerberos_enabled', 'false') + putLogSearchCommonEnvAttribute('logsearch_external_solr_kerberos_enabled', 'visible', 'false') + putLogSearchEnvAttribute('logsearch_external_solr_kerberos_keytab', 'visible', 'false') + putLogSearchEnvAttribute('logsearch_external_solr_kerberos_principal', 'visible', 'false') + putLogFeederEnvAttribute('logfeeder_external_solr_kerberos_keytab', 'visible', 'false') + putLogFeederEnvAttribute('logfeeder_external_solr_kerberos_principal', 'visible', 'false') + + def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts): + """ + Entry point. + Validate configurations for the service. Return a list of errors. + The code for this function should be the same for each Service Advisor. + """ + #Logger.info("Class: %s, Method: %s. Validating Configurations." % + # (self.__class__.__name__, inspect.stack()[0][3])) + + return [] diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/themes/credentials.json b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/themes/credentials.json new file mode 100644 index 00000000000..e1b18cccf14 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/themes/credentials.json @@ -0,0 +1,42 @@ +{ + "name": "credentials", + "configuration": { + "placement": { + "configs": [ + { + "config": "logsearch-admin-json/logsearch_admin_password", + "subsection-name": "subsection-log-search" + }, + { + "config": "logsearch-admin-json/logsearch_admin_username", + "subsection-name": "subsection-log-search" + } + ], + "configuration-layout": "credentials" + }, + "widgets": [], + "layouts": [ + { + "name": "credentials", + "tabs": [ + { + "name": "credentials", + "layout": { + "sections": [ + { + "subsections": [ + { + "name": "subsection-log-search", + "display-name": "Log Search" + } + ], + "name": "credentials" + } + ] + } + } + ] + } + ] + } +} \ No newline at end of file diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/themes/directories.json b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/themes/directories.json new file mode 100644 index 00000000000..3c180ddbfba --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/themes/directories.json @@ -0,0 +1,137 @@ +{ + "name": "directories", + "description": "Directories theme for AMBARI_LOGSEARCH service", + "configuration": { + "layouts": [ + { + "name": "directories", + "tabs": [ + { + "name": "directories", + "display-name": "Directories", + "layout": { + "tab-columns": "1", + "tab-rows": "4", + "sections": [ + { + "name": "subsection-data-dirs", + "display-name": "DATA DIRS", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-data-dirs", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "subsection-log-dirs", + "display-name": "LOG DIRS", + "row-index": "1", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-log-dirs", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "subsection-pid-dirs", + "display-name": "PID DIRS", + "row-index": "2", + "column-index": "0", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "1", + "subsections": [ + { + "name": "subsection-pid-dirs", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + } + ] + } + ] + } + } + ] + } + ], + "placement": { + "configuration-layout": "default", + "configs": [ + { + "config": "logfeeder-properties/logfeeder.checkpoint.folder", + "subsection-name": "subsection-data-dirs" + }, + { + "config": "logsearch-env/logsearch_log_dir", + "subsection-name": "subsection-log-dirs" + }, + { + "config": "logfeeder-env/logfeeder_log_dir", + "subsection-name": "subsection-log-dirs" + }, + { + "config": "logsearch-env/logsearch_pid_dir", + "subsection-name": "subsection-pid-dirs" + }, + { + "config": "logfeeder-env/logfeeder_pid_dir", + "subsection-name": "subsection-pid-dirs" + } + ] + }, + "widgets": [ + { + "config": "logfeeder-properties/logfeeder.checkpoint.folder", + "widget": { + "type": "text-field" + } + }, + { + "config": "logsearch-env/logsearch_log_dir", + "widget": { + "type": "text-field" + } + }, + { + "config": "logfeeder-env/logfeeder_log_dir", + "widget": { + "type": "text-field" + } + }, + { + "config": "logsearch-env/logsearch_pid_dir", + "widget": { + "type": "text-field" + } + }, + { + "config": "logfeeder-env/logfeeder_pid_dir", + "widget": { + "type": "text-field" + } + } + ] + } +} diff --git a/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/themes/theme.json b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/themes/theme.json new file mode 100644 index 00000000000..6de89f121c8 --- /dev/null +++ b/ambari-server/src/main/resources/common-services/AMBARI_LOGSEARCH/3.0.0/themes/theme.json @@ -0,0 +1,480 @@ +{ + "name": "default", + "description": "Default theme for AMBARI_LOGSEARCH service", + "configuration": { + "layouts": [ + { + "name": "default", + "tabs": [ + { + "name": "settings", + "display-name": "Settings", + "layout": { + "tab-columns": "2", + "tab-rows": "2", + "sections": [ + { + "name": "section-logsearch-server", + "display-name": "Log Search Server", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1", + "section-columns": "1", + "section-rows": "5", + "subsections": [ + { + "name": "subsection-logsearch-server-col1", + "row-index": "0", + "column-index": "0", + "row-span": "1", + "column-span": "1" + }, + { + "name": "subsection-logsearch-server-col2", + "row-index": "0", + "column-index": "1", + "row-span": "1", + "column-span": "1" + } + ] + }, + { + "name": "section-logsearch-sso", + "display-name": "Single Sign-On", + "row-index": "1", + "column-index": "1", + "row-span": "3", + "column-span": "1", + "section-columns": "1", + "section-rows": "5", + "subsections": [ + { + "name": "subsection-logsearch-server-sso-col1", + "row-index": "1", + "column-index": "0", + "row-span": "3", + "column-span": "1" + } + ] + }, + { + "name": "section-logsearch-solr-connection", + "display-name": "Solr Connection", + "row-index": "4", + "column-index": "1", + "row-span": "3", + "column-span": "1", + "section-columns": "1", + "section-rows": "4", + "subsections": [ + { + "name": "subsection-logsearch-solr-connection-col1", + "row-index": "4", + "column-index": "0", + "row-span": "3", + "column-span": "1" + } + ] + } + ] + } + } + ] + } + ], + "placement": { + "configuration-layout": "default", + "configs": [ + { + "config" : "logfeeder-properties/logfeeder.include.default.level", + "subsection-name": "subsection-logsearch-server-col1" + }, + { + "config": "logsearch-env/logsearch_app_max_memory", + "subsection-name": "subsection-logsearch-server-col1" + }, + { + "config": "logsearch-properties/logsearch.collection.service.logs.numshards", + "subsection-name": "subsection-logsearch-server-col2" + }, + { + "config": "logsearch-properties/logsearch.collection.service.logs.replication.factor", + "subsection-name": "subsection-logsearch-server-col2" + }, + { + "config": "logsearch-properties/logsearch.collection.audit.logs.numshards", + "subsection-name": "subsection-logsearch-server-col2" + }, + { + "config": "logsearch-properties/logsearch.collection.audit.logs.replication.factor", + "subsection-name": "subsection-logsearch-server-col2" + }, + { + "config": "logsearch-properties/logsearch.auth.jwt.enabled", + "subsection-name": "subsection-logsearch-server-sso-col1" + }, + { + "config": "logsearch-properties/logsearch.auth.jwt.provider_url", + "subsection-name": "subsection-logsearch-server-sso-col1", + "depends-on": [ + { + "configs":[ + "logsearch-properties/logsearch.auth.jwt.enabled" + ], + "if": "${logsearch-properties/logsearch.auth.jwt.enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logsearch-properties/logsearch.auth.jwt.public_key", + "subsection-name": "subsection-logsearch-server-sso-col1", + "depends-on": [ + { + "configs":[ + "logsearch-properties/logsearch.auth.jwt.enabled" + ], + "if": "${logsearch-properties/logsearch.auth.jwt.enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logsearch-common-env/logsearch_use_external_solr", + "subsection-name": "subsection-logsearch-solr-connection-col1" + }, + { + "config": "logsearch-common-env/logsearch_external_solr_zk_znode", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-common-env/logsearch_use_external_solr" + ], + "if": "${logsearch-common-env/logsearch_use_external_solr}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logsearch-common-env/logsearch_external_solr_zk_quorum", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-common-env/logsearch_use_external_solr" + ], + "if": "${logsearch-common-env/logsearch_use_external_solr}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logsearch-common-env/logsearch_external_solr_ssl_enabled", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-common-env/logsearch_use_external_solr" + ], + "if": "${logsearch-common-env/logsearch_use_external_solr}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logsearch-common-env/logsearch_external_solr_kerberos_enabled", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-common-env/logsearch_use_external_solr" + ], + "if": "${logsearch-common-env/logsearch_use_external_solr}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logsearch-env/logsearch_external_solr_kerberos_keytab", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-common-env/logsearch_external_solr_kerberos_enabled" + ], + "if": "${logsearch-common-env/logsearch_external_solr_kerberos_enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logsearch-env/logsearch_external_solr_kerberos_principal", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-common-env/logsearch_external_solr_kerberos_enabled" + ], + "if": "${logsearch-common-env/logsearch_external_solr_kerberos_enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logfeeder-env/logfeeder_external_solr_kerberos_keytab", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-common-env/logsearch_external_solr_kerberos_enabled" + ], + "if": "${logsearch-common-env/logsearch_external_solr_kerberos_enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + }, + { + "config": "logfeeder-env/logfeeder_external_solr_kerberos_principal", + "subsection-name": "subsection-logsearch-solr-connection-col1", + "depends-on": [ + { + "configs":[ + "logsearch-common-env/logsearch_external_solr_kerberos_enabled" + ], + "if": "${logsearch-common-env/logsearch_external_solr_kerberos_enabled}", + "then": { + "property_value_attributes": { + "visible": true + } + }, + "else": { + "property_value_attributes": { + "visible": false + } + } + } + ] + } + ] + }, + "widgets": [ + { + "config": "logsearch-env/logsearch_app_max_memory", + "widget": { + "type": "slider", + "units": [ + { + "unit-name": "GB" + } + ] + } + }, + { + "config": "logfeeder-properties/logfeeder.include.default.level", + "widget": { + "type": "text-field" + } + }, + { + "config": "logsearch-properties/logsearch.collection.service.logs.numshards", + "widget": { + "type": "slider", + "units": [ + { + "unit-name": "int" + } + ] + } + }, + { + "config": "logsearch-properties/logsearch.collection.service.logs.replication.factor", + "widget": { + "type": "slider", + "units": [ + { + "unit-name": "int" + } + ] + } + }, + { + "config": "logsearch-properties/logsearch.collection.audit.logs.numshards", + "widget": { + "type": "slider", + "units": [ + { + "unit-name": "int" + } + ] + } + }, + { + "config": "logsearch-properties/logsearch.collection.audit.logs.replication.factor", + "widget": { + "type": "slider", + "units": [ + { + "unit-name": "int" + } + ] + } + }, + { + "config": "logsearch-properties/logsearch.auth.jwt.enabled", + "widget": { + "type": "toggle" + } + }, + { + "config": "logsearch-properties/logsearch.auth.jwt.provider_url", + "widget": { + "type": "text-field" + } + }, + { + "config": "logsearch-properties/logsearch.auth.jwt.public_key", + "widget": { + "type": "directories" + } + }, + { + "config": "logsearch-common-env/logsearch_use_external_solr", + "widget": { + "type": "toggle" + } + }, + { + "config": "logsearch-common-env/logsearch_external_solr_zk_znode", + "widget": { + "type": "text-field" + } + }, + { + "config": "logsearch-common-env/logsearch_external_solr_zk_quorum", + "widget": { + "type": "text-field" + } + }, + { + "config": "logsearch-common-env/logsearch_external_solr_ssl_enabled", + "widget": { + "type": "toggle" + } + }, + { + "config": "logsearch-common-env/logsearch_external_solr_kerberos_enabled", + "widget": { + "type": "toggle" + } + }, + { + "config": "logsearch-env/logsearch_external_solr_kerberos_keytab", + "widget": { + "type": "text-field" + } + }, + { + "config": "logsearch-env/logsearch_external_solr_kerberos_principal", + "widget": { + "type": "text-field" + } + }, + { + "config": "logfeeder-env/logfeeder_external_solr_kerberos_keytab", + "widget": { + "type": "text-field" + } + }, + { + "config": "logfeeder-env/logfeeder_external_solr_kerberos_principal", + "widget": { + "type": "text-field" + } + } + ] + } +} + From a998acb7a4ff02d07b6f117373e8f270e3add305 Mon Sep 17 00:00:00 2001 From: rzuo Date: Fri, 11 Aug 2023 16:51:24 +0800 Subject: [PATCH 2/2] add AMBARI_LOGSEARCH service --- .../services/AMBARI_LOGSEARCH/metainfo.xml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100755 ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/AMBARI_LOGSEARCH/metainfo.xml diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/AMBARI_LOGSEARCH/metainfo.xml b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/AMBARI_LOGSEARCH/metainfo.xml new file mode 100755 index 00000000000..3bdedd15df6 --- /dev/null +++ b/ambari-server/src/main/resources/stacks/BIGTOP/3.2.0/services/AMBARI_LOGSEARCH/metainfo.xml @@ -0,0 +1,26 @@ + + + + 2.0 + + + AMBARI_LOGSEARCH + common-services/AMBARI_LOGSEARCH/3.0.0 + + +