From 7d62a94729aef08f9a1d9015e15ff51a15770a89 Mon Sep 17 00:00:00 2001 From: Peter Adkins <74542596+hcpadkins@users.noreply.github.com> Date: Mon, 24 Jul 2023 13:40:17 +0100 Subject: [PATCH] Add Tines audit log connector. Add template for Tines audit logs. Also ensures stdout connector handles multiple entries properly. --- README.md | 1 + docs/index.rst | 1 + grove/__about__.py | 2 +- grove/connectors/tines/__init__.py | 4 + grove/connectors/tines/api.py | 117 ++++++++++++++++++ grove/connectors/tines/audit_logs.py | 85 +++++++++++++ grove/outputs/local_stdout.py | 31 ++--- pyproject.toml | 1 + templates/configuration/tines/audit_logs.json | 6 + tests/fixtures/tines/audit_logs/001.json | 27 ++++ tests/fixtures/tines/audit_logs/002.json | 62 ++++++++++ tests/fixtures/tines/audit_logs/003.json | 43 +++++++ tests/test_connectors_tines_audit_logs.py | 91 ++++++++++++++ 13 files changed, 455 insertions(+), 16 deletions(-) create mode 100644 grove/connectors/tines/__init__.py create mode 100644 grove/connectors/tines/api.py create mode 100644 grove/connectors/tines/audit_logs.py create mode 100644 templates/configuration/tines/audit_logs.json create mode 100644 tests/fixtures/tines/audit_logs/001.json create mode 100644 tests/fixtures/tines/audit_logs/002.json create mode 100644 tests/fixtures/tines/audit_logs/003.json create mode 100644 tests/test_connectors_tines_audit_logs.py diff --git a/README.md b/README.md index 031fcc7..a36e80e 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ isn't listed here, support can be added by creating a custom connector! * SalesForce Marketing Cloud audit event logs * SalesForce Marketing Cloud security event logs * Slack audit logs +* Tines audit logs * Terraform Cloud audit trails * Torq activity logs * Torq audit logs diff --git a/docs/index.rst b/docs/index.rst index a7391f7..0aedb1d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -60,6 +60,7 @@ isn't listed here, support can be added by creating a custom connector! * SalesForce Marketing Cloud audit event logs * SalesForce Marketing Cloud security event logs * Slack audit logs +* Tines audit logs * Terraform Cloud audit trails * Torq activity logs * Torq audit logs diff --git a/grove/__about__.py b/grove/__about__.py index 16b634a..d75c765 100644 --- a/grove/__about__.py +++ b/grove/__about__.py @@ -1,6 +1,6 @@ """Grove metadata.""" -__version__ = "1.0.0rc5" +__version__ = "1.0.0rc6" __title__ = "grove" __license__ = "Mozilla Public License 2.0" __copyright__ = "Copyright 2023 HashiCorp, Inc." diff --git a/grove/connectors/tines/__init__.py b/grove/connectors/tines/__init__.py new file mode 100644 index 0000000..8f80a7f --- /dev/null +++ b/grove/connectors/tines/__init__.py @@ -0,0 +1,4 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +"""Tines connectors for Grove.""" diff --git a/grove/connectors/tines/api.py b/grove/connectors/tines/api.py new file mode 100644 index 0000000..4227771 --- /dev/null +++ b/grove/connectors/tines/api.py @@ -0,0 +1,117 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +"""Tines Audit API client. + +This is a bare-bones client designed to interact with audit related APIs only. +""" + +import logging +import time +from typing import Dict, Optional + +import jmespath +import requests + +from grove.exceptions import RateLimitException, RequestFailedException +from grove.types import AuditLogEntries, HTTPResponse + +API_BASE_URI = "https://{identity}.{domain}/api/v1" +API_PAGE_SIZE = 500 + + +class Client: + def __init__( + self, + identity: Optional[str] = None, + domain: str = "tines.com", + token: Optional[str] = None, + retry: Optional[bool] = True, + ): + """Setup a new Tines API client. + + :param token: Tines API token. + :param identity: The name of the Tines tenant to collect logs from. + :param domain: The Tines domain to use when constructing API URLs. This is not + usually required if using Tines hosted tenants. + :param retry: Automatically retry if recoverable errors are encountered, such as + rate-limiting. + """ + self.retry = retry + self.logger = logging.getLogger(__name__) + self.headers = { + "content-type": "application/json", + "x-user-token": token, + } + self._api_base_uri = API_BASE_URI.format(identity=identity, domain=domain) + + def _get( + self, + url: str, + params: Optional[Dict[str, Optional[str]]] = None, + ) -> HTTPResponse: + """A GET wrapper to handle retries for the caller. + + :param url: URL to perform the HTTP GET against. + :param params: HTTP parameters to add to the request. + + :raises RateLimitException: A rate limit was encountered. + :raises RequestFailedException: An HTTP request failed. + + :return: HTTP Response object containing the headers and body of a response. + """ + while True: + try: + response = requests.get( + url, + headers=self.headers, # type: ignore + params=params, + ) + response.raise_for_status() + break + except requests.exceptions.RequestException as err: + # Retry on rate-limit, but only if requested. + if getattr(err.response, "status_code", None) == 429: + self.logger.warning("Rate-limit was exceeded during request") + if self.retry: + time.sleep(int(err.response.headers.get("Retry-After", "1"))) + continue + else: + raise RateLimitException(err) + + raise RequestFailedException(err) + + return HTTPResponse(headers=response.headers, body=response.json()) + + def list_audit_logs( + self, + after: Optional[str] = None, + operation_name: Optional[str] = None, + cursor: Optional[str] = None, + ) -> AuditLogEntries: + """Fetches a list of audit logs which match the provided filters. + + :param after: An RFC3339 timestamp, without milliseconds, to collect logs after. + :param operation_name: An optional operation to collect logs for. + :param cursor: Cursor to use when fetching events (pagination). This is the + page number in the context of Tines. + + :return: AuditLogEntries object containing a pagination cursor, and log entries. + """ + # See psf/requests issue #2651 for why we can happily pass in None values and + # not have the request key added to the URI. + result = self._get( + f"{self._api_base_uri}/audit_logs", + params={ + "page": cursor, + "after": after, + "operation_name": operation_name, + "per_page": str(API_PAGE_SIZE), + }, + ) + + # Return the cursor and the results to allow the caller to page as required. + return AuditLogEntries( + cursor=jmespath.search("meta.next_page", result.body), + entries=jmespath.search("audit_logs", result.body), + ) diff --git a/grove/connectors/tines/audit_logs.py b/grove/connectors/tines/audit_logs.py new file mode 100644 index 0000000..d253ce6 --- /dev/null +++ b/grove/connectors/tines/audit_logs.py @@ -0,0 +1,85 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +"""Tines Audit connector for Grove.""" + +from datetime import datetime, timedelta + +from grove.connectors import BaseConnector +from grove.connectors.tines.api import Client +from grove.constants import OPERATION_DEFAULT, REVERSE_CHRONOLOGICAL +from grove.exceptions import NotFoundException + + +class Connector(BaseConnector): + NAME = "tines_audit_logs" + POINTER_PATH = "created_at" + LOG_ORDER = REVERSE_CHRONOLOGICAL + + @property + def domain(self): + """Fetches the Tines domain suffix from the configuration. + + This field is used to allow configuration of collection of log data from + specific non 'tines.com' instances. Usually, this will not need to be changed, + as the configured identity (tenant name) will be appended to this domain to form + the full FQDN. + + If the required tenant is under 'tines.com', only the usual identity field need + be set. + + :return: The "domain" portion of the connector's configuration. + """ + try: + return self.configuration.domain + except AttributeError: + return "tines.com" + + def collect(self): + """Collects all logs from the Tines Audit API. + + This will first check whether there are any pointers cached to indicate previous + collections. If not, the last week of data will be collected. + """ + client = Client( + token=self.key, + domain=self.domain, + identity=self.identity, + ) + cursor = None + + # If no pointer is stored then a previous run hasn't been performed, so set the + # pointer to a week ago. The Tines API returns timestamps as RFC3339, and + # without milliseconds. + now = datetime.utcnow() + + try: + _ = self.pointer + except NotFoundException: + self.pointer = (now - timedelta(days=7)).isoformat( + sep="T", + timespec="seconds", + ) + "Z" + + # Set the operation name to collect to 'None' if none is specified - as the + # Grove default is 'all'. + operation = None + + if self.operation != OPERATION_DEFAULT: + operation = self.operation + + # Page over data using the cursor, saving returned data page by page. + while True: + log = client.list_audit_logs( + after=self.pointer, + cursor=cursor, + operation_name=operation, + ) + + # Save this batch of log entries. + self.save(log.entries) + + # Check if we need to continue paging. + cursor = log.cursor + if cursor is None: + break diff --git a/grove/outputs/local_stdout.py b/grove/outputs/local_stdout.py index 49142ba..b05b9f6 100644 --- a/grove/outputs/local_stdout.py +++ b/grove/outputs/local_stdout.py @@ -39,21 +39,22 @@ def submit( """ datestamp = datetime.datetime.utcnow() - print( - json.dumps( - { - "part": part, - "kind": kind, - "descriptor": descriptor, - "connector": connector, - "identity": identity, - "operation": operation, - "datestamp": datestamp.strftime(DATESTAMP_FORMAT), - "message": json.loads(data.decode("utf-8")), - } - ), - flush=True, - ) + for entry in data.decode("utf-8").splitlines(): + print( + json.dumps( + { + "part": part, + "kind": kind, + "descriptor": descriptor, + "connector": connector, + "identity": identity, + "operation": operation, + "datestamp": datestamp.strftime(DATESTAMP_FORMAT), + "message": json.loads(entry), + } + ), + flush=True, + ) def serialize(self, data: List[Any], metadata: Dict[str, Any] = {}) -> bytes: """Serialize data to a standard format (NDJSON). diff --git a/pyproject.toml b/pyproject.toml index 39e11fc..fc3c04d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,7 @@ sf_event_log = "grove.connectors.sf.event_log:Connector" sfmc_audit_events = "grove.connectors.sfmc.audit_events:Connector" sfmc_security_events = "grove.connectors.sfmc.security_events:Connector" slack_audit_logs = "grove.connectors.slack.audit_logs:Connector" +tines_audit_logs = "grove.connectors.tines.audit_logs:Connector" tfc_audit_trails = "grove.connectors.tfc.audit_trails:Connector" torq_activity_logs = "grove.connectors.torq.activity_logs:Connector" torq_audit_logs = "grove.connectors.torq.audit_logs:Connector" diff --git a/templates/configuration/tines/audit_logs.json b/templates/configuration/tines/audit_logs.json new file mode 100644 index 0000000..9e07da3 --- /dev/null +++ b/templates/configuration/tines/audit_logs.json @@ -0,0 +1,6 @@ +{ + "key": "API_TOKEN_HERE", + "identity": "TENANT_ID_HERE", + "name": "tines-audit-logs-example", + "connector": "tines_audit_logs" +} \ No newline at end of file diff --git a/tests/fixtures/tines/audit_logs/001.json b/tests/fixtures/tines/audit_logs/001.json new file mode 100644 index 0000000..f79ce39 --- /dev/null +++ b/tests/fixtures/tines/audit_logs/001.json @@ -0,0 +1,27 @@ +{ + "audit_logs": [ + { + "created_at": "2023-07-21T14:21:30Z", + "operation_name": "UserCredentialDestruction", + "id": 1488526, + "inputs": { + "credentialId": 3013 + }, + "request_ip": "192.0.2.1", + "request_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0", + "tenant_id": 5044, + "updated_at": "2023-07-21T14:21:30Z", + "user_email": "user@example.org", + "user_id": 7642, + "user_name": "Example User" + } + ], + "meta": { + "current_page": "https://my-tenant.tines.com/api/v1/audit_logs?per_page=1&page=1", + "previous_page": null, + "next_page": "https://my-tenant.tines.com/api/v1/audit_logs?per_page=1&page=2", + "per_page": 1, + "pages": 2, + "count": 2 + } + } \ No newline at end of file diff --git a/tests/fixtures/tines/audit_logs/002.json b/tests/fixtures/tines/audit_logs/002.json new file mode 100644 index 0000000..c1afeac --- /dev/null +++ b/tests/fixtures/tines/audit_logs/002.json @@ -0,0 +1,62 @@ +{ + "audit_logs": [ + { + "created_at": "2023-07-21T14:00:30Z", + "operation_name": "UserCredentialSave", + "id": 1488227, + "inputs": { + "inputs": { + "name": "Example", + "mode": "TEXT", + "value": "REDACTED", + "description": "", + "jwtPayload": "REDACTED", + "jwtPrivateKey": "REDACTED", + "jwtAutoGenerateTimeClaims": "REDACTED", + "jwtAlgorithm": "REDACTED", + "jwtHs256Secret": "REDACTED", + "oauthGrantType": "REDACTED", + "oauthClientId": "REDACTED", + "oauthScope": "REDACTED", + "oauthClientSecret": "REDACTED", + "oauthUrl": "REDACTED", + "awsAuthenticationType": "REDACTED", + "oauthTokenUrl": "REDACTED", + "oauthPkceCodeChallengeMethod": "REDACTED", + "awsAccessKey": "REDACTED", + "awsSecretKey": "REDACTED", + "awsAssumedRoleArn": "REDACTED", + "awsAssumedRoleExternalId": "REDACTED", + "httpRequestLocationOfToken": "REDACTED", + "httpRequestTtl": "REDACTED", + "httpRequestOptions": "REDACTED", + "mtlsClientCertificate": "REDACTED", + "mtlsClientPrivateKey": "REDACTED", + "mtlsRootCertificate": "REDACTED", + "readAccess": "TEAM", + "allowedHosts": "REDACTED", + "icon": "REDACTED", + "emoji": "REDACTED", + "preserveNewlineCharacters": "REDACTED", + "teamId": 12723, + "folderId": 332 + } + }, + "request_ip": "192.0.2.1", + "request_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0", + "tenant_id": 5044, + "updated_at": "2023-07-21T14:00:30Z", + "user_email": "user@example.org", + "user_id": 7642, + "user_name": "Example User" + } + ], + "meta": { + "current_page": "https://my-tenant.tines.com/api/v1/audit_logs?per_page=1&page=2", + "previous_page": "https://my-tenant.tines.com/api/v1/audit_logs?per_page=1&page=1", + "next_page": null, + "per_page": 1, + "pages": 2, + "count": 2 + } + } \ No newline at end of file diff --git a/tests/fixtures/tines/audit_logs/003.json b/tests/fixtures/tines/audit_logs/003.json new file mode 100644 index 0000000..c317d47 --- /dev/null +++ b/tests/fixtures/tines/audit_logs/003.json @@ -0,0 +1,43 @@ +{ + "audit_logs": [ + { + "created_at": "2023-07-21T10:32:37Z", + "operation_name": "AuthenticationTokenCreation", + "id": 1484335, + "inputs": { + "inputs": { + "name": "Grove", + "isServiceToken": false + } + }, + "request_ip": "192.0.2.1", + "request_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0", + "tenant_id": 5044, + "updated_at": "2023-07-21T10:32:37Z", + "user_email": "user@example.org", + "user_id": 7642, + "user_name": "Peter Adkins" + }, + { + "created_at": "2023-07-21T10:31:57Z", + "operation_name": "Login", + "id": 1484317, + "inputs": null, + "request_ip": "192.0.2.1", + "request_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0", + "tenant_id": 5044, + "updated_at": "2023-07-21T10:31:57Z", + "user_email": "user@example.org", + "user_id": 7642, + "user_name": " " + } + ], + "meta": { + "current_page": "https://my-tenant.tines.com/api/v1/audit_logs?per_page=2&page=4", + "previous_page": null, + "next_page": null, + "per_page": 2, + "pages": 1, + "count": 2 + } + } \ No newline at end of file diff --git a/tests/test_connectors_tines_audit_logs.py b/tests/test_connectors_tines_audit_logs.py new file mode 100644 index 0000000..6960673 --- /dev/null +++ b/tests/test_connectors_tines_audit_logs.py @@ -0,0 +1,91 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: MPL-2.0 + +"""Implements integration tests for the Tines Audit Log collector.""" + +import os +import re +import unittest +from unittest.mock import patch + +import responses + +from grove.connectors.tines.audit_logs import Connector +from grove.models import ConnectorConfig +from tests import mocks + + +class TinesAuditTestCase(unittest.TestCase): + """Implements integration tests for the Tines Audit collector.""" + + @patch("grove.helpers.plugin.load_handler", mocks.load_handler) + def setUp(self): + """Ensure the application is setup for testing.""" + self.dir = os.path.dirname(os.path.abspath(__file__)) + self.connector = Connector( + config=ConnectorConfig( + identity="1FEEDFEED1", + key="token", + name="test", + connector="test", + ), + context={ + "runtime": "test_harness", + "runtime_id": "NA", + }, + ) + + @responses.activate + def test_collect_pagination(self): + """Ensure pagination is working as expected.""" + # Succeed with a cursor returned (to indicate paging is required). + responses.add( + responses.GET, + re.compile(r"https://.*"), + status=200, + content_type="application/json", + body=bytes( + open( + os.path.join(self.dir, "fixtures/tines/audit_logs/001.json"), "r" + ).read(), + "utf-8", + ), + ) + + # The last "page" returns an empty cursor. + responses.add( + responses.GET, + re.compile(r"https://.*"), + status=200, + content_type="application/json", + body=bytes( + open( + os.path.join(self.dir, "fixtures/tines/audit_logs/002.json"), "r" + ).read(), + "utf-8", + ), + ) + + self.connector.run() + self.assertEqual(self.connector._saved["logs"], 2) + self.assertEqual(self.connector.pointer, "2023-07-21T14:21:30Z") + + @responses.activate + def test_collect_no_pagination(self): + """Ensure collection without pagination is working as expected.""" + responses.add( + responses.GET, + re.compile(r"https://.*"), + status=200, + content_type="application/json", + body=bytes( + open( + os.path.join(self.dir, "fixtures/tines/audit_logs/003.json"), "r" + ).read(), + "utf-8", + ), + ) + + self.connector.run() + self.assertEqual(self.connector._saved["logs"], 2) + self.assertEqual(self.connector.pointer, "2023-07-21T10:32:37Z")