Add Tines audit log connector.

README.md

@@ -49,6 +49,7 @@ isn't listed here, support can be added by creating a custom connector!
 * SalesForce Marketing Cloud audit event logs
 * SalesForce Marketing Cloud security event logs
 * Slack audit logs
+* Tines audit logs
 * Terraform Cloud audit trails
 * Torq activity logs
 * Torq audit logs

docs/index.rst

@@ -60,6 +60,7 @@ isn't listed here, support can be added by creating a custom connector!
 * SalesForce Marketing Cloud audit event logs
 * SalesForce Marketing Cloud security event logs
 * Slack audit logs
+* Tines audit logs
 * Terraform Cloud audit trails
 * Torq activity logs
 * Torq audit logs

grove/__about__.py

@@ -1,6 +1,6 @@
 """Grove metadata."""
 
-__version__ = "1.0.0rc5"
+__version__ = "1.0.0rc6"
 __title__ = "grove"
 __license__ = "Mozilla Public License 2.0"
 __copyright__ = "Copyright 2023 HashiCorp, Inc."

grove/connectors/tines/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+"""Tines connectors for Grove."""

grove/connectors/tines/api.py

@@ -0,0 +1,117 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+"""Tines Audit API client.
+
+This is a bare-bones client designed to interact with audit related APIs only.
+"""
+
+import logging
+import time
+from typing import Dict, Optional
+
+import jmespath
+import requests
+
+from grove.exceptions import RateLimitException, RequestFailedException
+from grove.types import AuditLogEntries, HTTPResponse
+
+API_BASE_URI = "https://{identity}.{domain}/api/v1"
+API_PAGE_SIZE = 500
+
+
+class Client:
+    def __init__(
+        self,
+        identity: Optional[str] = None,
+        domain: str = "tines.com",
+        token: Optional[str] = None,
+        retry: Optional[bool] = True,
+    ):
+        """Setup a new Tines API client.
+
+        :param token: Tines API token.
+        :param identity: The name of the Tines tenant to collect logs from.
+        :param domain: The Tines domain to use when constructing API URLs. This is not
+            usually required if using Tines hosted tenants.
+        :param retry: Automatically retry if recoverable errors are encountered, such as
+            rate-limiting.
+        """
+        self.retry = retry
+        self.logger = logging.getLogger(__name__)
+        self.headers = {
+            "content-type": "application/json",
+            "x-user-token": token,
+        }
+        self._api_base_uri = API_BASE_URI.format(identity=identity, domain=domain)
+
+    def _get(
+        self,
+        url: str,
+        params: Optional[Dict[str, Optional[str]]] = None,
+    ) -> HTTPResponse:
+        """A GET wrapper to handle retries for the caller.
+
+        :param url: URL to perform the HTTP GET against.
+        :param params: HTTP parameters to add to the request.
+
+        :raises RateLimitException: A rate limit was encountered.
+        :raises RequestFailedException: An HTTP request failed.
+
+        :return: HTTP Response object containing the headers and body of a response.
+        """
+        while True:
+            try:
+                response = requests.get(
+                    url,
+                    headers=self.headers,  # type: ignore
+                    params=params,
+                )
+                response.raise_for_status()
+                break
+            except requests.exceptions.RequestException as err:
+                # Retry on rate-limit, but only if requested.
+                if getattr(err.response, "status_code", None) == 429:
+                    self.logger.warning("Rate-limit was exceeded during request")
+                    if self.retry:
+                        time.sleep(int(err.response.headers.get("Retry-After", "1")))
+                        continue
+                    else:
+                        raise RateLimitException(err)
+
+                raise RequestFailedException(err)
+
+        return HTTPResponse(headers=response.headers, body=response.json())
+
+    def list_audit_logs(
+        self,
+        after: Optional[str] = None,
+        operation_name: Optional[str] = None,
+        cursor: Optional[str] = None,
+    ) -> AuditLogEntries:
+        """Fetches a list of audit logs which match the provided filters.
+
+        :param after: An RFC3339 timestamp, without milliseconds, to collect logs after.
+        :param operation_name: An optional operation to collect logs for.
+        :param cursor: Cursor to use when fetching events (pagination). This is the
+            page number in the context of Tines.
+
+        :return: AuditLogEntries object containing a pagination cursor, and log entries.
+        """
+        # See psf/requests issue #2651 for why we can happily pass in None values and
+        # not have the request key added to the URI.
+        result = self._get(
+            f"{self._api_base_uri}/audit_logs",
+            params={
+                "page": cursor,
+                "after": after,
+                "operation_name": operation_name,
+                "per_page": str(API_PAGE_SIZE),
+            },
+        )
+
+        # Return the cursor and the results to allow the caller to page as required.
+        return AuditLogEntries(
+            cursor=jmespath.search("meta.next_page", result.body),
+            entries=jmespath.search("audit_logs", result.body),
+        )

grove/connectors/tines/audit_logs.py

@@ -0,0 +1,85 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: MPL-2.0
+
+"""Tines Audit connector for Grove."""
+
+from datetime import datetime, timedelta
+
+from grove.connectors import BaseConnector
+from grove.connectors.tines.api import Client
+from grove.constants import OPERATION_DEFAULT, REVERSE_CHRONOLOGICAL
+from grove.exceptions import NotFoundException
+
+
+class Connector(BaseConnector):
+    NAME = "tines_audit_logs"
+    POINTER_PATH = "created_at"
+    LOG_ORDER = REVERSE_CHRONOLOGICAL
+
+    @property
+    def domain(self):
+        """Fetches the Tines domain suffix from the configuration.
+
+        This field is used to allow configuration of collection of log data from
+        specific non 'tines.com' instances. Usually, this will not need to be changed,
+        as the configured identity (tenant name) will be appended to this domain to form
+        the full FQDN.
+
+        If the required tenant is under 'tines.com', only the usual identity field need
+        be set.
+
+        :return: The "domain" portion of the connector's configuration.
+        """
+        try:
+            return self.configuration.domain
+        except AttributeError:
+            return "tines.com"
+
+    def collect(self):
+        """Collects all logs from the Tines Audit API.
+
+        This will first check whether there are any pointers cached to indicate previous
+        collections. If not, the last week of data will be collected.
+        """
+        client = Client(
+            token=self.key,
+            domain=self.domain,
+            identity=self.identity,
+        )
+        cursor = None
+
+        # If no pointer is stored then a previous run hasn't been performed, so set the
+        # pointer to a week ago. The Tines API returns timestamps as RFC3339, and
+        # without milliseconds.
+        now = datetime.utcnow()
+
+        try:
+            _ = self.pointer
+        except NotFoundException:
+            self.pointer = (now - timedelta(days=7)).isoformat(
+                sep="T",
+                timespec="seconds",
+            ) + "Z"
+
+        # Set the operation name to collect to 'None' if none is specified - as the
+        # Grove default is 'all'.
+        operation = None
+
+        if self.operation != OPERATION_DEFAULT:
+            operation = self.operation
+
+        # Page over data using the cursor, saving returned data page by page.
+        while True:
+            log = client.list_audit_logs(
+                after=self.pointer,
+                cursor=cursor,
+                operation_name=operation,
+            )
+
+            # Save this batch of log entries.
+            self.save(log.entries)
+
+            # Check if we need to continue paging.
+            cursor = log.cursor
+            if cursor is None:
+                break

grove/outputs/local_stdout.py

@@ -39,21 +39,22 @@ def submit(
         """
         datestamp = datetime.datetime.utcnow()
 
-        print(
-            json.dumps(
-                {
-                    "part": part,
-                    "kind": kind,
-                    "descriptor": descriptor,
-                    "connector": connector,
-                    "identity": identity,
-                    "operation": operation,
-                    "datestamp": datestamp.strftime(DATESTAMP_FORMAT),
-                    "message": json.loads(data.decode("utf-8")),
-                }
-            ),
-            flush=True,
-        )
+        for entry in data.decode("utf-8").splitlines():
+            print(
+                json.dumps(
+                    {
+                        "part": part,
+                        "kind": kind,
+                        "descriptor": descriptor,
+                        "connector": connector,
+                        "identity": identity,
+                        "operation": operation,
+                        "datestamp": datestamp.strftime(DATESTAMP_FORMAT),
+                        "message": json.loads(entry),
+                    }
+                ),
+                flush=True,
+            )
 
     def serialize(self, data: List[Any], metadata: Dict[str, Any] = {}) -> bytes:
         """Serialize data to a standard format (NDJSON).

pyproject.toml

@@ -74,6 +74,7 @@ sf_event_log = "grove.connectors.sf.event_log:Connector"
 sfmc_audit_events = "grove.connectors.sfmc.audit_events:Connector"
 sfmc_security_events = "grove.connectors.sfmc.security_events:Connector"
 slack_audit_logs = "grove.connectors.slack.audit_logs:Connector"
+tines_audit_logs = "grove.connectors.tines.audit_logs:Connector"
 tfc_audit_trails = "grove.connectors.tfc.audit_trails:Connector"
 torq_activity_logs = "grove.connectors.torq.activity_logs:Connector"
 torq_audit_logs = "grove.connectors.torq.audit_logs:Connector"

templates/configuration/tines/audit_logs.json

@@ -0,0 +1,6 @@
+{
+    "key": "API_TOKEN_HERE",
+    "identity": "TENANT_ID_HERE",
+    "name": "tines-audit-logs-example",
+    "connector": "tines_audit_logs"
+}

tests/fixtures/tines/audit_logs/001.json

@@ -0,0 +1,27 @@
+{
+    "audit_logs": [
+      {
+        "created_at": "2023-07-21T14:21:30Z",
+        "operation_name": "UserCredentialDestruction",
+        "id": 1488526,
+        "inputs": {
+          "credentialId": 3013
+        },
+        "request_ip": "192.0.2.1",
+        "request_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0",
+        "tenant_id": 5044,
+        "updated_at": "2023-07-21T14:21:30Z",
+        "user_email": "user@example.org",
+        "user_id": 7642,
+        "user_name": "Example User"
+      }
+    ],
+    "meta": {
+      "current_page": "https://my-tenant.tines.com/api/v1/audit_logs?per_page=1&page=1",
+      "previous_page": null,
+      "next_page": "https://my-tenant.tines.com/api/v1/audit_logs?per_page=1&page=2",
+      "per_page": 1,
+      "pages": 2,
+      "count": 2
+    }
+  }

tests/fixtures/tines/audit_logs/002.json

@@ -0,0 +1,62 @@
+{
+    "audit_logs": [
+      {
+        "created_at": "2023-07-21T14:00:30Z",
+        "operation_name": "UserCredentialSave",
+        "id": 1488227,
+        "inputs": {
+          "inputs": {
+            "name": "Example",
+            "mode": "TEXT",
+            "value": "REDACTED",
+            "description": "",
+            "jwtPayload": "REDACTED",
+            "jwtPrivateKey": "REDACTED",
+            "jwtAutoGenerateTimeClaims": "REDACTED",
+            "jwtAlgorithm": "REDACTED",
+            "jwtHs256Secret": "REDACTED",
+            "oauthGrantType": "REDACTED",
+            "oauthClientId": "REDACTED",
+            "oauthScope": "REDACTED",
+            "oauthClientSecret": "REDACTED",
+            "oauthUrl": "REDACTED",
+            "awsAuthenticationType": "REDACTED",
+            "oauthTokenUrl": "REDACTED",
+            "oauthPkceCodeChallengeMethod": "REDACTED",
+            "awsAccessKey": "REDACTED",
+            "awsSecretKey": "REDACTED",
+            "awsAssumedRoleArn": "REDACTED",
+            "awsAssumedRoleExternalId": "REDACTED",
+            "httpRequestLocationOfToken": "REDACTED",
+            "httpRequestTtl": "REDACTED",
+            "httpRequestOptions": "REDACTED",
+            "mtlsClientCertificate": "REDACTED",
+            "mtlsClientPrivateKey": "REDACTED",
+            "mtlsRootCertificate": "REDACTED",
+            "readAccess": "TEAM",
+            "allowedHosts": "REDACTED",
+            "icon": "REDACTED",
+            "emoji": "REDACTED",
+            "preserveNewlineCharacters": "REDACTED",
+            "teamId": 12723,
+            "folderId": 332
+          }
+        },
+        "request_ip": "192.0.2.1",
+        "request_user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/100.0",
+        "tenant_id": 5044,
+        "updated_at": "2023-07-21T14:00:30Z",
+        "user_email": "user@example.org",
+        "user_id": 7642,
+        "user_name": "Example User"
+      }
+    ],
+    "meta": {
+      "current_page": "https://my-tenant.tines.com/api/v1/audit_logs?per_page=1&page=2",
+      "previous_page": "https://my-tenant.tines.com/api/v1/audit_logs?per_page=1&page=1",
+      "next_page": null,
+      "per_page": 1,
+      "pages": 2,
+      "count": 2
+    }
+  }