plgn-380 updated users - query improvment, add extra logs

plugins/salesforce/.CHECKSUM

@@ -1,7 +1,7 @@
 {
-	"spec": "c1dec7cfceff3e96fdcfbbb845b157b8",
-	"manifest": "71e58227b4db0b3e8df068c994e3b2c7",
-	"setup": "7f5033d434b3c7de40f80b08c8129187",
+	"spec": "c75170b8c11b24f04d86bcecd5964a6f",
+	"manifest": "c0b4255f3f00f804f388170f1434d562",
+	"setup": "53af104162251cbb28e7802a21869cee",
 	"schemas": [
 		{
 			"identifier": "advanced_search/schema.py",

plugins/salesforce/bin/komand_salesforce

@@ -6,7 +6,7 @@ from sys import argv
 
 Name = "Salesforce"
 Vendor = "rapid7"
-Version = "2.1.0"
+Version = "2.1.1"
 Description = "The Salesforce plugin allows you to search, update, and manage salesforce records"
 
 

plugins/salesforce/help.md

@@ -531,6 +531,7 @@ _This plugin does not contain any troubleshooting information._
 
 # Version History
 
+* 2.1.1 - Task Monitor Users: query improvement on updated users | Add extra logs on timestamp
 * 2.1.0 - Implemented token auto-refresh on expiration for continuous sessions | Task Monitor Users: add flag `remove_duplicates` for duplicated events | Task Monitor Users: removed formatting of task output and cleaning null 
 * 2.0.2 - Task Monitor Users: query improvement | Handle exception related with grant type
 * 2.0.1 - Add extra logs register

plugins/salesforce/komand_salesforce/tasks/monitor_users/task.py

@@ -12,9 +12,10 @@ class MonitorUsers(insightconnect_plugin_runtime.Task):
     USER_LOGIN_QUERY = "SELECT LoginTime, UserId, LoginType, LoginUrl, SourceIp, Status, Application, Browser FROM LoginHistory WHERE LoginTime >= {start_timestamp} AND LoginTime < {end_timestamp}"
     USERS_QUERY = "SELECT Id, FirstName, LastName, Email, Alias, IsActive FROM User WHERE UserType = 'Standard'"
     UPDATED_USER_QUERY = "SELECT Id, FirstName, LastName, Email, Alias, IsActive FROM User WHERE Id = '{user_id}' AND UserType = 'Standard'"
-    UPDATED_USERS_QUERY = "SELECT Id, FirstName, LastName, Email, Alias, IsActive FROM User WHERE UserType = 'Standard' AND Id IN ({user_ids})"
+    UPDATED_USERS_QUERY = "SELECT Id, FirstName, LastName, Email, Alias, IsActive FROM User WHERE UserType = 'Standard' AND LastModifiedDate >= {start_timestamp} AND LastModifiedDate < {end_timestamp}"
     USERS_NEXT_PAGE_ID = "users_next_page_id"
     USER_LOGIN_NEXT_PAGE_ID = "user_login_next_page_id"
+    UPDATED_USERS_NEXT_PAGE_ID = "updated_users_next_page_id"
     LAST_USER_UPDATE_COLLECTION_TIMESTAMP = "last_user_update_collection_timestamp"
     NEXT_USER_COLLECTION_TIMESTAMP = "next_user_collection_timestamp"
     NEXT_USER_LOGIN_COLLECTION_TIMESTAMP = "next_user_login_collection_timestamp"
@@ -40,10 +41,9 @@ def run(self, params={}, state={}):  # noqa: C901
             get_user_login_history = False
 
             remove_duplicates = state.pop(self.REMOVE_DUPLICATES, True)  # true as a default
-            users_next_page_id = state.get(self.USERS_NEXT_PAGE_ID)
-            state.pop(self.USERS_NEXT_PAGE_ID, None)
-            user_login_next_page_id = state.get(self.USER_LOGIN_NEXT_PAGE_ID)
-            state.pop(self.USER_LOGIN_NEXT_PAGE_ID, None)
+            users_next_page_id = state.pop(self.USERS_NEXT_PAGE_ID, None)
+            user_login_next_page_id = state.pop(self.USER_LOGIN_NEXT_PAGE_ID, None)
+            updated_users_next_page_id = state.pop(self.UPDATED_USERS_NEXT_PAGE_ID, None)
 
             user_update_end_timestamp = now
             user_update_start_timestamp = (user_update_end_timestamp - timedelta(hours=24)).isoformat(
@@ -67,12 +67,13 @@ def run(self, params={}, state={}):  # noqa: C901
                 get_user_login_history = True
                 state[self.NEXT_USER_LOGIN_COLLECTION_TIMESTAMP] = str(now + timedelta(hours=1))
                 state[self.LAST_USER_LOGIN_COLLECTION_TIMESTAMP] = str(user_login_end_timestamp)
-            elif users_next_page_id or user_login_next_page_id:
+            elif users_next_page_id or user_login_next_page_id or updated_users_next_page_id:
                 self.logger.info("Getting next page of results...")
                 if users_next_page_id:
                     get_users = True
                 if user_login_next_page_id:
                     get_user_login_history = True
+
             else:
                 self.logger.info("Subsequent run")
                 user_update_start_timestamp = state.get(self.LAST_USER_UPDATE_COLLECTION_TIMESTAMP)
@@ -98,49 +99,56 @@ def run(self, params={}, state={}):  # noqa: C901
             try:
                 records = []
 
-                if not users_next_page_id and not user_login_next_page_id:
-                    user_ids = self.connection.api.get_updated_users(
-                        {
-                            "start": user_update_start_timestamp,
-                            "end": user_update_end_timestamp,
-                        }
-                    ).get("ids", [])
-                    self.logger.info(f"Found {len(user_ids)} updated users")
-                    if user_ids:
-                        user_ids_quoted = ["'" + x + "'" for x in user_ids]
-                        concatenated_ids = ",".join(user_ids_quoted)
-                        updated_users = self.connection.api.query(
-                            self.UPDATED_USERS_QUERY.format(user_ids=concatenated_ids), None
-                        ).get("records", [])
-
-                        self.logger.info(f"{len(updated_users)} updated users added to output")
-                        records.extend(self.add_data_type_field(updated_users, "User Update"))
+                if not users_next_page_id and not user_login_next_page_id or updated_users_next_page_id:
+                    self.logger.info(
+                        f"Get updated users - start: {user_update_start_timestamp} end: {user_update_end_timestamp}"
+                    )
+                    response = self.connection.api.query(
+                        self.UPDATED_USERS_QUERY.format(
+                            start_timestamp=user_update_start_timestamp, end_timestamp=user_update_end_timestamp
+                        ),
+                        updated_users_next_page_id,
+                    )
+                    updated_users_next_page_id = response.get("next_page_id")
+                    updated_users = response.get("records")
+                    if updated_users_next_page_id:
+                        state[self.UPDATED_USERS_NEXT_PAGE_ID] = updated_users_next_page_id
+                        has_more_pages = True
+
+                    self.logger.info(f"{len(updated_users)} updated users added to output")
+                    records.extend(self.add_data_type_field(updated_users, "User Update"))
 
                 if get_users:
+                    self.logger.info("Get all internal users")
                     response = self.connection.api.query(self.USERS_QUERY, users_next_page_id)
+                    users = response.get("records", [])
                     users_next_page_id = response.get("next_page_id")
                     if users_next_page_id:
                         state[self.USERS_NEXT_PAGE_ID] = users_next_page_id
                         has_more_pages = True
 
-                    self.logger.info(f"{len(response.get('records'))} internal users added to output")
-                    records.extend(self.add_data_type_field(response.get("records", []), "User"))
+                    self.logger.info(f"{len(users)} internal users added to output")
+                    records.extend(self.add_data_type_field(users, "User"))
 
                 if get_user_login_history:
+                    self.logger.info(
+                        f"Get user login history - start: {user_login_start_timestamp} end: {user_login_end_timestamp}"
+                    )
                     response = self.connection.api.query(
                         self.USER_LOGIN_QUERY.format(
                             start_timestamp=user_login_start_timestamp.isoformat(),
                             end_timestamp=user_login_end_timestamp.isoformat(),
                         ),
                         user_login_next_page_id,
                     )
+                    users_login = response.get("records", [])
                     user_login_next_page_id = response.get("next_page_id")
                     if user_login_next_page_id:
                         state[self.USER_LOGIN_NEXT_PAGE_ID] = user_login_next_page_id
                         has_more_pages = True
 
-                    self.logger.info(f"{len(response.get('records'))} users login history added to output")
-                    records.extend(self.add_data_type_field(response.get("records", []), "User Login"))
+                    self.logger.info(f"{len(users_login)} users login history added to output")
+                    records.extend(self.add_data_type_field(users_login, "User Login"))
 
                 if remove_duplicates is True:
                     records = self.remove_duplicates(records)

plugins/salesforce/plugin.spec.yaml

@@ -4,7 +4,7 @@ products: [insightconnect]
 name: salesforce
 title: Salesforce
 description: The Salesforce plugin allows you to search, update, and manage salesforce records
-version: 2.1.0
+version: 2.1.1
 connection_version: 2
 vendor: rapid7
 support: community

plugins/salesforce/setup.py

@@ -3,7 +3,7 @@
 
 
 setup(name="salesforce-rapid7-plugin",
-      version="2.1.0",
+      version="2.1.1",
       description="The Salesforce plugin allows you to search, update, and manage salesforce records",
       author="rapid7",
       author_email="",

plugins/salesforce/unit_test/responses/get_specific_user_empty.json.resp

@@ -0,0 +1,5 @@
+{
+  "totalSize": 0,
+  "done": true,
+  "records": []
+}

plugins/salesforce/unit_test/util.py

@@ -138,9 +138,21 @@ def json(self):
                 }:
                     return MockResponse(200, "get_specific_user.json.resp")
                 if params == {
-                    "q": "SELECT Id, FirstName, LastName, Email, Alias, IsActive FROM User WHERE UserType = 'Standard' AND Id IN ('005Hn00000HVWwxIAH')"
+                    "q": "SELECT Id, FirstName, LastName, Email, Alias, IsActive FROM User WHERE UserType = 'Standard' AND LastModifiedDate >= 2023-07-19T16:21:15.340+00:00 AND LastModifiedDate < 2023-07-20T16:21:15.340+00:00"
                 }:
                     return MockResponse(200, "get_specific_user.json.resp")
+                if params == {
+                    "q": "SELECT Id, FirstName, LastName, Email, Alias, IsActive FROM User WHERE UserType = 'Standard' AND LastModifiedDate >= 2023-07-20T16:15:15.340+00:00 AND LastModifiedDate < 2023-07-20T16:21:15.340+00:00"
+                }:
+                    return MockResponse(200, "get_specific_user.json.resp")
+                if params == {
+                    "q": "SELECT Id, FirstName, LastName, Email, Alias, IsActive FROM User WHERE UserType = 'Standard' AND LastModifiedDate >= 2023-07-20T16:10:15.340+00:00 AND LastModifiedDate < 2023-07-20T16:21:15.340+00:00"
+                }:
+                    return MockResponse(200, "get_specific_user_empty.json.resp")
+                if params == {
+                    "q": "SELECT Id, FirstName, LastName, Email, Alias, IsActive FROM User WHERE UserType = 'Standard' AND LastModifiedDate >= invalid AND LastModifiedDate < 2023-07-20T16:21:15.340+00:00"
+                }:
+                    return MockResponse(400)
                 if params == {
                     "q": "SELECT Id, FirstName, LastName, Email, Alias, IsActive FROM User WHERE UserType = 'Standard'"
                 }: