Handle documents in root dir of projects

Author: hsyyid

.DS_Store

@@ -2,3 +2,4 @@ hidden-tests/
 .secrets/
 .pytest_cache/
 __pycache__/
+.DS_Store

.gitignore

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tap-procore"
-version = "0.0.6"
+version = "0.0.7"
 description = "`tap-procore` is Singer tap for procore, built with the Singer SDK."
 authors = ["hotglue"]
 license = "Apache 2.0"

pyproject.toml

@@ -113,6 +113,12 @@ def partitions(self) -> Optional[List[dict]]:
             })
         return result or None
 
+    def post_process(self, row: dict, context: Optional[dict] = None) -> dict:
+        """As needed, append or transform raw data to match expected structure."""
+        # Add project_id to response
+        row['company_id'] = context['company_id']
+        return row
+
     def get_url_params(
         self,
         partition: Optional[dict],
@@ -129,6 +135,7 @@ def get_url_params(
 
     schema = PropertiesList(
         Property("id", IntegerType),
+        Property("company_id", IntegerType),
         Property("name", StringType)
     ).to_dict()
 
@@ -372,6 +379,9 @@ def get_folders(self, headers):
             r = requests.get(endpoint, headers=headers)
             data = r.json().get('folders', [])
 
+            # Add root folder
+            folders.append({'folder': -1, 'project': project['id']})
+
             # Add these folders to final output
             folders.extend(
                 [{'folder': x['id'], 'project': project['id']} for x in data])
@@ -384,8 +394,13 @@ def get_folders(self, headers):
 
     def get_url(self, partition: Optional[dict]) -> str:
         url = super().get_url(partition)
-        sub_url = f"{url}/{partition['folder']}"
-        return sub_url
+
+        # Handle sub folders
+        if partition['folder'] != -1:
+            sub_url = f"{url}/{partition['folder']}"
+            return sub_url
+
+        return url
 
     @property
     def partitions(self) -> Optional[List[dict]]: