diff --git a/tap_gitlab/__init__.py b/tap_gitlab/__init__.py index 1e98095..6c84685 100644 --- a/tap_gitlab/__init__.py +++ b/tap_gitlab/__init__.py @@ -204,6 +204,13 @@ def load_schema(entity): 'schema': load_schema('group_variables'), 'key_properties': ['group_id', 'key'], 'replication_method': 'FULL_TABLE', + }, + 'merge_request_notes': { + 'url2': '/projects/{id}/merge_requests/{secondar_id}/notes?order_by=updated_at={start_date}', + 'schema': load_schema('merge_request_notes'), + 'key_properties': ['id', 'merge_request_iid'], + 'replication_method': 'INCREMENTAL', + 'replication_keys': ['updated_at'], } } @@ -469,6 +476,7 @@ def sync_merge_requests(project): # And then sync all the commits for this MR # (if it has changed, new commits may be there to fetch) sync_merge_request_commits(project, transformed_row) + sync_merge_request_notes(project, transformed_row) singer.write_state(STATE) @@ -491,6 +499,23 @@ def sync_merge_request_commits(project, merge_request): singer.write_record("merge_request_commits", transformed_row, time_extracted=utils.now()) +def sync_merge_request_notes(project, merge_request): + entity = "merge_request_notes" + stream = CATALOG.get_stream(entity) + if stream is None or not stream.is_selected(): + return + state_key = "project_{}_merge_request_notes".format(project['id']) + start_date=get_start(state_key) + mdata = metadata.to_map(stream.metadata) + + url = get_url(entity="merge_request_notes", id=project['id'], secondary_id=merge_request['iid'], start_date=start_date) + with Transformer(pre_hook=format_timestamp) as transformer: + for row in gen_request(url): + row['merge_request_iid'] = merge_request['iid'] + row['author_id'] = row['author']['id'] + transformed_row = transformer.transform(row, RESOURCES["merge_request_notes"]["schema"], mdata) + singer.write_record("merge_request_notes", transformed_row, time_extracted=utils.now()) + utils.update_state(STATE, state_key, row['updated_at']) def sync_releases(project): entity = "releases" stream = CATALOG.get_stream(entity) diff --git a/tap_gitlab/schemas/merge_requests_notes.json b/tap_gitlab/schemas/merge_requests_notes.json new file mode 100644 index 0000000..68d4ad9 --- /dev/null +++ b/tap_gitlab/schemas/merge_requests_notes.json @@ -0,0 +1,76 @@ +{ + "type": "object", + "properties": { + "id": { + "type": [ + "integer", + "null" + ] + }, + "type": { + "type": [ + "string", + "null" + ] + }, + "body": { + "type": "string" + }, + "attachment": { + "type": "null" + }, + "author_id": { + "type": [ + "integer", + "null" + ] + }, + "created_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "updated_at": { + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "system": { + "type": "boolean" + }, + "noteable_id": { + "type": "integer" + }, + "noteable_type": { + "type": "string" + }, + "project_id": { + "type": "integer" + }, + "resolvable": { + "type": "boolean" + }, + "confidential": { + "type": "boolean" + }, + "internal": { + "type": "boolean" + }, + "imported": { + "type": "boolean" + }, + "imported_from": { + "type": "string" + }, + "noteable_iid": { + "type": "integer" + }, + "commands_changes": { + "type": "object" + } + } +} \ No newline at end of file