Skip to content

Commit 88ff63e

Browse files
plessbdplessbd
authored
Merge pull request #10 from plessbd/column-renames
Make federation cloud simpler
2 parents d3f012e + b40a531 commit 88ff63e

File tree

8 files changed

+40
-109
lines changed

8 files changed

+40
-109
lines changed

‎configuration/etl/etl_action_defs.d/federated/cloud/account.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
{
2+
"#name": "fed.ingest-cloud.account",
23
"table_definition": [{
34
"$ref": "${table_definition_dir}/cloud_common/account.json#/table_definition"
45
}],
56
"source_query": {
67
"records": {
78
"resource_id": "drf.id",
9+
"account_id": "scaf.account_id",
810
"provider_account": "scaf.provider_account",
911
"display": "CONCAT(scaf.display, ' (', dof.abbrev, ')')"
1012
},

‎configuration/etl/etl_action_defs.d/federated/cloud/event.json

Lines changed: 16 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"#name": "fed.ingest-cloud.event",
23
"table_definition": [
34
{
45
"$ref": "${table_definition_dir}/cloud_common/event.json#/table_definition"
@@ -7,11 +8,14 @@
78
"source_query": {
89
"records": {
910
"resource_id": "drf.id",
10-
"instance_id": "dcif.instance_id",
11-
"event_time_utc": "scef.event_time_utc",
11+
"event_id": "scef.event_id",
12+
"instance_id": "scef.instance_id",
13+
"event_time_ts": "scef.event_time_ts",
1214
"event_type_id": "scef.event_type_id",
1315
"record_type_id": "scef.record_type_id",
14-
"host_id": "dchf.host_id",
16+
"host_id": "scef.host_id",
17+
"person_id": "upf.id",
18+
"systemaccount_id": "usa.id",
1519
"submission_venue_id": "scef.submission_venue_id"
1620
},
1721

@@ -21,35 +25,23 @@
2125
"schema": "${SOURCE_SCHEMA}",
2226
"alias": "scef"
2327
},
24-
{
25-
"name": "host",
26-
"schema": "${SOURCE_SCHEMA}",
27-
"alias": "schf",
28-
"on":"schf.host_id = scef.host_id"
29-
},
30-
{
31-
"name": "instance",
32-
"schema": "${SOURCE_SCHEMA}",
33-
"alias": "scif",
34-
"on": "scif.instance_id = scef.instance_id"
35-
},
3628
{
3729
"name": "resourcefact",
3830
"schema": "${UTILITY_SCHEMA}",
3931
"alias": "drf",
40-
"on": "drf.organization_id = (SELECT id FROM ${UTILITY_SCHEMA}.organization WHERE federation_instance_id = ${instance_id}) AND drf.resource_origin_id = scif.resource_id"
32+
"on": "drf.organization_id = (SELECT id FROM ${UTILITY_SCHEMA}.organization WHERE federation_instance_id = ${instance_id}) AND drf.resource_origin_id = scef.resource_id"
4133
},
4234
{
43-
"name": "instance",
44-
"schema": "${DESTINATION_SCHEMA}",
45-
"alias": "dcif",
46-
"on": "dcif.provider_identifier = scif.provider_identifier AND dcif.resource_id = drf.id"
35+
"schema": "${UTILITY_SCHEMA}",
36+
"name": "person",
37+
"alias": "upf",
38+
"on": "upf.person_origin_id = scef.person_id AND upf.organization_id = drf.organization_id"
4739
},
4840
{
49-
"name": "host",
50-
"schema": "${DESTINATION_SCHEMA}",
51-
"alias": "dchf",
52-
"on": "dchf.resource_id = drf.id AND dchf.hostname = schf.hostname"
41+
"schema": "${UTILITY_SCHEMA}",
42+
"name": "systemaccount",
43+
"alias": "usa",
44+
"on": "usa.person_id = upf.id and usa.resource_id = drf.id"
5345
}
5446
],
5547
"macros": [

‎configuration/etl/etl_action_defs.d/federated/cloud/host.json

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
{
2+
"#name": "fed.ingest-cloud.host",
23
"table_definition": [{
34
"$ref": "${table_definition_dir}/cloud_common/host.json#/table_definition"
45
}],
56
"source_query": {
67
"records": {
78
"resource_id": "drf.id",
8-
"hostname": "schf.hostname"
9+
"hostname": "schf.hostname",
10+
"host_id": "schf.host_id"
911
},
1012
"joins": [
1113
{
@@ -20,9 +22,6 @@
2022
"on": "drf.organization_id = (SELECT id FROM ${UTILITY_SCHEMA}.organization WHERE federation_instance_id = ${instance_id}) AND drf.resource_origin_id = schf.resource_id"
2123
}
2224
],
23-
"where": [
24-
"schf.host_id != -1"
25-
],
2625
"macros": [
2726
{
2827
"$ref": "etl_macros.d/federated/federated.json#/getInstanceId"

‎configuration/etl/etl_action_defs.d/federated/cloud/image.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
{
2+
"#name": "fed.ingest-cloud.images",
23
"table_definition": [{
34
"$ref": "${table_definition_dir}/cloud_common/image.json#/table_definition"
45
}],
56
"source_query": {
67
"records": {
78
"resource_id": "drf.id",
8-
"image": "scif.image"
9+
"image": "scif.image",
10+
"image_id": "scif.image_id"
911
},
1012
"joins": [
1113
{

‎configuration/etl/etl_action_defs.d/federated/cloud/instance-data.json

Lines changed: 5 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"#name": "fed.ingest-cloud.instance-data",
23
"table_definition": [
34
{
45
"$ref": "${table_definition_dir}/cloud_common/instance_data.json#/table_definition"
@@ -7,10 +8,10 @@
78
"source_query": {
89
"records": {
910
"resource_id": "drf.id",
10-
"event_id": "dcef.event_id",
11-
"image_id": "dcimf.image_id",
12-
"instance_type_id": "dcitf.instance_type_id",
13-
"host_id": "dchf.host_id"
11+
"event_id": "scidf.event_id",
12+
"image_id": "scidf.image_id",
13+
"instance_type_id": "scidf.instance_type_id",
14+
"host_id": "scidf.host_id"
1415
},
1516

1617
"joins": [
@@ -24,66 +25,6 @@
2425
"schema": "${UTILITY_SCHEMA}",
2526
"alias": "drf",
2627
"on": "drf.organization_id = (SELECT id FROM ${UTILITY_SCHEMA}.organization WHERE federation_instance_id = ${instance_id}) AND drf.resource_origin_id = scidf.resource_id"
27-
},
28-
{
29-
"name": "host",
30-
"schema": "${SOURCE_SCHEMA}",
31-
"alias": "schf",
32-
"on":"schf.host_id = scidf.host_id"
33-
},
34-
{
35-
"name": "host",
36-
"schema": "${DESTINATION_SCHEMA}",
37-
"alias": "dchf",
38-
"on": "dchf.resource_id = drf.id AND dchf.hostname = schf.hostname"
39-
},
40-
{
41-
"name": "instance_type",
42-
"schema": "${SOURCE_SCHEMA}",
43-
"alias": "scitf",
44-
"on": "scitf.instance_type_id = scidf.instance_type_id"
45-
},
46-
{
47-
"name": "instance_type",
48-
"schema": "${DESTINATION_SCHEMA}",
49-
"alias": "dcitf",
50-
"on": "dcitf.resource_id = drf.id AND dcitf.instance_type = scitf.instance_type AND dcitf.num_cores = scitf.num_cores AND dcitf.memory_mb = scitf.memory_mb AND dcitf.disk_gb = scitf.disk_gb"
51-
},
52-
{
53-
"name": "event",
54-
"schema": "${SOURCE_SCHEMA}",
55-
"alias": "scef",
56-
"on": "scef.event_id = scidf.event_id"
57-
},
58-
{
59-
"name": "instance",
60-
"schema": "${SOURCE_SCHEMA}",
61-
"alias": "scif",
62-
"on": "scif.instance_id = scef.instance_id"
63-
},
64-
{
65-
"name": "instance",
66-
"schema": "${DESTINATION_SCHEMA}",
67-
"alias": "dcif",
68-
"on": "dcif.resource_id = drf.id AND dcif.provider_identifier = scif.provider_identifier"
69-
},
70-
{
71-
"name": "event",
72-
"schema": "${DESTINATION_SCHEMA}",
73-
"alias": "dcef",
74-
"on": "dcef.resource_id = drf.id AND dcef.event_time_utc = scef.event_time_utc AND dcef.event_type_id = scef.event_type_id AND dcef.record_type_id = scef.record_type_id AND dcef.instance_id = dcif.instance_id"
75-
},
76-
{
77-
"name": "image",
78-
"schema": "${SOURCE_SCHEMA}",
79-
"alias": "scimf",
80-
"on": "scimf.image_id = scidf.image_id"
81-
},
82-
{
83-
"name": "image",
84-
"schema": "${DESTINATION_SCHEMA}",
85-
"alias": "dcimf",
86-
"on": "dcif.resource_id = drf.id AND dcif.provider_identifier = scif.provider_identifier"
8728
}
8829
],
8930
"macros": [

‎configuration/etl/etl_action_defs.d/federated/cloud/instance-type.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"#name": "fed.ingest-cloud.instance-type",
23
"table_definition": [
34
{
45
"$ref": "${table_definition_dir}/cloud_common/instance_type.json#/table_definition"
@@ -8,6 +9,7 @@
89
"records": {
910
"resource_id": "drf.id",
1011
"instance_type": "scitf.instance_type",
12+
"instance_type_id": "scitf.instance_type_id",
1113
"display": "scitf.display",
1214
"description": "scitf.description",
1315
"num_cores": "scitf.num_cores",

‎configuration/etl/etl_action_defs.d/federated/cloud/instance.json

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
2+
"#name": "fed.ingest-cloud.instance",
23
"table_definition": [
34
{
45
"$ref": "${table_definition_dir}/cloud_common/instance.json#/table_definition"
@@ -7,9 +8,9 @@
78
"source_query": {
89
"records": {
910
"resource_id": "drf.id",
10-
"account_id": "dcaf.account_id",
11+
"instance_id": "scif.instance_id",
12+
"account_id": "scif.account_id",
1113
"provider_identifier": "COALESCE(scif.provider_identifier, 'unknown')",
12-
"#": "person_id is not currently populated correctly in the cloud, this is going to be ignored for now.",
1314
"person_id": "upf.id"
1415
},
1516

@@ -25,18 +26,6 @@
2526
"alias": "drf",
2627
"on": "drf.organization_id = (SELECT id FROM ${UTILITY_SCHEMA}.organization WHERE federation_instance_id = ${instance_id}) AND drf.resource_origin_id = scif.resource_id"
2728
},
28-
{
29-
"name": "account",
30-
"schema": "${SOURCE_SCHEMA}",
31-
"alias": "scaf",
32-
"on": "scaf.account_id = scif.account_id"
33-
},
34-
{
35-
"name": "account",
36-
"schema": "${DESTINATION_SCHEMA}",
37-
"alias": "dcaf",
38-
"on": "dcaf.provider_account = scaf.provider_account AND dcaf.resource_id = drf.id"
39-
},
4029
{
4130
"schema": "${UTILITY_SCHEMA}",
4231
"name": "person",

‎configuration/etl/etl_pipelines.d/federated-cloud.json

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
"#": "Resources are ingested as part of regular federation",
33
"#": "fed.ingest.instance-ogranization",
44
"#": "fed.ingest.instance-resource",
5+
"#name": "fed.ingest-cloud",
56
"description": "Federation Jobs Ingest",
67
"vars": {
78
"required": [{
@@ -11,7 +12,7 @@
1112
"optional":[{
1213
"name": "instance_id",
1314
"type": "integer",
14-
"#": "the instane id from `modw`.`federation_instances`",
15+
"#": "the instance id from `modw`.`federation_instances`",
1516
"#": "for the prefix that equals instance_name",
1617
"#": "this is an optimization and will run the macro",
1718
"#": "that will find the id based on the name if not provided"
@@ -60,7 +61,10 @@
6061
"name": "instance_data",
6162
"definition_file": "federated/cloud/instance-data.json",
6263
"description": "Cloud Instance Data *requires: -d instance_name*",
63-
"truncate_destination": false
64+
"truncate_destination": false,
65+
"hide_sql_warning_codes": [
66+
1062
67+
]
6468
}
6569
]
6670
}

0 commit comments

Comments
 (0)