Skip to content

Commit c99dca6

Browse files
aaronmondalaaronmondal
authored
Make stores and schedulers lists of named specs (#1496)
To migrate, change your config like this (similar for schedulers): ``` // Old: "stores": { "SOMESTORE": { "memory": {} } } // New: "stores": [ { "name": "SOMESTORE", "memory": {} } ] ``` For now this is not a breaking change, but a backwards compatible one. The backwards-compatibility code will be removed after the next release in a breaking change. Closes #834
1 parent a67d4bd commit c99dca6

File tree

16 files changed

+303
-87
lines changed

16 files changed

+303
-87
lines changed

‎Cargo.lock

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎deployment-examples/docker-compose/local-storage-cas.json5

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,9 @@
44
// so objects are compressed, deduplicated and uses some in-memory
55
// optimizations for certain hot paths.
66
{
7-
"stores": {
8-
"CAS_MAIN_STORE": {
7+
"stores": [
8+
{
9+
"name": "CAS_MAIN_STORE",
910
"compression": {
1011
"compression_algorithm": {
1112
"lz4": {}
@@ -21,8 +22,8 @@
2122
}
2223
}
2324
}
24-
},
25-
"AC_MAIN_STORE": {
25+
}, {
26+
"name": "AC_MAIN_STORE",
2627
"filesystem": {
2728
"content_path": "~/.cache/nativelink/content_path-ac",
2829
"temp_path": "~/.cache/nativelink/tmp_path-ac",
@@ -32,7 +33,7 @@
3233
}
3334
}
3435
}
35-
},
36+
],
3637
"servers": [{
3738
"listener": {
3839
"http": {

‎deployment-examples/docker-compose/scheduler.json5

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
2-
"stores": {
3-
"GRPC_LOCAL_STORE": {
2+
"stores": [
3+
{
4+
"name": "GRPC_LOCAL_STORE",
45
// Note: This file is used to test GRPC store.
56
"grpc": {
67
"instance_name": "",
@@ -9,8 +10,8 @@
910
],
1011
"store_type": "cas"
1112
}
12-
},
13-
"GRPC_LOCAL_AC_STORE": {
13+
}, {
14+
"name": "GRPC_LOCAL_AC_STORE",
1415
// Note: This file is used to test GRPC store.
1516
"grpc": {
1617
"instance_name": "",
@@ -20,9 +21,10 @@
2021
"store_type": "ac"
2122
}
2223
}
23-
},
24-
"schedulers": {
25-
"MAIN_SCHEDULER": {
24+
],
25+
"schedulers": [
26+
{
27+
"name": "MAIN_SCHEDULER",
2628
"simple": {
2729
"supported_platform_properties": {
2830
"cpu_count": "minimum",
@@ -32,7 +34,7 @@
3234
}
3335
}
3436
}
35-
},
37+
],
3638
"servers": [{
3739
"listener": {
3840
"http": {

‎deployment-examples/docker-compose/worker.json5

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
2-
"stores": {
3-
"GRPC_LOCAL_STORE": {
2+
"stores": [
3+
{
4+
"name": "GRPC_LOCAL_STORE",
45
// Note: This file is used to test GRPC store.
56
"grpc": {
67
"instance_name": "",
@@ -9,8 +10,8 @@
910
],
1011
"store_type": "cas"
1112
}
12-
},
13-
"GRPC_LOCAL_AC_STORE": {
13+
}, {
14+
"name": "GRPC_LOCAL_AC_STORE",
1415
// Note: This file is used to test GRPC store.
1516
"grpc": {
1617
"instance_name": "",
@@ -19,8 +20,8 @@
1920
],
2021
"store_type": "ac"
2122
}
22-
},
23-
"WORKER_FAST_SLOW_STORE": {
23+
}, {
24+
"name": "WORKER_FAST_SLOW_STORE",
2425
"fast_slow": {
2526
"fast": {
2627
"filesystem": {
@@ -39,7 +40,7 @@
3940
}
4041
}
4142
}
42-
},
43+
],
4344
"workers": [{
4445
"local": {
4546
"worker_api_endpoint": {

‎kubernetes/components/worker/worker.json5

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
2-
"stores": {
3-
"GRPC_LOCAL_STORE": {
2+
"stores": [
3+
{
4+
"name": "GRPC_LOCAL_STORE",
45
// Note: This file is used to test GRPC store.
56
"grpc": {
67
"instance_name": "",
@@ -9,8 +10,8 @@
910
],
1011
"store_type": "cas"
1112
}
12-
},
13-
"GRPC_LOCAL_AC_STORE": {
13+
}, {
14+
"name": "GRPC_LOCAL_AC_STORE",
1415
// Note: This file is used to test GRPC store.
1516
"grpc": {
1617
"instance_name": "",
@@ -19,8 +20,8 @@
1920
],
2021
"store_type": "ac"
2122
}
22-
},
23-
"WORKER_FAST_SLOW_STORE": {
23+
}, {
24+
"name": "WORKER_FAST_SLOW_STORE",
2425
"fast_slow": {
2526
"fast": {
2627
"filesystem": {
@@ -38,7 +39,7 @@
3839
}
3940
}
4041
}
41-
},
42+
],
4243
"workers": [{
4344
"local": {
4445
"worker_api_endpoint": {

‎kubernetes/nativelink/nativelink-config.json5

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22
// `~/.cache/nativelink`. When this location is mounted as a PersistentVolume
33
// it persists the cache across restarts.
44
{
5-
"stores": {
6-
"CAS_MAIN_STORE": {
5+
"stores": [
6+
{
7+
"name": "CAS_MAIN_STORE",
78
"existence_cache": {
89
"backend": {
910
"compression": {
@@ -22,8 +23,8 @@
2223
}
2324
}
2425
}
25-
},
26-
"AC_MAIN_STORE": {
26+
}, {
27+
"name": "AC_MAIN_STORE",
2728
"completeness_checking": {
2829
"backend": {
2930
"filesystem": {
@@ -41,9 +42,10 @@
4142
}
4243
}
4344
}
44-
},
45-
"schedulers": {
46-
"MAIN_SCHEDULER": {
45+
],
46+
"schedulers": [
47+
{
48+
"name": "MAIN_SCHEDULER",
4749
// TODO(adams): use the right scheduler because reclient doesn't use the cached results?
4850
// TODO(adams): max_bytes_per_stream
4951
"simple": {
@@ -68,7 +70,7 @@
6870
}
6971
}
7072
}
71-
},
73+
],
7274
"servers": [{
7375
// Only publish metrics on a private port.
7476
"listener": {

‎nativelink-config/BUILD.bazel

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ rust_test_suite(
3131
name = "integration",
3232
timeout = "short",
3333
srcs = [
34+
"tests/backwards_compat_test.rs",
3435
"tests/deserialization_test.rs",
3536
],
3637
deps = [
@@ -40,6 +41,7 @@ rust_test_suite(
4041
"@crates//:humantime",
4142
"@crates//:pretty_assertions",
4243
"@crates//:serde",
44+
"@crates//:serde_json",
4345
"@crates//:serde_json5",
4446
],
4547
)

‎nativelink-config/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,3 +12,4 @@ shellexpand = { version = "3.1.0", default-features = false, features = ["base-0
1212

1313
[dev-dependencies]
1414
pretty_assertions = { version = "1.4.1", features = ["std"] }
15+
serde_json = { version = "1.0.138", default-features = false }

‎nativelink-config/examples/basic_cas.json5

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
2-
"stores": {
3-
"AC_MAIN_STORE": {
2+
"stores": [
3+
{
4+
"name": "AC_MAIN_STORE",
45
"filesystem": {
56
"content_path": "/tmp/nativelink/data-worker-test/content_path-ac",
67
"temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac",
@@ -9,8 +10,8 @@
910
"max_bytes": 1000000000
1011
}
1112
}
12-
},
13-
"WORKER_FAST_SLOW_STORE": {
13+
}, {
14+
"name": "WORKER_FAST_SLOW_STORE",
1415
"fast_slow": {
1516
// "fast" must be a "filesystem" store because the worker uses it to make
1617
// hardlinks on disk to a directory where the jobs are running.
@@ -34,9 +35,10 @@
3435
}
3536
}
3637
}
37-
},
38-
"schedulers": {
39-
"MAIN_SCHEDULER": {
38+
],
39+
"schedulers": [
40+
{
41+
"name": "MAIN_SCHEDULER",
4042
"simple": {
4143
"supported_platform_properties": {
4244
"cpu_count": "minimum",
@@ -59,7 +61,7 @@
5961
}
6062
}
6163
}
62-
},
64+
],
6365
"workers": [{
6466
"local": {
6567
"worker_api_endpoint": {

‎nativelink-config/examples/filesystem_cas.json5

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,9 @@
44
// so objects are compressed, deduplicated and uses some in-memory
55
// optimizations for certain hot paths.
66
{
7-
"stores": {
8-
"FS_CONTENT_STORE": {
7+
"stores": [
8+
{
9+
"name": "FS_CONTENT_STORE",
910
"compression": {
1011
"compression_algorithm": {
1112
"lz4": {}
@@ -21,8 +22,8 @@
2122
}
2223
}
2324
}
24-
},
25-
"CAS_MAIN_STORE": {
25+
}, {
26+
"name": "CAS_MAIN_STORE",
2627
"verify": {
2728
"backend": {
2829
// Because we are using a dedup store, we can bypass small objects
@@ -78,8 +79,8 @@
7879
"verify_size": true,
7980
"verify_hash": true
8081
}
81-
},
82-
"AC_MAIN_STORE": {
82+
}, {
83+
"name": "AC_MAIN_STORE",
8384
"filesystem": {
8485
"content_path": "/tmp/nativelink/data/content_path-ac",
8586
"temp_path": "/tmp/nativelink/data/tmp_path-ac",
@@ -89,9 +90,10 @@
8990
}
9091
}
9192
}
92-
},
93-
"schedulers": {
94-
"MAIN_SCHEDULER": {
93+
],
94+
"schedulers": [
95+
{
96+
"name": "MAIN_SCHEDULER",
9597
"simple": {
9698
"supported_platform_properties": {
9799
"cpu_count": "minimum",
@@ -113,7 +115,7 @@
113115
}
114116
}
115117
}
116-
},
118+
],
117119
"servers": [{
118120
"listener": {
119121
"http": {

‎nativelink-config/examples/redis.json

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,19 @@
11
{
2-
"stores": {
3-
"CAS_FAST_SLOW_STORE": {
2+
"stores": [
3+
{
4+
"name": "CAS_FAST_SLOW_STORE",
45
"redis_store": {
56
"addresses": ["redis://127.0.0.1:6379/"],
67
"mode": "cluster"
78
}
8-
},
9-
"AC_FAST_SLOW_STORE": {
9+
}, {
10+
"name": "AC_FAST_SLOW_STORE",
1011
"redis_store": {
1112
"addresses": ["redis://127.0.0.1:6379/"],
1213
"mode": "cluster"
1314
}
14-
},
15-
"AC_MAIN_STORE": {
15+
}, {
16+
"name": "AC_MAIN_STORE",
1617
"completeness_checking": {
1718
"backend": {
1819
"ref_store": {
@@ -25,8 +26,8 @@
2526
}
2627
}
2728
}
28-
},
29-
"CAS_MAIN_STORE": {
29+
}, {
30+
"name": "CAS_MAIN_STORE",
3031
"existence_cache": {
3132
"backend": {
3233
"compression": {
@@ -42,7 +43,7 @@
4243
}
4344
}
4445
}
45-
},
46+
],
4647
"servers": [
4748
{
4849
"listener": {

0 commit comments

Comments
 (0)