Skip to content

Commit 43e3ce3

Browse files
fionerafionera
committed
feat: add nix deployment flavor
This adds a deployment flavor for NixOS. It contains lots of duplicated config structs, which will get cleaned up in a later change. Change-Id: I58c94b3f911ae02b57af9ea3e3ff3a7b09687dc7
1 parent bb58a82 commit 43e3ce3

File tree

11 files changed

+401
-150
lines changed

11 files changed

+401
-150
lines changed

‎deploy/base/clickhouse/clickhouse.cue

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package clickhouse
2+
3+
import (
4+
schema "github.com/monogon-dev/NetMeta/deploy/single-node/schema"
5+
"netmeta.monogon.tech/xml"
6+
)
7+
8+
// A stripped down version of the #SamplerConfig found in deploy/single-node/config.cue
9+
#SamplerConfig: [string]: {
10+
device: string
11+
samplingRate: int
12+
anonymizeAddresses: bool
13+
description: string
14+
interface: [string]: {
15+
id: int
16+
description: string
17+
}
18+
vlan: [string]: {
19+
id: int
20+
description: string
21+
}
22+
host: [string]: {
23+
device: string
24+
description: string
25+
}
26+
...
27+
}
28+
29+
#UserData: autnums: [string]: {
30+
asn: int
31+
name: string
32+
country: string
33+
}
34+
35+
#Config: {
36+
sampler: #SamplerConfig
37+
userData: #UserData
38+
39+
dataPath: string
40+
risinfoURL: string
41+
}
42+
43+
files: {
44+
// Iterate over our required files from schema, e.g. the protobuf files
45+
for k, v in schema.file {
46+
"\(k)": v
47+
}
48+
49+
// Iterate over all defined files in _files and generate the config files for clickhouse
50+
for k, v in _files {
51+
"\(k).conf": (xml.#Marshal & {in: v._cfg}).out
52+
"\(k).tsv": v.data
53+
}
54+
}

‎deploy/base/clickhouse/external.cue

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
package clickhouse
2+
3+
import (
4+
"netmeta.monogon.tech/xml"
5+
)
6+
7+
files: "risinfo.conf": (xml.#Marshal & {in: yandex: dictionary: {
8+
name: "risinfo"
9+
source: http: {
10+
url: "\(#Config.risinfoURL)/rib.tsv"
11+
format: "TabSeparated"
12+
}
13+
lifetime: 3600
14+
layout: ip_trie: access_to_key_from_attributes: true
15+
structure: key: attribute: {
16+
name: "prefix"
17+
type: "String"
18+
}
19+
structure: attribute: {
20+
name: "asnum"
21+
type: "UInt32"
22+
null_value: 0
23+
}
24+
}}).out
25+
26+
files: "autnums.conf": (xml.#Marshal & {in: yandex: dictionary: {
27+
name: "autnums"
28+
source: clickhouse: query:
29+
#"""
30+
SELECT * FROM dictionaries.risinfo_autnums
31+
UNION ALL
32+
SELECT * FROM dictionaries.user_autnums
33+
"""#
34+
lifetime: 3600
35+
layout: flat: null
36+
structure: [{
37+
id: name: "asnum"
38+
}, {
39+
attribute: {
40+
name: "name"
41+
type: "String"
42+
null_value: null
43+
}
44+
}, {
45+
attribute: {
46+
name: "country"
47+
type: "String"
48+
null_value: null
49+
}
50+
}]
51+
}}).out
52+
53+
files: "risinfo_autnums.conf": (xml.#Marshal & {in: yandex: dictionary: {
54+
name: "risinfo_autnums"
55+
source: http: {
56+
url: "\(#Config.risinfoURL)/autnums.tsv"
57+
format: "TabSeparated"
58+
}
59+
lifetime: 86400
60+
layout: flat: null
61+
structure: [{
62+
id: name: "asnum"
63+
}, {
64+
attribute: {
65+
name: "name"
66+
type: "String"
67+
null_value: null
68+
}
69+
}, {
70+
attribute: {
71+
name: "country"
72+
type: "String"
73+
null_value: null
74+
}
75+
}]
76+
}}).out
77+
78+
files: "format_function.xml": (xml.#Marshal & {in: yandex: functions: {
79+
type: "executable"
80+
name: "formatQuery"
81+
return_type: "String"
82+
argument: [{
83+
type: "String"
84+
name: "query"
85+
}]
86+
format: "LineAsString"
87+
command: "clickhouse format --oneline"
88+
execute_direct: "0"
89+
}}).out

‎deploy/single-node/k8s/clickhouse/files.cue renamed to ‎deploy/base/clickhouse/files.cue

Lines changed: 6 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ package clickhouse
33
import (
44
"strings"
55
"strconv"
6-
"netmeta.monogon.tech/xml"
76
)
87

98
// template for TSV dictionaries
@@ -20,7 +19,7 @@ _files: [NAME=string]: {
2019
name: NAME
2120
source: [{
2221
file: {
23-
path: "/etc/clickhouse-server/config.d/\(NAME).tsv"
22+
path: "\(#Config.dataPath)/\(NAME).tsv"
2423
format: "TSV"
2524
}
2625
settings: format_tsv_null_representation: "NULL"
@@ -29,17 +28,9 @@ _files: [NAME=string]: {
2928
}
3029
}
3130

32-
// Iterate over all defined files in _files and generate the config files for clickhouse
33-
ClickHouseInstallation: netmeta: spec: configuration: files: {
34-
for k, v in _files {
35-
"\(k).conf": (xml.#Marshal & {in: v._cfg}).out
36-
"\(k).tsv": v.data
37-
}
38-
}
39-
4031
// Dictionary for user-defined interface name lookup
4132
_files: InterfaceNames: {
42-
data: strings.Join([ for s in #Config.sampler for i in s.interface {
33+
data: strings.Join([for s in #Config.sampler for i in s.interface {
4334
strings.Join([s.device, "\(i.id)", i.description], "\t")
4435
}], "\n")
4536

@@ -68,7 +59,7 @@ _files: InterfaceNames: {
6859

6960
// Dictionary for user-defined sampler settings lookup
7061
_files: SamplerConfig: {
71-
data: strings.Join([ for s in #Config.sampler {
62+
data: strings.Join([for s in #Config.sampler {
7263
let samplingRate = [
7364
if s.samplingRate == 0 {
7465
"NULL"
@@ -119,7 +110,7 @@ _files: SamplerConfig: {
119110

120111
// Dictionary for user-defined vlan name lookup
121112
_files: VlanNames: {
122-
data: strings.Join([ for s in #Config.sampler for v in s.vlan {
113+
data: strings.Join([for s in #Config.sampler for v in s.vlan {
123114
strings.Join([s.device, "\(v.id)", v.description], "\t")
124115
}], "\n")
125116

@@ -148,7 +139,7 @@ _files: VlanNames: {
148139

149140
// Dictionary for user-defined host name lookup
150141
_files: HostNames: {
151-
data: strings.Join([ for s in #Config.sampler for h in s.host {
142+
data: strings.Join([for s in #Config.sampler for h in s.host {
152143
strings.Join([s.device, h.device, h.description], "\t")
153144
}], "\n")
154145

@@ -176,7 +167,7 @@ _files: HostNames: {
176167
}
177168

178169
_files: user_autnums: {
179-
data: strings.Join([ for _, e in #Config.userData.autnums {
170+
data: strings.Join([for _, e in #Config.userData.autnums {
180171
strings.Join(["\(e.asn)", e.name, e.country], "\t")
181172
}], "\n")
182173

@@ -199,97 +190,3 @@ _files: user_autnums: {
199190
}]
200191
}
201192
}
202-
203-
ClickHouseInstallation: netmeta: spec: configuration: files: "risinfo.conf": (xml.#Marshal & {in: {
204-
yandex: dictionary: {
205-
name: "risinfo"
206-
source: http: {
207-
url: "http://risinfo/rib.tsv"
208-
format: "TabSeparated"
209-
}
210-
lifetime: 3600
211-
layout: ip_trie: access_to_key_from_attributes: true
212-
structure: key: attribute: {
213-
name: "prefix"
214-
type: "String"
215-
}
216-
structure: attribute: {
217-
name: "asnum"
218-
type: "UInt32"
219-
null_value: 0
220-
}
221-
}
222-
}}).out
223-
224-
ClickHouseInstallation: netmeta: spec: configuration: files: "autnums.conf": (xml.#Marshal & {in: {
225-
yandex: dictionary: {
226-
name: "autnums"
227-
source: clickhouse: {
228-
query:
229-
#"""
230-
SELECT * FROM dictionaries.risinfo_autnums
231-
UNION ALL
232-
SELECT * FROM dictionaries.user_autnums
233-
"""#
234-
}
235-
lifetime: 3600
236-
layout: flat: null
237-
structure: [{
238-
id: name: "asnum"
239-
}, {
240-
attribute: {
241-
name: "name"
242-
type: "String"
243-
null_value: null
244-
}
245-
}, {
246-
attribute: {
247-
name: "country"
248-
type: "String"
249-
null_value: null
250-
}
251-
}]
252-
}
253-
}}).out
254-
255-
ClickHouseInstallation: netmeta: spec: configuration: files: "risinfo_autnums.conf": (xml.#Marshal & {in: {
256-
yandex: dictionary: {
257-
name: "risinfo_autnums"
258-
source: http: {
259-
url: "http://risinfo/autnums.tsv"
260-
format: "TabSeparated"
261-
}
262-
lifetime: 86400
263-
layout: flat: null
264-
structure: [{
265-
id: name: "asnum"
266-
}, {
267-
attribute: {
268-
name: "name"
269-
type: "String"
270-
null_value: null
271-
}
272-
}, {
273-
attribute: {
274-
name: "country"
275-
type: "String"
276-
null_value: null
277-
}
278-
}]
279-
}
280-
}}).out
281-
282-
ClickHouseInstallation: netmeta: spec: configuration: files: "format_function.xml": (xml.#Marshal & {in: {
283-
yandex: functions: {
284-
type: "executable"
285-
name: "formatQuery"
286-
return_type: "String"
287-
argument: [{
288-
type: "String"
289-
name: "query"
290-
}]
291-
format: "LineAsString"
292-
command: "clickhouse format --oneline"
293-
execute_direct: "0"
294-
}
295-
}}).out

‎deploy/single-node/k8s/clickhouse/static_files.cue renamed to ‎deploy/base/clickhouse/static_files.cue

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -252,9 +252,3 @@ _files: TCPFlags: data: #"""
252252
64 ECE
253253
128 CWR
254254
"""#
255-
256-
ClickHouseInstallation: netmeta: spec: configuration: files: {
257-
for k, v in schema.file {
258-
"\(k)": v
259-
}
260-
}

0 commit comments

Comments
 (0)