@@ -11,18 +11,40 @@ import {
1111 BROKER_SERVICE ,
1212 BROKER_USER ,
1313 DB_FILE_STATUS ,
14+ OBJECT_STORAGE_ACCESS_KEY ,
1415 OBJECT_STORAGE_BUCKET ,
16+ OBJECT_STORAGE_ENABLED ,
17+ OBJECT_STORAGE_END_POINT ,
1518 OBJECT_STORAGE_SECRET_KEY ,
16- VAULT_CRED_KEY ,
19+ VAULT_CRED_KEYS_ACCESS_KEY ,
20+ VAULT_CRED_KEYS_BUCKET ,
21+ VAULT_CRED_KEYS_END_POINT ,
22+ VAULT_CRED_KEYS_SECRET_KEY ,
1723 VAULT_CRED_PATH ,
1824} from '../constants' ;
1925import { DatabaseService } from '../services/database.service' ;
2026import VaultService from '../broker/vault.service' ;
2127import BrokerService from '../broker/broker.service' ;
2228
29+ interface LogStatus {
30+ id : number ;
31+ basename : string ;
32+ path : string ;
33+ }
34+
35+ interface LogArtifact {
36+ id : number ;
37+ checksum : string ;
38+ name : string ;
39+ size : number ;
40+ type : string ;
41+ }
42+
43+ type FileUpdateCallback = ( id : number ) => Promise < any > ;
44+
2345export async function backup ( db : DatabaseService ) {
2446 console . log ( 'backup: start' ) ;
25- const result = await db . query < {
47+ const result = await db . all < {
2648 id : number ;
2749 basename : string ;
2850 path : string ;
@@ -41,67 +63,115 @@ export async function backup(db: DatabaseService) {
4163 return ;
4264 }
4365
44- if ( OBJECT_STORAGE_SECRET_KEY ) {
45- await backupWithSecret ( db , OBJECT_STORAGE_SECRET_KEY , result ) ;
46- } else {
47- const brokerService = new BrokerService ( BROKER_JWT ) ;
48- try {
49- const openResponse = await brokerService . open ( {
50- event : {
51- provider : 'nr-objectstore-rotate-backup' ,
52- reason : 'Cron triggered' ,
53- } ,
54- actions : [
55- {
56- action : 'backup' ,
57- id : 'backup' ,
58- provision : [ 'token/self' ] ,
59- service : {
60- name : BROKER_SERVICE ,
61- project : BROKER_PROJECT ,
62- environment : BROKER_ENVIRONMENT ,
63- } ,
64- } ,
65- ] ,
66- user : {
67- name : BROKER_USER ,
68- } ,
69- } ) ;
70- const actionToken = openResponse . actions [ 'backup' ] . token ;
71- const vaultAccessToken = await brokerService . provisionToken ( actionToken ) ;
72- const vault = new VaultService ( vaultAccessToken ) ;
73- const objectStorageCreds = await vault . read ( VAULT_CRED_PATH ) ;
74- const secretKey = objectStorageCreds [ VAULT_CRED_KEY ] ;
75- vault . revokeToken ( ) ;
76- const backupFiles = await backupWithSecret ( db , secretKey , result ) ;
77- for ( const fileObj of backupFiles ) {
78- await brokerService . attachArtifact ( actionToken , fileObj ) ;
66+ const fileUpdateCb : FileUpdateCallback = ( id ) => {
67+ return db . updatelogStatus ( id , DB_FILE_STATUS . CopiedToObjectStore ) ;
68+ } ;
69+
70+ try {
71+ if ( ! OBJECT_STORAGE_ENABLED ) {
72+ // Skip copy to object storage
73+ for ( const file of result . rows ) {
74+ await db . updatelogStatus ( file . id , DB_FILE_STATUS . CopiedToObjectStore ) ;
7975 }
80- brokerService . close ( true ) ;
81- } catch ( e : any ) {
82- // Error!
83- console . log ( e ) ;
76+ } else if ( BROKER_JWT === '' ) {
77+ await backupUsingEnv ( result . rows , fileUpdateCb ) ;
78+ } else {
79+ await backupUsingBroker ( BROKER_JWT , result . rows , fileUpdateCb ) ;
8480 }
81+ } catch ( e : any ) {
82+ // Error!
83+ console . log ( e ) ;
8584 }
8685}
8786
87+ async function backupUsingEnv (
88+ dbFileRows : LogStatus [ ] ,
89+ cb : FileUpdateCallback ,
90+ ) : Promise < LogArtifact [ ] > {
91+ const backupFiles = await backupWithSecret (
92+ dbFileRows ,
93+ OBJECT_STORAGE_END_POINT ,
94+ OBJECT_STORAGE_ACCESS_KEY ,
95+ OBJECT_STORAGE_SECRET_KEY ,
96+ OBJECT_STORAGE_BUCKET ,
97+ ) ;
98+
99+ for ( const file of backupFiles ) {
100+ await cb ( file . id ) ;
101+ }
102+ return backupFiles ;
103+ }
104+
105+ async function backupUsingBroker (
106+ brokerJwt : string ,
107+ dbFileRows : LogStatus [ ] ,
108+ cb : FileUpdateCallback ,
109+ ) : Promise < LogArtifact [ ] > {
110+ const brokerService = new BrokerService ( brokerJwt ) ;
111+ const openResponse = await brokerService . open ( {
112+ event : {
113+ provider : 'nr-objectstore-rotate-backup' ,
114+ reason : 'Cron triggered' ,
115+ } ,
116+ actions : [
117+ {
118+ action : 'backup' ,
119+ id : 'backup' ,
120+ provision : [ 'token/self' ] ,
121+ service : {
122+ name : BROKER_SERVICE ,
123+ project : BROKER_PROJECT ,
124+ environment : BROKER_ENVIRONMENT ,
125+ } ,
126+ } ,
127+ ] ,
128+ user : {
129+ name : BROKER_USER ,
130+ } ,
131+ } ) ;
132+ const actionToken = openResponse . actions [ 'backup' ] . token ;
133+ const vaultAccessToken = await brokerService . provisionToken ( actionToken ) ;
134+ const vault = new VaultService ( vaultAccessToken ) ;
135+ const objectStorageCreds = await vault . read ( VAULT_CRED_PATH ) ;
136+ vault . revokeToken ( ) ;
137+ const backupFiles = await backupWithSecret (
138+ dbFileRows ,
139+ VAULT_CRED_KEYS_END_POINT === ''
140+ ? OBJECT_STORAGE_END_POINT
141+ : objectStorageCreds [ VAULT_CRED_KEYS_END_POINT ] ,
142+ VAULT_CRED_KEYS_ACCESS_KEY === ''
143+ ? OBJECT_STORAGE_ACCESS_KEY
144+ : objectStorageCreds [ VAULT_CRED_KEYS_ACCESS_KEY ] ,
145+ VAULT_CRED_KEYS_BUCKET === ''
146+ ? OBJECT_STORAGE_SECRET_KEY
147+ : objectStorageCreds [ VAULT_CRED_KEYS_BUCKET ] ,
148+ VAULT_CRED_KEYS_SECRET_KEY === ''
149+ ? OBJECT_STORAGE_BUCKET
150+ : objectStorageCreds [ VAULT_CRED_KEYS_SECRET_KEY ] ,
151+ ) ;
152+
153+ for ( const file of backupFiles ) {
154+ await brokerService . attachArtifact ( actionToken , file ) ;
155+ await cb ( file . id ) ;
156+ }
157+ brokerService . close ( true ) ;
158+
159+ return backupFiles ;
160+ }
161+
88162async function backupWithSecret (
89- db : DatabaseService ,
90- secret : string ,
91- dbResult : {
92- rows : {
93- id : number ;
94- basename : string ;
95- path : string ;
96- } [ ] ;
97- } ,
98- ) : Promise < any [ ] > {
99- const client = getClient ( secret ) ;
100- const files = [ ] ;
101- for ( const row of dbResult . rows ) {
163+ dbFileRows : LogStatus [ ] ,
164+ endPoint : string ,
165+ accessKey : string ,
166+ secretKey : string ,
167+ bucket : string ,
168+ ) : Promise < LogArtifact [ ] > {
169+ const client = getClient ( endPoint , accessKey , secretKey ) ;
170+ const files : LogArtifact [ ] = [ ] ;
171+ for ( const row of dbFileRows ) {
102172 try {
103173 const response = await client . fPutObject (
104- OBJECT_STORAGE_BUCKET ,
174+ bucket ,
105175 path . basename ( row . path ) ,
106176 row . path ,
107177 ) ;
@@ -111,24 +181,15 @@ async function backupWithSecret(
111181 console . log ( info ) ;
112182 continue ;
113183 }
114- db . query < {
115- id : number ;
116- basename : string ;
117- path : string ;
118- } > (
119- `
120- UPDATE logs
121- SET status = ?
122- WHERE id = ?
123- ` ,
124- [ DB_FILE_STATUS . CopiedToObjectStore , row . id ] ,
125- ) ;
184+ const checksum = `sha256:${ await computeHash ( row . path ) } ;
126185 files . push ( {
127- checksum : `sha256:${ await computeHash ( row . path ) } ,
186+ id : row . id ,
187+ checksum,
128188 name : path . basename ( row . path ) ,
129189 size : fs . statSync ( row . path ) . size ,
130190 type : 'tgz' ,
131191 } ) ;
192+ console . log ( `backup: Sent ${ row . path } ${ checksum } ) ;
132193 }
133194 return files ;
134195}
0 commit comments