Merge branch 'templates-providers' of https://github.com/red-kite-sol…

…utions/stalker into templates-providers

devspace.base.yaml

@@ -52,7 +52,7 @@ vars:
   JM_JWT_SECRET: "123456" # The secret used to sign JWT for users
   JM_REFRESH_SECRET: "123456" # The secret used to sign JWT refresh token for users
   JM_MONGO_ADDRESS: mongodb://${MONGO_JM_USER}:${MONGO_JM_PASSWORD}@mongo-mongodb-headless:27017/ # The address of the FM's mongo database
-  JM_DATA_SOURCES: '[{"type": "git", "url": "https://github.com/red-kite-solutions/stalker-templates-community"}]'
+  JM_DATA_SOURCES: '[{"type": "git", "url": "https://github.com/red-kite-solutions/stalker-templates-community", "branch": "main"}]'
 
   # Cron
   CRON_VERSION: latest

devspace.tests.yaml

@@ -18,7 +18,7 @@ vars:
   JM_CERTIFICATES_KAFKA_CLIENT_KEY: LS0tLS1CRUdJTiBFTkNSWVBURUQgUFJJVkFURSBLRVktLS0tLQpNSUlKbkRCT0Jna3Foa2lHOXcwQkJRMHdRVEFwQmdrcWhraUc5dzBCQlF3d0hBUUlLZUEwajVzSFVSRUNBZ2dBCk1Bd0dDQ3FHU0liM0RRSUpCUUF3RkFZSUtvWklodmNOQXdjRUNNVENMc3BjUWxCOEJJSUpTT0FRall5eHl3OUUKVnZITElBS1I2Ly9MQjJXQ3d0aVBWTUwxckc2VkhSb3ZObENRZ3VOL3BPcjBHNGpkdmJrcGcyV3V2WExmaFBsYwpuaVY0WmhQaHEyWnU0bTFGRVNtVlhXRWFNMWFwc2l6Q1NLU1FDTkhrZEZCYktxZFlmbTVMTXpiUldpTHBERTdLCmdwSTBFTE05VVNaaVlGMjBrdUJXN0MrYnVtZTFsZWs0d0tZUnNDOE15eTcxQUhkaWNIUEFqOG1QL3pWT1h6Ti8KdUdkdUVCbGZFZ0hTTCt6VzJBZ1l2ZkhRU1BBd1Ezd1JWTUw2TEQ0ckpVQnp1c1VSRGJtekRsc3JGYlFHMXdSbgp1L0dUK05YMU5HdFlNQ015cXhudnd1ejJWaFlkanhRaHNnSXR0R1l2Zy9URUJtM0tvUXpOalYzQXBuYXV2b3UxCmU4ODZiaU43UXJPdDhGQTRoelJMZU5rNHVsVkZBdFZGb05uTHlDYlIvTUpxQk1VdHM3bEJRUFdKUmNpdUpQL28KdGQzdk5iS2xMTW9uWnlaSGZMMlM3UVF1YlBUcEE1SHdRNCtyVUJPMXh4NHhBVlBBMWVnYml4eVdTU1NZQnFIUwprcDdoeTJVV3N2ZFhiaStGY2plK0ZkWFJOTWZOaHhUWDZHaUoxUXNOeldDeERROXF6T0dGNlpnYm11REFXRDdCCllnR3FSUlZ1VUNWR09hVDFXNHkxU3Eva2VTM05ZTDVUUHpkYyt2MFo5T0NENGNpMmFnTXc2cElzcXdzNnJRSzEKWXNTUW55T1BtSFUvZjZaRUQvY1VXQm9qTS9LbXhpVkt2dDRhcVhyNm1nYTJabHdGQ1crT1pKQzNGSlhMSDR2eApReFJIVmgxdDRNUkRrWmNJNGI5RnZacXNvTjlKUG1vSmdicHFCWktkMVZUZ25STisyWitJL3RXRE42azRIRm9XCnkzTVFxOEt1Z3dnQ3ppZTc2dFRPNUUwUkJmR2JlSi9xNHkramNzMTFOSXhpajdTclpHVGVEUTJ0UmhHTUdpaDYKeW5QbUdyVkM0K1NXU3l4czJka29SUkRaOUx6cm54YmZBY3FWbWw4OFkzVHpiN2RpcFlOTUFrSUVWL1RKOWdoQQp4Q0dzNjNNOVZKTGJLYmprTXdtNndUVW5CQlQ1SzQvRHNlQkwvaitKZHZHcDF1UXpYdjhsSlNWNGJoQzFId1pYCk55V3MwSE13Y1lVK2xBMTdJSlBIRjdWZk44RXExT1lRQ3dCV1V0WlkwaFJEU2MzQi8wL0h5S1BEdXpSWGk0MDkKM0N6REVBN0VuVVhVNnU3dFhYUWF5RDdIdjhYSkxJaUNsbk51QjhKalV4d0UxZWNGSjNINkZtMjhGOTlsMHZlTAovblpCM25sTEh5R2drcVMwSkZHaExZcko5NlB1c0E2YlpPbTlzQmlmQWpEZWg2dW9DbTg4N2VEREtzaDFNT3FECmlXb1Zuc0NpYmgwTEIzTTBmb0U1T0svcitJNFFrajNFY0JIM0MwbWpOQ0IxNzFXMDQ3QTNRbTB6ZVFVM0M1SDAKWE5TVHpVTXdzTkdvMmNXMHJRVWJ3bllraDRYd2IzbjdZUmROT2V2M0N3aEsvNkVFZG1uQTNKUmxEWVpsOW95LwpvOVEyQzF0MmNwYjlkMUVKT1ZDZ2tzZ2xPbjFUSHFjZHlObS90a2RSRmNkaDNSSjFlbVpUa01KWGovM3dwWlZXCmtoOVNPWVEyRVB3QzhMVkxtR0JrS3FRaUEzQkhlTkJGdjZSK3NaTnFLRDJrZTY4Yzd4Y3FveHJzQ1FMRnFLakIKcjFGUHVWVUNmb2RDTFhmQkVXaTlWamZ2Wm9pSnBiZHh1YjFibTZuQU5SNitoWVBSTGNzRFRkOGMvZTAxUktyMwpWL2RzZStqTENuQkNWQmhaekF0enNLUmVMSmpMN1BzWjZFcFVwYkUyZHVjblJXTmZBV3JGR1RRcVpTNWhjU3FiCjhvRXVua3QwMzNLV1U4YTluM0Z0QnZkcFN4aFk4bUZCV3lEQ1NtcnZUcmlldEZmVy9UWmdtUmhjT0dId0VCeGQKVnJScE8xOU5RMlRiUURCME5IRmF2QncrVWlDQnpGL1NYOTVIMkQydStXTEpwQ0J3d090K3QvS2RsSXFqTm1QZwpwWTVMMFkxR3JxTEQ2NVFINkpRdmYvTSs3SHBVMk1wOGpOVWZ4K01OaE5tTFArS25zZElsZDBHZXZqUU1aZHBJCkR4dWUyc0txcTBNZnNWY01COGx4bEF2VEh5cjVoRkxVbnAzb0pnMFNuTkhnME1EWjc0M2FJK3BWdElRV05CNTkKRzRYblBkcXpGbmFySnJDaFdIaHF0bUxOZEpZL1pDWlpmeXBOSGdmdHhib1lpa01meWtqQlA5Y0dwcC9JWStFdApHTnl1K2JxaE5IRWkxejE3VkszZkw1ZGo2WWUvUFlhV21wM0Z5S25kcWJyRDcrZHBqRTJ6V0tmczM0Z1NNM0xYCjcrUVRVYmg4UU1SUTUzdkxWNVY3NUVYMjU0aGl0NmFkSmxtazRTQ0tEcHV1VUNvOEE5VTNnL21YeFdTazJuU0gKNHJ3bHhxYk1oRnI3N3FkVGdNaDVUVUVqNVd1QVE2MUtmNlRtRWUxL01teWZZL0JFekdYU1RuMmtiQ1ozd0Q4NgpmNFJ5MUZsWlNCY1htU0xyMkNxTGRPdUZTblF2R1hkeHhPS0pJV2NKc2RvenU5SDR1ZjkzakdRcmZFNnNlUmk4CjYxZEthYW1rSkYzbUVjS3IrZEw2SjVZbHhDd29YbXAwUEpzcmU2aDNZVGwxbE1xYjAxTzNsWkJBMU1UL1FRbHMKNDltRDRMa2g4cnBKQWZYenFxTXZLby9sN3hadVZJN0hSNFVIRTBVaDc0UVdnZ0NqRXEvdXZGTkhZYTFBdXlFYwpPY2Y4UEJYSVVMeUVYcGVFbk5UMlpRWTRRWk1RUUdjWGsrd2ZVQTIrZzBBUGZ0K20zN1l6c2lsOXNrQThzRUNrClg2S2h1TFV5Sm52eE5jSGNIQ1QwUS9NeG5kK2FYcytUM1MzcjhSTzUxdllNMzlOMk5LQVpXbWRpY2R3SG9mQ2MKS0l2YURuWnpWY2lFeldXMnN2VkxkcGpXUWQvQVBEaVc1TDZGRlFrN0w5dHc1OWV5emxrYmlTbFRSZjU0Y2piUgpPNXpqQWtjV0tNR0dOUDJ1cGFhRGp5L3FHeDFwWGpzSnptRXBydXJpU1llNDN3ZkxWbjN5VmhBVkR1NGYxZGd6CmVzN2tWbDZ5R25STENTTFR5MGxqK1JNVzE2aHF2UFE5Vkd5MFl0UjJuWXVxNG5SajB2NWk2MnlHVHdTVXpsVU8KalF2Q2dLa1RWcHgzRHlSd2dvbHErNGZjSG03MmlrWUtXMHhHQlRTaVJlYlZCeVE4VDdIbGhVUGJkMnMzWmFPUwpLbm0rZklrcEdKdHNBU2lscFFXOWQxZ05UODduVmQrZlZUOGZIZlpyWHRuVzBDVkhxOUtNb3d2eUVLd29sNytrCmxlRmtIMVFJUDNURytWZlJTOU02aHFlV2ZKazZWQUtNMTlyRTN0eWo0WEQ2TDZTL2xJaGRVZXk3OEpvQlludjMKaC9kdjAzVnlmU2xiR0FBcmR4U0x1dzQ3SXJhd2VJL1Z0STZhVGphdzZwRzVyVU9xb05ycUcwRWdnWG8yVCtPZgpUdC9talBOaDc0bExxamRyM2ZYNEVuRkhZQVZiMzhQNWFHK0FhUzdmbnpYc29EUS8vdkRXRVFESDhRS1J1RnBwCjBOS25lQ0FVWHNyVTZOS3J0YzJMb2lJTVlqNHBTSlNZWDVnNlArdlZiSGpUOEdLa3hjQ0F4OGprQzRReGx1T0sKRUNNWFZYbmZQMy9aNTVPZzdYTUJiNWw0MUJEWTBjc1RLN1FlNmRIcUU3Ni9qRitYMXE3UHpGWllpTTVIdG93bAo4SUwvd0czSTBLZFhPY2xEejZRWVBpWkJUbys4WTZKTUo1bjYvd3lKWGFJdnh4cDRBWFBac0VqdkY2TFNjRFpLCm1TbGtBU3pkRk5KYTMwQndhSkFjSVBQWEJLVGhyMkJhSXZSVmZXQ1lIenNYYkdWMWlueWU4YVErakhlZHlMTzgKSUdIbENJVGdSVW15cWNiTk01QnloNW1LVWhYcEJ2NEsrdXFsaEQrTDluaTAyRjRFTlhHL3V5a0VuZ1ExbUV6aApSZ0dJVXlMSGJzUXpkZzg0ZmJMQW5nPT0KLS0tLS1FTkQgRU5DUllQVEVEIFBSSVZBVEUgS0VZLS0tLS0=
   JM_CERTIFICATES_SSL_PRIVATE_KEY: ""
   JM_CERTIFICATES_SSL_CERTIFICATE_CHAIN_PEM: ""
-  JM_DATA_SOURCES: '[{"type": "git", "url": "https://github.com/red-kite-solutions/stalker-templates-community"}]'
+  JM_DATA_SOURCES: '[{"type": "git", "url": "https://github.com/red-kite-solutions/stalker-templates-community", "branch": "main"}]'
 
   # Cron
   CRON_CERTIFICATES_CA_PEM: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURaVENDQWsyZ0F3SUJBZ0lVUUxZaERvdE9idXZRQnhGMGRrYnI2SmZSOTVNd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1FqRUxNQWtHQTFVRUF3d0NRMEV4RmpBVUJnTlZCQXNNRFZOMFlXeHJaWElnVFc5dVoyOHhHekFaQmdOVgpCQW9NRWxKbFpDQkxhWFJsSUZOdmJIVjBhVzl1Y3pBZUZ3MHlOREF5TVRreU1UQXlOVE5hRncwek5EQXlNVFl5Ck1UQXlOVE5hTUVJeEN6QUpCZ05WQkFNTUFrTkJNUll3RkFZRFZRUUxEQTFUZEdGc2EyVnlJRTF2Ym1kdk1Sc3cKR1FZRFZRUUtEQkpTWldRZ1MybDBaU0JUYjJ4MWRHbHZibk13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQgpEd0F3Z2dFS0FvSUJBUUM3NGFjRTZhWkNVS2YrcXBkZlJQdEU4di9odGZNdzhkb2ZrUjZiUUwwWmtDMTBQb0VBCkFIeTRCTWtmMW9HR09GWTdVNkxOZEpackVUaVIrSDdkM2ptNUg0QndMQ1grcjRTRUFvc0N3SDVDUHVVSndwZk4KMS9JRTFIbjVxRXcrOFBqUzZqUTBtNHY5NW1lbUpDMTFUWEszMTMySk5VMFUxVm0yaHorMnZwSGpveTc5b1pSdwphNmJ1d1dTV1NwL2lMN2M0Y2dBQmtRTFdHYXhDcW9FWUpqbXJpTnhwQy9FTzk4V2FJeTl2WGdmaUxTSkkzUG5ECjFXc1BTbHgrOFplSEZXdlZqSllhU2dEZUowbXFXUS9aMTI5SVdidmtMelhkV2VYYXIwdDdaYzlOWWVuOVhPQkUKNGdlY3BTWjJTMG1SRFVadnRyNW4rbzM0RTRCRjlqYldRRzVuQWdNQkFBR2pVekJSTUIwR0ExVWREZ1FXQkJSUgpXaHBhREI4eVVPVGhaU1hyMm1rRzV5RnlYVEFmQmdOVkhTTUVHREFXZ0JSUldocGFEQjh5VU9UaFpTWHIybWtHCjV5RnlYVEFQQmdOVkhSTUJBZjhFQlRBREFRSC9NQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUJBUUF2c1N3bVJSN2QKSzZSRmtoVTZHOFp1VnFIMldSTDdCd0VHM0UrbE5xWHEzMVBLcjVyd3dGb3VIUHlBdDhFRk13dlZBd2prc0s5TwpoMzJDWmorZE1MN3NtVFliTE90T0xIbWkvUmxEeVdES1hNQ3dqcXh4aS9LdnVVTkhLVEV0c2pMVW5zNWxVUWpzCmI5MUVBNGlvdFM1UDE2ZEMzM2NqVUwwY1p5TkdsTGoyVzB2UjRuM3gwYjZrRkl0QXlHZnpxZ0NHbFpNY29wUDcKNENpeUNWZ0xhcDduTFdrNTdOV294Z2F2L3VnczZ0YXVrQ3dZcm5aT28xVkNqL3R1L0wwNUp3dTNTVWhXWHlJbApobENRMjNNYVRRK1hqVEk5TXNueEtSOWlOTWI3MWNkT2V4K0M3bHhMcm41K2xDSW9yVzJyWHRPY003VlI5TlJnClBGT2w5SE5zMU01egotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0t

packages/backend/jobs-manager/service/src/modules/database/custom-job-templates/custom-job-templates.model.ts

@@ -8,6 +8,7 @@ export class CustomJobTemplateSource {
   type: 'git';
   repoUrl: string;
   avatarUrl: string;
+  branch: string;
 }
 
 @Schema()

packages/backend/jobs-manager/service/src/modules/database/custom-job-templates/job-source.dto.ts

@@ -9,4 +9,7 @@ export class JobSourceDto {
 
   @IsString()
   avatarUrl: string;
+
+  @IsString()
+  branch: string;
 }

packages/backend/jobs-manager/service/src/modules/database/custom-jobs/custom-jobs.model.ts

@@ -8,6 +8,7 @@ export class CustomJobSource {
   type: 'git';
   repoUrl: string;
   avatarUrl: string;
+  branch: string;
 }
 
 @Schema()

packages/backend/jobs-manager/service/src/modules/database/custom-jobs/job-reader.ts

@@ -24,24 +24,6 @@ async function exists(fs: FsPromisesApi, path: string) {
 export class JobReader {
   private logger = new Logger(JobReader.name);
 
-  public async validatePath(
-    path: string,
-    fileName: string,
-    fs?: FsPromisesApi,
-  ): Promise<boolean> {
-    fs ??= realFs.promises;
-
-    const fileSplit = fileName.split('.');
-    if (fileSplit.length <= 1) return false;
-
-    const ext = fileSplit[fileSplit.length - 1].toLowerCase();
-    if (ext !== 'yml' && ext !== 'yaml') return false;
-
-    if (!(await exists(fs, path + fileName))) return false;
-
-    return true;
-  }
-
   /** Reads a job configuration using the given file system. */
   public async readJob(
     jobsPath: string,
@@ -122,10 +104,11 @@ export class JobReader {
           type: dataSource.type,
           avatarUrl: dataSource.avatarUrl,
           repoUrl: dataSource.repoUrl,
+          branch: dataSource.branch,
         },
       };
 
-      if (await exists(fs, handlerPath)) {
+      if (handlerPath && (await exists(fs, handlerPath))) {
         job.findingHandler = (await fs.readFile(handlerPath)).toString();
         job.findingHandlerEnabled = true;
         job.findingHandlerLanguage = jobMetadata.findingHandlerLanguage;

packages/backend/jobs-manager/service/src/modules/database/custom-jobs/jobs.source.ts

@@ -11,6 +11,7 @@ export interface JobSourceConfig {
     username: string;
     password: string;
   };
+  branch: string;
 }
 
 export interface JobSource {
@@ -30,6 +31,7 @@ export class GitJobSource implements JobSource {
     includeTemplates: boolean = false,
   ): Promise<CustomJobEntry[]> {
     const jobs = await this.listJobs('/jobs');
+
     if (includeTemplates) {
       jobs.push(...(await this.listJobs('/job-templates')));
     }

packages/backend/jobs-manager/service/src/modules/database/subscriptions/cron-subscriptions/cron-subscriptions.model.ts

@@ -27,6 +27,7 @@ export class CronSubscriptionSource {
   type: 'git';
   url: string;
   avatarUrl: string;
+  branch: string;
 }
 
 @Schema()

packages/backend/jobs-manager/service/src/modules/database/subscriptions/event-subscriptions/event-subscriptions.model.ts

@@ -26,6 +26,7 @@ export class EventSubscriptionSource {
   type: 'git';
   url: string;
   avatarUrl: string;
+  branch: string;
 }
 
 @Schema()

packages/backend/jobs-manager/service/src/modules/database/subscriptions/subscriptions.utils.ts

@@ -427,6 +427,7 @@ export class SubscriptionsUtils {
       type: dataSource.type,
       avatarUrl: dataSource.avatarUrl,
       url: dataSource.repoUrl,
+      branch: dataSource.branch,
     };
 
     switch (yaml.triggerType) {

packages/backend/jobs-manager/service/src/modules/datasources/data-source.ts

@@ -10,6 +10,7 @@ export interface DataSourceConfig {
     username: string;
     password: string;
   };
+  branch?: string;
 }
 
 export interface DataSource {
@@ -29,7 +30,7 @@ export class GitDataSource implements DataSource {
       url: this.config.url,
       dir: '/',
       onAuth: () => this.config.auth,
-      ref: 'main',
+      ref: this.config.branch ?? 'main',
       singleBranch: true,
       depth: 1,
     });

packages/backend/jobs-manager/service/src/modules/datasources/data-sources.ts

@@ -8,6 +8,7 @@ export interface DataSource {
   repoUrl: string;
   avatarUrl: string;
   fs: FsPromisesApi;
+  branch: string;
 }
 
 interface DataSourceCacheEntry extends DataSource {
@@ -53,6 +54,7 @@ export class DataSources {
       fs,
       avatarUrl: this.inferAvatar(config.url),
       repoUrl: config.url,
+      branch: config.branch,
     };
   }
 

packages/backend/jobs-manager/service/src/modules/datasources/git-data-source.ts

@@ -11,6 +11,7 @@ export interface GitDataSourceConfig {
     username: string;
     password: string;
   };
+  branch: string;
 }
 
 export class GitDataSource implements DataSource {
@@ -26,7 +27,7 @@ export class GitDataSource implements DataSource {
       url: this.config.url,
       dir: '/',
       onAuth: () => this.config.auth,
-      ref: 'main',
+      ref: this.config.branch,
       singleBranch: true,
       depth: 1,
     });