From e6d9a2b49a0d22f2e86802bcc701ccd631e93b8c Mon Sep 17 00:00:00 2001 From: sagIoTPower Date: Fri, 6 Sep 2024 19:27:51 +0200 Subject: [PATCH 01/11] review update of connector & status infos in ConnectorConfigurationComponent --- dynamic-mapping-ui/package.json | 2 +- .../service-configuration.component.ts | 1 - .../connector/broker-connector.component.ts | 80 +-------- .../mapping-connector.component.html | 11 -- .../mapping-connector.component.ts | 78 ++------- .../shared/connector-configuration.service.ts | 41 ++--- .../connector-grid.component.html | 11 ++ .../connector-grid.component.ts | 81 ++++++--- .../status-enabled-renderer.component.ts | 2 +- .../connector-status.component.html | 160 ++++++++---------- .../connector-status.component.ts | 15 +- .../src/shared/connector-status.service.ts | 39 +++-- .../src/shared/navigation.factory.ts | 2 +- pom.xml | 2 +- 14 files changed, 213 insertions(+), 312 deletions(-) diff --git a/dynamic-mapping-ui/package.json b/dynamic-mapping-ui/package.json index 263f2c88..0559d9e9 100644 --- a/dynamic-mapping-ui/package.json +++ b/dynamic-mapping-ui/package.json @@ -1,6 +1,6 @@ { "name": "dynamic-mapping", - "version": "4.4.1", + "version": "4.5.1", "author": "christof.strack@softwareag.com, stefan.witschel@softwareag.com", "description": "Cumulocity plugin to map custom JSON payloads to C8Y payloads.The plugin support both directions: inbound/outbound. Currently MQTT is supported ", "repository": { diff --git a/dynamic-mapping-ui/src/configuration/service-configuration.component.ts b/dynamic-mapping-ui/src/configuration/service-configuration.component.ts index 40f04b34..05f2b1a8 100644 --- a/dynamic-mapping-ui/src/configuration/service-configuration.component.ts +++ b/dynamic-mapping-ui/src/configuration/service-configuration.component.ts @@ -74,7 +74,6 @@ export class ServiceConfigurationComponent implements OnInit { async loadData(): Promise { this.serviceConfiguration = await this.sharedService.getServiceConfiguration(); - // this.connectorConfigurationService.startConnectorConfigurations(); } async clickedReconnect2NotificationEndpoint() { diff --git a/dynamic-mapping-ui/src/connector/broker-connector.component.ts b/dynamic-mapping-ui/src/connector/broker-connector.component.ts index 7f97282d..0ebdb2d6 100644 --- a/dynamic-mapping-ui/src/connector/broker-connector.component.ts +++ b/dynamic-mapping-ui/src/connector/broker-connector.component.ts @@ -18,27 +18,27 @@ * * @authors Christof Strack */ -import { Component, OnInit } from '@angular/core'; -import { AlertService, gettext } from '@c8y/ngx-components'; +import { Component, ViewChild } from '@angular/core'; +import { AlertService } from '@c8y/ngx-components'; import { BsModalService } from 'ngx-bootstrap/modal'; -import { from, Observable } from 'rxjs'; +import { Observable } from 'rxjs'; import packageJson from '../../package.json'; import { ConnectorConfiguration, ConnectorSpecification, ConnectorStatus, - Feature, - uuidCustom + Feature } from '../shared'; import { ConnectorConfigurationService } from '../shared/connector-configuration.service'; -import { ConfigurationConfigurationModalComponent } from '../shared'; +import { ConnectorConfigurationComponent } from '../shared/connector-configuration/connector-grid.component'; @Component({ selector: 'd11r-mapping-broker-connector', styleUrls: ['./broker-connector.component.style.css'], templateUrl: 'broker-connector.component.html' }) -export class BrokerConnectorComponent implements OnInit { +export class BrokerConnectorComponent { + @ViewChild(ConnectorConfigurationComponent) connectorGrid!: ConnectorConfigurationComponent; version: string = packageJson.version; monitoring$: Observable; feature: Feature; @@ -51,73 +51,11 @@ export class BrokerConnectorComponent implements OnInit { public alertService: AlertService ) {} - ngOnInit() { - // console.log('Running version', this.version); - from( - this.connectorConfigurationService.getConnectorSpecifications() - ).subscribe((specs) => { - this.specifications = specs; - }); - this.connectorConfigurationService - .getConnectorConfigurationsLive() - .subscribe((confs) => { - this.configurations = confs; - }); - this.loadData(); - } - refresh() { - this.connectorConfigurationService.resetCache(); - this.loadData(); - } - - loadData(): void { - this.connectorConfigurationService.startConnectorConfigurations(); + this.connectorGrid.refresh(); } async onConfigurationAdd() { - const configuration: Partial = { - properties: {}, - ident: uuidCustom() - }; - const initialState = { - add: true, - configuration: configuration, - specifications: this.specifications, - configurationsCount: this.configurations.length - }; - const modalRef = this.bsModalService.show( - ConfigurationConfigurationModalComponent, - { - initialState - } - ); - modalRef.content.closeSubject.subscribe(async (addedConfiguration) => { - // console.log('Configuration after edit:', addedConfiguration); - if (addedConfiguration) { - // avoid to include status$ - const clonedConfiguration = { - ident: addedConfiguration.ident, - connectorType: addedConfiguration.connectorType, - enabled: addedConfiguration.enabled, - name: addedConfiguration.name, - properties: addedConfiguration.properties - }; - const response = - await this.connectorConfigurationService.createConnectorConfiguration( - clonedConfiguration - ); - if (response.status < 300) { - this.alertService.success( - gettext('Added successfully configuration') - ); - } else { - this.alertService.danger( - gettext('Failed to update connector configuration') - ); - } - this.loadData(); - } - }); + this.connectorGrid.onConfigurationAdd(); } } diff --git a/dynamic-mapping-ui/src/mapping/step-connector/mapping-connector.component.html b/dynamic-mapping-ui/src/mapping/step-connector/mapping-connector.component.html index 36612005..e7948233 100644 --- a/dynamic-mapping-ui/src/mapping/step-connector/mapping-connector.component.html +++ b/dynamic-mapping-ui/src/mapping/step-connector/mapping-connector.component.html @@ -42,17 +42,6 @@ [(deploymentMapEntry)]="deploymentMapEntry" > -
- -
- - {{ findNameByIdent(ident) }} -
-
diff --git a/dynamic-mapping-ui/src/mapping/step-connector/mapping-connector.component.ts b/dynamic-mapping-ui/src/mapping/step-connector/mapping-connector.component.ts index 1338ce96..4eead8c8 100644 --- a/dynamic-mapping-ui/src/mapping/step-connector/mapping-connector.component.ts +++ b/dynamic-mapping-ui/src/mapping/step-connector/mapping-connector.component.ts @@ -25,24 +25,21 @@ import { OnDestroy, OnInit, Output, + ViewChild, ViewEncapsulation } from '@angular/core'; -import { AlertService, gettext } from '@c8y/ngx-components'; import { BehaviorSubject } from 'rxjs'; import { - ConfigurationConfigurationModalComponent, - ConnectorConfiguration, - ConnectorSpecification, DeploymentMapEntry, Direction, Feature, - StepperConfiguration, - uuidCustom + StepperConfiguration } from '../../shared'; import { EditorMode } from '../shared/stepper-model'; import { SharedService } from '../../shared/shared.service'; import { BsModalService } from 'ngx-bootstrap/modal'; import { ConnectorConfigurationService } from '../../connector'; +import { ConnectorConfigurationComponent } from '../../shared/connector-configuration/connector-grid.component'; @Component({ selector: 'd11r-mapping-connector', @@ -51,6 +48,8 @@ import { ConnectorConfigurationService } from '../../connector'; encapsulation: ViewEncapsulation.None }) export class MappingConnectorComponent implements OnInit, OnDestroy { + @ViewChild(ConnectorConfigurationComponent) + connectorGrid!: ConnectorConfigurationComponent; @Input() stepperConfiguration: StepperConfiguration; private _deploymentMapEntry: DeploymentMapEntry; @Input() @@ -68,82 +67,25 @@ export class MappingConnectorComponent implements OnInit, OnDestroy { readOnly: boolean; selectedResult$: BehaviorSubject = new BehaviorSubject(0); - specifications: ConnectorSpecification[] = []; - configurations: ConnectorConfiguration[]; constructor( - private alertService: AlertService, private sharedService: SharedService, public bsModalService: BsModalService, public connectorConfigurationService: ConnectorConfigurationService ) {} async ngOnInit() { - this.readOnly = this.stepperConfiguration.editorMode == EditorMode.READ_ONLY; + this.readOnly = + this.stepperConfiguration.editorMode == EditorMode.READ_ONLY; this.feature = await this.sharedService.getFeatures(); - this.specifications = - await this.connectorConfigurationService.getConnectorSpecifications(); - this.connectorConfigurationService - .getConnectorConfigurationsLive() - .subscribe((confs) => { - this.configurations = confs; - }); - this.loadData(); } async onConfigurationAdd() { - const configuration: Partial = { - properties: {}, - ident: uuidCustom() - }; - const initialState = { - add: true, - configuration: configuration, - specifications: this.specifications, - configurationsCount: this.configurations?.length - }; - const modalRef = this.bsModalService.show( - ConfigurationConfigurationModalComponent, - { - initialState - } - ); - modalRef.content.closeSubject.subscribe(async (addedConfiguration) => { - // console.log('Configuration after edit:', addedConfiguration); - if (addedConfiguration) { - this.configurations.push(addedConfiguration); - // avoid to include status$ - const clonedConfiguration = { - ident: addedConfiguration.ident, - connectorType: addedConfiguration.connectorType, - enabled: addedConfiguration.enabled, - name: addedConfiguration.name, - properties: addedConfiguration.properties - }; - const response = - await this.connectorConfigurationService.createConnectorConfiguration( - clonedConfiguration - ); - if (response.status < 300) { - this.alertService.success( - gettext('Added successfully configuration') - ); - } else { - this.alertService.danger( - gettext('Failed to update connector configuration') - ); - } - } - this.loadData(); - }); + this.connectorGrid.onConfigurationAdd(); } - loadData(): void { - this.connectorConfigurationService.startConnectorConfigurations(); - } - - findNameByIdent(ident: string): string { - return this.configurations?.find((conf) => conf.ident == ident)?.name; + refresh() { + this.connectorGrid.refresh(); } ngOnDestroy() { diff --git a/dynamic-mapping-ui/src/shared/connector-configuration.service.ts b/dynamic-mapping-ui/src/shared/connector-configuration.service.ts index ca9d6a13..e030afa9 100644 --- a/dynamic-mapping-ui/src/shared/connector-configuration.service.ts +++ b/dynamic-mapping-ui/src/shared/connector-configuration.service.ts @@ -41,7 +41,7 @@ import { Observable, Subject } from 'rxjs'; -import { map, switchMap } from 'rxjs/operators'; +import { map, shareReplay, switchMap, tap } from 'rxjs/operators'; import { Realtime } from '@c8y/ngx-components/api'; @Injectable({ providedIn: 'root' }) @@ -51,18 +51,19 @@ export class ConnectorConfigurationService { private sharedService: SharedService ) { this.initConnectorConfigurations(); + this.startConnectorStatusSubscriptions(); this.realtime = new Realtime(this.client); - // console.log("Constructor:BrokerConfigurationService"); } private _connectorConfigurations: ConnectorConfiguration[]; private _connectorSpecifications: ConnectorSpecification[]; private triggerConfigurations$: Subject = new Subject(); - private incomingRealtime$: Subject = new Subject(); + private realtimeConnectorStatus$: Subject = new Subject(); private connectorConfigurations$: Observable; private _agentId: string; + private initialized: boolean = false; private realtime: Realtime; private subscriptionEvents: any; @@ -77,29 +78,30 @@ export class ConnectorConfigurationService { } startConnectorConfigurations() { - this.triggerConfigurations$.next(''); - this.incomingRealtime$.next({} as any); - this.startConnectorStatusSubscriptions(); + const n = Date.now(); + if (!this.initialized) { + this.initConnectorConfigurations(); + this.startConnectorStatusSubscriptions(); + this.initialized = true; + } + this.realtimeConnectorStatus$.next({} as any); + this.triggerConfigurations$.next('start' + '/' + n); } - reloadConnectorConfigurations() { - this.triggerConfigurations$.next(''); + updateConnectorConfigurations() { + const n = Date.now(); + this.triggerConfigurations$.next('refresh' + '/' + n); } stopConnectorConfigurations() { this.realtime.unsubscribe(this.subscriptionEvents); } - refreshConnectorConfigurations() { - this.triggerConfigurations$.next(''); - } - initConnectorConfigurations() { - // console.log( - // 'Calling BrokerConfigurationService.initConnectorConfigurations()' - // ); const connectorConfig$ = this.triggerConfigurations$.pipe( - // tap(() => console.log('New triggerConfigurations!')), + // tap((state) => + // console.log('New triggerConfigurations:', state + '/' + Date.now()) + // ), switchMap(() => { const observableConfigurations = from( this.getConnectorConfigurations() @@ -120,11 +122,12 @@ export class ConnectorConfigurationService { } }); return configurations; - }) + }), + shareReplay(1) ); this.connectorConfigurations$ = combineLatest([ connectorConfig$, - this.incomingRealtime$ + this.realtimeConnectorStatus$ ]).pipe( map((vars) => { const [configurations, payload] = vars; @@ -245,6 +248,6 @@ export class ConnectorConfigurationService { private updateConnectorStatus = async (p: object) => { const payload = p['data']['data']; - this.incomingRealtime$.next(payload); + this.realtimeConnectorStatus$.next(payload); }; } diff --git a/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.html b/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.html index 229ff50f..a76528f8 100644 --- a/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.html +++ b/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.html @@ -31,3 +31,14 @@ (itemsSelect)="onSelectionChanged($event)" > +
+ +
+ + {{ findNameByIdent(ident) }} +
+
diff --git a/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.ts b/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.ts index 61d90ded..3c542d0a 100644 --- a/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.ts +++ b/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.ts @@ -67,7 +67,7 @@ import { CheckedRendererComponent } from './checked-renderer.component'; templateUrl: 'connector-grid.component.html' }) export class ConnectorConfigurationComponent - implements OnInit, OnDestroy, AfterViewInit, AfterViewChecked + implements OnInit, OnDestroy, AfterViewInit { @Input() selectable = true; @Input() readOnly = false; @@ -107,10 +107,6 @@ export class ConnectorConfigurationComponent private el: ElementRef, private renderer: Renderer2 ) {} - // eslint-disable-next-line @angular-eslint/no-empty-lifecycle-method - ngAfterViewChecked(): void { - // if (this.readOnly) this.updateCheckboxState(); - } ngAfterViewInit(): void { setTimeout(async () => { @@ -120,17 +116,6 @@ export class ConnectorConfigurationComponent }, 0); } - private updateCheckboxState() { - if (this.connectorGrid && this.el && this.el.nativeElement) { - const checkboxes = this.el.nativeElement.querySelectorAll( - 'label.c8y-checkbox > input[type="checkbox"]' - ); - checkboxes.forEach((checkbox: HTMLInputElement) => { - this.renderer.setProperty(checkbox, 'disabled', this.readOnly); - }); - } - } - ngOnInit() { // console.log('connector-configuration', this._deploymentMapEntry, this.deploymentMapEntry); @@ -242,7 +227,7 @@ export class ConnectorConfigurationComponent (conf) => (conf['checked'] = this.selected.includes(conf.ident)) ); }); - this.loadData(); + this.connectorConfigurationService.startConnectorConfigurations(); } public onSelectToggle(id: string) { @@ -278,17 +263,11 @@ export class ConnectorConfigurationComponent } refresh() { - this.connectorConfigurationService.stopConnectorConfigurations(); - this.connectorConfigurationService.resetCache(); - this.connectorConfigurationService.startConnectorConfigurations(); - } - - loadData(): void { - this.connectorConfigurationService.startConnectorConfigurations(); + this.connectorConfigurationService.updateConnectorConfigurations(); } reloadData(): void { - this.connectorConfigurationService.reloadConnectorConfigurations(); + this.connectorConfigurationService.updateConnectorConfigurations(); } async onConfigurationUpdate(config: ConnectorConfiguration) { @@ -429,6 +408,58 @@ export class ConnectorConfigurationComponent ); } + async onConfigurationAdd() { + const configuration: Partial = { + properties: {}, + ident: uuidCustom() + }; + const initialState = { + add: true, + configuration: configuration, + specifications: this.specifications, + configurationsCount: this.configurations?.length + }; + const modalRef = this.bsModalService.show( + ConfigurationConfigurationModalComponent, + { + initialState + } + ); + modalRef.content.closeSubject.subscribe(async (addedConfiguration) => { + // console.log('Configuration after edit:', addedConfiguration); + if (addedConfiguration) { + this.configurations.push(addedConfiguration); + // avoid to include status$ + const clonedConfiguration = { + ident: addedConfiguration.ident, + connectorType: addedConfiguration.connectorType, + enabled: addedConfiguration.enabled, + name: addedConfiguration.name, + properties: addedConfiguration.properties + }; + const response = + await this.connectorConfigurationService.createConnectorConfiguration( + clonedConfiguration + ); + if (response.status < 300) { + this.alertService.success( + gettext('Added successfully configuration') + ); + } else { + this.alertService.danger( + gettext('Failed to update connector configuration') + ); + } + } + this.refresh(); + }); + } + + + findNameByIdent(ident: string): string { + return this.configurations?.find((conf) => conf.ident == ident)?.name; + } + ngOnDestroy(): void { this.connectorConfigurationService.stopConnectorConfigurations(); } diff --git a/dynamic-mapping-ui/src/shared/connector-configuration/status-enabled-renderer.component.ts b/dynamic-mapping-ui/src/shared/connector-configuration/status-enabled-renderer.component.ts index 0e520540..a7022665 100644 --- a/dynamic-mapping-ui/src/shared/connector-configuration/status-enabled-renderer.component.ts +++ b/dynamic-mapping-ui/src/shared/connector-configuration/status-enabled-renderer.component.ts @@ -79,6 +79,6 @@ export class StatusEnabledRendererComponent { this.sharedService.refreshMappings(Direction.OUTBOUND); } reloadData(): void { - this.connectorConfigurationService.reloadConnectorConfigurations(); + this.connectorConfigurationService.updateConnectorConfigurations(); } } diff --git a/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.html b/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.html index d1b33d6f..69bd8188 100644 --- a/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.html +++ b/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.html @@ -19,93 +19,83 @@ ~ @authors Christof Strack --> -
+
Connector Logs
- -
-
- -
-
- + + + + + +
+
+ -
- -
- -
- -
-
- - {{ event.date | date: 'dd.MM.yy hh:mm:ss' }} - -
-
- -
-
-
-
- - {{ event.status }} - - {{ event.connectorName }} -
- {{ event.message }} -
-
-
-
-
-
-
+ {{ t.name }} + + +
+ + + +
+
+
+ + {{ event.date | date: 'dd.MM.yy hh:mm:ss' }} + +
+
+ +
+
+
+
+ + {{ event.status }} + + {{ event.connectorName }} +
+ {{ event.message }} +
+
+
+
+
+
+
diff --git a/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.ts b/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.ts index 9b0ac92f..7b4e845c 100644 --- a/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.ts +++ b/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.ts @@ -65,22 +65,15 @@ export class ConnectorStatusComponent implements OnInit, OnDestroy { async ngOnInit() { // console.log('Running version', this.version); this.feature = await this.sharedService.getFeatures(); + this.statusLogs$ = this.connectorStatusService.getStatusLogs(); + await this.connectorStatusService.startConnectorStatusLogs(); + this.connectorConfigurationService .getConnectorConfigurationsLive() .subscribe((confs) => { this.configurations = confs; }); - // if (!this.feature.userHasMappingAdminRole) { - // this.alertService.warning( - // "The configuration on this tab is not editable, as you don't have Mapping ADMIN permissions. Please assign Mapping ADMIN permissions to your user." - // ); - // } - - this.connectorStatusService.getStatusLogs()?.subscribe((logs) => { - this.statusLogs = logs; - }); - this.connectorStatusService.startConnectorStatusLogs(); - this.statusLogs$ = this.connectorStatusService.getStatusLogs(); + await this.connectorConfigurationService.startConnectorConfigurations(); } updateStatusLogs() { diff --git a/dynamic-mapping-ui/src/shared/connector-status.service.ts b/dynamic-mapping-ui/src/shared/connector-status.service.ts index 23e73341..a324a0a2 100644 --- a/dynamic-mapping-ui/src/shared/connector-status.service.ts +++ b/dynamic-mapping-ui/src/shared/connector-status.service.ts @@ -29,7 +29,7 @@ import { } from '../shared'; import { merge, Observable, Subject } from 'rxjs'; -import { filter, map, scan, switchMap } from 'rxjs/operators'; +import { filter, map, scan, share, switchMap, tap } from 'rxjs/operators'; @Injectable({ providedIn: 'root' }) export class ConnectorStatusService { @@ -39,11 +39,10 @@ export class ConnectorStatusService { private sharedService: SharedService ) { this.realtime = new Realtime(this.client); - this.initConnectorLogsRealtime(); - // console.log("Constructor:BrokerConfigurationService"); } private _agentId: string; + private initialized: boolean = false; private realtime: Realtime; private subscriptionEvents: any; private filterStatusLog = { @@ -52,17 +51,21 @@ export class ConnectorStatusService { connectorIdent: 'ALL' }; private triggerLogs$: Subject = new Subject(); - private incomingRealtime$: Subject = new Subject(); - private statusLogs$: Observable; + private realtimeConnectorStatus$: Subject = new Subject(); + private statusLogs$: Subject = new Subject(); getStatusLogs(): Observable { return this.statusLogs$; } - startConnectorStatusLogs() { - this.startConnectorStatusSubscriptions(); - this.triggerLogs$.next([{ type: 'reset' }]); - this.incomingRealtime$.next({} as any); + async startConnectorStatusLogs() { + // console.log('Calling: startConnectorStatusLogs'); + if (!this.initialized) { + this.startConnectorStatusSubscriptions(); + await this.initConnectorLogsRealtime(); + this.initialized = true; + } + this.triggerLogs$.next([{ type: 'reset' }]); } updateStatusLogs(filter: any) { @@ -78,10 +81,10 @@ export class ConnectorStatusService { if (!this._agentId) { this._agentId = await this.sharedService.getDynamicMappingServiceAgent(); } - // console.log( - // 'Calling: BrokerConfigurationService.initConnectorLogsRealtime()', - // this._agentId - // ); + console.log( + 'Calling: BrokerConfigurationService.initConnectorLogsRealtime()', + this._agentId + ); const sourceList$ = this.triggerLogs$.pipe( // tap((x) => console.log('TriggerLogs In', x)), switchMap(() => { @@ -110,11 +113,12 @@ export class ConnectorStatusService { ? true : event.connectorIdent == this.filterStatusLog.connectorIdent; }) - ) + ), + share() // tap((x) => console.log('TriggerLogs Out', x)) ); - const sourceRealtime$ = this.incomingRealtime$.pipe( + const sourceRealtime$ = this.realtimeConnectorStatus$.pipe( // tap((x) => console.log('IncomingRealtime In', x)), filter((event) => { return ( @@ -134,7 +138,7 @@ export class ConnectorStatusService { }) ); - this.statusLogs$ = merge( + const o: Observable = merge( sourceList$, sourceRealtime$, this.triggerLogs$ @@ -152,6 +156,7 @@ export class ConnectorStatusService { return sortedAcc; }, []) ); + o.subscribe((logs) => this.statusLogs$.next(logs)); } async startConnectorStatusSubscriptions(): Promise { @@ -169,7 +174,7 @@ export class ConnectorStatusService { private updateConnectorStatus = async (p: object) => { const payload = p['data']['data']; - this.incomingRealtime$.next(payload); + this.realtimeConnectorStatus$.next(payload); }; async getConnectorStatus(): Promise { diff --git a/dynamic-mapping-ui/src/shared/navigation.factory.ts b/dynamic-mapping-ui/src/shared/navigation.factory.ts index cbfe047f..70981459 100644 --- a/dynamic-mapping-ui/src/shared/navigation.factory.ts +++ b/dynamic-mapping-ui/src/shared/navigation.factory.ts @@ -48,7 +48,7 @@ export class MappingNavigationFactory implements NavigatorNodeFactory { public router: Router ) { this.appStateService.currentApplication.subscribe((cur) => { - console.log('AppName in MappingNavigationFactory', cur, this.router.url); + // console.log('AppName in MappingNavigationFactory', cur, this.router.url); if (_.has(cur?.manifest , 'exports')) { this.isStandaloneApp = true; diff --git a/pom.xml b/pom.xml index aecf9a13..0181858d 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ UTF-8 - 4.4.1 + 4.5.1-SNAPSHOT 1020.107.0 1.18.34 1.7.36 From c1826aebd15ee7c1c7b540836a6d943cd80ed1e7 Mon Sep 17 00:00:00 2001 From: sagIoTPower Date: Sun, 8 Sep 2024 09:18:02 +0200 Subject: [PATCH 02/11] fixing position of textarea source result, target result --- .../src/landing/landing.component.html | 42 ++++++++-------- .../src/mapping/mapping.module.ts | 9 ++-- .../mapping-properties.component.html | 0 .../mapping-properties.component.ts | 0 .../mapping-stepper.component.html | 48 +++++++++---------- 5 files changed, 52 insertions(+), 47 deletions(-) rename dynamic-mapping-ui/src/mapping/{step-topic => step-property}/mapping-properties.component.html (100%) rename dynamic-mapping-ui/src/mapping/{step-topic => step-property}/mapping-properties.component.ts (100%) diff --git a/dynamic-mapping-ui/src/landing/landing.component.html b/dynamic-mapping-ui/src/landing/landing.component.html index 6b7f3ccd..449dd819 100644 --- a/dynamic-mapping-ui/src/landing/landing.component.html +++ b/dynamic-mapping-ui/src/landing/landing.component.html @@ -28,9 +28,11 @@

Using the Cumulocity Dynamic Data Mapper you are able to connect to almost - any message broker.
- And map any payload received on a topic dynamically to the Cumulocity IoT - Domain Model in a graphical editor. + any message broker and map any payload to Cumulocity format.
+ To do so you define a mapping in a intuitive graphical editor. During + operation, your custom payload is automatically converted
+ to align with the Cumulocity IoT Domain Model, ensuring seamless + integration and data flow.
The following links help you get started with the Dynamic Data Mapper:
@@ -55,7 +57,8 @@

connectors and mappings.
If you want to receive messages from an Mqtt broker (Cumulocity Mqtt - Service, Hive MQ, mosquitto, ...) you use an inbound mapping.
+ Service, Hive MQ, mosquitto, ...) you use an inbound mapping. +
The rules of the mapping are applied and result in an transformed payload for any of the >

- For the other direction you can define an outbound mapping. At runtime it - listens for changes of any of the core + For the other direction you can define an outbound mapping. At + runtime it listens for changes of any of the core domain objects. @@ -89,7 +92,7 @@

- {{ 'How to define a mapping ...' | translate }} + {{ 'How to define a mapping?' | translate }}

@@ -144,7 +147,7 @@

- {{ 'How to define a substitution ...' | translate }} + {{ 'How to define a substitution?' | translate }}

@@ -187,22 +190,21 @@

- {{ - "How to you start when you don't know what your source payload looks like ..." - | translate - }} + {{ 'How to begin without knowing your source payload?' | translate }}

- If you don't know the payload of your device you start by snooping the - payload as a first step.
- Snooping will record the messages and you can use these in the next step - as source templates .
- When you select a mapping type you can enable snooping - Enable snooping for this mapping.
- You start the stepper by clicking on Add mapping (Mapping -> - Inbound Mapping -> Action Add mapping).
+ When dealing with an unknown device payload, the initial step is to + capture and analyze the incoming data.
+ This process, called snooping, records messages that can serve as source + templates for future mapping.
+ To activate snooping for a specific mapping, select the + Enable snooping for this mapping option when choosing a mapping + type.
. You start the stepper by clicking on the actionAdd mapping + (Mapping -> Inbound Mapping -> Action Add mapping).

The stepper when snooping leads you through the two steps:
diff --git a/dynamic-mapping-ui/src/mapping/mapping.module.ts b/dynamic-mapping-ui/src/mapping/mapping.module.ts index 35addf02..f5399cbb 100644 --- a/dynamic-mapping-ui/src/mapping/mapping.module.ts +++ b/dynamic-mapping-ui/src/mapping/mapping.module.ts @@ -46,7 +46,7 @@ import { FieldInputCustom } from './shared/formly/input-custom.type.component'; import { MessageField } from './shared/formly/message.type.component'; import { MappingStepperComponent } from './stepper-mapping/mapping-stepper.component'; import { SubstitutionRendererComponent } from './substitution/substitution-grid.component'; -import { MappingStepPropertiesComponent } from './step-topic/mapping-properties.component'; +import { MappingStepPropertiesComponent } from './step-property/mapping-properties.component'; import { MappingStepTestingComponent } from './step-testing/mapping-testing.component'; import { MappingSubscriptionComponent } from './subscription/mapping-subscription.component'; import { WrapperCustomFormField } from './shared/formly/custom-form-field.wrapper.component'; @@ -55,7 +55,10 @@ import { SnoopingStepperComponent } from './stepper-snooping/snooping-stepper.co import { MappingConnectorComponent } from './step-connector/mapping-connector.component'; import { FORMLY_CONFIG } from '@ngx-formly/core'; import { FieldTextareaCustom } from './shared/formly/textarea.type.component'; -import { checkTopicsOutboundAreValid, checkTopicsInboundAreValid } from './shared/util'; +import { + checkTopicsOutboundAreValid, + checkTopicsInboundAreValid +} from './shared/util'; import { NODE1 } from '../shared/model/util'; @NgModule({ @@ -86,7 +89,7 @@ import { NODE1 } from '../shared/model/util'; ], imports: [ CoreModule, - CommonModule, + CommonModule, AssetSelectorModule, PopoverModule, DynamicFormsModule, diff --git a/dynamic-mapping-ui/src/mapping/step-topic/mapping-properties.component.html b/dynamic-mapping-ui/src/mapping/step-property/mapping-properties.component.html similarity index 100% rename from dynamic-mapping-ui/src/mapping/step-topic/mapping-properties.component.html rename to dynamic-mapping-ui/src/mapping/step-property/mapping-properties.component.html diff --git a/dynamic-mapping-ui/src/mapping/step-topic/mapping-properties.component.ts b/dynamic-mapping-ui/src/mapping/step-property/mapping-properties.component.ts similarity index 100% rename from dynamic-mapping-ui/src/mapping/step-topic/mapping-properties.component.ts rename to dynamic-mapping-ui/src/mapping/step-property/mapping-properties.component.ts diff --git a/dynamic-mapping-ui/src/mapping/stepper-mapping/mapping-stepper.component.html b/dynamic-mapping-ui/src/mapping/stepper-mapping/mapping-stepper.component.html index d5e16dd7..10880010 100644 --- a/dynamic-mapping-ui/src/mapping/stepper-mapping/mapping-stepper.component.html +++ b/dynamic-mapping-ui/src/mapping/stepper-mapping/mapping-stepper.component.html @@ -589,8 +589,8 @@

>
-
-
-
-
- - -
-
+
+
+ + +
+

From eb117e368313a442ddde10222fb39af532f4c0f1 Mon Sep 17 00:00:00 2001 From: sagIoTPower Date: Sun, 8 Sep 2024 11:58:21 +0200 Subject: [PATCH 03/11] harmonized names of observable in ConnectorConfigurationService --- .../shared/connector-configuration.service.ts | 23 +++--- .../connector-grid.component.ts | 5 +- .../connector-status.component.ts | 2 +- .../src/shared/connector-status.service.ts | 72 ++++++++++--------- 4 files changed, 54 insertions(+), 48 deletions(-) diff --git a/dynamic-mapping-ui/src/shared/connector-configuration.service.ts b/dynamic-mapping-ui/src/shared/connector-configuration.service.ts index e030afa9..e13e0baa 100644 --- a/dynamic-mapping-ui/src/shared/connector-configuration.service.ts +++ b/dynamic-mapping-ui/src/shared/connector-configuration.service.ts @@ -60,15 +60,18 @@ export class ConnectorConfigurationService { private triggerConfigurations$: Subject = new Subject(); private realtimeConnectorStatus$: Subject = new Subject(); - private connectorConfigurations$: Observable; + private realtimeConnectorConfigurations$: Observable< + ConnectorConfiguration[] + >; + private enrichedConnectorConfiguration$: Observable; private _agentId: string; private initialized: boolean = false; private realtime: Realtime; private subscriptionEvents: any; - getConnectorConfigurationsLive(): Observable { - return this.connectorConfigurations$; + getRealtimeConnectorConfigurations(): Observable { + return this.realtimeConnectorConfigurations$; } resetCache() { @@ -84,8 +87,8 @@ export class ConnectorConfigurationService { this.startConnectorStatusSubscriptions(); this.initialized = true; } - this.realtimeConnectorStatus$.next({} as any); - this.triggerConfigurations$.next('start' + '/' + n); + this.realtimeConnectorStatus$.next({} as any); + this.triggerConfigurations$.next('start' + '/' + n); } updateConnectorConfigurations() { @@ -98,7 +101,7 @@ export class ConnectorConfigurationService { } initConnectorConfigurations() { - const connectorConfig$ = this.triggerConfigurations$.pipe( + this.enrichedConnectorConfiguration$ = this.triggerConfigurations$.pipe( // tap((state) => // console.log('New triggerConfigurations:', state + '/' + Date.now()) // ), @@ -125,8 +128,8 @@ export class ConnectorConfigurationService { }), shareReplay(1) ); - this.connectorConfigurations$ = combineLatest([ - connectorConfig$, + this.realtimeConnectorConfigurations$ = combineLatest([ + this.enrichedConnectorConfiguration$, this.realtimeConnectorStatus$ ]).pipe( map((vars) => { @@ -242,11 +245,11 @@ export class ConnectorConfigurationService { // subscribe to event stream this.subscriptionEvents = this.realtime.subscribe( `/events/${this._agentId}`, - this.updateConnectorStatus + this.updateRealtimeConnectorStatus ); } - private updateConnectorStatus = async (p: object) => { + private updateRealtimeConnectorStatus = async (p: object) => { const payload = p['data']['data']; this.realtimeConnectorStatus$.next(payload); }; diff --git a/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.ts b/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.ts index 3c542d0a..0d8cd662 100644 --- a/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.ts +++ b/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.ts @@ -217,7 +217,7 @@ export class ConnectorConfigurationComponent }); this.connectorConfigurationService - .getConnectorConfigurationsLive() + .getRealtimeConnectorConfigurations() .subscribe((confs) => this.configurations$.next(confs)); this.configurations$.subscribe((confs) => { @@ -227,7 +227,7 @@ export class ConnectorConfigurationComponent (conf) => (conf['checked'] = this.selected.includes(conf.ident)) ); }); - this.connectorConfigurationService.startConnectorConfigurations(); + this.connectorConfigurationService.startConnectorConfigurations(); } public onSelectToggle(id: string) { @@ -455,7 +455,6 @@ export class ConnectorConfigurationComponent }); } - findNameByIdent(ident: string): string { return this.configurations?.find((conf) => conf.ident == ident)?.name; } diff --git a/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.ts b/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.ts index 7b4e845c..15a79ddc 100644 --- a/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.ts +++ b/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.ts @@ -69,7 +69,7 @@ export class ConnectorStatusComponent implements OnInit, OnDestroy { await this.connectorStatusService.startConnectorStatusLogs(); this.connectorConfigurationService - .getConnectorConfigurationsLive() + .getRealtimeConnectorConfigurations() .subscribe((confs) => { this.configurations = confs; }); diff --git a/dynamic-mapping-ui/src/shared/connector-status.service.ts b/dynamic-mapping-ui/src/shared/connector-status.service.ts index a324a0a2..608b4462 100644 --- a/dynamic-mapping-ui/src/shared/connector-status.service.ts +++ b/dynamic-mapping-ui/src/shared/connector-status.service.ts @@ -60,12 +60,12 @@ export class ConnectorStatusService { async startConnectorStatusLogs() { // console.log('Calling: startConnectorStatusLogs'); - if (!this.initialized) { - this.startConnectorStatusSubscriptions(); - await this.initConnectorLogsRealtime(); - this.initialized = true; - } - this.triggerLogs$.next([{ type: 'reset' }]); + if (!this.initialized) { + this.startConnectorStatusSubscriptions(); + await this.initConnectorLogsRealtime(); + this.initialized = true; + } + this.triggerLogs$.next([{ type: 'reset' }]); } updateStatusLogs(filter: any) { @@ -81,11 +81,11 @@ export class ConnectorStatusService { if (!this._agentId) { this._agentId = await this.sharedService.getDynamicMappingServiceAgent(); } - console.log( - 'Calling: BrokerConfigurationService.initConnectorLogsRealtime()', - this._agentId - ); - const sourceList$ = this.triggerLogs$.pipe( + // console.log( + // 'Calling: BrokerConfigurationService.initConnectorLogsRealtime()', + // this._agentId + // ); + const filteredConnectorStatus$ = this.triggerLogs$.pipe( // tap((x) => console.log('TriggerLogs In', x)), switchMap(() => { const filter = { @@ -114,11 +114,11 @@ export class ConnectorStatusService { : event.connectorIdent == this.filterStatusLog.connectorIdent; }) ), - share() - // tap((x) => console.log('TriggerLogs Out', x)) + share(), + tap((x) => console.log('TriggerLogs Out', x)) ); - const sourceRealtime$ = this.realtimeConnectorStatus$.pipe( + const realtimeConnectorStatusRealtime$ = this.realtimeConnectorStatus$.pipe( // tap((x) => console.log('IncomingRealtime In', x)), filter((event) => { return ( @@ -138,25 +138,29 @@ export class ConnectorStatusService { }) ); - const o: Observable = merge( - sourceList$, - sourceRealtime$, + // const refreshedConnectorStatus$: Observable = + merge( + filteredConnectorStatus$, + realtimeConnectorStatusRealtime$, this.triggerLogs$ - ).pipe( - // tap((i) => console.log('Items', i)), - scan((acc, val) => { - let sortedAcc; - if (val[0]?.type == 'reset') { - // console.log('Reset loaded logs!'); - sortedAcc = []; - } else { - sortedAcc = val.concat(acc); - } - sortedAcc = sortedAcc.slice(0, 9); - return sortedAcc; - }, []) - ); - o.subscribe((logs) => this.statusLogs$.next(logs)); + ) + .pipe( + // tap((i) => console.log('Items', i)), + scan((acc, val) => { + let sortedAcc; + if (val[0]?.type == 'reset') { + // console.log('Reset loaded logs!'); + sortedAcc = []; + } else { + sortedAcc = val.concat(acc); + } + sortedAcc = sortedAcc.slice(0, 9); + return sortedAcc; + }, []), + tap((logs) => this.statusLogs$.next(logs)) + ) + .subscribe(); + // refreshedConnectorStatus$.subscribe((logs) => this.statusLogs$.next(logs)); } async startConnectorStatusSubscriptions(): Promise { @@ -168,11 +172,11 @@ export class ConnectorStatusService { // subscribe to event stream this.subscriptionEvents = this.realtime.subscribe( `/events/${this._agentId}`, - this.updateConnectorStatus + this.updateRealtimeConnectorStatus ); } - private updateConnectorStatus = async (p: object) => { + private updateRealtimeConnectorStatus = async (p: object) => { const payload = p['data']['data']; this.realtimeConnectorStatus$.next(payload); }; From 4de439a36df001b7c76dde201d05b53ffd890c08 Mon Sep 17 00:00:00 2001 From: sagIoTPower Date: Sun, 8 Sep 2024 12:40:03 +0200 Subject: [PATCH 04/11] mention limitation for issue #230 --- ARCHITECTURE.md | 28 +- INSTALLATION.md | 17 +- LIMITATIONS.md | 15 +- README.md | 45 +- .../java/dynamic/mapping/core/C8YAgent.java | 1164 +++++++++-------- 5 files changed, 642 insertions(+), 627 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index bd85166f..b785c518 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,11 +1,12 @@ # Architecture + The solution is composed of two major components: -* A **microservice** - exposes REST endpoints, provides a generic connector interface which can be used by broker clients to +- A **microservice** - exposes REST endpoints, provides a generic connector interface which can be used by broker clients to connect to a message broker, a generic data mapper, a comprehensive expression language for data mapping and the [Cumulocity Microservice SDK](https://cumulocity.com/guides/microservice-sdk/introduction/) to connect to Cumulocity. It also supports multi tenancy. -* A **frontend (plugin)** - uses the exposed endpoints of the microservice to configure a broker connection & to perform +- A **frontend (plugin)** - uses the exposed endpoints of the microservice to configure a broker connection & to perform graphical data mappings within the Cumumlocity IoT UI. The architecture of the components consists of the following components: @@ -16,22 +17,23 @@ The architecture of the components consists of the following components:
The light blue components are part of this project which are: -* three default connectors for.. - * **MQTT client** - using [hivemq-mqtt-client](https://github.com/hivemq/hivemq-mqtt-client) to connect and subscribe to MQTT brokers - * **MQTT Service client** - using hivemq-mqtt-client to connect to MQTT Service - * **Kafka connector** - to connect to Kafka brokers -* **Data mapper** - handling of received messages via connector and mapping them to a target data format for Cumulocity IoT. +- three default connectors for.. + - **MQTT client** - using [hivemq-mqtt-client](https://github.com/hivemq/hivemq-mqtt-client) to connect and subscribe to MQTT brokers + - **MQTT Service client** - using hivemq-mqtt-client to connect to MQTT Service + - **Kafka connector** - to connect to Kafka brokers +- **Data mapper** - handling of received messages via connector and mapping them to a target data format for Cumulocity IoT. Also includes an expression runtime [JSONata](https://jsonata.org) to execute expressions -* **C8Y client** - implements part of the Cumulocity IoT REST API to integrate data -* **REST endpoints** - custom endpoints which are used by the MQTT Frontend or can be used to add mappings programmatically -* **Mapper frontend** - A plugin for Cumulocity IoT to provide an UI for MQTT Configuration & Data Mapping +- **C8Y client** - implements part of the Cumulocity IoT REST API to integrate data +- **REST endpoints** - custom endpoints which are used by the MQTT Frontend or can be used to add mappings programmatically +- **Mapper frontend** - A plugin for Cumulocity IoT to provide an UI for MQTT Configuration & Data Mapping -> **Please Note:** When using MQTT or any other Message Broker beside MQTT Service you need to provide this broker available yourself to use the Dynamic Mapper. +> **Please Note:** When using MQTT or any other Message Broker beside MQTT Service you need to provide this broker available yourself to use the Dynamic Data Mapper. The mapper processes messages in both directions: + 1. `INBOUND`: from Message Broker to C8Y 2. `OUTBOUND`: from C8Y to Message Broker -The Dynamic Mapper can be deployed as a **multi tenant microservice** which means you can deploy it once in your enterprise tenant and subscribe additional tenants using the same hardware resources. +The Dynamic Data Mapper can be deployed as a **multi tenant microservice** which means you can deploy it once in your enterprise tenant and subscribe additional tenants using the same hardware resources. It is also implemented to support **multiple broker connections** at the same time. So you can combine multiple message brokers sharing the same mappings. -This implies of course that all of them use the same topic structure and payload otherwise the mappings will fail. \ No newline at end of file +This implies of course that all of them use the same topic structure and payload otherwise the mappings will fail. diff --git a/INSTALLATION.md b/INSTALLATION.md index 38194db5..28bd35ca 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -3,8 +3,9 @@ ## Prerequisites To use the mapper you must have the following: -* a Cumulocity Tenant with **microservice** feature subscribed (if not available please ask your contact or support) -* Admin privileges for your user in the tenant + +- a Cumulocity Tenant with **microservice** feature subscribed (if not available please ask your contact or support) +- Admin privileges for your user in the tenant ## Installation @@ -17,6 +18,7 @@ Both are provided as binaries in [releases](https://github.com/SoftwareAG/cumulo Download the binaries from the latest release and upload them to your Cumulocity IoT Tenant. ## Permissions + The solution defines one role:`ROLE_MAPPING_ADMIN` that must be assigned to the user accessing the Dynamic Mapping app. ### Microservice @@ -26,21 +28,20 @@ In your Enterprise Tenant or Tenant navigate to "Administration" App, go to "Eco Select the `dynamic-mapping-service.zip`. Make sure that you subscribe the microservice to your tenant when prompted - ### Web app The frontend can be used in two variants in your tenant: + 1. As a **UI Plugin** to extend existing applications 2. As a **Blueprint** standalone Application selectable from the App switcher #### Community store (Preferred) - - The Web App is part of the community store and should be available directly in your tenant under "Administration" -> "Ecosystem" -> "Extensions" -> "Dynamic-mapping". Here you have the choice to install it as a plugin or as a blueprint app. ##### Plugin + > **_NOTE:_** For a plugin we need to clone the Administration app to add the plugin to it Go to "All Applications" and click on "Add Application". Select "Duplicate existing application" and afterward "Administration". @@ -50,7 +51,7 @@ Go to "All Applications" and click on "Add Application". Select "Duplicate exist


-Now select the cloned Administration App and go to the "Plugin" Tab. Click on "Install Plugin" and select "Dynamic Mapper Widget" +Now select the cloned Administration App and go to the "Plugin" Tab. Click on "Install Plugin" and select "Dynamic Data Mapper Widget"

@@ -58,6 +59,7 @@ Now select the cloned Administration App and go to the "Plugin" Tab. Click on "I
After successfully adding the plugin you need to refresh the Administration App by pressing F5 and you should see a new navigation entry "Dynamic Mapping" +

@@ -65,6 +67,7 @@ After successfully adding the plugin you need to refresh the Administration App ##### Blueprint For the blueprint go to "Administration" -> "Ecosystem" -> "Dynamic-mapping" -> "Deploy application" +

@@ -83,4 +86,4 @@ If you made changes or your want to upload the plugin manually you can do that b 1. In "Administration" App go to "Ecosystem" -> "Packages" and click on "Add Application" on the top right. 2. Select `dynamic-mapping.zip` and wait until it is uploaded. -Follow the steps from the point above to assign the plugin to your Administration App. \ No newline at end of file +Follow the steps from the point above to assign the plugin to your Administration App. diff --git a/LIMITATIONS.md b/LIMITATIONS.md index dcefd448..79b4dd5b 100644 --- a/LIMITATIONS.md +++ b/LIMITATIONS.md @@ -3,17 +3,20 @@ As we already have a very good C8Y API coverage for mapping not all complex cases might be supported. Currently, the mappings to the following C8Y APIs are supported: -* inventory -* events -* measurements -* alarms -* operations (outbound to devices) +- inventory +- events +- measurements +- alarms +- operations (outbound to devices) A mapping is defined of mapping properties and substitutions. The substitutions are mapping rules copying date from the incoming payload to the payload in the target system. These substitutions are defined using the standard JSONata as JSONata expressions. These JSONata expressions are evaluated in two different libraries: + 1. `dynamic-mapping`: (nodejs) [npmjs JSONata](https://www.npmjs.com/package/jsonata) and 2. `dynamic-mapping-service` (java): [JSONata4Java](https://github.com/IBM/JSONata4Java) Please be aware that slight in differences in the evaluation of these expressions exist. Differences in more advanced expressions can occur. Please test your expressions before you use advanced elements. -For MQTT Service currently no wildcards topics (e.g. `topic/#` or `topic/+` ) for Inbound Mappings / Subscriptions are allowed. \ No newline at end of file +For Cumulocity MQTT Service currently no wildcards topics (e.g. `topic/#` or `topic/+` ) for Inbound Mappings / Subscriptions are allowed. + +The [java library for JSONata](https://github.com/IBM/JSONata4Java) uses the words `and`, `or ` and `in` as reserved words in their [expression language](https://github.com/IBM/JSONata4Java/issues/317), hence they can be used as key in an JSON payload, see [issue](https://github.com/SoftwareAG/cumulocity-dynamic-mapper/issues/230). diff --git a/README.md b/README.md index 61d40ce1..a9d89959 100644 --- a/README.md +++ b/README.md @@ -2,26 +2,27 @@ ## Overview -The Cumulocity Dynamic Mapper addresses the need to get **any** data provided by a message broker mapped to the Cumulocity IoT Domain model in a zero-code approach. +The Cumulocity Dynamic Data Mapper addresses the need to get **any** data provided by a message broker mapped to the Cumulocity IoT Domain model in a zero-code approach. It can connect to multiple message brokers likes **MQTT**, **MQTT Service** , **Kafka** and others, subscribes to specific topics and maps the data in a graphical editor to the domain model of Cumulocity. Per default the followings connectors are supported -* **MQTT** - any MQTT Broker -* **Cumulocity IoT MQTT Service** - Cumulocity IoT built-in MQTT Broker -* **Kafka** - Kafka Broker -Using the Cumulocity Dynamic Mapper you are able to connect to almost any message broker and map any payload on any topic dynamically to +- **MQTT** - any MQTT Broker +- **Cumulocity IoT MQTT Service** - Cumulocity IoT built-in MQTT Broker +- **Kafka** - Kafka Broker + +Using the Cumulocity Dynamic Data Mapper you are able to connect to almost any message broker and map any payload on any topic dynamically to the Cumulocity IoT Domain Model in a graphical editor. Here are the **core features** summarized: -* **Connect** to multiple message broker of your choice at the same time. -* **Map** any data to/from the Cumulocity IoT Domain Model in a graphical way. -* **Bidirectional mappings** are supported - so you can forward data to Cumulocity or subscribe on Cumulocity data and forward it to the broker -* **Transform** data with a comprehensive expression language supported by [JSONata](https://jsonata.org/) -* **Multiple payload formats** are supported, starting with **JSON**, **Protobuf**, **Binary**, **CSV**. -* **Extend** the mapper easily by using payload extensions or the provided connector interface -* Full support of **multi-tenancy** - deploy it in your enterprise tenant and subscribe it to sub-tenants. +- **Connect** to multiple message broker of your choice at the same time. +- **Map** any data to/from the Cumulocity IoT Domain Model in a graphical way. +- **Bidirectional mappings** are supported - so you can forward data to Cumulocity or subscribe on Cumulocity data and forward it to the broker +- **Transform** data with a comprehensive expression language supported by [JSONata](https://jsonata.org/) +- **Multiple payload formats** are supported, starting with **JSON**, **Protobuf**, **Binary**, **CSV**. +- **Extend** the mapper easily by using payload extensions or the provided connector interface +- Full support of **multi-tenancy** - deploy it in your enterprise tenant and subscribe it to sub-tenants.

@@ -31,11 +32,11 @@ Here are the **core features** summarized: ## Installation -Please check the [Installation Guide](/INSTALLATION.md) to find out how you can install the Dynamic Mapper. +Please check the [Installation Guide](/INSTALLATION.md) to find out how you can install the Dynamic Data Mapper. ## User Guide -Please check the [User Guide](/USERGUIDE.md) to find a comprehensive guidance how to use the Dynamic Mapper. +Please check the [User Guide](/USERGUIDE.md) to find a comprehensive guidance how to use the Dynamic Data Mapper. ## Architecture @@ -43,15 +44,15 @@ Please check the [Architecture overview](/ARCHITECTURE.md) if you are eager to u ## API -Please check the [REST API](/API.md) provided by the Dynamic Mapper. +Please check the [REST API](/API.md) provided by the Dynamic Data Mapper. ## Extensions -The Dynamic Mapper can be extended on multiple layers. Check out the [Extensions Guide](/EXTENSIONS.md) if you want to add customer mapper or connectors. +The Dynamic Data Mapper can be extended on multiple layers. Check out the [Extensions Guide](/EXTENSIONS.md) if you want to add customer mapper or connectors. ## Limitations -Please check the current [Limitations](/LIMITATIONS.md) of the Dynamic Mapper. +Please check the current [Limitations](/LIMITATIONS.md) of the Dynamic Data Mapper. ## Contribution @@ -59,26 +60,28 @@ We are always looking for additional [contribution](/CONTRIBUTING.md). ## Build & Deploy -If you want to make changes to the code or configuration check out this [Build & Deploy guide](/BUILDDEPLOY.md) +If you want to make changes to the code or configuration check out this [Build & Deploy guide](/BUILDDEPLOY.md) ## Tests & Sample Data ### Load Test -In the resource section you find a test profil [jmeter_test_01.jmx](./resources/script/performance/jmeter_test_01.jmx) using the performance tool ```jmeter``` and an extension for MQTT: [emqx/mqtt-jmeter](https://github.com/emqx/mqtt-jmeter). + +In the resource section you find a test profil [jmeter_test_01.jmx](./resources/script/performance/jmeter_test_01.jmx) using the performance tool `jmeter` and an extension for MQTT: [emqx/mqtt-jmeter](https://github.com/emqx/mqtt-jmeter). This was used to run simple loadtest. ## Setup Sample mappings A script to create sample mappings can be found [here](./resources/script/mapping/import_mappings_01.py). You have to start it as follows: + ``` #python3 resources/script/mapping/import_mappings_01.py -p -U -u -f resources/script/mapping/sampleMapping/sampleMappings_02.json ``` The mappings with inputs and substitutions are explained in the [sample document](./resources/script/mapping/sampleMapping/sampleMappings_02.html). -______________________ +--- + These tools are provided as-is and without warranty or support. They do not constitute part of the Software AG product suite. Users are free to use, fork and modify them, subject to the license agreement. While Software AG welcomes contributions, we cannot guarantee to include every contribution in the master project. Contact us at [TECHcommunity](mailto:technologycommunity@softwareag.com?subject=Github/SoftwareAG) if you have any questions. - diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/C8YAgent.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/C8YAgent.java index bfb4e4bd..f877e0e9 100644 --- a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/C8YAgent.java +++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/C8YAgent.java @@ -87,586 +87,590 @@ @Component public class C8YAgent implements ImportBeanDefinitionRegistrar { - ConnectorStatus previousConnectorStatus = ConnectorStatus.UNKNOWN; + ConnectorStatus previousConnectorStatus = ConnectorStatus.UNKNOWN; - @Autowired - private EventApi eventApi; - - @Autowired - private InventoryFacade inventoryApi; - - @Autowired - private BinariesApi binaryApi; - - @Autowired - private IdentityFacade identityApi; - - @Autowired - private MeasurementApi measurementApi; - - @Autowired - private AlarmApi alarmApi; - - @Autowired - private DeviceControlApi deviceControlApi; - - @Autowired - private Platform platform; - - @Autowired - private MicroserviceSubscriptionsService subscriptionsService; - - @Autowired - private ContextService contextService; - - private ExtensionsComponent extensionsComponent; - - @Autowired - public void setExtensionsComponent(ExtensionsComponent extensionsComponent) { - this.extensionsComponent = extensionsComponent; - } - - @Getter - private ConfigurationRegistry configurationRegistry; - - @Autowired - public void setConfigurationRegistry(@Lazy ConfigurationRegistry configurationRegistry) { - this.configurationRegistry = configurationRegistry; - } - - private JSONParser jsonParser = JSONBase.getJSONParser(); - - public static final String MAPPING_FRAGMENT = "d11r_mapping"; - - public static final String CONNECTOR_FRAGMENT = "d11r_connector"; - public static final String DEPLOYMENT_MAP_FRAGMENT = "d11r_deploymentMap"; - - public static final String STATUS_SUBSCRIPTION_EVENT_TYPE = "d11r_subscriptionEvent"; - public static final String STATUS_CONNECTOR_EVENT_TYPE = "d11r_connectorStatusEvent"; - public static final String STATUS_MAPPING_CHANGED_EVENT_TYPE = "d11r_mappingChangedEvent"; - public static final String STATUS_NOTIFICATION_EVENT_TYPE = "d11r_notificationStatusEvent"; - - private static final String EXTENSION_INTERNAL_FILE = "extension-internal.properties"; - private static final String EXTENSION_EXTERNAL_FILE = "extension-external.properties"; - - private static final String C8Y_NOTIFICATION_CONNECTOR = "C8YNotificationConnector"; - - private static final String PACKAGE_MAPPING_PROCESSOR_EXTENSION_EXTERNAL = "dynamic.mapping.processor.extension.external"; - - @Value("${application.version}") - private String version; - - public ExternalIDRepresentation resolveExternalId2GlobalId(String tenant, ID identity, - ProcessingContext context) { - if (identity.getType() == null) { - identity.setType("c8y_Serial"); - } - ExternalIDRepresentation result = subscriptionsService.callForTenant(tenant, () -> { - try { - return identityApi.resolveExternalId2GlobalId(identity, context); - } catch (SDKException e) { - log.warn("Tenant {} - External ID {} not found", tenant, identity.getValue()); - } - return null; - }); - return result; - } - - public ExternalIDRepresentation resolveGlobalId2ExternalId(String tenant, GId gid, String idType, - ProcessingContext context) { - if (idType == null) { - idType = "c8y_Serial"; - } - final String idt = idType; - ExternalIDRepresentation result = subscriptionsService.callForTenant(tenant, () -> { - try { - return identityApi.resolveGlobalId2ExternalId(gid, idt, context); - } catch (SDKException e) { - log.warn("Tenant {} - External ID type {} for {} not found", tenant, idt, gid.getValue()); - } - return null; - }); - return result; - } - - public MeasurementRepresentation createMeasurement(String name, String type, ManagedObjectRepresentation mor, - DateTime dateTime, HashMap mvMap, String tenant) { - MeasurementRepresentation measurementRepresentation = new MeasurementRepresentation(); - subscriptionsService.runForTenant(tenant, () -> { - MicroserviceCredentials context = removeAppKeyHeaderFromContext(contextService.getContext()); - contextService.runWithinContext(context, () -> { - try { - measurementRepresentation.set(mvMap, name); - measurementRepresentation.setType(type); - measurementRepresentation.setSource(mor); - measurementRepresentation.setDateTime(dateTime); - log.debug("Tenant {} - Creating Measurement {}", tenant, measurementRepresentation); - MeasurementRepresentation mrn = measurementApi.create(measurementRepresentation); - measurementRepresentation.setId(mrn.getId()); - } catch (SDKException e) { - log.error("Tenant {} - Error creating Measurement", tenant, e); - } - }); - }); - return measurementRepresentation; - } - - public AlarmRepresentation createAlarm(String severity, String message, String type, DateTime alarmTime, - ManagedObjectRepresentation parentMor, String tenant) { - AlarmRepresentation alarmRepresentation = subscriptionsService.callForTenant(tenant, () -> { - MicroserviceCredentials context = removeAppKeyHeaderFromContext(contextService.getContext()); - return contextService.callWithinContext(context, () -> { - AlarmRepresentation ar = new AlarmRepresentation(); - ar.setSeverity(severity); - ar.setSource(parentMor); - ar.setText(message); - ar.setDateTime(alarmTime); - ar.setStatus("ACTIVE"); - ar.setType(type); - return this.alarmApi.create(ar); - }); - }); - return alarmRepresentation; - } - - public void createEvent(String message, String type, DateTime eventTime, MappingServiceRepresentation source, - String tenant, Map properties) { - subscriptionsService.runForTenant(tenant, () -> { - MicroserviceCredentials context = removeAppKeyHeaderFromContext(contextService.getContext()); - contextService.runWithinContext(context, () -> { - EventRepresentation er = new EventRepresentation(); - ManagedObjectRepresentation mor = new ManagedObjectRepresentation(); - mor.setId(new GId(source.getId())); - er.setSource(mor); - er.setText(message); - er.setDateTime(eventTime); - er.setType(type); - if (properties != null) { - er.setProperty(C8YAgent.CONNECTOR_FRAGMENT, properties); - } - this.eventApi.createAsync(er); - }); - }); - } - - public AConnectorClient.Certificate loadCertificateByName(String certificateName, String fingerprint, - String tenant, String connectorName) { - TrustedCertificateRepresentation result = subscriptionsService.callForTenant(tenant, - () -> { - log.info("Tenant {} - Connector {} - Retrieving certificate {} ", tenant, connectorName, - certificateName); - TrustedCertificateRepresentation certResult = null; - try { - List certificatesList = new ArrayList<>(); - boolean next = true; - String nextUrl = String.format("/tenant/tenants/%s/trusted-certificates", tenant); - TrustedCertificateCollectionRepresentation certificatesResult; - while (next) { - certificatesResult = platform.rest().get( - nextUrl, - MediaType.APPLICATION_JSON_TYPE, TrustedCertificateCollectionRepresentation.class); - certificatesList.addAll(certificatesResult.getCertificates()); - nextUrl = certificatesResult.getNext(); - next = certificatesResult.getCertificates().size() > 0; - log.info("Tenant {} - Connector {} - Retrieved certificates {} - next {} - nextUrl {}", - tenant, - connectorName, certificatesList.size(), next, nextUrl); - } - for (int index = 0; index < certificatesList.size(); index++) { - TrustedCertificateRepresentation certificateIterate = certificatesList.get(index); - log.info("Tenant {} - Found certificate with fingerprint: {} with name: {}", tenant, - certificateIterate.getFingerprint(), - certificateIterate.getName()); - if (certificateIterate.getName().equals(certificateName) - && certificateIterate.getFingerprint().equals(fingerprint)) { - certResult = certificateIterate; - log.info("Tenant {} - Connector {} - Found certificate {} with fingerprint {} ", tenant, - connectorName, certificateName, certificateIterate.getFingerprint()); - break; - } - } - } catch (Exception e) { - log.error("Tenant {} - Connector {} - Exception when initializing connector: ", tenant, - connectorName, e); - } - return certResult; - }); - if (result != null) { - log.info("Tenant {} - Connector {} - Found certificate {} with fingerprint {} ", tenant, - connectorName, certificateName, result.getFingerprint()); - StringBuffer cert = new StringBuffer("-----BEGIN CERTIFICATE-----\n") - .append(result.getCertInPemFormat()) - .append("\n").append("-----END CERTIFICATE-----"); - return new AConnectorClient.Certificate(result.getFingerprint(), cert.toString()); - } else { - log.info("Tenant {} - Connector {} - No certificate found!", tenant, connectorName); - return null; - } - } - - public AbstractExtensibleRepresentation createMEAO(ProcessingContext context) - throws ProcessingException { - String tenant = context.getTenant(); - StringBuffer error = new StringBuffer(""); - C8YRequest currentRequest = context.getCurrentRequest(); - String payload = currentRequest.getRequest(); - API targetAPI = context.getMapping().getTargetAPI(); - AbstractExtensibleRepresentation result = subscriptionsService.callForTenant(tenant, () -> { - MicroserviceCredentials contextCredentials = removeAppKeyHeaderFromContext(contextService.getContext()); - return contextService.callWithinContext(contextCredentials, () -> { - AbstractExtensibleRepresentation rt = null; - try { - if (targetAPI.equals(API.EVENT)) { - EventRepresentation eventRepresentation = configurationRegistry.getObjectMapper().readValue(payload, - EventRepresentation.class); - rt = eventApi.create(eventRepresentation); - log.info("Tenant {} - New event posted: {}", tenant, rt); - } else if (targetAPI.equals(API.ALARM)) { - AlarmRepresentation alarmRepresentation = configurationRegistry.getObjectMapper().readValue(payload, - AlarmRepresentation.class); - rt = alarmApi.create(alarmRepresentation); - log.info("Tenant {} - New alarm posted: {}", tenant, rt); - } else if (targetAPI.equals(API.MEASUREMENT)) { - MeasurementRepresentation measurementRepresentation = jsonParser - .parse(MeasurementRepresentation.class, payload); - rt = measurementApi.create(measurementRepresentation); - log.info("Tenant {} - New measurement posted: {}", tenant, rt); - } else if (targetAPI.equals(API.OPERATION)) { - OperationRepresentation operationRepresentation = jsonParser - .parse(OperationRepresentation.class, payload); - rt = deviceControlApi.create(operationRepresentation); - log.info("Tenant {} - New operation posted: {}", tenant, rt); - } else { - log.error("Tenant {} - Not existing API!", tenant); - } - } catch (JsonProcessingException e) { - log.error("Tenant {} - Could not map payload: {} {}", tenant, targetAPI, payload); - error.append("Could not map payload: " + targetAPI + "/" + payload); - } catch (SDKException s) { - log.error("Tenant {} - Could not sent payload to c8y: {} {}: ", tenant, targetAPI, payload, s); - error.append("Could not sent payload to c8y: " + targetAPI + "/" + payload + "/" + s); - } - return rt; - }); - }); - if (!error.toString().equals("")) { - throw new ProcessingException(error.toString()); - } - return result; - } - - public ManagedObjectRepresentation upsertDevice(String tenant, ID identity, ProcessingContext context) - throws ProcessingException { - StringBuffer error = new StringBuffer(""); - C8YRequest currentRequest = context.getCurrentRequest(); - ManagedObjectRepresentation device = subscriptionsService.callForTenant(tenant, () -> { - MicroserviceCredentials contextCredentials = removeAppKeyHeaderFromContext(contextService.getContext()); - return contextService.callWithinContext(contextCredentials, () -> { - ManagedObjectRepresentation mor = configurationRegistry.getObjectMapper().readValue( - currentRequest.getRequest(), - ManagedObjectRepresentation.class); - try { - ExternalIDRepresentation extId = resolveExternalId2GlobalId(tenant, identity, context); - if (extId == null) { - // Device does not exist - // append external id to name - mor.setName(mor.getName()); - /* - mor.set(new Agent()); - HashMap agentFragments = new HashMap<>(); - agentFragments.put("name", "Dynamic Mapper"); - agentFragments.put("version", version); - agentFragments.put("url", "https://github.com/SoftwareAG/cumulocity-dynamic-mapper"); - agentFragments.put("maintainer", "Open-Source"); - mor.set(agentFragments, "c8y_Agent"); - */ - mor.set(new IsDevice()); - // remove id - mor.setId(null); - - mor = inventoryApi.create(mor, context); - log.info("Tenant {} - New device created: {}", tenant, mor); - identityApi.create(mor, identity, context); - } else { - // Device exists - update needed - mor.setId(extId.getManagedObject().getId()); - mor = inventoryApi.update(mor, context); - - log.info("Tenant {} - Device updated: {}", tenant, mor); - } - } catch (SDKException s) { - log.error("Tenant {} - Could not sent payload to c8y: {}: ", tenant, currentRequest.getRequest(), s); - error.append("Could not sent payload to c8y: " + currentRequest.getRequest() + " " + s); - } - return mor; - }); - }); - if (!error.toString().equals("")) { - throw new ProcessingException(error.toString()); - } - return device; - } - - public void loadProcessorExtensions(String tenant) { - ClassLoader internalClassloader = C8YAgent.class.getClassLoader(); - ClassLoader externalClassLoader = null; - - for (ManagedObjectRepresentation extension : extensionsComponent.get()) { - Map props = (Map) (extension.get(ExtensionsComponent.PROCESSOR_EXTENSION_TYPE)); - String extName = props.get("name").toString(); - boolean external = (Boolean) props.get("external"); - log.debug("Tenant {} - Trying to load extension id: {}, name: {}", tenant, extension.getId().getValue(), - extName); - try { - if (external) { - // step 1 download extension for binary repository - InputStream downloadInputStream = binaryApi.downloadFile(extension.getId()); - - // step 2 create temporary file,because classloader needs a url resource - File tempFile = File.createTempFile(extName, "jar"); - tempFile.deleteOnExit(); - String canonicalPath = tempFile.getCanonicalPath(); - String path = tempFile.getPath(); - String pathWithProtocol = "file://".concat(tempFile.getPath()); - log.debug("Tenant {} - CanonicalPath: {}, Path: {}, PathWithProtocol: {}", tenant, canonicalPath, - path, - pathWithProtocol); - FileOutputStream outputStream = new FileOutputStream(tempFile); - IOUtils.copy(downloadInputStream, outputStream); - - // step 3 parse list of extensions - URL[] urls = {tempFile.toURI().toURL()}; - externalClassLoader = new URLClassLoader(urls, App.class.getClassLoader()); - registerExtensionInProcessor(tenant, extension.getId().getValue(), extName, externalClassLoader, - external); - } else { - registerExtensionInProcessor(tenant, extension.getId().getValue(), extName, internalClassloader, - external); - } - } catch (IOException e) { - log.error("Tenant {} - Exception occurred, When loading extension, starting without extensions: ", - tenant, - e); - } - } - } - - private void registerExtensionInProcessor(String tenant, String id, String extensionName, ClassLoader dynamicLoader, - boolean external) - throws IOException { - ExtensibleProcessorInbound extensibleProcessor = configurationRegistry.getExtensibleProcessors().get(tenant); - extensibleProcessor.addExtension(tenant, new Extension(id, extensionName, external)); - String resource = external ? EXTENSION_EXTERNAL_FILE : EXTENSION_INTERNAL_FILE; - InputStream resourceAsStream = dynamicLoader.getResourceAsStream(resource); - InputStreamReader in = null; - try { - in = new InputStreamReader(resourceAsStream); - } catch (Exception e) { - log.error("Tenant {} - Registration file: {} missing, ignoring to load extensions from: {}", tenant, - resource, - (external ? "EXTERNAL" : "INTERNAL")); - throw new IOException("Registration file: " + resource + " missing, ignoring to load extensions from:" - + (external ? "EXTERNAL" : "INTERNAL")); - } - BufferedReader buffered = new BufferedReader(in); - Properties newExtensions = new Properties(); - - if (buffered != null) - newExtensions.load(buffered); - log.debug("Tenant {} - Preparing to load extensions:" + newExtensions.toString(), tenant); - - Enumeration extensions = newExtensions.propertyNames(); - while (extensions.hasMoreElements()) { - String key = (String) extensions.nextElement(); - Class clazz; - ExtensionEntry extensionEntry = new ExtensionEntry(key, newExtensions.getProperty(key), - null, true, "OK"); - extensibleProcessor.addExtensionEntry(tenant, extensionName, extensionEntry); - - try { - clazz = dynamicLoader.loadClass(newExtensions.getProperty(key)); - if (external && !clazz.getPackageName().startsWith(PACKAGE_MAPPING_PROCESSOR_EXTENSION_EXTERNAL)) { - extensionEntry.setMessage( - "Implementation must be in package: 'dynamic.mapping.processor.extension.external' instead of: " - + clazz.getPackageName()); - extensionEntry.setLoaded(false); - } else { - Object object = clazz.getDeclaredConstructor().newInstance(); - if (!(object instanceof ProcessorExtensionInbound)) { - String msg = String.format( - "Extension: %s=%s is not instance of ProcessorExtension, ignoring this entry!", key, - newExtensions.getProperty(key)); - log.warn(msg); - extensionEntry.setLoaded(false); - } else { - ProcessorExtensionInbound extensionImpl = (ProcessorExtensionInbound) clazz - .getDeclaredConstructor() - .newInstance(); - // springUtil.registerBean(key, clazz); - extensionEntry.setExtensionImplementation(extensionImpl); - log.debug("Tenant {} - Successfully registered bean: {} for key: {}", tenant, - newExtensions.getProperty(key), - key); - } - } - } catch (Exception e) { - String msg = String.format("Could not load extension: %s:%s, ignoring loading!", key, - newExtensions.getProperty(key)); - log.warn(msg); - e.printStackTrace(); - extensionEntry.setLoaded(false); - } - } - extensibleProcessor.updateStatusExtension(extensionName); - } - - public Map getProcessorExtensions(String tenant) { - ExtensibleProcessorInbound extensibleProcessor = configurationRegistry.getExtensibleProcessors().get(tenant); - return extensibleProcessor.getExtensions(); - } - - public Extension getProcessorExtension(String tenant, String extension) { - ExtensibleProcessorInbound extensibleProcessor = configurationRegistry.getExtensibleProcessors().get(tenant); - return extensibleProcessor.getExtension(extension); - } - - public Extension deleteProcessorExtension(String tenant, String extensionName) { - ExtensibleProcessorInbound extensibleProcessor = configurationRegistry.getExtensibleProcessors().get(tenant); - for (ManagedObjectRepresentation extensionRepresentation : extensionsComponent.get()) { - if (extensionName.equals(extensionRepresentation.getName())) { - binaryApi.deleteFile(extensionRepresentation.getId()); - log.info("Tenant {} - Deleted extension: {} permanently!", tenant, extensionName); - } - } - return extensibleProcessor.deleteExtension(extensionName); - } - - public void reloadExtensions(String tenant) { - ExtensibleProcessorInbound extensibleProcessor = configurationRegistry.getExtensibleProcessors().get(tenant); - extensibleProcessor.deleteExtensions(); - loadProcessorExtensions(tenant); - } - - public ManagedObjectRepresentation getManagedObjectForId(String tenant, String deviceId) { - ManagedObjectRepresentation device = subscriptionsService.callForTenant(tenant, () -> { - try { - return inventoryApi.get(GId.asGId(deviceId)); - } catch (SDKException exception) { - log.warn("Tenant {} - Device with id {} not found!", tenant, deviceId); - } - return null; - }); - - return device; - } - - public void updateOperationStatus(String tenant, OperationRepresentation op, OperationStatus status, - String failureReason) { - subscriptionsService.runForTenant(tenant, () -> { - MicroserviceCredentials contextCredentials = removeAppKeyHeaderFromContext(contextService.getContext()); - contextService.runWithinContext(contextCredentials, () -> { - try { - op.setStatus(status.toString()); - if (failureReason != null) - op.setFailureReason(failureReason); - deviceControlApi.update(op); - } catch (SDKException exception) { - log.error("Tenant {} - Operation with id {} could not be updated: {}", tenant, - op.getDeviceId().getValue(), - exception.getLocalizedMessage()); - } - }); - }); - } - - public ManagedObjectRepresentation initializeMappingServiceObject(String tenant) { - ExternalIDRepresentation mappingServiceIdRepresentation = resolveExternalId2GlobalId(tenant, - new ID(null, MappingServiceRepresentation.AGENT_ID), - null); - ; - ManagedObjectRepresentation amo = new ManagedObjectRepresentation(); - - if (mappingServiceIdRepresentation != null) { - amo = inventoryApi.get(mappingServiceIdRepresentation.getManagedObject().getId()); - log.info("Tenant {} - Agent with ID {} already exists {}", tenant, - MappingServiceRepresentation.AGENT_ID, - mappingServiceIdRepresentation, amo.getId()); - } else { - amo.setName(MappingServiceRepresentation.AGENT_NAME); - amo.setType(MappingServiceRepresentation.AGENT_TYPE); - amo.set(new Agent()); - HashMap agentFragments = new HashMap<>(); - agentFragments.put("name", "Dynamic Mapper"); - agentFragments.put("version", version); - agentFragments.put("url", "https://github.com/SoftwareAG/cumulocity-dynamic-mapper"); - agentFragments.put("maintainer", "Open-Source"); - amo.set(agentFragments, "c8y_Agent"); - amo.set(new IsDevice()); - amo.setProperty(C8YAgent.MAPPING_FRAGMENT, - new ArrayList<>()); - amo = inventoryApi.create(amo, null); - log.info("Tenant {} - Agent has been created with ID {}", tenant, amo.getId()); - ExternalIDRepresentation externalAgentId = identityApi.create(amo, - new ID("c8y_Serial", - MappingServiceRepresentation.AGENT_ID), - null); - log.debug("Tenant {} - ExternalId created: {}", tenant, externalAgentId.getExternalId()); - } - return amo; - } - - public void createExtensibleProcessor(String tenant) { - ExtensibleProcessorInbound extensibleProcessor = new ExtensibleProcessorInbound(configurationRegistry); - configurationRegistry.getExtensibleProcessors().put(tenant, extensibleProcessor); - log.debug("Tenant {} - Create ExtensibleProcessor {}", tenant, extensibleProcessor); - - // check if managedObject for internal mapping extension exists - List internalExtension = extensionsComponent.getInternal(); - ManagedObjectRepresentation ie = new ManagedObjectRepresentation(); - if (internalExtension == null || internalExtension.size() == 0) { - Map props = Map.of("name", - ExtensionsComponent.PROCESSOR_EXTENSION_INTERNAL_NAME, - "external", false); - ie.setProperty(ExtensionsComponent.PROCESSOR_EXTENSION_TYPE, - props); - ie.setName(ExtensionsComponent.PROCESSOR_EXTENSION_INTERNAL_NAME); - ie = inventoryApi.create(ie, null); - } else { - ie = internalExtension.get(0); - } - log.debug("Tenant {} - Internal extension: {} registered: {}", tenant, - ExtensionsComponent.PROCESSOR_EXTENSION_INTERNAL_NAME, - ie.getId().getValue(), ie); - } - - public void sendNotificationLifecycle(String tenant, ConnectorStatus connectorStatus, String message) { - if (configurationRegistry.getServiceConfigurations().get(tenant).sendNotificationLifecycle - && !(connectorStatus.equals(previousConnectorStatus))) { - previousConnectorStatus = connectorStatus; - DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - Date now = new Date(); - String date = dateFormat.format(now); - Map stMap = Map.ofEntries( - entry("status", connectorStatus.name()), - entry("message", - message == null ? C8Y_NOTIFICATION_CONNECTOR + ": " + connectorStatus.name() : message), - entry("connectorName", C8Y_NOTIFICATION_CONNECTOR), - entry("connectorIdent", "000000"), - entry("date", date)); - createEvent("Connector status:" + connectorStatus.name(), - C8YAgent.STATUS_NOTIFICATION_EVENT_TYPE, - DateTime.now(), configurationRegistry.getMappingServiceRepresentations().get(tenant), tenant, - stMap); - } - } - - public MicroserviceCredentials removeAppKeyHeaderFromContext(MicroserviceCredentials context) { - final MicroserviceCredentials clonedContext = new MicroserviceCredentials( - context.getTenant(), - context.getUsername(), context.getPassword(), - context.getOAuthAccessToken(), context.getXsrfToken(), - context.getTfaToken(), null); - return clonedContext; - } + @Autowired + private EventApi eventApi; + + @Autowired + private InventoryFacade inventoryApi; + + @Autowired + private BinariesApi binaryApi; + + @Autowired + private IdentityFacade identityApi; + + @Autowired + private MeasurementApi measurementApi; + + @Autowired + private AlarmApi alarmApi; + + @Autowired + private DeviceControlApi deviceControlApi; + + @Autowired + private Platform platform; + + @Autowired + private MicroserviceSubscriptionsService subscriptionsService; + + @Autowired + private ContextService contextService; + + private ExtensionsComponent extensionsComponent; + + @Autowired + public void setExtensionsComponent(ExtensionsComponent extensionsComponent) { + this.extensionsComponent = extensionsComponent; + } + + @Getter + private ConfigurationRegistry configurationRegistry; + + @Autowired + public void setConfigurationRegistry(@Lazy ConfigurationRegistry configurationRegistry) { + this.configurationRegistry = configurationRegistry; + } + + private JSONParser jsonParser = JSONBase.getJSONParser(); + + public static final String MAPPING_FRAGMENT = "d11r_mapping"; + + public static final String CONNECTOR_FRAGMENT = "d11r_connector"; + public static final String DEPLOYMENT_MAP_FRAGMENT = "d11r_deploymentMap"; + + public static final String STATUS_SUBSCRIPTION_EVENT_TYPE = "d11r_subscriptionEvent"; + public static final String STATUS_CONNECTOR_EVENT_TYPE = "d11r_connectorStatusEvent"; + public static final String STATUS_MAPPING_CHANGED_EVENT_TYPE = "d11r_mappingChangedEvent"; + public static final String STATUS_NOTIFICATION_EVENT_TYPE = "d11r_notificationStatusEvent"; + + private static final String EXTENSION_INTERNAL_FILE = "extension-internal.properties"; + private static final String EXTENSION_EXTERNAL_FILE = "extension-external.properties"; + + private static final String C8Y_NOTIFICATION_CONNECTOR = "C8YNotificationConnector"; + + private static final String PACKAGE_MAPPING_PROCESSOR_EXTENSION_EXTERNAL = "dynamic.mapping.processor.extension.external"; + + @Value("${application.version}") + private String version; + + public ExternalIDRepresentation resolveExternalId2GlobalId(String tenant, ID identity, + ProcessingContext context) { + if (identity.getType() == null) { + identity.setType("c8y_Serial"); + } + ExternalIDRepresentation result = subscriptionsService.callForTenant(tenant, () -> { + try { + return identityApi.resolveExternalId2GlobalId(identity, context); + } catch (SDKException e) { + log.warn("Tenant {} - External ID {} not found", tenant, identity.getValue()); + } + return null; + }); + return result; + } + + public ExternalIDRepresentation resolveGlobalId2ExternalId(String tenant, GId gid, String idType, + ProcessingContext context) { + if (idType == null) { + idType = "c8y_Serial"; + } + final String idt = idType; + ExternalIDRepresentation result = subscriptionsService.callForTenant(tenant, () -> { + try { + return identityApi.resolveGlobalId2ExternalId(gid, idt, context); + } catch (SDKException e) { + log.warn("Tenant {} - External ID type {} for {} not found", tenant, idt, gid.getValue()); + } + return null; + }); + return result; + } + + public MeasurementRepresentation createMeasurement(String name, String type, ManagedObjectRepresentation mor, + DateTime dateTime, HashMap mvMap, String tenant) { + MeasurementRepresentation measurementRepresentation = new MeasurementRepresentation(); + subscriptionsService.runForTenant(tenant, () -> { + MicroserviceCredentials context = removeAppKeyHeaderFromContext(contextService.getContext()); + contextService.runWithinContext(context, () -> { + try { + measurementRepresentation.set(mvMap, name); + measurementRepresentation.setType(type); + measurementRepresentation.setSource(mor); + measurementRepresentation.setDateTime(dateTime); + log.debug("Tenant {} - Creating Measurement {}", tenant, measurementRepresentation); + MeasurementRepresentation mrn = measurementApi.create(measurementRepresentation); + measurementRepresentation.setId(mrn.getId()); + } catch (SDKException e) { + log.error("Tenant {} - Error creating Measurement", tenant, e); + } + }); + }); + return measurementRepresentation; + } + + public AlarmRepresentation createAlarm(String severity, String message, String type, DateTime alarmTime, + ManagedObjectRepresentation parentMor, String tenant) { + AlarmRepresentation alarmRepresentation = subscriptionsService.callForTenant(tenant, () -> { + MicroserviceCredentials context = removeAppKeyHeaderFromContext(contextService.getContext()); + return contextService.callWithinContext(context, () -> { + AlarmRepresentation ar = new AlarmRepresentation(); + ar.setSeverity(severity); + ar.setSource(parentMor); + ar.setText(message); + ar.setDateTime(alarmTime); + ar.setStatus("ACTIVE"); + ar.setType(type); + return this.alarmApi.create(ar); + }); + }); + return alarmRepresentation; + } + + public void createEvent(String message, String type, DateTime eventTime, MappingServiceRepresentation source, + String tenant, Map properties) { + subscriptionsService.runForTenant(tenant, () -> { + MicroserviceCredentials context = removeAppKeyHeaderFromContext(contextService.getContext()); + contextService.runWithinContext(context, () -> { + EventRepresentation er = new EventRepresentation(); + ManagedObjectRepresentation mor = new ManagedObjectRepresentation(); + mor.setId(new GId(source.getId())); + er.setSource(mor); + er.setText(message); + er.setDateTime(eventTime); + er.setType(type); + if (properties != null) { + er.setProperty(C8YAgent.CONNECTOR_FRAGMENT, properties); + } + this.eventApi.createAsync(er); + }); + }); + } + + public AConnectorClient.Certificate loadCertificateByName(String certificateName, String fingerprint, + String tenant, String connectorName) { + TrustedCertificateRepresentation result = subscriptionsService.callForTenant(tenant, + () -> { + log.info("Tenant {} - Connector {} - Retrieving certificate {} ", tenant, connectorName, + certificateName); + TrustedCertificateRepresentation certResult = null; + try { + List certificatesList = new ArrayList<>(); + boolean next = true; + String nextUrl = String.format("/tenant/tenants/%s/trusted-certificates", tenant); + TrustedCertificateCollectionRepresentation certificatesResult; + while (next) { + certificatesResult = platform.rest().get( + nextUrl, + MediaType.APPLICATION_JSON_TYPE, TrustedCertificateCollectionRepresentation.class); + certificatesList.addAll(certificatesResult.getCertificates()); + nextUrl = certificatesResult.getNext(); + next = certificatesResult.getCertificates().size() > 0; + log.info("Tenant {} - Connector {} - Retrieved certificates {} - next {} - nextUrl {}", + tenant, + connectorName, certificatesList.size(), next, nextUrl); + } + for (int index = 0; index < certificatesList.size(); index++) { + TrustedCertificateRepresentation certificateIterate = certificatesList.get(index); + log.info("Tenant {} - Found certificate with fingerprint: {} with name: {}", tenant, + certificateIterate.getFingerprint(), + certificateIterate.getName()); + if (certificateIterate.getName().equals(certificateName) + && certificateIterate.getFingerprint().equals(fingerprint)) { + certResult = certificateIterate; + log.info("Tenant {} - Connector {} - Found certificate {} with fingerprint {} ", tenant, + connectorName, certificateName, certificateIterate.getFingerprint()); + break; + } + } + } catch (Exception e) { + log.error("Tenant {} - Connector {} - Exception when initializing connector: ", tenant, + connectorName, e); + } + return certResult; + }); + if (result != null) { + log.info("Tenant {} - Connector {} - Found certificate {} with fingerprint {} ", tenant, + connectorName, certificateName, result.getFingerprint()); + StringBuffer cert = new StringBuffer("-----BEGIN CERTIFICATE-----\n") + .append(result.getCertInPemFormat()) + .append("\n").append("-----END CERTIFICATE-----"); + return new AConnectorClient.Certificate(result.getFingerprint(), cert.toString()); + } else { + log.info("Tenant {} - Connector {} - No certificate found!", tenant, connectorName); + return null; + } + } + + public AbstractExtensibleRepresentation createMEAO(ProcessingContext context) + throws ProcessingException { + String tenant = context.getTenant(); + StringBuffer error = new StringBuffer(""); + C8YRequest currentRequest = context.getCurrentRequest(); + String payload = currentRequest.getRequest(); + API targetAPI = context.getMapping().getTargetAPI(); + AbstractExtensibleRepresentation result = subscriptionsService.callForTenant(tenant, () -> { + MicroserviceCredentials contextCredentials = removeAppKeyHeaderFromContext(contextService.getContext()); + return contextService.callWithinContext(contextCredentials, () -> { + AbstractExtensibleRepresentation rt = null; + try { + if (targetAPI.equals(API.EVENT)) { + EventRepresentation eventRepresentation = configurationRegistry.getObjectMapper().readValue( + payload, + EventRepresentation.class); + rt = eventApi.create(eventRepresentation); + log.info("Tenant {} - New event posted: {}", tenant, rt); + } else if (targetAPI.equals(API.ALARM)) { + AlarmRepresentation alarmRepresentation = configurationRegistry.getObjectMapper().readValue( + payload, + AlarmRepresentation.class); + rt = alarmApi.create(alarmRepresentation); + log.info("Tenant {} - New alarm posted: {}", tenant, rt); + } else if (targetAPI.equals(API.MEASUREMENT)) { + MeasurementRepresentation measurementRepresentation = jsonParser + .parse(MeasurementRepresentation.class, payload); + rt = measurementApi.create(measurementRepresentation); + log.info("Tenant {} - New measurement posted: {}", tenant, rt); + } else if (targetAPI.equals(API.OPERATION)) { + OperationRepresentation operationRepresentation = jsonParser + .parse(OperationRepresentation.class, payload); + rt = deviceControlApi.create(operationRepresentation); + log.info("Tenant {} - New operation posted: {}", tenant, rt); + } else { + log.error("Tenant {} - Not existing API!", tenant); + } + } catch (JsonProcessingException e) { + log.error("Tenant {} - Could not map payload: {} {}", tenant, targetAPI, payload); + error.append("Could not map payload: " + targetAPI + "/" + payload); + } catch (SDKException s) { + log.error("Tenant {} - Could not sent payload to c8y: {} {}: ", tenant, targetAPI, payload, s); + error.append("Could not sent payload to c8y: " + targetAPI + "/" + payload + "/" + s); + } + return rt; + }); + }); + if (!error.toString().equals("")) { + throw new ProcessingException(error.toString()); + } + return result; + } + + public ManagedObjectRepresentation upsertDevice(String tenant, ID identity, ProcessingContext context) + throws ProcessingException { + StringBuffer error = new StringBuffer(""); + C8YRequest currentRequest = context.getCurrentRequest(); + ManagedObjectRepresentation device = subscriptionsService.callForTenant(tenant, () -> { + MicroserviceCredentials contextCredentials = removeAppKeyHeaderFromContext(contextService.getContext()); + return contextService.callWithinContext(contextCredentials, () -> { + ManagedObjectRepresentation mor = configurationRegistry.getObjectMapper().readValue( + currentRequest.getRequest(), + ManagedObjectRepresentation.class); + try { + ExternalIDRepresentation extId = resolveExternalId2GlobalId(tenant, identity, context); + if (extId == null) { + // Device does not exist + // append external id to name + mor.setName(mor.getName()); + /* + * mor.set(new Agent()); + * HashMap agentFragments = new HashMap<>(); + * agentFragments.put("name", "Dynamic Data Mapper"); + * agentFragments.put("version", version); + * agentFragments.put("url", + * "https://github.com/SoftwareAG/cumulocity-dynamic-mapper"); + * agentFragments.put("maintainer", "Open-Source"); + * mor.set(agentFragments, "c8y_Agent"); + */ + mor.set(new IsDevice()); + // remove id + mor.setId(null); + + mor = inventoryApi.create(mor, context); + log.info("Tenant {} - New device created: {}", tenant, mor); + identityApi.create(mor, identity, context); + } else { + // Device exists - update needed + mor.setId(extId.getManagedObject().getId()); + mor = inventoryApi.update(mor, context); + + log.info("Tenant {} - Device updated: {}", tenant, mor); + } + } catch (SDKException s) { + log.error("Tenant {} - Could not sent payload to c8y: {}: ", tenant, currentRequest.getRequest(), + s); + error.append("Could not sent payload to c8y: " + currentRequest.getRequest() + " " + s); + } + return mor; + }); + }); + if (!error.toString().equals("")) { + throw new ProcessingException(error.toString()); + } + return device; + } + + public void loadProcessorExtensions(String tenant) { + ClassLoader internalClassloader = C8YAgent.class.getClassLoader(); + ClassLoader externalClassLoader = null; + + for (ManagedObjectRepresentation extension : extensionsComponent.get()) { + Map props = (Map) (extension.get(ExtensionsComponent.PROCESSOR_EXTENSION_TYPE)); + String extName = props.get("name").toString(); + boolean external = (Boolean) props.get("external"); + log.debug("Tenant {} - Trying to load extension id: {}, name: {}", tenant, extension.getId().getValue(), + extName); + try { + if (external) { + // step 1 download extension for binary repository + InputStream downloadInputStream = binaryApi.downloadFile(extension.getId()); + + // step 2 create temporary file,because classloader needs a url resource + File tempFile = File.createTempFile(extName, "jar"); + tempFile.deleteOnExit(); + String canonicalPath = tempFile.getCanonicalPath(); + String path = tempFile.getPath(); + String pathWithProtocol = "file://".concat(tempFile.getPath()); + log.debug("Tenant {} - CanonicalPath: {}, Path: {}, PathWithProtocol: {}", tenant, canonicalPath, + path, + pathWithProtocol); + FileOutputStream outputStream = new FileOutputStream(tempFile); + IOUtils.copy(downloadInputStream, outputStream); + + // step 3 parse list of extensions + URL[] urls = { tempFile.toURI().toURL() }; + externalClassLoader = new URLClassLoader(urls, App.class.getClassLoader()); + registerExtensionInProcessor(tenant, extension.getId().getValue(), extName, externalClassLoader, + external); + } else { + registerExtensionInProcessor(tenant, extension.getId().getValue(), extName, internalClassloader, + external); + } + } catch (IOException e) { + log.error("Tenant {} - Exception occurred, When loading extension, starting without extensions: ", + tenant, + e); + } + } + } + + private void registerExtensionInProcessor(String tenant, String id, String extensionName, ClassLoader dynamicLoader, + boolean external) + throws IOException { + ExtensibleProcessorInbound extensibleProcessor = configurationRegistry.getExtensibleProcessors().get(tenant); + extensibleProcessor.addExtension(tenant, new Extension(id, extensionName, external)); + String resource = external ? EXTENSION_EXTERNAL_FILE : EXTENSION_INTERNAL_FILE; + InputStream resourceAsStream = dynamicLoader.getResourceAsStream(resource); + InputStreamReader in = null; + try { + in = new InputStreamReader(resourceAsStream); + } catch (Exception e) { + log.error("Tenant {} - Registration file: {} missing, ignoring to load extensions from: {}", tenant, + resource, + (external ? "EXTERNAL" : "INTERNAL")); + throw new IOException("Registration file: " + resource + " missing, ignoring to load extensions from:" + + (external ? "EXTERNAL" : "INTERNAL")); + } + BufferedReader buffered = new BufferedReader(in); + Properties newExtensions = new Properties(); + + if (buffered != null) + newExtensions.load(buffered); + log.debug("Tenant {} - Preparing to load extensions:" + newExtensions.toString(), tenant); + + Enumeration extensions = newExtensions.propertyNames(); + while (extensions.hasMoreElements()) { + String key = (String) extensions.nextElement(); + Class clazz; + ExtensionEntry extensionEntry = new ExtensionEntry(key, newExtensions.getProperty(key), + null, true, "OK"); + extensibleProcessor.addExtensionEntry(tenant, extensionName, extensionEntry); + + try { + clazz = dynamicLoader.loadClass(newExtensions.getProperty(key)); + if (external && !clazz.getPackageName().startsWith(PACKAGE_MAPPING_PROCESSOR_EXTENSION_EXTERNAL)) { + extensionEntry.setMessage( + "Implementation must be in package: 'dynamic.mapping.processor.extension.external' instead of: " + + clazz.getPackageName()); + extensionEntry.setLoaded(false); + } else { + Object object = clazz.getDeclaredConstructor().newInstance(); + if (!(object instanceof ProcessorExtensionInbound)) { + String msg = String.format( + "Extension: %s=%s is not instance of ProcessorExtension, ignoring this entry!", key, + newExtensions.getProperty(key)); + log.warn(msg); + extensionEntry.setLoaded(false); + } else { + ProcessorExtensionInbound extensionImpl = (ProcessorExtensionInbound) clazz + .getDeclaredConstructor() + .newInstance(); + // springUtil.registerBean(key, clazz); + extensionEntry.setExtensionImplementation(extensionImpl); + log.debug("Tenant {} - Successfully registered bean: {} for key: {}", tenant, + newExtensions.getProperty(key), + key); + } + } + } catch (Exception e) { + String msg = String.format("Could not load extension: %s:%s, ignoring loading!", key, + newExtensions.getProperty(key)); + log.warn(msg); + e.printStackTrace(); + extensionEntry.setLoaded(false); + } + } + extensibleProcessor.updateStatusExtension(extensionName); + } + + public Map getProcessorExtensions(String tenant) { + ExtensibleProcessorInbound extensibleProcessor = configurationRegistry.getExtensibleProcessors().get(tenant); + return extensibleProcessor.getExtensions(); + } + + public Extension getProcessorExtension(String tenant, String extension) { + ExtensibleProcessorInbound extensibleProcessor = configurationRegistry.getExtensibleProcessors().get(tenant); + return extensibleProcessor.getExtension(extension); + } + + public Extension deleteProcessorExtension(String tenant, String extensionName) { + ExtensibleProcessorInbound extensibleProcessor = configurationRegistry.getExtensibleProcessors().get(tenant); + for (ManagedObjectRepresentation extensionRepresentation : extensionsComponent.get()) { + if (extensionName.equals(extensionRepresentation.getName())) { + binaryApi.deleteFile(extensionRepresentation.getId()); + log.info("Tenant {} - Deleted extension: {} permanently!", tenant, extensionName); + } + } + return extensibleProcessor.deleteExtension(extensionName); + } + + public void reloadExtensions(String tenant) { + ExtensibleProcessorInbound extensibleProcessor = configurationRegistry.getExtensibleProcessors().get(tenant); + extensibleProcessor.deleteExtensions(); + loadProcessorExtensions(tenant); + } + + public ManagedObjectRepresentation getManagedObjectForId(String tenant, String deviceId) { + ManagedObjectRepresentation device = subscriptionsService.callForTenant(tenant, () -> { + try { + return inventoryApi.get(GId.asGId(deviceId)); + } catch (SDKException exception) { + log.warn("Tenant {} - Device with id {} not found!", tenant, deviceId); + } + return null; + }); + + return device; + } + + public void updateOperationStatus(String tenant, OperationRepresentation op, OperationStatus status, + String failureReason) { + subscriptionsService.runForTenant(tenant, () -> { + MicroserviceCredentials contextCredentials = removeAppKeyHeaderFromContext(contextService.getContext()); + contextService.runWithinContext(contextCredentials, () -> { + try { + op.setStatus(status.toString()); + if (failureReason != null) + op.setFailureReason(failureReason); + deviceControlApi.update(op); + } catch (SDKException exception) { + log.error("Tenant {} - Operation with id {} could not be updated: {}", tenant, + op.getDeviceId().getValue(), + exception.getLocalizedMessage()); + } + }); + }); + } + + public ManagedObjectRepresentation initializeMappingServiceObject(String tenant) { + ExternalIDRepresentation mappingServiceIdRepresentation = resolveExternalId2GlobalId(tenant, + new ID(null, MappingServiceRepresentation.AGENT_ID), + null); + ; + ManagedObjectRepresentation amo = new ManagedObjectRepresentation(); + + if (mappingServiceIdRepresentation != null) { + amo = inventoryApi.get(mappingServiceIdRepresentation.getManagedObject().getId()); + log.info("Tenant {} - Agent with ID {} already exists {}", tenant, + MappingServiceRepresentation.AGENT_ID, + mappingServiceIdRepresentation, amo.getId()); + } else { + amo.setName(MappingServiceRepresentation.AGENT_NAME); + amo.setType(MappingServiceRepresentation.AGENT_TYPE); + amo.set(new Agent()); + HashMap agentFragments = new HashMap<>(); + agentFragments.put("name", "Dynamic Data Mapper"); + agentFragments.put("version", version); + agentFragments.put("url", "https://github.com/SoftwareAG/cumulocity-dynamic-mapper"); + agentFragments.put("maintainer", "Open-Source"); + amo.set(agentFragments, "c8y_Agent"); + amo.set(new IsDevice()); + amo.setProperty(C8YAgent.MAPPING_FRAGMENT, + new ArrayList<>()); + amo = inventoryApi.create(amo, null); + log.info("Tenant {} - Agent has been created with ID {}", tenant, amo.getId()); + ExternalIDRepresentation externalAgentId = identityApi.create(amo, + new ID("c8y_Serial", + MappingServiceRepresentation.AGENT_ID), + null); + log.debug("Tenant {} - ExternalId created: {}", tenant, externalAgentId.getExternalId()); + } + return amo; + } + + public void createExtensibleProcessor(String tenant) { + ExtensibleProcessorInbound extensibleProcessor = new ExtensibleProcessorInbound(configurationRegistry); + configurationRegistry.getExtensibleProcessors().put(tenant, extensibleProcessor); + log.debug("Tenant {} - Create ExtensibleProcessor {}", tenant, extensibleProcessor); + + // check if managedObject for internal mapping extension exists + List internalExtension = extensionsComponent.getInternal(); + ManagedObjectRepresentation ie = new ManagedObjectRepresentation(); + if (internalExtension == null || internalExtension.size() == 0) { + Map props = Map.of("name", + ExtensionsComponent.PROCESSOR_EXTENSION_INTERNAL_NAME, + "external", false); + ie.setProperty(ExtensionsComponent.PROCESSOR_EXTENSION_TYPE, + props); + ie.setName(ExtensionsComponent.PROCESSOR_EXTENSION_INTERNAL_NAME); + ie = inventoryApi.create(ie, null); + } else { + ie = internalExtension.get(0); + } + log.debug("Tenant {} - Internal extension: {} registered: {}", tenant, + ExtensionsComponent.PROCESSOR_EXTENSION_INTERNAL_NAME, + ie.getId().getValue(), ie); + } + + public void sendNotificationLifecycle(String tenant, ConnectorStatus connectorStatus, String message) { + if (configurationRegistry.getServiceConfigurations().get(tenant).sendNotificationLifecycle + && !(connectorStatus.equals(previousConnectorStatus))) { + previousConnectorStatus = connectorStatus; + DateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + Date now = new Date(); + String date = dateFormat.format(now); + Map stMap = Map.ofEntries( + entry("status", connectorStatus.name()), + entry("message", + message == null ? C8Y_NOTIFICATION_CONNECTOR + ": " + connectorStatus.name() : message), + entry("connectorName", C8Y_NOTIFICATION_CONNECTOR), + entry("connectorIdent", "000000"), + entry("date", date)); + createEvent("Connector status:" + connectorStatus.name(), + C8YAgent.STATUS_NOTIFICATION_EVENT_TYPE, + DateTime.now(), configurationRegistry.getMappingServiceRepresentations().get(tenant), tenant, + stMap); + } + } + + public MicroserviceCredentials removeAppKeyHeaderFromContext(MicroserviceCredentials context) { + final MicroserviceCredentials clonedContext = new MicroserviceCredentials( + context.getTenant(), + context.getUsername(), context.getPassword(), + context.getOAuthAccessToken(), context.getXsrfToken(), + context.getTfaToken(), null); + return clonedContext; + } } \ No newline at end of file From 5a14de3fd89ad05ba59af79e3559e00e5dfbaa0c Mon Sep 17 00:00:00 2001 From: sagIoTPower Date: Sun, 8 Sep 2024 22:36:22 +0200 Subject: [PATCH 05/11] moving from charts.js to echarts --- dynamic-mapping-ui/package-lock.json | 61 +++-- dynamic-mapping-ui/package.json | 3 +- .../src/monitoring/chart/chart.component.css | 16 +- .../src/monitoring/chart/chart.component.html | 12 +- .../src/monitoring/chart/chart.component.ts | 213 +++++++++++------- .../src/monitoring/chart/util.ts | 25 +- .../src/monitoring/monitoring.module.ts | 9 +- 7 files changed, 217 insertions(+), 122 deletions(-) diff --git a/dynamic-mapping-ui/package-lock.json b/dynamic-mapping-ui/package-lock.json index 9388eab7..6f4dc816 100644 --- a/dynamic-mapping-ui/package-lock.json +++ b/dynamic-mapping-ui/package-lock.json @@ -1,12 +1,12 @@ { "name": "dynamic-mapping", - "version": "4.4.1", + "version": "4.5.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dynamic-mapping", - "version": "4.4.1", + "version": "4.5.1", "license": "ISC", "dependencies": { "@angular/animations": "^17.3.0", @@ -23,10 +23,11 @@ "@c8y/ngx-components": "^1020.18.1", "@c8y/options": "^1020.18.1", "@ngx-translate/core": "15.0.0", - "chart.js": "^4.4.1", "css-loader": "^7.1.2", + "echarts": "^5.5.1", "jsonata": "^2.0.3", "ngx-bootstrap": "^12.0.0", + "ngx-echarts": "^18.0.0", "postcss-loader": "^8.1.0", "rxjs": "~7.8.1", "style-loader": "^4.0.0", @@ -4298,11 +4299,6 @@ "tslib": "2" } }, - "node_modules/@kurkle/color": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.2.tgz", - "integrity": "sha512-fuscdXJ9G1qb7W8VdHi+IwRqij3lBkosAm4ydQtEmbY58OzHXqQhvlxqEkoz0yssNVn38bcpRWgA9PP+OGoisw==" - }, "node_modules/@leichtgewicht/ip-codec": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz", @@ -6732,17 +6728,6 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, - "node_modules/chart.js": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/chart.js/-/chart.js-4.4.3.tgz", - "integrity": "sha512-qK1gkGSRYcJzqrrzdR6a+I0vQ4/R+SoODXyAjscQ/4mzuNzySaMCd+hyVxitSY1+L2fjPD1Gbn+ibNqRmwQeLw==", - "dependencies": { - "@kurkle/color": "^0.3.0" - }, - "engines": { - "pnpm": ">=8" - } - }, "node_modules/check-more-types": { "version": "2.24.0", "resolved": "https://registry.npmjs.org/check-more-types/-/check-more-types-2.24.0.tgz", @@ -8529,6 +8514,20 @@ "safer-buffer": "^2.1.0" } }, + "node_modules/echarts": { + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/echarts/-/echarts-5.5.1.tgz", + "integrity": "sha512-Fce8upazaAXUVUVsjgV6mBnGuqgO+JNDlcgF79Dksy4+wgGpQB2lmYoO4TSweFg/mZITdpGHomw/cNBJZj1icA==", + "dependencies": { + "tslib": "2.3.0", + "zrender": "5.6.0" + } + }, + "node_modules/echarts/node_modules/tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==" + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -12851,6 +12850,17 @@ "rxjs": "^6.5.3 || ^7.4.0" } }, + "node_modules/ngx-echarts": { + "version": "18.0.0", + "resolved": "https://registry.npmjs.org/ngx-echarts/-/ngx-echarts-18.0.0.tgz", + "integrity": "sha512-1rJW7vhMTTQMZNO5AhbHfTDorhP7dcvwRsDH5jFk2SPb/gjIFWvXBY9VSNAOKumuSBnopm2+uSz6BRO5oWxovA==", + "dependencies": { + "tslib": "^2.3.0" + }, + "peerDependencies": { + "echarts": ">=5.0.0" + } + }, "node_modules/nice-napi": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/nice-napi/-/nice-napi-1.0.2.tgz", @@ -18087,6 +18097,19 @@ "version": "0.14.8", "resolved": "https://registry.npmjs.org/zone.js/-/zone.js-0.14.8.tgz", "integrity": "sha512-48uh7MnVp4/OQDuCHeFdXw5d8xwPqFTvlHgPJ1LBFb5GaustLSZV+YUH0to5ygNyGpqTsjpbpt141U/j3pCfqQ==" + }, + "node_modules/zrender": { + "version": "5.6.0", + "resolved": "https://registry.npmjs.org/zrender/-/zrender-5.6.0.tgz", + "integrity": "sha512-uzgraf4njmmHAbEUxMJ8Oxg+P3fT04O+9p7gY+wJRVxo8Ge+KmYv0WJev945EH4wFuc4OY2NLXz46FZrWS9xJg==", + "dependencies": { + "tslib": "2.3.0" + } + }, + "node_modules/zrender/node_modules/tslib": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.0.tgz", + "integrity": "sha512-N82ooyxVNm6h1riLCoyS9e3fuJ3AMG2zIZs2Gd1ATcSFjSA23Q0fzjjZeh0jbJvWVDZ0cJT8yaNNaaXHzueNjg==" } } } diff --git a/dynamic-mapping-ui/package.json b/dynamic-mapping-ui/package.json index 0559d9e9..bcff6301 100644 --- a/dynamic-mapping-ui/package.json +++ b/dynamic-mapping-ui/package.json @@ -33,10 +33,11 @@ "@c8y/ngx-components": "^1020.18.1", "@c8y/options": "^1020.18.1", "@ngx-translate/core": "15.0.0", - "chart.js": "^4.4.1", "css-loader": "^7.1.2", + "echarts": "^5.5.1", "jsonata": "^2.0.3", "ngx-bootstrap": "^12.0.0", + "ngx-echarts": "^18.0.0", "postcss-loader": "^8.1.0", "rxjs": "~7.8.1", "style-loader": "^4.0.0", diff --git a/dynamic-mapping-ui/src/monitoring/chart/chart.component.css b/dynamic-mapping-ui/src/monitoring/chart/chart.component.css index 8f3572f1..658d6ae6 100644 --- a/dynamic-mapping-ui/src/monitoring/chart/chart.component.css +++ b/dynamic-mapping-ui/src/monitoring/chart/chart.component.css @@ -1,5 +1,11 @@ -#divChart{ - /* position: relative; */ - height:30vh; - width:60vw -} \ No newline at end of file +#divChart { + /* position: relative; */ + height: 30vh; + width: 60vw; +} + +.bar-chart { + /* height: 30vh; + width: 60vw; */ + margin-top: 30px; +} diff --git a/dynamic-mapping-ui/src/monitoring/chart/chart.component.html b/dynamic-mapping-ui/src/monitoring/chart/chart.component.html index df0f606e..c18a3c0e 100644 --- a/dynamic-mapping-ui/src/monitoring/chart/chart.component.html +++ b/dynamic-mapping-ui/src/monitoring/chart/chart.component.html @@ -18,7 +18,7 @@ ~ ~ @authors Christof Strack --> - + Monitoring

@@ -27,9 +27,13 @@

Messages processed by mappings

-
- -
+
diff --git a/dynamic-mapping-ui/src/monitoring/chart/chart.component.ts b/dynamic-mapping-ui/src/monitoring/chart/chart.component.ts index 4fa702e9..79ef75db 100644 --- a/dynamic-mapping-ui/src/monitoring/chart/chart.component.ts +++ b/dynamic-mapping-ui/src/monitoring/chart/chart.component.ts @@ -1,13 +1,10 @@ import { Component, ElementRef, OnDestroy, OnInit } from '@angular/core'; -import { Chart, registerables } from 'chart.js'; -import { CHART_COLORS, transparentize } from './util'; +import { CHART_COLORS } from './util'; import { Subject } from 'rxjs'; import { Direction, MappingStatus } from '../../shared'; import { map } from 'rxjs/operators'; import { MonitoringService } from '../shared/monitoring.service'; -Chart.register(...registerables); -// Chart.defaults.font.family = 'Roboto, Helvetica, Arial, sans-serif'; -// Chart.defaults.color = 'green'; +import { ECharts, EChartsOption } from 'echarts'; @Component({ selector: 'd11r-monitoring-chart', @@ -21,11 +18,13 @@ export class MonitoringChartComponent implements OnInit, OnDestroy { ) {} mappingStatus$: Subject = new Subject(); + echartOptions: EChartsOption; + echartUpdateOptions: EChartsOption; subscription: object; - statusMappingChart: Chart; + echartsInstance: any; textColor: string; fontFamily: string; - fontWeight: string; + fontWeight: number; fontSize: number; ngOnInit() { @@ -38,37 +37,16 @@ export class MonitoringChartComponent implements OnInit, OnDestroy { this.fontFamily = getComputedStyle(root) .getPropertyValue('--c8y-font-family-sans-serif') .trim(); - this.fontWeight = getComputedStyle(root) - .getPropertyValue('--c8y-font-weight-headings') - .trim(); + this.fontWeight = parseInt( + getComputedStyle(root) + .getPropertyValue('--c8y-font-weight-headings') + .trim() + ); this.fontSize = parseInt( - getComputedStyle(root).getPropertyValue('--c8y-font-size-base').trim(), - 12 + getComputedStyle(root).getPropertyValue('--c8y-font-size-base').trim() ); - // rgb(100, 31, 61), 'Roboto, Helvetica, Arial, sans-serif' - // console.log('Text Color', this.textColor); - const statistic = [0, 0, 0, 0]; - const data = { - labels: [ - 'Errors', - 'Messages received', - 'Snooped templates total', - 'Snooped templates active' - ], - datasets: [ - { - label: 'Inbound', - data: statistic, - backgroundColor: transparentize(CHART_COLORS.green, 0.3) - }, - { - label: 'Outbound', - data: statistic, - backgroundColor: transparentize(CHART_COLORS.orange, 0.3) - } - ] - }; + const data = [undefined, undefined]; this.mappingStatus$ .pipe( map((data01) => { @@ -121,64 +99,142 @@ export class MonitoringChartComponent implements OnInit, OnDestroy { // console.log('Statistic', total); if (total) { const [inbound, outbound] = total; - data.datasets[0].data = [ + data[0] = [ inbound.errors, inbound.messagesReceived, inbound.snoopedTemplatesTotal, inbound.snoopedTemplatesActive ]; - data.datasets[1].data = [ + data[1] = [ outbound.errors, outbound.messagesReceived, outbound.snoopedTemplatesTotal, outbound.snoopedTemplatesActive ]; - this.statusMappingChart.update(); + // this.statusMappingChart.update(); + this.echartUpdateOptions = { + series: [ + { + type: 'bar', + data: data[0] + }, + { + type: 'bar', + data: data[1] + } + ] + }; + this.echartsInstance?.setOption(this.echartUpdateOptions); } }); - const config = { - type: 'bar' as any, - data: data, - options: { - responsive: true, - maintainAspectRatio: false, - layout: { - padding: { - left: 0, - right: 0, - top: 0, - bottom: 0 - } + // const config = { + // type: 'bar' as any, + // data: data, + // options: { + // responsive: true, + // maintainAspectRatio: false, + // layout: { + // padding: { + // left: 0, + // right: 0, + // top: 0, + // bottom: 0 + // } + // }, + // plugins: { + // legend: { + // display: true, + // position: 'left' as any, + // font: { + // family: this.fontFamily, + // weight: 'normal' as any + // } + // } + // }, + // indexAxis: 'y' as any, + // color: this.textColor as any, + // scales: { + // y: { + // ticks: { + // color: this.textColor as any + // } + // }, + // x: { + // ticks: { + // color: this.textColor as any, + // stepSize: 0 + // } + // } + // } + // } + // }; + // this.statusMappingChart = new Chart('monitoringChart', config); + + const yAxisData = [ + 'Errors', + 'Messages received', + 'Snooped templates total', + 'Snooped templates active' + ]; + + this.echartOptions = { + title: { + show: false, + text: 'Messages processed by mappings', + textStyle: { + color: this.textColor, + fontFamily: this.fontFamily, + fontSize: this.fontSize + 4, + fontWeight: this.fontWeight + } + }, + legend: { + data: ['Inbound', 'Outbound'], + align: 'left' + }, + tooltip: {}, + grid: { + left: '20%' // Adjust this value as needed + }, + xAxis: { + type: 'value', + boundaryGap: [0, 0.01] + }, + yAxis: { + axisTick: { + show: false }, - plugins: { - legend: { - display: true, - position: 'left' as any, - font: { - family: this.fontFamily, - weight: 'normal' as any - } - } + type: 'category', + data: yAxisData, + silent: false, + splitLine: { + show: true + } + }, + series: [ + { + name: 'Inbound', + color: CHART_COLORS.green, + type: 'bar', + data: data[0] + // animationDelay: (idx) => idx * 10 + 100 }, - indexAxis: 'y' as any, - color: this.textColor as any, - scales: { - y: { - ticks: { - color: this.textColor as any - } - }, - x: { - ticks: { - color: this.textColor as any, - stepSize: 0 - } - } + { + name: 'Outbound', + color: CHART_COLORS.orange, + type: 'bar', + data: data[1] + // animationDelay: (idx) => idx * 10 } - } + ] + // animationEasing: 'elasticOut', + // animationDelayUpdate: (idx) => idx * 5 }; - this.statusMappingChart = new Chart('monitoringChart', config); + } + + onChartInit(ec: ECharts) { + this.echartsInstance = ec; } private async initializeMonitoringService() { @@ -193,4 +249,9 @@ export class MonitoringChartComponent implements OnInit, OnDestroy { // console.log('Stop subscription'); this.monitoringService.unsubscribeFromMonitoringChannel(this.subscription); } + + randomIntFromInterval(min, max) { + // min and max included + return Math.floor(Math.random() * (max - min + 1) + min); + } } diff --git a/dynamic-mapping-ui/src/monitoring/chart/util.ts b/dynamic-mapping-ui/src/monitoring/chart/util.ts index f17587bc..a7500c58 100644 --- a/dynamic-mapping-ui/src/monitoring/chart/util.ts +++ b/dynamic-mapping-ui/src/monitoring/chart/util.ts @@ -1,5 +1,3 @@ -import colorLib from '@kurkle/color'; - // Adapted from http://indiegamr.com/generate-repeatable-random-numbers-in-js/ let _seed = Date.now(); @@ -32,7 +30,7 @@ export function numbers(config) { for (i = 0; i < count; ++i) { value = (from[i] || 0) + rand(min, max); - if (rand(0,0) <= continuity) { + if (rand(0, 0) <= continuity) { data.push(Math.round(dfactor * value) / dfactor); } else { data.push(null); @@ -77,11 +75,6 @@ export function color(index) { return COLORS[index % COLORS.length]; } -export function transparentize(value, opacity) { - const alpha = opacity === undefined ? 0.5 : 1 - opacity; - return colorLib(value).alpha(alpha).rgbString(); -} - // export const CHART_COLORS = { // red: 'rgb(255, 99, 132)', // orange: 'rgb(255, 159, 64)', @@ -93,14 +86,14 @@ export function transparentize(value, opacity) { // }; export const CHART_COLORS = { - red: 'red', - orange: 'orange', - yellow: 'yellow', - green: 'green', - blue: 'blue', - purple: 'purple', - grey: 'grey' - }; + red: 'red', + orange: 'orange', + yellow: 'yellow', + green: 'green', + blue: 'blue', + purple: 'purple', + grey: 'grey' +}; const NAMED_COLORS = [ CHART_COLORS.red, diff --git a/dynamic-mapping-ui/src/monitoring/monitoring.module.ts b/dynamic-mapping-ui/src/monitoring/monitoring.module.ts index 178d41ef..7b1a6336 100644 --- a/dynamic-mapping-ui/src/monitoring/monitoring.module.ts +++ b/dynamic-mapping-ui/src/monitoring/monitoring.module.ts @@ -29,6 +29,7 @@ import { DirectionRendererComponent } from './renderer/direction.renderer.compon import { MonitoringChartComponent } from './chart/chart.component'; import { MonitoringTabFactory } from './monitoring-tab.factory'; import { NODE2 } from '../shared/model/util'; +import { NgxEchartsModule } from 'ngx-echarts'; @NgModule({ declarations: [ @@ -38,7 +39,13 @@ import { NODE2 } from '../shared/model/util'; DirectionRendererComponent, MonitoringChartComponent ], - imports: [CoreModule, BrokerConfigurationModule], + imports: [ + CoreModule, + BrokerConfigurationModule, + NgxEchartsModule.forRoot({ + echarts: () => import('echarts') + }) + ], exports: [], providers: [ hookRoute({ From 29a38db115a4ac6b9610dcbf9c426766252ea82e Mon Sep 17 00:00:00 2001 From: sagIoTPower Date: Mon, 9 Sep 2024 16:21:39 +0200 Subject: [PATCH 06/11] change initializing realtime updates: ConnectorConfigurationService, ConnectorStatusService --- .../src/monitoring/chart/chart.component.ts | 47 +---------------- .../shared/connector-configuration.service.ts | 52 ++++++++++--------- .../connector-grid.component.ts | 23 ++++---- .../connector-status.component.ts | 8 +-- .../src/shared/connector-status.service.ts | 24 ++++++--- 5 files changed, 58 insertions(+), 96 deletions(-) diff --git a/dynamic-mapping-ui/src/monitoring/chart/chart.component.ts b/dynamic-mapping-ui/src/monitoring/chart/chart.component.ts index 79ef75db..fbc15cb4 100644 --- a/dynamic-mapping-ui/src/monitoring/chart/chart.component.ts +++ b/dynamic-mapping-ui/src/monitoring/chart/chart.component.ts @@ -20,8 +20,8 @@ export class MonitoringChartComponent implements OnInit, OnDestroy { mappingStatus$: Subject = new Subject(); echartOptions: EChartsOption; echartUpdateOptions: EChartsOption; - subscription: object; echartsInstance: any; + subscription: object; textColor: string; fontFamily: string; fontWeight: number; @@ -124,53 +124,10 @@ export class MonitoringChartComponent implements OnInit, OnDestroy { } ] }; - this.echartsInstance?.setOption(this.echartUpdateOptions); + // this.echartsInstance?.setOption(this.echartUpdateOptions); } }); - // const config = { - // type: 'bar' as any, - // data: data, - // options: { - // responsive: true, - // maintainAspectRatio: false, - // layout: { - // padding: { - // left: 0, - // right: 0, - // top: 0, - // bottom: 0 - // } - // }, - // plugins: { - // legend: { - // display: true, - // position: 'left' as any, - // font: { - // family: this.fontFamily, - // weight: 'normal' as any - // } - // } - // }, - // indexAxis: 'y' as any, - // color: this.textColor as any, - // scales: { - // y: { - // ticks: { - // color: this.textColor as any - // } - // }, - // x: { - // ticks: { - // color: this.textColor as any, - // stepSize: 0 - // } - // } - // } - // } - // }; - // this.statusMappingChart = new Chart('monitoringChart', config); - const yAxisData = [ 'Errors', 'Messages received', diff --git a/dynamic-mapping-ui/src/shared/connector-configuration.service.ts b/dynamic-mapping-ui/src/shared/connector-configuration.service.ts index e13e0baa..53c9ae0c 100644 --- a/dynamic-mapping-ui/src/shared/connector-configuration.service.ts +++ b/dynamic-mapping-ui/src/shared/connector-configuration.service.ts @@ -39,6 +39,7 @@ import { forkJoin, from, Observable, + ReplaySubject, Subject } from 'rxjs'; import { map, shareReplay, switchMap, tap } from 'rxjs/operators'; @@ -50,8 +51,7 @@ export class ConnectorConfigurationService { private client: FetchClient, private sharedService: SharedService ) { - this.initConnectorConfigurations(); - this.startConnectorStatusSubscriptions(); + this.startConnectorConfigurations(); this.realtime = new Realtime(this.client); } @@ -60,9 +60,8 @@ export class ConnectorConfigurationService { private triggerConfigurations$: Subject = new Subject(); private realtimeConnectorStatus$: Subject = new Subject(); - private realtimeConnectorConfigurations$: Observable< - ConnectorConfiguration[] - >; + private realtimeConnectorConfigurations$: Subject = + new ReplaySubject(1); private enrichedConnectorConfiguration$: Observable; private _agentId: string; @@ -125,30 +124,33 @@ export class ConnectorConfigurationService { } }); return configurations; - }), - shareReplay(1) + }) + // shareReplay(1) ); - this.realtimeConnectorConfigurations$ = combineLatest([ + combineLatest([ this.enrichedConnectorConfiguration$, this.realtimeConnectorStatus$ - ]).pipe( - map((vars) => { - const [configurations, payload] = vars; - if (payload?.type == StatusEventTypes.STATUS_CONNECTOR_EVENT_TYPE) { - const statusLog: ConnectorStatusEvent = payload[CONNECTOR_FRAGMENT]; - configurations.forEach((cc) => { - if (statusLog['connectorIdent'] == cc.ident) { - if (!cc['status$']) { - cc['status$'] = new BehaviorSubject(statusLog.status); - } else { - cc['status$'].next(statusLog.status); + ]) + .pipe( + map((vars) => { + const [configurations, payload] = vars; + if (payload?.type == StatusEventTypes.STATUS_CONNECTOR_EVENT_TYPE) { + const statusLog: ConnectorStatusEvent = payload[CONNECTOR_FRAGMENT]; + configurations.forEach((cc) => { + if (statusLog['connectorIdent'] == cc.ident) { + if (!cc['status$']) { + cc['status$'] = new BehaviorSubject(statusLog.status); + } else { + cc['status$'].next(statusLog.status); + } } - } - }); - } - return configurations; - }) - ); + }); + } + return configurations; + }), + tap((confs) => this.realtimeConnectorConfigurations$.next(confs)) + ) + .subscribe(); } async getConnectorSpecifications(): Promise { diff --git a/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.ts b/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.ts index 0d8cd662..20e3ba20 100644 --- a/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.ts +++ b/dynamic-mapping-ui/src/shared/connector-configuration/connector-grid.component.ts @@ -41,7 +41,13 @@ import { Pagination } from '@c8y/ngx-components'; import { BsModalRef, BsModalService } from 'ngx-bootstrap/modal'; -import { BehaviorSubject, from, Observable, Subject } from 'rxjs'; +import { + BehaviorSubject, + from, + Observable, + ReplaySubject, + Subject +} from 'rxjs'; import * as _ from 'lodash'; import { ConfirmationModalComponent } from '../confirmation/confirmation-modal.component'; @@ -66,9 +72,7 @@ import { CheckedRendererComponent } from './checked-renderer.component'; styleUrls: ['./connector-grid.component.style.css'], templateUrl: 'connector-grid.component.html' }) -export class ConnectorConfigurationComponent - implements OnInit, OnDestroy, AfterViewInit -{ +export class ConnectorConfigurationComponent implements OnInit, AfterViewInit { @Input() selectable = true; @Input() readOnly = false; @Input() deploy: string[]; @@ -88,7 +92,7 @@ export class ConnectorConfigurationComponent monitoring$: Observable; specifications: ConnectorSpecification[] = []; configurations: ConnectorConfiguration[]; - configurations$: Subject = new Subject(); + configurations$: Subject = new ReplaySubject(1); StatusEventTypes = StatusEventTypes; pagination: Pagination = { pageSize: 30, @@ -218,7 +222,9 @@ export class ConnectorConfigurationComponent this.connectorConfigurationService .getRealtimeConnectorConfigurations() - .subscribe((confs) => this.configurations$.next(confs)); + .subscribe((confs) => { + this.configurations$.next(confs); + }); this.configurations$.subscribe((confs) => { this.configurations = confs; @@ -227,7 +233,6 @@ export class ConnectorConfigurationComponent (conf) => (conf['checked'] = this.selected.includes(conf.ident)) ); }); - this.connectorConfigurationService.startConnectorConfigurations(); } public onSelectToggle(id: string) { @@ -458,8 +463,4 @@ export class ConnectorConfigurationComponent findNameByIdent(ident: string): string { return this.configurations?.find((conf) => conf.ident == ident)?.name; } - - ngOnDestroy(): void { - this.connectorConfigurationService.stopConnectorConfigurations(); - } } diff --git a/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.ts b/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.ts index 15a79ddc..e260d20c 100644 --- a/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.ts +++ b/dynamic-mapping-ui/src/shared/connector-log/connector-status.component.ts @@ -39,7 +39,7 @@ import { ConnectorConfigurationService } from '../connector-configuration.servic styleUrls: ['./connector-status.component.style.css'], templateUrl: 'connector-status.component.html' }) -export class ConnectorStatusComponent implements OnInit, OnDestroy { +export class ConnectorStatusComponent implements OnInit { version: string = packageJson.version; monitorings$: Observable; feature: Feature; @@ -66,21 +66,15 @@ export class ConnectorStatusComponent implements OnInit, OnDestroy { // console.log('Running version', this.version); this.feature = await this.sharedService.getFeatures(); this.statusLogs$ = this.connectorStatusService.getStatusLogs(); - await this.connectorStatusService.startConnectorStatusLogs(); this.connectorConfigurationService .getRealtimeConnectorConfigurations() .subscribe((confs) => { this.configurations = confs; }); - await this.connectorConfigurationService.startConnectorConfigurations(); } updateStatusLogs() { this.connectorStatusService.updateStatusLogs(this.filterStatusLog); } - - ngOnDestroy(): void { - this.connectorStatusService.stopConnectorStatusLogs(); - } } diff --git a/dynamic-mapping-ui/src/shared/connector-status.service.ts b/dynamic-mapping-ui/src/shared/connector-status.service.ts index 608b4462..a6c28f7e 100644 --- a/dynamic-mapping-ui/src/shared/connector-status.service.ts +++ b/dynamic-mapping-ui/src/shared/connector-status.service.ts @@ -28,8 +28,16 @@ import { SharedService } from '../shared'; -import { merge, Observable, Subject } from 'rxjs'; -import { filter, map, scan, share, switchMap, tap } from 'rxjs/operators'; +import { merge, Observable, ReplaySubject, Subject } from 'rxjs'; +import { + filter, + map, + scan, + share, + shareReplay, + switchMap, + tap +} from 'rxjs/operators'; @Injectable({ providedIn: 'root' }) export class ConnectorStatusService { @@ -39,6 +47,7 @@ export class ConnectorStatusService { private sharedService: SharedService ) { this.realtime = new Realtime(this.client); + this.startConnectorStatusLogs(); } private _agentId: string; @@ -52,14 +61,14 @@ export class ConnectorStatusService { }; private triggerLogs$: Subject = new Subject(); private realtimeConnectorStatus$: Subject = new Subject(); - private statusLogs$: Subject = new Subject(); + private statusLogs$: Subject = new ReplaySubject(1); getStatusLogs(): Observable { return this.statusLogs$; } async startConnectorStatusLogs() { - // console.log('Calling: startConnectorStatusLogs'); + console.log('Calling: startConnectorStatusLogs', this.initialized); if (!this.initialized) { this.startConnectorStatusSubscriptions(); await this.initConnectorLogsRealtime(); @@ -113,9 +122,8 @@ export class ConnectorStatusService { ? true : event.connectorIdent == this.filterStatusLog.connectorIdent; }) - ), - share(), - tap((x) => console.log('TriggerLogs Out', x)) + ) + // tap((x) => console.log('TriggerLogs Out', x)) ); const realtimeConnectorStatusRealtime$ = this.realtimeConnectorStatus$.pipe( @@ -158,9 +166,9 @@ export class ConnectorStatusService { return sortedAcc; }, []), tap((logs) => this.statusLogs$.next(logs)) + // shareReplay(1) ) .subscribe(); - // refreshedConnectorStatus$.subscribe((logs) => this.statusLogs$.next(logs)); } async startConnectorStatusSubscriptions(): Promise { From 63817a2cc42481ed221a74d5691ce6cc0ccbad17 Mon Sep 17 00:00:00 2001 From: sagIoTPower Date: Mon, 9 Sep 2024 16:30:41 +0200 Subject: [PATCH 07/11] fix for bug #253 --- .../mapping/rest/MappingRestController.java | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/rest/MappingRestController.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/rest/MappingRestController.java index a3c5f946..a17b60c7 100644 --- a/dynamic-mapping-service/src/main/java/dynamic/mapping/rest/MappingRestController.java +++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/rest/MappingRestController.java @@ -161,7 +161,7 @@ public ResponseEntity createConnectorConfiguration( } // Remove sensitive data before printing to log ConnectorSpecification connectorSpecification = connectorRegistry - .getConnectorSpecification(configuration.connectorType); + .getConnectorSpecification(configuration.connectorType); ConnectorConfiguration clonedConfig = configuration.getCleanedConfig(connectorSpecification); log.info("Tenant {} - Post Connector configuration: {}", tenant, clonedConfig.toString()); try { @@ -186,7 +186,7 @@ public ResponseEntity> getConnectionConfigurations( // Remove sensitive data before sending to UI for (ConnectorConfiguration config : configurations) { ConnectorSpecification connectorSpecification = connectorRegistry - .getConnectorSpecification(config.connectorType); + .getConnectorSpecification(config.connectorType); ConnectorConfiguration cleanedConfig = config.getCleanedConfig(connectorSpecification); modifiedConfigs.add(cleanedConfig); } @@ -210,14 +210,24 @@ public ResponseEntity deleteConnectionConfiguration(@PathVariable String try { ConnectorConfiguration configuration = connectorConfigurationComponent.getConnectorConfiguration(ident, tenant); - AConnectorClient client = connectorRegistry.getClientForTenant(tenant, - configuration.getIdent()); - if (client == null) - return ResponseEntity.status(HttpStatus.BAD_REQUEST).body("Client with ident " + ident + " not found"); - client.disconnect(); - bootstrapService.shutdownAndRemoveConnector(tenant, client.getConnectorIdent()); + if (configuration.enabled) + return ResponseEntity.status(HttpStatus.BAD_REQUEST) + .body("Can't delete an enabled connector! Disable connector first."); connectorConfigurationComponent.deleteConnectorConfiguration(ident); mappingComponent.removeConnectorFromDeploymentMap(tenant, ident); + // NOTE this block was disabled since a disabled connector is not registered in + // connectorRegistry + + // AConnectorClient client = connectorRegistry.getClientForTenant(tenant, + // configuration.getIdent()); + // if (client == null) + // return ResponseEntity.status(HttpStatus.BAD_REQUEST).body("Client with ident + // " + ident + " not found"); + // client.disconnect(); + // bootstrapService.shutdownAndRemoveConnector(tenant, + // client.getConnectorIdent()); + // connectorConfigurationComponent.deleteConnectorConfiguration(ident); + // mappingComponent.removeConnectorFromDeploymentMap(tenant, ident); } catch (Exception ex) { log.error("Tenant {} - Error getting mqtt broker configuration {}", tenant, ex); throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, ex.getLocalizedMessage()); @@ -241,7 +251,7 @@ public ResponseEntity updateConnectionConfiguration(@Pat } // Remove sensitive data before printing to log ConnectorSpecification connectorSpecification = connectorRegistry - .getConnectorSpecification(configuration.connectorType); + .getConnectorSpecification(configuration.connectorType); ConnectorConfiguration clonedConfig = configuration.getCleanedConfig(connectorSpecification); log.info("Tenant {} - Post Connector configuration: {}", tenant, clonedConfig.toString()); try { @@ -420,8 +430,9 @@ public ResponseEntity runOperation(@Valid @RequestBody ServiceOperat configuration.setEnabled(true); connectorConfigurationComponent.saveConnectorConfiguration(configuration); - //Initialize Connector only when enabled. - ServiceConfiguration serviceConfiguration = serviceConfigurationComponent.getServiceConfiguration(tenant); + // Initialize Connector only when enabled. + ServiceConfiguration serviceConfiguration = serviceConfigurationComponent + .getServiceConfiguration(tenant); bootstrapService.initializeConnectorByConfiguration(configuration, serviceConfiguration, tenant); configurationRegistry.getNotificationSubscriber().notificationSubscriberReconnect(tenant); @@ -438,9 +449,9 @@ public ResponseEntity runOperation(@Valid @RequestBody ServiceOperat AConnectorClient client = connectorRegistry.getClientForTenant(tenant, connectorIdent); - //client.submitDisconnect(); + // client.submitDisconnect(); bootstrapService.disableConnector(tenant, client.getConnectorIdent()); - //We might need to Reconnect other Notification Clients for other connectors + // We might need to Reconnect other Notification Clients for other connectors configurationRegistry.getNotificationSubscriber().notificationSubscriberReconnect(tenant); } else if (operation.getOperation().equals(Operation.REFRESH_STATUS_MAPPING)) { mappingComponent.sendMappingStatus(tenant); From ddcdd47e01129089f47b20c0b7f87b1bbf2e8f21 Mon Sep 17 00:00:00 2001 From: sagIoTPower Date: Mon, 9 Sep 2024 16:36:15 +0200 Subject: [PATCH 08/11] disabling userRoles, #254 --- .../src/main/resources/application.properties | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dynamic-mapping-service/src/main/resources/application.properties b/dynamic-mapping-service/src/main/resources/application.properties index 83bcafe5..4d497441 100644 --- a/dynamic-mapping-service/src/main/resources/application.properties +++ b/dynamic-mapping-service/src/main/resources/application.properties @@ -24,7 +24,8 @@ application.version=@project.version@ server.port=8080 server.error.include-message=always APP.externalExtensionsEnabled=true -APP.userRolesEnabled=true +# ensure to leave this disabled, since the UI does not support this at the moment +APP.userRolesEnabled=false APP.mappingAdminRole=ROLE_MAPPING_ADMIN APP.mappingCreateRole=ROLE_MAPPING_CREATE # set to false to enable Open Telemetry Instrumentation From 40b0900426356a6bd224002f2721dcb7535ffae4 Mon Sep 17 00:00:00 2001 From: Stefan Witschel Date: Tue, 10 Sep 2024 09:15:39 +0200 Subject: [PATCH 09/11] Adding proper subscriber deletion on connector delete --- .../main/java/dynamic/mapping/rest/MappingRestController.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/rest/MappingRestController.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/rest/MappingRestController.java index a17b60c7..5f5ad4ef 100644 --- a/dynamic-mapping-service/src/main/java/dynamic/mapping/rest/MappingRestController.java +++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/rest/MappingRestController.java @@ -215,6 +215,7 @@ public ResponseEntity deleteConnectionConfiguration(@PathVariable String .body("Can't delete an enabled connector! Disable connector first."); connectorConfigurationComponent.deleteConnectorConfiguration(ident); mappingComponent.removeConnectorFromDeploymentMap(tenant, ident); + bootstrapService.shutdownAndRemoveConnector(tenant, ident); // NOTE this block was disabled since a disabled connector is not registered in // connectorRegistry From 99279c0b0757d7835c71e9f5d7115f65258c2912 Mon Sep 17 00:00:00 2001 From: Stefan Witschel Date: Tue, 10 Sep 2024 12:27:53 +0200 Subject: [PATCH 10/11] Adding proper subscriber deletion on connector delete --- .../main/java/dynamic/mapping/core/BootstrapService.java | 2 +- .../mapping/notification/C8YNotificationSubscriber.java | 9 +++++++-- .../java/dynamic/mapping/rest/MappingRestController.java | 6 +++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/BootstrapService.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/BootstrapService.java index 64f173f3..43605d3d 100644 --- a/dynamic-mapping-service/src/main/java/dynamic/mapping/core/BootstrapService.java +++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/core/BootstrapService.java @@ -203,7 +203,7 @@ public void initializeOutboundMapping(String tenant, ServiceConfiguration servic //shutdownAndRemoveConnector will unsubscribe the subscriber which drops all queues public void shutdownAndRemoveConnector(String tenant, String connectorIdent) throws ConnectorRegistryException { - connectorRegistry.unregisterClient(tenant, connectorIdent); + //connectorRegistry.unregisterClient(tenant, connectorIdent); ServiceConfiguration serviceConfiguration = serviceConfigurationComponent.getServiceConfiguration(tenant); if (serviceConfiguration.isOutboundMappingEnabled()) { configurationRegistry.getNotificationSubscriber().unsubscribeDeviceSubscriberByConnector(tenant, connectorIdent); diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/notification/C8YNotificationSubscriber.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/notification/C8YNotificationSubscriber.java index 439efed3..f27fd8b1 100644 --- a/dynamic-mapping-service/src/main/java/dynamic/mapping/notification/C8YNotificationSubscriber.java +++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/notification/C8YNotificationSubscriber.java @@ -465,8 +465,13 @@ public void unsubscribeDeviceSubscriber(String tenant) { public void unsubscribeDeviceSubscriberByConnector(String tenant, String connectorIdent) { if (deviceTokenPerConnector.get(tenant) != null) { if (deviceTokenPerConnector.get(tenant).get(connectorIdent) != null) { - tokenApi.unsubscribe(new Token(deviceTokenPerConnector.get(tenant).get(connectorIdent))); - deviceTokenPerConnector.get(tenant).remove(connectorIdent); + try { + tokenApi.unsubscribe(new Token(deviceTokenPerConnector.get(tenant).get(connectorIdent))); + log.info("Tenant {} - Subscriber for Connector {} successfully unsubscribed for Notification 2.0!", tenant, connectorIdent); + deviceTokenPerConnector.get(tenant).remove(connectorIdent); + } catch (SDKException e) { + log.error("Tenant {} - Could not unsubscribe subscriber for connector {}:", tenant, connectorIdent, e); + } } } } diff --git a/dynamic-mapping-service/src/main/java/dynamic/mapping/rest/MappingRestController.java b/dynamic-mapping-service/src/main/java/dynamic/mapping/rest/MappingRestController.java index 5f5ad4ef..ef196caf 100644 --- a/dynamic-mapping-service/src/main/java/dynamic/mapping/rest/MappingRestController.java +++ b/dynamic-mapping-service/src/main/java/dynamic/mapping/rest/MappingRestController.java @@ -273,9 +273,9 @@ public ResponseEntity updateConnectionConfiguration(@Pat } } connectorConfigurationComponent.saveConnectorConfiguration(configuration); - AConnectorClient client = connectorRegistry.getClientForTenant(tenant, - configuration.getIdent()); - client.reconnect(); + //AConnectorClient client = connectorRegistry.getClientForTenant(tenant, + // configuration.getIdent()); + //client.reconnect(); } catch (Exception ex) { log.error("Tenant {} - Error getting mqtt broker configuration {}", tenant, ex); throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, ex.getLocalizedMessage()); From b681ab150ff394abcc7dbed3c7f74c44efb212d2 Mon Sep 17 00:00:00 2001 From: Stefan Witschel Date: Tue, 10 Sep 2024 12:28:41 +0200 Subject: [PATCH 11/11] Prepare 4.5.1 release --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0181858d..049fc3fd 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ UTF-8 - 4.5.1-SNAPSHOT + 4.5.1 1020.107.0 1.18.34 1.7.36