Skip to content

Commit

Permalink
Grafana Loki support (#155)
Browse files Browse the repository at this point in the history
Signed-off-by: Sanjula Ganepola <Sanjula.Ganepola@ibm.com>
  • Loading branch information
SanjulaGanepola authored Oct 8, 2024
1 parent a981da3 commit 71587bc
Show file tree
Hide file tree
Showing 21 changed files with 205 additions and 38 deletions.
16 changes: 16 additions & 0 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "java",
"name": "Run Manzan",
"request": "launch",
"mainClass": "com.github.theprez.manzan.ManzanMainApp",
"projectName": "manzan",
"args": "--configdir=${workspaceFolder}/config"
}
]
}
3 changes: 2 additions & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -103,5 +103,6 @@
"compare": "cpp",
"concepts": "cpp",
"algorithm": "cpp"
}
},
"java.configuration.updateBuildConfiguration": "automatic"
}
6 changes: 5 additions & 1 deletion camel/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,11 @@
<artifactId>fluent-logger</artifactId>
<version>0.3.4</version>
</dependency>

<dependency>
<groupId>io.github.mjfryc</groupId>
<artifactId>mjaron-tinyloki-java</artifactId>
<version>0.3.11</version>
</dependency>

<!-- testing -->
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import com.github.theprez.manzan.routes.dest.EmailDestination;
import com.github.theprez.manzan.routes.dest.FileDestination;
import com.github.theprez.manzan.routes.dest.FluentDDestination;
import com.github.theprez.manzan.routes.dest.GrafanaLokiDestination;
import com.github.theprez.manzan.routes.dest.HttpDestination;
import com.github.theprez.manzan.routes.dest.KafkaDestination;
import com.github.theprez.manzan.routes.dest.SentryDestination;
Expand Down Expand Up @@ -83,6 +84,12 @@ public synchronized Map<String, ManzanRoute> getRoutes(CamelContext context) {
final String host = getRequiredString(name, "host");
final int port = getRequiredInt(name, "port");
ret.put(name, new FluentDDestination(name, tag, host, port));
}
case "loki": {
final String url = getRequiredString(name, "url");
final String username = getRequiredString(name, "username");
final String password = getRequiredString(name, "password");
ret.put(name, new GrafanaLokiDestination(name, url, username, password, format));
}
break;
case "smtp":
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,23 @@ public abstract class ManzanRoute extends RouteBuilder {

protected static final String EVENT_TYPE = "event_type";

protected static final String SESSION_ID = "SESSION_ID";
protected static final String MSG_MESSAGE_ID = "MESSAGE_ID";
protected static final String MSG_MESSAGE_TYPE = "MESSAGE_TYPE";
protected static final String MSG_SEVERITY = "SEVERITY";
protected static final String JOB = "JOB";
protected static final String MSG_SENDING_USRPRF = "SENDING_USRPRF";
protected static final String MSG_SENDING_PROGRAM_NAME = "SENDING_PROGRAM_NAME";
protected static final String MSG_SENDING_MODULE_NAME = "SENDING_MODULE_NAME";
protected static final String MSG_SENDING_PROCEDURE_NAME = "SENDING_PROCEDURE_NAME";
protected static final String MSG_ORDINAL_POSITION = "ORDINAL_POSITION";
protected static final String MSG_MESSAGE_TIMESTAMP = "MESSAGE_TIMESTAMP";
protected static final String MSG_MESSAGE = "MESSAGE";

protected static final int SEVERITY_LIMIT = 29;

protected static String getWatchName(final Exchange exchange) {
final Object watchNameObject = exchange.getIn().getHeader("session_id");
final Object watchNameObject = exchange.getIn().getHeader(SESSION_ID);
if (null == watchNameObject) {
throw new RuntimeException("Couldn't figure out watch ID");
}
Expand Down Expand Up @@ -45,6 +60,10 @@ protected Map<String, Object> getDataMap(final Exchange _exchange) {
return (Map<String, Object>) _exchange.getIn().getHeader("data_map");
}

protected <T> T getBody(final Exchange _exchange, Class<T> type) {
return _exchange.getIn().getBody(type);
}

protected String getInUri() {
return "direct:" + m_name;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
package com.github.theprez.manzan.routes.dest;

import java.sql.Timestamp;

import com.github.theprez.manzan.ManzanEventType;
import com.github.theprez.manzan.routes.ManzanRoute;

import pl.mjaron.tinyloki.ILogStream;
import pl.mjaron.tinyloki.Labels;
import pl.mjaron.tinyloki.LogController;
import pl.mjaron.tinyloki.StreamBuilder;
import pl.mjaron.tinyloki.TinyLoki;

public class GrafanaLokiDestination extends ManzanRoute {
private final LogController logController;
private final static String endpoint = "/loki/api/v1/push";
private final static String appLabelName = "app";
private final static String appLabelValue = "manzan";

public GrafanaLokiDestination(final String _name, final String _url, final String _username, final String _password,
final String _format) {
super(_name);

logController = TinyLoki
.withUrl(_url + endpoint)
.withBasicAuth(_username, _password)
.start();

Runtime.getRuntime().addShutdownHook(new Thread() {
@Override
public void run() {
try {
logController
.softStop()
.hardStop();
} catch (final Exception e) {
e.printStackTrace();
}
}
});
}

//@formatter:off
@Override
public void configure() {
from(getInUri())
.routeId(m_name).process(exchange -> {
final ManzanEventType type = (ManzanEventType) exchange.getIn().getHeader(EVENT_TYPE);
if(ManzanEventType.WATCH_MSG == type) {
StreamBuilder builder = logController
.stream()
.l(appLabelName, appLabelValue)
.l(Labels.LEVEL, ((Integer) get(exchange, MSG_SEVERITY)) > SEVERITY_LIMIT ? Labels.FATAL : Labels.INFO)
.l(SESSION_ID, getWatchName(exchange));

String[] keys = {
MSG_MESSAGE_ID,
MSG_MESSAGE_TYPE,
MSG_SEVERITY,
JOB,
MSG_SENDING_USRPRF,
MSG_SENDING_PROGRAM_NAME,
MSG_SENDING_MODULE_NAME,
MSG_SENDING_PROCEDURE_NAME
};

for (String key: keys) {
String value = getString(exchange, key);
if(!value.equals("")) {
builder.l(key, value);
}
}

ILogStream stream = builder.build();
stream.log(Timestamp.valueOf(getString(exchange, MSG_MESSAGE_TIMESTAMP)).getTime(), getBody(exchange, String.class));
} else {
throw new RuntimeException("Grafana Loki route doesn't know how to process type "+type);
}
});
}
//@formatter:on
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,6 @@

public class SentryDestination extends ManzanRoute {
private static SentryDestination m_singleton = null;
private static final String MSG_MESSAGE = "MESSAGE";
private static final String MSG_MESSAGE_ID = "MESSAGE_ID";
private static final String MSG_ORDINAL_POSITION = "ORDINAL_POSITION";
private static final String MSG_SENDING_MODULE_NAME = "SENDING_MODULE_NAME";

private static final String MSG_SENDING_PROCEDURE_NAME = "SENDING_PROCEDURE_NAME";
private static final String MSG_SENDING_PROGRAM_NAME = "SENDING_PROGRAM_NAME";
private static final String MSG_SENDING_USRPRF = "SENDING_USRPRF";

private static final String MSG_SEVERITY = "SEVERITY";

public SentryDestination(final String _name, final String _dsn) {
super(_name);
Expand Down Expand Up @@ -68,7 +58,7 @@ public void configure() {

SentryLevel level;
final int sev = (Integer) get(exchange, MSG_SEVERITY);
if (sev > 29) {
if (sev > SEVERITY_LIMIT) {
level = SentryLevel.ERROR;
} else {
level = SentryLevel.INFO;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,6 @@ public TwilioDestination(CamelContext context, final String _name, final String

@Override
protected void customPostProcess(Exchange exchange) {
exchange.getIn().setHeader("CamelTwilio.body", exchange.getIn().getBody(String.class));
exchange.getIn().setHeader("CamelTwilio.body", getBody(exchange, String.class));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -43,21 +43,21 @@ public void configure() {
.setHeader(EVENT_TYPE, constant(ManzanEventType.FILE))
.setBody(constant(m_file.getAbsolutePath()))
.process((exchange) -> {
final File f = new File(exchange.getIn().getBody().toString());
final File f = new File(getBody(exchange, String.class));
exchange.getIn().setBody(new FileNewContentsReader(f, "*TAG"));
})
.split(body().tokenize("\n")).streaming().parallelProcessing(false).stopOnException()
.convertBodyTo(String.class)
.process(exchange -> {
String bodyStr = exchange.getIn().getBody(String.class);
String bodyStr = getBody(exchange, String.class);
exchange.getIn().setHeader("abort", m_filter.matches(bodyStr) && StringUtils.isNonEmpty(bodyStr)?"continue":"abort");
})
.choice().when(simple("${header.abort} != 'abort'"))
.process(exchange -> {
final Map<String,Object> data_map = new LinkedHashMap<String,Object>();
data_map.put(FILE_NAME, m_file.getName());
data_map.put(FILE_PATH, m_file.getAbsolutePath());
data_map.put(FILE_DATA, exchange.getIn().getBody(String.class).replace("\r",""));
data_map.put(FILE_DATA, getBody(exchange, String.class).replace("\r",""));
exchange.getIn().setHeader("data_map", data_map);
exchange.getIn().setBody(data_map);
})
Expand Down
2 changes: 1 addition & 1 deletion docs/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ Many other destinations will be available. Examples include:
- [Google Drive](http://drive.google.com)
- [Google Mail (gmail)](http://gmail.com)
- [Google Pub/Sub](http://cloud.google.com/pubsub)
- [Grafana Loki](https://grafana.com/oss/loki/)
- [Grafana Loki](https://grafana.com/oss/loki/)
- HTTP endpoints (REST, etc) ✅
- HTTPS endpoints (REST, etc) ✅
- [Internet of Things (mqtt)](https://www.eclipse.org/paho/)
Expand Down
3 changes: 2 additions & 1 deletion docs/_sidebar.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,5 +9,6 @@
* Examples
* [File](config/examples/file.md)
* [Twilio](config/examples/twilio.md)
* [Sentry watch](config/examples/sentry.md)
* [Sentry](config/examples/sentry.md)
* [Grafana Loki](config/examples/grafanaLoki.md)
* [Contributing](contributing.md)
27 changes: 17 additions & 10 deletions docs/config/dests.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,16 @@ As well, each section can provide `format` as an optional type.

Some types have additional properties that they require.

| id | Description | Required properties | Optional properties |
|------------------|---------------------------------|------------------------------------------------------------| |
| `stdout` | Write all data to standard out. | None. | |
| `slack` | Send data to a Slack channel | * `webhook` <br> * `channel` <br> | |
| `fluentd` | Sent data to FluentD | * `tag` <br> * `host` <br> * `port` <br> | |
| `smtp`/`smtps` | Sent data via email | * `server` <br> * `subject` <br> * `to` <br> * `from` <br> | * `port` |
| `sentry` | Send data into Sentry | * `dsn` | |
| `twilio` | Send via SMS | * `sid` <br> * `token` <br> * `to` <br> * `from` | |
| `http`/`https` | Send data via http/https | * `url` | * `httpMethod` <br> * `x` where x is any query parameter |
| id | Description | Required properties | Optional properties |
|------------------|---------------------------------|------------------------------------------------------------| -------------------------------------------------------- |
| `stdout` | Write all data to standard out. | None. | |
| `slack` | Send data to a Slack channel | * `webhook` <br> * `channel` | |
| `fluentd` | Sent data to FluentD | * `tag` <br> * `host` <br> * `port` | |
| `smtp`/`smtps` | Sent data via email | * `server` <br> * `subject` <br> * `to` <br> * `from` | * `port` |
| `sentry` | Send data into Sentry | * `dsn` | |
| `twilio` | Send via SMS | * `sid` <br> * `token` <br> * `to` <br> * `from` | |
| `loki` | Send data into Grafana Loki | * `url` <br> * `username` <br> * `password` <br> | |
| `http`/`https` | Send data via http/https | * `url` | * `httpMethod` <br> * `x` where x is any query parameter |

### Example

Expand All @@ -46,7 +47,7 @@ type=stdout

[sentry_out]
type=sentry
dsn=<slackdsn>
dsn=<sentry_dsn>

[twilio_sms]
type=twilio
Expand All @@ -55,6 +56,12 @@ token=x
to=+x
from=+x

[loki_out]
type=loki
url=<loki_url>
username=<loki_username>
password=<loki_password>

[slackme]
type=slack
channel=open-source-system-status
Expand Down
4 changes: 3 additions & 1 deletion docs/config/examples/file.md
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
# File

This example shows how to use `file` as a data source.

## Configuration

Be sure to have read up on [Manzan configuration](/config/index.md) to understand where this files exist on your system.
Be sure to have read up on [Manzan configuration](/config/index.md) to understand where these files exist on your system.

### `app.ini`

Expand Down
34 changes: 34 additions & 0 deletions docs/config/examples/grafanaLoki.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Grafana Loki

This example shows how to use `loki` as a destination for a `watch` data source.

## Configuration

Be sure to have read up on [Manzan configuration](/config/index.md) to understand where these files exist on your system.

### `data.ini`

```ini
type=watch
id=sanjula
destinations=loki_out
format=$MESSAGE_ID$ (severity $SEVERITY$): $MESSAGE$
strwch=WCHMSG((*ALL)) WCHMSGQ((*HSTLOG))
```

### `dests.ini`

```ini
[loki_out]
type=loki
url=<loki_url>
username=<loki_username>
password=<loki_password>
```

## Result

<div style="text-align: center; margin: 20px;">
<img src="../../images/grafanaLoki1.png" alt="Grafana Loki 1" style="box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2); border-radius: 8px; max-width: 100%; display: block; margin-bottom: 20px;">
<img src="../../images/grafanaLoki2.png" alt="Grafana Loki 2" style="box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2); border-radius: 8px; max-width: 100%;">
</div>
12 changes: 7 additions & 5 deletions docs/config/examples/sentry.md
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
This example shows how to use `sentry` as a destination for a `watch` data source
# Sentry

This example shows how to use `sentry` as a destination for a `watch` data source.

## Configuration

Be sure to have read up on [Manzan configuration](/config/index.md) to understand where this files exist on your system.
Be sure to have read up on [Manzan configuration](/config/index.md) to understand where these files exist on your system.

### `data.ini`

Expand All @@ -20,11 +22,11 @@ strwch=WCHMSG((*ALL)) WCHMSGQ((*HSTLOG))
```ini
[sentry_out]
type=sentry
dsn=<sentryDSN>
dsn=<sentry_dsn>
```

## Result

![](./images/sentry1.png)
![](../../images/sentry1.png)

![](./images/sentry2.png)
![](../../images/sentry2.png)
Loading

0 comments on commit 71587bc

Please sign in to comment.