Skip to content

Commit

Permalink
Process measurements in parallel for InfluxDB sinks
Browse files Browse the repository at this point in the history
  • Loading branch information
@Scrin
Scrin committed Jan 10, 2024
1 parent 9c8173e commit 61aa237
Show file tree
Hide file tree
Showing 2 changed files with 70 additions and 66 deletions.
68 changes: 35 additions & 33 deletions data_sinks/influxdb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,39 +45,41 @@ func InfluxDB(conf config.InfluxDBPublisher) chan<- parser.Measurement {
log.WithFields(log.Fields{"mac": measurement.Mac}).Trace("Skipping InfluxDB publish due to interval limit")
continue
}
p := influxdb.NewPointWithMeasurement(measurementName).
AddTag("dataFormat", fmt.Sprintf("%d", measurement.DataFormat)).
AddTag("mac", strings.ReplaceAll(measurement.Mac, ":", ""))
if measurement.Name != nil {
p.AddTag("name", *measurement.Name)
}
for tag, value := range conf.AdditionalTags {
p.AddTag(tag, value)
}
addFloat(p, "temperature", measurement.Temperature)
addFloat(p, "humidity", measurement.Humidity)
addFloat(p, "pressure", measurement.Pressure)
addFloat(p, "accelerationX", measurement.AccelerationX)
addFloat(p, "accelerationY", measurement.AccelerationY)
addFloat(p, "accelerationZ", measurement.AccelerationZ)
addFloat(p, "batteryVoltage", measurement.BatteryVoltage)
addInt(p, "txPower", measurement.TxPower)
addInt(p, "rssi", measurement.Rssi)
addInt(p, "movementCounter", measurement.MovementCounter)
addInt(p, "measurementSequenceNumber", measurement.MeasurementSequenceNumber)
addFloat(p, "accelerationTotal", measurement.AccelerationTotal)
addFloat(p, "absoluteHumidity", measurement.AbsoluteHumidity)
addFloat(p, "dewPoint", measurement.DewPoint)
addFloat(p, "equilibriumVaporPressure", measurement.EquilibriumVaporPressure)
addFloat(p, "airDensity", measurement.AirDensity)
addFloat(p, "accelerationAngleFromX", measurement.AccelerationAngleFromX)
addFloat(p, "accelerationAngleFromY", measurement.AccelerationAngleFromY)
addFloat(p, "accelerationAngleFromZ", measurement.AccelerationAngleFromZ)
p.SetTime(time.Now())
err := writeAPI.WritePoint(context.Background(), p)
if err != nil {
log.WithError(err).Error("Failed to send data to InfluxDB")
}
go func(measurement parser.Measurement) {
p := influxdb.NewPointWithMeasurement(measurementName).
AddTag("dataFormat", fmt.Sprintf("%d", measurement.DataFormat)).
AddTag("mac", strings.ReplaceAll(measurement.Mac, ":", ""))
if measurement.Name != nil {
p.AddTag("name", *measurement.Name)
}
for tag, value := range conf.AdditionalTags {
p.AddTag(tag, value)
}
addFloat(p, "temperature", measurement.Temperature)
addFloat(p, "humidity", measurement.Humidity)
addFloat(p, "pressure", measurement.Pressure)
addFloat(p, "accelerationX", measurement.AccelerationX)
addFloat(p, "accelerationY", measurement.AccelerationY)
addFloat(p, "accelerationZ", measurement.AccelerationZ)
addFloat(p, "batteryVoltage", measurement.BatteryVoltage)
addInt(p, "txPower", measurement.TxPower)
addInt(p, "rssi", measurement.Rssi)
addInt(p, "movementCounter", measurement.MovementCounter)
addInt(p, "measurementSequenceNumber", measurement.MeasurementSequenceNumber)
addFloat(p, "accelerationTotal", measurement.AccelerationTotal)
addFloat(p, "absoluteHumidity", measurement.AbsoluteHumidity)
addFloat(p, "dewPoint", measurement.DewPoint)
addFloat(p, "equilibriumVaporPressure", measurement.EquilibriumVaporPressure)
addFloat(p, "airDensity", measurement.AirDensity)
addFloat(p, "accelerationAngleFromX", measurement.AccelerationAngleFromX)
addFloat(p, "accelerationAngleFromY", measurement.AccelerationAngleFromY)
addFloat(p, "accelerationAngleFromZ", measurement.AccelerationAngleFromZ)
p.SetTime(time.Now())
err := writeAPI.WritePoint(context.Background(), p)
if err != nil {
log.WithError(err).Error("Failed to send data to InfluxDB")
}
}(measurement)
}
client.Close()
}()
Expand Down
68 changes: 35 additions & 33 deletions data_sinks/influxdb3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,39 +46,41 @@ func InfluxDB3(conf config.InfluxDB3Publisher) chan<- parser.Measurement {
log.WithFields(log.Fields{"mac": measurement.Mac}).Trace("Skipping InfluxDB3 publish due to interval limit")
continue
}
p := influxdb3.NewPointWithMeasurement(measurementName).
AddTag("dataFormat", fmt.Sprintf("%d", measurement.DataFormat)).
AddTag("mac", strings.ReplaceAll(measurement.Mac, ":", ""))
if measurement.Name != nil {
p.AddTag("name", *measurement.Name)
}
for tag, value := range conf.AdditionalTags {
p.AddTag(tag, value)
}
influx3AddFloat(p, "temperature", measurement.Temperature)
influx3AddFloat(p, "humidity", measurement.Humidity)
influx3AddFloat(p, "pressure", measurement.Pressure)
influx3AddFloat(p, "accelerationX", measurement.AccelerationX)
influx3AddFloat(p, "accelerationY", measurement.AccelerationY)
influx3AddFloat(p, "accelerationZ", measurement.AccelerationZ)
influx3AddFloat(p, "batteryVoltage", measurement.BatteryVoltage)
influx3AddInt(p, "txPower", measurement.TxPower)
influx3AddInt(p, "rssi", measurement.Rssi)
influx3AddInt(p, "movementCounter", measurement.MovementCounter)
influx3AddInt(p, "measurementSequenceNumber", measurement.MeasurementSequenceNumber)
influx3AddFloat(p, "accelerationTotal", measurement.AccelerationTotal)
influx3AddFloat(p, "absoluteHumidity", measurement.AbsoluteHumidity)
influx3AddFloat(p, "dewPoint", measurement.DewPoint)
influx3AddFloat(p, "equilibriumVaporPressure", measurement.EquilibriumVaporPressure)
influx3AddFloat(p, "airDensity", measurement.AirDensity)
influx3AddFloat(p, "accelerationAngleFromX", measurement.AccelerationAngleFromX)
influx3AddFloat(p, "accelerationAngleFromY", measurement.AccelerationAngleFromY)
influx3AddFloat(p, "accelerationAngleFromZ", measurement.AccelerationAngleFromZ)
p.SetTimestamp(time.Now())
err := client.WritePoints(context.Background(), p)
if err != nil {
log.WithError(err).Error("Failed to send data to InfluxDB3")
}
go func(measurement parser.Measurement) {
p := influxdb3.NewPointWithMeasurement(measurementName).
AddTag("dataFormat", fmt.Sprintf("%d", measurement.DataFormat)).
AddTag("mac", strings.ReplaceAll(measurement.Mac, ":", ""))
if measurement.Name != nil {
p.AddTag("name", *measurement.Name)
}
for tag, value := range conf.AdditionalTags {
p.AddTag(tag, value)
}
influx3AddFloat(p, "temperature", measurement.Temperature)
influx3AddFloat(p, "humidity", measurement.Humidity)
influx3AddFloat(p, "pressure", measurement.Pressure)
influx3AddFloat(p, "accelerationX", measurement.AccelerationX)
influx3AddFloat(p, "accelerationY", measurement.AccelerationY)
influx3AddFloat(p, "accelerationZ", measurement.AccelerationZ)
influx3AddFloat(p, "batteryVoltage", measurement.BatteryVoltage)
influx3AddInt(p, "txPower", measurement.TxPower)
influx3AddInt(p, "rssi", measurement.Rssi)
influx3AddInt(p, "movementCounter", measurement.MovementCounter)
influx3AddInt(p, "measurementSequenceNumber", measurement.MeasurementSequenceNumber)
influx3AddFloat(p, "accelerationTotal", measurement.AccelerationTotal)
influx3AddFloat(p, "absoluteHumidity", measurement.AbsoluteHumidity)
influx3AddFloat(p, "dewPoint", measurement.DewPoint)
influx3AddFloat(p, "equilibriumVaporPressure", measurement.EquilibriumVaporPressure)
influx3AddFloat(p, "airDensity", measurement.AirDensity)
influx3AddFloat(p, "accelerationAngleFromX", measurement.AccelerationAngleFromX)
influx3AddFloat(p, "accelerationAngleFromY", measurement.AccelerationAngleFromY)
influx3AddFloat(p, "accelerationAngleFromZ", measurement.AccelerationAngleFromZ)
p.SetTimestamp(time.Now())
err := client.WritePoints(context.Background(), p)
if err != nil {
log.WithError(err).Error("Failed to send data to InfluxDB3")
}
}(measurement)
}
client.Close()
}()
Expand Down

0 comments on commit 61aa237

Please sign in to comment.