Skip to content

Commit

Permalink
docker: reset timer after collecting stats
Browse files Browse the repository at this point in the history
In ##23966 when we switched to using the official Docker SDK client, we had to
rework the stats collection loop for the new client. But we missed resetting the
timer on the collection loop, which meant that we'd only collect stats once and
then never again.

* Ref: [NET-11202 (comment)](https://hashicorp.atlassian.net/browse/NET-11202?focusedCommentId=550814)
* This has shipped in Nomad 1.9.0-beta.1 but not production yet.
  • Loading branch information
tgross committed Sep 30, 2024
1 parent 52b2711 commit 2ecc6d3
Showing 1 changed file with 20 additions and 21 deletions.
41 changes: 20 additions & 21 deletions drivers/docker/stats.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,34 +97,33 @@ func (h *taskHandle) collectStats(ctx context.Context, destCh *usageSender, inte
timer, cancel := helper.NewSafeTimer(interval)
defer cancel()

collectOnce := func() {
defer timer.Reset(interval)
statsReader, err := h.dockerClient.ContainerStatsOneShot(ctx, h.containerID)
if err != nil && err != io.EOF {
h.logger.Debug("error collecting stats from container", "error", err)
return
}
defer statsReader.Body.Close()

var stats containerapi.Stats
if err := json.NewDecoder(statsReader.Body).Decode(&stats); err != nil {
h.logger.Error("error decoding stats data for container", "error", err)
return
}

resourceUsage := util.DockerStatsToTaskResourceUsage(&stats, compute)
destCh.send(resourceUsage)
}

for {
select {
case <-ctx.Done():
return
case <-h.doneCh:
return
case <-timer.C:
// ContainerStats returns a StatsResponseReader. Body of that reader
// contains the stats and implements io.Reader
statsReader, err := h.dockerClient.ContainerStatsOneShot(ctx, h.containerID)
if err != nil && err != io.EOF {
// An error occurred during stats collection, retry with backoff
h.logger.Debug("error collecting stats from container", "error", err)
continue
}

var stats containerapi.Stats

if err := json.NewDecoder(statsReader.Body).Decode(&stats); err != nil {
h.logger.Error("error unmarshalling stats data for container", "error", err)
_ = statsReader.Body.Close()
continue
}

resourceUsage := util.DockerStatsToTaskResourceUsage(&stats, compute)
destCh.send(resourceUsage)

_ = statsReader.Body.Close()
collectOnce()
}
}
}

0 comments on commit 2ecc6d3

Please sign in to comment.