serve/rest: improve prometheus metrics encoding (#5143)

This tweaks the logic behind the `/metrics` endpoint in order to
properly log encoding errors. It also switches the existing code
to use a pre-allocated String buffer, skipping some small
reallocations while encoding and avoiding double-parsing an already
valid UTF-8 string.

quickwit/quickwit-common/src/metrics.rs

@@ -26,7 +26,7 @@ pub use prometheus::{
     IntCounter, IntCounterVec as PrometheusIntCounterVec, IntGauge,
     IntGaugeVec as PrometheusIntGaugeVec,
 };
-use prometheus::{Encoder, Gauge, HistogramOpts, Opts, TextEncoder};
+use prometheus::{Gauge, HistogramOpts, Opts, TextEncoder};
 
 #[derive(Clone)]
 pub struct HistogramVec<const N: usize> {
@@ -305,12 +305,16 @@ impl Drop for OwnedGaugeGuard {
     }
 }
 
-pub fn metrics_text_payload() -> String {
+pub fn metrics_text_payload() -> Result<String, String> {
     let metric_families = prometheus::gather();
-    let mut buffer = Vec::new();
+    // Arbitrary non-zero size in order to skip a bunch of
+    // buffer growth-reallocations when encoding metrics.
+    let mut buffer = String::with_capacity(1024);
     let encoder = TextEncoder::new();
-    let _ = encoder.encode(&metric_families, &mut buffer); // TODO avoid ignoring the error.
-    String::from_utf8_lossy(&buffer).to_string()
+    match encoder.encode_utf8(&metric_families, &mut buffer) {
+        Ok(()) => Ok(buffer),
+        Err(e) => Err(e.to_string()),
+    }
 }
 
 #[derive(Clone)]

quickwit/quickwit-serve/src/metrics_api.rs

@@ -17,6 +17,10 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program. If not, see <http://www.gnu.org/licenses/>.
 
+use hyper::StatusCode;
+use tracing::error;
+use warp::reply::with_status;
+
 #[derive(utoipa::OpenApi)]
 #[openapi(paths(metrics_handler))]
 /// Endpoints which are weirdly tied to another crate with no
@@ -32,11 +36,18 @@ pub struct MetricsApi;
     path = "/",
     responses(
         (status = 200, description = "Successfully fetched metrics.", body = String),
+        (status = 500, description = "Metrics not available.", body = String),
     ),
 )]
 /// Get Node Metrics
 ///
 /// These are in the form of prometheus metrics.
 pub fn metrics_handler() -> impl warp::Reply {
-    quickwit_common::metrics::metrics_text_payload()
+    match quickwit_common::metrics::metrics_text_payload() {
+        Ok(metrics) => with_status(metrics, StatusCode::OK),
+        Err(e) => {
+            error!("failed to encode prometheus metrics: {e}");
+            with_status(String::new(), StatusCode::INTERNAL_SERVER_ERROR)
+        }
+    }
 }