Issues and Errors 2.0

.github/workflows/haskell.yml

@@ -81,7 +81,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v6
         with:
-          node-version: '18'
+          node-version: '22'
 
       - name: Generate required static files
         run: |

.github/workflows/pullrequest.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Setup node env
         uses: actions/setup-node@v6
         with:
-          node-version: 18
+          node-version: 22
 
       - name: Generate required static files
         run: |
@@ -125,7 +125,7 @@ jobs:
       - name: Setup node env
         uses: actions/setup-node@v6
         with:
-          node-version: 18
+          node-version: 22
 
       - name: Generate required static files
         run: |
@@ -198,7 +198,7 @@ jobs:
       - name: Setup node env
         uses: actions/setup-node@v6
         with:
-          node-version: 20
+          node-version: 22
       - name: Run UI tests
         run: |
           cd web-components

.gitignore

@@ -46,3 +46,5 @@ cabal.project.local
 stack-work
 tests.log
 tests_optimized.log
+.claude
+.playwright-mcp/

.hlint.yaml

@@ -17,6 +17,7 @@
   # - {name: System.Directory, as: Dir, importStyle: qualified, qualifiedStyle: post, asRequired: true}
 
 - arguments:
+  - "-XMultilineStrings"
   - "-XConstraintKinds"
   - "-XDeriveGeneric"
   - "-XGeneralizedNewtypeDeriving"

Dockerfile

@@ -1,5 +1,5 @@
 # Stage 1: Build frontend assets
-FROM node:18-alpine AS frontend-builder
+FROM node:22-alpine AS frontend-builder
 
 WORKDIR /build
 

cabal.project

@@ -39,8 +39,8 @@ source-repository-package
 
 source-repository-package
   type: git
-  location: https://github.com/tonyalaribe/proto-lens
-  tag: da3a3c7d8f43b7b22a3325a6706eb2aad98f41be
+  location: https://github.com/dawkaka/proto-lens
+  tag: cf4e060ea3376b68b65a9e207115f55b38cf1a12
   subdir: proto-lens-setup
   subdir: proto-lens-protoc
   subdir: proto-lens-runtime

monoscope.cabal

@@ -80,6 +80,8 @@ library
       Models.Apis.Monitors
       Models.Apis.Reports
       Models.Apis.RequestDumps
+      Models.Apis.Errors 
+      Models.Apis.LogPatterns
       Models.Apis.Shapes
       Models.Apis.Slack
       Models.Projects.Dashboards

package.yaml

@@ -13,7 +13,6 @@ custom-setup:
   dependencies:
     - base
     - Cabal
-    - proto-lens-setup
 
 build-tools:
   - proto-lens-protoc:proto-lens-protoc

src/Models/Apis/Anomalies.hs

@@ -1,7 +1,6 @@
 module Models.Apis.Anomalies (
   AnomalyVM (..),
   AnomalyActions (..),
-  Issue (..),
   IssueL (..),
   IssueEventAgg (..),
   AnomalyTypes (..),
@@ -350,42 +349,6 @@ instance Default IssuesData where
   def = IDEmpty
 
 
-data Issue = Issue
-  { id :: AnomalyId
-  , createdAt :: ZonedTime
-  , updatedAt :: ZonedTime
-  , projectId :: Projects.ProjectId
-  , acknowlegedAt :: Maybe ZonedTime
-  , anomalyType :: AnomalyTypes
-  , targetHash :: Text
-  , issueData :: IssuesData
-  , endpointId :: Maybe Endpoints.EndpointId
-  , acknowlegedBy :: Maybe Users.UserId
-  , archivedAt :: Maybe ZonedTime
-  , -- Enhanced UI fields
-    title :: Text
-  , service :: Text
-  , critical :: Bool
-  , breakingChanges :: Int
-  , incrementalChanges :: Int
-  , affectedRequests :: Int
-  , affectedClients :: Int
-  , estimatedRequests :: Text
-  , migrationComplexity :: Text
-  , recommendedAction :: Text
-  , requestPayloads :: Aeson [PayloadChange]
-  , responsePayloads :: Aeson [PayloadChange]
-  , -- New fields for anomaly grouping
-    anomalyHashes :: V.Vector Text
-  , endpointHash :: Text
-  }
-  deriving stock (Generic, Show)
-  deriving anyclass (Default, FromRow, NFData, ToRow)
-  deriving
-    (Entity)
-    via (GenericEntity '[Schema "apis", TableName "issues", PrimaryKey "id", FieldModifiers '[CamelToSnake]] Issue)
-
-
 data IssueEventAgg = IssueEventAgg
   { count :: Int
   , lastSeen :: UTCTime

src/Models/Apis/Endpoints.hs

@@ -8,10 +8,17 @@ module Models.Apis.Endpoints (
   EndpointRequestStats (..),
   Host (..),
   HostEvents (..),
+  EndpointWithCurrentRates (..),
+  EndpointStats (..),
   bulkInsertEndpoints,
   dependenciesAndEventsCount,
   endpointRequestStatsByProject,
   countEndpointInbox,
+  getEndpointsWithCurrentRates,
+  getEndpointStats,
+  updateEndpointBaseline,
+  getEndpointByHash,
+  getActiveEndpoints,
 )
 where
 
@@ -33,7 +40,7 @@ import Effectful.PostgreSQL (withConnection)
 import Effectful.PostgreSQL qualified as PG
 import Models.Projects.Projects qualified as Projects
 import NeatInterpolation (text)
-import Pkg.DeriveUtils (UUIDId (..))
+import Pkg.DeriveUtils (BaselineState (..), UUIDId (..))
 import Relude
 import System.Types (DB)
 
@@ -59,6 +66,20 @@ data Endpoint = Endpoint
   , hash :: Text
   , outgoing :: Bool
   , description :: Text
+  , firstTraceId :: Maybe Text
+  , recentTraceId :: Maybe Text
+  , service :: Maybe Text
+  , baselineState :: BaselineState
+  , baselineSamples :: Int
+  , baselineUpdatedAt :: Maybe UTCTime
+  , baselineErrorRateMean :: Maybe Double
+  , baselineErrorRateStddev :: Maybe Double
+  , baselineLatencyMean :: Maybe Double
+  , baselineLatencyStddev :: Maybe Double
+  , baselineLatencyP95 :: Maybe Double
+  , baselineLatencyP99 :: Maybe Double
+  , baselineVolumeHourlyMean :: Maybe Double
+  , baselineVolumeHourlyStddev :: Maybe Double
   }
   deriving stock (Eq, Generic, Show)
   deriving anyclass (Default, FromRow, NFData, ToRow)
@@ -70,8 +91,8 @@ bulkInsertEndpoints :: DB es => V.Vector Endpoint -> Eff es ()
 bulkInsertEndpoints endpoints = void $ PG.executeMany q $ V.toList rowsToInsert
   where
     q =
-      [sql| INSERT INTO apis.endpoints (project_id, url_path, url_params, method, host, hash, outgoing)
-          VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT (hash) DO  NOTHING;
+      [sql| INSERT INTO apis.endpoints (project_id, url_path, url_params, method, host, hash, outgoing, first_trace_id, recent_trace_id, service)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (hash) DO  NOTHING;
       |]
     rowsToInsert =
       endpoints <&> \endpoint ->
@@ -82,6 +103,9 @@ bulkInsertEndpoints endpoints = void $ PG.executeMany q $ V.toList rowsToInsert
         , endpoint.host
         , endpoint.hash
         , endpoint.outgoing
+        , endpoint.firstTraceId
+        , endpoint.recentTraceId
+        , endpoint.service
         )
 
 
@@ -233,3 +257,189 @@ countEndpointInbox pid host requestType = do
             AND ann.acknowledged_at IS NULL
             AND host = ?
      |]
+
+
+-- | Endpoint with current hourly rates for anomaly detection
+data EndpointWithCurrentRates = EndpointWithCurrentRates
+  { endpointId :: EndpointId
+  , endpointHash :: Text
+  , method :: Text
+  , urlPath :: Text
+  , host :: Text
+  , baselineState :: BaselineState
+  , baselineErrorRateMean :: Maybe Double
+  , baselineErrorRateStddev :: Maybe Double
+  , baselineLatencyMean :: Maybe Double
+  , baselineLatencyStddev :: Maybe Double
+  , baselineLatencyP95 :: Maybe Double
+  , baselineLatencyP99 :: Maybe Double
+  , baselineVolumeHourlyMean :: Maybe Double
+  , baselineVolumeHourlyStddev :: Maybe Double
+  , currentHourRequests :: Int
+  , currentHourErrors :: Int
+  , currentHourLatencyP50 :: Maybe Double
+  , currentHourLatencyP95 :: Maybe Double
+  , currentHourLatencyP99 :: Maybe Double
+  }
+  deriving stock (Generic, Show)
+  deriving anyclass (Default, FromRow, NFData)
+
+
+-- | Endpoint stats for baseline calculation
+data EndpointStats = EndpointStats
+  { totalHours :: Int
+  , hourlyMeanRequests :: Double
+  , hourlyStddevRequests :: Double
+  , hourlyMeanErrors :: Double
+  , hourlyStddevErrors :: Double
+  , meanLatency :: Double
+  , stddevLatency :: Double
+  , p95Latency :: Double
+  , p99Latency :: Double
+  }
+  deriving stock (Generic, Show)
+  deriving anyclass (Default, FromRow, NFData)
+
+
+-- | Get endpoints with their current hourly rates for anomaly detection
+getEndpointsWithCurrentRates :: DB es => Projects.ProjectId -> Eff es [EndpointWithCurrentRates]
+getEndpointsWithCurrentRates pid = PG.query q (pid, pid)
+  where
+    q =
+      [sql|
+        WITH current_hour_stats AS (
+          SELECT
+            attributes->'http'->>'route' AS url_path,
+            attributes___http___request___method AS method,
+            COUNT(*) AS request_count,
+            COUNT(*) FILTER (WHERE attributes___http___response___status_code::int >= 500 OR attributes___http___response___status_code::int = 0) AS error_count,
+            PERCENTILE_CONT(0.50) WITHIN GROUP (ORDER BY duration) AS p50_latency,
+            PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY duration) AS p95_latency,
+            PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY duration) AS p99_latency
+          FROM otel_logs_and_spans
+          WHERE project_id = ?
+            AND name = 'monoscope.http'
+            AND timestamp >= NOW() - INTERVAL '1 hour'
+          GROUP BY url_path, method
+        )
+        SELECT
+          e.id AS endpoint_id,
+          e.hash AS endpoint_hash,
+          e.method,
+          e.url_path,
+          e.host,
+          e.baseline_state,
+          e.baseline_error_rate_mean,
+          e.baseline_error_rate_stddev,
+          e.baseline_latency_mean,
+          e.baseline_latency_stddev,
+          e.baseline_latency_p95,
+          e.baseline_latency_p99,
+          e.baseline_volume_hourly_mean,
+          e.baseline_volume_hourly_stddev,
+          COALESCE(chs.request_count, 0)::int AS current_hour_requests,
+          COALESCE(chs.error_count, 0)::int AS current_hour_errors,
+          chs.p50_latency AS current_hour_latency_p50,
+          chs.p95_latency AS current_hour_latency_p95,
+          chs.p99_latency AS current_hour_latency_p99
+        FROM apis.endpoints e
+        LEFT JOIN current_hour_stats chs
+          ON e.url_path = chs.url_path AND e.method = chs.method
+        WHERE e.project_id = ?
+          AND e.baseline_state = 'established'
+      |]
+
+
+-- | Get endpoint stats for baseline calculation over N hours
+getEndpointStats :: DB es => Projects.ProjectId -> Text -> Int -> Eff es (Maybe EndpointStats)
+getEndpointStats pid endpointHash hours = listToMaybe <$> PG.query q (pid, endpointHash, hours)
+  where
+    q =
+      [sql|
+        WITH hourly_stats AS (
+          SELECT
+            DATE_TRUNC('hour', timestamp) AS hour,
+            COUNT(*) AS request_count,
+            COUNT(*) FILTER (WHERE attributes___http___response___status_code::int >= 500 OR attributes___http___response___status_code::int = 0) AS error_count,
+            AVG(duration) AS avg_latency,
+            PERCENTILE_CONT(0.95) WITHIN GROUP (ORDER BY duration) AS p95_latency,
+            PERCENTILE_CONT(0.99) WITHIN GROUP (ORDER BY duration) AS p99_latency
+          FROM otel_logs_and_spans ols
+          JOIN apis.endpoints e ON e.url_path = (ols.attributes->'http'->>'route')
+            AND e.method = ols.attributes___http___request___method
+          WHERE ols.project_id = ?
+            AND e.hash = ?
+            AND ols.name = 'monoscope.http'
+            AND ols.timestamp >= NOW() - MAKE_INTERVAL(hours => ?)
+          GROUP BY DATE_TRUNC('hour', timestamp)
+        )
+        SELECT
+          COUNT(*)::int AS total_hours,
+          COALESCE(AVG(request_count), 0)::float8 AS hourly_mean_requests,
+          COALESCE(STDDEV(request_count), 0)::float8 AS hourly_stddev_requests,
+          COALESCE(AVG(error_count::float / NULLIF(request_count, 0)), 0)::float8 AS hourly_mean_errors,
+          COALESCE(STDDEV(error_count::float / NULLIF(request_count, 0)), 0)::float8 AS hourly_stddev_errors,
+          COALESCE(AVG(avg_latency), 0)::float8 AS mean_latency,
+          COALESCE(STDDEV(avg_latency), 0)::float8 AS stddev_latency,
+          COALESCE(AVG(p95_latency), 0)::float8 AS p95_latency,
+          COALESCE(AVG(p99_latency), 0)::float8 AS p99_latency
+        FROM hourly_stats
+        HAVING COUNT(*) > 0
+      |]
+
+
+-- | Update endpoint baseline values
+updateEndpointBaseline :: DB es => EndpointId -> BaselineState -> Double -> Double -> Double -> Double -> Double -> Double -> Double -> Double -> Int -> Eff es ()
+updateEndpointBaseline eid bState errMean errStddev latMean latStddev latP95 latP99 volMean volStddev samples =
+  void $ PG.execute q (bState, errMean, errStddev, latMean, latStddev, latP95, latP99, volMean, volStddev, samples, eid)
+  where
+    q =
+      [sql|
+        UPDATE apis.endpoints
+        SET baseline_state = ?,
+            baseline_error_rate_mean = ?,
+            baseline_error_rate_stddev = ?,
+            baseline_latency_mean = ?,
+            baseline_latency_stddev = ?,
+            baseline_latency_p95 = ?,
+            baseline_latency_p99 = ?,
+            baseline_volume_hourly_mean = ?,
+            baseline_volume_hourly_stddev = ?,
+            baseline_samples = ?,
+            baseline_updated_at = NOW()
+        WHERE id = ?
+      |]
+
+
+-- | Get endpoint by hash
+getEndpointByHash :: DB es => Projects.ProjectId -> Text -> Eff es (Maybe Endpoint)
+getEndpointByHash pid hash = listToMaybe <$> PG.query q (pid, hash)
+  where
+    q =
+      [sql|
+        SELECT id, created_at, updated_at, project_id, url_path, url_params, method, host, hash, outgoing, description,
+               first_trace_id, recent_trace_id, service,
+               baseline_state, baseline_samples, baseline_updated_at,
+               baseline_error_rate_mean, baseline_error_rate_stddev,
+               baseline_latency_mean, baseline_latency_stddev, baseline_latency_p95, baseline_latency_p99,
+               baseline_volume_hourly_mean, baseline_volume_hourly_stddev
+        FROM apis.endpoints
+        WHERE project_id = ? AND hash = ?
+      |]
+
+
+-- | Get all active endpoints for a project (for baseline calculation)
+getActiveEndpoints :: DB es => Projects.ProjectId -> Eff es [Endpoint]
+getActiveEndpoints pid = PG.query q (Only pid)
+  where
+    q =
+      [sql|
+        SELECT id, created_at, updated_at, project_id, url_path, url_params, method, host, hash, outgoing, description,
+               first_trace_id, recent_trace_id, service,
+               baseline_state, baseline_samples, baseline_updated_at,
+               baseline_error_rate_mean, baseline_error_rate_stddev,
+               baseline_latency_mean, baseline_latency_stddev, baseline_latency_p95, baseline_latency_p99,
+               baseline_volume_hourly_mean, baseline_volume_hourly_stddev
+        FROM apis.endpoints
+        WHERE project_id = ?
+      |]