Distributed queries and carves do expire

admin/handlers/json-carves.go

@@ -112,6 +112,9 @@ func (h *HandlersAdmin) JSONCarvesHandler(w http.ResponseWriter, r *http.Request
 		if q.Completed {
 			status = queries.StatusComplete
 		}
+		if q.Expired {
+			status = queries.StatusExpired
+		}
 		progress := make(CarveProgress)
 		progress["expected"] = q.Expected
 		progress["executions"] = q.Executions

admin/handlers/json-queries.go

@@ -20,6 +20,7 @@ var (
 		queries.TargetActive:    true,
 		queries.TargetCompleted: true,
 		queries.TargetSaved:     true,
+		queries.TargetExpired:   true,
 	}
 )
 
@@ -83,6 +84,9 @@ func (h *HandlersAdmin) JSONQueryJSON(q queries.DistributedQuery, env string) Qu
 	if q.Completed {
 		status = queries.StatusComplete
 	}
+	if q.Expired {
+		status = queries.StatusExpired
+	}
 	// Preparing query targets
 	ts, _ := h.Queries.GetTargets(q.Name)
 	_ts := []QueryTarget{}

admin/handlers/post.go

@@ -4,6 +4,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"strings"
+	"time"
 
 	"fmt"
 	"net/http"
@@ -139,7 +140,11 @@ func (h *HandlersAdmin) QueryRunPOSTHandler(w http.ResponseWriter, r *http.Reque
 	}
 	// FIXME check if query is carve and user has permissions to carve
 	// Prepare and create new query
-	newQuery := newQueryReady(ctx[sessions.CtxUser], q.Query, env.ID)
+	expTime := h.queryExpiration(q.ExpHours)
+	if q.ExpHours == 0 {
+		expTime = time.Time{}
+	}
+	newQuery := newQueryReady(ctx[sessions.CtxUser], q.Query, expTime, env.ID)
 	if err := h.Queries.Create(newQuery); err != nil {
 		adminErrorResponse(w, "error creating query", http.StatusInternalServerError, err)
 		h.Inc(metricAdminErr)
@@ -301,6 +306,8 @@ func (h *HandlersAdmin) CarvesRunPOSTHandler(w http.ResponseWriter, r *http.Requ
 		Active:        true,
 		Completed:     false,
 		Deleted:       false,
+		Expired:       false,
+		Expiration:    h.queryExpiration(c.ExpHours),
 		Type:          queries.CarveQueryType,
 		Path:          c.Path,
 		EnvironmentID: env.ID,

admin/handlers/templates.go

@@ -679,18 +679,6 @@ func (h *HandlersAdmin) QueryLogsHandler(w http.ResponseWriter, r *http.Request)
 		log.Info().Msg("error getting name")
 		return
 	}
-	// Custom functions to handle formatting
-	funcMap := template.FuncMap{
-		"queryResultLink": h.queryResultLink,
-	}
-	// Prepare template
-	tempateFiles := h.NewTemplateFiles(h.TemplatesFolder, "queries-logs.html").filepaths
-	t, err := template.New("queries-logs.html").Funcs(funcMap).ParseFiles(tempateFiles...)
-	if err != nil {
-		h.Inc(metricAdminErr)
-		log.Err(err).Msg("error getting table template")
-		return
-	}
 	// Get all environments
 	envAll, err := h.Envs.All()
 	if err != nil {
@@ -724,6 +712,19 @@ func (h *HandlersAdmin) QueryLogsHandler(w http.ResponseWriter, r *http.Request)
 		Query:     true,
 		QueryName: query.Name,
 	}
+	// Custom functions to handle formatting
+	funcMap := template.FuncMap{
+		"queryResultLink": h.queryResultLink,
+		"inFutureTime":    utils.InFutureTime,
+	}
+	// Prepare template
+	tempateFiles := h.NewTemplateFiles(h.TemplatesFolder, "queries-logs.html").filepaths
+	t, err := template.New("queries-logs.html").Funcs(funcMap).ParseFiles(tempateFiles...)
+	if err != nil {
+		h.Inc(metricAdminErr)
+		log.Err(err).Msg("error getting table template")
+		return
+	}
 	// Prepare template data
 	templateData := QueryLogsTemplateData{
 		Title:         "Query logs " + query.Name,
@@ -780,9 +781,13 @@ func (h *HandlersAdmin) CarvesDetailsHandler(w http.ResponseWriter, r *http.Requ
 		log.Info().Msg("error getting name")
 		return
 	}
+	// Custom functions to handle formatting
+	funcMap := template.FuncMap{
+		"inFutureTime":    utils.InFutureTime,
+	}
 	// Prepare template
 	tempateFiles := h.NewTemplateFiles(h.TemplatesFolder, "carves-details.html").filepaths
-	t, err := template.ParseFiles(tempateFiles...)
+	t, err := template.New("carves-details.html").Funcs(funcMap).ParseFiles(tempateFiles...)
 	if err != nil {
 		h.Inc(metricAdminErr)
 		log.Err(err).Msg("error getting table template")

admin/handlers/types-requests.go

@@ -21,6 +21,7 @@ type DistributedQueryRequest struct {
 	Save         bool     `json:"save"`
 	Name         string   `json:"name"`
 	Query        string   `json:"query"`
+	ExpHours     int      `json:"exp_hours"`
 }
 
 // DistributedCarveRequest to receive carve requests
@@ -31,6 +32,7 @@ type DistributedCarveRequest struct {
 	UUIDs        []string `json:"uuid_list"`
 	Hosts        []string `json:"host_list"`
 	Path         string   `json:"path"`
+	ExpHours     int      `json:"exp_hours"`
 }
 
 // DistributedQueryActionRequest to receive query requests

admin/handlers/utils.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"net/http"
 	"strings"
+	"time"
 
 	"github.com/jmpsec/osctrl/environments"
 	"github.com/jmpsec/osctrl/queries"
@@ -79,7 +80,7 @@ func generateCarveQuery(file string, glob bool) string {
 }
 
 // Helper to determine if a query may be a carve
-func newQueryReady(user, query string, envid uint) queries.DistributedQuery {
+func newQueryReady(user, query string, exp time.Time, envid uint) queries.DistributedQuery {
 	if strings.Contains(query, "carve(") || strings.Contains(query, "carve=1") {
 		return queries.DistributedQuery{
 			Query:         query,
@@ -90,6 +91,8 @@ func newQueryReady(user, query string, envid uint) queries.DistributedQuery {
 			Active:        true,
 			Completed:     false,
 			Deleted:       false,
+			Expired:       false,
+			Expiration:    exp,
 			Type:          queries.CarveQueryType,
 			Path:          query,
 			EnvironmentID: envid,
@@ -104,6 +107,8 @@ func newQueryReady(user, query string, envid uint) queries.DistributedQuery {
 		Active:        true,
 		Completed:     false,
 		Deleted:       false,
+		Expired:       false,
+		Expiration:    exp,
 		Type:          queries.StandardQueryType,
 		EnvironmentID: envid,
 	}
@@ -203,3 +208,8 @@ func (h *HandlersAdmin) generateFlags(flagsRaw, secretFile, certFile string) str
 	replaced := strings.Replace(flagsRaw, "__SECRET_FILE__", secretFile, 1)
 	return strings.Replace(replaced, "__CERT_FILE__", certFile, 1)
 }
+
+// Helper to generate the time.Time for the expiration of a query or carve based on hours
+func (h *HandlersAdmin) queryExpiration(exp int) time.Time {
+	return time.Now().Add(time.Duration(exp) * time.Hour)
+}

admin/main.go

@@ -97,6 +97,8 @@ const (
 	defCarvedFolder string = "./carved_files/"
 	// Default refreshing interval in seconds
 	defaultRefresh int = 300
+	// Default interval in seconds to expire queries/carves
+	defaultExpiration int = 900
 	// Default hours to classify nodes as inactive
 	defaultInactive int = -72
 )
@@ -748,6 +750,32 @@ func osctrlAdminService() {
 		}
 	}()
 
+	// Goroutine to cleanup expired queries and carves
+	go func() {
+		_t := settingsmgr.CleanupExpired()
+		if _t == 0 {
+			_t = int64(defaultExpiration)
+		}
+		for {
+			if settingsmgr.DebugService(settings.ServiceAdmin) {
+				log.Debug().Msg("DebugService: Cleaning up expired queries/carves")
+			}
+			allEnvs, err := envs.All()
+			if err != nil {
+				log.Err(err).Msg("Error getting all environments")
+			}
+			for _, e := range allEnvs {
+				if err:= queriesmgr.CleanupExpiredQueries(e.ID); err != nil {
+					log.Err(err).Msg("Error cleaning up expired queries")
+				}
+				if err:= queriesmgr.CleanupExpiredCarves(e.ID); err != nil {
+					log.Err(err).Msg("Error cleaning up expired carves")
+				}
+			}
+			time.Sleep(time.Duration(_t) * time.Second)
+		}
+	}()
+
 	var loggerDBConfig *backend.JSONConfigurationDB
 	// Set the logger configuration file if we have a DB logger
 	if adminConfig.Logger == settings.LoggingDB {

admin/settings.go

@@ -60,6 +60,12 @@ func loadingSettings(mgr *settings.Settings) error {
 			return fmt.Errorf("Failed to add %s to configuration: %v", settings.CleanupSessions, err)
 		}
 	}
+	// Check if service settings for queries/carves cleanup is ready
+	if !mgr.IsValue(settings.ServiceAdmin, settings.CleanupExpired, settings.NoEnvironmentID) {
+		if err := mgr.NewIntegerValue(settings.ServiceAdmin, settings.CleanupExpired, int64(defaultExpiration), settings.NoEnvironmentID); err != nil {
+			return fmt.Errorf("Failed to add %s to configuration: %v", settings.CleanupExpired, err)
+		}
+	}
 	// Check if service settings for node inactive hours is ready
 	if !mgr.IsValue(settings.ServiceAdmin, settings.InactiveHours, settings.NoEnvironmentID) {
 		if err := mgr.NewIntegerValue(settings.ServiceAdmin, settings.InactiveHours, int64(defaultInactive), settings.NoEnvironmentID); err != nil {

admin/static/js/carves.js

@@ -4,6 +4,7 @@ function sendCarve(_url, _redir) {
   var _platform_list = $("#target_platform").val();
   var _uuid_list = $("#target_uuids").val();
   var _host_list = $("#target_hosts").val();
+  var _exp_hours = parseInt($("#expiration_hours").val());
   var _repeat = $('#target_repeat').prop('checked') ? 1 : 0;
   var _path = $("#carve").val();
 
@@ -45,7 +46,8 @@ function sendCarve(_url, _redir) {
     uuid_list: _uuid_list,
     host_list: _host_list,
     path: _path,
-    repeat: _repeat
+    exp_hours: _exp_hours,
+    repeat: _repeat,
   };
   sendPostRequest(data, _url, _redir, false);
 }