From e322305fd327d0e659fdd08e46ca13a02fbf4243 Mon Sep 17 00:00:00 2001 From: Egari Date: Thu, 12 Dec 2024 15:09:49 +0100 Subject: [PATCH 01/10] Only do pagination after filtering out releases we're not searching for --- internal/v3/api/release.go | 79 ++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 21 deletions(-) diff --git a/internal/v3/api/release.go b/internal/v3/api/release.go index 88a0236..dbf76c0 100644 --- a/internal/v3/api/release.go +++ b/internal/v3/api/release.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "errors" "fmt" + "github.com/dadav/gorge/internal/log" "net/http" "net/url" "os" @@ -261,36 +262,58 @@ func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, off } results := []gen.Release{} - filtered := []gen.Release{} + filtered := []*gen.Release{} allReleases, _ := backend.ConfiguredBackend.GetAllReleases() params := url.Values{} params.Add("offset", strconv.Itoa(int(offset))) params.Add("limit", strconv.Itoa(int(limit))) + filterSet := false - if int(offset)+1 > len(allReleases) { - return gen.Response(404, GetRelease404Response{ - Message: "Invalid offset", - Errors: []string{"The given offset is larger than the total number of modules."}, - }), nil - } + if module != "" { + filterSet = true + params.Add("module", module) - for _, r := range allReleases[offset:] { - var filterMatched, filterSet bool + // Perform an early query to see if the module even exists in the backend + _, err := backend.ConfiguredBackend.GetModuleBySlug(module) + if err != nil { + log.Log.Debugf("Could not find module with slug '%s' in backend, returning 404 so we can proxy if desired\n", module) - if module != "" && r.Module.Slug != module { - filterSet = true - filterMatched = r.Module.Slug == module - params.Add("module", module) - } - if owner != "" && r.Module.Owner.Slug != owner { - filterSet = true - filterMatched = r.Module.Owner.Slug == owner - params.Add("owner", owner) + return gen.Response(http.StatusNotFound, GetRelease404Response{ + Message: "No releases found", + Errors: []string{"No module(s) found for given query."}, + }), nil } + } + + if owner != "" { + filterSet = true + params.Add("owner", owner) + } + + prefiltered := []*gen.Release{} + if len(allReleases) > int(offset) { + prefiltered = allReleases[offset:] + } - if !filterSet || filterMatched { - filtered = append(filtered, *r) + if filterSet { + // We search through all available releases to see if they match the filter + for _, r := range prefiltered { + var filterMatched bool + + if module != "" && r.Module.Slug == module { + filterMatched = r.Module.Slug == module + } + + if owner != "" && r.Module.Owner.Slug == owner { + filterMatched = r.Module.Owner.Slug == owner + } + + if filterMatched { + filtered = append(filtered, r) + } } + } else { + filtered = prefiltered } i := 1 @@ -298,10 +321,24 @@ func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, off if i > int(limit) { break } - results = append(results, release) + + results = append(results, *release) i++ } + if len(results) == 0 { + if module != "" { + log.Log.Debugf("No releases for '%s' found in backend\n", module) + } else { + log.Log.Debugf("No releases found in backend\n") + } + + return gen.Response(http.StatusNotFound, GetRelease404Response{ + Message: "No releases found", + Errors: []string{"No release(s) found for given query."}, + }), nil + } + base, _ := url.Parse("/v3/releases") base.RawQuery = params.Encode() currentInf := interface{}(base.String()) From 402f518314d25f1028d39f7b6e560a610c22013e Mon Sep 17 00:00:00 2001 From: Egari Date: Thu, 12 Dec 2024 17:10:49 +0100 Subject: [PATCH 02/10] if a fallback-proxy is configured, throw more 404s to ensure we actually _do_ fallback --- internal/v3/api/release.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/internal/v3/api/release.go b/internal/v3/api/release.go index dbf76c0..6ee92d4 100644 --- a/internal/v3/api/release.go +++ b/internal/v3/api/release.go @@ -269,6 +269,16 @@ func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, off params.Add("limit", strconv.Itoa(int(limit))) filterSet := false + // We know there's no releases and a fallback proxy, so we should return a 404 to let the proxy handle it + if config.FallbackProxyUrl != "" && len(allReleases) == 0 { + log.Log.Debugln("Could not find *any* releases in the backend, returning 404 so we can proxy if desired") + + return gen.Response(http.StatusNotFound, GetRelease404Response{ + Message: "No releases found", + Errors: []string{"Did not retrieve any releases from the backend."}, + }), nil + } + if module != "" { filterSet = true params.Add("module", module) @@ -326,11 +336,12 @@ func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, off i++ } - if len(results) == 0 { + // If we're using a fallback-proxy, we should return a 404 so the proxy can handle the request + if config.FallbackProxyUrl != "" && len(results) == 0 { if module != "" { log.Log.Debugf("No releases for '%s' found in backend\n", module) } else { - log.Log.Debugf("No releases found in backend\n") + log.Log.Debugln("No releases found in backend") } return gen.Response(http.StatusNotFound, GetRelease404Response{ From 1bd06fa69aca053e1810c50ef36431e4008b8729 Mon Sep 17 00:00:00 2001 From: Egari Date: Thu, 12 Dec 2024 17:13:56 +0100 Subject: [PATCH 03/10] Track proxied requests to fallback URLs --- cmd/serve.go | 11 +- internal/middleware/stats.go | 48 +++++---- internal/v3/ui/components/statistics.templ | 3 + internal/v3/ui/components/statistics_templ.go | 100 +++++++++++------- 4 files changed, 103 insertions(+), 59 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index f25f668..feb35bd 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -209,10 +209,14 @@ You can also enable the caching functionality to speed things up.`, slices.Reverse(proxies) for _, proxy := range proxies { - r.Use(customMiddleware.ProxyFallback(proxy, func(status int) bool { - return status == http.StatusNotFound - }, + r.Use(customMiddleware.ProxyFallback( + proxy, + func(status int) bool { + return status == http.StatusNotFound + }, func(r *http.Response) { + r.Header.Add("X-Proxied-To", proxy) + if config.ImportProxiedReleases && strings.HasPrefix(r.Request.URL.Path, "/v3/files/") && r.StatusCode == http.StatusOK { body, err := io.ReadAll(r.Body) if err != nil { @@ -228,6 +232,7 @@ You can also enable the caching functionality to speed things up.`, log.Log.Error(err) return } + log.Log.Infof("Imported release %s\n", release.Slug) } }, diff --git a/internal/middleware/stats.go b/internal/middleware/stats.go index a0cdcd6..fc4d4fb 100644 --- a/internal/middleware/stats.go +++ b/internal/middleware/stats.go @@ -7,29 +7,33 @@ import ( ) type Statistics struct { - ActiveConnections int - TotalConnections int - TotalResponseTime time.Duration - TotalCacheHits int - TotalCacheMisses int - ConnectionsPerEndpoint map[string]int - ResponseTimePerEndpoint map[string]time.Duration - CacheHitsPerEndpoint map[string]int - CacheMissesPerEndpoint map[string]int - Mutex sync.Mutex + ActiveConnections int + TotalConnections int + TotalResponseTime time.Duration + TotalCacheHits int + TotalCacheMisses int + ConnectionsPerEndpoint map[string]int + ResponseTimePerEndpoint map[string]time.Duration + CacheHitsPerEndpoint map[string]int + CacheMissesPerEndpoint map[string]int + Mutex sync.Mutex + ProxiedConnections int + ProxiedConnectionsPerEndpoint map[string]int } func NewStatistics() *Statistics { return &Statistics{ - ActiveConnections: 0, - TotalConnections: 0, - TotalResponseTime: 0, - TotalCacheHits: 0, - TotalCacheMisses: 0, - ConnectionsPerEndpoint: make(map[string]int), - CacheHitsPerEndpoint: make(map[string]int), - CacheMissesPerEndpoint: make(map[string]int), - ResponseTimePerEndpoint: make(map[string]time.Duration), + ActiveConnections: 0, + TotalConnections: 0, + TotalResponseTime: 0, + TotalCacheHits: 0, + TotalCacheMisses: 0, + ConnectionsPerEndpoint: make(map[string]int), + CacheHitsPerEndpoint: make(map[string]int), + CacheMissesPerEndpoint: make(map[string]int), + ResponseTimePerEndpoint: make(map[string]time.Duration), + ProxiedConnections: 0, + ProxiedConnectionsPerEndpoint: make(map[string]int), } } @@ -59,6 +63,12 @@ func StatisticsMiddleware(stats *Statistics) func(next http.Handler) http.Handle stats.TotalResponseTime += duration stats.ResponseTimePerEndpoint[r.URL.Path] += duration + + if w.Header().Get("X-Proxied-To") != "" { + stats.ProxiedConnections++ + stats.ProxiedConnectionsPerEndpoint[r.URL.Path]++ + } + stats.Mutex.Unlock() }() diff --git a/internal/v3/ui/components/statistics.templ b/internal/v3/ui/components/statistics.templ index 6146ca7..c1bc867 100644 --- a/internal/v3/ui/components/statistics.templ +++ b/internal/v3/ui/components/statistics.templ @@ -10,6 +10,7 @@ templ StatisticsView(stats *customMiddleware.Statistics) {

Statistics

ActiveConnections: { strconv.Itoa(stats.ActiveConnections) }

+

ProxiedConnections: { strconv.Itoa(stats.ProxiedConnections) }

TotalConnections: { strconv.Itoa(stats.TotalConnections) }

TotalResponseTime: { stats.TotalResponseTime.String() }

TotalCacheHits: { strconv.Itoa(stats.TotalCacheHits) }

@@ -19,6 +20,7 @@ templ StatisticsView(stats *customMiddleware.Statistics) { Path Connections + Proxied Connections Average ResponseTime Total ResponseTime Cache (Hits/Misses) @@ -29,6 +31,7 @@ templ StatisticsView(stats *customMiddleware.Statistics) { { path } { strconv.Itoa(connections) } + { strconv.Itoa(stats.ProxiedConnectionsPerEndpoint[path]) } { (stats.ResponseTimePerEndpoint[path] / time.Duration(connections)).String() } { stats.ResponseTimePerEndpoint[path].String() } if stats.CacheHitsPerEndpoint[path] > 0 || stats.CacheMissesPerEndpoint[path] > 0 { diff --git a/internal/v3/ui/components/statistics_templ.go b/internal/v3/ui/components/statistics_templ.go index 534b1bc..3ecaa96 100644 --- a/internal/v3/ui/components/statistics_templ.go +++ b/internal/v3/ui/components/statistics_templ.go @@ -45,59 +45,72 @@ func StatisticsView(stats *customMiddleware.Statistics) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

TotalConnections: ") + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

ProxiedConnections: ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var3 string - templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(stats.TotalConnections)) + templ_7745c5c3_Var3, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(stats.ProxiedConnections)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 13, Col: 61} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 13, Col: 65} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var3)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

TotalResponseTime: ") + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

TotalConnections: ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var4 string - templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(stats.TotalResponseTime.String()) + templ_7745c5c3_Var4, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(stats.TotalConnections)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 14, Col: 58} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 14, Col: 61} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var4)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

TotalCacheHits: ") + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

TotalResponseTime: ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var5 string - templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(stats.TotalCacheHits)) + templ_7745c5c3_Var5, templ_7745c5c3_Err = templ.JoinStringErrs(stats.TotalResponseTime.String()) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 15, Col: 57} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 15, Col: 58} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var5)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

TotalCacheMisses: ") + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

TotalCacheHits: ") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } var templ_7745c5c3_Var6 string - templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(stats.TotalCacheMisses)) + templ_7745c5c3_Var6, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(stats.TotalCacheHits)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 16, Col: 61} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 16, Col: 57} } _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var6)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

") + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

TotalCacheMisses: ") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var7 string + templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(stats.TotalCacheMisses)) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 17, Col: 61} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("

PathConnectionsAverage ResponseTimeTotal ResponseTimeCache (Hits/Misses)
") if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -106,12 +119,12 @@ func StatisticsView(stats *customMiddleware.Statistics) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var7 string - templ_7745c5c3_Var7, templ_7745c5c3_Err = templ.JoinStringErrs(path) + var templ_7745c5c3_Var8 string + templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(path) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 30, Col: 16} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 32, Col: 16} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var7)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -119,12 +132,12 @@ func StatisticsView(stats *customMiddleware.Statistics) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var8 string - templ_7745c5c3_Var8, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(connections)) + var templ_7745c5c3_Var9 string + templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(connections)) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 31, Col: 37} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 33, Col: 37} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var8)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -132,12 +145,12 @@ func StatisticsView(stats *customMiddleware.Statistics) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var9 string - templ_7745c5c3_Var9, templ_7745c5c3_Err = templ.JoinStringErrs((stats.ResponseTimePerEndpoint[path] / time.Duration(connections)).String()) + var templ_7745c5c3_Var10 string + templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(stats.ProxiedConnectionsPerEndpoint[path])) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 32, Col: 87} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 34, Col: 67} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var9)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -145,12 +158,25 @@ func StatisticsView(stats *customMiddleware.Statistics) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var10 string - templ_7745c5c3_Var10, templ_7745c5c3_Err = templ.JoinStringErrs(stats.ResponseTimePerEndpoint[path].String()) + var templ_7745c5c3_Var11 string + templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs((stats.ResponseTimePerEndpoint[path] / time.Duration(connections)).String()) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 33, Col: 56} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 35, Col: 87} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var10)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString("
PathConnectionsProxied ConnectionsAverage ResponseTimeTotal ResponseTimeCache (Hits/Misses)
") + if templ_7745c5c3_Err != nil { + return templ_7745c5c3_Err + } + var templ_7745c5c3_Var12 string + templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(stats.ResponseTimePerEndpoint[path].String()) + if templ_7745c5c3_Err != nil { + return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 36, Col: 56} + } + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -163,12 +189,12 @@ func StatisticsView(stats *customMiddleware.Statistics) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var11 string - templ_7745c5c3_Var11, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(stats.CacheHitsPerEndpoint[path])) + var templ_7745c5c3_Var13 string + templ_7745c5c3_Var13, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(stats.CacheHitsPerEndpoint[path])) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 35, Col: 59} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 38, Col: 59} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var11)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var13)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } @@ -176,12 +202,12 @@ func StatisticsView(stats *customMiddleware.Statistics) templ.Component { if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } - var templ_7745c5c3_Var12 string - templ_7745c5c3_Var12, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(stats.CacheMissesPerEndpoint[path])) + var templ_7745c5c3_Var14 string + templ_7745c5c3_Var14, templ_7745c5c3_Err = templ.JoinStringErrs(strconv.Itoa(stats.CacheMissesPerEndpoint[path])) if templ_7745c5c3_Err != nil { - return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 35, Col: 112} + return templ.Error{Err: templ_7745c5c3_Err, FileName: `internal/v3/ui/components/statistics.templ`, Line: 38, Col: 112} } - _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var12)) + _, templ_7745c5c3_Err = templ_7745c5c3_Buffer.WriteString(templ.EscapeString(templ_7745c5c3_Var14)) if templ_7745c5c3_Err != nil { return templ_7745c5c3_Err } From 73ae456ccbb819793d54220f9ce5db37cfc8ec4e Mon Sep 17 00:00:00 2001 From: Egari Date: Thu, 12 Dec 2024 17:34:26 +0100 Subject: [PATCH 04/10] Allow stampede to cache on request path _or_ full request URI (configurable via 'usual' means) --- README.md | 2 ++ cmd/serve.go | 9 ++++++++- internal/config/config.go | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9dec623..cb4c2e1 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,7 @@ Flags: --bind string host to listen to (default "127.0.0.1") --cache-max-age int max number of seconds responses should be cached (default 86400) --cache-prefixes string url prefixes to cache (default "/v3/files") + --cache-by-full-request-uri will cache responses by the full request URI (incl. query fragments) instead of only the request path --cors string allowed cors origins separated by comma (default "*") --dev enables dev mode --drop-privileges drops privileges to the given user/group @@ -207,6 +208,7 @@ GORGE_BACKEND=filesystem GORGE_BIND=127.0.0.1 GORGE_CACHE_MAX_AGE=86400 GORGE_CACHE_PREFIXES=/v3/files +GORGE_CACHE_BY_FULL_REQUEST_URI=false GORGE_CORS="*" GORGE_DEV=false GORGE_DROP_PRIVILEGES=false diff --git a/cmd/serve.go b/cmd/serve.go index feb35bd..ab47623 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -159,7 +159,13 @@ You can also enable the caching functionality to speed things up.`, log.Log.Debug("Setting up cache middleware") customKeyFunc := func(r *http.Request) uint64 { token := r.Header.Get("Authorization") - return stampede.StringToHash(r.Method, r.URL.Path, strings.ToLower(token)) + requestURI := r.URL.Path + + if config.CacheByFullRequestURI { + requestURI = r.URL.RequestURI() + } + + return stampede.StringToHash(r.Method, requestURI, strings.ToLower(token)) } cachedMiddleware := stampede.HandlerWithKey( @@ -384,4 +390,5 @@ func init() { serveCmd.Flags().Int64Var(&config.CacheMaxAge, "cache-max-age", 86400, "max number of seconds responses should be cached") serveCmd.Flags().BoolVar(&config.NoCache, "no-cache", false, "disables the caching functionality") serveCmd.Flags().BoolVar(&config.ImportProxiedReleases, "import-proxied-releases", false, "add every proxied modules to local store") + serveCmd.Flags().BoolVar(&config.CacheByFullRequestURI, "cache-by-full-request-uri", false, "will cache responses by the full request URI (incl. query fragments) instead of only the request path") } diff --git a/internal/config/config.go b/internal/config/config.go index b2ae88c..d6b721d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -16,6 +16,7 @@ var ( FallbackProxyUrl string NoCache bool CachePrefixes string + CacheByFullRequestURI bool CacheMaxAge int64 ImportProxiedReleases bool JwtSecret string From 312639337a757cda8817fb9c9c6da27f7644f394 Mon Sep 17 00:00:00 2001 From: Egari Date: Thu, 12 Dec 2024 18:31:05 +0100 Subject: [PATCH 05/10] fix logic error in /v3/releases search filter --- internal/v3/api/release.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/v3/api/release.go b/internal/v3/api/release.go index 6ee92d4..5687779 100644 --- a/internal/v3/api/release.go +++ b/internal/v3/api/release.go @@ -308,14 +308,14 @@ func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, off if filterSet { // We search through all available releases to see if they match the filter for _, r := range prefiltered { - var filterMatched bool + filterMatched := true - if module != "" && r.Module.Slug == module { - filterMatched = r.Module.Slug == module + if module != "" && r.Module.Slug != module { + filterMatched = false } - if owner != "" && r.Module.Owner.Slug == owner { - filterMatched = r.Module.Owner.Slug == owner + if owner != "" && r.Module.Owner.Slug != owner { + filterMatched = false } if filterMatched { From bf495bac3e1bb8a92e2675012e0f6d0b4fe1bab1 Mon Sep 17 00:00:00 2001 From: Egari Date: Thu, 12 Dec 2024 18:32:08 +0100 Subject: [PATCH 06/10] simplify filter loop --- internal/v3/api/release.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/internal/v3/api/release.go b/internal/v3/api/release.go index 5687779..09a8c75 100644 --- a/internal/v3/api/release.go +++ b/internal/v3/api/release.go @@ -308,19 +308,15 @@ func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, off if filterSet { // We search through all available releases to see if they match the filter for _, r := range prefiltered { - filterMatched := true - if module != "" && r.Module.Slug != module { - filterMatched = false + continue } if owner != "" && r.Module.Owner.Slug != owner { - filterMatched = false + continue } - if filterMatched { - filtered = append(filtered, r) - } + filtered = append(filtered, r) } } else { filtered = prefiltered From 39270b917aec8e79ee0faa9fe47580dd8435c19f Mon Sep 17 00:00:00 2001 From: Egari Date: Fri, 13 Dec 2024 20:50:47 +0100 Subject: [PATCH 07/10] If the module does not exist with fallback-proxy enabled early return with a 404, return 'empty' 200 result if fallback-proxy is not enabled --- internal/v3/api/release.go | 55 +++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/internal/v3/api/release.go b/internal/v3/api/release.go index 09a8c75..4dc123e 100644 --- a/internal/v3/api/release.go +++ b/internal/v3/api/release.go @@ -5,7 +5,6 @@ import ( "encoding/base64" "errors" "fmt" - "github.com/dadav/gorge/internal/log" "net/http" "net/url" "os" @@ -14,6 +13,7 @@ import ( "strings" "github.com/dadav/gorge/internal/config" + "github.com/dadav/gorge/internal/log" "github.com/dadav/gorge/internal/v3/backend" "github.com/dadav/gorge/internal/v3/utils" gen "github.com/dadav/gorge/pkg/gen/v3/openapi" @@ -264,10 +264,27 @@ func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, off results := []gen.Release{} filtered := []*gen.Release{} allReleases, _ := backend.ConfiguredBackend.GetAllReleases() + + base, _ := url.Parse("/v3/releases") params := url.Values{} + + filterSet := false + + if module != "" { + filterSet = true + params.Add("module", module) + } + + if owner != "" { + filterSet = true + params.Add("owner", owner) + } + params.Add("offset", strconv.Itoa(int(offset))) params.Add("limit", strconv.Itoa(int(limit))) - filterSet := false + + base.RawQuery = params.Encode() + currentInf := interface{}(base.String()) // We know there's no releases and a fallback proxy, so we should return a 404 to let the proxy handle it if config.FallbackProxyUrl != "" && len(allReleases) == 0 { @@ -280,26 +297,33 @@ func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, off } if module != "" { - filterSet = true - params.Add("module", module) - - // Perform an early query to see if the module even exists in the backend + // Perform an early query to see if the module even exists in the backend, optimization for instances with _many_ modules _, err := backend.ConfiguredBackend.GetModuleBySlug(module) if err != nil { log.Log.Debugf("Could not find module with slug '%s' in backend, returning 404 so we can proxy if desired\n", module) - return gen.Response(http.StatusNotFound, GetRelease404Response{ - Message: "No releases found", - Errors: []string{"No module(s) found for given query."}, - }), nil + if config.FallbackProxyUrl != "" { + return gen.Response(http.StatusNotFound, GetRelease404Response{ + Message: "No releases found", + Errors: []string{"No module(s) found for given query."}, + }), nil + } else { + return gen.Response(http.StatusOK, gen.GetReleases200Response{ + Pagination: gen.GetReleases200ResponsePagination{ + Limit: limit, + Offset: offset, + First: ¤tInf, + Previous: nil, + Current: ¤tInf, + Next: nil, + Total: 0, + }, + Results: []gen.Release{}, + }), nil + } } } - if owner != "" { - filterSet = true - params.Add("owner", owner) - } - prefiltered := []*gen.Release{} if len(allReleases) > int(offset) { prefiltered = allReleases[offset:] @@ -346,7 +370,6 @@ func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, off }), nil } - base, _ := url.Parse("/v3/releases") base.RawQuery = params.Encode() currentInf := interface{}(base.String()) params.Set("offset", "0") From 09725190f09e19e3e43ce903995fa2c04d0fa217 Mon Sep 17 00:00:00 2001 From: Egari Date: Fri, 13 Dec 2024 20:52:36 +0100 Subject: [PATCH 08/10] testing on forge.puppet.com/v3/releases shows that module & owner filters apply before offset, despite documentation implying otherwise --- internal/v3/api/release.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/internal/v3/api/release.go b/internal/v3/api/release.go index 4dc123e..83ffba2 100644 --- a/internal/v3/api/release.go +++ b/internal/v3/api/release.go @@ -324,14 +324,9 @@ func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, off } } - prefiltered := []*gen.Release{} - if len(allReleases) > int(offset) { - prefiltered = allReleases[offset:] - } - if filterSet { // We search through all available releases to see if they match the filter - for _, r := range prefiltered { + for _, r := range allReleases { if module != "" && r.Module.Slug != module { continue } @@ -343,17 +338,19 @@ func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, off filtered = append(filtered, r) } } else { - filtered = prefiltered + filtered = allReleases } - i := 1 - for _, release := range filtered { - if i > int(limit) { - break - } + if len(filtered) > int(offset) { + i := 1 + for _, release := range filtered[offset:] { + if i > int(limit) { + break + } - results = append(results, *release) - i++ + results = append(results, *release) + i++ + } } // If we're using a fallback-proxy, we should return a 404 so the proxy can handle the request From 98547b801f991b0688d7f28d92f0a84eff834b2f Mon Sep 17 00:00:00 2001 From: dadav <33197631+dadav@users.noreply.github.com> Date: Sat, 28 Dec 2024 10:22:37 +0100 Subject: [PATCH 09/10] fix: no walrus --- internal/v3/api/release.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/v3/api/release.go b/internal/v3/api/release.go index 83ffba2..e18bd3a 100644 --- a/internal/v3/api/release.go +++ b/internal/v3/api/release.go @@ -115,7 +115,6 @@ type GetFile400Response struct { // GetFile - Download module release func (s *ReleaseOperationsApi) GetFile(ctx context.Context, filename string) (gen.ImplResponse, error) { - if filename == "" { return gen.Response(400, gen.GetFile400Response{ Message: "No filename provided", @@ -368,7 +367,7 @@ func (s *ReleaseOperationsApi) GetReleases(ctx context.Context, limit int32, off } base.RawQuery = params.Encode() - currentInf := interface{}(base.String()) + currentInf = interface{}(base.String()) params.Set("offset", "0") firstInf := interface{}(base.String()) From 32ec930142adf1a1989d606660d51aad5bf5fb23 Mon Sep 17 00:00:00 2001 From: dadav <33197631+dadav@users.noreply.github.com> Date: Sat, 28 Dec 2024 10:39:42 +0100 Subject: [PATCH 10/10] fix: Adjust dockerfile --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7e0074e..ce7d473 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,7 @@ FROM alpine:3.21@sha256:21dc6063fd678b478f57c0e13f47560d0ea4eeba26dfc947b2a4f81f # Create non-root user and set up permissions in a single layer RUN adduser -k /dev/null -u 10001 -D gorge \ && chgrp 0 /home/gorge \ - && chmod -R g+rwX /home/gorge \ - # Add additional security hardening - && chmod 755 /gorge + && chmod -R g+rwX /home/gorge # Copy application binary with explicit permissions COPY --chmod=755 gorge /