From e2500c93debd55c134c4a1be996ab17fd210b26d Mon Sep 17 00:00:00 2001 From: Tit Petric Date: Thu, 12 Sep 2024 08:24:05 +0200 Subject: [PATCH] [TT-12865] URL matching prefixes/explicit, regex support (#6475) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### **User description** https://tyktech.atlassian.net/browse/TT-12865 --- ### **PR Type** Bug fix, Tests ___ ### **Description** - Enhanced URL path matching to handle both stripped and full URL paths, ensuring backward compatibility. - Improved error handling and logging for regex matching in `ProcessRequest`. - Updated test cases to reflect changes in URL path handling and added new tests for regex matching. - Improved regex pattern handling by supporting patterns starting with '^' in `GetPathRegexp`. ___ ### **Changes walkthrough** 📝
Relevant files
Bug fix
mw_granular_access.go
Enhance URL path matching for backward compatibility         
gateway/mw_granular_access.go
  • Introduced handling for both stripped and full URL paths.
  • Added error handling for regex matching.
  • Refactored logger initialization.
    		•  +17/-8   
    mux.go
    Improve regex pattern handling in path matching                   
    internal/httputil/mux.go
  • Added handling for patterns starting with '^'.
  • Ensured backward compatibility with existing regex patterns.
    		•  +3/-0     
    Tests
    mw_granular_access_test.go
    Update tests for enhanced URL path matching                           
    gateway/mw_granular_access_test.go
  • Updated test cases to include regex matching for listen paths.
  • Adjusted paths in test cases to reflect new listen path.
  • Added new test cases for regex matching.
    		•  +46/-27 
    mux_test.go
    Add test for regex pattern handling improvement                   
    internal/httputil/mux_test.go - Added test case for regex pattern starting with '^'.
    		+1/-0     
    ___ > 💡 **PR-Agent usage**: >Comment `/help` on the PR to get a list of all available PR-Agent tools and their descriptions --------- Co-authored-by: Tit Petric --- cli/linter/schema.json | 6 + config/config.go | 43 ++ gateway/api_definition.go | 56 ++- gateway/api_definition_test.go | 23 +- gateway/model_apispec.go | 10 +- gateway/model_urlspec.go | 22 + gateway/mw_granular_access.go | 99 +++- gateway/mw_granular_access_test.go | 78 ++-- gateway/session_manager.go | 52 ++- gateway/session_manager_test.go | 155 +++++++ internal/httputil/mux.go | 97 ++-- internal/httputil/mux_test.go | 36 +- tests/regression/issue_12865_test.go | 171 +++++++ tests/regression/testdata/issue-12865.json | 501 +++++++++++++++++++++ user/session.go | 51 +-- user/session_test.go | 147 ------ 16 files changed, 1237 insertions(+), 310 deletions(-) create mode 100644 tests/regression/issue_12865_test.go create mode 100644 tests/regression/testdata/issue-12865.json diff --git a/cli/linter/schema.json b/cli/linter/schema.json index d10f8ff8d81..061d55508f0 100644 --- a/cli/linter/schema.json +++ b/cli/linter/schema.json @@ -550,6 +550,12 @@ "enable_strict_routes": { "type": "boolean" }, + "enable_prefix_matching": { + "type": "boolean" + }, + "enable_suffix_matching": { + "type": "boolean" + }, "flush_interval": { "type": "integer" }, diff --git a/config/config.go b/config/config.go index eb9163df993..b658742b4f3 100644 --- a/config/config.go +++ b/config/config.go @@ -408,6 +408,49 @@ type HttpServerOptionsConfig struct { // Regular expressions and parameterized routes will be left alone regardless of this setting. EnableStrictRoutes bool `json:"enable_strict_routes"` + // EnablePrefixMatching changes the URL matching from wildcard mode to prefix mode. + // For example, `/json` matches `*/json*` by current default behaviour. + // If prefix matching is enabled, the match will be performed as a prefix match (`/json*`). + // + // The `/json` url would be matched as `^/json` against the following paths: + // + // - Full listen path and versioning URL (`/listen-path/v4/json`) + // - Stripped listen path URL (`/v4/json`) + // - Stripped version information (`/json`) - match. + // + // If versioning is disabled then the following URLs are considered: + // + // - Full listen path and endpoint (`/listen-path/json`) + // - Stripped listen path (`/json`) - match. + // + // For inputs that start with `/`, a prefix match is ensured by prepending + // the start of string `^` caret. + // + // For all other cases, the pattern remains unmodified. + // + // Combine this option with EnableSuffixMatching to achieve strict + // url matching with `/json` being evaluated as `^/json$`. + EnablePrefixMatching bool `json:"enable_prefix_matching"` + + // EnableSuffixMatching changes the URL matching to match as a suffix. + // For example: `/json` is matched as `/json$` against the following paths: + // + // - Full listen path and versioning URL (`/listen-path/v4/json`) + // - Stripped listen path URL (`/v4/json`) + // - Stripped version information (`/json`) - match. + // + // If versioning is disabled then the following URLs are considered: + // + // - Full listen path and endpoint (`/listen-path/json`) + // - Stripped listen path (`/json`) - match. + // + // If the input pattern already ends with a `$` (`/json$`) then + // the pattern remains unmodified. + // + // Combine this option with EnablePrefixMatching to achieve strict + // url matching with `/json` being evaluated as `^/json$`. + EnableSuffixMatching bool `json:"enable_suffix_matching"` + // Disable TLS verification. Required if you are using self-signed certificates. SSLInsecureSkipVerify bool `json:"ssl_insecure_skip_verify"` diff --git a/gateway/api_definition.go b/gateway/api_definition.go index d4880bc6d70..5f9f768882e 100644 --- a/gateway/api_definition.go +++ b/gateway/api_definition.go @@ -134,7 +134,8 @@ const ( // path is on any of the white, black or ignored lists. This is generated as part of the // configuration init type URLSpec struct { - Spec *regexp.Regexp + spec *regexp.Regexp + Status URLStatus MethodActions map[string]apidef.EndpointMethodMeta Whitelist apidef.EndPointMeta @@ -817,21 +818,28 @@ func (a APIDefinitionLoader) getPathSpecs(apiVersionDef apidef.VersionInfo, conf return combinedPath, len(whiteListPaths) > 0 } -// match mux tags, `{id}`. -var apiLangIDsRegex = regexp.MustCompile(`{([^}]+)}`) - func (a APIDefinitionLoader) generateRegex(stringSpec string, newSpec *URLSpec, specType URLStatus, conf config.Config) { - // replace mux named parameters with regex path match - asRegexStr := apiLangIDsRegex.ReplaceAllString(stringSpec, `([^/]+)`) + var ( + pattern string + err error + ) + // Hook per-api settings here via newSpec *URLSpec + isPrefixMatch := conf.HttpServerOptions.EnablePrefixMatching + isSuffixMatch := conf.HttpServerOptions.EnableSuffixMatching + isIgnoreCase := newSpec.IgnoreCase || conf.IgnoreEndpointCase + + pattern = httputil.PreparePathRegexp(stringSpec, isPrefixMatch, isSuffixMatch) // Case insensitive match - if newSpec.IgnoreCase || conf.IgnoreEndpointCase { - asRegexStr = "(?i)" + asRegexStr + if isIgnoreCase { + pattern = "(?i)" + pattern } - asRegex, _ := regexp.Compile(asRegexStr) + asRegex, err := regexp.Compile(pattern) + log.WithError(err).Debugf("URLSpec: %s => %s type=%d", stringSpec, pattern, specType) + newSpec.Status = specType - newSpec.Spec = asRegex + newSpec.spec = asRegex } func (a APIDefinitionLoader) compilePathSpec(paths []string, specType URLStatus, conf config.Config) []URLSpec { @@ -1493,7 +1501,7 @@ func (a *APISpec) getURLStatus(stat URLStatus) RequestStatus { // URLAllowedAndIgnored checks if a url is allowed and ignored. func (a *APISpec) URLAllowedAndIgnored(r *http.Request, rxPaths []URLSpec, whiteListStatus bool) (RequestStatus, interface{}) { for i := range rxPaths { - if !rxPaths[i].Spec.MatchString(r.URL.Path) { + if !rxPaths[i].matchesPath(r.URL.Path, a) { continue } @@ -1504,7 +1512,7 @@ func (a *APISpec) URLAllowedAndIgnored(r *http.Request, rxPaths []URLSpec, white // Check if ignored for i := range rxPaths { - if !rxPaths[i].Spec.MatchString(r.URL.Path) { + if !rxPaths[i].matchesPath(r.URL.Path, a) { continue } @@ -1776,10 +1784,34 @@ func (a *APISpec) Version(r *http.Request) (*apidef.VersionInfo, RequestStatus) return &version, StatusOk } +// StripListenPath will strip the listen path from the URL, keeping version in tact. func (a *APISpec) StripListenPath(reqPath string) string { return httputil.StripListenPath(a.Proxy.ListenPath, reqPath) } +// StripVersionPath will strip the version from the URL. The input URL +// should already have listen path stripped. +func (a *APISpec) StripVersionPath(reqPath string) string { + // First part of the url is the version fragment + part := strings.Split(strings.Trim(reqPath, "/"), "/")[0] + + matchesUrlVersioningPattern := true + if a.VersionDefinition.UrlVersioningPattern != "" { + re, err := regexp.Compile(a.VersionDefinition.UrlVersioningPattern) + if err != nil { + log.Error("Error compiling versioning pattern: ", err) + } else { + matchesUrlVersioningPattern = re.Match([]byte(part)) + } + } + + if (a.VersionDefinition.StripVersioningData || a.VersionDefinition.StripPath) && matchesUrlVersioningPattern { + return strings.Replace(reqPath, "/"+part+"/", "/", 1) + } + + return reqPath +} + func (a *APISpec) SanitizeProxyPaths(r *http.Request) { if !a.Proxy.StripListenPath { return diff --git a/gateway/api_definition_test.go b/gateway/api_definition_test.go index e7e3dec783d..7817b641cc3 100644 --- a/gateway/api_definition_test.go +++ b/gateway/api_definition_test.go @@ -412,13 +412,15 @@ func TestConflictingPaths(t *testing.T) { ts.Run(t, []test.TestCase{ // Should ignore auth check - {Method: "POST", Path: "/customer-servicing/documents/metadata/purge", Code: http.StatusOK}, - {Method: "GET", Path: "/customer-servicing/documents/metadata/{id}", Code: http.StatusOK}, + {Method: "POST", Path: "/metadata/purge", Code: http.StatusOK}, + {Method: "GET", Path: "/metadata/{id}", Code: http.StatusOK}, }...) } func TestIgnored(t *testing.T) { - ts := StartTest(nil) + ts := StartTest(func(c *config.Config) { + c.HttpServerOptions.EnablePrefixMatching = true + }) defer ts.Close() t.Run("Extended Paths", func(t *testing.T) { @@ -445,14 +447,13 @@ func TestIgnored(t *testing.T) { {Path: "/ignored/literal", Code: http.StatusOK}, {Path: "/ignored/123/test", Code: http.StatusOK}, // Only GET is ignored - {Method: "POST", Path: "/ext/ignored/literal", Code: 401}, + {Method: "POST", Path: "/ext/ignored/literal", Code: http.StatusUnauthorized}, - {Path: "/", Code: 401}, + {Path: "/", Code: http.StatusUnauthorized}, }...) }) t.Run("Simple Paths", func(t *testing.T) { - ts.Gw.BuildAndLoadAPI(func(spec *APISpec) { UpdateAPIVersion(spec, "v1", func(v *apidef.VersionInfo) { v.Paths.Ignored = []string{"/ignored/literal", "/ignored/{id}/test"} @@ -467,15 +468,14 @@ func TestIgnored(t *testing.T) { // Should ignore auth check {Path: "/ignored/literal", Code: http.StatusOK}, {Path: "/ignored/123/test", Code: http.StatusOK}, - // All methods ignored - {Method: "POST", Path: "/ext/ignored/literal", Code: http.StatusOK}, - {Path: "/", Code: 401}, + {Method: "POST", Path: "/ext/ignored/literal", Code: http.StatusUnauthorized}, + + {Path: "/", Code: http.StatusUnauthorized}, }...) }) t.Run("With URL rewrite", func(t *testing.T) { - ts.Gw.BuildAndLoadAPI(func(spec *APISpec) { UpdateAPIVersion(spec, "v1", func(v *apidef.VersionInfo) { v.ExtendedPaths.URLRewrite = []apidef.URLRewriteMeta{{ @@ -510,7 +510,6 @@ func TestIgnored(t *testing.T) { }) t.Run("Case Sensitivity", func(t *testing.T) { - spec := BuildAPI(func(spec *APISpec) { UpdateAPIVersion(spec, "v1", func(v *apidef.VersionInfo) { v.ExtendedPaths.Ignored = []apidef.EndPointMeta{{Path: "/Foo", IgnoreCase: false}, {Path: "/bar", IgnoreCase: true}} @@ -1032,7 +1031,7 @@ func (ts *Test) testPrepareDefaultVersion() (string, *APISpec) { func TestGetVersionFromRequest(t *testing.T) { versionInfo := apidef.VersionInfo{} - versionInfo.Paths.WhiteList = []string{"/foo"} + versionInfo.Paths.WhiteList = []string{"/foo", "/v3/foo"} versionInfo.Paths.BlackList = []string{"/bar"} t.Run("Header location", func(t *testing.T) { diff --git a/gateway/model_apispec.go b/gateway/model_apispec.go index 2054a5e0cea..b3a6a5eae23 100644 --- a/gateway/model_apispec.go +++ b/gateway/model_apispec.go @@ -14,10 +14,10 @@ func (a *APISpec) CheckSpecMatchesStatus(r *http.Request, rxPaths []URLSpec, mod if rxPaths[i].Status != mode { continue } - if !rxPaths[i].Spec.MatchString(matchPath) { + if !rxPaths[i].matchesMethod(method) { continue } - if !rxPaths[i].matchesMethod(method) { + if !rxPaths[i].matchesPath(matchPath, a) { continue } @@ -36,10 +36,10 @@ func (a *APISpec) FindSpecMatchesStatus(r *http.Request, rxPaths []URLSpec, mode if rxPaths[i].Status != mode { continue } - if !rxPaths[i].Spec.MatchString(matchPath) { + if !rxPaths[i].matchesMethod(method) { continue } - if !rxPaths[i].matchesMethod(method) { + if !rxPaths[i].matchesPath(matchPath, a) { continue } @@ -64,7 +64,7 @@ func (a *APISpec) getMatchPathAndMethod(r *http.Request, mode URLStatus) (string } if a.Proxy.ListenPath != "/" { - matchPath = strings.TrimPrefix(matchPath, a.Proxy.ListenPath) + matchPath = a.StripListenPath(matchPath) } if !strings.HasPrefix(matchPath, "/") { diff --git a/gateway/model_urlspec.go b/gateway/model_urlspec.go index 8d04388a6f6..5ea94655519 100644 --- a/gateway/model_urlspec.go +++ b/gateway/model_urlspec.go @@ -98,3 +98,25 @@ func (u *URLSpec) matchesMethod(method string) bool { return false } } + +// matchesPath takes the input string and matches it against an internal regex. +// it will match the regex against the clean URL with stripped listen path first, +// then it will match against the full URL including the listen path as provided. +// APISpec to provide URL sanitization of the input is passed along. +func (a *URLSpec) matchesPath(reqPath string, api *APISpec) bool { + clean := api.StripListenPath(reqPath) + noVersion := api.StripVersionPath(clean) + // match /users + if noVersion != clean && a.spec.MatchString(noVersion) { + return true + } + // match /v3/users + if a.spec.MatchString(clean) { + return true + } + // match /listenpath/v3/users + if a.spec.MatchString(reqPath) { + return true + } + return false +} diff --git a/gateway/mw_granular_access.go b/gateway/mw_granular_access.go index 52dfbf13e3f..090d8c0a014 100644 --- a/gateway/mw_granular_access.go +++ b/gateway/mw_granular_access.go @@ -3,8 +3,13 @@ package gateway import ( "errors" "net/http" + "slices" + "strings" + + "github.com/sirupsen/logrus" "github.com/TykTechnologies/tyk/internal/httputil" + "github.com/TykTechnologies/tyk/regexp" ) // GranularAccessMiddleware will check if a URL is specifically enabled for the key @@ -22,7 +27,6 @@ func (m *GranularAccessMiddleware) ProcessRequest(w http.ResponseWriter, r *http return nil, http.StatusOK } - logger := m.Logger().WithField("path", r.URL.Path) session := ctxGetSession(r) sessionVersionData, foundAPI := session.AccessRights[m.Spec.APIID] @@ -34,33 +38,96 @@ func (m *GranularAccessMiddleware) ProcessRequest(w http.ResponseWriter, r *http return nil, http.StatusOK } - urlPath := m.Spec.StripListenPath(r.URL.Path) + gwConfig := m.Gw.GetConfig() + + // Hook per-api settings here (m.Spec...) + isPrefixMatch := gwConfig.HttpServerOptions.EnablePrefixMatching + isSuffixMatch := gwConfig.HttpServerOptions.EnableSuffixMatching + + if isPrefixMatch { + urlPaths := []string{ + m.Spec.StripListenPath(r.URL.Path), + r.URL.Path, + } + + logger := m.Logger().WithField("paths", urlPaths) + + for _, accessSpec := range sessionVersionData.AllowedURLs { + if !slices.Contains(accessSpec.Methods, r.Method) { + continue + } + + // Append $ if so configured to match end of request path. + pattern := httputil.PreparePathRegexp(accessSpec.URL, isPrefixMatch, isSuffixMatch) + if isSuffixMatch && !strings.HasSuffix(pattern, "$") { + pattern += "$" + } + + match, err := httputil.MatchPaths(pattern, urlPaths) + + // unconditional log of err/match/url + // if loglevel is set to debug verbosity increases and all requests are logged, + // regardless if an error occured or not. + if gwConfig.LogLevel == "debug" || err != nil { + logger = logger.WithError(err).WithField("pattern", pattern).WithField("match", match) + if err != nil { + logger.Error("error matching endpoint") + } else { + logger.Debug("matching endpoint") + } + } + + if err != nil || !match { + continue + } + return m.pass() + } + + return m.block(logger) + } + + logger := m.Logger().WithField("paths", []string{r.URL.Path}) + + // Legacy behaviour (5.5.0 and earlier), wildcard match against full request path. + // Fixed error handling in regex compilation to continue to next pattern (block). + urlPath := r.URL.Path for _, accessSpec := range sessionVersionData.AllowedURLs { - url := accessSpec.URL - match, err := httputil.MatchEndpoint(url, urlPath) - if err != nil { - logger.WithError(err).Errorf("error matching path regex: %q, skipping", url) + if !slices.Contains(accessSpec.Methods, r.Method) { continue } - if !match { - continue + pattern := accessSpec.URL + + // Extends legacy by honoring isSuffixMatch. + // Append $ if so configured to match end of request path. + if isSuffixMatch && !strings.HasSuffix(pattern, "$") { + pattern += "$" } - logger.WithField("pattern", url).WithField("match", match).Debug("checking allowed url") + logger.Debug("Checking: ", urlPath, " Against:", pattern) - // if a path is matched, but isn't matched on method, - // then we continue onto the next path for evaluation. - for _, method := range accessSpec.Methods { - if method == r.Method { - return nil, http.StatusOK - } + // Wildcard match (user supplied, as-is) + asRegex, err := regexp.Compile(pattern) + if err != nil { + logger.WithError(err).Error("error compiling regex") + continue + } + + match := asRegex.MatchString(r.URL.Path) + if match { + return m.pass() } } - logger.Info("Attempted access to unauthorised endpoint (Granular).") + return m.block(logger) +} +func (m *GranularAccessMiddleware) block(logger *logrus.Entry) (error, int) { + logger.Info("Attempted access to unauthorised endpoint (Granular).") return errors.New("Access to this resource has been disallowed"), http.StatusForbidden +} +func (m *GranularAccessMiddleware) pass() (error, int) { + return nil, http.StatusOK } diff --git a/gateway/mw_granular_access_test.go b/gateway/mw_granular_access_test.go index e4cc9d6c6ff..edbc6c5826c 100644 --- a/gateway/mw_granular_access_test.go +++ b/gateway/mw_granular_access_test.go @@ -4,35 +4,40 @@ import ( "net/http" "testing" + "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/header" "github.com/TykTechnologies/tyk/test" "github.com/TykTechnologies/tyk/user" ) func TestGranularAccessMiddleware_ProcessRequest(t *testing.T) { - g := StartTest(nil) + g := StartTest(func(c *config.Config) { + c.HttpServerOptions.EnablePrefixMatching = true + }) defer g.Close() api := g.Gw.BuildAndLoadAPI(func(spec *APISpec) { - spec.Proxy.ListenPath = "/" + spec.Proxy.ListenPath = "/test" spec.UseKeylessAccess = false })[0] + allowedURLs := []user.AccessSpec{ + { + URL: "^/valid_path", + Methods: []string{"GET"}, + }, + { + URL: "^/test/try_valid_path", + Methods: []string{"GET"}, + }, + } + _, directKey := g.CreateSession(func(s *user.SessionState) { s.AccessRights = map[string]user.AccessDefinition{ api.APIID: { - APIID: api.APIID, - APIName: api.Name, - AllowedURLs: []user.AccessSpec{ - { - URL: "^/*.$", - Methods: []string{"GET"}, - }, - { - URL: "^/valid_path.*", - Methods: []string{"GET"}, - }, - }, + APIID: api.APIID, + APIName: api.Name, + AllowedURLs: allowedURLs, }, } }) @@ -40,14 +45,9 @@ func TestGranularAccessMiddleware_ProcessRequest(t *testing.T) { pID := g.CreatePolicy(func(p *user.Policy) { p.AccessRights = map[string]user.AccessDefinition{ api.APIID: { - APIID: api.APIID, - APIName: api.Name, - AllowedURLs: []user.AccessSpec{ - { - URL: "^/valid_path.*", - Methods: []string{"GET"}, - }, - }, + APIID: api.APIID, + APIName: api.Name, + AllowedURLs: allowedURLs, }, } }) @@ -61,10 +61,21 @@ func TestGranularAccessMiddleware_ProcessRequest(t *testing.T) { header.Authorization: directKey, } + t.Run("should return 200 OK on regex matching listen path", func(t *testing.T) { + _, _ = g.Run(t, []test.TestCase{ + { + Path: "/test/try_valid_path", + Method: http.MethodGet, + Code: http.StatusOK, + Headers: authHeaderWithDirectKey, + }, + }...) + }) + t.Run("should return 200 OK on allowed path with allowed method", func(t *testing.T) { _, _ = g.Run(t, []test.TestCase{ { - Path: "/valid_path", + Path: "/test/valid_path", Method: http.MethodGet, Code: http.StatusOK, Headers: authHeaderWithDirectKey, @@ -75,7 +86,7 @@ func TestGranularAccessMiddleware_ProcessRequest(t *testing.T) { t.Run("should return 403 Forbidden on allowed path with disallowed method", func(t *testing.T) { _, _ = g.Run(t, []test.TestCase{ { - Path: "/valid_path", + Path: "/test/valid_path", Method: http.MethodPost, Code: http.StatusForbidden, Headers: authHeaderWithDirectKey, @@ -86,7 +97,7 @@ func TestGranularAccessMiddleware_ProcessRequest(t *testing.T) { t.Run("should return 403 Forbidden on disallowed path with allowed method", func(t *testing.T) { _, _ = g.Run(t, []test.TestCase{ { - Path: "/invalid_path", + Path: "/test/invalid_path", Method: http.MethodGet, Code: http.StatusForbidden, Headers: authHeaderWithDirectKey, @@ -101,10 +112,21 @@ func TestGranularAccessMiddleware_ProcessRequest(t *testing.T) { header.Authorization: policyAppliedKey, } + t.Run("should return 200 OK on regex matching listen path", func(t *testing.T) { + _, _ = g.Run(t, []test.TestCase{ + { + Path: "/test/try_valid_path", + Method: http.MethodGet, + Code: http.StatusOK, + Headers: authHeaderWithPolicyAppliedKey, + }, + }...) + }) + t.Run("should return 200 OK on allowed path with allowed method", func(t *testing.T) { _, _ = g.Run(t, []test.TestCase{ { - Path: "/valid_path", + Path: "/test/valid_path", Method: http.MethodGet, Code: http.StatusOK, Headers: authHeaderWithPolicyAppliedKey, @@ -115,7 +137,7 @@ func TestGranularAccessMiddleware_ProcessRequest(t *testing.T) { t.Run("should return 403 Forbidden on allowed path with disallowed method", func(t *testing.T) { _, _ = g.Run(t, []test.TestCase{ { - Path: "/valid_path", + Path: "/test/valid_path", Method: http.MethodPost, Code: http.StatusForbidden, Headers: authHeaderWithPolicyAppliedKey, @@ -126,7 +148,7 @@ func TestGranularAccessMiddleware_ProcessRequest(t *testing.T) { t.Run("should return 403 Forbidden on disallowed path with allowed method", func(t *testing.T) { _, _ = g.Run(t, []test.TestCase{ { - Path: "/invalid_path", + Path: "/test/invalid_path", Method: http.MethodGet, Code: http.StatusForbidden, Headers: authHeaderWithPolicyAppliedKey, diff --git a/gateway/session_manager.go b/gateway/session_manager.go index 72f996d8681..bce1f784741 100644 --- a/gateway/session_manager.go +++ b/gateway/session_manager.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "strings" "time" "github.com/sirupsen/logrus" @@ -14,9 +15,11 @@ import ( "github.com/TykTechnologies/leakybucket/memorycache" "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/internal/httputil" "github.com/TykTechnologies/tyk/internal/rate" "github.com/TykTechnologies/tyk/internal/rate/limiter" "github.com/TykTechnologies/tyk/internal/redis" + "github.com/TykTechnologies/tyk/regexp" "github.com/TykTechnologies/tyk/storage" "github.com/TykTechnologies/tyk/user" ) @@ -225,6 +228,52 @@ func (sfr sessionFailReason) String() string { } } +func (l *SessionLimiter) RateLimitInfo(r *http.Request, api *APISpec, endpoints user.Endpoints) (*user.EndpointRateLimitInfo, bool) { + // Hook per-api settings here (m.Spec...) + isPrefixMatch := l.config.HttpServerOptions.EnablePrefixMatching + isSuffixMatch := l.config.HttpServerOptions.EnableSuffixMatching + + urlPaths := []string{ + api.StripListenPath(r.URL.Path), + r.URL.Path, + } + + for _, endpoint := range endpoints { + if !endpoint.Methods.Contains(r.Method) { + continue + } + + pattern := httputil.PreparePathRegexp(endpoint.Path, isPrefixMatch, isSuffixMatch) + + asRegex, err := regexp.Compile(pattern) + if err != nil { + log.WithError(err).Error("endpoint rate limit: error compiling regex") + continue + } + + for _, urlPath := range urlPaths { + match := asRegex.MatchString(urlPath) + if !match { + break + } + + for _, endpointMethod := range endpoint.Methods { + if !strings.EqualFold(endpointMethod.Name, r.Method) { + continue + } + + return &user.EndpointRateLimitInfo{ + KeySuffix: storage.HashStr(fmt.Sprintf("%s:%s", endpointMethod.Name, endpoint.Path)), + Rate: endpointMethod.Limit.Rate, + Per: endpointMethod.Limit.Per, + }, true + } + } + } + return nil, false + +} + // ForwardMessage will enforce rate limiting, returning a non-zero // sessionFailReason if session limits have been exceeded. // Key values to manage rate are Rate and Per, e.g. Rate of 10 messages @@ -242,8 +291,7 @@ func (l *SessionLimiter) ForwardMessage(r *http.Request, session *user.SessionSt endpointRLKeySuffix = "" ) - reqEndpoint := api.StripListenPath(r.URL.Path) - endpointRLInfo, doEndpointRL := accessDef.Endpoints.RateLimitInfo(r.Method, reqEndpoint) + endpointRLInfo, doEndpointRL := l.RateLimitInfo(r, api, accessDef.Endpoints) if doEndpointRL { apiLimit.Rate = endpointRLInfo.Rate apiLimit.Per = endpointRLInfo.Per diff --git a/gateway/session_manager_test.go b/gateway/session_manager_test.go index 847ef644b5d..9f54069ec77 100644 --- a/gateway/session_manager_test.go +++ b/gateway/session_manager_test.go @@ -1,11 +1,15 @@ package gateway import ( + "net/http" + "net/http/httptest" "testing" "github.com/stretchr/testify/assert" "github.com/TykTechnologies/tyk/apidef" + "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/storage" "github.com/TykTechnologies/tyk/user" ) @@ -140,3 +144,154 @@ func TestGetAccessDefinitionByAPIIDOrSession(t *testing.T) { assert.NoError(t, err) }) } + +func TestSessionLimiter_RateLimitInfo(t *testing.T) { + limiter := &SessionLimiter{config: &config.Default} + spec := BuildAPI(func(a *APISpec) { + a.Proxy.ListenPath = "/" + })[0] + + tests := []struct { + name string + method string + path string + endpoints user.Endpoints + expected *user.EndpointRateLimitInfo + found bool + }{ + { + name: "Matching endpoint and method", + method: http.MethodGet, + path: "/api/v1/users", + endpoints: user.Endpoints{ + { + Path: "/api/v1/users", + Methods: []user.EndpointMethod{ + {Name: "GET", Limit: user.RateLimit{Rate: 100, Per: 60}}, + }, + }, + }, + expected: &user.EndpointRateLimitInfo{ + KeySuffix: storage.HashStr("GET:/api/v1/users"), + Rate: 100, + Per: 60, + }, + found: true, + }, + { + name: "Matching endpoint, non-matching method", + path: "/api/v1/users", + method: http.MethodPost, + endpoints: []user.Endpoint{ + { + Path: "/api/v1/users", + Methods: []user.EndpointMethod{ + {Name: "GET", Limit: user.RateLimit{Rate: 100, Per: 60}}, + }, + }, + }, + expected: nil, + found: false, + }, + { + name: "Non-matching endpoint", + method: http.MethodGet, + path: "/api/v1/products", + endpoints: []user.Endpoint{ + { + Path: "/api/v1/users", + Methods: []user.EndpointMethod{ + {Name: "GET", Limit: user.RateLimit{Rate: 100, Per: 60}}, + }, + }, + }, + expected: nil, + found: false, + }, + { + name: "Regex path matching", + path: "/api/v1/users/123", + method: http.MethodGet, + endpoints: []user.Endpoint{ + { + Path: "/api/v1/users/[0-9]+", + Methods: []user.EndpointMethod{ + {Name: "GET", Limit: user.RateLimit{Rate: 50, Per: 30}}, + }, + }, + }, + expected: &user.EndpointRateLimitInfo{ + KeySuffix: storage.HashStr("GET:/api/v1/users/[0-9]+"), + Rate: 50, + Per: 30, + }, + found: true, + }, + { + name: "Invalid regex path", + path: "/api/v1/users", + method: http.MethodGet, + endpoints: []user.Endpoint{ + { + Path: "[invalid regex", + Methods: []user.EndpointMethod{ + {Name: "GET", Limit: user.RateLimit{Rate: 100, Per: 60}}, + }, + }, + }, + expected: nil, + found: false, + }, + { + name: "Invalid regex path and valid url", + path: "/api/v1/users", + method: http.MethodGet, + endpoints: []user.Endpoint{ + { + Path: "[invalid regex", + Methods: []user.EndpointMethod{ + {Name: "GET", Limit: user.RateLimit{Rate: 100, Per: 60}}, + }, + }, + { + Path: "/api/v1/users", + Methods: []user.EndpointMethod{ + {Name: "GET", Limit: user.RateLimit{Rate: 100, Per: 60}}, + }, + }, + }, + expected: &user.EndpointRateLimitInfo{ + KeySuffix: storage.HashStr("GET:/api/v1/users"), + Rate: 100, + Per: 60, + }, + found: true, + }, + { + name: "nil endpoints", + path: "/api/v1/users", + method: http.MethodGet, + endpoints: nil, + expected: nil, + found: false, + }, + { + name: "empty endpoints", + path: "/api/v1/users", + method: http.MethodGet, + endpoints: user.Endpoints{}, + expected: nil, + found: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest(tt.method, tt.path, nil) + + result, found := limiter.RateLimitInfo(req, spec, tt.endpoints) + assert.Equal(t, tt.found, found) + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/internal/httputil/mux.go b/internal/httputil/mux.go index bee0b5d7c7d..e59fb0a1fd2 100644 --- a/internal/httputil/mux.go +++ b/internal/httputil/mux.go @@ -1,6 +1,8 @@ package httputil import ( + "errors" + "fmt" "regexp" "strings" @@ -9,34 +11,55 @@ import ( "github.com/TykTechnologies/tyk/internal/maps" ) -// routeCache holds the raw routes as they are mapped to mux regular expressions. -// e.g. `/foo` becomes `^/foo$` or similar, and parameters get matched and replaced. +// routeCache holds the raw routes as they are mapped from mux parameters to regular expressions. +// e.g. `/foo/{id}` becomes `^/foo/([^/]+)$` or similar. var pathRegexpCache = maps.NewStringMap() -// GetPathRegexp will convert a mux route url to a regular expression string. -// The results for subsequent invocations with the same parameters are cached. -func GetPathRegexp(pattern string) (string, error) { - val, ok := pathRegexpCache.Get(pattern) +// apiLandIDsRegex matches mux-style parameters like `{id}`. +var apiLangIDsRegex = regexp.MustCompile(`{([^}]+)}`) + +// PreparePathRexep will replace mux-style parameters in input with a compatible regular expression. +// Parameters like `{id}` would be replaced to `([^/]+)`. If the input pattern provides a starting +// or ending delimiters (`^` or `$`), the pattern is returned. +// If prefix is true, and pattern starts with /, the returned pattern prefixes a `^` to the regex. +// No other prefix matches are possible so only `/` to `^/` conversion is considered. +// If suffix is true, the returned pattern suffixes a `$` to the regex. +// If both prefix and suffixes are achieved, an explicit match is made. +func PreparePathRegexp(pattern string, prefix bool, suffix bool) string { + // Construct cache key from pattern and flags + key := fmt.Sprintf("%s:%v:%v", pattern, prefix, suffix) + val, ok := pathRegexpCache.Get(key) if ok { - return val, nil + return val } + // Replace mux named parameters with regex path match. if IsMuxTemplate(pattern) { - dummyRouter := mux.NewRouter() - route := dummyRouter.PathPrefix(pattern) - result, err := route.GetPathRegexp() - if err != nil { - return "", err - } + pattern = apiLangIDsRegex.ReplaceAllString(pattern, `([^/]+)`) + } - pathRegexpCache.Set(pattern, result) - return result, nil + // Replace mux wildcard path with a `.*` (match 0 or more characters) + if strings.Contains(pattern, "/*") { + pattern = strings.ReplaceAll(pattern, "/*/", "/[^/]+/") + pattern = strings.ReplaceAll(pattern, "/*", "/.*") } - if strings.HasPrefix(pattern, "/") { - return "^" + pattern, nil + // Pattern `/users` becomes `^/users`. + if prefix && strings.HasPrefix(pattern, "/") { + pattern = "^" + pattern } - return "^.*" + pattern, nil + + // Append $ if necessary to enforce suffix matching. + // Pattern `/users` becomes `/users$`. + // Pattern `^/users` becomes `^/users$`. + if suffix && !strings.HasSuffix(pattern, "$") { + pattern = pattern + "$" + } + + // Save cache for following invocations. + pathRegexpCache.Set(key, pattern) + + return pattern } // IsMuxTemplate determines if a pattern is a mux template by counting the number of opening and closing braces. @@ -78,25 +101,39 @@ func StripListenPath(listenPath, urlPath string) (res string) { return reg.ReplaceAllString(res, "") } -// MatchEndpoint matches pattern with request endpoint. -func MatchEndpoint(pattern string, endpoint string) (bool, error) { - if pattern == endpoint { - return true, nil - } - - if pattern == "" { +// MatchPath matches regexp pattern with request endpoint. +func MatchPath(pattern string, endpoint string) (bool, error) { + if strings.Trim(pattern, "^$") == "" || endpoint == "" { return false, nil } - - clean, err := GetPathRegexp(pattern) - if err != nil { - return false, err + if pattern == endpoint || pattern == "^"+endpoint+"$" { + return true, nil } - asRegex, err := regexp.Compile(clean) + asRegex, err := regexp.Compile(pattern) if err != nil { return false, err } return asRegex.MatchString(endpoint), nil } + +// MatchPaths matches regexp pattern with multiple request URLs endpoint paths. +// It will return true if any of them is correctly matched, with no error. +// If no matches occur, any errors will be retured joined with errors.Join. +func MatchPaths(pattern string, endpoints []string) (bool, error) { + var errs []error + + for _, endpoint := range endpoints { + match, err := MatchPath(pattern, endpoint) + if err != nil { + errs = append(errs, err) + continue + } + if match { + return true, nil + } + } + + return false, errors.Join(errs...) +} diff --git a/internal/httputil/mux_test.go b/internal/httputil/mux_test.go index 189d7b805e6..94e7914fe1d 100644 --- a/internal/httputil/mux_test.go +++ b/internal/httputil/mux_test.go @@ -9,37 +9,36 @@ import ( "github.com/TykTechnologies/tyk/internal/httputil" ) -func testPathRegexp(tb testing.TB, in string, want string) string { +func pathRegexp(tb testing.TB, in string, want string) string { tb.Helper() - res, err := httputil.GetPathRegexp(in) - assert.NoError(tb, err) - if want != "" { - assert.Equal(tb, want, res) - } + res := httputil.PreparePathRegexp(in, true, false) + assert.Equal(tb, want, res) return res } -func TestGetPathRegexp(t *testing.T) { +func TestPreparePathRegexp(t *testing.T) { tests := map[string]string{ "/users*.": "^/users*.", "/users": "^/users", - "users": "^.*users", + "users": "users", + "^/test/users": "^/test/users", "/users$": "^/users$", "/users/.*": "^/users/.*", - "/users/{id}": "^/users/(?P[^/]+)", - "/users/{id}/profile/{type:[a-zA-Z]+}": "^/users/(?P[^/]+)/profile/(?P[a-zA-Z]+)", - "/static/{path}/assets/{file}": "^/static/(?P[^/]+)/assets/(?P[^/]+)", - "/items/{itemID:[0-9]+}/details/{detail}": "^/items/(?P[0-9]+)/details/(?P[^/]+)", + "/users/{id}": "^/users/([^/]+)", + "/users/{id}$": "^/users/([^/]+)$", + "/users/{id}/profile/{type:[a-zA-Z]+}": "^/users/([^/]+)/profile/([^/]+)", + "/static/{path}/assets/{file}": "^/static/([^/]+)/assets/([^/]+)", + "/items/{itemID:[0-9]+}/details/{detail}": "^/items/([^/]+)/details/([^/]+)", } for k, v := range tests { - testPathRegexp(t, k, v) + pathRegexp(t, k, v) } } func TestGetPathRegexpWithRegexCompile(t *testing.T) { - pattern := testPathRegexp(t, "/api/v1/users/{userId}/roles/{roleId}", "") + pattern := pathRegexp(t, "/api/v1/users/{userId}/roles/{roleId}", "^/api/v1/users/([^/]+)/roles/([^/]+)") matched, err := regexp.MatchString(pattern, "/api/v1/users/10512/roles/32587") assert.NoError(t, err) @@ -63,7 +62,7 @@ func TestStripListenPath(t *testing.T) { assert.Equal(t, "/anything/get", httputil.StripListenPath("/{myPattern:foo|bar}", "/anything/get")) } -func TestMatchEndpoint(t *testing.T) { +func TestMatchPaths(t *testing.T) { tests := []struct { name string pattern string @@ -138,7 +137,7 @@ func TestMatchEndpoint(t *testing.T) { name: "both empty endpoints", pattern: "", endpoint: "", - match: true, + match: false, isErr: false, }, { @@ -152,7 +151,10 @@ func TestMatchEndpoint(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result, err := httputil.MatchEndpoint(tt.pattern, tt.endpoint) + // explicit match inputs as `^/path$` + pattern := httputil.PreparePathRegexp(tt.pattern, true, true) + + result, err := httputil.MatchPaths(pattern, []string{tt.endpoint}) assert.Equal(t, tt.match, result) assert.Equal(t, tt.isErr, err != nil) }) diff --git a/tests/regression/issue_12865_test.go b/tests/regression/issue_12865_test.go new file mode 100644 index 00000000000..4a3de8b0324 --- /dev/null +++ b/tests/regression/issue_12865_test.go @@ -0,0 +1,171 @@ +package regression + +import ( + "net/http" + "testing" + + "github.com/TykTechnologies/tyk/config" + "github.com/TykTechnologies/tyk/gateway" + "github.com/TykTechnologies/tyk/header" + "github.com/TykTechnologies/tyk/test" + "github.com/TykTechnologies/tyk/user" +) + +func Test_Issue12865(t *testing.T) { + t.Run("Wildcard", func(t *testing.T) { + ts := gateway.StartTest(func(c *config.Config) { + c.HttpServerOptions.EnablePrefixMatching = false + c.HttpServerOptions.EnableSuffixMatching = false + }) + defer ts.Close() + + // load api definition from file + api := loadAPISpec(t, "testdata/issue-12865.json") + + ts.Gw.LoadAPI(api) + + _, directKey := ts.CreateSession(func(s *user.SessionState) { + s.AccessRights = map[string]user.AccessDefinition{ + api.APIID: { + APIID: api.APIID, + APIName: api.Name, + Limit: user.APILimit{ + QuotaMax: 30, + }, + }, + } + }) + + headers := map[string]string{ + header.Authorization: directKey, + } + + // issue request against /test to trigger panic + ts.Run(t, []test.TestCase{ + {Path: "/test/anything", Method: http.MethodGet, Code: http.StatusOK}, + {Path: "/test/anything/health", Method: http.MethodGet, Code: http.StatusOK}, + {Path: "/test/anything/status/200", Method: http.MethodGet, Code: http.StatusOK}, + {Path: "/test/status/200", Method: http.MethodGet, Code: http.StatusUnauthorized}, + {Headers: headers, Path: "/test/status/200", Method: http.MethodGet, Code: http.StatusOK}, + {Path: "/test/status/200/anything", Method: http.MethodGet, Code: http.StatusOK}, + }...) + }) + + t.Run("Prefix", func(t *testing.T) { + ts := gateway.StartTest(func(c *config.Config) { + c.HttpServerOptions.EnablePrefixMatching = true + c.HttpServerOptions.EnableSuffixMatching = false + }) + defer ts.Close() + + // load api definition from file + api := loadAPISpec(t, "testdata/issue-12865.json") + + ts.Gw.LoadAPI(api) + + _, directKey := ts.CreateSession(func(s *user.SessionState) { + s.AccessRights = map[string]user.AccessDefinition{ + api.APIID: { + APIID: api.APIID, + APIName: api.Name, + Limit: user.APILimit{ + QuotaMax: 30, + }, + }, + } + }) + + headers := map[string]string{ + header.Authorization: directKey, + } + + // issue request against /test to trigger panic + ts.Run(t, []test.TestCase{ + {Path: "/test/anything", Method: http.MethodGet, Code: http.StatusOK}, + {Path: "/test/anything/health", Method: http.MethodGet, Code: http.StatusOK}, + {Path: "/test/anything/status/200", Method: http.MethodGet, Code: http.StatusOK}, + {Path: "/test/status/200", Method: http.MethodGet, Code: http.StatusUnauthorized}, + {Headers: headers, Path: "/test/status/200", Method: http.MethodGet, Code: http.StatusOK}, + {Path: "/test/status/200/anything", Method: http.MethodGet, Code: http.StatusUnauthorized}, + }...) + }) + + t.Run("Prefix and Suffix", func(t *testing.T) { + ts := gateway.StartTest(func(c *config.Config) { + c.HttpServerOptions.EnablePrefixMatching = true + c.HttpServerOptions.EnableSuffixMatching = true + }) + defer ts.Close() + + // load api definition from file + api := loadAPISpec(t, "testdata/issue-12865.json") + + ts.Gw.LoadAPI(api) + + _, directKey := ts.CreateSession(func(s *user.SessionState) { + s.AccessRights = map[string]user.AccessDefinition{ + api.APIID: { + APIID: api.APIID, + APIName: api.Name, + Limit: user.APILimit{ + QuotaMax: 30, + }, + }, + } + }) + + headers := map[string]string{ + header.Authorization: directKey, + } + + // issue request against /test to trigger panic + ts.Run(t, []test.TestCase{ + {Path: "/test/anything", Method: http.MethodGet, Code: http.StatusOK}, + {Path: "/test/anything/health", Method: http.MethodGet, Code: http.StatusUnauthorized}, + {Path: "/test/anything/status/200", Method: http.MethodGet, Code: http.StatusUnauthorized}, + {Path: "/test/status/200", Method: http.MethodGet, Code: http.StatusUnauthorized}, + {Headers: headers, Path: "/test/status/200", Method: http.MethodGet, Code: http.StatusOK}, + {Path: "/test/status/200/anything", Method: http.MethodGet, Code: http.StatusUnauthorized}, + }...) + }) + + t.Run("Suffix", func(t *testing.T) { + ts := gateway.StartTest(func(c *config.Config) { + c.HttpServerOptions.EnablePrefixMatching = false + c.HttpServerOptions.EnableSuffixMatching = true + }) + defer ts.Close() + + // load api definition from file + api := loadAPISpec(t, "testdata/issue-12865.json") + + ts.Gw.LoadAPI(api) + + _, directKey := ts.CreateSession(func(s *user.SessionState) { + s.AccessRights = map[string]user.AccessDefinition{ + api.APIID: { + APIID: api.APIID, + APIName: api.Name, + Limit: user.APILimit{ + QuotaMax: 30, + }, + }, + } + }) + + headers := map[string]string{ + header.Authorization: directKey, + } + + // issue request against /test to trigger panic + ts.Run(t, []test.TestCase{ + {Path: "/test/anything", Method: http.MethodGet, Code: http.StatusOK}, + {Path: "/test/anything/health", Method: http.MethodGet, Code: http.StatusUnauthorized}, + {Path: "/test/anything/status/200", Method: http.MethodGet, Code: http.StatusUnauthorized}, + {Path: "/test/status/200", Method: http.MethodGet, Code: http.StatusUnauthorized}, + {Headers: headers, Path: "/test/status/200", Method: http.MethodGet, Code: http.StatusOK}, + {Path: "/test/status/200/anything", Method: http.MethodGet, Code: http.StatusNotFound}, + }...) + }) + +} diff --git a/tests/regression/testdata/issue-12865.json b/tests/regression/testdata/issue-12865.json new file mode 100644 index 00000000000..123d0ebccdc --- /dev/null +++ b/tests/regression/testdata/issue-12865.json @@ -0,0 +1,501 @@ +{ + "id": "66c5cebd5b600879903f1ad3", + "name": "test", + "slug": "temp", + "listen_port": 0, + "protocol": "", + "enable_proxy_protocol": false, + "api_id": "c88e2d8ed982400469a2d1738f055902", + "org_id": "645b3db586341f751f4258aa", + "use_keyless": false, + "use_oauth2": false, + "external_oauth": { + "enabled": false, + "providers": [] + }, + "use_openid": false, + "openid_options": { + "providers": [], + "segregate_by_client": false + }, + "oauth_meta": { + "allowed_access_types": [], + "allowed_authorize_types": [], + "auth_login_redirect": "" + }, + "auth": { + "name": "", + "use_param": false, + "param_name": "", + "use_cookie": false, + "cookie_name": "", + "disable_header": false, + "auth_header_name": "Authorization", + "use_certificate": false, + "validate_signature": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + } + }, + "auth_configs": { + "authToken": { + "name": "", + "use_param": false, + "param_name": "", + "use_cookie": false, + "cookie_name": "", + "disable_header": false, + "auth_header_name": "Authorization", + "use_certificate": false, + "validate_signature": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + } + }, + "basic": { + "name": "", + "use_param": false, + "param_name": "", + "use_cookie": false, + "cookie_name": "", + "disable_header": false, + "auth_header_name": "Authorization", + "use_certificate": false, + "validate_signature": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + } + }, + "coprocess": { + "name": "", + "use_param": false, + "param_name": "", + "use_cookie": false, + "cookie_name": "", + "disable_header": false, + "auth_header_name": "Authorization", + "use_certificate": false, + "validate_signature": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + } + }, + "hmac": { + "name": "", + "use_param": false, + "param_name": "", + "use_cookie": false, + "cookie_name": "", + "disable_header": false, + "auth_header_name": "Authorization", + "use_certificate": false, + "validate_signature": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + } + }, + "jwt": { + "name": "", + "use_param": false, + "param_name": "", + "use_cookie": false, + "cookie_name": "", + "disable_header": false, + "auth_header_name": "Authorization", + "use_certificate": false, + "validate_signature": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + } + }, + "oauth": { + "name": "", + "use_param": false, + "param_name": "", + "use_cookie": false, + "cookie_name": "", + "disable_header": false, + "auth_header_name": "Authorization", + "use_certificate": false, + "validate_signature": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + } + }, + "oidc": { + "name": "", + "use_param": false, + "param_name": "", + "use_cookie": false, + "cookie_name": "", + "disable_header": false, + "auth_header_name": "Authorization", + "use_certificate": false, + "validate_signature": false, + "signature": { + "algorithm": "", + "header": "", + "use_param": false, + "param_name": "", + "secret": "", + "allowed_clock_skew": 0, + "error_code": 0, + "error_message": "" + } + } + }, + "use_basic_auth": false, + "basic_auth": { + "disable_caching": false, + "cache_ttl": 0, + "extract_from_body": false, + "body_user_regexp": "", + "body_password_regexp": "" + }, + "use_mutual_tls_auth": false, + "client_certificates": [], + "upstream_certificates": {}, + "pinned_public_keys": {}, + "enable_jwt": false, + "use_standard_auth": true, + "use_go_plugin_auth": false, + "enable_coprocess_auth": false, + "custom_plugin_auth_enabled": false, + "jwt_signing_method": "", + "jwt_source": "", + "jwt_identity_base_field": "", + "jwt_client_base_field": "", + "jwt_policy_field_name": "", + "jwt_default_policies": [], + "jwt_issued_at_validation_skew": 0, + "jwt_expires_at_validation_skew": 0, + "jwt_not_before_validation_skew": 0, + "jwt_skip_kid": false, + "scopes": { + "jwt": {}, + "oidc": {} + }, + "idp_client_id_mapping_disabled": false, + "jwt_scope_to_policy_mapping": {}, + "jwt_scope_claim_name": "", + "notifications": { + "shared_secret": "", + "oauth_on_keychange_url": "" + }, + "enable_signature_checking": false, + "hmac_allowed_clock_skew": -1, + "hmac_allowed_algorithms": [], + "request_signing": { + "is_enabled": false, + "secret": "", + "key_id": "", + "algorithm": "", + "header_list": [], + "certificate_id": "", + "signature_header": "" + }, + "base_identity_provided_by": "", + "definition": { + "enabled": false, + "name": "", + "default": "", + "location": "header", + "key": "x-api-version", + "strip_path": false, + "strip_versioning_data": false, + "url_versioning_pattern": "", + "fallback_to_default": false, + "versions": {} + }, + "version_data": { + "not_versioned": true, + "default_version": "", + "versions": { + "Default": { + "name": "Default", + "expires": "", + "paths": { + "ignored": [], + "white_list": [], + "black_list": [] + }, + "use_extended_paths": true, + "extended_paths": { + "ignored": [ + { + "disabled": false, + "path": "/anything", + "method": "", + "ignore_case": false, + "method_actions": { + "GET": { + "action": "no_action", + "code": 200, + "data": "", + "headers": {} + } + } + } + ], + "url_rewrites": [ + { + "disabled": false, + "path": "/status", + "method": "GET", + "match_pattern": "/status/(.*)", + "rewrite_to": "http://google.com", + "triggers": [] + } + ], + "persist_graphql": [], + "rate_limit": [] + }, + "global_headers": {}, + "global_headers_remove": [], + "global_headers_disabled": false, + "global_response_headers": {}, + "global_response_headers_remove": [], + "global_response_headers_disabled": false, + "ignore_endpoint_case": false, + "global_size_limit": 0, + "override_target": "" + } + } + }, + "uptime_tests": { + "check_list": [], + "config": { + "expire_utime_after": 0, + "service_discovery": { + "use_discovery_service": false, + "query_endpoint": "", + "use_nested_query": false, + "parent_data_path": "", + "data_path": "", + "port_data_path": "", + "target_path": "", + "use_target_list": false, + "cache_disabled": false, + "cache_timeout": 60, + "endpoint_returns_list": false + }, + "recheck_wait": 0 + } + }, + "proxy": { + "preserve_host_header": false, + "listen_path": "/test/", + "target_url": "http://httpbin.org/", + "disable_strip_slash": true, + "strip_listen_path": true, + "enable_load_balancing": false, + "target_list": [], + "check_host_against_uptime_tests": false, + "service_discovery": { + "use_discovery_service": false, + "query_endpoint": "", + "use_nested_query": false, + "parent_data_path": "", + "data_path": "", + "port_data_path": "", + "target_path": "", + "use_target_list": false, + "cache_disabled": false, + "cache_timeout": 0, + "endpoint_returns_list": false + }, + "transport": { + "ssl_insecure_skip_verify": false, + "ssl_ciphers": [], + "ssl_min_version": 0, + "ssl_max_version": 0, + "ssl_force_common_name_check": false, + "proxy_url": "" + } + }, + "disable_rate_limit": false, + "disable_quota": false, + "custom_middleware": { + "pre": [], + "post": [], + "post_key_auth": [], + "auth_check": { + "disabled": false, + "name": "", + "path": "", + "require_session": false, + "raw_body_only": false + }, + "response": [], + "driver": "", + "id_extractor": { + "disabled": false, + "extract_from": "", + "extract_with": "", + "extractor_config": {} + } + }, + "custom_middleware_bundle": "", + "custom_middleware_bundle_disabled": false, + "cache_options": { + "cache_timeout": 60, + "enable_cache": true, + "cache_all_safe_requests": false, + "cache_response_codes": [], + "enable_upstream_cache_control": false, + "cache_control_ttl_header": "", + "cache_by_headers": [] + }, + "session_lifetime": 0, + "active": true, + "internal": false, + "auth_provider": { + "name": "", + "storage_engine": "", + "meta": {} + }, + "session_provider": { + "name": "", + "storage_engine": "", + "meta": {} + }, + "event_handlers": { + "events": {} + }, + "enable_batch_request_support": false, + "enable_ip_whitelisting": false, + "allowed_ips": [], + "enable_ip_blacklisting": false, + "blacklisted_ips": [], + "dont_set_quota_on_create": false, + "expire_analytics_after": 0, + "response_processors": [], + "CORS": { + "enable": false, + "allowed_origins": [ + "*" + ], + "allowed_methods": [ + "GET", + "POST", + "HEAD" + ], + "allowed_headers": [ + "Origin", + "Accept", + "Content-Type", + "X-Requested-With", + "Authorization" + ], + "exposed_headers": [], + "allow_credentials": false, + "max_age": 24, + "options_passthrough": false, + "debug": false + }, + "domain": "", + "certificates": [], + "do_not_track": false, + "enable_context_vars": false, + "config_data": {}, + "config_data_disabled": false, + "tag_headers": [], + "global_rate_limit": { + "disabled": false, + "rate": 0, + "per": 0 + }, + "strip_auth_data": false, + "enable_detailed_recording": false, + "graphql": { + "enabled": false, + "execution_mode": "proxyOnly", + "version": "2", + "schema": "", + "type_field_configurations": [], + "playground": { + "enabled": false, + "path": "" + }, + "engine": { + "field_configs": [], + "data_sources": [], + "global_headers": [] + }, + "proxy": { + "features": { + "use_immutable_headers": true + }, + "auth_headers": {}, + "request_headers": {}, + "use_response_extensions": { + "on_error_forwarding": false + }, + "request_headers_rewrite": {} + }, + "subgraph": { + "sdl": "" + }, + "supergraph": { + "subgraphs": [], + "merged_sdl": "", + "global_headers": {}, + "disable_query_batching": false + }, + "introspection": { + "disabled": false + } + }, + "analytics_plugin": {}, + "tags": [], + "detailed_tracing": false +} \ No newline at end of file diff --git a/user/session.go b/user/session.go index e59a93acea6..929e8efe2d3 100644 --- a/user/session.go +++ b/user/session.go @@ -7,10 +7,6 @@ import ( "strings" "time" - "github.com/TykTechnologies/tyk/internal/httputil" - - "github.com/TykTechnologies/tyk/storage" - "github.com/TykTechnologies/graphql-go-tools/pkg/graphql" "github.com/TykTechnologies/tyk/apidef" @@ -214,11 +210,6 @@ type Endpoint struct { Methods EndpointMethods `json:"methods,omitempty" msg:"methods"` } -// match matches supplied endpoint with endpoint path. -func (e Endpoint) match(endpoint string) (bool, error) { - return httputil.MatchEndpoint(e.Path, endpoint) -} - // EndpointMethods is a collection of EndpointMethod. type EndpointMethods []EndpointMethod @@ -237,6 +228,16 @@ func (em EndpointMethods) Swap(i, j int) { em[i], em[j] = em[j], em[i] } +// Contains is used to assert if a method exists in EndpointMethods. +func (em EndpointMethods) Contains(method string) bool { + for _, v := range em { + if strings.EqualFold(v.Name, method) { + return true + } + } + return false +} + // EndpointMethod holds the configuration on endpoint method level. type EndpointMethod struct { Name string `json:"name,omitempty" msg:"name,omitempty"` @@ -527,38 +528,6 @@ type EndpointRateLimitInfo struct { Per float64 } -// RateLimitInfo returns EndpointRateLimitInfo for endpoint rate limiting. -func (es Endpoints) RateLimitInfo(method string, reqEndpoint string) (*EndpointRateLimitInfo, bool) { - if len(es) == 0 { - return nil, false - } - - for _, endpoint := range es { - match, err := endpoint.match(reqEndpoint) - if err != nil { - log.WithError(err).Errorf("error matching path regex: %q, skipping", endpoint.Path) - } - - if !match { - continue - } - - for _, endpointMethod := range endpoint.Methods { - if !strings.EqualFold(endpointMethod.Name, method) { - continue - } - - return &EndpointRateLimitInfo{ - KeySuffix: storage.HashStr(fmt.Sprintf("%s:%s", endpointMethod.Name, endpoint.Path)), - Rate: endpointMethod.Limit.Rate, - Per: endpointMethod.Limit.Per, - }, true - } - } - - return nil, false -} - // Map returns EndpointsMap of Endpoints using the key format [method:path]. // If duplicate entries are found, it would get overwritten with latest entries Endpoints. func (es Endpoints) Map() EndpointsMap { diff --git a/user/session_test.go b/user/session_test.go index 25ce4478db4..9195e7f1112 100644 --- a/user/session_test.go +++ b/user/session_test.go @@ -2,13 +2,10 @@ package user import ( "encoding/json" - "net/http" "reflect" "testing" "time" - "github.com/TykTechnologies/tyk/storage" - "github.com/TykTechnologies/tyk/apidef" "github.com/stretchr/testify/assert" @@ -377,150 +374,6 @@ func TestAPILimit_Clone(t *testing.T) { } } -func TestEndpoints_RateLimitInfo(t *testing.T) { - tests := []struct { - name string - method string - path string - endpoints Endpoints - expected *EndpointRateLimitInfo - found bool - }{ - { - name: "Matching endpoint and method", - method: http.MethodGet, - path: "/api/v1/users", - endpoints: Endpoints{ - { - Path: "/api/v1/users", - Methods: []EndpointMethod{ - {Name: "GET", Limit: RateLimit{Rate: 100, Per: 60}}, - }, - }, - }, - expected: &EndpointRateLimitInfo{ - KeySuffix: storage.HashStr("GET:/api/v1/users"), - Rate: 100, - Per: 60, - }, - found: true, - }, - { - name: "Matching endpoint, non-matching method", - path: "/api/v1/users", - method: http.MethodPost, - endpoints: []Endpoint{ - { - Path: "/api/v1/users", - Methods: []EndpointMethod{ - {Name: "GET", Limit: RateLimit{Rate: 100, Per: 60}}, - }, - }, - }, - expected: nil, - found: false, - }, - { - name: "Non-matching endpoint", - method: http.MethodGet, - path: "/api/v1/products", - endpoints: []Endpoint{ - { - Path: "/api/v1/users", - Methods: []EndpointMethod{ - {Name: "GET", Limit: RateLimit{Rate: 100, Per: 60}}, - }, - }, - }, - expected: nil, - found: false, - }, - { - name: "Regex path matching", - path: "/api/v1/users/123", - method: http.MethodGet, - endpoints: []Endpoint{ - { - Path: "/api/v1/users/[0-9]+", - Methods: []EndpointMethod{ - {Name: "GET", Limit: RateLimit{Rate: 50, Per: 30}}, - }, - }, - }, - expected: &EndpointRateLimitInfo{ - KeySuffix: storage.HashStr("GET:/api/v1/users/[0-9]+"), - Rate: 50, - Per: 30, - }, - found: true, - }, - { - name: "Invalid regex path", - path: "/api/v1/users", - method: http.MethodGet, - endpoints: []Endpoint{ - { - Path: "[invalid regex", - Methods: []EndpointMethod{ - {Name: "GET", Limit: RateLimit{Rate: 100, Per: 60}}, - }, - }, - }, - expected: nil, - found: false, - }, - { - name: "Invalid regex path and valid url", - path: "/api/v1/users", - method: http.MethodGet, - endpoints: []Endpoint{ - { - Path: "[invalid regex", - Methods: []EndpointMethod{ - {Name: "GET", Limit: RateLimit{Rate: 100, Per: 60}}, - }, - }, - { - Path: "/api/v1/users", - Methods: []EndpointMethod{ - {Name: "GET", Limit: RateLimit{Rate: 100, Per: 60}}, - }, - }, - }, - expected: &EndpointRateLimitInfo{ - KeySuffix: storage.HashStr("GET:/api/v1/users"), - Rate: 100, - Per: 60, - }, - found: true, - }, - { - name: "nil endpoints", - path: "/api/v1/users", - method: http.MethodGet, - endpoints: nil, - expected: nil, - found: false, - }, - { - name: "empty endpoints", - path: "/api/v1/users", - method: http.MethodGet, - endpoints: Endpoints{}, - expected: nil, - found: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result, found := tt.endpoints.RateLimitInfo(tt.method, tt.path) - assert.Equal(t, tt.found, found) - assert.Equal(t, tt.expected, result) - }) - } -} - func TestEndpoints_Map(t *testing.T) { tests := []struct { name string