diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 76a20c23403..4ee45a0091c 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -21,7 +21,9 @@ jobs: steps: - name: Checkout Tyk - uses: actions/checkout@v3 + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} - name: Setup Golang uses: actions/setup-go@v2 @@ -42,7 +44,7 @@ jobs: python -m pip install --upgrade pip pip install setuptools pip install google - pip install protobuf + pip install 'protobuf==4.24.4' - name: Start Redis uses: supercharge/redis-github-action@1.1.0 diff --git a/coprocess/python/coprocess_id_extractor_python_test.go b/coprocess/python/coprocess_id_extractor_python_test.go index 208b3b62dc1..41e896f7739 100644 --- a/coprocess/python/coprocess_id_extractor_python_test.go +++ b/coprocess/python/coprocess_id_extractor_python_test.go @@ -3,16 +3,40 @@ package python import ( "net/http" "net/url" + "strings" "testing" - "time" "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/gateway" + "github.com/TykTechnologies/tyk/internal/uuid" "github.com/TykTechnologies/tyk/test" ) -var pythonIDExtractorHeaderValue = map[string]string{ - "manifest.json": ` +var pythonIDExtractorHeaderValue = func(token string) map[string]string { + middleware := ` +import time +from tyk.decorators import * +from gateway import TykGateway as tyk + +counter = 0 + +@Hook +def MyAuthHook(request, session, metadata, spec): + global counter + counter = counter + 1 + auth_header = request.get_header('Authorization') + if auth_header == 'valid_token' and counter < 2: + session.rate = 1000.0 + session.per = 1.0 + session.id_extractor_deadline = int(time.time()) + 60 + metadata["token"] = "valid_token" + return request, session, metadata + ` + + middleware = strings.ReplaceAll(middleware, "valid_token", token) + + return map[string]string{ + "manifest.json": ` { "file_list": [ "middleware.py" @@ -32,10 +56,16 @@ var pythonIDExtractorHeaderValue = map[string]string{ } } `, - "middleware.py": ` + "middleware.py": middleware, + } +} + +var pythonIDExtractorFormValue = func(token string) map[string]string { + middleware := ` import time from tyk.decorators import * from gateway import TykGateway as tyk +from urllib import parse counter = 0 @@ -43,18 +73,19 @@ counter = 0 def MyAuthHook(request, session, metadata, spec): global counter counter = counter + 1 - auth_header = request.get_header('Authorization') - if auth_header == 'valid_token' and counter < 2: + auth_param = parse.parse_qs(request.object.body)["auth"] + if auth_param and auth_param[0] == 'valid_token' and counter < 2: session.rate = 1000.0 session.per = 1.0 session.id_extractor_deadline = int(time.time()) + 60 metadata["token"] = "valid_token" return request, session, metadata - `, -} +` -var pythonIDExtractorFormValue = map[string]string{ - "manifest.json": ` + middleware = strings.ReplaceAll(middleware, "valid_token", token) + + return map[string]string{ + "manifest.json": ` { "file_list": [ "middleware.py" @@ -74,30 +105,36 @@ var pythonIDExtractorFormValue = map[string]string{ } } `, - "middleware.py": ` + "middleware.py": middleware, + } +} + +var pythonIDExtractorHeaderRegex = func(token string) map[string]string { + middleware := ` import time from tyk.decorators import * from gateway import TykGateway as tyk -from urllib import parse counter = 0 @Hook def MyAuthHook(request, session, metadata, spec): + print("MyAuthHook3 is called") global counter counter = counter + 1 - auth_param = parse.parse_qs(request.object.body)["auth"] - if auth_param and auth_param[0] == 'valid_token' and counter < 2: + _, auth_header = request.get_header('Authorization').split('-') + if auth_header and auth_header == '12345' and counter < 2: session.rate = 1000.0 session.per = 1.0 session.id_extractor_deadline = int(time.time()) + 60 metadata["token"] = "valid_token" return request, session, metadata -`, -} + ` -var pythonIDExtractorHeaderRegex = map[string]string{ - "manifest.json": ` + middleware = strings.ReplaceAll(middleware, "valid_token", token) + + return map[string]string{ + "manifest.json": ` { "file_list": [ "middleware.py" @@ -119,26 +156,8 @@ var pythonIDExtractorHeaderRegex = map[string]string{ } } `, - "middleware.py": ` -import time -from tyk.decorators import * -from gateway import TykGateway as tyk - -counter = 0 - -@Hook -def MyAuthHook(request, session, metadata, spec): - print("MyAuthHook3 is called") - global counter - counter = counter + 1 - _, auth_header = request.get_header('Authorization').split('-') - if auth_header and auth_header == '12345' and counter < 2: - session.rate = 1000.0 - session.per = 1.0 - session.id_extractor_deadline = int(time.time()) + 60 - metadata["token"] = "valid_token" - return request, session, metadata - `, + "middleware.py": middleware, + } } /* Value Extractor tests, using "header" source */ @@ -146,59 +165,79 @@ def MyAuthHook(request, session, metadata, spec): // Our `pythonBundleWithAuthCheck` plugin restrict more then 1 call // With ID extractor, it should run multiple times (because cache) func TestValueExtractorHeaderSource(t *testing.T) { - test.Flaky(t) - ts := gateway.StartTest(nil, gateway.TestConfig{ - CoprocessConfig: config.CoProcessConfig{ - EnableCoProcess: true, - PythonPathPrefix: pkgPath, - PythonVersion: "3.5", - }, - Delay: 10 * time.Millisecond, - }) - defer ts.Close() - - spec := gateway.BuildAPI(func(spec *gateway.APISpec) { - spec.Proxy.ListenPath = "/" - spec.UseKeylessAccess = false - spec.EnableCoProcessAuth = true - })[0] t.Run("Header value", func(t *testing.T) { - bundleID := ts.RegisterBundle("id_extractor_header_value", pythonIDExtractorHeaderValue) - spec.CustomMiddlewareBundle = bundleID - spec.APIID = "api1" - - ts.Gw.LoadAPI(spec) - time.Sleep(1 * time.Second) + ts := gateway.StartTest(nil, gateway.TestConfig{ + CoprocessConfig: config.CoProcessConfig{ + EnableCoProcess: true, + PythonPathPrefix: pkgPath, + }, + }) + defer ts.Close() + + validToken := uuid.NewHex() + bundleID := ts.RegisterBundle("id_extractor_header_value", pythonIDExtractorHeaderValue(validToken)) + + ts.Gw.BuildAndLoadAPI(func(spec *gateway.APISpec) { + spec.Proxy.ListenPath = "/" + spec.UseKeylessAccess = false + spec.EnableCoProcessAuth = true + spec.CustomMiddlewareBundle = bundleID + spec.APIID = "api1" + }) ts.Run(t, []test.TestCase{ - {Path: "/", Headers: map[string]string{"Authorization": "valid_token"}, Code: http.StatusOK}, - {Path: "/", Headers: map[string]string{"Authorization": "valid_token"}, Code: http.StatusOK}, + {Path: "/", Headers: map[string]string{"Authorization": validToken}, Code: http.StatusOK}, + {Path: "/", Headers: map[string]string{"Authorization": validToken}, Code: http.StatusOK}, {Path: "/", Headers: map[string]string{"Authorization": "invalid_token"}, Code: http.StatusForbidden}, }...) }) t.Run("Form value", func(t *testing.T) { - bundleID := ts.RegisterBundle("id_extractor_form_value", pythonIDExtractorFormValue) - spec.CustomMiddlewareBundle = bundleID - spec.APIID = "api2" - - ts.Gw.LoadAPI(spec) - time.Sleep(1 * time.Second) + ts := gateway.StartTest(nil, gateway.TestConfig{ + CoprocessConfig: config.CoProcessConfig{ + EnableCoProcess: true, + PythonPathPrefix: pkgPath, + }, + }) + defer ts.Close() + + validToken := uuid.NewHex() + bundleID := ts.RegisterBundle("id_extractor_form_value", pythonIDExtractorFormValue(validToken)) + + ts.Gw.BuildAndLoadAPI(func(spec *gateway.APISpec) { + spec.Proxy.ListenPath = "/" + spec.UseKeylessAccess = false + spec.EnableCoProcessAuth = true + spec.CustomMiddlewareBundle = bundleID + spec.APIID = "api2" + }) formHeaders := map[string]string{"Content-Type": "application/x-www-form-urlencoded"} ts.Run(t, []test.TestCase{ - {Method: "POST", Path: "/", Headers: formHeaders, Data: url.Values{"auth": []string{"valid_token"}}.Encode(), Code: http.StatusOK}, - {Method: "POST", Path: "/", Headers: formHeaders, Data: url.Values{"auth": []string{"valid_token"}}.Encode(), Code: http.StatusOK}, + {Method: "POST", Path: "/", Headers: formHeaders, Data: url.Values{"auth": []string{validToken}}.Encode(), Code: http.StatusOK}, + {Method: "POST", Path: "/", Headers: formHeaders, Data: url.Values{"auth": []string{validToken}}.Encode(), Code: http.StatusOK}, {Method: "POST", Path: "/", Headers: formHeaders, Data: url.Values{"auth": []string{"invalid_token"}}.Encode(), Code: http.StatusForbidden}, }...) }) t.Run("Header regex", func(t *testing.T) { - bundleID := ts.RegisterBundle("id_extractor_header_regex", pythonIDExtractorHeaderRegex) - spec.CustomMiddlewareBundle = bundleID - spec.APIID = "api3" - - ts.Gw.LoadAPI(spec) - time.Sleep(1 * time.Second) + ts := gateway.StartTest(nil, gateway.TestConfig{ + CoprocessConfig: config.CoProcessConfig{ + EnableCoProcess: true, + PythonPathPrefix: pkgPath, + }, + }) + defer ts.Close() + + validToken := uuid.NewHex() + bundleID := ts.RegisterBundle("id_extractor_header_regex", pythonIDExtractorHeaderRegex(validToken)) + + ts.Gw.BuildAndLoadAPI(func(spec *gateway.APISpec) { + spec.Proxy.ListenPath = "/" + spec.UseKeylessAccess = false + spec.EnableCoProcessAuth = true + spec.CustomMiddlewareBundle = bundleID + spec.APIID = "api3" + }) ts.Run(t, []test.TestCase{ {Path: "/", Headers: map[string]string{"Authorization": "prefix-12345"}, Code: http.StatusOK}, diff --git a/coprocess/python/coprocess_python_test.go b/coprocess/python/coprocess_python_test.go index 2740d063fea..d1778b38e68 100644 --- a/coprocess/python/coprocess_python_test.go +++ b/coprocess/python/coprocess_python_test.go @@ -8,13 +8,13 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" - "time" "github.com/TykTechnologies/tyk/apidef" - "github.com/TykTechnologies/tyk/config" "github.com/TykTechnologies/tyk/gateway" + "github.com/TykTechnologies/tyk/internal/uuid" "github.com/TykTechnologies/tyk/test" "github.com/TykTechnologies/tyk/user" ) @@ -26,21 +26,8 @@ func init() { pkgPath = filepath.Dir(filename) + "./../../" } -var pythonBundleWithAuthCheck = map[string]string{ - "manifest.json": ` - { - "file_list": [ - "middleware.py" - ], - "custom_middleware": { - "driver": "python", - "auth_check": { - "name": "MyAuthHook" - } - } - } -`, - "middleware.py": ` +var pythonBundleWithAuthCheck = func(token1, token2 string) map[string]string { + middleware := ` from tyk.decorators import * from gateway import TykGateway as tyk @@ -54,12 +41,32 @@ def MyAuthHook(request, session, metadata, spec): session.quota_max = 1 session.quota_renewal_rate = 60 metadata["token"] = "valid_token" - if auth_header == '47a0c79c427728b3df4af62b9228c8ae11': + if auth_header == 'token_without_quota': policy_id = request.get_header('Policy') session.apply_policy_id = policy_id - metadata["token"] = "47a0c79c427728b3df4af62b9228c8ae11" + metadata["token"] = "token_without_quota" return request, session, metadata +` + + middleware = strings.ReplaceAll(middleware, "valid_token", token1) + middleware = strings.ReplaceAll(middleware, "token_without_quota", token2) + + return map[string]string{ + "manifest.json": ` + { + "file_list": [ + "middleware.py" + ], + "custom_middleware": { + "driver": "python", + "auth_check": { + "name": "MyAuthHook" + } + } + } `, + "middleware.py": middleware, + } } var pythonBundleWithPostHook = map[string]string{ @@ -213,21 +220,26 @@ func TestMain(m *testing.M) { os.Exit(gateway.InitTestMain(context.Background(), m)) } -func TestPythonBundles(t *testing.T) { +func setupGateway() *gateway.Test { ts := gateway.StartTest(nil, gateway.TestConfig{ CoprocessConfig: config.CoProcessConfig{ EnableCoProcess: true, PythonPathPrefix: pkgPath, }}) - defer ts.Close() - authCheckBundle := ts.RegisterBundle("python_with_auth_check", pythonBundleWithAuthCheck) - postHookBundle := ts.RegisterBundle("python_with_post_hook", pythonBundleWithPostHook) - preHookBundle := ts.RegisterBundle("python_with_pre_hook", pythonBundleWithPreHook) - responseHookBundle := ts.RegisterBundle("python_with_response_hook", pythonBundleWithResponseHook) - postRequestTransformHookBundle := ts.RegisterBundle("python_post_with_request_transform_hook", pythonPostRequestTransform) + return ts +} + +func TestPythonBundles(t *testing.T) { + unused := uuid.NewHex() t.Run("Single-file bundle with authentication hook", func(t *testing.T) { + ts := setupGateway() + defer ts.Close() + + validToken := uuid.NewHex() + authCheckBundle := ts.RegisterBundle("python_with_auth_check", pythonBundleWithAuthCheck(validToken, unused)) + ts.Gw.BuildAndLoadAPI(func(spec *gateway.APISpec) { spec.Proxy.ListenPath = "/test-api/" spec.UseKeylessAccess = false @@ -236,9 +248,7 @@ func TestPythonBundles(t *testing.T) { spec.VersionData.NotVersioned = true }) - time.Sleep(1 * time.Second) - - validAuth := map[string]string{"Authorization": "valid_token"} + validAuth := map[string]string{"Authorization": validToken} invalidAuth := map[string]string{"Authorization": "invalid_token"} ts.Run(t, []test.TestCase{ @@ -249,6 +259,12 @@ func TestPythonBundles(t *testing.T) { }) t.Run("Auth with policy", func(t *testing.T) { + ts := setupGateway() + defer ts.Close() + + validToken := uuid.NewHex() + authCheckBundle := ts.RegisterBundle("python_with_auth_check", pythonBundleWithAuthCheck(unused, validToken)) + specs := ts.Gw.BuildAndLoadAPI(func(spec *gateway.APISpec) { spec.Auth.AuthHeaderName = "Authorization" spec.Proxy.ListenPath = "/test-api/" @@ -258,8 +274,6 @@ func TestPythonBundles(t *testing.T) { spec.VersionData.NotVersioned = true }) - time.Sleep(1 * time.Second) - pID := ts.CreatePolicy(func(p *user.Policy) { p.QuotaMax = 1 p.QuotaRenewalRate = 60 @@ -267,9 +281,11 @@ func TestPythonBundles(t *testing.T) { APIID: specs[0].APIID, Versions: []string{"Default"}, }} + + t.Log(p.ID) }) - policyAuth := map[string]string{"authorization": "47a0c79c427728b3df4af62b9228c8ae11", "policy": pID} + policyAuth := map[string]string{"authorization": validToken, "policy": pID} ts.Run(t, []test.TestCase{ {Path: "/test-api/", Code: http.StatusOK, Headers: policyAuth}, @@ -278,6 +294,10 @@ func TestPythonBundles(t *testing.T) { }) t.Run("Single-file bundle with post hook", func(t *testing.T) { + ts := setupGateway() + defer ts.Close() + + postHookBundle := ts.RegisterBundle("python_with_post_hook", pythonBundleWithPostHook) keyID := gateway.CreateSession(ts.Gw, func(s *user.SessionState) { s.MetaData = map[string]interface{}{ @@ -294,8 +314,6 @@ func TestPythonBundles(t *testing.T) { spec.VersionData.NotVersioned = true }) - time.Sleep(1 * time.Second) - auth := map[string]string{"Authorization": keyID} ts.Run(t, []test.TestCase{ @@ -304,6 +322,10 @@ func TestPythonBundles(t *testing.T) { }) t.Run("Single-file bundle with response hook", func(t *testing.T) { + ts := setupGateway() + defer ts.Close() + + responseHookBundle := ts.RegisterBundle("python_with_response_hook", pythonBundleWithResponseHook) keyID := gateway.CreateSession(ts.Gw, func(s *user.SessionState) { s.MetaData = map[string]interface{}{ @@ -320,8 +342,6 @@ func TestPythonBundles(t *testing.T) { spec.VersionData.NotVersioned = true }) - time.Sleep(1 * time.Second) - auth := map[string]string{"Authorization": keyID} ts.Run(t, []test.TestCase{ @@ -330,6 +350,11 @@ func TestPythonBundles(t *testing.T) { }) t.Run("Single-file bundle with pre hook and UTF-8/non-UTF-8 request data", func(t *testing.T) { + ts := setupGateway() + defer ts.Close() + + preHookBundle := ts.RegisterBundle("python_with_pre_hook", pythonBundleWithPreHook) + ts.Gw.BuildAndLoadAPI(func(spec *gateway.APISpec) { spec.Proxy.ListenPath = "/test-api-2/" spec.UseKeylessAccess = true @@ -338,8 +363,6 @@ func TestPythonBundles(t *testing.T) { spec.VersionData.NotVersioned = true }) - time.Sleep(1 * time.Second) - fileData := gateway.GenerateTestBinaryData() var buf bytes.Buffer multipartWriter := multipart.NewWriter(&buf) @@ -371,6 +394,11 @@ func TestPythonBundles(t *testing.T) { }) t.Run("python post hook with url rewrite and method transform", func(t *testing.T) { + ts := setupGateway() + defer ts.Close() + + postRequestTransformHookBundle := ts.RegisterBundle("python_post_with_request_transform_hook", pythonPostRequestTransform) + ts.Gw.BuildAndLoadAPI(func(spec *gateway.APISpec) { spec.Proxy.ListenPath = "/test-api-1/" spec.UseKeylessAccess = true @@ -402,8 +430,6 @@ func TestPythonBundles(t *testing.T) { spec.UseKeylessAccess = true }) - time.Sleep(1 * time.Second) - ts.Run(t, []test.TestCase{ {Path: "/test-api-1/get", Code: http.StatusOK, BodyMatch: "newpath"}, {Path: "/test-api-1/get", Code: http.StatusOK, BodyMatch: "GET"},