From 7614b51369d2e0677bf4c568bacba597503c6f39 Mon Sep 17 00:00:00 2001 From: bubbajoe Date: Wed, 19 Jun 2024 23:05:13 +0900 Subject: [PATCH] update test to be inclusive for distributed nodes as well, service url validation, improve document loading and storing --- .github/workflows/e2e.yml | 37 +++++++-- functional-tests/admin_tests/admin_test.sh | 10 +-- .../admin_tests/iphash_load_balancer_test.sh | 6 +- .../admin_tests/merge_responses_test.sh | 2 +- .../admin_tests/modify_request_test.sh | 4 +- .../admin_tests/modify_response_test.sh | 2 +- .../admin_tests/multi_module_test.sh | 6 +- .../admin_tests/performance_test_prep.sh | 6 +- .../admin_tests/url_shortener_test.sh | 2 +- functional-tests/raft_tests/raft_test.sh | 14 ++-- internal/admin/routes/service_routes.go | 22 ++++++ internal/proxy/change_log.go | 76 ++++++++++++++----- internal/proxy/dynamic_proxy.go | 13 ++-- internal/proxy/proxystore/proxy_store.go | 17 +++++ pkg/modules/dgate/state/state_mod.go | 8 +- pkg/util/sliceutil/slice.go | 18 +++++ 16 files changed, 182 insertions(+), 61 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 55ffbc2..08f21b6 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -36,22 +36,47 @@ jobs: dgate-cli --version go install github.com/dgate-io/dgate/cmd/dgate-server dgate-server --version - - - run: go run cmd/dgate-server/main.go & - - - name: Wait for server to start - run: sleep 5 - name: Install jq run: | sudo apt install -y jq jq --version - - name: Functional Tests + - name: Install goreman run: | + go get go install github.com/mattn/goreman@latest + goreman version + + - run: go run cmd/dgate-server/main.go & + + - run: | + cd functional-tests/raft_tests + goreman start & + + - name: Wait for server to start + run: sleep 5 + + - name: Functional Standalone Tests + run: | + for i in functional-tests/admin_tests/*.sh; \ + do bash -c $i; done + + - name: Functional Distributed Tests [Node 1] + run: | + export ADMIN_URL=http://localhost:9081 + export PROXY_URL=http://localhost:81 + export TEST_URL=http://localhost:8081 for i in functional-tests/admin_tests/*.sh; \ do bash -c $i; done + - name: Functional Distributed Tests [Node 2] + run: | + export ADMIN_URL=http://localhost:9083 + export PROXY_URL=http://localhost:83 + export TEST_URL=http://localhost:8083 + for i in functional-tests/admin_tests/*.sh; \ + do bash -c $i; done + - name: Run local k6 test uses: grafana/k6-action@v0.3.1 with: diff --git a/functional-tests/admin_tests/admin_test.sh b/functional-tests/admin_tests/admin_test.sh index d038c30..1b3e91a 100755 --- a/functional-tests/admin_tests/admin_test.sh +++ b/functional-tests/admin_tests/admin_test.sh @@ -11,7 +11,7 @@ DIR="$( cd "$( dirname "$0" )" && pwd )" # domain setup # check if uuid is available if ! command -v uuid > /dev/null; then - id=x-$RANDOM-$RANDOM-$RANDOM + id=X$RANDOM-$RANDOM-$RANDOM else id=$(uuid) fi @@ -23,7 +23,7 @@ dgate-cli -Vf domain create name=dm-$id \ dgate-cli -Vf service create \ name=svc-$id namespace=ns-$id \ - urls="$TEST/$RANDOM" + urls="$TEST_URL/$RANDOM" dgate-cli -Vf module create name=module1 \ payload@=$DIR/admin_test.ts \ @@ -33,14 +33,14 @@ dgate-cli -Vf route create \ name=rt-$id \ service=svc-$id \ namespace=ns-$id \ - paths="/,/{},/$id,/$id/{id}" \ + paths="/,/{id},/$id,/$id/{id}" \ methods=GET,POST,PUT \ modules=module1 \ preserveHost:=false \ stripPath:=false -curl -f $ADMIN_URL/readyz +curl -sf $ADMIN_URL/readyz > /dev/null -curl -f ${PROXY_URL}/$id/$RANDOM-$j -H Host:$id.example.com +curl -f ${PROXY_URL}/$id/$RANDOM -H Host:$id.example.com echo "Admin Test Succeeded" diff --git a/functional-tests/admin_tests/iphash_load_balancer_test.sh b/functional-tests/admin_tests/iphash_load_balancer_test.sh index 782cd08..1c55aae 100755 --- a/functional-tests/admin_tests/iphash_load_balancer_test.sh +++ b/functional-tests/admin_tests/iphash_load_balancer_test.sh @@ -27,7 +27,7 @@ dgate-cli -Vf module create \ dgate-cli -Vf service create \ name=base_svc \ - urls:="$TEST/a","$TEST/b","$TEST/c" \ + urls:="$TEST_URL/a","$TEST_URL/b","$TEST_URL/c" \ namespace=test-lb-ns dgate-cli -Vf route create \ @@ -40,9 +40,9 @@ dgate-cli -Vf route create \ preserveHost:=true \ namespace=test-lb-ns -path1="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.example.com | jq -r '.data.path')" +path1="$(curl -sf ${PROXY_URL}/test-lb -H Host:test-lb.example.com | jq -r '.data.path')" -path2="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.example.com -H X-Forwarded-For:192.168.0.1 | jq -r '.data.path')" +path2="$(curl -sf ${PROXY_URL}/test-lb -H Host:test-lb.example.com -H X-Forwarded-For:192.168.0.1 | jq -r '.data.path')" if [ "$path1" != "$path2" ]; then echo "IP Hash Load Balancer Test Passed" diff --git a/functional-tests/admin_tests/merge_responses_test.sh b/functional-tests/admin_tests/merge_responses_test.sh index d3160ad..7774d91 100755 --- a/functional-tests/admin_tests/merge_responses_test.sh +++ b/functional-tests/admin_tests/merge_responses_test.sh @@ -32,6 +32,6 @@ dgate-cli -Vf route create \ preserveHost:=true \ namespace=test-ns -curl -s --fail-with-body ${PROXY_URL}/hello -H Host:test.example.com +curl -sf ${PROXY_URL}/hello -H Host:test.example.com echo "Merge Responses Test Passed" diff --git a/functional-tests/admin_tests/modify_request_test.sh b/functional-tests/admin_tests/modify_request_test.sh index 5117895..73853bf 100755 --- a/functional-tests/admin_tests/modify_request_test.sh +++ b/functional-tests/admin_tests/modify_request_test.sh @@ -25,7 +25,7 @@ dgate-cli -Vf module create \ dgate-cli -Vf service create \ name=base_svc \ - urls:="$TEST" \ + urls:="$TEST_URL" \ namespace=modify_request_test-ns dgate-cli -Vf route create \ @@ -38,7 +38,7 @@ dgate-cli -Vf route create \ namespace=modify_request_test-ns \ service='base_svc' -curl -s --fail-with-body ${PROXY_URL}/modify_request_test \ +curl -sf ${PROXY_URL}/modify_request_test \ -H Host:modify_request_test.example.com \ -H X-Forwarded-For:1.1.1.1 diff --git a/functional-tests/admin_tests/modify_response_test.sh b/functional-tests/admin_tests/modify_response_test.sh index bf18294..9b29d31 100755 --- a/functional-tests/admin_tests/modify_response_test.sh +++ b/functional-tests/admin_tests/modify_response_test.sh @@ -25,7 +25,7 @@ dgate-cli -Vf module create \ dgate-cli -Vf service create \ name=base_svc \ - urls:="$TEST"\ + urls:="$TEST_URL"\ namespace=test-ns dgate-cli -Vf route create \ diff --git a/functional-tests/admin_tests/multi_module_test.sh b/functional-tests/admin_tests/multi_module_test.sh index 2c43cb7..8b54c5a 100755 --- a/functional-tests/admin_tests/multi_module_test.sh +++ b/functional-tests/admin_tests/multi_module_test.sh @@ -58,7 +58,7 @@ dgate-cli -Vf module create name=multimod2 \ payload="$MOD_B64" namespace=multimod-test-ns dgate-cli -Vf service create name=base_svc \ - urls="$TEST/a","$TEST/b","$TEST/c" \ + urls="$TEST_URL/a","$TEST_URL/b","$TEST_URL/c" \ namespace=multimod-test-ns dgate-cli -Vf route create name=base_rt \ @@ -71,7 +71,7 @@ dgate-cli -Vf route create name=base_rt \ namespace=multimod-test-ns -curl -s --fail-with-body ${PROXY_URL}/ -H Host:multimod-test.example.com -curl -s --fail-with-body ${PROXY_URL}/multimod-test -H Host:multimod-test.example.com +curl -sf ${PROXY_URL}/ -H Host:multimod-test.example.com +curl -sf ${PROXY_URL}/multimod-test -H Host:multimod-test.example.com echo "Multi Module Test Passed" \ No newline at end of file diff --git a/functional-tests/admin_tests/performance_test_prep.sh b/functional-tests/admin_tests/performance_test_prep.sh index 9d8cd85..ae03733 100755 --- a/functional-tests/admin_tests/performance_test_prep.sh +++ b/functional-tests/admin_tests/performance_test_prep.sh @@ -18,7 +18,7 @@ dgate-cli -Vf domain create \ namespace=test-ns1 priority:=100 dgate-cli -Vf service create \ - name=test-svc urls:="$TEST" \ + name=test-svc urls:="$TEST_URL" \ namespace=test-ns1 retries:=3 retryTimeout=50ms MOD_B64="$(base64 < $DIR/performance_test_prep.ts)" @@ -53,9 +53,9 @@ dgate-cli -Vf route create \ namespace=test-ns1 -curl -s --fail-with-body ${PROXY_URL}/svctest -H Host:performance.example.com +curl -sf ${PROXY_URL}/svctest -H Host:performance.example.com -curl -s --fail-with-body ${PROXY_URL}/modtest -H Host:performance.example.com +curl -sf ${PROXY_URL}/modtest -H Host:performance.example.com curl -s ${PROXY_URL}/blank -H Host:performance.example.com diff --git a/functional-tests/admin_tests/url_shortener_test.sh b/functional-tests/admin_tests/url_shortener_test.sh index 7a4ecac..5e92794 100755 --- a/functional-tests/admin_tests/url_shortener_test.sh +++ b/functional-tests/admin_tests/url_shortener_test.sh @@ -39,6 +39,6 @@ JSON_RESP=$(curl -fsG -X POST \ URL_ID=$(echo $JSON_RESP | jq -r '.id') -curl -s --fail-with-body \ +curl -sf \ ${PROXY_URL}/$URL_ID \ -H Host:url_shortener.example.com diff --git a/functional-tests/raft_tests/raft_test.sh b/functional-tests/raft_tests/raft_test.sh index e163c1d..ae4f3bb 100755 --- a/functional-tests/raft_tests/raft_test.sh +++ b/functional-tests/raft_tests/raft_test.sh @@ -4,6 +4,7 @@ set -eo xtrace ADMIN_URL1=${ADMIN_URL1:-"http://localhost:9081"} PROXY_URL1=${PROXY_URL1:-"http://localhost:81"} +TEST_URL1=${TEST_URL1:-"http://localhost:8081"} ADMIN_URL2=${ADMIN_URL2:-"http://localhost:9082"} PROXY_URL2=${PROXY_URL2:-"http://localhost:82"} @@ -24,8 +25,11 @@ DIR="$( cd "$( dirname "$0" )" && pwd )" export DGATE_ADMIN_API=$ADMIN_URL1 - -id=$(uuid) +if ! command -v uuid > /dev/null; then + id=X$RANDOM-$RANDOM-$RANDOM +else + id=$(uuid) +fi dgate-cli -Vf namespace create name=ns-$id @@ -34,7 +38,7 @@ dgate-cli -Vf domain create name=dm-$id \ dgate-cli -Vf service create \ name=svc-$id namespace=ns-$id \ - urls="http://localhost:8081/$RANDOM" + urls="$TEST_URL1/$RANDOM" dgate-cli -Vf route create \ name=rt-$id \ @@ -45,12 +49,12 @@ dgate-cli -Vf route create \ preserveHost:=false \ stripPath:=false -curl -f $ADMIN_URL1/readyz +curl -sf $ADMIN_URL1/readyz for i in {1..1}; do for j in {1..3}; do proxy_url=PROXY_URL$i - curl -f ${!proxy_url}/$id/$RANDOM-$j -H Host:$id.example.com + curl -sf ${!proxy_url}/$id/$RANDOM-$j -H Host:$id.example.com done done diff --git a/internal/admin/routes/service_routes.go b/internal/admin/routes/service_routes.go index 09dba42..3a41a1d 100644 --- a/internal/admin/routes/service_routes.go +++ b/internal/admin/routes/service_routes.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "net/http" + urllib "net/url" "github.com/dgate-io/chi-router" "github.com/dgate-io/dgate/internal/admin/changestate" @@ -50,6 +51,26 @@ func ConfigureServiceAPI(server chi.Router, logger *zap.Logger, cs changestate.C util.JsonError(w, http.StatusBadRequest, "retry timeout must be greater than 0") return } + if len(svc.URLs) == 0 { + util.JsonError(w, http.StatusBadRequest, "urls are required") + return + } else { + for i, url := range svc.URLs { + errPrefix := fmt.Sprintf(`error on urls["%d"]: `, i) + if url, err := urllib.Parse(url); err != nil { + util.JsonError(w, http.StatusBadRequest, errPrefix+err.Error()) + } else { + if url.Scheme == "" { + util.JsonError(w, http.StatusBadRequest, errPrefix+"url scheme cannot be empty") + return + } + if url.Host == "" { + util.JsonError(w, http.StatusBadRequest, errPrefix+"url host cannot be empty") + return + } + } + } + } if svc.NamespaceName == "" { if appConfig.DisableDefaultNamespace { util.JsonError(w, http.StatusBadRequest, "namespace is required") @@ -57,6 +78,7 @@ func ConfigureServiceAPI(server chi.Router, logger *zap.Logger, cs changestate.C } svc.NamespaceName = spec.DefaultNamespace.Name } + cl := spec.NewChangeLog(&svc, svc.NamespaceName, spec.AddServiceCommand) err = cs.ApplyChangeLog(cl) if err != nil { diff --git a/internal/proxy/change_log.go b/internal/proxy/change_log.go index c6f4484..57d8fc8 100644 --- a/internal/proxy/change_log.go +++ b/internal/proxy/change_log.go @@ -74,9 +74,16 @@ func (ps *ProxyState) processChangeLog(cl *spec.ChangeLog, reload, store bool) ( }) } }() - if err = ps.processResource(cl); err != nil { - ps.logger.Error("decoding or processing change log", zap.Error(err)) - return + if cl.Cmd.Resource() == spec.Documents { + if err = ps.processDocument(cl.Item.(*spec.Document), cl, store); err != nil { + ps.logger.Error("error processing document change log", zap.Error(err)) + return + } + } else { + if err = ps.processResource(cl); err != nil { + ps.logger.Error("error processing change log", zap.Error(err)) + return + } } } @@ -84,7 +91,12 @@ func (ps *ProxyState) processChangeLog(cl *spec.ChangeLog, reload, store bool) ( if reload { overrideReload := cl.Cmd.IsNoop() || ps.pendingChanges if overrideReload || cl.Cmd.Resource().IsRelatedTo(spec.Routes) { - ps.logger.Debug("Reloading change log at...", zap.String("id", cl.ID)) + ps.logger.Debug("Storing cached documents", zap.String("id", cl.ID)) + if err := ps.storeCachedDocuments(); err != nil { + ps.logger.Error("error storing cached documents", zap.Error(err)) + return err + } + ps.logger.Debug("Reloading change log", zap.String("id", cl.ID)) if err = ps.reconfigureState(cl); err != nil { ps.logger.Error("Error registering change log", zap.Error(err)) return @@ -150,11 +162,6 @@ func (ps *ProxyState) processResource(cl *spec.ChangeLog) (err error) { if item, err = decode[spec.Collection](cl.Item); err == nil { err = ps.processCollection(&item, cl) } - case spec.Documents: - var item spec.Document - if item, err = decode[spec.Document](cl.Item); err == nil { - err = ps.processDocument(&item, cl) - } case spec.Secrets: var item spec.Secret if item, err = decode[spec.Secret](cl.Item); err == nil { @@ -253,17 +260,44 @@ func (ps *ProxyState) processCollection(col *spec.Collection, cl *spec.ChangeLog return err } -func (ps *ProxyState) processDocument(doc *spec.Document, cl *spec.ChangeLog) (err error) { +var docCache = []*spec.Document{} + +func (ps *ProxyState) storeCachedDocuments() error { + err := ps.store.StoreDocuments(docCache) + if err != nil { + return err + } + docCache = []*spec.Document{} + return nil +} + +func (ps *ProxyState) processDocument(doc *spec.Document, cl *spec.ChangeLog, store bool) (err error) { if doc.NamespaceName == "" { doc.NamespaceName = cl.Namespace } - switch cl.Cmd.Action() { - case spec.Add: - err = ps.store.StoreDocument(doc) - case spec.Delete: - err = ps.store.DeleteDocument(doc.ID, doc.CollectionName, doc.NamespaceName) - default: - err = fmt.Errorf("unknown command: %s", cl.Cmd) + if store { + switch cl.Cmd.Action() { + case spec.Add: + err = ps.store.StoreDocument(doc) + case spec.Delete: + err = ps.store.DeleteDocument(doc.ID, doc.CollectionName, doc.NamespaceName) + default: + err = fmt.Errorf("unknown command: %s", cl.Cmd) + } + } else { + switch cl.Cmd.Action() { + case spec.Add: + docCache = append(docCache, doc) + case spec.Delete: + deletedIndex := sliceutil.BinarySearch(docCache, doc, func(doc1 *spec.Document, doc2 *spec.Document) bool { + return doc1.ID < doc2.ID + }) + if deletedIndex >= 0 { + docCache = append(docCache[:deletedIndex], docCache[deletedIndex+1:]...) + } + default: + err = fmt.Errorf("unknown command: %s", cl.Cmd) + } } return err } @@ -291,10 +325,10 @@ func (ps *ProxyState) restoreFromChangeLogs(directApply bool) error { ps.logger.Info("restoring state change logs from storage", zap.Int("count", len(logs))) // we might need to sort the change logs by timestamp for _, cl := range logs { - // ps.logger.Debug("restoring change log", - // zap.Int("index", i), - // zap.Stringer("changeLog", cl.Cmd), - // ) + // skip documents as they are persisted in the store + if cl.Cmd.Resource() == spec.Documents { + continue + } if err = ps.processChangeLog(cl, false, false); err != nil { return err } else { diff --git a/internal/proxy/dynamic_proxy.go b/internal/proxy/dynamic_proxy.go index 7727c05..9094169 100644 --- a/internal/proxy/dynamic_proxy.go +++ b/internal/proxy/dynamic_proxy.go @@ -12,6 +12,7 @@ import ( "github.com/dgate-io/dgate/pkg/modules/extractors" "github.com/dgate-io/dgate/pkg/spec" "github.com/dgate-io/dgate/pkg/typescript" + "github.com/dgate-io/dgate/pkg/util/tree/avl" "github.com/dop251/goja" "go.uber.org/zap" "golang.org/x/net/http2" @@ -54,7 +55,7 @@ func (ps *ProxyState) setupModules(log *spec.ChangeLog) error { } else { routes = ps.rm.GetRoutesByNamespace(log.Namespace) } - programMap := make(map[string]*goja.Program) + programs := avl.NewTree[string, *goja.Program]() grp, ctx := errgroup.WithContext(context.TODO()) grp.SetLimit(16) for _, rt := range routes { @@ -90,7 +91,7 @@ func (ps *ProxyState) setupModules(log *spec.ChangeLog) error { ) return err } - programMap[mod.Name+"/"+route.Namespace.Name] = program + programs.Insert(mod.Name+"/"+route.Namespace.Name, program) return nil }) } @@ -99,10 +100,10 @@ func (ps *ProxyState) setupModules(log *spec.ChangeLog) error { if err := grp.Wait(); err != nil { return err } - - for k, v := range programMap { - ps.modPrograms.Insert(k, v) - } + programs.Each(func(s string, p *goja.Program) bool { + ps.modPrograms.Insert(s, p) + return true + }) return nil } diff --git a/internal/proxy/proxystore/proxy_store.go b/internal/proxy/proxystore/proxy_store.go index 742ceca..58c877f 100644 --- a/internal/proxy/proxystore/proxy_store.go +++ b/internal/proxy/proxystore/proxy_store.go @@ -166,6 +166,23 @@ func (store *ProxyStore) StoreDocument(doc *spec.Document) error { return nil } +func (store *ProxyStore) StoreDocuments(docs []*spec.Document) error { + for _, doc := range docs { + docBytes, err := json.Marshal(doc) + if err != nil { + return err + } + key := docKey(doc.ID, doc.CollectionName, doc.NamespaceName) + err = store.storage.Txn(true, func(txn storage.StorageTxn) error { + return txn.Set(key, docBytes) + }) + if err != nil { + return err + } + } + return nil +} + func (store *ProxyStore) DeleteDocument(id, colName, nsName string) error { return store.storage.Delete(docKey(id, colName, nsName)) } diff --git a/pkg/modules/dgate/state/state_mod.go b/pkg/modules/dgate/state/state_mod.go index 44ed32b..652fe5e 100644 --- a/pkg/modules/dgate/state/state_mod.go +++ b/pkg/modules/dgate/state/state_mod.go @@ -103,9 +103,9 @@ func (hp *ResourcesModule) getDocuments(payload FetchDocumentsPayload) (*goja.Pr } namespace := namespaceVal.(string) - docPromise, resolve, reject := rt.NewPromise() + prom, resolve, reject := rt.NewPromise() loop.RunOnLoop(func(rt *goja.Runtime) { - doc, err := state.DocumentManager(). + docs, err := state.DocumentManager(). GetDocuments( payload.Collection, namespace, @@ -116,9 +116,9 @@ func (hp *ResourcesModule) getDocuments(payload FetchDocumentsPayload) (*goja.Pr reject(rt.NewGoError(err)) return } - resolve(rt.ToValue(doc)) + resolve(rt.ToValue(docs)) }) - return docPromise, nil + return prom, nil } func writeFunc[T spec.Named](hp *ResourcesModule, cmd spec.Command) func(map[string]any) (*goja.Promise, error) { diff --git a/pkg/util/sliceutil/slice.go b/pkg/util/sliceutil/slice.go index f9c5569..4f7eae9 100644 --- a/pkg/util/sliceutil/slice.go +++ b/pkg/util/sliceutil/slice.go @@ -67,3 +67,21 @@ func SliceCopy[T any](arr []T) []T { } return append([]T(nil), arr...) } + + +// BinarySearch searches for a value in a sorted slice and returns the index of the value. +// If the value is not found, it returns -1 +func BinarySearch[T any](slice []T, val T, less func(T, T) bool) int { + low, high := 0, len(slice)-1 + for low <= high { + mid := low + (high-low)/2 + if less(slice[mid], val) { + low = mid + 1 + } else if less(val, slice[mid]) { + high = mid - 1 + } else { + return mid + } + } + return -1 +} \ No newline at end of file