improve extractor promise wait timeout

add prevotereq to rafthttp optimize promise waiting change example domains so they work on raft instances add admin test
dgate-io · Jun 17, 2024 · e7d03bf · e7d03bf
1 parent da7e866
commit e7d03bf
Show file tree

Hide file tree

Showing 50 changed files with 934 additions and 822 deletions.
diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@ DGate is a distributed API Gateway built for developers. DGate allows you to use
 
 ## Getting Started
 
-Coming soon @ http://dgate.io/docs/getting-started
+http://dgate.io/docs/getting-started
 
 ### Installing
 

diff --git a/TODO.md b/TODO.md
@@ -8,6 +8,12 @@
   - cluster management (raft commands, replica commands, etc.) (low priority)
   - other commands (backup, restore, etc.) (low priority)
 
+# Raft Snapshots
+
+- Add support for Raft snapshots to reduce the size of the Raft log. This can be used to reduce the size of the Raft log and improve the performance of the cluster.
+  - [ ] - Snapshot documents
+  - [ ] - Snapshot resources (with correlactions)
+
 ## Add Module Tests
 
 - Test multiple modules being used at the same time
@@ -138,13 +144,12 @@ Make it easier to debug modules by adding more logging and error handling. This
 
 Add stack tracing for typescript modules.
 
-
-## Decouple Admin API from Raft Implementation
-
-Currently, Raft Implementation is tightly coupled with the Admin API. This makes it difficult to change the Raft Implementation without changing the Admin API. Decouple the Raft Implementation from the Admin API to make it easier to change the Raft Implementation.
-
 ## Add Telemetry (sentry, datadog, etc.)
 
 ## ResourceManager callback for resource changes
 
-Add a callback to the ResourceManager that is called when a resource is changed. This can be used to invalidate caches, update modules, and more.
+Add a callback to the ResourceManager that is called when a resource is changed. This can be used to invalidate caches, update modules, and more.
+
+## Enable WAF
+
+https://github.com/corazawaf/coraza
diff --git a/config.dgate.yaml b/config.dgate.yaml
@@ -1,9 +1,8 @@
 version: v1
 debug: true
-log_level: ${LOG_LEVEL:-info}
+log_level: ${LOG_LEVEL:-debug}
 disable_default_namespace: true
-tags:
-  - debug
+tags: [debug, local, test]
 storage:
   type: file
   dir: .dgate/data/
@@ -15,7 +14,7 @@ test_server:
 proxy:
   port: ${PORT:-80}
   host: 0.0.0.0
-  enable_console_logger: true
+  console_log_level: info
   transport:
     dns_prefer_go: true
   init_resources:

diff --git a/functional-tests/admin_tests/admin_test.sh b/functional-tests/admin_tests/admin_test.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -eo xtrace
+
+ADMIN_URL=${ADMIN_URL:-"http://localhost:9080"}
+PROXY_URL=${PROXY_URL:-"http://localhost"}
+
+DIR="$( cd "$( dirname "$0" )" && pwd )"
+
+# domain setup
+
+id=$(uuid)
+
+dgate-cli namespace create name=ns-$id
+
+dgate-cli domain create name=dm-$id \
+    namespace=ns-$id priority:=$RANDOM patterns="$id.example.com"
+
+dgate-cli service create \
+    name=svc-$id namespace=ns-$id \
+    urls="http://localhost:8888/$RANDOM"
+
+dgate-cli module create name=module1 \
+    payload@=$DIR/admin_test.ts \
+    namespace=ns-$id
+
+dgate-cli route create \
+    name=rt-$id \
+    service=svc-$id \
+    namespace=ns-$id \
+    paths="/,/{},/$id,/$id/{id}" \
+    methods=GET,POST,PUT \
+    modules=module1 \
+    preserveHost:=false \
+    stripPath:=false
+
+curl -f $ADMIN_URL/readyz
+
+curl -f ${PROXY_URL}/$id/$RANDOM-$j -H Host:$id.example.com
+
+echo "Admin Test Succeeded"
diff --git a/functional-tests/admin_tests/admin_test.ts b/functional-tests/admin_tests/admin_test.ts
@@ -0,0 +1,4 @@
+
+export const responseModifier = async (ctx: any) => {
+    console.log("responseModifier -> path params", ctx.pathParams());
+}
diff --git a/functional-tests/admin_tests/change_checker.sh b/functional-tests/admin_tests/change_checker.sh
@@ -14,7 +14,7 @@ dgate-cli namespace create \
 
 dgate-cli domain create \
     name=change_checker-dm \
-    patterns:='["change_checker.com"]' \
+    patterns:='["change_checker.example.com"]' \
     namespace=change_checker-ns
 
 dgate-cli module create name=change_checker-mod \
@@ -29,7 +29,7 @@ dgate-cli route create \
     preserveHost:=true \
     namespace=change_checker-ns
 
-MODID1=$(curl -sG -H Host:change_checker.com ${PROXY_URL}/ | jq -r '.mod')
+MODID1=$(curl -sG -H Host:change_checker.example.com ${PROXY_URL}/ | jq -r '.mod')
 
 if [ "$MODID1" != "module1" ]; then
     echo "Initial assert failed"
@@ -43,7 +43,7 @@ dgate-cli module create name=change_checker-mod \
 
 # dgate-cli r.ker-ns
 
-MODID2=$(curl -sG -H Host:change_checker.com ${PROXY_URL}/ | jq -r '.mod')
+MODID2=$(curl -sG -H Host:change_checker.example.com ${PROXY_URL}/ | jq -r '.mod')
 
 if [ "$MODID2" != "module2" ]; then
     echo "module update failed"

diff --git a/functional-tests/admin_tests/iphash_load_balancer_test.sh b/functional-tests/admin_tests/iphash_load_balancer_test.sh
@@ -14,7 +14,7 @@ dgate-cli namespace create \
 
 dgate-cli domain create \
     name=test-lb-dm \
-    patterns:='["test-lb.com"]' \
+    patterns:='["test-lb.example.com"]' \
     namespace=test-lb-ns
 
 MOD_B64="$(base64 < $DIR/iphash_load_balancer.ts)"
@@ -39,9 +39,9 @@ dgate-cli route create \
     preserveHost:=true \
     namespace=test-lb-ns
 
-path1="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.com | jq -r '.data.path')"
+path1="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.example.com | jq -r '.data.path')"
 
-path2="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.com -H X-Forwarded-For:192.168.0.1 | jq -r '.data.path')"
+path2="$(curl -s --fail-with-body ${PROXY_URL}/test-lb -H Host:test-lb.example.com -H X-Forwarded-For:192.168.0.1 | jq -r '.data.path')"
 
 if [ "$path1" != "$path2" ]; then
     echo "IP Hash Load Balancer Test Passed"

diff --git a/functional-tests/admin_tests/merge_responses_test.sh b/functional-tests/admin_tests/merge_responses_test.sh
@@ -14,7 +14,7 @@ dgate-cli namespace create \
 
 dgate-cli domain create \
     name=test-dm \
-    patterns:='["test.com"]' \
+    patterns:='["test.example.com"]' \
     namespace=test-ns
 
 MOD_B64="$(base64 < $DIR/merge_responses.ts)"
@@ -32,6 +32,6 @@ dgate-cli route create \
     preserveHost:=true \
     namespace=test-ns
 
-curl -s --fail-with-body ${PROXY_URL}/hello -H Host:test.com
+curl -s --fail-with-body ${PROXY_URL}/hello -H Host:test.example.com
 
 echo "Merge Responses Test Passed"
diff --git a/functional-tests/admin_tests/modify_request_test.sh b/functional-tests/admin_tests/modify_request_test.sh
@@ -14,7 +14,7 @@ dgate-cli namespace create \
 
 dgate-cli domain create \
     name=modify_request_test-dm \
-    patterns:='["modify_request_test.com"]' \
+    patterns:='["modify_request_test.example.com"]' \
     namespace=modify_request_test-ns
 
 MOD_B64="$(base64 < $DIR/modify_request.ts)"
@@ -38,7 +38,7 @@ dgate-cli route create \
     service='base_svc'
 
 curl -s --fail-with-body ${PROXY_URL}/modify_request_test \
-    -H Host:modify_request_test.com \
+    -H Host:modify_request_test.example.com \
     -H X-Forwarded-For:1.1.1.1
 
 echo "Modify Request Test Passed"
diff --git a/functional-tests/admin_tests/modify_response_test.sh b/functional-tests/admin_tests/modify_response_test.sh
@@ -14,7 +14,7 @@ dgate-cli namespace create \
 
 dgate-cli domain create \
     name=test-dm \
-    patterns:='["test.com"]' \
+    patterns:='["test.example.com"]' \
     namespace=test-ns
 
 MOD_B64="$(base64 < $DIR/modify_response.ts)"
@@ -37,6 +37,6 @@ dgate-cli route create \
     namespace=test-ns \
     service='base_svc'
 
-curl -s ${PROXY_URL}/test -H Host:test.com
+curl -s ${PROXY_URL}/test -H Host:test.example.com
 
 echo "Modify Response Test Passed"
diff --git a/functional-tests/admin_tests/multi_module_test.sh b/functional-tests/admin_tests/multi_module_test.sh
@@ -14,7 +14,7 @@ dgate-cli namespace create \
 
 dgate-cli domain create \
     name=multimod-test-dm \
-    patterns:='["multimod-test.com"]' \
+    patterns:='["multimod-test.example.com"]' \
     namespace=multimod-test-ns
 
 MOD_B64=$(base64 <<-END
@@ -71,7 +71,7 @@ dgate-cli route create name=base_rt \
     namespace=multimod-test-ns
 
 
-curl -s --fail-with-body ${PROXY_URL}/ -H Host:multimod-test.com
-curl -s --fail-with-body ${PROXY_URL}/multimod-test -H Host:multimod-test.com
+curl -s --fail-with-body ${PROXY_URL}/ -H Host:multimod-test.example.com
+curl -s --fail-with-body ${PROXY_URL}/multimod-test -H Host:multimod-test.example.com
 
 echo "Multi Module Test Passed"
diff --git a/functional-tests/admin_tests/performance_test_prep.sh b/functional-tests/admin_tests/performance_test_prep.sh
@@ -14,7 +14,7 @@ dgate-cli -V -f namespace create \
     name=test-ns1
 
 dgate-cli domain create \
-    name=test-dm patterns:='["dgate.dev"]' \
+    name=test-dm patterns:='["performance.example.com"]' \
     namespace=test-ns1 priority:=100
 
 dgate-cli service create \
@@ -53,10 +53,10 @@ dgate-cli route create \
     namespace=test-ns1
 
 
-curl -s --fail-with-body ${PROXY_URL}/svctest -H Host:dgate.dev
+curl -s --fail-with-body ${PROXY_URL}/svctest -H Host:performance.example.com
 
-curl -s --fail-with-body ${PROXY_URL}/modtest -H Host:dgate.dev
+curl -s --fail-with-body ${PROXY_URL}/modtest -H Host:performance.example.com
 
-curl -s ${PROXY_URL}/blank -H Host:dgate.dev
+curl -s ${PROXY_URL}/blank -H Host:performance.example.com
 
 echo "Performance Test Prep Done"
diff --git a/functional-tests/admin_tests/url_shortener_test.sh b/functional-tests/admin_tests/url_shortener_test.sh
@@ -14,14 +14,12 @@ dgate-cli namespace create \
 
 dgate-cli domain create \
     name=url_shortener-dm \
-    patterns:='["url_shortener.com"]' \
+    patterns:='["url_shortener.example.com"]' \
     namespace=url_shortener-ns
 
 dgate-cli collection create \
     schema:='{"type":"object","properties":{"url":{"type":"string"}}}' \
-    name=short_link \
-    type=document \
-    namespace=url_shortener-ns
+    name=short_link type=document namespace=url_shortener-ns
 
 dgate-cli module create name=url_shortener-mod \
     payload@=$DIR/url_shortener.ts \
@@ -36,11 +34,11 @@ dgate-cli route create \
     namespace=url_shortener-ns
 
 JSON_RESP=$(curl -sG -X POST \
-    -H Host:url_shortener.com ${PROXY_URL}/ \
-    --data-urlencode 'url=https://dgate.io')
+    -H Host:url_shortener.example.com ${PROXY_URL}/ \
+    --data-urlencode 'url=https://dgate.io/'$(uuid))
 
 URL_ID=$(echo $JSON_RESP | jq -r '.id')
 
 curl -s --fail-with-body \
     ${PROXY_URL}/$URL_ID \
-    -H Host:url_shortener.com
+    -H Host:url_shortener.example.com
diff --git a/functional-tests/raft_tests/raft_test.sh b/functional-tests/raft_tests/raft_test.sh
@@ -40,30 +40,30 @@ dgate-cli -f route create \
     name=rt-$id \
     service=svc-$id \
     namespace=ns-$id \
-    paths="/$id/{id}" \
-    methods:='["GET"]' \
+    paths="/,/{},/$id,/$id/{id}" \
+    methods=GET,POST,PUT \
     preserveHost:=false \
-    stripPath:=true
+    stripPath:=false
 
 curl -f $ADMIN_URL1/readyz
 
-for i in {1..5}; do
+for i in {1..1}; do
     for j in {1..3}; do
         proxy_url=PROXY_URL$i
-        curl -f ${!proxy_url}/$id/$j -H Host:$id.example.com
+        curl -f ${!proxy_url}/$id/$RANDOM-$j -H Host:$id.example.com
     done
 done
 
-if dgate-cli --admin $ADMIN_URL4 namespace create name=0; then
-    echo "Expected error when creating namespace on non-voter"
-    exit 1
-fi
+# if dgate-cli --admin $ADMIN_URL4 namespace create name=0; then
+#     echo "Expected error when creating namespace on non-voter"
+#     exit 1
+# fi
 
-export DGATE_ADMIN_API=$ADMIN_URL5
+# export DGATE_ADMIN_API=$ADMIN_URL5
 
-if dgate-cli --admin $ADMIN_URL5 namespace create name=0; then
-    echo "Expected error when creating namespace on non-voter"
-    exit 1
-fi
+# if dgate-cli --admin $ADMIN_URL5 namespace create name=0; then
+#     echo "Expected error when creating namespace on non-voter"
+#     exit 1
+# fi
 
 echo "Raft Test Succeeded"
diff --git a/go.mod b/go.mod
@@ -5,13 +5,13 @@ go 1.22.0
 require (
 	github.com/clarkmcc/go-typescript v0.7.0
 	github.com/dgate-io/chi-router v0.0.0-20231217131951-d154152d5115
-	github.com/dgate-io/raft-badger v0.0.0-20231217131807-c5eb3f9eafa5
 	github.com/dgraph-io/badger/v4 v4.2.0
 	github.com/dop251/goja v0.0.0-20240220182346-e401ed450204
 	github.com/dop251/goja_nodejs v0.0.0-20231122114759-e84d9a924c5c
 	github.com/google/uuid v1.3.1
-	github.com/hashicorp/go-hclog v1.6.2
-	github.com/hashicorp/raft v1.6.0
+	github.com/hashicorp/go-hclog v1.6.3
+	github.com/hashicorp/raft v1.7.0
+	github.com/hashicorp/raft-boltdb/v2 v2.3.0
 	github.com/knadh/koanf/parsers/json v0.1.0
 	github.com/knadh/koanf/parsers/toml v0.1.0
 	github.com/knadh/koanf/parsers/yaml v0.1.0
@@ -38,14 +38,16 @@ require (
 require (
 	github.com/armon/go-metrics v0.4.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
+	github.com/dgryski/go-farm v0.0.0-20191112170834-c2139c5d712b // indirect
 	github.com/dlclark/regexp2 v1.10.0 // indirect
 	github.com/dop251/base64dec v0.0.0-20231022112746-c6c9f9a96217 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
-	github.com/fatih/color v1.14.1 // indirect
+	github.com/fatih/color v1.17.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -58,12 +60,14 @@ require (
 	github.com/google/flatbuffers v1.12.1 // indirect
 	github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
-	github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/hashicorp/go-msgpack v1.1.5 // indirect
+	github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
+	github.com/hashicorp/golang-lru v1.0.2 // indirect
+	github.com/hashicorp/raft-boltdb v0.0.0-20231211162105-6c830fa4535e // indirect
 	github.com/klauspost/compress v1.17.0 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/pelletier/go-toml v1.9.5 // indirect
@@ -76,11 +80,13 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
+	go.etcd.io/bbolt v1.3.10 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.26.0 // indirect
 	go.opentelemetry.io/otel/trace v1.26.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/sys v0.19.0 // indirect
+	golang.org/x/sync v0.7.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect