Skip to content

Commit

Permalink
[Tests] Fix scenario where nethermind stops before capturing extra lo…
Browse files Browse the repository at this point in the history
…gs - convert script to Python (#7718)
  • Loading branch information
kamilchodola authored Nov 5, 2024
1 parent 6409179 commit 001e424
Show file tree
Hide file tree
Showing 2 changed files with 71 additions and 74 deletions.
82 changes: 8 additions & 74 deletions .github/workflows/sync-supported-chains.yml
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,11 @@ jobs:
check-latest: true
cache: true

- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.X"

- name: Install Sedge environment
run: |
echo "Downloading sedge sources..."
Expand Down Expand Up @@ -187,81 +192,10 @@ jobs:
- name: Wait for ${{ matrix.config.network }} to sync
id: wait
env:
NETWORK: ${{ matrix.config.network }}
run: |
declare -A bad_logs
declare -A good_logs
declare -A required_count
bad_logs["Exception"]=1
bad_logs["Missing node found!"]=1
good_logs["Processed"]=0
good_logs["Stats after finishing state"]=0
required_count["Processed"]=20
required_count["Stats after finishing state"]=1
network="${{ matrix.config.network }}"
if [[ "$network" != "joc-mainnet" && "$network" != "joc-testnet" && "$network" != "linea-mainnet" && "$network" != "linea-sepolia" ]]; then
good_logs["Synced Chain Head"]=0
required_count["Synced Chain Head"]=20
fi
counter=0
found_bad_log=false
if [[ "$network" == base-* || "$network" == op-* ]]; then
container_name="sedge-execution-op-l2-client"
else
container_name="sedge-execution-client"
fi
docker logs -f "$container_name" | while read -r line; do
echo "$line"
if [[ "$line" == *"All done"* ]]; then
echo "Unexpected termination detected: $line"
exit 1
fi
if [ "$found_bad_log" = true ]; then
counter=$((counter + 1))
if [ $counter -ge 100 ]; then
echo "Exiting after capturing extra logs due to error."
exit 1
else
continue
fi
fi

for bad_log in "${!bad_logs[@]}"; do
if [[ "$line" == *"$bad_log"* ]]; then
echo "Error: $bad_log found in Docker logs."
found_bad_log=true
break
fi
done

for good_log in "${!good_logs[@]}"; do
if [[ "$line" == *"$good_log"* ]]; then
good_logs["$good_log"]=$((good_logs["$good_log"]+1))
fi
done

# Check if all good logs have reached the required count
all_reached_required_count=true
for good_log in "${!good_logs[@]}"; do
if [[ ${good_logs[$good_log]} -lt ${required_count[$good_log]} ]]; then
all_reached_required_count=false
break
fi
done

if $all_reached_required_count; then
echo "All required logs found."
exit 0
fi
done
python scripts/waitForSync.py
- name: Get Consensus Logs
if: always() && matrix.config.network != 'joc-mainnet' && matrix.config.network != 'joc-testnet' && matrix.config.network != 'linea-mainnet' && matrix.config.network != 'linea-sepolia'
Expand Down
63 changes: 63 additions & 0 deletions scripts/waitForSync.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
import os
import subprocess
import sys

network = os.getenv("NETWORK")

bad_logs = {"Exception": 1, "Missing node found!": 1}
good_logs = {"Processed": 0, "Stats after finishing state": 0}
required_count = {"Processed": 20, "Stats after finishing state": 1}

if network not in {"joc-mainnet", "joc-testnet", "linea-mainnet", "linea-sepolia"}:
good_logs["Synced Chain Head"] = 0
required_count["Synced Chain Head"] = 20

container_mapping = {
"base-": "sedge-execution-op-l2-client",
"op-": "sedge-execution-op-l2-client",
}
default_container_name = "sedge-execution-client"

container_name = next(
(name for prefix, name in container_mapping.items() if network.startswith(prefix)),
default_container_name
)

process = subprocess.Popen(["docker", "logs", "-f", container_name], stdout=subprocess.PIPE, text=True)

found_bad_log = False
counter = 0

try:
for line in process.stdout:
print(line.strip())

if found_bad_log:
counter += 1
if counter >= 100:
print("Exiting after capturing extra logs due to error.")
sys.exit(1)
continue

if any(bad_log in line for bad_log in bad_logs):
print(f"Error: Found bad log in line: {line.strip()}")
found_bad_log = True
continue

for good_log in good_logs:
if good_log in line:
good_logs[good_log] += 1

if all(good_logs[log] >= required_count[log] for log in required_count):
print("All required logs found.")
sys.exit(0)

except Exception as e:
print(f"An error occurred: {e}")
sys.exit(1)
finally:
process.terminate()

# Final exit if we did not reach required lines after an error
print("Unhandled termination. Probably critical issue in client. Stopping...")
sys.exit(1)

0 comments on commit 001e424

Please sign in to comment.