diff --git a/.github/workflows/Update.yml b/.github/workflows/Update.yml index 650c1c31..1799602c 100644 --- a/.github/workflows/Update.yml +++ b/.github/workflows/Update.yml @@ -47,3 +47,43 @@ jobs: with: commit_message: "[BOT] [OCI] changes - ${{ steps.oracle-extract-timestamp.outputs.oracle_utc }}" push_options: '--force' + + - name: Run Azure Script + run: | + chmod +x ./microsoft-azure/start.sh + ./microsoft-azure/start.sh + + - name: Extract Timestamp from File (Azure) + id: azure-extract-timestamp + run: echo "azure_utc=$(<./microsoft-azure/timestamp.txt)" >> "$GITHUB_OUTPUT" + + - name: Git Auto Commit (Azure) + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "[BOT] [Azure] changes - ${{ steps.azure-extract-timestamp.outputs.azure_utc }}" + push_options: '--force' + + - name: Run Github Script + run: | + chmod +x ./github/start.sh + ./github/start.sh + + - name: Extract Timestamp from File (Github) + id: github-extract-timestamp + run: echo "github_utc=$(<./github/timestamp.txt)" >> "$GITHUB_OUTPUT" + + - name: Git Auto Commit (Github) + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "[BOT] [Github] changes - ${{ steps.github-extract-timestamp.outputs.github_utc }}" + push_options: '--force' + + - name: Extract Timestamp from File (Cloudflare) + id: cloudflare-extract-timestamp + run: echo "cloudflare_utc=$(<./cloudflare/timestamp.txt)" >> "$GITHUB_OUTPUT" + + - name: Git Auto Commit (Cloudflare) + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: "[BOT] [Cloudflare] changes - ${{ steps.cloudflare-extract-timestamp.outputs.cloudflare_utc }}" + push_options: '--force' diff --git a/amazon/start.sh b/amazon/start.sh new file mode 100755 index 00000000..bf70f57a --- /dev/null +++ b/amazon/start.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# https://docs.aws.amazon.com/general/latest/gr/aws-ip-ranges.html + +set -euo pipefail +set -x + +# Define file paths +json_file="/tmp/amazon.json" +timestamp_file="amazon/timestamp.txt" +ipv4_file="/tmp/amazon-ipv4.txt" +ipv6_file="/tmp/amazon-ipv6.txt" +ipv4_output="amazon/ipv4.txt" +ipv6_output="amazon/ipv6.txt" + +# get from public ranges +if curl -s https://ip-ranges.amazonaws.com/ip-ranges.json > "$json_file"; then + echo "AWS IP ranges fetched successfully." +else + echo "Error: Failed to fetch AWS IP ranges." >&2 + exit 1 +fi + +# Extract "createDate" value +create_date=$(jq -r '.createDate' "$json_file") + +# Save "createDate" value to file +echo "$create_date" > "$timestamp_file" + +# save ipv4 +if jq '.prefixes[] | [.ip_prefix][] | select(. != null)' -r "$json_file" > "$ipv4_file"; then + echo "AWS IPv4 addresses saved successfully." +else + echo "Error: Failed to save AWS IPv4 addresses." >&2 + exit 1 +fi + +# save ipv6 +if jq '.ipv6_prefixes[] | [.ipv6_prefix][] | select(. != null)' -r "$json_file" > "$ipv6_file"; then + echo "AWS IPv6 addresses saved successfully." +else + echo "Error: Failed to save AWS IPv6 addresses." >&2 + exit 1 +fi + +# sort & uniq +if sort -V "$ipv4_file" | uniq > "$ipv4_output"; then + echo "AWS IPv4 addresses sorted and duplicates removed successfully." +else + echo "Error: Failed to sort AWS IPv4 addresses." >&2 + exit 1 +fi + +if sort -V "$ipv6_file" | uniq > "$ipv6_output"; then + echo "AWS IPv6 addresses sorted and duplicates removed successfully." +else + echo "Error: Failed to sort AWS IPv6 addresses." >&2 + exit 1 +fi diff --git a/cloudflare/start.sh b/cloudflare/start.sh new file mode 100755 index 00000000..02c5df9b --- /dev/null +++ b/cloudflare/start.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# https://www.cloudflare.com/ips/ + +# Define file paths +timestamp_file="cloudflare/timestamp.txt" + +ipv4_file="/tmp/cf-ipv4.txt" +ipv4_output="cloudflare/ipv4.txt" +ipv4_comma_output="cloudflare/ipv4_comma.txt" + +ipv6_file="/tmp/cf-ipv6.txt" +ipv6_output="cloudflare/ipv6.txt" +ipv6_comma_output="cloudflare/ipv6_comma.txt" + +# Check if the timestamp_file exists and remove it if it does +if [ -e "$timestamp_file" ]; then + rm "$timestamp_file" + echo "Step 0: File $timestamp_file removed successfully." +else + echo "Step 0: File $timestamp_file does not exist. Skip." +fi + +# get from public ranges +if curl -s https://www.cloudflare.com/ips-v4/ > "$ipv4_file"; then + echo "IPv4 addresses fetched successfully." +else + echo "Error: Failed to fetch IPv4 addresses." >&2 + exit 1 +fi + +if curl -s https://www.cloudflare.com/ips-v6/ > "$ipv6_file"; then + echo "IPv6 addresses fetched successfully." +else + echo "Error: Failed to fetch IPv6 addresses." >&2 + exit 1 +fi + +# sort & uniq +if sort -V "$ipv4_file" | uniq > "$ipv4_output"; then + echo "IPv4 addresses sorted and duplicates removed successfully." +else + echo "Error: Failed to sort IPv4 addresses." >&2 + exit 1 +fi + +if sort -V "$ipv6_file" | uniq > "$ipv6_output"; then + echo "IPv6 addresses sorted and duplicates removed successfully." +else + echo "Error: Failed to sort IPv6 addresses." >&2 + exit 1 +fi + +# Save IPv4 addresses with comma separation +if paste -sd "," "$ipv4_output" > "$ipv4_comma_output"; then + echo "IPv4 addresses saved with comma separation successfully." +else + echo "Error: Failed to save IPv4 addresses with comma separation." >&2 + exit 1 +fi + +# Save IPv6 addresses with comma separation +if paste -sd "," "$ipv6_output" > "$ipv6_comma_output"; then + echo "IPv6 addresses saved with comma separation successfully." +else + echo "Error: Failed to save IPv6 addresses with comma separation." >&2 + exit 1 +fi + +# Step 5: Save timestamp to the timestamp file +timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S.000000Z") +echo "$timestamp" > "$timestamp_file" +echo "Step 5: Timestamp saved successfully." + +echo "Cloudflare Complete!" \ No newline at end of file diff --git a/github/start.sh b/github/start.sh new file mode 100755 index 00000000..ae5f977d --- /dev/null +++ b/github/start.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -euo pipefail +set -x + +# Define file paths +json_file="/tmp/github.json" +timestamp_file="github/timestamp.txt" + +github_all_txt="/tmp/github-all.txt" + +github_ipv4_txt="github/ipv4.txt" +github_ipv6_txt="github/ipv6.txt" + +# Check if the timestamp_file exists and remove it if it does +if [ -e "$timestamp_file" ]; then + rm "$timestamp_file" + echo "Step 0: File $timestamp_file removed successfully." +else + echo "Step 0: File $timestamp_file does not exist. Skip." +fi + +# Step 1: Fetch data from GitHub API with error handling +if curl -s https://api.github.com/meta > "$json_file"; then + echo "Step 1: Data fetched from GitHub API successfully." +else + echo "Step 1: Unable to fetch data from GitHub API." >&2 + exit 1 +fi + +# Step 2: Extract data without specified keys and flatten the structure +filtered_data=$(jq ' + with_entries( + select( + .key | . != "ssh_keys" and + . != "verifiable_password_authentication" and + . != "ssh_key_fingerprints" and + . != "domains" + ) + ) | .[] | .[] +' -r "$json_file") + +# Save the filtered data to the github_all_txt file +echo "$filtered_data" > "$github_all_txt" + +# Step 3: Save IPv4 addresses +if grep -v ':' "$github_all_txt" | sort -V | uniq > "$github_ipv4_txt"; then + echo "Step 3: IPv4 addresses saved successfully." +else + echo "Step 3: Failed to save IPv4 addresses." >&2 + exit 1 +fi + +# Step 4: Save IPv6 addresses +if grep ':' "$github_all_txt" | sort -V | uniq > "$github_ipv6_txt"; then + echo "Step 4: IPv6 addresses saved successfully." +else + echo "Step 3: Failed to save IPv6 addresses." >&2 + exit 1 +fi + +# Step 5: Save timestamp to the timestamp file +timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S.000000Z") +echo "$timestamp" > "$timestamp_file" +echo "Step 5: Timestamp saved successfully." + +echo "Github Complete!" \ No newline at end of file diff --git a/googlebot/ipv4.txt b/googlebot/ipv4.txt deleted file mode 100644 index 5b04c75b..00000000 --- a/googlebot/ipv4.txt +++ /dev/null @@ -1,128 +0,0 @@ -34.22.85.0/27 -34.64.82.64/28 -34.65.242.112/28 -34.80.50.80/28 -34.88.194.0/28 -34.89.10.80/28 -34.89.198.80/28 -34.96.162.48/28 -34.100.182.96/28 -34.101.50.144/28 -34.118.66.0/28 -34.118.254.0/28 -34.126.178.96/28 -34.146.150.144/28 -34.147.110.144/28 -34.151.74.144/28 -34.152.50.64/28 -34.154.114.144/28 -34.155.98.32/28 -34.165.18.176/28 -34.175.160.64/28 -34.176.130.16/28 -35.247.243.240/28 -66.249.64.0/27 -66.249.64.32/27 -66.249.64.64/27 -66.249.64.96/27 -66.249.64.128/27 -66.249.64.160/27 -66.249.64.192/27 -66.249.64.224/27 -66.249.65.0/27 -66.249.65.32/27 -66.249.65.64/27 -66.249.65.96/27 -66.249.65.160/27 -66.249.65.192/27 -66.249.65.224/27 -66.249.66.0/27 -66.249.66.32/27 -66.249.66.64/27 -66.249.66.96/27 -66.249.66.128/27 -66.249.66.160/27 -66.249.66.192/27 -66.249.68.0/27 -66.249.68.32/27 -66.249.68.64/27 -66.249.69.0/27 -66.249.69.32/27 -66.249.69.64/27 -66.249.69.96/27 -66.249.69.128/27 -66.249.69.160/27 -66.249.69.192/27 -66.249.69.224/27 -66.249.70.0/27 -66.249.70.32/27 -66.249.70.64/27 -66.249.70.96/27 -66.249.70.128/27 -66.249.70.160/27 -66.249.70.192/27 -66.249.70.224/27 -66.249.71.0/27 -66.249.71.32/27 -66.249.71.64/27 -66.249.71.96/27 -66.249.71.128/27 -66.249.71.160/27 -66.249.71.192/27 -66.249.71.224/27 -66.249.72.0/27 -66.249.72.32/27 -66.249.72.64/27 -66.249.72.96/27 -66.249.72.128/27 -66.249.72.160/27 -66.249.72.192/27 -66.249.72.224/27 -66.249.73.0/27 -66.249.73.32/27 -66.249.73.64/27 -66.249.73.96/27 -66.249.73.128/27 -66.249.73.160/27 -66.249.73.192/27 -66.249.73.224/27 -66.249.74.0/27 -66.249.74.32/27 -66.249.74.64/27 -66.249.74.96/27 -66.249.74.128/27 -66.249.75.0/27 -66.249.75.32/27 -66.249.75.64/27 -66.249.75.96/27 -66.249.75.128/27 -66.249.75.160/27 -66.249.75.192/27 -66.249.75.224/27 -66.249.76.0/27 -66.249.76.32/27 -66.249.76.64/27 -66.249.76.96/27 -66.249.76.128/27 -66.249.76.160/27 -66.249.76.192/27 -66.249.76.224/27 -66.249.77.0/27 -66.249.77.32/27 -66.249.77.64/27 -66.249.77.96/27 -66.249.77.128/27 -66.249.77.160/27 -66.249.77.192/27 -66.249.77.224/27 -66.249.78.0/27 -66.249.78.32/27 -66.249.79.0/27 -66.249.79.32/27 -66.249.79.64/27 -66.249.79.96/27 -66.249.79.128/27 -66.249.79.160/27 -66.249.79.192/27 -66.249.79.224/27 -192.178.5.0/27 diff --git a/googlebot/ipv4_comma.txt b/googlebot/ipv4_comma.txt deleted file mode 100644 index 4a52e72d..00000000 --- a/googlebot/ipv4_comma.txt +++ /dev/null @@ -1 +0,0 @@ -34.22.85.0/27,34.64.82.64/28,34.65.242.112/28,34.80.50.80/28,34.88.194.0/28,34.89.10.80/28,34.89.198.80/28,34.96.162.48/28,34.100.182.96/28,34.101.50.144/28,34.118.66.0/28,34.118.254.0/28,34.126.178.96/28,34.146.150.144/28,34.147.110.144/28,34.151.74.144/28,34.152.50.64/28,34.154.114.144/28,34.155.98.32/28,34.165.18.176/28,34.175.160.64/28,34.176.130.16/28,35.247.243.240/28,66.249.64.0/27,66.249.64.32/27,66.249.64.64/27,66.249.64.96/27,66.249.64.128/27,66.249.64.160/27,66.249.64.192/27,66.249.64.224/27,66.249.65.0/27,66.249.65.32/27,66.249.65.64/27,66.249.65.96/27,66.249.65.160/27,66.249.65.192/27,66.249.65.224/27,66.249.66.0/27,66.249.66.32/27,66.249.66.64/27,66.249.66.96/27,66.249.66.128/27,66.249.66.160/27,66.249.66.192/27,66.249.68.0/27,66.249.68.32/27,66.249.68.64/27,66.249.69.0/27,66.249.69.32/27,66.249.69.64/27,66.249.69.96/27,66.249.69.128/27,66.249.69.160/27,66.249.69.192/27,66.249.69.224/27,66.249.70.0/27,66.249.70.32/27,66.249.70.64/27,66.249.70.96/27,66.249.70.128/27,66.249.70.160/27,66.249.70.192/27,66.249.70.224/27,66.249.71.0/27,66.249.71.32/27,66.249.71.64/27,66.249.71.96/27,66.249.71.128/27,66.249.71.160/27,66.249.71.192/27,66.249.71.224/27,66.249.72.0/27,66.249.72.32/27,66.249.72.64/27,66.249.72.96/27,66.249.72.128/27,66.249.72.160/27,66.249.72.192/27,66.249.72.224/27,66.249.73.0/27,66.249.73.32/27,66.249.73.64/27,66.249.73.96/27,66.249.73.128/27,66.249.73.160/27,66.249.73.192/27,66.249.73.224/27,66.249.74.0/27,66.249.74.32/27,66.249.74.64/27,66.249.74.96/27,66.249.74.128/27,66.249.75.0/27,66.249.75.32/27,66.249.75.64/27,66.249.75.96/27,66.249.75.128/27,66.249.75.160/27,66.249.75.192/27,66.249.75.224/27,66.249.76.0/27,66.249.76.32/27,66.249.76.64/27,66.249.76.96/27,66.249.76.128/27,66.249.76.160/27,66.249.76.192/27,66.249.76.224/27,66.249.77.0/27,66.249.77.32/27,66.249.77.64/27,66.249.77.96/27,66.249.77.128/27,66.249.77.160/27,66.249.77.192/27,66.249.77.224/27,66.249.78.0/27,66.249.78.32/27,66.249.79.0/27,66.249.79.32/27,66.249.79.64/27,66.249.79.96/27,66.249.79.128/27,66.249.79.160/27,66.249.79.192/27,66.249.79.224/27,192.178.5.0/27 diff --git a/googlebot/ipv6.txt b/googlebot/ipv6.txt deleted file mode 100644 index 7683e3ad..00000000 --- a/googlebot/ipv6.txt +++ /dev/null @@ -1,105 +0,0 @@ -2001:4860:4801:1a::/64 -2001:4860:4801:1b::/64 -2001:4860:4801:1c::/64 -2001:4860:4801:1d::/64 -2001:4860:4801:1e::/64 -2001:4860:4801:2a::/64 -2001:4860:4801:2b::/64 -2001:4860:4801:2c::/64 -2001:4860:4801:2d::/64 -2001:4860:4801:2e::/64 -2001:4860:4801:2f::/64 -2001:4860:4801:2::/64 -2001:4860:4801:3a::/64 -2001:4860:4801:3b::/64 -2001:4860:4801:3c::/64 -2001:4860:4801:3d::/64 -2001:4860:4801:3e::/64 -2001:4860:4801:3::/64 -2001:4860:4801:4a::/64 -2001:4860:4801:6a::/64 -2001:4860:4801:6b::/64 -2001:4860:4801:6c::/64 -2001:4860:4801:6d::/64 -2001:4860:4801:6e::/64 -2001:4860:4801:6f::/64 -2001:4860:4801:10::/64 -2001:4860:4801:11::/64 -2001:4860:4801:12::/64 -2001:4860:4801:13::/64 -2001:4860:4801:14::/64 -2001:4860:4801:15::/64 -2001:4860:4801:16::/64 -2001:4860:4801:17::/64 -2001:4860:4801:18::/64 -2001:4860:4801:19::/64 -2001:4860:4801:20::/64 -2001:4860:4801:21::/64 -2001:4860:4801:22::/64 -2001:4860:4801:23::/64 -2001:4860:4801:24::/64 -2001:4860:4801:25::/64 -2001:4860:4801:26::/64 -2001:4860:4801:27::/64 -2001:4860:4801:28::/64 -2001:4860:4801:29::/64 -2001:4860:4801:30::/64 -2001:4860:4801:31::/64 -2001:4860:4801:32::/64 -2001:4860:4801:33::/64 -2001:4860:4801:34::/64 -2001:4860:4801:35::/64 -2001:4860:4801:36::/64 -2001:4860:4801:37::/64 -2001:4860:4801:38::/64 -2001:4860:4801:39::/64 -2001:4860:4801:40::/64 -2001:4860:4801:41::/64 -2001:4860:4801:42::/64 -2001:4860:4801:43::/64 -2001:4860:4801:44::/64 -2001:4860:4801:45::/64 -2001:4860:4801:46::/64 -2001:4860:4801:47::/64 -2001:4860:4801:48::/64 -2001:4860:4801:49::/64 -2001:4860:4801:50::/64 -2001:4860:4801:51::/64 -2001:4860:4801:53::/64 -2001:4860:4801:54::/64 -2001:4860:4801:55::/64 -2001:4860:4801:60::/64 -2001:4860:4801:61::/64 -2001:4860:4801:62::/64 -2001:4860:4801:63::/64 -2001:4860:4801:64::/64 -2001:4860:4801:65::/64 -2001:4860:4801:66::/64 -2001:4860:4801:67::/64 -2001:4860:4801:68::/64 -2001:4860:4801:69::/64 -2001:4860:4801:70::/64 -2001:4860:4801:71::/64 -2001:4860:4801:72::/64 -2001:4860:4801:73::/64 -2001:4860:4801:74::/64 -2001:4860:4801:75::/64 -2001:4860:4801:76::/64 -2001:4860:4801:77::/64 -2001:4860:4801:78::/64 -2001:4860:4801:79::/64 -2001:4860:4801:80::/64 -2001:4860:4801:81::/64 -2001:4860:4801:82::/64 -2001:4860:4801:83::/64 -2001:4860:4801:84::/64 -2001:4860:4801:85::/64 -2001:4860:4801:86::/64 -2001:4860:4801:87::/64 -2001:4860:4801:88::/64 -2001:4860:4801:90::/64 -2001:4860:4801:91::/64 -2001:4860:4801:92::/64 -2001:4860:4801:93::/64 -2001:4860:4801:c::/64 -2001:4860:4801:f::/64 diff --git a/googlebot/ipv6_comma.txt b/googlebot/ipv6_comma.txt deleted file mode 100644 index 23c9be8e..00000000 --- a/googlebot/ipv6_comma.txt +++ /dev/null @@ -1 +0,0 @@ -2001:4860:4801:1a::/64,2001:4860:4801:1b::/64,2001:4860:4801:1c::/64,2001:4860:4801:1d::/64,2001:4860:4801:1e::/64,2001:4860:4801:2a::/64,2001:4860:4801:2b::/64,2001:4860:4801:2c::/64,2001:4860:4801:2d::/64,2001:4860:4801:2e::/64,2001:4860:4801:2f::/64,2001:4860:4801:2::/64,2001:4860:4801:3a::/64,2001:4860:4801:3b::/64,2001:4860:4801:3c::/64,2001:4860:4801:3d::/64,2001:4860:4801:3e::/64,2001:4860:4801:3::/64,2001:4860:4801:4a::/64,2001:4860:4801:6a::/64,2001:4860:4801:6b::/64,2001:4860:4801:6c::/64,2001:4860:4801:6d::/64,2001:4860:4801:6e::/64,2001:4860:4801:6f::/64,2001:4860:4801:10::/64,2001:4860:4801:11::/64,2001:4860:4801:12::/64,2001:4860:4801:13::/64,2001:4860:4801:14::/64,2001:4860:4801:15::/64,2001:4860:4801:16::/64,2001:4860:4801:17::/64,2001:4860:4801:18::/64,2001:4860:4801:19::/64,2001:4860:4801:20::/64,2001:4860:4801:21::/64,2001:4860:4801:22::/64,2001:4860:4801:23::/64,2001:4860:4801:24::/64,2001:4860:4801:25::/64,2001:4860:4801:26::/64,2001:4860:4801:27::/64,2001:4860:4801:28::/64,2001:4860:4801:29::/64,2001:4860:4801:30::/64,2001:4860:4801:31::/64,2001:4860:4801:32::/64,2001:4860:4801:33::/64,2001:4860:4801:34::/64,2001:4860:4801:35::/64,2001:4860:4801:36::/64,2001:4860:4801:37::/64,2001:4860:4801:38::/64,2001:4860:4801:39::/64,2001:4860:4801:40::/64,2001:4860:4801:41::/64,2001:4860:4801:42::/64,2001:4860:4801:43::/64,2001:4860:4801:44::/64,2001:4860:4801:45::/64,2001:4860:4801:46::/64,2001:4860:4801:47::/64,2001:4860:4801:48::/64,2001:4860:4801:49::/64,2001:4860:4801:50::/64,2001:4860:4801:51::/64,2001:4860:4801:53::/64,2001:4860:4801:54::/64,2001:4860:4801:55::/64,2001:4860:4801:60::/64,2001:4860:4801:61::/64,2001:4860:4801:62::/64,2001:4860:4801:63::/64,2001:4860:4801:64::/64,2001:4860:4801:65::/64,2001:4860:4801:66::/64,2001:4860:4801:67::/64,2001:4860:4801:68::/64,2001:4860:4801:69::/64,2001:4860:4801:70::/64,2001:4860:4801:71::/64,2001:4860:4801:72::/64,2001:4860:4801:73::/64,2001:4860:4801:74::/64,2001:4860:4801:75::/64,2001:4860:4801:76::/64,2001:4860:4801:77::/64,2001:4860:4801:78::/64,2001:4860:4801:79::/64,2001:4860:4801:80::/64,2001:4860:4801:81::/64,2001:4860:4801:82::/64,2001:4860:4801:83::/64,2001:4860:4801:84::/64,2001:4860:4801:85::/64,2001:4860:4801:86::/64,2001:4860:4801:87::/64,2001:4860:4801:88::/64,2001:4860:4801:90::/64,2001:4860:4801:91::/64,2001:4860:4801:92::/64,2001:4860:4801:93::/64,2001:4860:4801:c::/64,2001:4860:4801:f::/64 diff --git a/googlebot/start.sh b/googlebot/start.sh index 0bec5417..896815fc 100755 --- a/googlebot/start.sh +++ b/googlebot/start.sh @@ -1,5 +1,8 @@ #!/bin/bash +set -euo pipefail +set -x + # Define file paths json_file="/tmp/googlebot.json" timestamp_file="googlebot/timestamp.txt" @@ -12,17 +15,19 @@ ipv6_file="/tmp/googlebot-ipv6.txt" ipv6_output="googlebot/ipv6.txt" ipv6_comma_output="googlebot/ipv6_comma.txt" -# Check if timestamp file exists and remove it +# Check if the timestamp_file exists and remove it if it does if [ -e "$timestamp_file" ]; then rm "$timestamp_file" + echo "Step 0: File $timestamp_file removed successfully." +else + echo "Step 0: File $timestamp_file does not exist. Skip." fi # Download public GoogleBot IP ranges -curl -s https://developers.google.com/search/apis/ipranges/googlebot.json > "$json_file" - -# Check if the curl command was successful -if [ $? -ne 0 ]; then - echo "Error: Failed to download GoogleBot IP ranges using curl." +if curl -s https://developers.google.com/search/apis/ipranges/googlebot.json > "$json_file"; then + echo "Step 1: Data fetched from GoogleBot IP ranges using curl successfully." +else + echo "Step 1: Failed to download ORACLE IP ranges using curl." >&2 exit 1 fi @@ -33,59 +38,55 @@ googlebot_utc=$(date -d "$timestamp" -u +"%Y-%m-%dT%H:%M:%S.000000Z") # Save timestamp to file echo "$googlebot_utc" > "$timestamp_file" -# Save IPv4 addresses -jq '.prefixes[] | [.ipv4Prefix][] | select(. != null)' -r "$json_file" > "$ipv4_file" - -# Check if the jq command was successful -if [ $? -ne 0 ]; then - echo "Error: Failed to extract IPv4 addresses using jq." +# Save IPv4 addresses and check if the jq command was successful +if jq '.prefixes[] | [.ipv4Prefix][] | select(. != null)' -r "$json_file" > "$ipv4_file"; then + echo "Step 2: IPv4 addresses extracted successfully." +else + echo "Step 2: Error: Failed to extract IPv4 addresses using jq." exit 1 fi -# Save IPv6 addresses -jq '.prefixes[] | [.ipv6Prefix][] | select(. != null)' -r "$json_file" > "$ipv6_file" - -# Check if the jq command was successful -if [ $? -ne 0 ]; then - echo "Error: Failed to extract IPv6 addresses using jq." +# Save IPv6 addresses and check if the jq command was successful +if jq '.prefixes[] | [.ipv6Prefix][] | select(. != null)' -r "$json_file" > "$ipv6_file"; then + echo "Step 3: IPv6 addresses extracted successfully." +else + echo "Step 3: Error: Failed to extract IPv6 addresses using jq." exit 1 fi -# Sort and remove duplicates for IPv4 -sort -V "$ipv4_file" | uniq > "$ipv4_output" - -# Check if the sort command was successful -if [ $? -ne 0 ]; then - echo "Error: Failed to sort IPv4 addresses." +# Sort and remove duplicates for IPv4 and check if the sort command was successful +if sort -V "$ipv4_file" | uniq > "$ipv4_output"; then + echo "Step 4: IPv4 addresses sorted and duplicates removed successfully." +else + echo "Step 4: Error: Failed to sort IPv4 addresses." exit 1 fi -# Sort and remove duplicates for IPv6 -sort -V "$ipv6_file" | uniq > "$ipv6_output" - -# Check if the sort command was successful -if [ $? -ne 0 ]; then - echo "Error: Failed to sort IPv6 addresses." +# Sort and remove duplicates for IPv6 and check if the sort command was successful +if sort -V "$ipv6_file" | uniq > "$ipv6_output"; then + echo "Step 5: IPv6 addresses sorted and duplicates removed successfully." +else + echo "Step 5: Error: Failed to sort IPv6 addresses." exit 1 fi -# Save IPv4 addresses with comma separation -jq -r '.prefixes[] | [.ipv4Prefix][] | select(. != null)' "$json_file" | sort -V | uniq | paste -sd "," - > "$ipv4_comma_output" - -# Check if the jq and sort commands were successful -if [ $? -ne 0 ]; then - echo "Error: Failed to process IPv4 addresses for comma separation." +# Save IPv4 addresses with comma separation and check if the jq and sort commands were successful +if jq -r '.prefixes[] | [.ipv4Prefix][] | select(. != null)' "$json_file" | sort -V | uniq | paste -sd "," - > "$ipv4_comma_output"; then + echo "Step 6: IPv4 addresses processed for comma separation successfully." +else + echo "Step 6: Failed to process IPv4 addresses for comma separation." exit 1 fi -# Save IPv6 addresses with comma separation -jq -r '.prefixes[] | [.ipv6Prefix][] | select(. != null)' "$json_file" | sort -V | uniq | paste -sd "," - > "$ipv6_comma_output" - -# Check if the jq and sort commands were successful -if [ $? -ne 0 ]; then - echo "Error: Failed to process IPv6 addresses for comma separation." +# Save IPv6 addresses with comma separation and check if the jq and sort commands were successful +if jq -r '.prefixes[] | [.ipv6Prefix][] | select(. != null)' "$json_file" | sort -V | uniq | paste -sd "," - > "$ipv6_comma_output"; then + echo "Step 7: IPv6 addresses processed for comma separation successfully." +else + echo "Step 7: Error: Failed to process IPv6 addresses for comma separation." exit 1 fi # Clean up temporary files rm "$json_file" "$ipv4_file" "$ipv6_file" + +echo "Googlebot Complete!" \ No newline at end of file diff --git a/googlebot/timestamp.txt b/googlebot/timestamp.txt deleted file mode 100644 index ca2ffd6d..00000000 --- a/googlebot/timestamp.txt +++ /dev/null @@ -1 +0,0 @@ -2024-01-30T23:00:31.000000Z diff --git a/microsoft-azure/start.sh b/microsoft-azure/start.sh new file mode 100755 index 00000000..6ee9182e --- /dev/null +++ b/microsoft-azure/start.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +# https://azure.microsoft.com/en-us/updates/service-tag-discovery-api-in-preview/ +# https://docs.microsoft.com/en-us/microsoft-365/enterprise/urls-and-ip-address-ranges?view=o365-worldwide +# From: https://github.com/jensihnow/AzurePublicIPAddressRanges/blob/main/.github/workflows/main.yml + +set -euo pipefail +set -x + +timestamp_file="microsoft-azure/timestamp.txt" + +# Check if the timestamp_file exists and remove it if it does +if [ -e "$timestamp_file" ]; then + rm "$timestamp_file" + echo "Step 0: File $timestamp_file removed successfully." +else + echo "Step 0: File $timestamp_file does not exist. Skip." +fi + +# Function to download and parse Microsoft IP ranges for a given region +download_and_parse() { + local REGION_ID="$1" + local REGION_NAME="$2" + local OUTPUT_DIR="microsoft-azure/${REGION_NAME}" + + # Create output directory if not exists + mkdir -p "${OUTPUT_DIR}" + + # Download Microsoft IP ranges JSON + URL="$(curl -s https://www.microsoft.com/en-us/download/confirmation.aspx?id="${REGION_ID}" | grep -o ' "/tmp/microsoft_${REGION_NAME}.json" + + # Parse and save IPv4 addresses + jq '.values[] | [.properties] | .[].addressPrefixes[] | select(. != null)' -r "/tmp/microsoft_${REGION_NAME}.json" | grep -v ':' | sort -V | uniq > "${OUTPUT_DIR}/ipv4.txt" + + # Parse and save IPv6 addresses + jq '.values[] | [.properties] | .[].addressPrefixes[] | select(. != null)' -r "/tmp/microsoft_${REGION_NAME}.json" | grep ':' | sort -V | uniq > "${OUTPUT_DIR}/ipv6.txt" + + # Create comma-separated files + paste -sd ',' "${OUTPUT_DIR}/ipv4.txt" > "${OUTPUT_DIR}/ipv4_comma.txt" + paste -sd ',' "${OUTPUT_DIR}/ipv6.txt" > "${OUTPUT_DIR}/ipv6_comma.txt" + + # Save timestamp to the directory + echo "${TIMESTAMP}" > "${OUTPUT_DIR}/timestamp.txt" +} + +# Function to create consolidated files for all regions +consolidate_files() { + local OUTPUT_DIR="microsoft-azure" + + # Concatenate all IPv4 and IPv6 addresses + cat "${OUTPUT_DIR}"/*/ipv4.txt > "${OUTPUT_DIR}/all_ipv4.txt" + cat "${OUTPUT_DIR}"/*/ipv6.txt > "${OUTPUT_DIR}/all_ipv6.txt" + + # Concatenate all comma-separated files + cat "${OUTPUT_DIR}"/*/ipv4_comma.txt > "${OUTPUT_DIR}/all_ipv4_comma.txt" + cat "${OUTPUT_DIR}"/*/ipv6_comma.txt > "${OUTPUT_DIR}/all_ipv6_comma.txt" +} + +# Download and parse IP ranges for each region +download_and_parse "56519" "public-cloud" +download_and_parse "57063" "us-gov" +download_and_parse "57064" "germany" +download_and_parse "57062" "china" + +# Consolidate files for all regions +consolidate_files + +timestamp=$(date -u +"%Y-%m-%dT%H:%M:%S.000000Z") +echo "$timestamp" > "$timestamp_file" +echo "Timestamp saved successfully." + +echo "Azure Complete!" \ No newline at end of file diff --git a/oracle/ipv4.txt b/oracle/ipv4.txt deleted file mode 100644 index 269d6477..00000000 --- a/oracle/ipv4.txt +++ /dev/null @@ -1,673 +0,0 @@ -38.104.155.92/31 -40.233.0.0/19 -40.233.64.0/18 -62.115.179.220/31 -62.115.179.228/31 -64.110.64.0/19 -64.110.96.0/20 -64.110.112.0/21 -68.233.96.0/20 -68.233.112.0/21 -68.233.120.0/21 -81.208.160.0/20 -81.208.188.0/22 -84.235.160.0/19 -84.235.192.0/23 -84.235.208.0/20 -84.235.224.0/20 -84.235.240.0/20 -89.168.0.0/21 -89.168.16.0/20 -89.168.32.0/19 -89.168.64.0/18 -129.80.0.0/16 -129.91.16.0/21 -129.146.0.0/21 -129.146.8.0/22 -129.146.12.128/25 -129.146.13.128/25 -129.146.14.128/25 -129.146.16.0/20 -129.146.32.0/19 -129.146.64.0/18 -129.146.128.0/17 -129.148.0.0/21 -129.148.8.0/22 -129.148.12.0/22 -129.148.16.0/20 -129.148.32.0/19 -129.148.128.0/23 -129.148.132.0/22 -129.148.144.0/23 -129.148.148.0/25 -129.148.148.192/26 -129.148.150.0/23 -129.148.152.0/23 -129.148.156.0/22 -129.148.160.0/23 -129.148.164.0/25 -129.148.166.0/23 -129.148.168.0/21 -129.148.176.0/22 -129.148.180.0/25 -129.148.184.0/22 -129.148.200.0/21 -129.148.208.0/22 -129.148.212.0/23 -129.148.214.0/24 -129.148.215.0/25 -129.148.219.192/26 -129.148.222.0/23 -129.149.0.0/22 -129.149.6.0/25 -129.149.16.0/23 -129.149.20.0/23 -129.149.22.0/25 -129.149.22.192/26 -129.149.23.0/24 -129.149.28.0/22 -129.149.32.0/23 -129.149.36.0/24 -129.149.37.128/25 -129.149.38.0/23 -129.149.48.0/22 -129.149.52.0/25 -129.149.56.0/22 -129.149.63.192/26 -129.149.64.0/22 -129.149.68.0/25 -129.149.80.0/22 -129.149.84.0/25 -129.149.96.0/22 -129.149.100.0/25 -129.149.112.0/22 -129.149.118.0/25 -129.149.120.0/22 -129.149.126.0/25 -129.150.32.0/19 -129.151.0.0/19 -129.151.32.0/21 -129.151.40.0/21 -129.151.48.0/20 -129.151.64.0/19 -129.151.96.0/19 -129.151.128.0/19 -129.151.160.0/19 -129.151.192.0/19 -129.151.224.0/19 -129.152.0.0/19 -129.153.0.0/19 -129.153.32.0/20 -129.153.48.0/20 -129.153.64.0/18 -129.153.128.0/18 -129.153.192.0/19 -129.153.224.0/20 -129.153.240.0/23 -129.153.243.192/26 -129.154.32.0/20 -129.154.48.0/20 -129.154.192.0/19 -129.154.224.0/19 -129.158.32.0/19 -129.158.192.0/18 -129.159.0.0/20 -129.159.16.0/21 -129.159.24.0/21 -129.159.32.0/20 -129.159.48.0/20 -129.159.64.0/18 -129.159.128.0/19 -129.159.160.0/19 -129.159.192.0/19 -129.159.224.0/20 -129.159.240.0/20 -129.213.0.128/25 -129.213.2.128/25 -129.213.4.128/25 -129.213.8.0/21 -129.213.16.0/20 -129.213.32.0/19 -129.213.64.0/18 -129.213.128.0/18 -129.213.192.0/20 -129.213.208.0/21 -130.35.0.0/22 -130.35.16.0/22 -130.35.96.0/21 -130.35.112.0/22 -130.35.116.0/25 -130.35.128.0/22 -130.35.144.0/22 -130.35.200.0/22 -130.35.228.0/22 -130.61.0.128/25 -130.61.2.128/25 -130.61.4.128/25 -130.61.8.0/21 -130.61.16.0/20 -130.61.32.0/19 -130.61.64.0/18 -130.61.128.0/17 -130.162.32.0/19 -130.162.92.0/23 -130.162.128.0/19 -130.162.160.0/19 -130.162.192.0/21 -130.162.208.0/20 -130.162.224.0/19 -131.186.0.0/21 -131.186.8.0/22 -131.186.12.0/25 -131.186.16.0/20 -131.186.32.0/20 -131.186.56.0/21 -132.145.0.128/25 -132.145.2.128/25 -132.145.4.128/25 -132.145.8.0/21 -132.145.16.0/20 -132.145.32.0/19 -132.145.64.0/20 -132.145.80.0/20 -132.145.96.0/20 -132.145.112.0/20 -132.145.128.0/18 -132.145.192.0/19 -132.145.224.0/19 -132.226.0.0/20 -132.226.16.0/21 -132.226.24.0/21 -132.226.32.0/19 -132.226.64.0/18 -132.226.128.0/21 -132.226.144.0/20 -132.226.160.0/21 -132.226.168.0/21 -132.226.176.0/21 -132.226.184.0/21 -132.226.192.0/20 -132.226.208.0/21 -132.226.216.0/21 -132.226.224.0/20 -132.226.240.0/20 -134.65.16.0/20 -134.65.48.0/22 -134.65.52.128/25 -134.65.56.0/21 -134.65.208.0/20 -134.65.224.0/19 -134.70.8.0/21 -134.70.16.0/22 -134.70.24.0/21 -134.70.32.0/22 -134.70.40.0/21 -134.70.48.0/22 -134.70.56.0/21 -134.70.64.0/22 -134.70.72.0/22 -134.70.76.0/22 -134.70.80.0/22 -134.70.84.0/22 -134.70.88.0/22 -134.70.92.0/22 -134.70.96.0/22 -134.70.100.0/22 -134.70.104.0/22 -134.70.108.0/22 -134.70.112.0/22 -134.70.116.0/22 -134.70.120.0/22 -134.70.124.0/22 -134.70.128.0/22 -134.70.132.0/22 -134.70.136.0/22 -134.70.140.0/22 -134.70.144.0/22 -134.70.148.0/22 -134.70.152.0/22 -134.70.156.0/22 -134.70.160.0/22 -134.70.164.0/22 -134.70.168.0/22 -134.70.172.0/22 -134.70.176.0/22 -134.70.180.0/22 -134.70.184.0/22 -134.70.188.0/22 -134.70.192.0/21 -134.70.200.0/23 -138.1.0.0/22 -138.1.16.0/22 -138.1.32.0/21 -138.1.40.0/21 -138.1.48.0/21 -138.1.64.0/22 -138.1.80.0/22 -138.1.108.0/25 -138.1.112.0/20 -138.2.0.0/19 -138.2.32.0/19 -138.2.64.0/19 -138.2.96.0/20 -138.2.112.0/20 -138.2.128.0/18 -138.2.208.0/20 -138.2.224.0/20 -138.2.240.0/21 -138.3.208.0/20 -138.3.240.0/20 -139.177.96.0/21 -139.177.104.0/22 -139.177.108.0/25 -139.185.32.0/19 -140.83.32.0/21 -140.83.44.0/22 -140.83.48.0/20 -140.83.80.0/21 -140.84.160.0/19 -140.86.192.0/20 -140.86.208.0/21 -140.86.216.0/21 -140.91.4.0/22 -140.91.8.0/23 -140.91.10.0/23 -140.91.12.0/22 -140.91.16.0/22 -140.91.20.0/23 -140.91.22.0/23 -140.91.24.0/22 -140.91.28.0/23 -140.91.30.0/23 -140.91.32.0/23 -140.91.34.0/23 -140.91.36.0/23 -140.91.38.0/23 -140.91.40.0/23 -140.91.42.0/23 -140.91.44.0/23 -140.91.46.0/23 -140.91.48.0/23 -140.91.50.0/23 -140.91.52.0/23 -140.91.54.0/23 -140.91.56.0/23 -140.91.58.0/23 -140.91.60.0/23 -140.91.62.0/23 -140.91.64.0/23 -140.91.66.0/23 -140.91.68.0/23 -140.91.70.0/23 -140.91.72.0/23 -140.91.74.0/23 -140.91.76.0/23 -140.91.78.0/23 -140.91.80.0/23 -140.91.82.0/23 -140.91.84.0/22 -140.91.88.0/23 -140.91.90.0/23 -140.204.0.128/25 -140.204.4.128/25 -140.204.8.128/25 -140.204.12.128/25 -140.204.16.128/25 -140.204.20.128/25 -140.204.24.128/25 -140.204.30.128/25 -140.204.34.128/25 -140.204.36.128/25 -140.204.38.128/25 -140.204.40.128/25 -140.204.42.128/25 -140.204.46.128/25 -140.204.50.128/25 -140.204.52.128/25 -140.204.54.128/25 -140.204.58.128/25 -140.204.66.128/25 -140.204.70.128/25 -140.204.76.128/25 -140.204.80.128/25 -140.204.84.0/23 -140.204.86.128/25 -140.204.92.128/25 -140.204.96.128/25 -140.204.100.128/25 -140.204.104.128/25 -140.204.108.128/25 -140.204.112.128/25 -140.204.116.128/25 -140.204.120.128/25 -140.204.122.128/25 -140.204.124.128/25 -140.204.126.128/25 -140.204.132.128/25 -140.238.0.0/19 -140.238.32.0/19 -140.238.64.0/18 -140.238.128.0/19 -140.238.160.0/21 -140.238.168.0/21 -140.238.176.0/20 -140.238.192.0/20 -140.238.208.0/20 -140.238.224.0/21 -140.238.232.0/22 -140.238.236.0/22 -140.238.240.0/20 -141.144.192.0/19 -141.144.224.0/19 -141.145.192.0/19 -141.147.0.0/18 -141.147.64.0/18 -141.147.128.0/20 -141.147.144.0/20 -141.147.160.0/19 -141.147.240.0/20 -141.148.0.0/18 -141.148.64.0/19 -141.148.128.0/18 -141.148.192.0/19 -141.148.224.0/19 -143.47.32.0/19 -143.47.96.0/19 -143.47.176.0/20 -143.47.224.0/19 -144.21.32.0/20 -144.21.48.0/20 -144.22.32.0/19 -144.22.64.0/18 -144.22.128.0/17 -144.24.0.0/18 -144.24.64.0/19 -144.24.96.0/19 -144.24.128.0/19 -144.24.160.0/19 -144.24.192.0/20 -144.24.208.0/20 -144.24.224.0/21 -144.24.232.0/21 -144.24.240.0/20 -146.56.32.0/20 -146.56.48.0/21 -146.56.61.192/26 -146.56.96.0/20 -146.56.112.0/21 -146.56.120.0/22 -146.56.127.192/26 -146.56.128.0/18 -146.235.0.0/20 -146.235.16.0/21 -146.235.24.0/21 -146.235.32.0/19 -146.235.192.0/19 -146.235.224.0/20 -146.235.240.0/21 -146.235.251.192/26 -146.235.253.0/24 -146.235.254.0/24 -146.235.255.192/26 -147.154.0.0/19 -147.154.32.0/25 -147.154.36.0/22 -147.154.40.0/21 -147.154.48.0/21 -147.154.56.0/22 -147.154.96.0/20 -147.154.112.0/21 -147.154.120.0/22 -147.154.128.0/19 -147.154.176.0/21 -147.154.184.0/22 -147.154.189.128/25 -147.154.224.0/20 -147.154.240.0/21 -147.154.255.128/25 -149.130.160.0/19 -149.130.208.0/20 -149.130.224.0/19 -150.136.0.0/16 -150.230.0.0/21 -150.230.8.0/21 -150.230.20.0/22 -150.230.24.0/21 -150.230.32.0/20 -150.230.48.0/21 -150.230.56.0/21 -150.230.64.0/19 -150.230.96.0/20 -150.230.112.0/20 -150.230.128.0/20 -150.230.144.0/20 -150.230.160.0/19 -150.230.192.0/19 -150.230.224.0/21 -150.230.232.0/21 -150.230.240.0/21 -150.230.248.0/21 -151.145.32.0/19 -151.145.64.0/20 -151.145.80.0/20 -152.67.0.0/19 -152.67.32.0/19 -152.67.64.0/19 -152.67.96.0/19 -152.67.128.0/19 -152.67.160.0/19 -152.67.192.0/19 -152.67.224.0/19 -152.69.160.0/20 -152.69.176.0/20 -152.69.192.0/20 -152.69.208.0/20 -152.69.224.0/20 -152.70.0.0/19 -152.70.32.0/22 -152.70.36.0/22 -152.70.40.0/21 -152.70.48.0/20 -152.70.64.0/20 -152.70.80.0/21 -152.70.88.0/21 -152.70.96.0/20 -152.70.112.0/20 -152.70.128.0/19 -152.70.160.0/19 -152.70.192.0/20 -152.70.208.0/20 -152.70.224.0/22 -152.70.228.0/22 -152.70.232.0/21 -152.70.240.0/20 -155.248.128.0/22 -155.248.132.0/23 -155.248.135.128/25 -155.248.136.0/22 -155.248.140.0/25 -155.248.144.0/22 -155.248.148.0/25 -155.248.160.0/19 -155.248.192.0/20 -155.248.208.0/21 -155.248.216.0/21 -155.248.224.0/20 -155.248.240.0/20 -158.101.0.0/18 -158.101.64.0/19 -158.101.96.0/19 -158.101.128.0/19 -158.101.160.0/19 -158.101.192.0/19 -158.101.224.0/19 -158.178.128.0/22 -158.178.136.0/21 -158.178.144.0/20 -158.178.192.0/20 -158.178.208.0/21 -158.178.216.0/22 -158.178.220.0/22 -158.178.224.0/20 -158.178.240.0/21 -158.178.248.0/22 -158.179.4.0/22 -158.179.8.0/22 -158.179.15.192/26 -158.179.16.0/20 -158.179.160.0/20 -158.179.176.0/20 -158.179.192.0/22 -158.179.196.0/24 -158.179.198.0/23 -158.179.200.0/21 -158.179.208.0/20 -158.180.4.0/22 -158.180.8.0/21 -158.180.16.0/20 -158.180.32.0/19 -158.180.64.0/19 -158.180.226.0/23 -158.180.228.0/22 -158.180.232.0/21 -158.247.96.0/22 -158.247.100.0/25 -158.247.104.0/22 -158.247.112.0/23 -158.247.114.128/25 -158.247.120.0/21 -159.13.32.0/19 -159.54.128.0/19 -159.112.128.0/20 -159.112.144.0/21 -159.112.162.0/23 -159.112.166.0/24 -159.112.172.0/22 -159.112.176.0/20 -164.152.16.0/20 -164.152.32.0/19 -164.152.104.0/21 -164.152.192.0/21 -164.152.240.0/20 -165.1.64.0/20 -165.1.96.0/22 -165.1.100.0/25 -165.1.104.0/22 -165.1.112.0/23 -165.1.114.128/25 -165.1.120.0/21 -168.75.64.0/19 -168.75.96.0/20 -168.138.0.0/19 -168.138.32.0/19 -168.138.64.0/19 -168.138.96.0/20 -168.138.112.0/21 -168.138.120.0/22 -168.138.124.0/22 -168.138.128.0/19 -168.138.160.0/19 -168.138.192.0/19 -168.138.224.0/19 -169.155.128.0/19 -192.9.128.0/19 -192.9.160.0/19 -192.9.224.0/19 -192.18.128.0/20 -192.18.144.0/20 -192.18.200.0/21 -192.29.8.0/21 -192.29.20.0/22 -192.29.24.0/21 -192.29.36.0/22 -192.29.40.0/21 -192.29.48.0/22 -192.29.56.0/23 -192.29.60.0/23 -192.29.64.0/21 -192.29.72.0/25 -192.29.80.0/22 -192.29.88.0/23 -192.29.91.192/26 -192.29.92.0/22 -192.29.96.0/20 -192.29.112.0/20 -192.29.128.0/23 -192.29.130.0/24 -192.29.134.0/23 -192.29.137.192/26 -192.29.138.0/23 -192.29.140.0/22 -192.29.144.0/23 -192.29.148.0/23 -192.29.151.0/24 -192.29.152.0/22 -192.29.158.0/23 -192.29.160.0/21 -192.29.168.0/22 -192.29.172.0/24 -192.29.178.0/23 -192.29.180.0/22 -192.29.192.0/22 -192.29.200.0/21 -192.29.208.0/22 -192.29.216.0/21 -192.29.224.0/22 -192.29.232.0/25 -192.29.232.192/26 -192.29.240.0/22 -192.29.248.0/21 -193.122.0.0/20 -193.122.16.0/20 -193.122.32.0/19 -193.122.64.0/19 -193.122.96.0/19 -193.122.128.0/17 -193.123.0.0/19 -193.123.32.0/19 -193.123.64.0/19 -193.123.96.0/19 -193.123.128.0/19 -193.123.160.0/20 -193.123.176.0/20 -193.123.192.0/19 -193.123.224.0/19 -193.227.135.0/24 -204.216.104.0/21 -204.216.112.0/23 -204.216.119.192/26 -204.216.120.0/22 -204.216.127.192/26 -204.216.128.0/18 -204.216.192.0/20 -204.216.208.0/20 -205.147.88.0/23 -207.127.72.0/22 -207.127.88.0/21 -207.127.96.0/21 -207.127.107.192/26 -207.127.109.192/26 -207.127.124.0/23 -207.135.0.0/22 -207.135.4.0/22 -207.135.8.0/23 -207.135.10.0/23 -207.135.12.0/22 -207.135.16.0/23 -207.135.18.0/23 -207.135.20.0/23 -207.135.25.0/24 -207.135.26.0/24 -207.135.27.0/24 -207.135.28.0/24 -207.135.29.0/24 -207.135.30.0/23 -207.211.140.0/22 -207.211.144.0/20 -207.211.160.0/19 -209.17.60.0/22 -213.19.198.156/31 -213.19.198.164/31 -213.35.96.0/19 -217.142.128.0/19 -217.142.224.0/19 diff --git a/oracle/ipv4_comma.txt b/oracle/ipv4_comma.txt deleted file mode 100644 index b3a8fcdd..00000000 --- a/oracle/ipv4_comma.txt +++ /dev/null @@ -1 +0,0 @@ -38.104.155.92/31,40.233.0.0/19,40.233.64.0/18,62.115.179.220/31,62.115.179.228/31,64.110.64.0/19,64.110.96.0/20,64.110.112.0/21,68.233.96.0/20,68.233.112.0/21,68.233.120.0/21,81.208.160.0/20,81.208.188.0/22,84.235.160.0/19,84.235.192.0/23,84.235.208.0/20,84.235.224.0/20,84.235.240.0/20,89.168.0.0/21,89.168.16.0/20,89.168.32.0/19,89.168.64.0/18,129.80.0.0/16,129.91.16.0/21,129.146.0.0/21,129.146.8.0/22,129.146.12.128/25,129.146.13.128/25,129.146.14.128/25,129.146.16.0/20,129.146.32.0/19,129.146.64.0/18,129.146.128.0/17,129.148.0.0/21,129.148.8.0/22,129.148.12.0/22,129.148.16.0/20,129.148.32.0/19,129.148.128.0/23,129.148.132.0/22,129.148.144.0/23,129.148.148.0/25,129.148.148.192/26,129.148.150.0/23,129.148.152.0/23,129.148.156.0/22,129.148.160.0/23,129.148.164.0/25,129.148.166.0/23,129.148.168.0/21,129.148.176.0/22,129.148.180.0/25,129.148.184.0/22,129.148.200.0/21,129.148.208.0/22,129.148.212.0/23,129.148.214.0/24,129.148.215.0/25,129.148.219.192/26,129.148.222.0/23,129.149.0.0/22,129.149.6.0/25,129.149.16.0/23,129.149.20.0/23,129.149.22.0/25,129.149.22.192/26,129.149.23.0/24,129.149.28.0/22,129.149.32.0/23,129.149.36.0/24,129.149.37.128/25,129.149.38.0/23,129.149.48.0/22,129.149.52.0/25,129.149.56.0/22,129.149.63.192/26,129.149.64.0/22,129.149.68.0/25,129.149.80.0/22,129.149.84.0/25,129.149.96.0/22,129.149.100.0/25,129.149.112.0/22,129.149.118.0/25,129.149.120.0/22,129.149.126.0/25,129.150.32.0/19,129.151.0.0/19,129.151.32.0/21,129.151.40.0/21,129.151.48.0/20,129.151.64.0/19,129.151.96.0/19,129.151.128.0/19,129.151.160.0/19,129.151.192.0/19,129.151.224.0/19,129.152.0.0/19,129.153.0.0/19,129.153.32.0/20,129.153.48.0/20,129.153.64.0/18,129.153.128.0/18,129.153.192.0/19,129.153.224.0/20,129.153.240.0/23,129.153.243.192/26,129.154.32.0/20,129.154.48.0/20,129.154.192.0/19,129.154.224.0/19,129.158.32.0/19,129.158.192.0/18,129.159.0.0/20,129.159.16.0/21,129.159.24.0/21,129.159.32.0/20,129.159.48.0/20,129.159.64.0/18,129.159.128.0/19,129.159.160.0/19,129.159.192.0/19,129.159.224.0/20,129.159.240.0/20,129.213.0.128/25,129.213.2.128/25,129.213.4.128/25,129.213.8.0/21,129.213.16.0/20,129.213.32.0/19,129.213.64.0/18,129.213.128.0/18,129.213.192.0/20,129.213.208.0/21,130.35.0.0/22,130.35.16.0/22,130.35.96.0/21,130.35.112.0/22,130.35.116.0/25,130.35.128.0/22,130.35.144.0/22,130.35.200.0/22,130.35.228.0/22,130.61.0.128/25,130.61.2.128/25,130.61.4.128/25,130.61.8.0/21,130.61.16.0/20,130.61.32.0/19,130.61.64.0/18,130.61.128.0/17,130.162.32.0/19,130.162.92.0/23,130.162.128.0/19,130.162.160.0/19,130.162.192.0/21,130.162.208.0/20,130.162.224.0/19,131.186.0.0/21,131.186.8.0/22,131.186.12.0/25,131.186.16.0/20,131.186.32.0/20,131.186.56.0/21,132.145.0.128/25,132.145.2.128/25,132.145.4.128/25,132.145.8.0/21,132.145.16.0/20,132.145.32.0/19,132.145.64.0/20,132.145.80.0/20,132.145.96.0/20,132.145.112.0/20,132.145.128.0/18,132.145.192.0/19,132.145.224.0/19,132.226.0.0/20,132.226.16.0/21,132.226.24.0/21,132.226.32.0/19,132.226.64.0/18,132.226.128.0/21,132.226.144.0/20,132.226.160.0/21,132.226.168.0/21,132.226.176.0/21,132.226.184.0/21,132.226.192.0/20,132.226.208.0/21,132.226.216.0/21,132.226.224.0/20,132.226.240.0/20,134.65.16.0/20,134.65.48.0/22,134.65.52.128/25,134.65.56.0/21,134.65.208.0/20,134.65.224.0/19,134.70.8.0/21,134.70.16.0/22,134.70.24.0/21,134.70.32.0/22,134.70.40.0/21,134.70.48.0/22,134.70.56.0/21,134.70.64.0/22,134.70.72.0/22,134.70.76.0/22,134.70.80.0/22,134.70.84.0/22,134.70.88.0/22,134.70.92.0/22,134.70.96.0/22,134.70.100.0/22,134.70.104.0/22,134.70.108.0/22,134.70.112.0/22,134.70.116.0/22,134.70.120.0/22,134.70.124.0/22,134.70.128.0/22,134.70.132.0/22,134.70.136.0/22,134.70.140.0/22,134.70.144.0/22,134.70.148.0/22,134.70.152.0/22,134.70.156.0/22,134.70.160.0/22,134.70.164.0/22,134.70.168.0/22,134.70.172.0/22,134.70.176.0/22,134.70.180.0/22,134.70.184.0/22,134.70.188.0/22,134.70.192.0/21,134.70.200.0/23,138.1.0.0/22,138.1.16.0/22,138.1.32.0/21,138.1.40.0/21,138.1.48.0/21,138.1.64.0/22,138.1.80.0/22,138.1.108.0/25,138.1.112.0/20,138.2.0.0/19,138.2.32.0/19,138.2.64.0/19,138.2.96.0/20,138.2.112.0/20,138.2.128.0/18,138.2.208.0/20,138.2.224.0/20,138.2.240.0/21,138.3.208.0/20,138.3.240.0/20,139.177.96.0/21,139.177.104.0/22,139.177.108.0/25,139.185.32.0/19,140.83.32.0/21,140.83.44.0/22,140.83.48.0/20,140.83.80.0/21,140.84.160.0/19,140.86.192.0/20,140.86.208.0/21,140.86.216.0/21,140.91.4.0/22,140.91.8.0/23,140.91.10.0/23,140.91.12.0/22,140.91.16.0/22,140.91.20.0/23,140.91.22.0/23,140.91.24.0/22,140.91.28.0/23,140.91.30.0/23,140.91.32.0/23,140.91.34.0/23,140.91.36.0/23,140.91.38.0/23,140.91.40.0/23,140.91.42.0/23,140.91.44.0/23,140.91.46.0/23,140.91.48.0/23,140.91.50.0/23,140.91.52.0/23,140.91.54.0/23,140.91.56.0/23,140.91.58.0/23,140.91.60.0/23,140.91.62.0/23,140.91.64.0/23,140.91.66.0/23,140.91.68.0/23,140.91.70.0/23,140.91.72.0/23,140.91.74.0/23,140.91.76.0/23,140.91.78.0/23,140.91.80.0/23,140.91.82.0/23,140.91.84.0/22,140.91.88.0/23,140.91.90.0/23,140.204.0.128/25,140.204.4.128/25,140.204.8.128/25,140.204.12.128/25,140.204.16.128/25,140.204.20.128/25,140.204.24.128/25,140.204.30.128/25,140.204.34.128/25,140.204.36.128/25,140.204.38.128/25,140.204.40.128/25,140.204.42.128/25,140.204.46.128/25,140.204.50.128/25,140.204.52.128/25,140.204.54.128/25,140.204.58.128/25,140.204.66.128/25,140.204.70.128/25,140.204.76.128/25,140.204.80.128/25,140.204.84.0/23,140.204.86.128/25,140.204.92.128/25,140.204.96.128/25,140.204.100.128/25,140.204.104.128/25,140.204.108.128/25,140.204.112.128/25,140.204.116.128/25,140.204.120.128/25,140.204.122.128/25,140.204.124.128/25,140.204.126.128/25,140.204.132.128/25,140.238.0.0/19,140.238.32.0/19,140.238.64.0/18,140.238.128.0/19,140.238.160.0/21,140.238.168.0/21,140.238.176.0/20,140.238.192.0/20,140.238.208.0/20,140.238.224.0/21,140.238.232.0/22,140.238.236.0/22,140.238.240.0/20,141.144.192.0/19,141.144.224.0/19,141.145.192.0/19,141.147.0.0/18,141.147.64.0/18,141.147.128.0/20,141.147.144.0/20,141.147.160.0/19,141.147.240.0/20,141.148.0.0/18,141.148.64.0/19,141.148.128.0/18,141.148.192.0/19,141.148.224.0/19,143.47.32.0/19,143.47.96.0/19,143.47.176.0/20,143.47.224.0/19,144.21.32.0/20,144.21.48.0/20,144.22.32.0/19,144.22.64.0/18,144.22.128.0/17,144.24.0.0/18,144.24.64.0/19,144.24.96.0/19,144.24.128.0/19,144.24.160.0/19,144.24.192.0/20,144.24.208.0/20,144.24.224.0/21,144.24.232.0/21,144.24.240.0/20,146.56.32.0/20,146.56.48.0/21,146.56.61.192/26,146.56.96.0/20,146.56.112.0/21,146.56.120.0/22,146.56.127.192/26,146.56.128.0/18,146.235.0.0/20,146.235.16.0/21,146.235.24.0/21,146.235.32.0/19,146.235.192.0/19,146.235.224.0/20,146.235.240.0/21,146.235.251.192/26,146.235.253.0/24,146.235.254.0/24,146.235.255.192/26,147.154.0.0/19,147.154.32.0/25,147.154.36.0/22,147.154.40.0/21,147.154.48.0/21,147.154.56.0/22,147.154.96.0/20,147.154.112.0/21,147.154.120.0/22,147.154.128.0/19,147.154.176.0/21,147.154.184.0/22,147.154.189.128/25,147.154.224.0/20,147.154.240.0/21,147.154.255.128/25,149.130.160.0/19,149.130.208.0/20,149.130.224.0/19,150.136.0.0/16,150.230.0.0/21,150.230.8.0/21,150.230.20.0/22,150.230.24.0/21,150.230.32.0/20,150.230.48.0/21,150.230.56.0/21,150.230.64.0/19,150.230.96.0/20,150.230.112.0/20,150.230.128.0/20,150.230.144.0/20,150.230.160.0/19,150.230.192.0/19,150.230.224.0/21,150.230.232.0/21,150.230.240.0/21,150.230.248.0/21,151.145.32.0/19,151.145.64.0/20,151.145.80.0/20,152.67.0.0/19,152.67.32.0/19,152.67.64.0/19,152.67.96.0/19,152.67.128.0/19,152.67.160.0/19,152.67.192.0/19,152.67.224.0/19,152.69.160.0/20,152.69.176.0/20,152.69.192.0/20,152.69.208.0/20,152.69.224.0/20,152.70.0.0/19,152.70.32.0/22,152.70.36.0/22,152.70.40.0/21,152.70.48.0/20,152.70.64.0/20,152.70.80.0/21,152.70.88.0/21,152.70.96.0/20,152.70.112.0/20,152.70.128.0/19,152.70.160.0/19,152.70.192.0/20,152.70.208.0/20,152.70.224.0/22,152.70.228.0/22,152.70.232.0/21,152.70.240.0/20,155.248.128.0/22,155.248.132.0/23,155.248.135.128/25,155.248.136.0/22,155.248.140.0/25,155.248.144.0/22,155.248.148.0/25,155.248.160.0/19,155.248.192.0/20,155.248.208.0/21,155.248.216.0/21,155.248.224.0/20,155.248.240.0/20,158.101.0.0/18,158.101.64.0/19,158.101.96.0/19,158.101.128.0/19,158.101.160.0/19,158.101.192.0/19,158.101.224.0/19,158.178.128.0/22,158.178.136.0/21,158.178.144.0/20,158.178.192.0/20,158.178.208.0/21,158.178.216.0/22,158.178.220.0/22,158.178.224.0/20,158.178.240.0/21,158.178.248.0/22,158.179.4.0/22,158.179.8.0/22,158.179.15.192/26,158.179.16.0/20,158.179.160.0/20,158.179.176.0/20,158.179.192.0/22,158.179.196.0/24,158.179.198.0/23,158.179.200.0/21,158.179.208.0/20,158.180.4.0/22,158.180.8.0/21,158.180.16.0/20,158.180.32.0/19,158.180.64.0/19,158.180.226.0/23,158.180.228.0/22,158.180.232.0/21,158.247.96.0/22,158.247.100.0/25,158.247.104.0/22,158.247.112.0/23,158.247.114.128/25,158.247.120.0/21,159.13.32.0/19,159.54.128.0/19,159.112.128.0/20,159.112.144.0/21,159.112.162.0/23,159.112.166.0/24,159.112.172.0/22,159.112.176.0/20,164.152.16.0/20,164.152.32.0/19,164.152.104.0/21,164.152.192.0/21,164.152.240.0/20,165.1.64.0/20,165.1.96.0/22,165.1.100.0/25,165.1.104.0/22,165.1.112.0/23,165.1.114.128/25,165.1.120.0/21,168.75.64.0/19,168.75.96.0/20,168.138.0.0/19,168.138.32.0/19,168.138.64.0/19,168.138.96.0/20,168.138.112.0/21,168.138.120.0/22,168.138.124.0/22,168.138.128.0/19,168.138.160.0/19,168.138.192.0/19,168.138.224.0/19,169.155.128.0/19,192.9.128.0/19,192.9.160.0/19,192.9.224.0/19,192.18.128.0/20,192.18.144.0/20,192.18.200.0/21,192.29.8.0/21,192.29.20.0/22,192.29.24.0/21,192.29.36.0/22,192.29.40.0/21,192.29.48.0/22,192.29.56.0/23,192.29.60.0/23,192.29.64.0/21,192.29.72.0/25,192.29.80.0/22,192.29.88.0/23,192.29.91.192/26,192.29.92.0/22,192.29.96.0/20,192.29.112.0/20,192.29.128.0/23,192.29.130.0/24,192.29.134.0/23,192.29.137.192/26,192.29.138.0/23,192.29.140.0/22,192.29.144.0/23,192.29.148.0/23,192.29.151.0/24,192.29.152.0/22,192.29.158.0/23,192.29.160.0/21,192.29.168.0/22,192.29.172.0/24,192.29.178.0/23,192.29.180.0/22,192.29.192.0/22,192.29.200.0/21,192.29.208.0/22,192.29.216.0/21,192.29.224.0/22,192.29.232.0/25,192.29.232.192/26,192.29.240.0/22,192.29.248.0/21,193.122.0.0/20,193.122.16.0/20,193.122.32.0/19,193.122.64.0/19,193.122.96.0/19,193.122.128.0/17,193.123.0.0/19,193.123.32.0/19,193.123.64.0/19,193.123.96.0/19,193.123.128.0/19,193.123.160.0/20,193.123.176.0/20,193.123.192.0/19,193.123.224.0/19,193.227.135.0/24,204.216.104.0/21,204.216.112.0/23,204.216.119.192/26,204.216.120.0/22,204.216.127.192/26,204.216.128.0/18,204.216.192.0/20,204.216.208.0/20,205.147.88.0/23,207.127.72.0/22,207.127.88.0/21,207.127.96.0/21,207.127.107.192/26,207.127.109.192/26,207.127.124.0/23,207.135.0.0/22,207.135.4.0/22,207.135.8.0/23,207.135.10.0/23,207.135.12.0/22,207.135.16.0/23,207.135.18.0/23,207.135.20.0/23,207.135.25.0/24,207.135.26.0/24,207.135.27.0/24,207.135.28.0/24,207.135.29.0/24,207.135.30.0/23,207.211.140.0/22,207.211.144.0/20,207.211.160.0/19,209.17.60.0/22,213.19.198.156/31,213.19.198.164/31,213.35.96.0/19,217.142.128.0/19,217.142.224.0/19 diff --git a/oracle/start.sh b/oracle/start.sh index 9a6ad230..beaff2b7 100755 --- a/oracle/start.sh +++ b/oracle/start.sh @@ -1,5 +1,8 @@ #!/bin/bash +set -euo pipefail +set -x + # Define file paths json_file="/tmp/oracle.json" timestamp_file="oracle/timestamp.txt" @@ -14,17 +17,20 @@ ipv4_comma_output="oracle/ipv4_comma.txt" # ipv6_comma_output="oracle/ipv6_comma.txt" -# Check if timestamp file exists and remove it +# Check if the timestamp_file exists and remove it if it does if [ -e "$timestamp_file" ]; then rm "$timestamp_file" + echo "Step 0: File $timestamp_file removed successfully." +else + echo "Step 0: File $timestamp_file does not exist. Skip." fi -# Download public ORACLE IP ranges -curl -s https://docs.oracle.com/en-us/iaas/tools/public_ip_ranges.json > "$json_file" -# Check if the curl command was successful -if [ $? -ne 0 ]; then - echo "Error: Failed to download ORACLE IP ranges using curl." +# Download public ORACLE IP ranges +if curl -s https://docs.oracle.com/en-us/iaas/tools/public_ip_ranges.json > "$json_file"; then + echo "Step 1: Data fetched from ORACLE IP ranges using curl successfully." +else + echo "Step 1: Failed to download ORACLE IP ranges using curl." >&2 exit 1 fi @@ -35,32 +41,31 @@ oracle_utc=$(date -d "$timestamp" -u +"%Y-%m-%dT%H:%M:%S.000000Z") # Save timestamp to file echo "$oracle_utc" > "$timestamp_file" -# Save IPv4 addresses -jq '.regions[] | [.cidrs][] | .[].cidr | select(. != null)' -r "$json_file" > "$ipv4_file" - -# Check if the jq command was successful -if [ $? -ne 0 ]; then - echo "Error: Failed to extract IPv4 addresses using jq." +# Save IPv4 addresses and check if the jq command was successful +if jq '.regions[] | [.cidrs][] | .[].cidr | select(. != null)' -r "$json_file" > "$ipv4_file"; then + echo "Step 2: IPv4 addresses extracted successfully." +else + echo "Step 2: Failed to extract IPv4 addresses using jq." exit 1 fi -# Sort and remove duplicates for IPv4 -sort -V "$ipv4_file" | uniq > "$ipv4_output" - -# Check if the sort command was successful -if [ $? -ne 0 ]; then - echo "Error: Failed to sort IPv4 addresses." +# Sort and remove duplicates for IPv4 and check if the sort command was successful +if sort -V "$ipv4_file" | uniq > "$ipv4_output"; then + echo "Step 3: IPv4 addresses sorted and duplicates removed successfully." +else + echo "Step 3: Failed to sort IPv4 addresses." exit 1 fi -# Save IPv4 addresses with comma separation -jq -r '.regions[] | [.cidrs][] | .[].cidr | select(. != null)' "$json_file" | sort -V | uniq | paste -sd "," - > "$ipv4_comma_output" - -# Check if the jq and sort commands were successful -if [ $? -ne 0 ]; then - echo "Error: Failed to process IPv4 addresses for comma separation." +# Save IPv4 addresses with comma separation and check if the jq and sort commands were successful +if jq -r '.regions[] | [.cidrs][] | .[].cidr | select(. != null)' "$json_file" | sort -V | uniq | paste -sd "," - > "$ipv4_comma_output"; then + echo "Step 4: IPv4 addresses processed for comma separation successfully." +else + echo "Step 4: Failed to process IPv4 addresses for comma separation." exit 1 fi # Clean up temporary files rm "$json_file" "$ipv4_file" + +echo "OCI Complete!" \ No newline at end of file diff --git a/oracle/timestamp.txt b/oracle/timestamp.txt deleted file mode 100644 index c88c1154..00000000 --- a/oracle/timestamp.txt +++ /dev/null @@ -1 +0,0 @@ -2024-01-08T13:45:39.000000Z