Skip to content

Commit

Permalink
Improved the script
Browse files Browse the repository at this point in the history 
Added a new file for tracking blocked domains
Removed unnecessary comments and codes
Made the script faster
  • Loading branch information
@HotCakeX
HotCakeX committed May 1, 2024
1 parent 80f0853 commit 901c003
Showing 1 changed file with 58 additions and 68 deletions.
126 changes: 58 additions & 68 deletions NextDNS API/Stream the logs - Customized Output for Microsoft.ps1
View file
Original file line number Diff line number Diff line change
@@ -1,93 +1,75 @@
$ErrorActionPreference = 'Stop'

# Make a new file to store the items that belong to Microsoft but are not in the Whitelisted domains list
if (-NOT (Test-Path -Path .\NotWhitelisted.txt)) {
New-Item -ItemType File -Path .\NotWhitelisted.txt -Force | Out-Null
}

# Make a new file to store the items that possibly belong to Microsoft but got blocked
if (-NOT (Test-Path -Path .\MicrosoftPossibleBlocked.txt)) {
New-Item -ItemType File -Path .\MicrosoftPossibleBlocked.txt -Force | Out-Null
}

# Make a new file to store all of the domains that were connected to, no duplicates. Excluding the domains stored in other Microsoft related lists and blocked domains
if (-NOT (Test-Path -Path .\AllDomains.txt)) {
New-Item -ItemType File -Path .\AllDomains.txt -Force | Out-Null
# 1) Domains that belong to Microsoft but are not whitelisted
# 2) Domains possibly belong to Microsoft but got blocked
# 3) All contacted domains, excluding the domains stored in other Microsoft related lists and blocked domains
foreach ($File in 'NotWhitelisted.txt', 'MicrosoftPossibleBlocked.txt', 'AllDomains.txt') {
if (-NOT (Test-Path -Path ".\$File")) {
New-Item -ItemType File -Path ".\$File" -Force | Out-Null
}
}

# Make a new file to store all of the domains that were connected to, with their counts as a hashtable. Excluding the domains stored in other Microsoft related lists and blocked domains
# a file to store all of the domains that were connected to, with their counts as a hashtable. Excluding the domains stored in other Microsoft related lists and blocked domains
if (-NOT (Test-Path -Path .\AllDomainsCount.txt)) {
New-Item -ItemType File -Path .\AllDomainsCount.txt -Force | Out-Null

# Create an empty ordered hashtable to store the domains and their counts
$DomainCount = [System.Management.Automation.OrderedHashtable]::new()
}
# If the file and hashtable already exists, read it
# If the file already exists, convert it to JSON and read it as hashtable
else {
# Read the JSON string from the file and convert it to a hashtable
[System.Management.Automation.OrderedHashtable]$DomainCount = Get-Content -Path '.\AllDomainsCount.txt' | ConvertFrom-Json -AsHashtable
}

# a file to store all of the blocked domains that were connected to, with their counts as a hashtable
if (-NOT (Test-Path -Path .\AllBlockedCount.txt)) {
New-Item -ItemType File -Path .\AllBlockedCount.txt -Force | Out-Null
$BlockedCount = [System.Management.Automation.OrderedHashtable]::new()
}
# If the file already exists, convert it to JSON and read it as hashtable
else {
[System.Management.Automation.OrderedHashtable]$BlockedCount = Get-Content -Path '.\AllBlockedCount.txt' | ConvertFrom-Json -AsHashtable
}

# Try-Finally to loop through the stream and process each line as a JSON object
try {

# Read the Whitelisted Domains, it's always located here: https://github.com/HotCakeX/MicrosoftDomains/blob/main/Microsoft%20Domains.txt
[System.String[]]$WhiteListedDomains = Get-Content -Path '.\Microsoft Domains.txt'
$WhiteListedDomains = [System.Collections.Generic.HashSet[System.String]] @(Get-Content -Path '.\Microsoft Domains.txt')

# Define the API key and the profile ID
# These are the credentials that you need to access the NextDNS API
# Get your API key from here: https://my.nextdns.io/account
# Define the API key and the profile ID to access the NextDNS API - https://my.nextdns.io/account
[System.String]$ApiKey = ''
[System.String]$ProfileId = ''

# Define the URL for streaming the logs
# This is the endpoint that you need to send a web request to get the logs as a SSE stream
# https://nextdns.github.io/api/#streaming
# Define the endpoint that you need to send a web request to get the logs as a SSE stream - https://nextdns.github.io/api/#streaming
[System.Uri]$URL = "https://api.nextdns.io/profiles/$ProfileId/logs/stream"

# Create a header with the API key as a hashtable
# This is a key-value pair that you need to include in the web request to authenticate yourself
[System.Collections.Hashtable]$Header = @{
'X-Api-Key' = $ApiKey
}
# Create an empty NameValueCollection
# This is a special type of collection that can store multiple values for each key, and it is used by the web request object to set the header
$HeaderNVC = [System.Collections.Specialized.NameValueCollection]::new()

# Loop over the hashtable keys and values and add them to the NameValueCollection
# This is a way of converting the hashtable to a NameValueCollection, by iterating over each key and value and adding them to the collection
foreach ($key in $Header.Keys) {
$Value = $Header[$key]
$HeaderNVC.Add($key, $Value)
}
# Add the X-Api-Key header for authentication to the NameValueCollection
$HeaderNVC.Add('X-Api-Key', $ApiKey)

# Create a web request object
# This is an object that represents a HTTP request that can be sent to a server and get a response
# Create a web request object that represents a HTTP request that can be sent to a server and get a response
$WebRequest = [System.Net.HttpWebRequest]::Create($URL)

# Set the header with the API key
# This is a way of adding the header to the web request object, by using the NameValueCollection that we created earlier
# Adding the header to the web request object
$WebRequest.Headers.Add($HeaderNVC)

# Set the timeout to infinite
# This is a way of telling the web request object to wait indefinitely for a response, because we are expecting a continuous stream of data from the server
# Set the timeout to infinite, tells the web request object to wait indefinitely for a response, because we are expecting a continuous stream of data from the server
$WebRequest.Timeout = -1

# Get the web response object
# This is an object that represents a HTTP response that is received from the server after sending the web request
# Get the web response (HTTP response) object from the server
[System.Net.HttpWebResponse]$Response = $WebRequest.GetResponse()

# Get the response stream
# This is an object that represents a stream of data that is sent by the server as part of the response, and it can be read line by line using a stream reader object
# Get the response stream, represents a stream of data that is sent by the server as part of the response, and it can be read line by line using a stream reader object
$ResponseStream = $Response.GetResponseStream()

# Create a stream reader object
# This is an object that can read data from a stream, such as the response stream, and convert it to text
# Create a stream reader object that can read data from a stream, such as the response stream, and convert it to text
$StreamReader = [System.IO.StreamReader]::new($ResponseStream)

while ($true) {
# Read one line from the stream
# This is a way of getting one line of text from the stream reader object, which corresponds to one event from the server
# Read one line from the stream, which corresponds to one event from the server
$Line = $StreamReader.ReadLine()

# Split the line by colon and space characters
Expand All @@ -97,20 +79,16 @@ try {
# Check if the line has two parts
# This is a way of validating that the line has both a prefix and a JSON data, and not something else. We use the Count property of the array to check this.
if ($Parts.Count -eq 2) {
# Use the second part as the JSON data
# This is a way of getting only the JSON data from the line, by using index 1 of the array (index 0 is for prefix)
# Use the second part so we only get the JSON data from the line (index 0 is for prefix)
$JsonData = $Parts[1]

# Check if the JSON data is not empty
# This is a way of validating that there is some data in the JSON part, and not just an empty string. We use the -ne operator to compare the JSON data with an empty string.
if ($JsonData -ne '') {

# Remove any characters from the beginning of the JSON data that may make it invalid or malformed
# This is a way of fixing the JSON data if it has some extra characters at the beginning, such as colon or space, that may prevent it from being parsed as a JSON object.
# Remove any characters from the beginning of the JSON data that may make it invalid or malformed such as colon or space
$JsonData = $JsonData.TrimStart(': ')

# Test if the JSON data is a valid JSON object using the Test-Json cmdlet
# Using the -ErrorAction SilentlyContinue parameter to suppress any error messages and return false instead.
[System.Boolean]$IsValidJson = Test-Json -Json $JsonData -ErrorAction SilentlyContinue

# Check if the JSON data is a valid JSON object
Expand All @@ -119,10 +97,6 @@ try {
# Convert the JSON data to a hashtable
[System.Management.Automation.OrderedHashtable]$Log = $JsonData | ConvertFrom-Json -AsHashtable

# Select only the properties that you are interested in
# This is a way of filtering the hashtable and getting only the properties that you want
# $Log = $Log | Select-Object timestamp, domain, root, clientIp, status | Format-Table

# Making sure the root in the log is actually the root domain and not a sub-domain
# Sometimes it identifies sub-domains as root domains, so here we make sure it doesn't happen
# first see if the root domain has more than 1 dot in it, indicating that it contains sub-domains
Expand Down Expand Up @@ -166,24 +140,25 @@ try {
$($Log | Select-Object -Property timestamp, domain, root, clientIp, status | Format-Table)

# Make sure the domain isn't already available in the file
[System.String[]]$CurrentItemsMicrosoft = Get-Content -Path '.\MicrosoftPossibleBlocked.txt' -Force
$CurrentItemsMicrosoft = [System.Collections.Generic.HashSet[System.String]] @(Get-Content -Path '.\MicrosoftPossibleBlocked.txt' -Force)

# Add the Blocked domain to the MicrosoftPossibleBlocked.txt list for later review
if ($RootDomain -notin $CurrentItemsMicrosoft) {
if (-NOT $CurrentItemsMicrosoft.Contains($RootDomain)) {
Add-Content -Value $RootDomain -Path '.\MicrosoftPossibleBlocked.txt' -Force
}
}
# If the domain was not blocked but also wasn't in the Microsoft domains Whitelist
elseif ($RootDomain -notin $WhiteListedDomains) {
elseif (-NOT $WhiteListedDomains.Contains($RootDomain)) {

# Display it with cyan text on the console
Write-Host -Object 'Microsoft Domain Not Whitelisted' -ForegroundColor Cyan
$($Log | Select-Object -Property timestamp, domain, root, clientIp, status | Format-Table)

# Make sure the domain isn't already available in the NotWhitelisted.Txt file
[System.String[]]$CurrentItemsNotWhitelisted = Get-Content -Path '.\NotWhitelisted.txt' -Force
$CurrentItemsNotWhitelisted = [System.Collections.Generic.HashSet[System.String]] @(Get-Content -Path '.\NotWhitelisted.txt' -Force)

# Add the detected domain to the NotWhitelisted.Txt list for later review
if ($RootDomain -notin $CurrentItemsNotWhitelisted) {
if (-NOT $CurrentItemsNotWhitelisted.Contains($RootDomain)) {
Add-Content -Value $RootDomain -Path '.\NotWhitelisted.txt' -Force
}
}
Expand All @@ -197,20 +172,32 @@ try {
elseif ($Log.status -eq 'blocked') {
Write-Host -Object 'BLOCKED' -ForegroundColor Red
$($Log | Select-Object -Property timestamp, domain, root, clientIp, status | Format-Table)

# Check if the domain already exists in the blocked domains hashtable
if ($BlockedCount.ContainsKey($RootDomain)) {
# Increment its count by one
$BlockedCount[$RootDomain] += 1
}
else {
# Add it to the hashtable with a count of one
$BlockedCount.Add($RootDomain, 1)
}

# Convert the hashtable to a JSON string and write it to .\AllBlockedCount.txt
$BlockedCount | ConvertTo-Json | Set-Content -Path '.\AllBlockedCount.txt' -Force
}
# Display any allowed domain with green text on the console
else {
Write-Host -Object 'Allowed' -ForegroundColor Green
$($Log | Select-Object -Property timestamp, domain, root, clientIp, status | Format-Table)

# if the domain is neither blocked, belongs to Microsoft nor is it in the whitelisted domains list
if ($RootDomain -notin $WhiteListedDomains) {
if (-NOT $WhiteListedDomains.Contains($RootDomain)) {

# Get the content of the .\AllDomains.txt
[System.String[]]$CurrentItemsAllDomains = Get-Content -Path '.\AllDomains.txt' -Force
$CurrentItemsAllDomains = [System.Collections.Generic.HashSet[System.String]] @(Get-Content -Path '.\AllDomains.txt' -Force)

# Add the domain to .\AllDomains.txt , make sure it's unique and not already in the list
if ($RootDomain -notin $CurrentItemsAllDomains) {
if (-NOT $CurrentItemsAllDomains.Contains($RootDomain)) {
Add-Content -Value $RootDomain -Path '.\AllDomains.txt' -Force
}

Expand All @@ -233,6 +220,9 @@ try {
}
}
}
catch {
$_
}
finally {
# If an error occurred while reading from the stream
Write-Error -Message "An error occurred while reading from the stream: $_" -ErrorAction Continue
Expand Down

0 comments on commit 901c003

Please sign in to comment.