Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Improved download method #228

Merged
merged 14 commits into from
Nov 15, 2024
Prev Previous commit
Next Next commit
feat: Improved download method
kroese authored Nov 15, 2024
commit 5517a6b91c776c5e0fa97e7d6730c0c668a7a5f0
82 changes: 42 additions & 40 deletions src/mido.sh
Original file line number Diff line number Diff line change
@@ -64,31 +64,31 @@ download_windows() {
local desc="$3"
local sku_id=""
local sku_url=""
local iso_url=""
local iso_json=""
local language=""
local session_id=""
local user_agent=""
local download_type=""
local windows_version=""
local iso_download_link=""
local download_page_html=""
local product_edition_id=""
local language_skuid_json=""
local iso_download_link_html=""
local iso_download_page_html=""
local profile="606624d44113"

user_agent=$(get_agent)
language=$(getLanguage "$lang" "name")

case "${id,,}" in
"win11x64" ) windows_version="11" ;;
"win10x64" ) windows_version="10" ;;
"win81x64" ) windows_version="8" ;;
"win11arm64" ) windows_version="11arm64" ;;
"win11x64" ) windows_version="11" && download_type="1" ;;
"win10x64" ) windows_version="10" && download_type="1" ;;
"win11arm64" ) windows_version="11arm64" && download_type="2" ;;
* ) error "Invalid VERSION specified, value \"$id\" is not recognized!" && return 1 ;;
esac

user_agent=$(get_agent)
language=$(getLanguage "$lang" "name")

local url="https://www.microsoft.com/en-us/software-download/windows$windows_version"
case "$windows_version" in
8 | 10) url+="ISO";;
esac
[[ "${id,,}" == "win10"* ]] && url+="ISO"

# uuidgen: For MacOS (installed by default) and other systems (e.g. with no /proc) that don't have a kernel interface for generating random UUIDs
session_id=$(cat /proc/sys/kernel/random/uuid 2> /dev/null || uuidgen --random)
@@ -98,38 +98,40 @@ download_windows() {
# This is the *only* request we make that Fido doesn't. Fido manually maintains a list of all the Windows release/edition product edition IDs in its script (see: $WindowsVersions array). This is helpful for downloading older releases (e.g. Windows 10 1909, 21H1, etc.) but we always want to get the newest release which is why we get this value dynamically
# Also, keeping a "$WindowsVersions" array like Fido does would be way too much of a maintenance burden
# Remove "Accept" header that curl sends by default
[[ "$DEBUG" == [Yy1]* ]] && echo " - Parsing download page: ${url}"
iso_download_page_html=$(curl --silent --max-time 30 --user-agent "$user_agent" --header "Accept:" --max-filesize 1M --fail --proto =https --tlsv1.2 --http1.1 -- "$url") || {
[[ "$DEBUG" == [Yy1]* ]] && echo "Parsing download page: ${url}"
download_page_html=$(curl --silent --max-time 30 --user-agent "$user_agent" --header "Accept:" --max-filesize 1M --fail --proto =https --tlsv1.2 --http1.1 -- "$url") || {
handle_curl_error $?
return $?
}

[[ "$DEBUG" == [Yy1]* ]] && echo -n "Getting Product edition ID: "
# tr: Filter for only numerics to prevent HTTP parameter injection
# head -c was recently added to POSIX: https://austingroupbugs.net/view.php?id=407
product_edition_id=$(echo "$iso_download_page_html" | grep -Eo '<option value="[0-9]+">Windows' | cut -d '"' -f 2 | head -n 1 | tr -cd '0-9' | head -c 16)
product_edition_id=$(echo "$download_page_html" | grep -Eo '<option value="[0-9]+">Windows' | cut -d '"' -f 2 | head -n 1 | tr -cd '0-9' | head -c 16)
[[ "$DEBUG" == [Yy1]* ]] && echo "$product_edition_id"

if [ -z "$product_edition_id" ]; then
error "Product edition ID not found!"
return 1
fi


[[ "$DEBUG" == [Yy1]* ]] && echo "Permit Session ID: $session_id"
# Permit Session ID
# "org_id" is always the same value
curl --silent --max-time 30 --output /dev/null --user-agent "$user_agent" --header "Accept:" --max-filesize 100K --fail --proto =https --tlsv1.2 --http1.1 -- "https://vlscppe.microsoft.com/tags?org_id=y6jn8c31&session_id=$session_id" || {
# This should only happen if there's been some change to how this API works
handle_curl_error $?
return $?
}

[[ "$DEBUG" == [Yy1]* ]] && echo -n "Getting language SKU ID: "
sku_url="https://www.microsoft.com/software-download-connector/api/getskuinformationbyproductedition?profile=1234&ProductEditionId=$product_edition_id&SKU=undefined&friendlyFileName=undefined&Locale=en-US&sessionID=$session_id"

language_skuid_json=$(curl --silent --max-time 30 --request POST --user-agent "$user_agent" --data "" --header "Accept:" --max-filesize 10K --fail --proto =https --tlsv1.2 --http1.1 -- "$sku_url") || {
sku_url="https://www.microsoft.com/software-download-connector/api/getskuinformationbyproductedition?profile=$profile&ProductEditionId=$product_edition_id&SKU=undefined&friendlyFileName=undefined&Locale=en-US&sessionID=$session_id"
language_skuid_json=$(curl --silent --max-time 30 --request GET --user-agent "$user_agent" --referer "$url" --header "Accept:" --max-filesize 100K --fail --proto =https --tlsv1.2 --http1.1 -- "$sku_url") || {
handle_curl_error $?
return $?
}

{ sku_id=$(echo "$language_skuid_json" | jq -r '.Skus[] | select(.Language=="${language}").Id'); rc=$?; } || :
{ sku_id=$(echo "$language_skuid_json" | jq --arg LANG "$language" -r '.Skus[] | select(.Language==$LANG).Id') 2>/dev/null; rc=$?; } || :

if [ -z "$sku_id" ] || [[ "$sku_id" == "null" ]] || (( rc != 0 )); then
if [ -z "$sku_id" ] || [[ "${sku_id,,}" == "null" ]] || (( rc != 0 )); then
language=$(getLanguage "$lang" "desc")
error "No download in the $language language available for $desc!"
return 1
@@ -138,33 +140,33 @@ download_windows() {
[[ "$DEBUG" == [Yy1]* ]] && echo "$sku_id"
[[ "$DEBUG" == [Yy1]* ]] && echo "Getting ISO download link..."

# Extract everything after the last slash
local url_segment_parameter="${url##*/}"

# Get ISO download link
# If any request is going to be blocked by Microsoft it's always this last one (the previous requests always seem to succeed)
# --referer: Required by Microsoft servers to allow request
iso_download_link_html=$(curl --silent --max-time 30 --request POST --user-agent "$user_agent" --data "" --referer "$url" --header "Accept:" --max-filesize 100K --fail --proto =https --tlsv1.2 --http1.1 -- "https://www.microsoft.com/en-US/api/controls/contentinclude/html?pageId=6e2a1789-ef16-4f27-a296-74ef7ef5d96b&host=www.microsoft.com&segments=software-download,$url_segment_parameter&query=&action=GetProductDownloadLinksBySku&sessionId=$session_id&skuId=$sku_id&language=English&sdVersion=2")

if ! [ "$iso_download_link_html" ]; then
iso_url="https://www.microsoft.com/software-download-connector/api/GetProductDownloadLinksBySku?profile=$profile&ProductEditionId=undefined&SKU=$sku_id&friendlyFileName=undefined&Locale=en-US&sessionID=$session_id"
iso_json=$(curl --silent --max-time 30 --request GET --user-agent "$user_agent" --referer "$url" --header "Accept:" --max-filesize 100K --fail --proto =https --tlsv1.2 --http1.1 -- "$iso_url")

if ! [ "$iso_json" ]; then
# This should only happen if there's been some change to how this API works
error "Microsoft servers gave us an empty response to our request for an automated download."
return 1
fi

if echo "$iso_download_link_html" | grep -q "We are unable to complete your request at this time."; then
if echo "$iso_json" | grep -q "Sentinel marked this request as rejected."; then
error "Microsoft blocked the automated download request based on your IP address."
return 1
fi

# Filter for 64-bit ISO download URL
# sed: HTML decode "&" character
# tr: Filter for only alphanumerics or punctuation
iso_download_link=$(echo "$iso_download_link_html" | grep -o "https://software.download.prss.microsoft.com.*IsoX64" | cut -d '"' -f 1 | sed 's/&amp;/\&/g' | tr -cd '[:alnum:][:punct:]')
if echo "$iso_json" | grep -q "We are unable to complete your request at this time."; then
error "Microsoft blocked the automated download request based on your IP address."
return 1
fi

if ! [ "$iso_download_link" ]; then
# This should only happen if there's been some change to the download endpoint web address
{ iso_download_link=$(echo "$iso_json" | jq --argjson TYPE "$download_type" -r '.ProductDownloadOptions[] | select(.DownloadType==$TYPE).Uri') 2>/dev/null; rc=$?; } || :

if [ -z "$iso_download_link" ] || [[ "${iso_download_link,,}" == "null" ]] || (( rc != 0 )); then
error "Microsoft servers gave us no download link to our request for an automated download!"
info "Response: $iso_json"
return 1
fi

@@ -282,8 +284,8 @@ download_windows_eval() {
[[ "$DEBUG" == [Yy1]* ]] && echo "Found download link: $iso_download_link"

# Follow redirect so proceeding log message is useful
# This is a request we make this Fido doesn't
# We don't need to set "--max-filesize" here because this is a HEAD request and the output is to /dev/null anyway
# This is a request we make that Fido doesn't

iso_download_link=$(curl --silent --max-time 30 --user-agent "$user_agent" --location --output /dev/null --silent --write-out "%{url_effective}" --head --fail --proto =https --tlsv1.2 --http1.1 -- "$iso_download_link") || {
# This should only happen if the Microsoft servers are down
handle_curl_error $?
@@ -316,7 +318,7 @@ getWindows() {
esac

case "${version,,}" in
"win11${PLATFORM,,}" ) ;;
"win11${PLATFORM,,}" ) ;;
"win11${PLATFORM,,}-enterprise-iot"* ) ;;
"win11${PLATFORM,,}-enterprise-ltsc"* ) ;;
* )
@@ -327,7 +329,7 @@ getWindows() {
esac

case "${version,,}" in
"win81${PLATFORM,,}" | "win10${PLATFORM,,}" | "win11${PLATFORM,,}" )
"win10${PLATFORM,,}" | "win11${PLATFORM,,}" )
download_windows "$version" "$lang" "$edition" && return 0
;;
"win11${PLATFORM,,}-enterprise"* | "win10${PLATFORM,,}-enterprise"* )