Skip to content

Commit

Permalink
✅ chore(deps): Update dependencies to latest versions. Adds NewPipeEx…
Browse files Browse the repository at this point in the history 
…tractor for improved signature decoding
  • Loading branch information
@maxrave-dev
maxrave-dev committed Jan 18, 2025
1 parent 8325e55 commit fdd97d0
Show file tree
Hide file tree
Showing 10 changed files with 297 additions and 168 deletions.
12 changes: 7 additions & 5 deletions gradle/libs.versions.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ android = "8.8.0"
kotlin = "2.1.0"
serialization = "2.1.0"
ksp = "2.1.0-1.0.29"
compose-bom = "2024.12.01"
compose-bom = "2025.01.00"
constraintlayout-compose = "1.1.0"
activity-compose = "1.9.3"
activity-compose = "1.10.0"
lifecycle-viewmodel-compose = "2.8.7"
lifecycle-runtime-ktx = "2.8.7"
core-ktx = "1.15.0"
Expand All @@ -30,16 +30,16 @@ androidx-junit = "1.2.1"
espresso-core = "3.6.1"
room = "2.6.1"
legacy-support-v4 = "1.0.0"
coroutines-android = "1.9.0"
coroutines-guava = "1.9.0"
coroutines-android = "1.10.1"
coroutines-guava = "1.10.1"
navigation = "2.8.5"
gson = "2.10.1"
coil3 = "3.0.4"
kmpalette = "3.1.0"
easypermissions = "3.0.0"
preference-ktx = "1.2.1"
fragment-ktx = "1.8.5"
datastore-preferences = "1.1.1"
datastore-preferences = "1.1.2"
swiperefreshlayout = "1.2.0-alpha01"
insetter = "0.6.1"
shimmer = "0.5.0"
Expand Down Expand Up @@ -145,6 +145,8 @@ kotlin-reflect = { module = "org.jetbrains.kotlin:kotlin-reflect" }
ksoup-html = { group = "com.mohamedrejeb.ksoup", name = "ksoup-html", version.ref = "ksoup" }
ksoup-entities = { group = "com.mohamedrejeb.ksoup", name = "ksoup-entities", version.ref = "ksoup" }
store = { module = "org.mobilenativefoundation.store:store5", version.ref = "store" }
# Use NewPipeExtractor from the Outertune project (Thanks to @DD3Boh)
newpipe-extractor = { group = "com.github.gechoto", name = "NewPipeExtractor", version = "00cf7dad9e49933c7faaa382c9c7b1e7afb28a7d" }

#Koin
koin-bom = { module = "io.insert-koin:koin-bom", version.ref = "koin-bom" }
Expand Down
3 changes: 3 additions & 0 deletions kotlinYtmusicScraper/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ dependencies {
testImplementation(libs.junit)
androidTestImplementation(libs.androidx.junit)
androidTestImplementation(libs.espresso.core)
coreLibraryDesugaring(libs.desugaring)

implementation(libs.ktor.client.core)
implementation(libs.ktor.client.cio)
Expand All @@ -136,6 +137,8 @@ dependencies {
implementation(libs.ktor.serialization.kotlinx.xml)
implementation(libs.ktor.serialization.kotlinx.protobuf)

implementation(libs.newpipe.extractor)

implementation(libs.brotli.dec)

implementation(libs.kotlin.reflect)
Expand Down
239 changes: 158 additions & 81 deletions kotlinYtmusicScraper/src/main/java/com/maxrave/kotlinytmusicscraper/YouTube.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,19 +64,25 @@ import com.maxrave.kotlinytmusicscraper.parser.getPlaylistContinuation
import com.maxrave.kotlinytmusicscraper.parser.getReloadParams
import com.maxrave.kotlinytmusicscraper.parser.getSuggestionSongItems
import com.maxrave.kotlinytmusicscraper.parser.hasReloadParams
import com.maxrave.kotlinytmusicscraper.utils.CustomNewPipeDownloader
import com.mohamedrejeb.ksoup.html.parser.KsoupHtmlHandler
import com.mohamedrejeb.ksoup.html.parser.KsoupHtmlParser
import io.ktor.client.call.body
import io.ktor.client.engine.ProxyBuilder
import io.ktor.client.engine.http
import io.ktor.client.statement.bodyAsText
import io.ktor.http.URLBuilder
import io.ktor.http.parseQueryString
import kotlinx.coroutines.delay
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonPrimitive
import okhttp3.Interceptor
import org.json.JSONArray
import org.schabi.newpipe.extractor.NewPipe
import org.schabi.newpipe.extractor.exceptions.ParsingException
import org.schabi.newpipe.extractor.services.youtube.YoutubeJavaScriptPlayerManager
import java.io.File
import kotlin.random.Random

Expand All @@ -102,6 +108,7 @@ private fun List<PipedResponse.AudioStream>.toListFormat(): List<PlayerResponse.
audioChannels = 0,
loudnessDb = 0.0,
lastModified = 0,
signatureCipher = null,
),
)
}
Expand All @@ -119,6 +126,12 @@ private fun List<PipedResponse.AudioStream>.toListFormat(): List<PlayerResponse.
*/
class YouTube {
private val ytMusic = Ytmusic()
private val newPipeDownloader = CustomNewPipeDownloader(ytMusic.proxy)

// Init NewPipe
init {
NewPipe.init(newPipeDownloader)
}

var cachePath: File?
get() = ytMusic.cachePath
Expand Down Expand Up @@ -191,6 +204,7 @@ class YouTube {
*/
fun removeProxy() {
ytMusic.proxy = null
newPipeDownloader.updateProxy(null)
}

/**
Expand All @@ -205,6 +219,7 @@ class YouTube {
if (isHttp) ProxyBuilder.http("$host:$port") else ProxyBuilder.socks(host, port)
}.onSuccess {
ytMusic.proxy = it
newPipeDownloader.updateProxy(it)
}.onFailure {
it.printStackTrace()
}
Expand Down Expand Up @@ -1161,7 +1176,6 @@ class YouTube {
playlistId: String? = null,
): Result<Triple<String?, PlayerResponse, MediaType>> =
runCatching {
val (tempCookie, visitorData, playbackTracking) = getVisitorData(videoId, playlistId)
val cpn =
(1..16)
.map {
Expand All @@ -1172,96 +1186,131 @@ class YouTube {
),
]
}.joinToString("")
val now = System.currentTimeMillis()
val poToken =
if (now < poTokenObject.second) {
println("Use saved PoToken")
poTokenObject.first
} else {
ytMusic
.createPoTokenChallenge()
.bodyAsText()
.let { challenge ->
val listChallenge = poTokenJsonDeserializer.decodeFromString<List<String?>>(challenge)
listChallenge.filterIsInstance<String>().firstOrNull()
}?.let { poTokenChallenge ->
ytMusic.generatePoToken(poTokenChallenge).bodyAsText().getPoToken().also { poToken ->
if (poToken != null) {
poTokenObject = Pair(poToken, now + 21600000)
val sigTimestamp = getSignatureTimestamp(videoId)
val sigResponse = ytMusic.player(WEB_REMIX, videoId, playlistId, cpn, signatureTimestamp = sigTimestamp).body<PlayerResponse>()
val decodedSigResponse =
sigResponse.copy(
streamingData =
sigResponse.streamingData?.copy(
formats =
sigResponse.streamingData.formats?.map { format ->
format.copy(
url = format.signatureCipher?.let { decodeSignatureCipher(videoId, it) },
)
},
adaptiveFormats =
sigResponse.streamingData.adaptiveFormats.map { adaptiveFormats ->
adaptiveFormats.copy(
url = adaptiveFormats.signatureCipher?.let { decodeSignatureCipher(videoId, it) },
)
},
),
)
val listUrlSig =
(
decodedSigResponse.streamingData
?.adaptiveFormats
?.mapNotNull { it.url }
?.toMutableList() ?: mutableListOf()
).apply {
decodedSigResponse.streamingData
?.formats
?.mapNotNull { it.url }
?.let { addAll(it) }
}
if (listUrlSig.isEmpty()) {
val (tempCookie, visitorData, playbackTracking) = getVisitorData(videoId, playlistId)
val now = System.currentTimeMillis()
val poToken =
if (now < poTokenObject.second) {
println("Use saved PoToken")
poTokenObject.first
} else {
ytMusic
.createPoTokenChallenge()
.bodyAsText()
.let { challenge ->
val listChallenge = poTokenJsonDeserializer.decodeFromString<List<String?>>(challenge)
listChallenge.filterIsInstance<String>().firstOrNull()
}?.let { poTokenChallenge ->
ytMusic.generatePoToken(poTokenChallenge).bodyAsText().getPoToken().also { poToken ->
if (poToken != null) {
poTokenObject = Pair(poToken, now + 21600000)
}
}
}
}
println("PoToken $poToken")
val playerResponse = ytMusic.noLogInPlayer(videoId, tempCookie, visitorData, poToken ?: "").body<PlayerResponse>()
println("Player Response $playerResponse")
println("Thumbnails " + playerResponse.videoDetails?.thumbnail)
println("Player Response status: ${playerResponse.playabilityStatus.status}")
val firstThumb =
playerResponse.videoDetails
?.thumbnail
?.thumbnails
?.firstOrNull()
val thumbnails =
if (firstThumb?.height == firstThumb?.width && firstThumb != null) MediaType.Song else MediaType.Video
val formatList = playerResponse.streamingData?.formats?.map { Pair(it.itag, it.isAudio) }
println("Player Response formatList $formatList")
val adaptiveFormatsList = playerResponse.streamingData?.adaptiveFormats?.map { Pair(it.itag, it.isAudio) }
println("Player Response adaptiveFormat $adaptiveFormatsList")

if (playerResponse.playabilityStatus.status == "OK" && (formatList != null || adaptiveFormatsList != null)) {
return@runCatching Triple(
cpn,
playerResponse.copy(
videoDetails = playerResponse.videoDetails?.copy(),
playbackTracking = playbackTracking ?: playerResponse.playbackTracking,
),
thumbnails,
)
} else {
for (instance in listPipedInstances) {
try {
val piped = ytMusic.pipedStreams(videoId, instance).body<PipedResponse>()
val audioStreams = piped.audioStreams
val videoStreams = piped.videoStreams
val stream = audioStreams + videoStreams
return@runCatching Triple(
null,
playerResponse.copy(
streamingData =
PlayerResponse.StreamingData(
formats = stream.toListFormat(),
adaptiveFormats = stream.toListFormat(),
expiresInSeconds = 0,
),
videoDetails = playerResponse.videoDetails?.copy(),
playbackTracking = playbackTracking ?: playerResponse.playbackTracking,
),
thumbnails,
)
} catch (e: Exception) {
e.printStackTrace()
continue
}
}
}
println("PoToken $poToken")
val playerResponse = ytMusic.noLogInPlayer(videoId, tempCookie, visitorData, poToken ?: "").body<PlayerResponse>()
// try {
// println("Start logged in request")
// ytMusic.player(
// IOS,
// videoId,
// playlistId,
// cpn,
// poToken
// ).body<PlayerResponse>().also {
// if (it.playabilityStatus.status != "OK") throw Exception(it.playabilityStatus.status)
// }
// } catch (e: Exception) {
// println("Player Response Error $e")
// println("Start no logged in request")
// ytMusic.noLogInPlayer(videoId, tempCookie, visitorData, poToken ?: "").body<PlayerResponse>()
// }
println("Player Response $playerResponse")
println("Thumbnails " + playerResponse.videoDetails?.thumbnail)
println("Player Response status: ${playerResponse.playabilityStatus.status}")
val firstThumb =
playerResponse.videoDetails
?.thumbnail
?.thumbnails
?.firstOrNull()
val thumbnails =
if (firstThumb?.height == firstThumb?.width && firstThumb != null) MediaType.Song else MediaType.Video
val formatList = playerResponse.streamingData?.formats?.map { Pair(it.itag, it.isAudio) }
println("Player Response formatList $formatList")
val adaptiveFormatsList = playerResponse.streamingData?.adaptiveFormats?.map { Pair(it.itag, it.isAudio) }
println("Player Response adaptiveFormat $adaptiveFormatsList")

if (playerResponse.playabilityStatus.status == "OK" && (formatList != null || adaptiveFormatsList != null)) {
throw Exception(playerResponse.playabilityStatus.status ?: "Unknown error")
} else {
val firstThumb =
decodedSigResponse.videoDetails
?.thumbnail
?.thumbnails
?.firstOrNull()
val thumbnails =
if (firstThumb?.height == firstThumb?.width && firstThumb != null) MediaType.Song else MediaType.Video
return@runCatching Triple(
cpn,
playerResponse.copy(
videoDetails = playerResponse.videoDetails?.copy(),
playbackTracking = playbackTracking ?: playerResponse.playbackTracking,
decodedSigResponse.copy(
videoDetails = decodedSigResponse.videoDetails?.copy(),
playbackTracking = decodedSigResponse.playbackTracking,
),
thumbnails,
)
} else {
for (instance in listPipedInstances) {
try {
val piped = ytMusic.pipedStreams(videoId, instance).body<PipedResponse>()
val audioStreams = piped.audioStreams
val videoStreams = piped.videoStreams
val stream = audioStreams + videoStreams
return@runCatching Triple(
null,
playerResponse.copy(
streamingData =
PlayerResponse.StreamingData(
formats = stream.toListFormat(),
adaptiveFormats = stream.toListFormat(),
expiresInSeconds = 0,
),
videoDetails = playerResponse.videoDetails?.copy(),
playbackTracking = playbackTracking ?: playerResponse.playbackTracking,
),
thumbnails,
)
} catch (e: Exception) {
e.printStackTrace()
continue
}
}
}
throw Exception(playerResponse.playabilityStatus.status ?: "Unknown error")
}

suspend fun updateWatchTime(
Expand Down Expand Up @@ -1683,6 +1732,34 @@ class YouTube {
ytMusic.removeFromLiked(mediaId).status.value
}

/**
* NewPipeExtractor implement
*/
private fun getSignatureTimestamp(videoId: String): Int? =
try {
YoutubeJavaScriptPlayerManager.getSignatureTimestamp(videoId)
} catch (e: Exception) {
e.printStackTrace()
null
}

private fun decodeSignatureCipher(
videoId: String,
signatureCipher: String,
): String? =
try {
val params = parseQueryString(signatureCipher)
val obfuscatedSignature = params["s"] ?: throw ParsingException("Could not parse cipher signature")
val signatureParam = params["sp"] ?: throw ParsingException("Could not parse cipher signature parameter")
val url = params["url"]?.let { URLBuilder(it) } ?: throw ParsingException("Could not parse cipher url")
url.parameters[signatureParam] = YoutubeJavaScriptPlayerManager.deobfuscateSignature(videoId, obfuscatedSignature)
print("URL $url")
YoutubeJavaScriptPlayerManager.getUrlWithThrottlingParameterDeobfuscated(videoId, url.toString())
} catch (e: Exception) {
e.printStackTrace()
null
}

companion object {
const val MAX_GET_QUEUE_SIZE = 1000

Expand Down
Loading

0 comments on commit fdd97d0

Please sign in to comment.