Skip to content

Commit

Permalink
Fix broken object storage playlist on file removal
Browse files Browse the repository at this point in the history
  • Loading branch information
@Chocobozzz
Chocobozzz committed Aug 19, 2024
1 parent bd60f17 commit b2bb45c
Show file tree
Hide file tree
Showing 4 changed files with 162 additions and 134 deletions.
260 changes: 144 additions & 116 deletions packages/tests/src/api/videos/video-files.ts
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */

import { expect } from 'chai'
import { HttpStatusCode } from '@peertube/peertube-models'
import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils'
import {
cleanupTests,
createMultipleServers,
doubleFollow,
makeRawRequest,
ObjectStorageCommand,
PeerTubeServer,
setAccessTokensToServers,
waitJobs
} from '@peertube/peertube-server-commands'
import { expect } from 'chai'

describe('Test videos files', function () {
let servers: PeerTubeServer[]
Expand All @@ -28,173 +30,199 @@ describe('Test videos files', function () {
await servers[0].config.enableTranscoding({ hls: true, webVideo: true, resolutions: 'max' })
})

describe('When deleting all files', function () {
let validId1: string
let validId2: string
function runTests (objectStorage?: ObjectStorageCommand) {

before(async function () {
this.timeout(360_000)
describe('When deleting all files', function () {
let validId1: string
let validId2: string

{
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
validId1 = uuid
}
before(async function () {
this.timeout(360_000)

{
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 2' })
validId2 = uuid
}
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
validId1 = uuid
}

await waitJobs(servers)
})
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'video 2' })
validId2 = uuid
}

it('Should delete web video files', async function () {
this.timeout(30_000)
await waitJobs(servers)
})

await servers[0].videos.removeAllWebVideoFiles({ videoId: validId1 })
it('Should delete web video files', async function () {
this.timeout(30_000)

await waitJobs(servers)
await servers[0].videos.removeAllWebVideoFiles({ videoId: validId1 })

for (const server of servers) {
const video = await server.videos.get({ id: validId1 })
await waitJobs(servers)

expect(video.files).to.have.lengthOf(0)
expect(video.streamingPlaylists).to.have.lengthOf(1)
}
})
for (const server of servers) {
const video = await server.videos.get({ id: validId1 })

expect(video.files).to.have.lengthOf(0)
expect(video.streamingPlaylists).to.have.lengthOf(1)
}
})

it('Should delete HLS files', async function () {
this.timeout(30_000)
it('Should delete HLS files', async function () {
this.timeout(30_000)

await servers[0].videos.removeHLSPlaylist({ videoId: validId2 })
await servers[0].videos.removeHLSPlaylist({ videoId: validId2 })

await waitJobs(servers)
await waitJobs(servers)

for (const server of servers) {
const video = await server.videos.get({ id: validId2 })
for (const server of servers) {
const video = await server.videos.get({ id: validId2 })

expect(video.files).to.have.length.above(0)
expect(video.streamingPlaylists).to.have.lengthOf(0)
}
expect(video.files).to.have.length.above(0)
expect(video.streamingPlaylists).to.have.lengthOf(0)
}
})
})
})

describe('When deleting a specific file', function () {
let webVideoId: string
let hlsId: string
describe('When deleting a specific file', function () {
let webVideoId: string
let hlsId: string

before(async function () {
this.timeout(120_000)
before(async function () {
this.timeout(120_000)

{
const { uuid } = await servers[0].videos.quickUpload({ name: 'web-video' })
webVideoId = uuid
}
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'web-video' })
webVideoId = uuid
}

{
const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' })
hlsId = uuid
}
{
const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' })
hlsId = uuid
}

await waitJobs(servers)
})
await waitJobs(servers)
})

it('Shoulde delete a web video file', async function () {
this.timeout(30_000)
it('Shoulde delete a web video file', async function () {
this.timeout(30_000)

const video = await servers[0].videos.get({ id: webVideoId })
const files = video.files
const video = await servers[0].videos.get({ id: webVideoId })
const files = video.files

const toDelete = files[0]
await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: toDelete.id })
const toDelete = files[0]
await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: toDelete.id })

await waitJobs(servers)
await waitJobs(servers)

for (const server of servers) {
const video = await server.videos.get({ id: webVideoId })
for (const server of servers) {
const video = await server.videos.get({ id: webVideoId })

expect(video.files).to.have.lengthOf(files.length - 1)
expect(video.files.find(f => f.resolution.id === toDelete.resolution.id)).to.not.exist
}
})
expect(video.files).to.have.lengthOf(files.length - 1)
expect(video.files.find(f => f.resolution.id === toDelete.resolution.id)).to.not.exist
}
})

it('Should delete all web video files', async function () {
this.timeout(30_000)
it('Should delete all web video files', async function () {
this.timeout(30_000)

const video = await servers[0].videos.get({ id: webVideoId })
const files = video.files
const video = await servers[0].videos.get({ id: webVideoId })
const files = video.files

for (const file of files) {
await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: file.id })
}
for (const file of files) {
await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: file.id })
}

await waitJobs(servers)
await waitJobs(servers)

for (const server of servers) {
const video = await server.videos.get({ id: webVideoId })
for (const server of servers) {
const video = await server.videos.get({ id: webVideoId })

expect(video.files).to.have.lengthOf(0)
}
})
expect(video.files).to.have.lengthOf(0)
}
})

it('Should delete a hls file', async function () {
this.timeout(30_000)
it('Should delete a hls file', async function () {
this.timeout(30_000)

const video = await servers[0].videos.get({ id: hlsId })
const files = video.streamingPlaylists[0].files
const toDelete = files[0]
const video = await servers[0].videos.get({ id: hlsId })
const files = video.streamingPlaylists[0].files
const toDelete = files[0]

await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: toDelete.id })
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: toDelete.id })

await waitJobs(servers)
await waitJobs(servers)

for (const server of servers) {
const video = await server.videos.get({ id: hlsId })
for (const server of servers) {
const video = await server.videos.get({ id: hlsId })

expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1)
expect(video.streamingPlaylists[0].files.find(f => f.resolution.id === toDelete.resolution.id)).to.not.exist
expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1)
expect(video.streamingPlaylists[0].files.find(f => f.resolution.id === toDelete.resolution.id)).to.not.exist

const { text } = await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
const m3u8Content = await servers[0].streamingPlaylists.get({ url: video.streamingPlaylists[0].playlistUrl })
await makeRawRequest({ url: video.streamingPlaylists[0].segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })

expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false
expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true
}
})
expect(m3u8Content.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false
expect(m3u8Content.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true
}
})

it('Should delete all hls files', async function () {
this.timeout(30_000)

const video = await servers[0].videos.get({ id: hlsId })
const files = video.streamingPlaylists[0].files

for (const file of files) {
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: file.id })
}

await waitJobs(servers)

it('Should delete all hls files', async function () {
this.timeout(30_000)
for (const server of servers) {
const video = await server.videos.get({ id: hlsId })

const video = await servers[0].videos.get({ id: hlsId })
const files = video.streamingPlaylists[0].files
expect(video.streamingPlaylists).to.have.lengthOf(0)
}
})

for (const file of files) {
await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: file.id })
}
it('Should not delete last file of a video', async function () {
this.timeout(60_000)

await waitJobs(servers)
const webVideoOnly = await servers[0].videos.get({ id: hlsId })
const hlsOnly = await servers[0].videos.get({ id: webVideoId })

for (const server of servers) {
const video = await server.videos.get({ id: hlsId })
for (let i = 0; i < 4; i++) {
await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[i].id })
await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[i].id })
}

expect(video.streamingPlaylists).to.have.lengthOf(0)
}
const expectedStatus = HttpStatusCode.BAD_REQUEST_400
await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[4].id, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[4].id, expectedStatus })
})
})
}

it('Should not delete last file of a video', async function () {
this.timeout(60_000)
describe('Using filesystem', function () {
runTests()
})

describe('Using object storage', function () {
if (areMockObjectStorageTestsDisabled()) return

const webVideoOnly = await servers[0].videos.get({ id: hlsId })
const hlsOnly = await servers[0].videos.get({ id: webVideoId })
before(async function () {
this.timeout(120000)

for (let i = 0; i < 4; i++) {
await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[i].id })
await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[i].id })
}
const configOverride = objectStorage.getDefaultMockConfig()
await objectStorage.prepareDefaultMockBuckets()

const expectedStatus = HttpStatusCode.BAD_REQUEST_400
await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[4].id, expectedStatus })
await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[4].id, expectedStatus })
await servers[0].kill()
await servers[0].run(configOverride)
})

const objectStorage = new ObjectStorageCommand()

runTests(objectStorage)
})

after(async function () {
Expand Down
26 changes: 14 additions & 12 deletions server/core/lib/hls.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers
import { sequelizeTypescript } from '../initializers/database.js'
import { VideoFileModel } from '../models/video/video-file.js'
import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist.js'
import { storeHLSFileFromFilename } from './object-storage/index.js'
import { storeHLSFileFromContent } from './object-storage/index.js'
import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths.js'
import { VideoPathManager } from './video-path-manager.js'

Expand Down Expand Up @@ -121,14 +121,17 @@ export function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingP
}
playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive)

const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename)
await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n')

logger.info('Updating %s master playlist file of video %s', masterPlaylistPath, video.uuid, lTags(video.uuid))
const masterPlaylistContent = masterPlaylists.join('\n') + '\n'

if (playlist.storage === FileStorage.OBJECT_STORAGE) {
playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename)
await remove(masterPlaylistPath)
playlist.playlistUrl = await storeHLSFileFromContent(playlist, playlist.playlistFilename, masterPlaylistContent)

logger.info(`Updated master playlist file of video ${video.uuid} to object storage ${playlist.playlistUrl}`, lTags(video.uuid))
} else {
const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename)
await writeFile(masterPlaylistPath, masterPlaylistContent)

logger.info(`Updated master playlist file ${masterPlaylistPath} of video ${video.uuid}`, lTags(video.uuid))
}

return playlist.save()
Expand Down Expand Up @@ -174,12 +177,11 @@ export function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingP
}
playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive)

const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
await outputJSON(outputPath, json)

if (playlist.storage === FileStorage.OBJECT_STORAGE) {
playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename)
await remove(outputPath)
playlist.segmentsSha256Url = await storeHLSFileFromContent(playlist, playlist.segmentsSha256Filename, JSON.stringify(json))
} else {
const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename)
await outputJSON(outputPath, json)
}

return playlist.save()
Expand Down
Loading

0 comments on commit b2bb45c

Please sign in to comment.