diff --git a/packages/tests/src/api/videos/video-files.ts b/packages/tests/src/api/videos/video-files.ts index a5f3abd001c..b455307a349 100644 --- a/packages/tests/src/api/videos/video-files.ts +++ b/packages/tests/src/api/videos/video-files.ts @@ -1,16 +1,18 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { expect } from 'chai' import { HttpStatusCode } from '@peertube/peertube-models' +import { areMockObjectStorageTestsDisabled } from '@peertube/peertube-node-utils' import { cleanupTests, createMultipleServers, doubleFollow, makeRawRequest, + ObjectStorageCommand, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@peertube/peertube-server-commands' +import { expect } from 'chai' describe('Test videos files', function () { let servers: PeerTubeServer[] @@ -28,173 +30,199 @@ describe('Test videos files', function () { await servers[0].config.enableTranscoding({ hls: true, webVideo: true, resolutions: 'max' }) }) - describe('When deleting all files', function () { - let validId1: string - let validId2: string + function runTests (objectStorage?: ObjectStorageCommand) { - before(async function () { - this.timeout(360_000) + describe('When deleting all files', function () { + let validId1: string + let validId2: string - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) - validId1 = uuid - } + before(async function () { + this.timeout(360_000) - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'video 2' }) - validId2 = uuid - } + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) + validId1 = uuid + } - await waitJobs(servers) - }) + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'video 2' }) + validId2 = uuid + } - it('Should delete web video files', async function () { - this.timeout(30_000) + await waitJobs(servers) + }) - await servers[0].videos.removeAllWebVideoFiles({ videoId: validId1 }) + it('Should delete web video files', async function () { + this.timeout(30_000) - await waitJobs(servers) + await servers[0].videos.removeAllWebVideoFiles({ videoId: validId1 }) - for (const server of servers) { - const video = await server.videos.get({ id: validId1 }) + await waitJobs(servers) - expect(video.files).to.have.lengthOf(0) - expect(video.streamingPlaylists).to.have.lengthOf(1) - } - }) + for (const server of servers) { + const video = await server.videos.get({ id: validId1 }) + + expect(video.files).to.have.lengthOf(0) + expect(video.streamingPlaylists).to.have.lengthOf(1) + } + }) - it('Should delete HLS files', async function () { - this.timeout(30_000) + it('Should delete HLS files', async function () { + this.timeout(30_000) - await servers[0].videos.removeHLSPlaylist({ videoId: validId2 }) + await servers[0].videos.removeHLSPlaylist({ videoId: validId2 }) - await waitJobs(servers) + await waitJobs(servers) - for (const server of servers) { - const video = await server.videos.get({ id: validId2 }) + for (const server of servers) { + const video = await server.videos.get({ id: validId2 }) - expect(video.files).to.have.length.above(0) - expect(video.streamingPlaylists).to.have.lengthOf(0) - } + expect(video.files).to.have.length.above(0) + expect(video.streamingPlaylists).to.have.lengthOf(0) + } + }) }) - }) - describe('When deleting a specific file', function () { - let webVideoId: string - let hlsId: string + describe('When deleting a specific file', function () { + let webVideoId: string + let hlsId: string - before(async function () { - this.timeout(120_000) + before(async function () { + this.timeout(120_000) - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'web-video' }) - webVideoId = uuid - } + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'web-video' }) + webVideoId = uuid + } - { - const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' }) - hlsId = uuid - } + { + const { uuid } = await servers[0].videos.quickUpload({ name: 'hls' }) + hlsId = uuid + } - await waitJobs(servers) - }) + await waitJobs(servers) + }) - it('Shoulde delete a web video file', async function () { - this.timeout(30_000) + it('Shoulde delete a web video file', async function () { + this.timeout(30_000) - const video = await servers[0].videos.get({ id: webVideoId }) - const files = video.files + const video = await servers[0].videos.get({ id: webVideoId }) + const files = video.files - const toDelete = files[0] - await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: toDelete.id }) + const toDelete = files[0] + await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: toDelete.id }) - await waitJobs(servers) + await waitJobs(servers) - for (const server of servers) { - const video = await server.videos.get({ id: webVideoId }) + for (const server of servers) { + const video = await server.videos.get({ id: webVideoId }) - expect(video.files).to.have.lengthOf(files.length - 1) - expect(video.files.find(f => f.resolution.id === toDelete.resolution.id)).to.not.exist - } - }) + expect(video.files).to.have.lengthOf(files.length - 1) + expect(video.files.find(f => f.resolution.id === toDelete.resolution.id)).to.not.exist + } + }) - it('Should delete all web video files', async function () { - this.timeout(30_000) + it('Should delete all web video files', async function () { + this.timeout(30_000) - const video = await servers[0].videos.get({ id: webVideoId }) - const files = video.files + const video = await servers[0].videos.get({ id: webVideoId }) + const files = video.files - for (const file of files) { - await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: file.id }) - } + for (const file of files) { + await servers[0].videos.removeWebVideoFile({ videoId: webVideoId, fileId: file.id }) + } - await waitJobs(servers) + await waitJobs(servers) - for (const server of servers) { - const video = await server.videos.get({ id: webVideoId }) + for (const server of servers) { + const video = await server.videos.get({ id: webVideoId }) - expect(video.files).to.have.lengthOf(0) - } - }) + expect(video.files).to.have.lengthOf(0) + } + }) - it('Should delete a hls file', async function () { - this.timeout(30_000) + it('Should delete a hls file', async function () { + this.timeout(30_000) - const video = await servers[0].videos.get({ id: hlsId }) - const files = video.streamingPlaylists[0].files - const toDelete = files[0] + const video = await servers[0].videos.get({ id: hlsId }) + const files = video.streamingPlaylists[0].files + const toDelete = files[0] - await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: toDelete.id }) + await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: toDelete.id }) - await waitJobs(servers) + await waitJobs(servers) - for (const server of servers) { - const video = await server.videos.get({ id: hlsId }) + for (const server of servers) { + const video = await server.videos.get({ id: hlsId }) - expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1) - expect(video.streamingPlaylists[0].files.find(f => f.resolution.id === toDelete.resolution.id)).to.not.exist + expect(video.streamingPlaylists[0].files).to.have.lengthOf(files.length - 1) + expect(video.streamingPlaylists[0].files.find(f => f.resolution.id === toDelete.resolution.id)).to.not.exist - const { text } = await makeRawRequest({ url: video.streamingPlaylists[0].playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) + const m3u8Content = await servers[0].streamingPlaylists.get({ url: video.streamingPlaylists[0].playlistUrl }) + await makeRawRequest({ url: video.streamingPlaylists[0].segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) - expect(text.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false - expect(text.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true - } - }) + expect(m3u8Content.includes(`-${toDelete.resolution.id}.m3u8`)).to.be.false + expect(m3u8Content.includes(`-${video.streamingPlaylists[0].files[0].resolution.id}.m3u8`)).to.be.true + } + }) + + it('Should delete all hls files', async function () { + this.timeout(30_000) + + const video = await servers[0].videos.get({ id: hlsId }) + const files = video.streamingPlaylists[0].files + + for (const file of files) { + await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: file.id }) + } + + await waitJobs(servers) - it('Should delete all hls files', async function () { - this.timeout(30_000) + for (const server of servers) { + const video = await server.videos.get({ id: hlsId }) - const video = await servers[0].videos.get({ id: hlsId }) - const files = video.streamingPlaylists[0].files + expect(video.streamingPlaylists).to.have.lengthOf(0) + } + }) - for (const file of files) { - await servers[0].videos.removeHLSFile({ videoId: hlsId, fileId: file.id }) - } + it('Should not delete last file of a video', async function () { + this.timeout(60_000) - await waitJobs(servers) + const webVideoOnly = await servers[0].videos.get({ id: hlsId }) + const hlsOnly = await servers[0].videos.get({ id: webVideoId }) - for (const server of servers) { - const video = await server.videos.get({ id: hlsId }) + for (let i = 0; i < 4; i++) { + await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[i].id }) + await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[i].id }) + } - expect(video.streamingPlaylists).to.have.lengthOf(0) - } + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[4].id, expectedStatus }) + await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[4].id, expectedStatus }) + }) }) + } - it('Should not delete last file of a video', async function () { - this.timeout(60_000) + describe('Using filesystem', function () { + runTests() + }) + + describe('Using object storage', function () { + if (areMockObjectStorageTestsDisabled()) return - const webVideoOnly = await servers[0].videos.get({ id: hlsId }) - const hlsOnly = await servers[0].videos.get({ id: webVideoId }) + before(async function () { + this.timeout(120000) - for (let i = 0; i < 4; i++) { - await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[i].id }) - await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[i].id }) - } + const configOverride = objectStorage.getDefaultMockConfig() + await objectStorage.prepareDefaultMockBuckets() - const expectedStatus = HttpStatusCode.BAD_REQUEST_400 - await servers[0].videos.removeWebVideoFile({ videoId: webVideoOnly.id, fileId: webVideoOnly.files[4].id, expectedStatus }) - await servers[0].videos.removeHLSFile({ videoId: hlsOnly.id, fileId: hlsOnly.streamingPlaylists[0].files[4].id, expectedStatus }) + await servers[0].kill() + await servers[0].run(configOverride) }) + + const objectStorage = new ObjectStorageCommand() + + runTests(objectStorage) }) after(async function () { diff --git a/server/core/lib/hls.ts b/server/core/lib/hls.ts index 61c9608e3f7..87f503eeadf 100644 --- a/server/core/lib/hls.ts +++ b/server/core/lib/hls.ts @@ -17,7 +17,7 @@ import { P2P_MEDIA_LOADER_PEER_VERSION, REQUEST_TIMEOUTS } from '../initializers import { sequelizeTypescript } from '../initializers/database.js' import { VideoFileModel } from '../models/video/video-file.js' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist.js' -import { storeHLSFileFromFilename } from './object-storage/index.js' +import { storeHLSFileFromContent } from './object-storage/index.js' import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename, getHlsResolutionPlaylistFilename } from './paths.js' import { VideoPathManager } from './video-path-manager.js' @@ -121,14 +121,17 @@ export function updateMasterHLSPlaylist (video: MVideo, playlistArg: MStreamingP } playlist.playlistFilename = generateHLSMasterPlaylistFilename(video.isLive) - const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename) - await writeFile(masterPlaylistPath, masterPlaylists.join('\n') + '\n') - - logger.info('Updating %s master playlist file of video %s', masterPlaylistPath, video.uuid, lTags(video.uuid)) + const masterPlaylistContent = masterPlaylists.join('\n') + '\n' if (playlist.storage === FileStorage.OBJECT_STORAGE) { - playlist.playlistUrl = await storeHLSFileFromFilename(playlist, playlist.playlistFilename) - await remove(masterPlaylistPath) + playlist.playlistUrl = await storeHLSFileFromContent(playlist, playlist.playlistFilename, masterPlaylistContent) + + logger.info(`Updated master playlist file of video ${video.uuid} to object storage ${playlist.playlistUrl}`, lTags(video.uuid)) + } else { + const masterPlaylistPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.playlistFilename) + await writeFile(masterPlaylistPath, masterPlaylistContent) + + logger.info(`Updated master playlist file ${masterPlaylistPath} of video ${video.uuid}`, lTags(video.uuid)) } return playlist.save() @@ -174,12 +177,11 @@ export function updateSha256VODSegments (video: MVideo, playlistArg: MStreamingP } playlist.segmentsSha256Filename = generateHlsSha256SegmentsFilename(video.isLive) - const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) - await outputJSON(outputPath, json) - if (playlist.storage === FileStorage.OBJECT_STORAGE) { - playlist.segmentsSha256Url = await storeHLSFileFromFilename(playlist, playlist.segmentsSha256Filename) - await remove(outputPath) + playlist.segmentsSha256Url = await storeHLSFileFromContent(playlist, playlist.segmentsSha256Filename, JSON.stringify(json)) + } else { + const outputPath = VideoPathManager.Instance.getFSHLSOutputPath(video, playlist.segmentsSha256Filename) + await outputJSON(outputPath, json) } return playlist.save() diff --git a/server/core/lib/object-storage/object-storage-helpers.ts b/server/core/lib/object-storage/object-storage-helpers.ts index 1194772036c..758d064eb03 100644 --- a/server/core/lib/object-storage/object-storage-helpers.ts +++ b/server/core/lib/object-storage/object-storage-helpers.ts @@ -62,14 +62,13 @@ async function storeObject (options: { async function storeContent (options: { content: string - inputPath: string objectStorageKey: string bucketInfo: BucketInfo isPrivate: boolean }): Promise { - const { content, objectStorageKey, bucketInfo, inputPath, isPrivate } = options + const { content, objectStorageKey, bucketInfo, isPrivate } = options - logger.debug('Uploading %s content to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) + logger.debug('Uploading %s content to %s%s in bucket %s', content, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) return uploadToStorage({ objectStorageKey, content, bucketInfo, isPrivate }) } diff --git a/server/core/lib/object-storage/videos.ts b/server/core/lib/object-storage/videos.ts index 06e8b39de5f..009949256ca 100644 --- a/server/core/lib/object-storage/videos.ts +++ b/server/core/lib/object-storage/videos.ts @@ -49,11 +49,10 @@ export function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: s }) } -export function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) { +export function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, pathOrFilename: string, content: string) { return storeContent({ content, - inputPath: path, - objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), + objectStorageKey: generateHLSObjectStorageKey(playlist, basename(pathOrFilename)), bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, isPrivate: playlist.Video.hasPrivateStaticPath() })