Fix CI tests

Chocobozzz · Jun 29, 2023 · c37e305 · c37e305
1 parent 881958d
commit c37e305
Show file tree

Hide file tree

Showing 41 changed files with 133 additions and 72 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -46,6 +46,7 @@ jobs:
       PGHOST: localhost
       NODE_PENDING_JOB_WAIT: 250
       ENABLE_OBJECT_STORAGE_TESTS: true
+      ENABLE_FFMPEG_THUMBNAIL_PIXEL_COMPARISON_TESTS: true
       OBJECT_STORAGE_SCALEWAY_KEY_ID: ${{ secrets.OBJECT_STORAGE_SCALEWAY_KEY_ID }}
       OBJECT_STORAGE_SCALEWAY_ACCESS_KEY: ${{ secrets.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY }}
       YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN: ${{ secrets.GITHUB_TOKEN }}

diff --git a/server/controllers/api/videos/upload.ts b/server/controllers/api/videos/upload.ts
@@ -111,7 +111,7 @@ async function addVideoLegacy (req: express.Request, res: express.Response) {
 async function addVideoResumable (req: express.Request, res: express.Response) {
   const videoPhysicalFile = res.locals.videoFileResumable
   const videoInfo = videoPhysicalFile.metadata
-  const files = { previewfile: videoInfo.previewfile }
+  const files = { previewfile: videoInfo.previewfile, thumbnailfile: videoInfo.thumbnailfile }
 
   const response = await addVideo({ req, res, videoPhysicalFile, videoInfo, files })
   await Redis.Instance.setUploadSession(req.query.upload_id, response)

diff --git a/server/lib/object-storage/pre-signed-urls.ts b/server/lib/object-storage/pre-signed-urls.ts
@@ -0,0 +1,41 @@
+import { GetObjectCommand } from '@aws-sdk/client-s3'
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner'
+import { CONFIG } from '@server/initializers/config'
+import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models'
+import { generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
+import { buildKey, getClient } from './shared'
+
+export function generateWebVideoPresignedUrl (options: {
+  file: MVideoFile
+  downloadFilename: string
+}) {
+  const { file, downloadFilename } = options
+
+  const key = generateWebTorrentObjectStorageKey(file.filename)
+
+  const command = new GetObjectCommand({
+    Bucket: CONFIG.OBJECT_STORAGE.VIDEOS.BUCKET_NAME,
+    Key: buildKey(key, CONFIG.OBJECT_STORAGE.VIDEOS),
+    ResponseContentDisposition: `attachment; filename=${downloadFilename}`
+  })
+
+  return getSignedUrl(getClient(), command, { expiresIn: 3600 * 24 })
+}
+
+export function generateHLSFilePresignedUrl (options: {
+  streamingPlaylist: MStreamingPlaylistVideo
+  file: MVideoFile
+  downloadFilename: string
+}) {
+  const { streamingPlaylist, file, downloadFilename } = options
+
+  const key = generateHLSObjectStorageKey(streamingPlaylist, file.filename)
+
+  const command = new GetObjectCommand({
+    Bucket: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME,
+    Key: buildKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS),
+    ResponseContentDisposition: `attachment; filename=${downloadFilename}`
+  })
+
+  return getSignedUrl(getClient(), command, { expiresIn: 3600 * 24 })
+}
diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts
@@ -223,10 +223,12 @@ const videosAddResumableInitValidator = getCommonVideoEditAttributes().concat([
 
     if (!isValidPasswordProtectedPrivacy(req, res)) return cleanup()
 
-    // multer required unsetting the Content-Type, now we can set it for node-uploadx
+    // Multer required unsetting the Content-Type, now we can set it for node-uploadx
     req.headers['content-type'] = 'application/json; charset=utf-8'
-    // place previewfile in metadata so that uploadx saves it in .META
+
+    // Place thumbnail/previewfile in metadata so that uploadx saves it in .META
     if (req.files?.['previewfile']) req.body.previewfile = req.files['previewfile']
+    if (req.files?.['thumbnailfile']) req.body.thumbnailfile = req.files['thumbnailfile']
 
     return next()
   }

diff --git a/server/tests/api/check-params/live.ts b/server/tests/api/check-params/live.ts
@@ -194,7 +194,7 @@ describe('Test video lives API validator', function () {
     it('Should fail with a big thumbnail file', async function () {
       const fields = baseCorrectParams
       const attaches = {
-        thumbnailfile: buildAbsoluteFixturePath('preview-big.png')
+        thumbnailfile: buildAbsoluteFixturePath('custom-preview-big.png')
       }
 
       await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches })
@@ -212,7 +212,7 @@ describe('Test video lives API validator', function () {
     it('Should fail with a big preview file', async function () {
       const fields = baseCorrectParams
       const attaches = {
-        previewfile: buildAbsoluteFixturePath('preview-big.png')
+        previewfile: buildAbsoluteFixturePath('custom-preview-big.png')
       }
 
       await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches })

diff --git a/server/tests/api/check-params/runners.ts b/server/tests/api/check-params/runners.ts
@@ -752,7 +752,7 @@ describe('Test managing runners', function () {
         })
 
         it('Should fail with an invalid vod audio merge payload', async function () {
-          const attributes = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
+          const attributes = { name: 'audio_with_preview', previewfile: 'custom-preview.jpg', fixture: 'sample.ogg' }
           await server.videos.upload({ attributes, mode: 'legacy' })
 
           await waitJobs([ server ])

diff --git a/server/tests/api/check-params/video-imports.ts b/server/tests/api/check-params/video-imports.ts
@@ -244,7 +244,7 @@ describe('Test video imports API validator', function () {
     it('Should fail with a big thumbnail file', async function () {
       const fields = baseCorrectParams
       const attaches = {
-        thumbnailfile: buildAbsoluteFixturePath('preview-big.png')
+        thumbnailfile: buildAbsoluteFixturePath('custom-preview-big.png')
       }
 
       await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches })
@@ -262,7 +262,7 @@ describe('Test video imports API validator', function () {
     it('Should fail with a big preview file', async function () {
       const fields = baseCorrectParams
       const attaches = {
-        previewfile: buildAbsoluteFixturePath('preview-big.png')
+        previewfile: buildAbsoluteFixturePath('custom-preview-big.png')
       }
 
       await makeUploadRequest({ url: server.url, path, token: server.accessToken, fields, attaches })

diff --git a/server/tests/api/check-params/video-playlists.ts b/server/tests/api/check-params/video-playlists.ts
@@ -196,7 +196,7 @@ describe('Test video playlists API validator', function () {
         attributes: {
           displayName: 'display name',
           privacy: VideoPlaylistPrivacy.UNLISTED,
-          thumbnailfile: 'thumbnail.jpg',
+          thumbnailfile: 'custom-thumbnail.jpg',
           videoChannelId: server.store.channel.id,
 
           ...attributes
@@ -260,7 +260,7 @@ describe('Test video playlists API validator', function () {
     })
 
     it('Should fail with a thumbnail file too big', async function () {
-      const params = getBase({ thumbnailfile: 'preview-big.png' })
+      const params = getBase({ thumbnailfile: 'custom-preview-big.png' })
 
       await command.create(params)
       await command.update(getUpdate(params, playlist.shortUUID))

diff --git a/server/tests/api/check-params/video-studio.ts b/server/tests/api/check-params/video-studio.ts
@@ -293,7 +293,7 @@ describe('Test video studio API validator', function () {
       it('Should succeed with the correct params', async function () {
         this.timeout(120000)
 
-        await addWatermark('thumbnail.jpg', HttpStatusCode.NO_CONTENT_204)
+        await addWatermark('custom-thumbnail.jpg', HttpStatusCode.NO_CONTENT_204)
 
         await waitJobs([ server ])
       })
@@ -322,8 +322,8 @@ describe('Test video studio API validator', function () {
       })
 
       it('Should fail with an invalid file', async function () {
-        await addIntroOutro('add-intro', 'thumbnail.jpg')
-        await addIntroOutro('add-outro', 'thumbnail.jpg')
+        await addIntroOutro('add-intro', 'custom-thumbnail.jpg')
+        await addIntroOutro('add-outro', 'custom-thumbnail.jpg')
       })
 
       it('Should fail with a file that does not contain video stream', async function () {

diff --git a/server/tests/api/check-params/videos.ts b/server/tests/api/check-params/videos.ts
@@ -384,7 +384,7 @@ describe('Test videos API validator', function () {
       it('Should fail with a big thumbnail file', async function () {
         const fields = baseCorrectParams
         const attaches = {
-          thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'),
+          thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'custom-preview-big.png'),
           fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4')
         }
 
@@ -404,7 +404,7 @@ describe('Test videos API validator', function () {
       it('Should fail with a big preview file', async function () {
         const fields = baseCorrectParams
         const attaches = {
-          previewfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'),
+          previewfile: join(root(), 'server', 'tests', 'fixtures', 'custom-preview-big.png'),
           fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4')
         }
 
@@ -615,7 +615,7 @@ describe('Test videos API validator', function () {
     it('Should fail with a big thumbnail file', async function () {
       const fields = baseCorrectParams
       const attaches = {
-        thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png')
+        thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'custom-preview-big.png')
       }
 
       await makeUploadRequest({
@@ -647,7 +647,7 @@ describe('Test videos API validator', function () {
     it('Should fail with a big preview file', async function () {
       const fields = baseCorrectParams
       const attaches = {
-        previewfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png')
+        previewfile: join(root(), 'server', 'tests', 'fixtures', 'custom-preview-big.png')
       }
 
       await makeUploadRequest({

diff --git a/server/tests/api/live/live.ts b/server/tests/api/live/live.ts
@@ -2,7 +2,7 @@
 
 import { expect } from 'chai'
 import { basename, join } from 'path'
-import { SQLCommand, testImage, testLiveVideoResolutions } from '@server/tests/shared'
+import { SQLCommand, testImageGeneratedByFFmpeg, testLiveVideoResolutions } from '@server/tests/shared'
 import { getAllFiles, wait } from '@shared/core-utils'
 import { ffprobePromise, getVideoStream } from '@shared/ffmpeg'
 import {
@@ -121,8 +121,8 @@ describe('Test live', function () {
         expect(video.downloadEnabled).to.be.false
         expect(video.privacy.id).to.equal(VideoPrivacy.PUBLIC)
 
-        await testImage(server.url, 'video_short1-preview.webm', video.previewPath)
-        await testImage(server.url, 'video_short1.webm', video.thumbnailPath)
+        await testImageGeneratedByFFmpeg(server.url, 'video_short1-preview.webm', video.previewPath)
+        await testImageGeneratedByFFmpeg(server.url, 'video_short1.webm', video.thumbnailPath)
 
         const live = await server.live.get({ videoId: liveVideoUUID })
 

diff --git a/server/tests/api/runners/runner-studio-transcoding.ts b/server/tests/api/runners/runner-studio-transcoding.ts
@@ -104,7 +104,7 @@ describe('Test runner video studio transcoding', function () {
       {
         name: 'add-watermark' as 'add-watermark',
         options: {
-          file: 'thumbnail.png'
+          file: 'custom-thumbnail.png'
         }
       },
       {

diff --git a/server/tests/api/runners/runner-vod-transcoding.ts b/server/tests/api/runners/runner-vod-transcoding.ts
@@ -424,7 +424,7 @@ describe('Test runner VOD transcoding', function () {
 
       await servers[0].config.enableTranscoding(true, true)
 
-      const attributes = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
+      const attributes = { name: 'audio_with_preview', previewfile: 'custom-preview.jpg', fixture: 'sample.ogg' }
       const { uuid } = await servers[0].videos.upload({ attributes, mode: 'legacy' })
       videoUUID = uuid
 

diff --git a/server/tests/api/transcoding/transcoder.ts b/server/tests/api/transcoding/transcoder.ts
@@ -353,7 +353,7 @@ describe('Test video transcoding', function () {
       it('Should merge an audio file with the preview file', async function () {
         this.timeout(60_000)
 
-        const attributes = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
+        const attributes = { name: 'audio_with_preview', previewfile: 'custom-preview.jpg', fixture: 'sample.ogg' }
         await servers[1].videos.upload({ attributes, mode })
 
         await waitJobs(servers)
@@ -416,7 +416,7 @@ describe('Test video transcoding', function () {
           }
         })
 
-        const attributes = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
+        const attributes = { name: 'audio_with_preview', previewfile: 'custom-preview.jpg', fixture: 'sample.ogg' }
         const { id } = await servers[1].videos.upload({ attributes, mode })
 
         await waitJobs(servers)

diff --git a/server/tests/api/transcoding/video-studio.ts b/server/tests/api/transcoding/video-studio.ts
@@ -241,7 +241,7 @@ describe('Test video studio', function () {
         {
           name: 'add-watermark',
           options: {
-            file: 'thumbnail.png'
+            file: 'custom-thumbnail.png'
           }
         }
       ])

diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts
@@ -9,7 +9,7 @@ import {
   completeVideoCheck,
   dateIsValid,
   saveVideoInServers,
-  testImage
+  testImageGeneratedByFFmpeg
 } from '@server/tests/shared'
 import { buildAbsoluteFixturePath, wait } from '@shared/core-utils'
 import { HttpStatusCode, VideoCommentThreadTree, VideoPrivacy } from '@shared/models'
@@ -70,8 +70,9 @@ describe('Test multiple servers', function () {
   })
 
   describe('Should upload the video and propagate on each server', function () {
+
     it('Should upload the video on server 1 and propagate on each server', async function () {
-      this.timeout(25000)
+      this.timeout(60000)
 
       const attributes = {
         name: 'my super name for server 1',
@@ -175,8 +176,8 @@ describe('Test multiple servers', function () {
         support: 'my super support text for server 2',
         tags: [ 'tag1p2', 'tag2p2', 'tag3p2' ],
         fixture: 'video_short2.webm',
-        thumbnailfile: 'thumbnail.jpg',
-        previewfile: 'preview.jpg'
+        thumbnailfile: 'custom-thumbnail.jpg',
+        previewfile: 'custom-preview.jpg'
       }
       await servers[1].videos.upload({ token: userAccessToken, attributes, mode: 'resumable' })
 
@@ -229,8 +230,8 @@ describe('Test multiple servers', function () {
               size: 750000
             }
           ],
-          thumbnailfile: 'thumbnail',
-          previewfile: 'preview'
+          thumbnailfile: 'custom-thumbnail',
+          previewfile: 'custom-preview'
         }
 
         const { data } = await server.videos.list()
@@ -619,9 +620,9 @@ describe('Test multiple servers', function () {
         description: 'my super description updated',
         support: 'my super support text updated',
         tags: [ 'tag_up_1', 'tag_up_2' ],
-        thumbnailfile: 'thumbnail.jpg',
+        thumbnailfile: 'custom-thumbnail.jpg',
         originallyPublishedAt: '2019-02-11T13:38:14.449Z',
-        previewfile: 'preview.jpg'
+        previewfile: 'custom-preview.jpg'
       }
 
       updatedAtMin = new Date()
@@ -674,8 +675,8 @@ describe('Test multiple servers', function () {
               size: 292677
             }
           ],
-          thumbnailfile: 'thumbnail',
-          previewfile: 'preview'
+          thumbnailfile: 'custom-thumbnail',
+          previewfile: 'custom-preview'
         }
         await completeVideoCheck({ server, originServer: servers[2], videoUUID: videoUpdated.uuid, attributes: checkAttributes })
       }
@@ -685,7 +686,7 @@ describe('Test multiple servers', function () {
       this.timeout(30000)
 
       const attributes = {
-        thumbnailfile: 'thumbnail.jpg'
+        thumbnailfile: 'custom-thumbnail.jpg'
       }
 
       updatedAtMin = new Date()
@@ -761,7 +762,7 @@ describe('Test multiple servers', function () {
       for (const server of servers) {
         const video = await server.videos.get({ id: videoUUID })
 
-        await testImage(server.url, 'video_short1-preview.webm', video.previewPath)
+        await testImageGeneratedByFFmpeg(server.url, 'video_short1-preview.webm', video.previewPath)
       }
     })
   })

diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts
@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
 import { expect } from 'chai'
-import { checkVideoFilesWereRemoved, completeVideoCheck, testImage } from '@server/tests/shared'
+import { checkVideoFilesWereRemoved, completeVideoCheck, testImageGeneratedByFFmpeg } from '@server/tests/shared'
 import { wait } from '@shared/core-utils'
 import { Video, VideoPrivacy } from '@shared/models'
 import {
@@ -260,7 +260,7 @@ describe('Test a single server', function () {
 
       for (const video of data) {
         const videoName = video.name.replace(' name', '')
-        await testImage(server.url, videoName, video.thumbnailPath)
+        await testImageGeneratedByFFmpeg(server.url, videoName, video.thumbnailPath)
       }
     })
 

diff --git a/server/tests/api/videos/video-imports.ts b/server/tests/api/videos/video-imports.ts
@@ -3,7 +3,7 @@
 import { expect } from 'chai'
 import { pathExists, readdir, remove } from 'fs-extra'
 import { join } from 'path'
-import { FIXTURE_URLS, testCaptionFile, testImage } from '@server/tests/shared'
+import { FIXTURE_URLS, testCaptionFile, testImageGeneratedByFFmpeg } from '@server/tests/shared'
 import { areHttpImportTestsDisabled } from '@shared/core-utils'
 import { CustomConfig, HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models'
 import {
@@ -67,7 +67,7 @@ async function checkVideoServer2 (server: PeerTubeServer, id: number | string) {
   expect(video.description).to.equal('my super description')
   expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ])
 
-  await testImage(server.url, 'thumbnail', video.thumbnailPath)
+  await testImageGeneratedByFFmpeg(server.url, 'custom-thumbnail', video.thumbnailPath)
 
   expect(video.files).to.have.lengthOf(1)
 
@@ -126,8 +126,8 @@ describe('Test video imports', function () {
               ? '_yt_dlp'
               : ''
 
-            await testImage(servers[0].url, 'video_import_thumbnail' + suffix, video.thumbnailPath)
-            await testImage(servers[0].url, 'video_import_preview' + suffix, video.previewPath)
+            await testImageGeneratedByFFmpeg(servers[0].url, 'video_import_thumbnail' + suffix, video.thumbnailPath)
+            await testImageGeneratedByFFmpeg(servers[0].url, 'video_import_preview' + suffix, video.previewPath)
           }
 
           const bodyCaptions = await servers[0].captions.list({ videoId: video.id })
@@ -266,7 +266,7 @@ describe('Test video imports', function () {
             name: 'my super name',
             description: 'my super description',
             tags: [ 'supertag1', 'supertag2' ],
-            thumbnailfile: 'thumbnail.jpg'
+            thumbnailfile: 'custom-thumbnail.jpg'
           }
         })
         expect(video.name).to.equal('my super name')