From ae89d4527250947333faaa61f522f508951f30e4 Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Sat, 9 Apr 2022 00:28:30 +0000 Subject: [PATCH] working script Signed-off-by: James Ketrenos --- Dockerfile | 112 ++++++++------------------- Dockerfile.solution | 9 ++- SOLUTION | 8 +- video-convert | 179 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 219 insertions(+), 89 deletions(-) create mode 100755 video-convert diff --git a/Dockerfile b/Dockerfile index 4903647..cc219b4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,73 +17,7 @@ # Template from templates/ubuntu/00-from.in # # Pull from ubuntu on Docker Hub -FROM amr-registry.caas.intel.com/cache/library/ubuntu:21.10 AS xe-base-stage - - -# -# Template from templates/ubuntu/05-intel-proxy.in -# -# Pre-install proxy configuration values -# -# This uses the linux-ftp.ostc.intel.com as a mirror. -RUN { \ - for suite in 21.10 21.10-updates 21.10-security 21.10-backports; do \ - for component in main restricted universe multiverse; do \ - echo "deb http://linux-ftp.ostc.intel.com/pub/mirrors/ubuntu ${suite} ${component}" ; \ - done \ - done \ -} > /dev/null -#/etc/apt/sources.list - -# Retry downloads up to 100 times to work around linux-ftp issue... -RUN echo 'APT::Acquire::Retries "100";' > /etc/apt/apt.conf.d/80-retries - -ARG http_proxy="http://proxy-chain.intel.com:911/" -ARG https_proxy="http://proxy-chain.intel.com:912/" -ARG ftp_proxy="ftp://proxy-chain.intel.com:911/" -ARG socks_proxy="socks://proxy-chain.intel.com:1080/" -ARG no_proxy="localhost,*.*.intel.com" - -ENV http_proxy="${http_proxy}" -ENV https_proxy="${https_proxy}" -ENV no_proxy="${no_proxy}" -ENV ftp_proxy="${http_proxy}" -ENV socks_proxy="${socks_proxy}" - -ENV HTTP_PROXY="${http_proxy}" -ENV HTTPS_PROXY="${http_proxy}" -ENV NO_PROXY="${no_proxy}" - -RUN { \ - echo "Acquire::http::proxy \"${http_proxy}\";\n" ; \ - echo "Acquire::https::proxy \"${https_proxy}\";\n" ; \ - echo "Acquire::ftp::proxy \"${ftp_proxy}\";\n" ; \ - echo "Acquire::socks::proxy \"${socks_proxy}\";\n" ; \ - echo "Acquire::http::Proxy::linux-ftp.ostc.intel.com DIRECT;\n" ; \ -} > /etc/apt/apt.conf - - -# -# Template from templates/ubuntu/08-intel-certs.in -# -# Install Intel CA5A cert so Intel certs are recognized - -RUN apt-get -q update \ - && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ - ca-certificates \ - wget \ - unzip \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} \ - && mkdir -p /usr/local/share/ca-certificates \ - && wget -qO tmp.zip http://certificates.intel.com/repository/certificates/IntelSHA2RootChain-Base64.zip \ - && unzip tmp.zip -d /usr/local/share/ca-certificates \ - && rm tmp.zip \ - && wget -qO tmp.zip http://certificates.intel.com/repository/certificates/Intel%20Root%20Certificate%20Chain%20Base64.zip \ - && unzip tmp.zip -d /usr/local/share/ca-certificates \ - && rm tmp.zip \ - && apt-get remove -y wget unzip \ - && update-ca-certificates --fresh +FROM ubuntu:focal AS xe-base-stage # @@ -135,6 +69,8 @@ RUN groupadd -g $USER_GID user \ # Set 'sudo' to NOPASSWD for all container users RUN sed -i -e 's,%sudo.*,%sudo ALL=(ALL) NOPASSWD:ALL,g' /etc/sudoers +RUN echo "Defaults !env_reset" >> /etc/sudoers + RUN apt-get clean \ && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} @@ -170,7 +106,7 @@ RUN apt-add-repository \ ENV GPGPU_PACKAGE_REPOSITORY "https://repositories.intel.com/graphics" ENV GPGPU_PACKAGE_STREAM "focal" ENV GPGPU_PACKAGE_DISTRO "ubuntu" -ENV GPGPU_PACKAGE_DISTRO_RELEASE "21.10" +ENV GPGPU_PACKAGE_DISTRO_RELEASE "focal" USER user @@ -202,10 +138,16 @@ RUN apt-get update \ RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive \ && apt-get -q -y install \ - libva-dev$LIBVA_DEV_VERSION \ - libmfx-dev$LIBMFX_DEV_VERSION \ - libmfx1$LIBMFX1_VERSION \ - vainfo$VAINFO_VERSION + libva-dev \ + libmfx-dev \ + libmfx1 \ + vainfo + +RUN sed -i -e 's,^# deb-src,deb-src,g' /etc/apt/sources.list +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive \ + && apt-get build-dep -y \ + ffmpeg ENV LIBVA_DRIVER_NAME=iHD @@ -213,15 +155,26 @@ USER user # Clone ffmpeg # NOTE: This explicitly clones the FFMPEG_TAG_VERSION (see SOLUTIONS) -RUN git clone --depth 1 --branch n4.2.1 https://github.com/ffmpeg/ffmpeg /home/user/ffmpeg +RUN git clone --depth 1 --branch n5.0.1 https://github.com/ffmpeg/ffmpeg /home/user/ffmpeg # Build FFmpeg WORKDIR /home/user/ffmpeg -RUN ./configure --arch=x86_64 --disable-x86asm --enable-vaapi --enable-libmfx \ +RUN ./configure --arch=x86_64 \ + --disable-x86asm --enable-vaapi --enable-libmfx --enable-nonfree \ +--arch=amd64 --enable-gpl --disable-stripping --enable-swresample --disable-filter=resample --enable-gnutls \ +--enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 \ +--enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libjack \ +--enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librsvg \ +--enable-librubberband --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora \ +--enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwebp --enable-libx265 \ +--enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-lv2 --enable-omx --enable-openal --enable-opencl --enable-opengl \ +--enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-nvenc --enable-chromaprint --enable-frei0r --enable-libx264 \ && make -j $(nproc --all) \ && sudo make install +# --enable-shared \ + FROM xe-user-stage USER root @@ -236,13 +189,14 @@ USER root RUN apt-get -q update \ && DEBIAN_FRONTEND=noninteractive \ && apt-get --no-install-recommends -q -y install \ - intel-media-va-driver-non-free$INTEL_MEDIA_VA_DRIVER_NON_FREE_VERSION \ - libva2$LIBVA2_VERSION \ - libmfx1$LIBMFX1_VERSION \ - vainfo$VAINFO_VERSION \ + intel-media-va-driver-non-free \ + libva2 \ + libmfx1 \ + vainfo \ libpciaccess0 \ pciutils \ - curl + curl \ + ffmpeg # Copy ffmpeg and ffprobe from build container COPY --from=solution-build /usr/local/bin/ /usr/local/bin/ diff --git a/Dockerfile.solution b/Dockerfile.solution index bc4d782..b333696 100644 --- a/Dockerfile.solution +++ b/Dockerfile.solution @@ -46,19 +46,20 @@ RUN git clone --depth 1 --branch $FFMPEG_TAG_VERSION https://github.com/ffmpeg/f WORKDIR /home/user/ffmpeg RUN ./configure --arch=x86_64 \ - --disable-x86asm --enable-vaapi --enable-libmfx \ ---arch=amd64 --enable-gpl --disable-stripping --enable-avresample --disable-filter=resample --enable-avisynth --enable-gnutls \ + --disable-x86asm --enable-vaapi --enable-libmfx --enable-nonfree \ +--arch=amd64 --enable-gpl --disable-stripping --enable-swresample --disable-filter=resample --enable-gnutls \ --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 \ --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libjack \ --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librsvg \ --enable-librubberband --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora \ ---enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 \ +--enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwebp --enable-libx265 \ --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-lv2 --enable-omx --enable-openal --enable-opencl --enable-opengl \ --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-nvenc --enable-chromaprint --enable-frei0r --enable-libx264 \ ---enable-shared \ && make -j $(nproc --all) \ && sudo make install +# --enable-shared \ + FROM xe-user-stage USER root diff --git a/SOLUTION b/SOLUTION index 4480a3a..e4d1d36 100644 --- a/SOLUTION +++ b/SOLUTION @@ -1,11 +1,7 @@ -# Until focal is published to repositories.intel.com, use the -# internal repository hosted on osgc.jf.intel.com/internal -PACKAGE_REPOSITORY=${PACKAGE_REPOSITORY:-https://osgc.jf.intel.com/internal} - -#FFMPEG_TAG_VERSION=n4.2.1 FFMPEG_TAG_VERSION=n5.0.1 -REGISTRY_URL=${REGISTRY_URL:-amr-registry.caas.intel.com/vtt-osgc/solutions} +REGISTRY_OS=external RELEASE_INFO=${RELEASE_INFO:-N/A} +TEMPLATE_IGNORE='[0-9]+-intel.*' CONTAINER="intel-media-ffmpeg" OS_DISTRO=${OS_DISTRO:-ubuntu} OS_RELEASE=${OS_RELEASE:-focal} diff --git a/video-convert b/video-convert new file mode 100755 index 0000000..a135346 --- /dev/null +++ b/video-convert @@ -0,0 +1,179 @@ +#!/bin/bash +# Look into HW video encoding to vp9 instead of h264: +# https://www.reddit.com/r/VP9/comments/g9uzzv/hardware_encoding_vp9_on_intel/ + +video_detect() { + for file in "${*}"; do + INFO=($(ffprobe -v error \ + -select_streams v:0 \ + -show_entries stream=codec_name:format=format_name \ + -of default=noprint_wrappers=1:nokey=1 "${file}")) + if (( ${#INFO[*]} != 2 )); then + echo "${file}|none|none" + else + echo "${file}|${INFO[0]}|${INFO[1]}" + fi + done +} + +# mstodate MILLISECONDS +# +# Given MILLISECONDS, convert to DAYS:HOURS:MINUTES:SECONDS.MS +mstodate() { + scales=( "86400-d:" "3600-h:-02" "60-m:-02" "1-.-02" ) + echo $1 | sed -E 's/([[:digit:]]{3})$/ \1/' | while read sec msec; do + for scale in ${scales[@]}; do + parts=(${scale//-/ }) + divisor=${parts[0]} + suffix=${parts[1]} + min=${parts[2]} + num_scale=$((sec / divisor)) + sec=$((sec - (num_scale * divisor))) + printf "%${min}d%s" ${num_scale} ${suffix} + done + echo "${msec}s" + done +} + +function move { + base="$(dirname "${IN}")" + [ ! -d "${base}" ] && continue + base=/multimedia/backup/"${base}" + [ ! -d "${base}" ] && mkdir -p "${base}" + [ -e "${base}" ] && mv "${IN}" "${base}"/ +} + +function convert { + IN="$1" + SRC_CODEC="$2" + FORMAT="$3" + OUT="${IN/%.???/.mkv}" + ORIG=${OUT} + if [[ "${OUT}" == "${IN}" ]]; then + OUT=${OUT/.mkv/.transcoded.mkv} + [ -e "${OUT}" ] && rm "${OUT}" + else + [ -e "${OUT}" ] && move && return 0 + [ -e /multimedia/backup/"${OUT}" ] && return 0 + fi + + TITLE="${IN%.*}" + TITLE="${TITLE##*/}" + if [[ "${SRC_CODEC}" == "h264" ]]; then + echo "Content will have format changed from ${FORMAT}/h264 to mkv/h264." + # Just change container to matroska + input_flags=" + -c:v copy + -c:a copy + " + output_flags="" + else + echo "Content will be transcoded from ${FORMAT}/${SRC_CODEC} to mkv/h264." + # Convert all video streams to h264. Copy all other streams unchanged (audio, subtitle, etc.) + # Move headers to start of the file for fast start + # Use "reasonable" quality level of 20, slow encode + # + input_flags=" + -hwaccel qsv + -c:v hevc_qsv + " + output_flags=" + -vf vpp_qsv=format=nv12 + -c:v h264_qsv + -preset slow + " + fi + + echo -n "Counting frames in ${IN}: " + FRAMES=$(ffprobe -v error -select_streams v:0 -count_packets -show_entries stream=nb_read_packets -of csv=p=0 "${IN}") + echo "${FRAMES} frames." + + # Use stdbuf to flush stdout/stderr every line + STARTTIME=$(date +%s) + LASTTIME=${STARTTIME} + LASTPOS=0 + IN=$(realpath "${IN}") + OUT=$(realpath "${OUT}") + DIN=$(dirname "${IN}") + DOUT=$(dirname "${OUT}") + NO_COLOR=1 \ + docker run --device=/dev/dri \ + --rm \ + -v "${DIN}:${DIN}" \ + -v "${DOUT}:${DOUT}" \ + intel-media-ffmpeg \ + ffmpeg \ + -nostdin \ + -v quiet \ + -loglevel fatal \ + ${input_flags} \ + -i "${IN}" \ + -progress /dev/stdout \ + -metadata title="${TITLE/.\//}" \ + ${output_flags} \ + -movflags +faststart \ + "${OUT}" \ + -y | while read line; do + POS=$(echo $line | sed -n 's/^frame=*\(.*\)/\1/p') + if [[ "${POS}" != "" ]]; then + NOW=$(date +%s) + ELAPSEDTIME=$(( NOW - LASTTIME )) + if (( ELAPSEDTIME > 5 )); then + ELAPSEDFRAMES=$(( POS - LASTPOS )) + REMAININGFRAMES=$(( FRAMES - POS )) + REMAININGFRAMES=$(( REMAININGFRAMES * 1000 )) # convert to ms + FRAMERATE=$(( ELAPSEDFRAMES / ELAPSEDTIME )) + if (( FRAMERATE != 0 )); then + REMAININGMS=$(( REMAININGFRAMES / FRAMERATE )) + printf "\r%*s\r%s" $(tput cols) " " "Processing for $(mstodate $(( NOW - STARTTIME ))000). Processing at ($((ELAPSEDFRAMES / ELAPSEDTIME ))fps). Now at frame $POS of $FRAMES $(( $(( 100 * POS)) / FRAMES ))%. Estimating $(mstodate ${REMAININGMS}) remaining." + else + printf "\r%*s\r%s" $(tput cols) " " "Processing for $(mstodate $(( NOW - STARTTIME ))000). Processing at ($((ELAPSEDFRAMES / ELAPSEDTIME ))fps). Now at frame $POS of $FRAMES $(( $(( 100 * POS)) / FRAMES ))%. No frames processed in last ${ELAPSEDTIME} seconds." + fi + LASTTIME=${NOW} + LASTPOS=${POS} + fi + fi + done && move && { + if [[ "${ORIG}" == "${IN}" ]]; then + # If the original file was an '.mkv' then OUT was '.transcoded.mkv' + # during transcode. Now that the transcode is complete, set the + # file back to the original name. + mv "${OUT}" "${IN}" + fi + } + NOW=$(date +%s) + echo -e "\nCompleted in $(mstodate $(( NOW - STARTTIME ))000)" +# -vcodec copy \ +# -acodec aac \ +} + +function check_and_convert { + video_detect "$1" | while read entry; do + # # file:codec:format + file="${entry%%|*}" # file + entry="${entry#*|}" # codec:format + codec="${entry%%|*}" # codec + format="${entry##*|}" # format + codec="${codec,,}" + if [[ "${codec}" == "none" ]]; then + continue + fi + if [[ "${codec}" == "h264" ]] && [[ "${format}" =~ mkv|matroska ]]; then + # If container is not mkv, convert to mkv container + echo "Skipping ${file} as it is already mkv/h264." + else + convert "${file}" "${codec}" "${format}" + fi + done +} + +if [ -z "$1" ]; then + find . -type f \ + -and -not -path "./backup/*" | sort | while read file; do + check_and_convert "${file}" + done +else + for file in "$*"; do + check_and_convert "${file}" + done +fi