working script

Signed-off-by: James Ketrenos <jketreno@media.ketrenos.com>

Dockerfile

@@ -17,73 +17,7 @@
 # Template from templates/ubuntu/00-from.in
 #
 # Pull from ubuntu on Docker Hub
-FROM amr-registry.caas.intel.com/cache/library/ubuntu:21.10 AS xe-base-stage
+FROM ubuntu:focal AS xe-base-stage
-
-
-#
-# Template from templates/ubuntu/05-intel-proxy.in
-#
-# Pre-install proxy configuration values
-#
-# This uses the linux-ftp.ostc.intel.com as a mirror.
-RUN { \
-  for suite in 21.10 21.10-updates 21.10-security 21.10-backports; do \
-    for component in main restricted universe multiverse; do \
-      echo "deb http://linux-ftp.ostc.intel.com/pub/mirrors/ubuntu ${suite} ${component}" ; \
-    done \
-  done \
-} > /dev/null
-#/etc/apt/sources.list
-
-# Retry downloads up to 100 times to work around linux-ftp issue...
-RUN echo 'APT::Acquire::Retries "100";' > /etc/apt/apt.conf.d/80-retries
-
-ARG http_proxy="http://proxy-chain.intel.com:911/"
-ARG https_proxy="http://proxy-chain.intel.com:912/"
-ARG ftp_proxy="ftp://proxy-chain.intel.com:911/"
-ARG socks_proxy="socks://proxy-chain.intel.com:1080/"
-ARG no_proxy="localhost,*.*.intel.com"
-
-ENV http_proxy="${http_proxy}"
-ENV https_proxy="${https_proxy}"
-ENV no_proxy="${no_proxy}"
-ENV ftp_proxy="${http_proxy}"
-ENV socks_proxy="${socks_proxy}"
-
-ENV HTTP_PROXY="${http_proxy}"
-ENV HTTPS_PROXY="${http_proxy}"
-ENV NO_PROXY="${no_proxy}"
-
-RUN { \
-  echo "Acquire::http::proxy \"${http_proxy}\";\n" ; \
-  echo "Acquire::https::proxy \"${https_proxy}\";\n" ; \
-  echo "Acquire::ftp::proxy \"${ftp_proxy}\";\n" ; \
-  echo "Acquire::socks::proxy \"${socks_proxy}\";\n" ; \
-  echo "Acquire::http::Proxy::linux-ftp.ostc.intel.com DIRECT;\n" ; \
-} > /etc/apt/apt.conf
-
-
-#
-# Template from templates/ubuntu/08-intel-certs.in
-#
-# Install Intel CA5A cert so Intel certs are recognized
-
-RUN apt-get -q update \
-  && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
-    ca-certificates \
-    wget \
-    unzip \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} \
-  && mkdir -p /usr/local/share/ca-certificates \
-  && wget -qO tmp.zip http://certificates.intel.com/repository/certificates/IntelSHA2RootChain-Base64.zip \
-     && unzip tmp.zip -d /usr/local/share/ca-certificates \
-     && rm tmp.zip \
-  && wget -qO tmp.zip http://certificates.intel.com/repository/certificates/Intel%20Root%20Certificate%20Chain%20Base64.zip \
-     && unzip tmp.zip -d /usr/local/share/ca-certificates \
-     && rm tmp.zip \
-  && apt-get remove -y wget unzip \
-  && update-ca-certificates --fresh
 
 
 #
@@ -135,6 +69,8 @@ RUN groupadd -g $USER_GID user \
 # Set 'sudo' to NOPASSWD for all container users
 RUN sed -i -e 's,%sudo.*,%sudo ALL=(ALL) NOPASSWD:ALL,g' /etc/sudoers
 
+RUN echo "Defaults !env_reset" >> /etc/sudoers
+
 RUN apt-get clean \
   && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log}
 
@@ -170,7 +106,7 @@ RUN apt-add-repository \
 ENV GPGPU_PACKAGE_REPOSITORY "https://repositories.intel.com/graphics"
 ENV GPGPU_PACKAGE_STREAM "focal"
 ENV GPGPU_PACKAGE_DISTRO "ubuntu"
-ENV GPGPU_PACKAGE_DISTRO_RELEASE "21.10"
+ENV GPGPU_PACKAGE_DISTRO_RELEASE "focal"
 
 USER user
 
@@ -202,10 +138,16 @@ RUN apt-get update \
 RUN apt-get update \
   && DEBIAN_FRONTEND=noninteractive \
   && apt-get -q -y install \
-    libva-dev$LIBVA_DEV_VERSION \
+    libva-dev \
-    libmfx-dev$LIBMFX_DEV_VERSION \
+    libmfx-dev \
-    libmfx1$LIBMFX1_VERSION \
+    libmfx1 \
-    vainfo$VAINFO_VERSION
+    vainfo
+
+RUN sed -i -e 's,^# deb-src,deb-src,g' /etc/apt/sources.list
+RUN apt-get update \
+  && DEBIAN_FRONTEND=noninteractive \
+  && apt-get build-dep -y \
+    ffmpeg
 
 ENV LIBVA_DRIVER_NAME=iHD
 
@@ -213,15 +155,26 @@ USER user
 
 # Clone ffmpeg
 # NOTE: This explicitly clones the FFMPEG_TAG_VERSION (see SOLUTIONS)
-RUN git clone --depth 1 --branch n4.2.1 https://github.com/ffmpeg/ffmpeg /home/user/ffmpeg
+RUN git clone --depth 1 --branch n5.0.1 https://github.com/ffmpeg/ffmpeg /home/user/ffmpeg
 
 # Build FFmpeg
 WORKDIR /home/user/ffmpeg
 
-RUN ./configure --arch=x86_64 --disable-x86asm --enable-vaapi --enable-libmfx \
+RUN ./configure --arch=x86_64 \
+  --disable-x86asm --enable-vaapi --enable-libmfx --enable-nonfree \
+--arch=amd64 --enable-gpl --disable-stripping --enable-swresample --disable-filter=resample --enable-gnutls \
+--enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 \
+--enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libjack \
+--enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librsvg \
+--enable-librubberband --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora \
+--enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwebp --enable-libx265 \
+--enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-lv2 --enable-omx --enable-openal --enable-opencl --enable-opengl \
+--enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-nvenc --enable-chromaprint --enable-frei0r --enable-libx264 \
   && make -j $(nproc --all) \
   && sudo make install
 
+# --enable-shared \
+
 FROM xe-user-stage
 
 USER root
@@ -236,13 +189,14 @@ USER root
 RUN apt-get -q update \
   && DEBIAN_FRONTEND=noninteractive \
   && apt-get --no-install-recommends -q -y install \
-    intel-media-va-driver-non-free$INTEL_MEDIA_VA_DRIVER_NON_FREE_VERSION \
+    intel-media-va-driver-non-free \
-    libva2$LIBVA2_VERSION \
+    libva2 \
-    libmfx1$LIBMFX1_VERSION \
+    libmfx1 \
-    vainfo$VAINFO_VERSION \
+    vainfo \
     libpciaccess0 \
     pciutils \
-    curl
+    curl \
+    ffmpeg
 
 # Copy ffmpeg and ffprobe from build container
 COPY --from=solution-build /usr/local/bin/ /usr/local/bin/

Dockerfile.solution

@@ -46,19 +46,20 @@ RUN git clone --depth 1 --branch $FFMPEG_TAG_VERSION https://github.com/ffmpeg/f
 WORKDIR /home/user/ffmpeg
 
 RUN ./configure --arch=x86_64 \
-  --disable-x86asm --enable-vaapi --enable-libmfx \
+  --disable-x86asm --enable-vaapi --enable-libmfx --enable-nonfree \
---arch=amd64 --enable-gpl --disable-stripping --enable-avresample --disable-filter=resample --enable-avisynth --enable-gnutls \
+--arch=amd64 --enable-gpl --disable-stripping --enable-swresample --disable-filter=resample --enable-gnutls \
 --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 \
 --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libjack \
 --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librsvg \
 --enable-librubberband --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libssh --enable-libtheora \
---enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwavpack --enable-libwebp --enable-libx265 \
+--enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwebp --enable-libx265 \
 --enable-libxml2 --enable-libxvid --enable-libzmq --enable-libzvbi --enable-lv2 --enable-omx --enable-openal --enable-opencl --enable-opengl \
 --enable-sdl2 --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-nvenc --enable-chromaprint --enable-frei0r --enable-libx264 \
---enable-shared \
   && make -j $(nproc --all) \
   && sudo make install
 
+# --enable-shared \
+
 FROM xe-user-stage
 
 USER root

SOLUTION

@@ -1,11 +1,7 @@
-# Until focal is published to repositories.intel.com, use the
-# internal repository hosted on osgc.jf.intel.com/internal
-PACKAGE_REPOSITORY=${PACKAGE_REPOSITORY:-https://osgc.jf.intel.com/internal}
-
-#FFMPEG_TAG_VERSION=n4.2.1
 FFMPEG_TAG_VERSION=n5.0.1
-REGISTRY_URL=${REGISTRY_URL:-amr-registry.caas.intel.com/vtt-osgc/solutions}
+REGISTRY_OS=external
 RELEASE_INFO=${RELEASE_INFO:-N/A}
+TEMPLATE_IGNORE='[0-9]+-intel.*'
 CONTAINER="intel-media-ffmpeg"
 OS_DISTRO=${OS_DISTRO:-ubuntu}
 OS_RELEASE=${OS_RELEASE:-focal}

video-convert

@@ -0,0 +1,179 @@
+#!/bin/bash
+# Look into HW video encoding to vp9 instead of h264:
+# https://www.reddit.com/r/VP9/comments/g9uzzv/hardware_encoding_vp9_on_intel/
+
+video_detect() {
+  for file in "${*}"; do
+    INFO=($(ffprobe -v error \
+      -select_streams v:0 \
+      -show_entries stream=codec_name:format=format_name \
+      -of default=noprint_wrappers=1:nokey=1 "${file}"))
+    if (( ${#INFO[*]} != 2 )); then
+      echo "${file}|none|none"
+    else
+      echo "${file}|${INFO[0]}|${INFO[1]}"
+    fi
+  done
+}
+
+# mstodate MILLISECONDS
+#
+# Given MILLISECONDS, convert to DAYS:HOURS:MINUTES:SECONDS.MS
+mstodate() {
+  scales=( "86400-d:" "3600-h:-02" "60-m:-02" "1-.-02" )
+  echo $1 | sed -E 's/([[:digit:]]{3})$/ \1/' | while read sec msec; do
+    for scale in ${scales[@]}; do
+      parts=(${scale//-/ })
+      divisor=${parts[0]}
+      suffix=${parts[1]}
+      min=${parts[2]}
+      num_scale=$((sec / divisor))
+      sec=$((sec - (num_scale * divisor)))
+      printf "%${min}d%s" ${num_scale} ${suffix}
+    done
+    echo "${msec}s"
+  done
+}
+
+function move {
+  base="$(dirname "${IN}")"
+  [ ! -d "${base}" ] && continue
+  base=/multimedia/backup/"${base}"
+  [ ! -d "${base}" ] && mkdir -p "${base}"
+  [ -e "${base}" ] && mv "${IN}" "${base}"/
+}
+
+function convert {
+  IN="$1"
+  SRC_CODEC="$2"
+  FORMAT="$3"
+  OUT="${IN/%.???/.mkv}"
+  ORIG=${OUT}
+  if [[ "${OUT}" == "${IN}" ]]; then
+    OUT=${OUT/.mkv/.transcoded.mkv}
+    [ -e "${OUT}" ] && rm "${OUT}"
+  else
+    [ -e "${OUT}" ] && move && return 0
+    [ -e /multimedia/backup/"${OUT}" ] && return 0
+  fi
+
+  TITLE="${IN%.*}"
+  TITLE="${TITLE##*/}"
+  if [[ "${SRC_CODEC}" == "h264" ]]; then
+    echo "Content will have format changed from ${FORMAT}/h264 to mkv/h264."
+    # Just change container to matroska
+    input_flags="
+      -c:v copy
+      -c:a copy
+    "
+    output_flags=""
+  else
+    echo "Content will be transcoded from ${FORMAT}/${SRC_CODEC} to mkv/h264."
+    # Convert all video streams to h264. Copy all other streams unchanged (audio, subtitle, etc.)
+    # Move headers to start of the file for fast start
+    # Use "reasonable" quality level of 20, slow encode
+    #
+   input_flags="
+     -hwaccel qsv
+     -c:v hevc_qsv
+   "
+   output_flags="
+     -vf vpp_qsv=format=nv12
+     -c:v h264_qsv
+     -preset slow
+   "
+  fi
+
+  echo -n "Counting frames in ${IN}: "
+  FRAMES=$(ffprobe -v error -select_streams v:0 -count_packets -show_entries stream=nb_read_packets -of csv=p=0 "${IN}")
+  echo "${FRAMES} frames."
+
+  # Use stdbuf to flush stdout/stderr every line
+  STARTTIME=$(date +%s)
+  LASTTIME=${STARTTIME}
+  LASTPOS=0
+  IN=$(realpath "${IN}")
+  OUT=$(realpath "${OUT}")
+  DIN=$(dirname "${IN}")
+  DOUT=$(dirname "${OUT}")
+  NO_COLOR=1 \
+    docker run --device=/dev/dri \
+    --rm \
+    -v "${DIN}:${DIN}" \
+    -v "${DOUT}:${DOUT}" \
+    intel-media-ffmpeg \
+    ffmpeg \
+      -nostdin \
+      -v quiet \
+      -loglevel fatal \
+    ${input_flags} \
+      -i "${IN}" \
+      -progress /dev/stdout \
+      -metadata title="${TITLE/.\//}" \
+    ${output_flags} \
+      -movflags +faststart \
+    "${OUT}" \
+      -y | while read line; do
+      POS=$(echo $line | sed -n 's/^frame=*\(.*\)/\1/p')
+      if [[ "${POS}" != "" ]]; then
+        NOW=$(date +%s)
+        ELAPSEDTIME=$(( NOW - LASTTIME ))
+        if (( ELAPSEDTIME > 5 )); then
+          ELAPSEDFRAMES=$(( POS - LASTPOS ))
+          REMAININGFRAMES=$(( FRAMES - POS ))
+          REMAININGFRAMES=$(( REMAININGFRAMES * 1000 )) # convert to ms
+          FRAMERATE=$(( ELAPSEDFRAMES / ELAPSEDTIME ))
+          if (( FRAMERATE != 0 )); then
+            REMAININGMS=$(( REMAININGFRAMES / FRAMERATE ))
+            printf "\r%*s\r%s" $(tput cols) " " "Processing for $(mstodate $(( NOW - STARTTIME ))000). Processing at ($((ELAPSEDFRAMES / ELAPSEDTIME ))fps). Now at frame $POS of $FRAMES $(( $(( 100 * POS)) / FRAMES ))%. Estimating $(mstodate ${REMAININGMS}) remaining."
+          else
+            printf "\r%*s\r%s" $(tput cols) " " "Processing for $(mstodate $(( NOW - STARTTIME ))000). Processing at ($((ELAPSEDFRAMES / ELAPSEDTIME ))fps). Now at frame $POS of $FRAMES $(( $(( 100 * POS)) / FRAMES ))%. No frames processed in last ${ELAPSEDTIME} seconds."
+          fi
+          LASTTIME=${NOW}
+          LASTPOS=${POS}
+        fi
+      fi
+    done && move && {
+      if [[ "${ORIG}" == "${IN}" ]]; then
+        # If the original file was an '.mkv' then OUT was '.transcoded.mkv'
+        # during transcode. Now that the transcode is complete, set the
+        # file back to the original name.
+        mv "${OUT}" "${IN}"
+      fi
+    }
+    NOW=$(date +%s)
+    echo -e "\nCompleted in $(mstodate $(( NOW - STARTTIME ))000)"
+#    -vcodec copy \
+#    -acodec aac \
+}
+
+function check_and_convert {
+  video_detect "$1" | while read entry; do
+    #                     # file:codec:format
+    file="${entry%%|*}"   # file
+    entry="${entry#*|}"   # codec:format
+    codec="${entry%%|*}"  # codec
+    format="${entry##*|}" # format
+    codec="${codec,,}"
+    if [[ "${codec}" == "none" ]]; then
+      continue
+    fi
+    if [[ "${codec}" == "h264" ]] && [[ "${format}" =~ mkv|matroska ]]; then
+      # If container is not mkv, convert to mkv container
+      echo "Skipping ${file} as it is already mkv/h264."
+    else
+      convert "${file}" "${codec}" "${format}"
+    fi
+  done
+}
+
+if [ -z "$1" ]; then
+  find . -type f \
+    -and -not -path "./backup/*" | sort | while read file; do
+    check_and_convert "${file}"
+  done
+else
+  for file in "$*"; do
+    check_and_convert "${file}"
+  done
+fi