diff --git a/normalize-video b/normalize-video new file mode 100755 index 0000000..37335d1 --- /dev/null +++ b/normalize-video @@ -0,0 +1,414 @@ +#!/bin/bash + +log_file= +fail() { + echo "$*" >&2 + if [[ -e ${log_file} ]]; then + rm ${log_file} + fi + exit -1 +} + +VIDEO=$(getent group video | sed -E 's,^video:[^:]*:([^:]*):.*$,\1,') +RENDER=$(getent group render | sed -E 's,^render:[^:]*:([^:]*):.*$,\1,') + +sw_only=0 +if [[ "${VIDEO}" != "" ]]; then + sw_only=0 + ADD_GROUPS="--group-add ${VIDEO}" + if [[ "${RENDER}" != "" ]]; then + ADD_GROUPS+=" --group-add ${RENDER}" + fi +else + sw_only=1 +fi + + +quiet=" + -v quiet + -loglevel error +" + +# mstodate MILLISECONDS +# +# Given MILLISECONDS, convert to DAYS:HOURS:MINUTES:SECONDS.MS +mstodate() { + scales=( "86400-d:" "3600-h:-02" "60-m:-02" "1-.-02" ) + echo $1 | sed -E 's/([[:digit:]]{3})$/ \1/' | while read sec msec; do + for scale in ${scales[@]}; do + parts=(${scale//-/ }) + divisor=${parts[0]} + suffix=${parts[1]} + min=${parts[2]} + num_scale=$((sec / divisor)) + sec=$((sec - (num_scale * divisor))) + printf "%${min}d%s" ${num_scale} ${suffix} + done + echo "${msec}s" + done +} + +# Works: +# docker run --device=/dev/dri --user=1000 --rm --group-add 44 --group-add 109 -v /multimedia/Downloads/Infinite.Storm.2022/test:/multimedia/Downloads/Infinite.Storm.2022/test -v /multimedia/Downloads/Infinite.Storm.2022/test:/multimedia/Downloads/Infinite.Storm.2022/test --entrypoint=ffmpeg intel-media-ffmpeg -v error -loglevel error -nostdin -hwaccel qsv -c:v h264_qsv -i /multimedia/Downloads/Infinite.Storm.2022/test/Infinite.Storm.2022.mkv -progress /dev/stdout -metadata title='Infinite Storm 2022' -vf 'scale_qsv=w=1920:h=802,hwdownload,format=nv12' -map 0 -map -0:d -c:a copy -c:s copy -c:v h264_qsv -preset veryslow -crf 20 -movflags +faststart /multimedia/Downloads/Infinite.Storm.2022/test/Infinite.Storm.2022.transcoded.mkv -y^ + +# Convert yuv420p10le to nv12 +#docker run --device=/dev/dri --user=1000 --rm --group-add 44 --group-add 109 -v /multimedia/Downloads/Infinite.Storm.2022/test:/multimedia/Downloads/Infinite.Storm.2022/test -v /multimedia/Downloads/Infinite.Storm.2022/test:/multimedia/Downloads/Infinite.Storm.2022/test --entrypoint=ffmpeg intel-media-ffmpeg -v quiet -loglevel debug -nostdin -hwaccel qsv -resize 1920x1080 -i /multimedia/Downloads/Infinite.Storm.2022/test/Foundation.S01E02.Preparing.to.Live.mkv -progress /dev/stdout -metadata title='Foundation.S01E02.Preparing.to.Live' -pix_fmt nv12 -vf hwupload=extra_hw_frames=64,vpp_qsv=format=nv12 -map 0 -map -0:d -c:a copy -c:s copy -global_quality 20 -c:v h264_qsv -movflags +faststart /multimedia/Downloads/Infinite.Storm.2022/test/Foundation.S01E02.Preparing.to.Live.transcoded.mkv -y 2>&1 +ffmpeg_hw_or_sw() { + IN="${1}" + shift + OUT="${1}" + shift + TITLE="${1}" + shift + width=${1} + shift + height=${1} + shift + codec="${1}" + shift + pix_fmt="${1}" + shift + + if [[ "${pix_fmt}" == "yuv420p10le" ]]; then + sw_decode=1 + else + sw_decode=0 + fi + + DIN="$(dirname "${IN}")" + DOUT="$(dirname "${OUT}")" + + if (( sw_only == 1 )); then + echo "${PREFIX} Content will be transcoded to Matroska(yuv420p:h264:${width}x${height}) using software." >&2 + + input_flags=" + " + + output_flags=" + -map 0 + -map -0:d + -c:a copy + -c:s copy + -c:v libx264 + -vf scale=${width}:${height},format=yuv420p \ + -preset veryslow \ + -crf 20 \ + " + ffmpeg \ + ${quiet} \ + -nostdin \ + ${input_flags} \ + -i "${IN}" \ + -progress /dev/stdout \ + -metadata "title='${TITLE}'" \ + ${output_flags} \ + -movflags +faststart \ + "${OUT}" \ + -y + else + output_flags=" + -map 0 + -map -0:d + -c:a copy + -c:s copy + " + + if (( sw_decode == 1 )); then + echo "${PREFIX} Content will be transcoded from ${codec} to Matroska(yuv420p:h264:${width}x${height}) with sw decode:hw encode." >&2 + input_flags=" + -hwaccel qsv + " + output_flags=" + -pix_fmt nv12 + -vf scale=${width}:${height} + -vf hwupload=extra_hw_frames=64,scale_qsv=w=${width}:h=${height},vpp_qsv=format=nv12 + ${output_flags} + -global_quality 20 + -c:v h264_qsv + " + else + if [[ "${codec}" =~ .*264.* ]]; then + echo "${PREFIX} Content will be transcoded from ${codec} to Matroska(yuv420p:h264:${width}x${height}) with hw decode:hw encode." >&2 + input_flags=" + -hwaccel qsv + -c:v h264_qsv + " + output_flags=" + -vf scale_qsv=w=${width}:h=${height},hwdownload,format=nv12 + ${output_flags} + -global_quality 20 + -c:v h264_qsv + " + elif [[ "${codec}" = .*265.* ]]; then + echo "${PREFIX} Content will be transcoded from ${codec} to Matroska(yuv420p:h264:${width}x${height}) with hw decode:hw encode." >&2 + input_flags=" + -hwaccel qsv + -c:v hevc_qsv + " + output_flags=" + -vf scale_qsv=w=${width}:h=${height},hwdownload,format=nv12 + ${output_flags} + -global_quality 20 + -c:v h264_qsv + " + else + echo "${PREFIX} Content will be transcoded from ${codec} to Matroska(yuv420p:h264:${width}x${height}) using software only." >&2 + input_flags=" + " + output_flags=" + ${output_flags} + -c:v libx264 + -vf scale=${width}:${height},format=yuv420p + -preset veryslow + -crf 20 + " + fi + fi + + docker run \ + --device=/dev/dri \ + --user=$(id -u) \ + --rm \ + ${ADD_GROUPS} \ + -v "${DIN}":"${DIN}" \ + -v "${DOUT}":"${DOUT}" \ + --entrypoint=ffmpeg \ + intel-media-ffmpeg \ + ${quiet} \ + -nostdin \ + ${input_flags} \ + -i "${IN}" \ + -progress /dev/stdout \ + -metadata "title='${TITLE}'" \ + ${output_flags} \ + -movflags +faststart \ + "${OUT}" \ + -y + fi +} + +function move_to_backup { + IN="$1" + base="$(dirname "${IN}")" + if [ ! -d "${base}" ]; then + continue + fi + base=/multimedia/backup"${base}" + if [ ! -d "${base}" ]; then + mkdir -p "${base}" || fail "Unable to mkdir '${base}'" + fi + if [ -d "${base}" ]; then + mv "${IN}" "${base}"/ || fail "Unable to move '$IN'" + fi +} + +function check_and_convert { + IN="$1" + PREFIX="$2" + + width=$(ffprobe \ + -v error \ + -select_streams v \ + -show_entries "stream=width" \ + "${IN}" | sed -ne 's,width=,,p' + ) + + height=$(ffprobe \ + -v error \ + -select_streams v \ + -show_entries "stream=height" \ + "${IN}" | sed -ne 's,height=,,p' + ) + + pix_fmt=$(ffprobe \ + -v error \ + -select_streams v \ + -show_entries "stream=pix_fmt" \ + "${IN}" | sed -ne 's,pix_fmt=,,p' + ) + + title=$(ffprobe -v error -show_entries format_tags=title -of default=noprint_wrappers=1 "${IN}" | sed -e s,^TAG:title=,,) + if [[ "${title}" == "" ]]; then + title=$(basename ${IN}) + title=${title%.???} + title="${title//./ }" + fi + + info=($(ffprobe -v error \ + -select_streams v:0 \ + -show_entries stream=codec_name:format=format_name \ + -of default=noprint_wrappers=1:nokey=1 "${IN}")) + codec=${info[0]} + container=${info[1]} + + update=0 + transcode=0 + if (( width > 1920 )); then + echo "${PREFIX} Has a width of ${width} and will be scaled to 1920." + height=$((((((1920*height/width))*2))/2)) + width=1920 + update=1 + transcode=1 + fi + + if [[ "${pix_fmt}" != "yuv420p" ]]; then + echo "${PREFIX} Has pixel format of ${pix_fmt} and will be converted to yuv420p." + update=1 + transcode=1 + fi + + if [[ "$codec" != "h264" ]]; then + echo "${PREFIX} Is using ${codec} and will be converted to h264." + update=1 + transcode=1 + fi + + if [[ ! "${container}" =~ mkv|matroska ]]; then + echo "${PREFIX} Is contained in ${container} and will be repacked to matroska." + update=1 + fi + + if (( update == 0 )); then + echo "${PREFIX} Skipping ${IN} as it is already normalized." + return 0; + fi + + echo -n "${PREFIX} Counting frames in ${IN}: " + FRAMES=$(ffprobe -v error -select_streams v:0 -count_packets -show_entries stream=nb_read_packets -of csv=p=0 "${IN}") || fail "Unable to count frames" + echo -e "\n${PREFIX} ${FRAMES} frames." + + IN="$(realpath "${IN}")" + OUT="${IN/%.???/.mkv}" + ORIG="${OUT}" + if [[ "${OUT}" == "${IN}" ]]; then + OUT="${OUT/.mkv/.transcoded.mkv}" + fi + + echo "${PREFIX} Output file is $(basename "${OUT}")." + + # Use stdbuf to flush stdout/stderr every line + STARTTIME=$(date +%s) + LASTTIME=${STARTTIME} + LASTPOS=0 + + log_file=$(mktemp) + if (( transcode == 0 )); then + echo "${PREFIX} Content will be repacked to Matroska(yuv420p:h264:${width}x${height})." + + input_flags=" + " + + output_flags=" + -map 0 + -map -0:d + -c:a copy + -c:s copy + -c:v copy + " + + command=" + ffmpeg \ + ${quiet} \ + -nostdin \ + ${input_flags} \ + -i \"${IN}\" \ + -progress /dev/stdout \ + -metadata title=\"${title}\" \ + ${output_flags} \ + -movflags +faststart \ + \"${OUT}\" \ + -y + " + else + command=" + ffmpeg_hw_or_sw \ + \"${IN}\" \ + \"${OUT}\" \ + \"${title}\" \ + ${width} \ + ${height} \ + \"${codec}\" \ + \"${pix_fmt}\" + " + fi + echo "Running: $command" + { eval ${command} || { + echo "${PREFIX} FFMPEG failed" | tee ${log_file} + false + } + } | while read line; do + if [[ "${line}" == "${PREFIX} FFMPEG failed" ]]; then + cat ${log_file} + fail "${PREFIX} Terminating" + false + break + fi + POS=$(echo $line | sed -n 's/^frame=*\(.*\)/\1/p') + if [[ "${POS}" == "" ]]; then + true + continue + fi + + NOW=$(date +%s) + ELAPSEDTIME=$(( NOW - LASTTIME )) + if (( ELAPSEDTIME <= 5 )); then + true + continue + fi + + ELAPSEDFRAMES=$(( POS - LASTPOS )) + REMAININGFRAMES=$(( FRAMES - POS )) + REMAININGFRAMES=$(( REMAININGFRAMES * 1000 )) # convert to ms + FRAMERATE=$(( ELAPSEDFRAMES / ELAPSEDTIME )) + if (( FRAMERATE != 0 )); then + REMAININGMS=$(( REMAININGFRAMES / FRAMERATE )) + printf "\r%*s\r%s" $(tput cols) " " "${PREFIX} $(mstodate $(( NOW - STARTTIME ))000). Transcode $((ELAPSEDFRAMES / ELAPSEDTIME ))fps. Frame $POS of $FRAMES $(( $(( 100 * POS)) / FRAMES ))%. ETA $(mstodate ${REMAININGMS}) remaining." + else + printf "\r%*s\r%s" $(tput cols) " " "${PREFIX} $(mstodate $(( NOW - STARTTIME ))000). Transcode $((ELAPSEDFRAMES / ELAPSEDTIME ))fps. Frame $POS of $FRAMES $(( $(( 100 * POS)) / FRAMES ))%. No frames processed in last ${ELAPSEDTIME} seconds." + fi + LASTTIME=${NOW} + LASTPOS=${POS} + done || fail "${PREFIX} Unable to transcode" + + if grep -q "${PREFIX} FFMPEG failed" ${log_file}; then + exit -1 + fi + + move_to_backup "${IN}" + + if [[ "${ORIG}" == "${IN}" ]]; then + # If the original file was an '.mkv' then OUT was '.transcoded.mkv' + # during transcode. Now that the transcode is complete, set the + # file back to the original name. + mv "${OUT}" "${IN}" || fail "${PREFIX} Unable to mv '${OUT}' -> '${IN}'" + fi + + rm ${log_file} + NOW=$(date +%s) + echo -e "${PREFIX} Completed in $(mstodate $(( NOW - STARTTIME ))000)" +} + +if [ -z "$1" ]; then + find . -name '* *' -type d | while read file; do mv "${file}" "${file// /_}" || fail "Unable to rename dir ${file}"; done + find . -name '* *' -type f | while read file; do mv "${file}" "${file// /_}" || fail "Unable to rename file ${file}"; done + total=$(find . -type f \ + -and -not -path "./backup/*" | wc -l) + current=1 + find . -type f -name '*mkv' -or -name '*mp4' -or -name '*avi' \ + -and -not -path "./backup/*" | sort | while read file; do + check_and_convert "${file}" "${current}/${total}:" + current=$((current+1)) + done +else + for file in "$*"; do + check_and_convert "${file}" 1 1 + done +fi && { + cat << EOF +To transfer the updated files: + +rsync -avprlP --remove-source-files /multimedia/Downloads/ azurite:/multimedia/ +find /multimedia/Downloads/ -type d -empty -delete + +EOF +} diff --git a/video-convert b/video-convert index 55a8b2c..b98c547 100755 --- a/video-convert +++ b/video-convert @@ -3,6 +3,13 @@ # https://www.reddit.com/r/VP9/comments/g9uzzv/hardware_encoding_vp9_on_intel/ log_file="" + +force=0 +if [[ "$1" == "-f" ]]; then + force=1 + shift +fi + fail() { echo "$*" >&2 if [[ -e ${log_file} ]]; then @@ -52,7 +59,8 @@ mstodate() { done } -function move { +function move_to_backup { + IN="$1" base="$(dirname "${IN}")" if [ ! -d "${base}" ]; then continue @@ -70,49 +78,75 @@ function convert { IN="$1" SRC_CODEC="$2" FORMAT="$3" - OUT="${IN/%.???/.mkv}" + current="$4" + total="$5" IN="$(realpath "${IN}")" - OUT="$(realpath "${OUT}")" + OUT="${IN/%.???/.mkv}" ORIG="${OUT}" if [[ "${OUT}" == "${IN}" ]]; then OUT="${OUT/.mkv/.transcoded.mkv}" - [ -e "${OUT}" ] && rm "${OUT}" - else - [ -e "${OUT}" ] && move && return 0 - [ -e /multimedia/backup/"${OUT}" ] && return 0 + fi + + width=0 + eval $(ffprobe -v error -show_entries 'stream=width' -select_streams v "${IN}" | grep width) + if (( width == 0 )); then + fail "Unable to determine width of ${IN}" + fi + if (( width > 1920 )); then + echo "Limiting width to 1920" + width=1920 fi TITLE="${IN%.*}" TITLE="${TITLE##*/}" if [[ "${SRC_CODEC}" == "h264" ]]; then - echo "Content will have format changed from ${FORMAT}/h264 to mkv/h264." - # Just change container to matroska - input_flags=" - " - output_flags=" - -c copy - " + if (( force == 1 )); then + input_flags=" + -hwaccel qsv + " +# HW decode here gives a MXF device error. Not sure why. +# -c:v h264_qsv + output_flags=" + -c copy + -c:v h264_qsv + -vf hwupload=extra_hw_frames=64 + -vf vpp_qsv=format=nv12,scale_qsv=w=${width} + -global_quality 20 + " + else + echo "${current}/${total}: Content will have format changed from ${FORMAT}/h264 to mkv/h264." + # Just change container to matroska + input_flags=" + " + output_flags=" + -c copy + " + fi else - echo "Content will be transcoded from ${FORMAT}/${SRC_CODEC} to mkv/h264." + echo "${current}/${total}: Content will be transcoded from ${FORMAT}/${SRC_CODEC} to mkv/h264." # Convert all video streams to h264. Copy all other streams unchanged (audio, subtitle, etc.) # Move headers to start of the file for fast start # Use "reasonable" quality level of 20, slow encode # - input_flags=" - -hwaccel qsv - -c:v hevc_qsv - " - output_flags=" - -c copy - -vf vpp_qsv=format=nv12 - -c:v h264_qsv - -global_quality 20 - " + input_flags=" + -hwaccel qsv + -c:v hevc_qsv + " + output_flags=" + -c copy + -vf vpp_qsv=format=nv12,scale_qsv=${width} + -c:v h264_qsv + -global_quality 20 + " fi - echo -n "Counting frames in ${IN}: " + echo -n "${current}/${total}: Counting frames in ${IN}: " FRAMES=$(ffprobe -v error -select_streams v:0 -count_packets -show_entries stream=nb_read_packets -of csv=p=0 "${IN}") || fail "Unable to count frames" - echo "${FRAMES} frames." + echo "${current}/${total}: ${FRAMES} frames." + + if [[ "${ORIG}" != "${OUT}" ]]; then + echo "${current}/${total}: Output file will be $(basename "${OUT}")." + fi # Use stdbuf to flush stdout/stderr every line STARTTIME=$(date +%s) @@ -146,13 +180,12 @@ function convert { ${output_flags} \ -movflags +faststart \ "${OUT}" \ - -y || echo "FFMPEG failed" | tee ${log_file} + -y || { echo "FFMPEG failed" | tee ${log_file} ; false ; } } | while read line; do - if [[ "${line}" == "ffmpeg failed" ]]; then + if [[ "${line}" == "FFMPEG failed" ]]; then cat ${log_file} fail "Terminating" false - exit -1 break fi POS=$(echo $line | sed -n 's/^frame=*\(.*\)/\1/p') @@ -174,26 +207,35 @@ function convert { FRAMERATE=$(( ELAPSEDFRAMES / ELAPSEDTIME )) if (( FRAMERATE != 0 )); then REMAININGMS=$(( REMAININGFRAMES / FRAMERATE )) - printf "\r%*s\r%s" $(tput cols) " " "Processing for $(mstodate $(( NOW - STARTTIME ))000). Processing at ($((ELAPSEDFRAMES / ELAPSEDTIME ))fps). Now at frame $POS of $FRAMES $(( $(( 100 * POS)) / FRAMES ))%. Estimating $(mstodate ${REMAININGMS}) remaining." + printf "\r%*s\r%s" $(tput cols) " " "${current}/${total}: $(mstodate $(( NOW - STARTTIME ))000). Transcode $((ELAPSEDFRAMES / ELAPSEDTIME ))fps. Frame $POS of $FRAMES $(( $(( 100 * POS)) / FRAMES ))%. ETA $(mstodate ${REMAININGMS}) remaining." else - printf "\r%*s\r%s" $(tput cols) " " "Processing for $(mstodate $(( NOW - STARTTIME ))000). Processing at ($((ELAPSEDFRAMES / ELAPSEDTIME ))fps). Now at frame $POS of $FRAMES $(( $(( 100 * POS)) / FRAMES ))%. No frames processed in last ${ELAPSEDTIME} seconds." + printf "\r%*s\r%s" $(tput cols) " " "${current}/${total}: $(mstodate $(( NOW - STARTTIME ))000). Transcode $((ELAPSEDFRAMES / ELAPSEDTIME ))fps. Frame $POS of $FRAMES $(( $(( 100 * POS)) / FRAMES ))%. No frames processed in last ${ELAPSEDTIME} seconds." fi LASTTIME=${NOW} LASTPOS=${POS} - done && move && { - if [[ "${ORIG}" == "${IN}" ]]; then - # If the original file was an '.mkv' then OUT was '.transcoded.mkv' - # during transcode. Now that the transcode is complete, set the - # file back to the original name. - mv "${OUT}" "${IN}" || fail "Unable to mv '${OUT}' -> '${IN}'" - fi - } + done || fail 'Unable to transcode' + + if grep -q "FFMPEG failed" ${log_file}; then + exit -1 + fi + + move_to_backup "${IN}" + + if [[ "${ORIG}" == "${IN}" ]]; then + # If the original file was an '.mkv' then OUT was '.transcoded.mkv' + # during transcode. Now that the transcode is complete, set the + # file back to the original name. + mv "${OUT}" "${IN}" || fail "Unable to mv '${OUT}' -> '${IN}'" + fi + rm ${log_file} NOW=$(date +%s) - echo -e "\nCompleted in $(mstodate $(( NOW - STARTTIME ))000)" + echo -e "\n${current}/${total}: Completed in $(mstodate $(( NOW - STARTTIME ))000)" } function check_and_convert { + current="$2" + total="$3" video_detect "$1" | while read entry; do # # file:codec:format file="${entry%%|*}" # file @@ -204,11 +246,11 @@ function check_and_convert { if [[ "${codec}" == "none" ]]; then continue fi - if [[ "${codec}" == "h264" ]] && [[ "${format}" =~ mkv|matroska ]]; then + if (( force == 0 )) && [[ "${codec}" == "h264" ]] && [[ "${format}" =~ mkv|matroska ]]; then # If container is not mkv, convert to mkv container - echo "Skipping ${file} as it is already mkv/h264." + echo "${current}/${total}: Skipping ${file} as it is already mkv/h264." else - convert "${file}" "${codec}" "${format}" + convert "${file}" "${codec}" "${format}" "$current" "$total" fi done } @@ -216,20 +258,24 @@ function check_and_convert { if [ -z "$1" ]; then find . -name '* *' -type d | while read file; do mv "${file}" "${file// /_}" || fail "Unable to rename dir ${file}"; done find . -name '* *' -type f | while read file; do mv "${file}" "${file// /_}" || fail "Unable to rename file ${file}"; done + total=$(find . -type f \ + -and -not -path "./backup/*" | wc -l) + current=0 find . -type f \ -and -not -path "./backup/*" | sort | while read file; do - check_and_convert "${file}" + check_and_convert "${file}" ${current} ${total} + current=$((current+1)) done else for file in "$*"; do - check_and_convert "${file}" + check_and_convert "${file}" 1 1 done -fi - -cat << EOF +fi && { + cat << EOF To transfer the updated files: rsync -avprlP --remove-source-files /multimedia/Downloads/ azurite:/multimedia/ find /multimedia/Downloads/ -type d -empty -delete EOF +}