Merge branch 'fix-6' into 'master'

Refactored scripts/* and .gitlab-ci.yml

Closes #6

See merge request vtt/sws/osgc/solutions/intel-media-ffmpeg!6

.dockerignore

@@ -1,6 +1,6 @@
 *
 !assets
-!Dockerfile
-!Dockerfile-*
+!Dockerfile*
 !SOLUTION
 !MANIFEST
+Dockerfile.solution*

.gitignore

@@ -1,2 +1,2 @@
 media
-
+SECRETS

.gitlab-ci.yml

@@ -1,45 +1,46 @@
 stages:
-- build-devel
-- test-devel
-- publish-rolling
+- build
+- test
+- publish
 
 # Required Input From Trigger
 # OS_DISTRO: ex. ubuntu
 # OS_RELEASE: ex. eoan
 # PACKAGE_STREAM: ex. eoan
+# BUILD: Agama Build number
+# BRANCH: Agama CI branch (releases_*, master, embargo)
 
 variables:
   ci_enabled: "true"
-  PACKAGE_REPOSITORY: "https://repositories.intel.com/graphics"
-  REGISTRY_URL: "amr-registry.caas.intel.com/vtt-osgc/solutions"
   CONTAINER: "intel-media-ffmpeg"
+  TARGET_TAG: "${OS_DISTRO}-${PACKAGE_STREAM}"
+  TAG: "${CI_PIPELINE_IID}-${CI_COMMIT_REF_NAME}-${OS_DISTRO}-${PACKAGE_STREAM}"
 
-build_devel_image:
-  stage: "build-devel"
+build_image:
+  stage: "build"
   script:
   - ./scripts/build-dockerfile.sh 
-  - ./scripts/build-image.sh
-  - ./scripts/push-test-image.sh
+  - ./scripts/build-image.sh --no-cache
+  - ./scripts/push-image.sh
   tags:
   - builder
   - ubuntu
   only:
   - triggers
 
-test_devel_image:
-  stage: "test-devel"
+test_image:
+  stage: "test"
   script:
-  - ./scripts/test-tag.sh
+  - ./scripts/test-image.sh
   tags:
   - gen9
   only:
   - triggers
 
-publish_rolling_image:
-  stage: "publish-rolling"
+publish_image:
+  stage: "publish"
   script:
-  - ./scripts/push-rolling-image.sh
-  - ./scripts/remove-pipeline-tag.sh
+  - ./scripts/promote-image.sh
   tags:
   - builder
   - ubuntu

Dockerfile

@@ -132,7 +132,7 @@ RUN echo "deb [trusted=yes arch=amd64] https://osgc.jf.intel.com/internal/ubuntu
 
 
 #
-# Template from templates/ubuntu/25-graphics-user.in
+# Template from templates/ubuntu/25-create-user.in
 #
 # Create user 'user' and add them to 'sudo' for sudo access and set
 # the passwd to 'user'
@@ -150,7 +150,7 @@ RUN groupadd -r user \
     -s /bin/bash \
     -r -m \
     -g user \
-    -G sudo \
+    -G sudo,video \
     -p $(echo "user" | openssl passwd -stdin) user
 
 # Set 'sudo' to NOPASSWD for all container users

Dockerfile.rhel-8.0

@@ -48,11 +48,9 @@ RUN { \
   echo "no_proxy=${no_proxy}" ; \
 } | tee -a /etc/dnf/dnf.conf /etc/yum.conf
 
-
 #
 # Template from templates/rhel/10-rhel-partner.in
 #
-
 # Remove any pre-configured repositories
 RUN rm -rf /etc/yum.repos.d/*
 
@@ -214,6 +212,16 @@ RUN { \
   echo "" ; \
 } > /etc/yum.repos.d/RHEL-8-rt-intel-partner.repo
 
+#
+# Template from templates/rhel/15-upgrade.in
+#
+# Update package lists, and upgrade to the latest packages
+#
+# Failure to do this will result in GPG errors later
+RUN dnf clean all \
+  && dnf -y upgrade
+
+
 #
 # Template from templates/rhel/20-repositories-intel-com.in
 #
@@ -236,14 +244,14 @@ RUN { \
 RUN { \
   echo "[intel-graphics]" ; \
   echo "name=Intel Graphics Drivers Repository" ; \
-  echo "baseurl=https://repositories.intel.com/graphics/rhel/8.0/" ; \
+  echo "baseurl=https://osgc.jf.intel.com/internal/rhel/8.0/" ; \
   echo "sslverify=0" ; \
   echo "enabled=1" ; \
   echo "gpgcheck=0" ; \
 } > /etc/yum.repos.d/intel-graphics.repo
 
 #
-# Template from templates/rhel/25-graphics-user.in
+# Template from templates/rhel/25-create-user.in
 #
 # Create user 'user' and add them to 'sudo' for sudo access and set
 # the passwd to 'user'
@@ -258,11 +266,11 @@ RUN groupadd -r user \
     -s /bin/bash \
     -r -m \
     -g user \
-    -G sudo \
+    -G wheel,video \
     -p $(echo "user" | openssl passwd -stdin) user
 
-# Set 'sudo' to NOPASSWD for all container users
-RUN sed -i -e 's,%sudo.*,%sudo ALL=(ALL) NOPASSWD:ALL,g' /etc/sudoers
+# Set 'wheel' to NOPASSWD for all container users
+RUN sed -i -e 's,%wheel.*,%wheel ALL=(ALL) NOPASSWD:ALL,g' /etc/sudoers
 
 USER user
 
@@ -351,13 +359,6 @@ ENTRYPOINT [ "/assets/entry" ]
 
 # Ensure that each Docker container self-documents the 
 # versions included in it
-
-RUN { \
-  echo "PACKAGE_REPOSITORY=https://repositories.intel.com/graphics" ; \
-  echo "RELEASE_INFO=N/A" ; \
-  echo "PACKAGE_STREAM=8.0" ; \
-  echo "OS_DISTRO=rhel" ; \
-  echo "OS_RELEASE=8.0" ; \
-} | sudo tee /assets/SOLUTION
+COPY SOLUTION /assets/
 COPY MANIFEST /assets/
 COPY Dockerfile.rhel-8.0 /assets/Dockerfile

scripts/build-image.sh

@@ -17,7 +17,6 @@ RELEASE_INFO=${RELEASE_INFO:-N/A}
 } || {
   TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}}
 }
-TAG=${CI_PIPELINE_IID}-${TAG}
 
 case "${OS_DISTRO}" in
   rhel)

scripts/clean-image.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-# Determine if it is Mac OS and switch to use gxargs instead
-CMD=xargs
-if [ $(which system_profiler) ]; then
-  CMD=gxargs
-fi
-
-# Bring in the variables from SOLUTION file, supporting
-# nested substitution
-. SOLUTION
-. MANIFEST
-
-VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION))
-VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST))
-for var in ${VARS[@]}; do
-  export ${var}
-done
-
-docker rmi ${REGISTRY_URL}/${CONTAINER}:devel

scripts/promote-image.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# Bring in the variables from SOLUTION file, supporting
+# nested substitution
+. SOLUTION
+. MANIFEST
+
+VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION))
+VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST))
+for var in ${VARS[@]}; do
+  export ${var}
+done
+
+RELEASE_INFO=${RELEASE_INFO:-N/A}
+[[ "${RELEASE_INFO}" == "N/A" ]] && {
+  TAG=${TAG:-${OS_DISTRO}-${PACKAGE_STREAM}-$(date +%Y%m%d)}
+} || {
+  TAG=${TAG:-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}}
+}
+
+[[ "${TARGET_TAG}" != "" ]] || {
+  echo >&2 "TARGET_TAG needs to be set to the tag to push to Harbor"
+  exit 1
+}
+
+function fail {
+  >&2 echo "$*"
+  exit -1
+}
+
+cat << EOF
+
+The following will publish latest rolling image:
+
+1. Pull ${CONTAINER}:${TAG}
+2. Tag #1 as ${CONTAINER}:${TARGET_TAG}
+3. Tag #1 as ${CONTAINER}:latest-${TARGET_TAG}
+4. Delete ${CONTAINER}:${TAG}
+5. Push #2
+6. Push #3
+
+EOF
+
+docker pull ${REGISTRY_URL}/${CONTAINER}:${TAG} ||
+  fail "Unable to pull image"
+
+docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \
+  ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG} ||
+  fail "Unable to tag image (1/2)"
+
+docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \
+  ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG} ||
+  fail "Unable to tag image (2/2)"
+
+# Parse out the FQDN from the REGISTRY_URL and escape
+# the group/project/path
+FQDN=${REGISTRY_URL%%/*}
+PROJECT=$(echo ${REGISTRY_URL#*/}/${CONTAINER} | sed s,/,%2F,g)
+
+echo -e "Deleting tag:\n  ${CONTAINER}:${TAG}\nFrom:\n  ${PROJECT}"
+
+RESULTS=$(curl --noproxy '*' -s -k \
+  -u ${HARBOR_USER}:${HARBOR_PASSWD} \
+  -i \
+  -X DELETE \
+  -H "accept: application/json" \
+  "https://${FQDN}/api/repositories/${PROJECT}/tags/${TAG}")
+
+echo ${RESULTS} | grep -q "HTTP.*200" && {
+  echo "Tag deleted successfully."
+} || {
+  >&2 echo "Error deleting tag:"
+  >&2 echo "${RESULTS}"
+  exit 1
+}
+
+docker push ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG} ||
+  fail "Unable to push ${CONTAINER}:${TARGET_TAG}"
+
+docker push ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG} ||
+  fail "Unable to push ${CONTAINER}:latest-${TARGET_TAG}"
+
+echo "Done tagging and pushing ${CONTAINER} as ${TARGET_TAG}"

scripts/push-image.sh

@@ -17,7 +17,6 @@ RELEASE_INFO=${RELEASE_INFO:-N/A}
 } || {
   TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}}
 }
-TAG=${CI_PIPELINE_IID}-${TAG}
 
 docker tag ${CONTAINER}:${TAG} ${REGISTRY_URL}/${CONTAINER}:${TAG} \
   && docker push ${REGISTRY_URL}/${CONTAINER}:${TAG} \

scripts/push-rolling-image.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-# Determine if it is Mac OS and switch to use gxargs instead
-CMD=xargs
-if [ $(which system_profiler) ]; then
-  CMD=gxargs
-fi
-
-# Bring in the variables from SOLUTION file, supporting
-# nested substitution
-. SOLUTION
-. MANIFEST
-
-VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION))
-VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST))
-for var in ${VARS[@]}; do
-  export ${var}
-done
-
-RELEASE_INFO=${RELEASE_INFO:-N/A}
-[[ "${RELEASE_INFO}" == "N/A" ]] && {
-  TAG=${TAG:-${OS_DISTRO}-${PACKAGE_STREAM}-$(date +%Y%m%d)}
-} || {
-  TAG=${TAG:-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}}
-}
-
-docker pull ${REGISTRY_URL}/${CONTAINER}:${CI_PIPELINE_IID}-test-build-${TAG}
-docker tag ${REGISTRY_URL}/${CONTAINER}:${CI_PIPELINE_IID}-test-build-${TAG} ${REGISTRY_URL}/${CONTAINER}:${TAG}
-docker tag ${REGISTRY_URL}/${CONTAINER}:${CI_PIPELINE_IID}-test-build-${TAG} ${REGISTRY_URL}/${CONTAINER}:latest-${OS_DISTRO}-${PACKAGE_STREAM}
-docker push ${REGISTRY_URL}/${CONTAINER}:${TAG}
-docker push ${REGISTRY_URL}/${CONTAINER}:latest-${OS_DISTRO}-${PACKAGE_STREAM}

scripts/remove-pipeline-tag.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-# Bring in the variables from SOLUTION file, supporting
-# nested substitution
-. SOLUTION
-. MANIFEST
-
-VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION))
-VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST))
-for var in ${VARS[@]}; do
-  export ${var}
-done
-
-RELEASE_INFO=${RELEASE_INFO:-N/A}
-[[ "${RELEASE_INFO}" == "N/A" ]] && {
-  TAG=${TAG:-test-build-${OS_DISTRO}-${PACKAGE_STREAM}-$(date +%Y%m%d)}
-} || {
-  TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}}
-}
-TAG=${CI_PIPELINE_IID}-${TAG}
-
-curl -i -k -u ${HARBOR_USER}:${HARBOR_PASSWD} \
-    -X DELETE "https://amr-registry.caas.intel.com/api/repositories/vtt-osgc%2Fsolutions%2F${CONTAINER}/tags/${TAG}" \
-    -H "accept: application/json"

scripts/test-image.sh

@@ -1,11 +1,5 @@
 #!/bin/bash
 
-# Determine if it is Mac OS and switch to use gxargs instead
-CMD=xargs
-if [ $(which system_profiler) ]; then
-  CMD=gxargs
-fi
-
 # Bring in the variables from SOLUTION file, supporting
 # nested substitution
 . SOLUTION
@@ -23,19 +17,42 @@ RELEASE_INFO=${RELEASE_INFO:-N/A}
 } || {
   TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}}
 }
-TAG=${CI_PIPELINE_IID}-${TAG}
 
 docker pull ${REGISTRY_URL}/${CONTAINER}:${TAG}
 
+function fail {
+  >&2 echo "$*"
+  exit -1
+}
+
+VIDEO=$(getent group video | sed -E 's,^video:[^:]*:([^:]*):.*$,\1,')
+RENDER=$(getent group render | sed -E 's,^render:[^:]*:([^:]*):.*$,\1,')
+
+cat << EOF
+
+Running test with:
+
+video group : ${VIDEO}
+render group: ${RENDER}
+
+-------------------------------------------------------------------------------
+
+EOF
+
+echo -n "Downloading test content: "
 mkdir $(pwd)/media
-wget -O $(pwd)/media/AUD_MW_E.264 \
-  https://fate-suite.libav.org/h264-conformance/AUD_MW_E.264
-chmod -R 777 $(pwd)/media
+wget -q -O $(pwd)/media/AUD_MW_E.264 \
+  https://fate-suite.libav.org/h264-conformance/AUD_MW_E.264 ||
+  fail "Unable to download test content."
+chmod -R 777 $(pwd)/media || fail "Unable to set permissions"
+echo "done"
+
 docker run \
+  --group-add ${VIDEO} \
+  --group-add ${RENDER} \
   --rm \
-  -a STDOUT \
   --device=/dev/dri \
   -e QSV_DEVICE=${QSV_DEVICE:-/dev/dri/renderD128} \
   --volume $(pwd)/media:/media \
   ${REGISTRY_URL}/${CONTAINER}:${TAG} \
-  test
+  test || fail "Unable to execute 'test' on image."

scripts/trigger.sh

@@ -0,0 +1,153 @@
+#!/bin/bash
+
+# 1. List project triggers, looking for 'sys_osgc CI trigger'
+# 2. If not found, create the 'sys_osgc CI trigger'
+# 3. Invoke the trigger
+
+# Bring in the variables from SOLUTION file, supporting
+# nested substitution
+. SOLUTION
+. MANIFEST
+
+VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION))
+VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST))
+for var in ${VARS[@]}; do
+  export ${var}
+done
+
+# The following constants are used so we can run trigger.sh
+# against a known build
+BUILD=${BUILD:-7517585}
+BRANCH=${BRANCH:-releases_19.4}
+REF=${REF:-master}
+
+PROJECT="vtt/sws/osgc/solutions/$(basename $(pwd))"
+URI_PROJECT=$(echo ${PROJECT} | sed -E 's,/,%2F,g')
+
+eval $(grep ^SYS_OSGC_TOKEN SECRETS)
+
+[[ "$SYS_OSGC_TOKEN" == "" ]] && {
+  >&2 echo "SECRETS needs to contain a GitLab token for SYS_OSGC_TOKEN."
+  exit -1
+}
+
+GITLAB="https://gitlab.devtools.intel.com"
+
+echo "Looking for CI trigger"
+
+RESULTS=$(curl --noproxy '*' -s -X GET \
+      --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \
+      --header "Content-Type: application/json" \
+      ${GITLAB}/api/v4/projects/${URI_PROJECT}/triggers)
+
+TOKEN=$(echo ${RESULTS} | sed -E 's#([[{},])#\1\n#g' | {
+  STATE=0
+  MATCH=
+  TOKEN=
+  # Parse the JSON looking for the trigger token and description of
+  # 'sys_osgc CI Trigger'
+  while read line; do
+    case ${STATE} in
+    0)
+      [[ ${line} =~ ^[[:space:]]*\{ ]] && STATE=1
+      ;;
+    1)
+      [[ "${MATCH}" != "sys_osgc CI trigger" ]] &&
+        MATCH=$(echo ${line} |
+          sed -nE 's,^.*"description":\s*"([^"]+)".*$,\1,p')
+      [[ "${TOKEN}" == "" ]] &&
+        TOKEN=$(echo ${line} |
+          sed -nE 's,^.*"token":\s*"([^"]+)".*$,\1,p')
+      [[ "${TOKEN}" != "" ]] && [[ "${MATCH}" == "sys_osgc CI trigger" ]] && break
+      [[ "${line}" =~ .*\s*} ]] && {
+        MATCH=
+        TOKEN=
+        STATE=0
+      }
+      ;;
+    esac
+  done
+  [[ "${TOKEN}" == "" ]] && echo -n ""
+  echo -n "${TOKEN}"
+})
+
+[[ "${TOKEN}" == "" ]] && {
+  echo "'sys_osgc CI trigger' does not exist on ${PROJECT}."
+
+  while read -n1 -r -p "Create new trigger [y|N]?"; do
+    case $REPLY in
+    y|Y)
+      echo ""
+      break
+      ;;
+    n|N|"")
+      echo " Exiting."
+      exit 1
+      ;;
+    *)
+      echo ""
+      ;;
+    esac
+  done
+
+
+  RESULTS=$(curl --noproxy '*' -s -X POST \
+      --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \
+      -F 'description="sys_osgc CI trigger"' \
+      ${GITLAB}/api/v4/projects/${URI_PROJECT}/triggers)
+
+  TOKEN=$(echo ${RESULTS} | sed -nE 's,.*"token":\s*"([^"]*).*,\1,p')
+
+  [[ "${TOKEN}" == "" ]] && {
+    >&2 echo "Unable to create trigger"
+    >&2 echo "${RESULTS}"
+    exit 1
+  }
+
+  echo "Created token: ${TOKEN}"
+}
+
+echo "Using token: ${TOKEN}"
+
+function post {
+  RESULTS=$(curl --noproxy '*' -s -X POST \
+      -F "token=${TOKEN}" \
+      -F "ref=${REF}" \
+      -F "variables[OS_DISTRO]=${OS_DISTRO}" \
+      -F "variables[OS_RELEASE]=${OS_RELEASE}" \
+      -F "variables[PACKAGE_STREAM]=${PACKAGE_STREAM}" \
+      -F "variables[PACKAGE_REPOSITORY]=${PACKAGE_REPOSITORY}" \
+      -F "variables[BRANCH]=${BRANCH}" \
+      -F "variables[BUILD]=${BUILD}" \
+      -F "variables[BUILDNUMBER]=${BUILD}" \
+      ${GITLAB}/api/v4/projects/${URI_PROJECT}/trigger/pipeline)
+
+  PIPELINE=$(echo ${RESULTS} | sed -nE 's,^.*details_path":"([^"]*)".*,\1,p')
+
+  [[ "${PIPELINE}" == "" ]] && {
+    >&2 echo "Unable to find pipeline details URL"
+    >&2 echo ${RESULTS}
+    return 1
+  }
+
+  echo -e "Pipeline URL:\n\n  ${GITLAB}${PIPELINE}\n"
+  true
+}
+
+echo "Triggering pipeline on project ${PROJECT} for BUILD=${BUILD} and BRANCH=${BRANCH}"
+while read -n1 -r -p "Is this correct [y|N]?"; do
+  case $REPLY in
+  y|Y)
+    echo ""
+    post
+    break
+    ;;
+  n|N|"")
+    echo " Exiting."
+    exit 1
+    ;;
+  *)
+    echo ""
+    ;;
+  esac
+done

scripts/watch.sh

@@ -0,0 +1,146 @@
+#!/bin/bash
+
+PIPELINE=$1
+[[ "${PIPELINE}" == "" ]] && {
+  >&2 echo "usage: watch PIPELINE"
+  exit -1
+}
+
+# Bring in the variables from SOLUTION file, supporting
+# nested substitution
+. SOLUTION
+
+VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION))
+for var in ${VARS[@]}; do
+  export ${var}
+done
+
+PROJECT="vtt/sws/osgc/solutions/$(basename $(pwd))"
+URI_PROJECT=$(echo ${PROJECT} | sed -E 's,/,%2F,g')
+
+eval $(grep ^SYS_OSGC_TOKEN SECRETS)
+
+[[ "$SYS_OSGC_TOKEN" == "" ]] && {
+  >&2 echo "SECRETS needs to contain a GitLab token for SYS_OSGC_TOKEN."
+  exit -1
+}
+
+GITLAB="https://gitlab.devtools.intel.com"
+
+echo "Looking for pipeline jobs for ${PIPELINE}"
+
+function get_jobs {
+  ECHO=$1
+  # Split the JSON results into line-by-line breakouts for
+  # easy sed matching
+  sed -E 's#([[{},]|])#\n\1\n#g' | grep '^[^,]' | {
+    SCOPE=0
+    JOB=""
+    STATUS=""
+    while read line; do
+      case ${SCOPE} in
+      2)
+        echo $line | grep -qE '^([{[])' && {
+          SCOPE=$((SCOPE+1))
+          (( ECHO )) && echo "Scope: ${SCOPE}"
+          continue
+        }
+        echo $line | grep -qE '^([}]|])' && {
+          SCOPE=$((SCOPE-1))
+          (( ECHO )) && echo "Scope: ${SCOPE}"
+          continue
+        }
+
+        [[ "${JOB}" == "" ]] &&
+          JOB=$(echo ${line} |
+            sed -nE 's,^"id"\s*:\s*([0-9]+).*$,\1,p')
+        [[ "${STATUS}" == "" ]] &&
+          STATUS=$(echo ${line} |
+            sed -nE 's,^.*"status"\s*:\s*"([^"]+).*$,\1,p')
+        [[ "${JOB}" != "" ]] && [[ "${STATUS}" != "" ]] && {
+          echo "${JOB}:${STATUS} "
+          JOB=""
+          STATUS=""
+        }
+        ;;
+      *)
+        echo $line | grep -qE '^([{[])' && {
+          SCOPE=$((SCOPE+1))
+          (( ECHO )) && echo "Scope: ${SCOPE}"
+          continue
+        }
+        echo $line | grep -qE '^([}]|])' && {
+          (( SCOPE == 0 )) && {
+            return 0
+          }
+          SCOPE=$((SCOPE-1))
+          (( ECHO )) && echo "Scope: ${SCOPE}"
+          continue
+        }
+      esac
+    done
+  }
+}
+
+function update_log {
+  ID=$1
+  [[ ! -d "logs" ]] && mkdir "logs"
+
+  curl --noproxy '*' -s -X GET \
+      --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \
+      "${GITLAB}/api/v4/projects/${URI_PROJECT}/jobs/${ID}/trace" > "logs/${ID}-latest"
+
+  LINES=1
+  [[ -e "logs/${ID}" ]] && {
+    LINES=$(wc -l "logs/${ID}")
+    LINES=${LINES%% *}
+  }
+
+  tail -n +${LINES} "logs/${ID}-latest"
+  mv "logs/${ID}-latest" "logs/${ID}"
+}
+
+while true; do
+  RESULTS=$(curl --noproxy '*' -s -X GET \
+      --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \
+      --header "Content-Type: application/json" \
+      "${GITLAB}/api/v4/projects/${URI_PROJECT}/pipelines/${PIPELINE}/jobs")
+
+  JOBS=($(echo "${RESULTS}" | get_jobs))
+  echo "JOBS: ${JOBS[@]}"
+
+  MORE=0
+  FAIL=0
+  for JOB in ${JOBS[@]}; do
+    ID=${JOB%%:*}
+    STATUS=${JOB##*:}
+    case ${STATUS,,} in
+    pending)
+      MORE=$((MORE+1))
+      ;;
+    created)
+      MORE=$((MORE+1))
+      ;;
+    running)
+      update_log ${ID}
+      MORE=$((MORE+1))
+      ;;
+    success)
+      update_log ${ID}
+      ;;
+    cancelled)
+      FAIL=1
+      ;;
+    failed)
+      FAIL=1
+      ;;
+    esac
+  done
+  (( MORE == 0 )) && {
+    echo "No more jobs. Exiting."
+    exit ${FAIL}
+  }
+
+  echo "Sleeping 5 seconds for ${MORE} jobs."
+  sleep 5
+done

templates/centos/25-create-user.in

@@ -11,7 +11,7 @@ RUN groupadd -r user \
     -s /bin/bash \
     -r -m \
     -g user \
-    -G wheel \
+    -G wheel,video \
     -p $(echo "user" | openssl passwd -stdin) user
 
 # Set 'wheel' to NOPASSWD for all container users

templates/rhel/25-create-user.in

@@ -11,7 +11,7 @@ RUN groupadd -r user \
     -s /bin/bash \
     -r -m \
     -g user \
-    -G wheel \
+    -G wheel,video \
     -p $(echo "user" | openssl passwd -stdin) user
 
 # Set 'wheel' to NOPASSWD for all container users

templates/ubuntu/25-create-user.in

@@ -14,7 +14,7 @@ RUN groupadd -r user \
     -s /bin/bash \
     -r -m \
     -g user \
-    -G sudo \
+    -G sudo,video \
     -p $(echo "user" | openssl passwd -stdin) user
 
 # Set 'sudo' to NOPASSWD for all container users