Refactored scripts/* and .gitlab-ci.yml

Fix #6: Define TAG in .gitlab-ci.yml and update scripts to defer to it.

renamed:    scripts/push-test-image.sh -> scripts/push-image.sh
renamed:    scripts/test-tag.sh -> scripts/test-image.sh

Refactored some of the scripts/* so they are generic and not tied to CI usages.

Added error checking and output to push-rolling-image.sh and remove-tag.sh

Added scripts/trigger.sh to auto-trigger builds.

Signed-off-by: James Ketrenos <james.p.ketrenos@intel.com>

.gitignore

@@ -1,2 +1,2 @@
 media
-
+SECRETS

.gitlab-ci.yml

@@ -13,13 +13,15 @@ variables:
   PACKAGE_REPOSITORY: "https://repositories.intel.com/graphics"
   REGISTRY_URL: "amr-registry.caas.intel.com/vtt-osgc/solutions"
   CONTAINER: "intel-media-ffmpeg"
+  TARGET_TAG: "${OS_DISTRO}-${PACKAGE_STREAM}"
+  TAG: "${CI_PIPELINE_IID}-${CI_COMMIT_REF_NAME}-${OS_DISTRO}-${PACKAGE_STREAM}"
 
 build_devel_image:
   stage: "build-devel"
   script:
   - ./scripts/build-dockerfile.sh 
   - ./scripts/build-image.sh
-  - ./scripts/push-test-image.sh
+  - ./scripts/push-image.sh
   tags:
   - builder
   - ubuntu
@@ -29,7 +31,7 @@ build_devel_image:
 test_devel_image:
   stage: "test-devel"
   script:
-  - ./scripts/test-tag.sh
+  - ./scripts/test-image.sh
   tags:
   - gen9
   only:
@@ -39,9 +41,9 @@ publish_rolling_image:
   stage: "publish-rolling"
   script:
   - ./scripts/push-rolling-image.sh
-  - ./scripts/remove-pipeline-tag.sh
+  - ./scripts/remove-tag.sh
   tags:
   - builder
   - ubuntu
   only:
-  - triggers
+  - triggers

scripts/build-image.sh

@@ -17,7 +17,6 @@ RELEASE_INFO=${RELEASE_INFO:-N/A}
 } || {
   TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}}
 }
-TAG=${CI_PIPELINE_IID}-${TAG}
 
 case "${OS_DISTRO}" in
   rhel)

scripts/push-image.sh

@@ -17,7 +17,6 @@ RELEASE_INFO=${RELEASE_INFO:-N/A}
 } || {
   TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}}
 }
-TAG=${CI_PIPELINE_IID}-${TAG}
 
 docker tag ${CONTAINER}:${TAG} ${REGISTRY_URL}/${CONTAINER}:${TAG} \
   && docker push ${REGISTRY_URL}/${CONTAINER}:${TAG} \

scripts/push-rolling-image.sh

@@ -1,11 +1,5 @@
 #!/bin/bash
 
-# Determine if it is Mac OS and switch to use gxargs instead
-CMD=xargs
-if [ $(which system_profiler) ]; then
-  CMD=gxargs
-fi
-
 # Bring in the variables from SOLUTION file, supporting
 # nested substitution
 . SOLUTION
@@ -24,8 +18,44 @@ RELEASE_INFO=${RELEASE_INFO:-N/A}
   TAG=${TAG:-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}}
 }
 
-docker pull ${REGISTRY_URL}/${CONTAINER}:${CI_PIPELINE_IID}-test-build-${TAG}
+[[ "${TARGET_TAG}" != "" ]] || {
-docker tag ${REGISTRY_URL}/${CONTAINER}:${CI_PIPELINE_IID}-test-build-${TAG} ${REGISTRY_URL}/${CONTAINER}:${TAG}
+  echo >&2 "TARGET_TAG needs to be set to the tag to push to Harbor"
-docker tag ${REGISTRY_URL}/${CONTAINER}:${CI_PIPELINE_IID}-test-build-${TAG} ${REGISTRY_URL}/${CONTAINER}:latest-${OS_DISTRO}-${PACKAGE_STREAM}
+  exit 1
-docker push ${REGISTRY_URL}/${CONTAINER}:${TAG}
+}
-docker push ${REGISTRY_URL}/${CONTAINER}:latest-${OS_DISTRO}-${PACKAGE_STREAM}
+
+function fail {
+  >&2 echo "$*"
+  exit -1
+}
+
+cat << EOF
+
+The following will done to publish latest rolling image:
+
+1. Pull ${CONTAINER}:${TAG}
+2. Tag #1 as ${CONTAINER}:${TARGET_TAG}
+3. Tag #1 as ${CONTAINER}:latest-${TARGET_TAG}
+4. Push #2
+5. Push #3
+6. Delete ${CONTAINER}:${TAG}
+
+EOF
+
+docker pull ${REGISTRY_URL}/${CONTAINER}:${TAG} ||
+  fail "Unable to pull image"
+
+docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \
+  ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG} ||
+  fail "Unable to tag image (1/2)"
+
+docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \
+  ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG} ||
+  fail "Unable to tag image (2/2)"
+
+docker push ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG} ||
+  fail "Unable to push ${CONTAINER}:${TARGET_TAG}"
+
+docker push ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG} ||
+  fail "Unable to push ${CONTAINER}:latest-${TARGET_TAG}"
+
+echo "Done tagging and pushing ${CONTAINER} as ${TARGET_TAG}"

scripts/remove-pipeline-tag.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-# Bring in the variables from SOLUTION file, supporting
-# nested substitution
-. SOLUTION
-. MANIFEST
-
-VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION))
-VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST))
-for var in ${VARS[@]}; do
-  export ${var}
-done
-
-RELEASE_INFO=${RELEASE_INFO:-N/A}
-[[ "${RELEASE_INFO}" == "N/A" ]] && {
-  TAG=${TAG:-test-build-${OS_DISTRO}-${PACKAGE_STREAM}-$(date +%Y%m%d)}
-} || {
-  TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}}
-}
-TAG=${CI_PIPELINE_IID}-${TAG}
-
-curl -i -k -u ${HARBOR_USER}:${HARBOR_PASSWD} \
-    -X DELETE "https://amr-registry.caas.intel.com/api/repositories/vtt-osgc%2Fsolutions%2F${CONTAINER}/tags/${TAG}" \
-    -H "accept: application/json"

scripts/remove-tag.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# Bring in the variables from SOLUTION file, supporting
+# nested substitution
+. SOLUTION
+. MANIFEST
+
+VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION))
+VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST))
+for var in ${VARS[@]}; do
+  export ${var}
+done
+
+RELEASE_INFO=${RELEASE_INFO:-N/A}
+[[ "${RELEASE_INFO}" == "N/A" ]] && {
+  TAG=${TAG:-test-build-${OS_DISTRO}-${PACKAGE_STREAM}-$(date +%Y%m%d)}
+} || {
+  TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}}
+}
+
+# Parse out the FQDN from the REGISTRY_URL and escape
+# the group/project/path
+FQDN=${REGISTRY_URL%%/*}
+PROJECT=$(echo ${REGISTRY_URL#*/}/${CONTAINER} | sed s,/,%2F,g)
+
+echo -e "Deleting tag:\n  ${CONTAINER}:${TAG}\nFrom:\n  ${PROJECT}"
+
+RESULTS=$(curl --noproxy '*' -s -k \
+  -u ${HARBOR_USER}:${HARBOR_PASSWD} \
+  -i \
+  -X DELETE \
+  -H "accept: application/json" \
+  "https://${FQDN}/api/repositories/${PROJECT}/tags/${TAG}")
+
+echo ${RESULTS} | grep -q "HTTP.*200" && {
+  echo "Tag deleted successfully."
+} || {
+  >&2 echo "Error deleting tag:"
+  >&2 echo "${RESULTS}"
+  exit 1
+}

scripts/test-image.sh

@@ -1,11 +1,5 @@
 #!/bin/bash
 
-# Determine if it is Mac OS and switch to use gxargs instead
-CMD=xargs
-if [ $(which system_profiler) ]; then
-  CMD=gxargs
-fi
-
 # Bring in the variables from SOLUTION file, supporting
 # nested substitution
 . SOLUTION
@@ -23,19 +17,25 @@ RELEASE_INFO=${RELEASE_INFO:-N/A}
 } || {
   TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}}
 }
-TAG=${CI_PIPELINE_IID}-${TAG}
 
 docker pull ${REGISTRY_URL}/${CONTAINER}:${TAG}
 
+echo "Downloading test content"
+
+function fail {
+  >&2 echo "$*"
+  exit -1
+}
+
 mkdir $(pwd)/media
-wget -O $(pwd)/media/AUD_MW_E.264 \
+wget -q -O $(pwd)/media/AUD_MW_E.264 \
-  https://fate-suite.libav.org/h264-conformance/AUD_MW_E.264
+  https://fate-suite.libav.org/h264-conformance/AUD_MW_E.264 ||
+  fail "Unable to download test content."
 chmod -R 777 $(pwd)/media
 docker run \
   --rm \
-  -a STDOUT \
   --device=/dev/dri \
   -e QSV_DEVICE=${QSV_DEVICE:-/dev/dri/renderD128} \
   --volume $(pwd)/media:/media \
   ${REGISTRY_URL}/${CONTAINER}:${TAG} \
-  test
+  test || fail "Unable to execute docker"

scripts/trigger.sh

@@ -0,0 +1,139 @@
+#!/bin/bash
+
+# 1. List project triggers, looking for 'sys_osgc CI trigger'
+# 2. If not found, create the 'sys_osgc CI trigger'
+# 3. Invoke the trigger
+
+# The following constants are used so we can run trigger.sh
+# against a known build
+BUILD=${BUILD:-7517585}
+BRANCH=${BRANCH:-releases_19.4}
+PROJECT=${PROJECT:-50552}
+REF=${REF:-master}
+
+eval $(grep ^SYS_OSGC_TOKEN SECRETS)
+
+[[ "$SYS_OSGC_TOKEN" == "" ]] && {
+  >&2 echo "SECRETS needs to contain a GitLab token for SYS_OSGC_TOKEN."
+  exit -1
+}
+
+GITLAB="https://gitlab.devtools.intel.com"
+
+echo "Looking for CI trigger"
+
+RESULTS=$(curl --noproxy '*' -s -X GET \
+      --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \
+      --header "Content-Type: application/json" \
+      ${GITLAB}/api/v4/projects/${PROJECT}/triggers)
+
+TOKEN=$(echo ${RESULTS} | sed -E 's#([{},])#\1\n#g' | {
+  STATE=0
+  MATCH=
+  TOKEN=
+  # Parse the JSON looking for the trigger token and description of
+  # 'sys_osgc CI Trigger'
+  while read line; do
+    case ${STATE} in
+    0)
+      [[ ${line} =~ ^[[:space:]]*\{ ]] && STATE=1
+      ;;
+    1)
+      [[ "${MATCH}" != "sys_osgc CI trigger" ]] &&
+        MATCH=$(echo ${line} |
+          sed -nE 's,^.*"description":\s*"([^"]+)".*$,\1,p')
+      [[ "${TOKEN}" == "" ]] &&
+        TOKEN=$(echo ${line} |
+          sed -nE 's,^.*"token":\s*"([^"]+)".*$,\1,p')
+      [[ "${TOKEN}" != "" ]] && [[ "${MATCH}" == "sys_osgc CI trigger" ]] && break
+      [[ "${line}" =~ .*\s*} ]] && {
+        MATCH=
+        TOKEN=
+        STATE=0
+      }
+      ;;
+    esac
+  done
+  [[ "${TOKEN}" == "" ]] && echo -n ""
+  echo -n "${TOKEN}"
+})
+
+[[ "${TOKEN}" == "" ]] && {
+  echo "'sys_osgc CI trigger' does not exist on ${PROJECT}."
+
+  while read -n1 -r -p "Create new trigger [y|N]?"; do
+    case $REPLY in
+    y|Y)
+      echo ""
+      break
+      ;;
+    n|N|"")
+      echo " Exiting."
+      exit 1
+      ;;
+    *)
+      echo ""
+      ;;
+    esac
+  done
+
+
+  RESULTS=$(curl --noproxy '*' -s -X POST \
+      --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \
+      -F 'description="sys_osgc CI trigger"' \
+      ${GITLAB}/api/v4/projects/${PROJECT}/triggers)
+
+  TOKEN=$(echo ${RESULTS} | sed -nE 's,.*"token":\s*"([^"]*).*,\1,p')
+
+  [[ "${TOKEN}" == "" ]] && {
+    >&2 echo "Unable to create trigger"
+    >&2 echo "${RESULTS}"
+    exit 1
+  }
+
+  echo "Created token: ${TOKEN}"
+}
+
+echo "Using token: ${TOKEN}"
+
+function post {
+  RESULTS=$(curl --noproxy '*' -s -X POST \
+      -F "token=${TOKEN}" \
+      -F "ref=${REF}" \
+      -F "variables[OS_DISTRO]=ubuntu" \
+      -F "variables[OS_RELEASE]=disco" \
+      -F "variables[PACKAGE_STREAM]=disco" \
+      -F "variables[BRANCH]=${BRANCH}" \
+      -F "variables[BUILD]=${BUILD}" \
+      -F "variables[BUILDNUMBER]=${BUILD}" \
+      ${GITLAB}/api/v4/projects/${PROJECT}/trigger/pipeline)
+
+  PIPELINE=$(echo ${RESULTS} | sed -nE 's,^.*details_path":"([^"]*)".*,\1,p')
+
+  [[ "${PIPELINE}" == "" ]] && {
+    >&2 echo "Unable to find pipeline details URL"
+    >&2 echo ${RESULTS}
+    return 1
+  }
+
+  echo -e "Pipeline URL:\n\n  ${GITLAB}${PIPELINE}\n"
+  true
+}
+
+echo "Triggering pipeline on project ${PROJECT} for BUILD=${BUILD} and BRANCH=${BRANCH}"
+while read -n1 -r -p "Is this correct [y|N]?"; do
+  case $REPLY in
+  y|Y)
+    echo ""
+    post
+    break
+    ;;
+  n|N|"")
+    echo " Exiting."
+    exit 1
+    ;;
+  *)
+    echo ""
+    ;;
+  esac
+done