From 363f40de52d882aee551635957addc6ec3ff8c2e Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Tue, 17 Dec 2019 19:35:27 -0800 Subject: [PATCH 1/9] Refactored scripts/* and .gitlab-ci.yml Fix #6: Define TAG in .gitlab-ci.yml and update scripts to defer to it. renamed: scripts/push-test-image.sh -> scripts/push-image.sh renamed: scripts/test-tag.sh -> scripts/test-image.sh Refactored some of the scripts/* so they are generic and not tied to CI usages. Added error checking and output to push-rolling-image.sh and remove-tag.sh Added scripts/trigger.sh to auto-trigger builds. Signed-off-by: James Ketrenos --- .gitignore | 2 +- .gitlab-ci.yml | 10 +- scripts/build-image.sh | 1 - scripts/{push-test-image.sh => push-image.sh} | 1 - scripts/push-rolling-image.sh | 52 +++++-- scripts/remove-pipeline-tag.sh | 24 --- scripts/remove-tag.sh | 41 ++++++ scripts/{test-tag.sh => test-image.sh} | 22 +-- scripts/trigger.sh | 139 ++++++++++++++++++ 9 files changed, 239 insertions(+), 53 deletions(-) rename scripts/{push-test-image.sh => push-image.sh} (98%) delete mode 100755 scripts/remove-pipeline-tag.sh create mode 100755 scripts/remove-tag.sh rename scripts/{test-tag.sh => test-image.sh} (73%) create mode 100755 scripts/trigger.sh diff --git a/.gitignore b/.gitignore index 8f87a49..ff20920 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ media - +SECRETS diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4cbeadd..a58765e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,13 +13,15 @@ variables: PACKAGE_REPOSITORY: "https://repositories.intel.com/graphics" REGISTRY_URL: "amr-registry.caas.intel.com/vtt-osgc/solutions" CONTAINER: "intel-media-ffmpeg" + TARGET_TAG: "${OS_DISTRO}-${PACKAGE_STREAM}" + TAG: "${CI_PIPELINE_IID}-${CI_COMMIT_REF_NAME}-${OS_DISTRO}-${PACKAGE_STREAM}" build_devel_image: stage: "build-devel" script: - ./scripts/build-dockerfile.sh - ./scripts/build-image.sh - - ./scripts/push-test-image.sh + - ./scripts/push-image.sh tags: - builder - ubuntu @@ -29,7 +31,7 @@ build_devel_image: test_devel_image: stage: "test-devel" script: - - ./scripts/test-tag.sh + - ./scripts/test-image.sh tags: - gen9 only: @@ -39,9 +41,9 @@ publish_rolling_image: stage: "publish-rolling" script: - ./scripts/push-rolling-image.sh - - ./scripts/remove-pipeline-tag.sh + - ./scripts/remove-tag.sh tags: - builder - ubuntu only: - - triggers \ No newline at end of file + - triggers diff --git a/scripts/build-image.sh b/scripts/build-image.sh index c5bc518..2affa83 100755 --- a/scripts/build-image.sh +++ b/scripts/build-image.sh @@ -17,7 +17,6 @@ RELEASE_INFO=${RELEASE_INFO:-N/A} } || { TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} } -TAG=${CI_PIPELINE_IID}-${TAG} case "${OS_DISTRO}" in rhel) diff --git a/scripts/push-test-image.sh b/scripts/push-image.sh similarity index 98% rename from scripts/push-test-image.sh rename to scripts/push-image.sh index 6743105..c92dd0a 100755 --- a/scripts/push-test-image.sh +++ b/scripts/push-image.sh @@ -17,7 +17,6 @@ RELEASE_INFO=${RELEASE_INFO:-N/A} } || { TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} } -TAG=${CI_PIPELINE_IID}-${TAG} docker tag ${CONTAINER}:${TAG} ${REGISTRY_URL}/${CONTAINER}:${TAG} \ && docker push ${REGISTRY_URL}/${CONTAINER}:${TAG} \ diff --git a/scripts/push-rolling-image.sh b/scripts/push-rolling-image.sh index 823d990..4fe4b3b 100755 --- a/scripts/push-rolling-image.sh +++ b/scripts/push-rolling-image.sh @@ -1,11 +1,5 @@ #!/bin/bash -# Determine if it is Mac OS and switch to use gxargs instead -CMD=xargs -if [ $(which system_profiler) ]; then - CMD=gxargs -fi - # Bring in the variables from SOLUTION file, supporting # nested substitution . SOLUTION @@ -24,8 +18,44 @@ RELEASE_INFO=${RELEASE_INFO:-N/A} TAG=${TAG:-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} } -docker pull ${REGISTRY_URL}/${CONTAINER}:${CI_PIPELINE_IID}-test-build-${TAG} -docker tag ${REGISTRY_URL}/${CONTAINER}:${CI_PIPELINE_IID}-test-build-${TAG} ${REGISTRY_URL}/${CONTAINER}:${TAG} -docker tag ${REGISTRY_URL}/${CONTAINER}:${CI_PIPELINE_IID}-test-build-${TAG} ${REGISTRY_URL}/${CONTAINER}:latest-${OS_DISTRO}-${PACKAGE_STREAM} -docker push ${REGISTRY_URL}/${CONTAINER}:${TAG} -docker push ${REGISTRY_URL}/${CONTAINER}:latest-${OS_DISTRO}-${PACKAGE_STREAM} \ No newline at end of file +[[ "${TARGET_TAG}" != "" ]] || { + echo >&2 "TARGET_TAG needs to be set to the tag to push to Harbor" + exit 1 +} + +function fail { + >&2 echo "$*" + exit -1 +} + +cat << EOF + +The following will done to publish latest rolling image: + +1. Pull ${CONTAINER}:${TAG} +2. Tag #1 as ${CONTAINER}:${TARGET_TAG} +3. Tag #1 as ${CONTAINER}:latest-${TARGET_TAG} +4. Push #2 +5. Push #3 +6. Delete ${CONTAINER}:${TAG} + +EOF + +docker pull ${REGISTRY_URL}/${CONTAINER}:${TAG} || + fail "Unable to pull image" + +docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \ + ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG} || + fail "Unable to tag image (1/2)" + +docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \ + ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG} || + fail "Unable to tag image (2/2)" + +docker push ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG} || + fail "Unable to push ${CONTAINER}:${TARGET_TAG}" + +docker push ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG} || + fail "Unable to push ${CONTAINER}:latest-${TARGET_TAG}" + +echo "Done tagging and pushing ${CONTAINER} as ${TARGET_TAG}" diff --git a/scripts/remove-pipeline-tag.sh b/scripts/remove-pipeline-tag.sh deleted file mode 100755 index 812903f..0000000 --- a/scripts/remove-pipeline-tag.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# Bring in the variables from SOLUTION file, supporting -# nested substitution -. SOLUTION -. MANIFEST - -VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION)) -VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST)) -for var in ${VARS[@]}; do - export ${var} -done - -RELEASE_INFO=${RELEASE_INFO:-N/A} -[[ "${RELEASE_INFO}" == "N/A" ]] && { - TAG=${TAG:-test-build-${OS_DISTRO}-${PACKAGE_STREAM}-$(date +%Y%m%d)} -} || { - TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} -} -TAG=${CI_PIPELINE_IID}-${TAG} - -curl -i -k -u ${HARBOR_USER}:${HARBOR_PASSWD} \ - -X DELETE "https://amr-registry.caas.intel.com/api/repositories/vtt-osgc%2Fsolutions%2F${CONTAINER}/tags/${TAG}" \ - -H "accept: application/json" \ No newline at end of file diff --git a/scripts/remove-tag.sh b/scripts/remove-tag.sh new file mode 100755 index 0000000..0ef4779 --- /dev/null +++ b/scripts/remove-tag.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Bring in the variables from SOLUTION file, supporting +# nested substitution +. SOLUTION +. MANIFEST + +VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION)) +VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST)) +for var in ${VARS[@]}; do + export ${var} +done + +RELEASE_INFO=${RELEASE_INFO:-N/A} +[[ "${RELEASE_INFO}" == "N/A" ]] && { + TAG=${TAG:-test-build-${OS_DISTRO}-${PACKAGE_STREAM}-$(date +%Y%m%d)} +} || { + TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} +} + +# Parse out the FQDN from the REGISTRY_URL and escape +# the group/project/path +FQDN=${REGISTRY_URL%%/*} +PROJECT=$(echo ${REGISTRY_URL#*/}/${CONTAINER} | sed s,/,%2F,g) + +echo -e "Deleting tag:\n ${CONTAINER}:${TAG}\nFrom:\n ${PROJECT}" + +RESULTS=$(curl --noproxy '*' -s -k \ + -u ${HARBOR_USER}:${HARBOR_PASSWD} \ + -i \ + -X DELETE \ + -H "accept: application/json" \ + "https://${FQDN}/api/repositories/${PROJECT}/tags/${TAG}") + +echo ${RESULTS} | grep -q "HTTP.*200" && { + echo "Tag deleted successfully." +} || { + >&2 echo "Error deleting tag:" + >&2 echo "${RESULTS}" + exit 1 +} diff --git a/scripts/test-tag.sh b/scripts/test-image.sh similarity index 73% rename from scripts/test-tag.sh rename to scripts/test-image.sh index 97e6c13..bc4e67b 100755 --- a/scripts/test-tag.sh +++ b/scripts/test-image.sh @@ -1,11 +1,5 @@ #!/bin/bash -# Determine if it is Mac OS and switch to use gxargs instead -CMD=xargs -if [ $(which system_profiler) ]; then - CMD=gxargs -fi - # Bring in the variables from SOLUTION file, supporting # nested substitution . SOLUTION @@ -23,19 +17,25 @@ RELEASE_INFO=${RELEASE_INFO:-N/A} } || { TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} } -TAG=${CI_PIPELINE_IID}-${TAG} docker pull ${REGISTRY_URL}/${CONTAINER}:${TAG} +echo "Downloading test content" + +function fail { + >&2 echo "$*" + exit -1 +} + mkdir $(pwd)/media -wget -O $(pwd)/media/AUD_MW_E.264 \ - https://fate-suite.libav.org/h264-conformance/AUD_MW_E.264 +wget -q -O $(pwd)/media/AUD_MW_E.264 \ + https://fate-suite.libav.org/h264-conformance/AUD_MW_E.264 || + fail "Unable to download test content." chmod -R 777 $(pwd)/media docker run \ --rm \ - -a STDOUT \ --device=/dev/dri \ -e QSV_DEVICE=${QSV_DEVICE:-/dev/dri/renderD128} \ --volume $(pwd)/media:/media \ ${REGISTRY_URL}/${CONTAINER}:${TAG} \ - test + test || fail "Unable to execute docker" diff --git a/scripts/trigger.sh b/scripts/trigger.sh new file mode 100755 index 0000000..463be66 --- /dev/null +++ b/scripts/trigger.sh @@ -0,0 +1,139 @@ +#!/bin/bash + +# 1. List project triggers, looking for 'sys_osgc CI trigger' +# 2. If not found, create the 'sys_osgc CI trigger' +# 3. Invoke the trigger + +# The following constants are used so we can run trigger.sh +# against a known build +BUILD=${BUILD:-7517585} +BRANCH=${BRANCH:-releases_19.4} +PROJECT=${PROJECT:-50552} +REF=${REF:-master} + +eval $(grep ^SYS_OSGC_TOKEN SECRETS) + +[[ "$SYS_OSGC_TOKEN" == "" ]] && { + >&2 echo "SECRETS needs to contain a GitLab token for SYS_OSGC_TOKEN." + exit -1 +} + +GITLAB="https://gitlab.devtools.intel.com" + +echo "Looking for CI trigger" + +RESULTS=$(curl --noproxy '*' -s -X GET \ + --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \ + --header "Content-Type: application/json" \ + ${GITLAB}/api/v4/projects/${PROJECT}/triggers) + +TOKEN=$(echo ${RESULTS} | sed -E 's#([{},])#\1\n#g' | { + STATE=0 + MATCH= + TOKEN= + # Parse the JSON looking for the trigger token and description of + # 'sys_osgc CI Trigger' + while read line; do + case ${STATE} in + 0) + [[ ${line} =~ ^[[:space:]]*\{ ]] && STATE=1 + ;; + 1) + [[ "${MATCH}" != "sys_osgc CI trigger" ]] && + MATCH=$(echo ${line} | + sed -nE 's,^.*"description":\s*"([^"]+)".*$,\1,p') + [[ "${TOKEN}" == "" ]] && + TOKEN=$(echo ${line} | + sed -nE 's,^.*"token":\s*"([^"]+)".*$,\1,p') + [[ "${TOKEN}" != "" ]] && [[ "${MATCH}" == "sys_osgc CI trigger" ]] && break + [[ "${line}" =~ .*\s*} ]] && { + MATCH= + TOKEN= + STATE=0 + } + ;; + esac + done + [[ "${TOKEN}" == "" ]] && echo -n "" + echo -n "${TOKEN}" +}) + +[[ "${TOKEN}" == "" ]] && { + echo "'sys_osgc CI trigger' does not exist on ${PROJECT}." + + while read -n1 -r -p "Create new trigger [y|N]?"; do + case $REPLY in + y|Y) + echo "" + break + ;; + n|N|"") + echo " Exiting." + exit 1 + ;; + *) + echo "" + ;; + esac + done + + + RESULTS=$(curl --noproxy '*' -s -X POST \ + --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \ + -F 'description="sys_osgc CI trigger"' \ + ${GITLAB}/api/v4/projects/${PROJECT}/triggers) + + TOKEN=$(echo ${RESULTS} | sed -nE 's,.*"token":\s*"([^"]*).*,\1,p') + + [[ "${TOKEN}" == "" ]] && { + >&2 echo "Unable to create trigger" + >&2 echo "${RESULTS}" + exit 1 + } + + echo "Created token: ${TOKEN}" +} + +echo "Using token: ${TOKEN}" + +function post { + RESULTS=$(curl --noproxy '*' -s -X POST \ + -F "token=${TOKEN}" \ + -F "ref=${REF}" \ + -F "variables[OS_DISTRO]=ubuntu" \ + -F "variables[OS_RELEASE]=disco" \ + -F "variables[PACKAGE_STREAM]=disco" \ + -F "variables[BRANCH]=${BRANCH}" \ + -F "variables[BUILD]=${BUILD}" \ + -F "variables[BUILDNUMBER]=${BUILD}" \ + ${GITLAB}/api/v4/projects/${PROJECT}/trigger/pipeline) + + PIPELINE=$(echo ${RESULTS} | sed -nE 's,^.*details_path":"([^"]*)".*,\1,p') + + [[ "${PIPELINE}" == "" ]] && { + >&2 echo "Unable to find pipeline details URL" + >&2 echo ${RESULTS} + return 1 + } + + echo -e "Pipeline URL:\n\n ${GITLAB}${PIPELINE}\n" + true +} + +echo "Triggering pipeline on project ${PROJECT} for BUILD=${BUILD} and BRANCH=${BRANCH}" +while read -n1 -r -p "Is this correct [y|N]?"; do + case $REPLY in + y|Y) + echo "" + post + break + ;; + n|N|"") + echo " Exiting." + exit 1 + ;; + *) + echo "" + ;; + esac +done From fa52c35027f1030bf17c290093e5dd04d18505df Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Wed, 18 Dec 2019 09:49:22 -0800 Subject: [PATCH 2/9] Updated to version being used in intel-compute-clinfo Signed-off-by: James Ketrenos --- .gitlab-ci.yml | 24 ++++++++++++------------ scripts/trigger.sh | 29 +++++++++++++++++++++-------- 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a58765e..ef61ae9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,26 +1,26 @@ stages: -- build-devel -- test-devel -- publish-rolling +- build +- test +- publish # Required Input From Trigger # OS_DISTRO: ex. ubuntu # OS_RELEASE: ex. eoan # PACKAGE_STREAM: ex. eoan +# BUILD: Agama Build number +# BRANCH: Agama CI branch (releases_*, master, embargo) variables: ci_enabled: "true" - PACKAGE_REPOSITORY: "https://repositories.intel.com/graphics" - REGISTRY_URL: "amr-registry.caas.intel.com/vtt-osgc/solutions" CONTAINER: "intel-media-ffmpeg" TARGET_TAG: "${OS_DISTRO}-${PACKAGE_STREAM}" TAG: "${CI_PIPELINE_IID}-${CI_COMMIT_REF_NAME}-${OS_DISTRO}-${PACKAGE_STREAM}" -build_devel_image: - stage: "build-devel" +build_image: + stage: "build" script: - ./scripts/build-dockerfile.sh - - ./scripts/build-image.sh + - ./scripts/build-image.sh --no-cache - ./scripts/push-image.sh tags: - builder @@ -28,8 +28,8 @@ build_devel_image: only: - triggers -test_devel_image: - stage: "test-devel" +test_image: + stage: "test" script: - ./scripts/test-image.sh tags: @@ -37,8 +37,8 @@ test_devel_image: only: - triggers -publish_rolling_image: - stage: "publish-rolling" +publish_image: + stage: "publish" script: - ./scripts/push-rolling-image.sh - ./scripts/remove-tag.sh diff --git a/scripts/trigger.sh b/scripts/trigger.sh index 463be66..b16348e 100755 --- a/scripts/trigger.sh +++ b/scripts/trigger.sh @@ -4,13 +4,26 @@ # 2. If not found, create the 'sys_osgc CI trigger' # 3. Invoke the trigger +# Bring in the variables from SOLUTION file, supporting +# nested substitution +. SOLUTION +. MANIFEST + +VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION)) +VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST)) +for var in ${VARS[@]}; do + export ${var} +done + # The following constants are used so we can run trigger.sh # against a known build BUILD=${BUILD:-7517585} BRANCH=${BRANCH:-releases_19.4} -PROJECT=${PROJECT:-50552} REF=${REF:-master} +PROJECT="vtt/sws/osgc/solutions/$(basename $(pwd))" +URI_PROJECT=$(echo ${PROJECT} | sed -E 's,/,%2F,g') + eval $(grep ^SYS_OSGC_TOKEN SECRETS) [[ "$SYS_OSGC_TOKEN" == "" ]] && { @@ -25,9 +38,9 @@ echo "Looking for CI trigger" RESULTS=$(curl --noproxy '*' -s -X GET \ --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \ --header "Content-Type: application/json" \ - ${GITLAB}/api/v4/projects/${PROJECT}/triggers) + ${GITLAB}/api/v4/projects/${URI_PROJECT}/triggers) -TOKEN=$(echo ${RESULTS} | sed -E 's#([{},])#\1\n#g' | { +TOKEN=$(echo ${RESULTS} | sed -E 's#([[{},])#\1\n#g' | { STATE=0 MATCH= TOKEN= @@ -81,7 +94,7 @@ TOKEN=$(echo ${RESULTS} | sed -E 's#([{},])#\1\n#g' | { RESULTS=$(curl --noproxy '*' -s -X POST \ --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \ -F 'description="sys_osgc CI trigger"' \ - ${GITLAB}/api/v4/projects/${PROJECT}/triggers) + ${GITLAB}/api/v4/projects/${URI_PROJECT}/triggers) TOKEN=$(echo ${RESULTS} | sed -nE 's,.*"token":\s*"([^"]*).*,\1,p') @@ -100,13 +113,13 @@ function post { RESULTS=$(curl --noproxy '*' -s -X POST \ -F "token=${TOKEN}" \ -F "ref=${REF}" \ - -F "variables[OS_DISTRO]=ubuntu" \ - -F "variables[OS_RELEASE]=disco" \ - -F "variables[PACKAGE_STREAM]=disco" \ + -F "variables[OS_DISTRO]=${OS_DISTRO}" \ + -F "variables[OS_RELEASE]=${OS_RELEASE}" \ + -F "variables[PACKAGE_STREAM]=${PACKAGE_STREAM}" \ -F "variables[BRANCH]=${BRANCH}" \ -F "variables[BUILD]=${BUILD}" \ -F "variables[BUILDNUMBER]=${BUILD}" \ - ${GITLAB}/api/v4/projects/${PROJECT}/trigger/pipeline) + ${GITLAB}/api/v4/projects/${URI_PROJECT}/trigger/pipeline) PIPELINE=$(echo ${RESULTS} | sed -nE 's,^.*details_path":"([^"]*)".*,\1,p') From 9d7b325a9dd5c60badf5e82c79b58828fdb1be82 Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Wed, 18 Dec 2019 14:33:16 -0800 Subject: [PATCH 3/9] Add video group from host Signed-off-by: James Ketrenos --- scripts/test-image.sh | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/scripts/test-image.sh b/scripts/test-image.sh index bc4e67b..c314bc3 100755 --- a/scripts/test-image.sh +++ b/scripts/test-image.sh @@ -20,22 +20,33 @@ RELEASE_INFO=${RELEASE_INFO:-N/A} docker pull ${REGISTRY_URL}/${CONTAINER}:${TAG} -echo "Downloading test content" - function fail { >&2 echo "$*" exit -1 } +VIDEO=$(getent group video | sed -E 's,^video:[^:]*:([^:]*):.*$,\1,') + +cat << EOF + +Running test with video GID as ${VIDEO} + +------------------------------------------------------------------------------- + +EOF + +echo -e "Downloading test content: " mkdir $(pwd)/media wget -q -O $(pwd)/media/AUD_MW_E.264 \ https://fate-suite.libav.org/h264-conformance/AUD_MW_E.264 || fail "Unable to download test content." -chmod -R 777 $(pwd)/media -docker run \ - --rm \ - --device=/dev/dri \ - -e QSV_DEVICE=${QSV_DEVICE:-/dev/dri/renderD128} \ - --volume $(pwd)/media:/media \ - ${REGISTRY_URL}/${CONTAINER}:${TAG} \ - test || fail "Unable to execute docker" +chmod -R 777 $(pwd)/media || fail "Unable to set permissions" +echo "done" + +docker run --group-add ${VIDEO} \ + --rm \ + --device=/dev/dri \ + -e QSV_DEVICE=${QSV_DEVICE:-/dev/dri/renderD128} \ + --volume $(pwd)/media:/media \ + ${REGISTRY_URL}/${CONTAINER}:${TAG} \ + test || fail "Unable to execute 'test' on image." From 0b55f53410e1d1ef3f4105f631ccff1e4c7e1e5c Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Wed, 18 Dec 2019 14:36:46 -0800 Subject: [PATCH 4/9] Remove unused scripts/clean-image.sh Signed-off-by: James Ketrenos --- scripts/clean-image.sh | 20 -------------------- 1 file changed, 20 deletions(-) delete mode 100755 scripts/clean-image.sh diff --git a/scripts/clean-image.sh b/scripts/clean-image.sh deleted file mode 100755 index 7f58065..0000000 --- a/scripts/clean-image.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -# Determine if it is Mac OS and switch to use gxargs instead -CMD=xargs -if [ $(which system_profiler) ]; then - CMD=gxargs -fi - -# Bring in the variables from SOLUTION file, supporting -# nested substitution -. SOLUTION -. MANIFEST - -VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION)) -VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST)) -for var in ${VARS[@]}; do - export ${var} -done - -docker rmi ${REGISTRY_URL}/${CONTAINER}:devel From f57b15c5557e0786f7aca7f368b50a705bab4616 Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Wed, 18 Dec 2019 14:43:11 -0800 Subject: [PATCH 5/9] Merge 'publish-rolling-image' and 'remove-tag' into single sequence 'promote-image' Signed-off-by: James Ketrenos --- .gitlab-ci.yml | 3 +- ...push-rolling-image.sh => promote-image.sh} | 30 ++++++++++++-- scripts/remove-tag.sh | 41 ------------------- 3 files changed, 27 insertions(+), 47 deletions(-) rename scripts/{push-rolling-image.sh => promote-image.sh} (67%) delete mode 100755 scripts/remove-tag.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ef61ae9..3a9574d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,8 +40,7 @@ test_image: publish_image: stage: "publish" script: - - ./scripts/push-rolling-image.sh - - ./scripts/remove-tag.sh + - ./scripts/promote-image.sh tags: - builder - ubuntu diff --git a/scripts/push-rolling-image.sh b/scripts/promote-image.sh similarity index 67% rename from scripts/push-rolling-image.sh rename to scripts/promote-image.sh index 4fe4b3b..db20a59 100755 --- a/scripts/push-rolling-image.sh +++ b/scripts/promote-image.sh @@ -30,14 +30,14 @@ function fail { cat << EOF -The following will done to publish latest rolling image: +The following will publish latest rolling image: 1. Pull ${CONTAINER}:${TAG} 2. Tag #1 as ${CONTAINER}:${TARGET_TAG} 3. Tag #1 as ${CONTAINER}:latest-${TARGET_TAG} -4. Push #2 -5. Push #3 -6. Delete ${CONTAINER}:${TAG} +4. Delete ${CONTAINER}:${TAG} +5. Push #2 +6. Push #3 EOF @@ -52,6 +52,28 @@ docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \ ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG} || fail "Unable to tag image (2/2)" +# Parse out the FQDN from the REGISTRY_URL and escape +# the group/project/path +FQDN=${REGISTRY_URL%%/*} +PROJECT=$(echo ${REGISTRY_URL#*/}/${CONTAINER} | sed s,/,%2F,g) + +echo -e "Deleting tag:\n ${CONTAINER}:${TAG}\nFrom:\n ${PROJECT}" + +RESULTS=$(curl --noproxy '*' -s -k \ + -u ${HARBOR_USER}:${HARBOR_PASSWD} \ + -i \ + -X DELETE \ + -H "accept: application/json" \ + "https://${FQDN}/api/repositories/${PROJECT}/tags/${TAG}") + +echo ${RESULTS} | grep -q "HTTP.*200" && { + echo "Tag deleted successfully." +} || { + >&2 echo "Error deleting tag:" + >&2 echo "${RESULTS}" + exit 1 +} + docker push ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG} || fail "Unable to push ${CONTAINER}:${TARGET_TAG}" diff --git a/scripts/remove-tag.sh b/scripts/remove-tag.sh deleted file mode 100755 index 0ef4779..0000000 --- a/scripts/remove-tag.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -# Bring in the variables from SOLUTION file, supporting -# nested substitution -. SOLUTION -. MANIFEST - -VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION)) -VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST)) -for var in ${VARS[@]}; do - export ${var} -done - -RELEASE_INFO=${RELEASE_INFO:-N/A} -[[ "${RELEASE_INFO}" == "N/A" ]] && { - TAG=${TAG:-test-build-${OS_DISTRO}-${PACKAGE_STREAM}-$(date +%Y%m%d)} -} || { - TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} -} - -# Parse out the FQDN from the REGISTRY_URL and escape -# the group/project/path -FQDN=${REGISTRY_URL%%/*} -PROJECT=$(echo ${REGISTRY_URL#*/}/${CONTAINER} | sed s,/,%2F,g) - -echo -e "Deleting tag:\n ${CONTAINER}:${TAG}\nFrom:\n ${PROJECT}" - -RESULTS=$(curl --noproxy '*' -s -k \ - -u ${HARBOR_USER}:${HARBOR_PASSWD} \ - -i \ - -X DELETE \ - -H "accept: application/json" \ - "https://${FQDN}/api/repositories/${PROJECT}/tags/${TAG}") - -echo ${RESULTS} | grep -q "HTTP.*200" && { - echo "Tag deleted successfully." -} || { - >&2 echo "Error deleting tag:" - >&2 echo "${RESULTS}" - exit 1 -} From b34b1d18fddf33790390460fb0f781aaa93122cb Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Wed, 18 Dec 2019 15:13:54 -0800 Subject: [PATCH 6/9] Updated to latest xe-solutions Signed-off-by: James Ketrenos --- .dockerignore | 4 +-- Dockerfile | 4 +-- Dockerfile.rhel-8.0 | 31 ++++++++++--------- templates/centos/25-create-user.in | 2 +- templates/rhel/25-create-user.in | 2 +- ...{25-graphics-user.in => 25-create-user.in} | 2 +- 6 files changed, 23 insertions(+), 22 deletions(-) rename templates/ubuntu/{25-graphics-user.in => 25-create-user.in} (97%) diff --git a/.dockerignore b/.dockerignore index 3d1deed..c4dfb92 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,6 @@ * !assets -!Dockerfile -!Dockerfile-* +!Dockerfile* !SOLUTION !MANIFEST +Dockerfile.solution* diff --git a/Dockerfile b/Dockerfile index df4447d..64742e8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -132,7 +132,7 @@ RUN echo "deb [trusted=yes arch=amd64] https://osgc.jf.intel.com/internal/ubuntu # -# Template from templates/ubuntu/25-graphics-user.in +# Template from templates/ubuntu/25-create-user.in # # Create user 'user' and add them to 'sudo' for sudo access and set # the passwd to 'user' @@ -150,7 +150,7 @@ RUN groupadd -r user \ -s /bin/bash \ -r -m \ -g user \ - -G sudo \ + -G sudo,video \ -p $(echo "user" | openssl passwd -stdin) user # Set 'sudo' to NOPASSWD for all container users diff --git a/Dockerfile.rhel-8.0 b/Dockerfile.rhel-8.0 index dcc170a..31491a4 100644 --- a/Dockerfile.rhel-8.0 +++ b/Dockerfile.rhel-8.0 @@ -48,11 +48,9 @@ RUN { \ echo "no_proxy=${no_proxy}" ; \ } | tee -a /etc/dnf/dnf.conf /etc/yum.conf - # # Template from templates/rhel/10-rhel-partner.in # - # Remove any pre-configured repositories RUN rm -rf /etc/yum.repos.d/* @@ -214,6 +212,16 @@ RUN { \ echo "" ; \ } > /etc/yum.repos.d/RHEL-8-rt-intel-partner.repo +# +# Template from templates/rhel/15-upgrade.in +# +# Update package lists, and upgrade to the latest packages +# +# Failure to do this will result in GPG errors later +RUN dnf clean all \ + && dnf -y upgrade + + # # Template from templates/rhel/20-repositories-intel-com.in # @@ -236,14 +244,14 @@ RUN { \ RUN { \ echo "[intel-graphics]" ; \ echo "name=Intel Graphics Drivers Repository" ; \ - echo "baseurl=https://repositories.intel.com/graphics/rhel/8.0/" ; \ + echo "baseurl=https://osgc.jf.intel.com/internal/rhel/8.0/" ; \ echo "sslverify=0" ; \ echo "enabled=1" ; \ echo "gpgcheck=0" ; \ } > /etc/yum.repos.d/intel-graphics.repo # -# Template from templates/rhel/25-graphics-user.in +# Template from templates/rhel/25-create-user.in # # Create user 'user' and add them to 'sudo' for sudo access and set # the passwd to 'user' @@ -258,11 +266,11 @@ RUN groupadd -r user \ -s /bin/bash \ -r -m \ -g user \ - -G sudo \ + -G wheel,video \ -p $(echo "user" | openssl passwd -stdin) user -# Set 'sudo' to NOPASSWD for all container users -RUN sed -i -e 's,%sudo.*,%sudo ALL=(ALL) NOPASSWD:ALL,g' /etc/sudoers +# Set 'wheel' to NOPASSWD for all container users +RUN sed -i -e 's,%wheel.*,%wheel ALL=(ALL) NOPASSWD:ALL,g' /etc/sudoers USER user @@ -351,13 +359,6 @@ ENTRYPOINT [ "/assets/entry" ] # Ensure that each Docker container self-documents the # versions included in it - -RUN { \ - echo "PACKAGE_REPOSITORY=https://repositories.intel.com/graphics" ; \ - echo "RELEASE_INFO=N/A" ; \ - echo "PACKAGE_STREAM=8.0" ; \ - echo "OS_DISTRO=rhel" ; \ - echo "OS_RELEASE=8.0" ; \ -} | sudo tee /assets/SOLUTION +COPY SOLUTION /assets/ COPY MANIFEST /assets/ COPY Dockerfile.rhel-8.0 /assets/Dockerfile diff --git a/templates/centos/25-create-user.in b/templates/centos/25-create-user.in index dd03c45..0ef9ba3 100644 --- a/templates/centos/25-create-user.in +++ b/templates/centos/25-create-user.in @@ -11,7 +11,7 @@ RUN groupadd -r user \ -s /bin/bash \ -r -m \ -g user \ - -G wheel \ + -G wheel,video \ -p $(echo "user" | openssl passwd -stdin) user # Set 'wheel' to NOPASSWD for all container users diff --git a/templates/rhel/25-create-user.in b/templates/rhel/25-create-user.in index f2fecdd..508db9d 100644 --- a/templates/rhel/25-create-user.in +++ b/templates/rhel/25-create-user.in @@ -11,7 +11,7 @@ RUN groupadd -r user \ -s /bin/bash \ -r -m \ -g user \ - -G wheel \ + -G wheel,video \ -p $(echo "user" | openssl passwd -stdin) user # Set 'wheel' to NOPASSWD for all container users diff --git a/templates/ubuntu/25-graphics-user.in b/templates/ubuntu/25-create-user.in similarity index 97% rename from templates/ubuntu/25-graphics-user.in rename to templates/ubuntu/25-create-user.in index 1c15620..4410873 100644 --- a/templates/ubuntu/25-graphics-user.in +++ b/templates/ubuntu/25-create-user.in @@ -14,7 +14,7 @@ RUN groupadd -r user \ -s /bin/bash \ -r -m \ -g user \ - -G sudo \ + -G sudo,video \ -p $(echo "user" | openssl passwd -stdin) user # Set 'sudo' to NOPASSWD for all container users From dbbf9c0c168c04f829a64d8876c65dbf39e1747f Mon Sep 17 00:00:00 2001 From: "James P. Ketrenos" Date: Wed, 18 Dec 2019 16:04:33 -0800 Subject: [PATCH 7/9] Add render group to Docker container run line Signed-off-by: James P. Ketrenos --- scripts/test-image.sh | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/scripts/test-image.sh b/scripts/test-image.sh index c314bc3..4a352db 100755 --- a/scripts/test-image.sh +++ b/scripts/test-image.sh @@ -25,17 +25,21 @@ function fail { exit -1 } -VIDEO=$(getent group video | sed -E 's,^video:[^:]*:([^:]*):.*$,\1,') +VIDEO=$(getent group video | sed -E 's,^video:[^:]*:([^:]*):.*$,\1,') +RENDER=$(getent group render | sed -E 's,^render:[^:]*:([^:]*):.*$,\1,') cat << EOF -Running test with video GID as ${VIDEO} +Running test with: + +video group : ${VIDEO} +render group: ${RENDER} ------------------------------------------------------------------------------- EOF -echo -e "Downloading test content: " +echo -n "Downloading test content: " mkdir $(pwd)/media wget -q -O $(pwd)/media/AUD_MW_E.264 \ https://fate-suite.libav.org/h264-conformance/AUD_MW_E.264 || @@ -43,10 +47,12 @@ wget -q -O $(pwd)/media/AUD_MW_E.264 \ chmod -R 777 $(pwd)/media || fail "Unable to set permissions" echo "done" -docker run --group-add ${VIDEO} \ - --rm \ - --device=/dev/dri \ - -e QSV_DEVICE=${QSV_DEVICE:-/dev/dri/renderD128} \ - --volume $(pwd)/media:/media \ - ${REGISTRY_URL}/${CONTAINER}:${TAG} \ - test || fail "Unable to execute 'test' on image." +docker run \ + --group-add ${VIDEO} \ + --group-add ${RENDER} \ + --rm \ + --device=/dev/dri \ + -e QSV_DEVICE=${QSV_DEVICE:-/dev/dri/renderD128} \ + --volume $(pwd)/media:/media \ + ${REGISTRY_URL}/${CONTAINER}:${TAG} \ + test || fail "Unable to execute 'test' on image." From 4ed8e2fb818bd9271a5073bf175bef61cbe0d329 Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Wed, 18 Dec 2019 17:29:07 -0800 Subject: [PATCH 8/9] Added pipeline watcher script Signed-off-by: James Ketrenos --- scripts/watch.sh | 146 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100755 scripts/watch.sh diff --git a/scripts/watch.sh b/scripts/watch.sh new file mode 100755 index 0000000..4a8ce70 --- /dev/null +++ b/scripts/watch.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +PIPELINE=$1 +[[ "${PIPELINE}" == "" ]] && { + >&2 echo "usage: watch PIPELINE" + exit -1 +} + +# Bring in the variables from SOLUTION file, supporting +# nested substitution +. SOLUTION + +VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION)) +for var in ${VARS[@]}; do + export ${var} +done + +PROJECT="vtt/sws/osgc/solutions/$(basename $(pwd))" +URI_PROJECT=$(echo ${PROJECT} | sed -E 's,/,%2F,g') + +eval $(grep ^SYS_OSGC_TOKEN SECRETS) + +[[ "$SYS_OSGC_TOKEN" == "" ]] && { + >&2 echo "SECRETS needs to contain a GitLab token for SYS_OSGC_TOKEN." + exit -1 +} + +GITLAB="https://gitlab.devtools.intel.com" + +echo "Looking for pipeline jobs for ${PIPELINE}" + +function get_jobs { + ECHO=$1 + # Split the JSON results into line-by-line breakouts for + # easy sed matching + sed -E 's#([[{},]|])#\n\1\n#g' | grep '^[^,]' | { + SCOPE=0 + JOB="" + STATUS="" + while read line; do + case ${SCOPE} in + 2) + echo $line | grep -qE '^([{[])' && { + SCOPE=$((SCOPE+1)) + (( ECHO )) && echo "Scope: ${SCOPE}" + continue + } + echo $line | grep -qE '^([}]|])' && { + SCOPE=$((SCOPE-1)) + (( ECHO )) && echo "Scope: ${SCOPE}" + continue + } + + [[ "${JOB}" == "" ]] && + JOB=$(echo ${line} | + sed -nE 's,^"id"\s*:\s*([0-9]+).*$,\1,p') + [[ "${STATUS}" == "" ]] && + STATUS=$(echo ${line} | + sed -nE 's,^.*"status"\s*:\s*"([^"]+).*$,\1,p') + [[ "${JOB}" != "" ]] && [[ "${STATUS}" != "" ]] && { + echo "${JOB}:${STATUS} " + JOB="" + STATUS="" + } + ;; + *) + echo $line | grep -qE '^([{[])' && { + SCOPE=$((SCOPE+1)) + (( ECHO )) && echo "Scope: ${SCOPE}" + continue + } + echo $line | grep -qE '^([}]|])' && { + (( SCOPE == 0 )) && { + return 0 + } + SCOPE=$((SCOPE-1)) + (( ECHO )) && echo "Scope: ${SCOPE}" + continue + } + esac + done + } +} + +function update_log { + ID=$1 + [[ ! -d "logs" ]] && mkdir "logs" + + curl --noproxy '*' -s -X GET \ + --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \ + "${GITLAB}/api/v4/projects/${URI_PROJECT}/jobs/${ID}/trace" > "logs/${ID}-latest" + + LINES=1 + [[ -e "logs/${ID}" ]] && { + LINES=$(wc -l "logs/${ID}") + LINES=${LINES%% *} + } + + tail -n +${LINES} "logs/${ID}-latest" + mv "logs/${ID}-latest" "logs/${ID}" +} + +while true; do + RESULTS=$(curl --noproxy '*' -s -X GET \ + --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \ + --header "Content-Type: application/json" \ + "${GITLAB}/api/v4/projects/${URI_PROJECT}/pipelines/${PIPELINE}/jobs") + + JOBS=($(echo "${RESULTS}" | get_jobs)) + echo "JOBS: ${JOBS[@]}" + + MORE=0 + FAIL=0 + for JOB in ${JOBS[@]}; do + ID=${JOB%%:*} + STATUS=${JOB##*:} + case ${STATUS,,} in + pending) + MORE=$((MORE+1)) + ;; + created) + MORE=$((MORE+1)) + ;; + running) + update_log ${ID} + MORE=$((MORE+1)) + ;; + success) + update_log ${ID} + ;; + cancelled) + FAIL=1 + ;; + failed) + FAIL=1 + ;; + esac + done + (( MORE == 0 )) && { + echo "No more jobs. Exiting." + exit ${FAIL} + } + + echo "Sleeping 5 seconds for ${MORE} jobs." + sleep 5 +done From f413f837ae76fed9b179ceeee2dad8387135282a Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Thu, 19 Dec 2019 09:39:27 -0800 Subject: [PATCH 9/9] PACKAGE_REPOSITORY can now be used with scripts/trigger Signed-off-by: James Ketrenos --- scripts/trigger.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/trigger.sh b/scripts/trigger.sh index b16348e..f3467c9 100755 --- a/scripts/trigger.sh +++ b/scripts/trigger.sh @@ -116,6 +116,7 @@ function post { -F "variables[OS_DISTRO]=${OS_DISTRO}" \ -F "variables[OS_RELEASE]=${OS_RELEASE}" \ -F "variables[PACKAGE_STREAM]=${PACKAGE_STREAM}" \ + -F "variables[PACKAGE_REPOSITORY]=${PACKAGE_REPOSITORY}" \ -F "variables[BRANCH]=${BRANCH}" \ -F "variables[BUILD]=${BUILD}" \ -F "variables[BUILDNUMBER]=${BUILD}" \