From 363f40de52d882aee551635957addc6ec3ff8c2e Mon Sep 17 00:00:00 2001 From: James Ketrenos Date: Tue, 17 Dec 2019 19:35:27 -0800 Subject: [PATCH] Refactored scripts/* and .gitlab-ci.yml Fix #6: Define TAG in .gitlab-ci.yml and update scripts to defer to it. renamed: scripts/push-test-image.sh -> scripts/push-image.sh renamed: scripts/test-tag.sh -> scripts/test-image.sh Refactored some of the scripts/* so they are generic and not tied to CI usages. Added error checking and output to push-rolling-image.sh and remove-tag.sh Added scripts/trigger.sh to auto-trigger builds. Signed-off-by: James Ketrenos --- .gitignore | 2 +- .gitlab-ci.yml | 10 +- scripts/build-image.sh | 1 - scripts/{push-test-image.sh => push-image.sh} | 1 - scripts/push-rolling-image.sh | 52 +++++-- scripts/remove-pipeline-tag.sh | 24 --- scripts/remove-tag.sh | 41 ++++++ scripts/{test-tag.sh => test-image.sh} | 22 +-- scripts/trigger.sh | 139 ++++++++++++++++++ 9 files changed, 239 insertions(+), 53 deletions(-) rename scripts/{push-test-image.sh => push-image.sh} (98%) delete mode 100755 scripts/remove-pipeline-tag.sh create mode 100755 scripts/remove-tag.sh rename scripts/{test-tag.sh => test-image.sh} (73%) create mode 100755 scripts/trigger.sh diff --git a/.gitignore b/.gitignore index 8f87a49..ff20920 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ media - +SECRETS diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4cbeadd..a58765e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -13,13 +13,15 @@ variables: PACKAGE_REPOSITORY: "https://repositories.intel.com/graphics" REGISTRY_URL: "amr-registry.caas.intel.com/vtt-osgc/solutions" CONTAINER: "intel-media-ffmpeg" + TARGET_TAG: "${OS_DISTRO}-${PACKAGE_STREAM}" + TAG: "${CI_PIPELINE_IID}-${CI_COMMIT_REF_NAME}-${OS_DISTRO}-${PACKAGE_STREAM}" build_devel_image: stage: "build-devel" script: - ./scripts/build-dockerfile.sh - ./scripts/build-image.sh - - ./scripts/push-test-image.sh + - ./scripts/push-image.sh tags: - builder - ubuntu @@ -29,7 +31,7 @@ build_devel_image: test_devel_image: stage: "test-devel" script: - - ./scripts/test-tag.sh + - ./scripts/test-image.sh tags: - gen9 only: @@ -39,9 +41,9 @@ publish_rolling_image: stage: "publish-rolling" script: - ./scripts/push-rolling-image.sh - - ./scripts/remove-pipeline-tag.sh + - ./scripts/remove-tag.sh tags: - builder - ubuntu only: - - triggers \ No newline at end of file + - triggers diff --git a/scripts/build-image.sh b/scripts/build-image.sh index c5bc518..2affa83 100755 --- a/scripts/build-image.sh +++ b/scripts/build-image.sh @@ -17,7 +17,6 @@ RELEASE_INFO=${RELEASE_INFO:-N/A} } || { TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} } -TAG=${CI_PIPELINE_IID}-${TAG} case "${OS_DISTRO}" in rhel) diff --git a/scripts/push-test-image.sh b/scripts/push-image.sh similarity index 98% rename from scripts/push-test-image.sh rename to scripts/push-image.sh index 6743105..c92dd0a 100755 --- a/scripts/push-test-image.sh +++ b/scripts/push-image.sh @@ -17,7 +17,6 @@ RELEASE_INFO=${RELEASE_INFO:-N/A} } || { TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} } -TAG=${CI_PIPELINE_IID}-${TAG} docker tag ${CONTAINER}:${TAG} ${REGISTRY_URL}/${CONTAINER}:${TAG} \ && docker push ${REGISTRY_URL}/${CONTAINER}:${TAG} \ diff --git a/scripts/push-rolling-image.sh b/scripts/push-rolling-image.sh index 823d990..4fe4b3b 100755 --- a/scripts/push-rolling-image.sh +++ b/scripts/push-rolling-image.sh @@ -1,11 +1,5 @@ #!/bin/bash -# Determine if it is Mac OS and switch to use gxargs instead -CMD=xargs -if [ $(which system_profiler) ]; then - CMD=gxargs -fi - # Bring in the variables from SOLUTION file, supporting # nested substitution . SOLUTION @@ -24,8 +18,44 @@ RELEASE_INFO=${RELEASE_INFO:-N/A} TAG=${TAG:-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} } -docker pull ${REGISTRY_URL}/${CONTAINER}:${CI_PIPELINE_IID}-test-build-${TAG} -docker tag ${REGISTRY_URL}/${CONTAINER}:${CI_PIPELINE_IID}-test-build-${TAG} ${REGISTRY_URL}/${CONTAINER}:${TAG} -docker tag ${REGISTRY_URL}/${CONTAINER}:${CI_PIPELINE_IID}-test-build-${TAG} ${REGISTRY_URL}/${CONTAINER}:latest-${OS_DISTRO}-${PACKAGE_STREAM} -docker push ${REGISTRY_URL}/${CONTAINER}:${TAG} -docker push ${REGISTRY_URL}/${CONTAINER}:latest-${OS_DISTRO}-${PACKAGE_STREAM} \ No newline at end of file +[[ "${TARGET_TAG}" != "" ]] || { + echo >&2 "TARGET_TAG needs to be set to the tag to push to Harbor" + exit 1 +} + +function fail { + >&2 echo "$*" + exit -1 +} + +cat << EOF + +The following will done to publish latest rolling image: + +1. Pull ${CONTAINER}:${TAG} +2. Tag #1 as ${CONTAINER}:${TARGET_TAG} +3. Tag #1 as ${CONTAINER}:latest-${TARGET_TAG} +4. Push #2 +5. Push #3 +6. Delete ${CONTAINER}:${TAG} + +EOF + +docker pull ${REGISTRY_URL}/${CONTAINER}:${TAG} || + fail "Unable to pull image" + +docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \ + ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG} || + fail "Unable to tag image (1/2)" + +docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \ + ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG} || + fail "Unable to tag image (2/2)" + +docker push ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG} || + fail "Unable to push ${CONTAINER}:${TARGET_TAG}" + +docker push ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG} || + fail "Unable to push ${CONTAINER}:latest-${TARGET_TAG}" + +echo "Done tagging and pushing ${CONTAINER} as ${TARGET_TAG}" diff --git a/scripts/remove-pipeline-tag.sh b/scripts/remove-pipeline-tag.sh deleted file mode 100755 index 812903f..0000000 --- a/scripts/remove-pipeline-tag.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -# Bring in the variables from SOLUTION file, supporting -# nested substitution -. SOLUTION -. MANIFEST - -VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION)) -VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST)) -for var in ${VARS[@]}; do - export ${var} -done - -RELEASE_INFO=${RELEASE_INFO:-N/A} -[[ "${RELEASE_INFO}" == "N/A" ]] && { - TAG=${TAG:-test-build-${OS_DISTRO}-${PACKAGE_STREAM}-$(date +%Y%m%d)} -} || { - TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} -} -TAG=${CI_PIPELINE_IID}-${TAG} - -curl -i -k -u ${HARBOR_USER}:${HARBOR_PASSWD} \ - -X DELETE "https://amr-registry.caas.intel.com/api/repositories/vtt-osgc%2Fsolutions%2F${CONTAINER}/tags/${TAG}" \ - -H "accept: application/json" \ No newline at end of file diff --git a/scripts/remove-tag.sh b/scripts/remove-tag.sh new file mode 100755 index 0000000..0ef4779 --- /dev/null +++ b/scripts/remove-tag.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# Bring in the variables from SOLUTION file, supporting +# nested substitution +. SOLUTION +. MANIFEST + +VARS=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" SOLUTION)) +VARS+=($(sed -nE "s,(^[^#][^=]*).*$,\1,pg" MANIFEST)) +for var in ${VARS[@]}; do + export ${var} +done + +RELEASE_INFO=${RELEASE_INFO:-N/A} +[[ "${RELEASE_INFO}" == "N/A" ]] && { + TAG=${TAG:-test-build-${OS_DISTRO}-${PACKAGE_STREAM}-$(date +%Y%m%d)} +} || { + TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} +} + +# Parse out the FQDN from the REGISTRY_URL and escape +# the group/project/path +FQDN=${REGISTRY_URL%%/*} +PROJECT=$(echo ${REGISTRY_URL#*/}/${CONTAINER} | sed s,/,%2F,g) + +echo -e "Deleting tag:\n ${CONTAINER}:${TAG}\nFrom:\n ${PROJECT}" + +RESULTS=$(curl --noproxy '*' -s -k \ + -u ${HARBOR_USER}:${HARBOR_PASSWD} \ + -i \ + -X DELETE \ + -H "accept: application/json" \ + "https://${FQDN}/api/repositories/${PROJECT}/tags/${TAG}") + +echo ${RESULTS} | grep -q "HTTP.*200" && { + echo "Tag deleted successfully." +} || { + >&2 echo "Error deleting tag:" + >&2 echo "${RESULTS}" + exit 1 +} diff --git a/scripts/test-tag.sh b/scripts/test-image.sh similarity index 73% rename from scripts/test-tag.sh rename to scripts/test-image.sh index 97e6c13..bc4e67b 100755 --- a/scripts/test-tag.sh +++ b/scripts/test-image.sh @@ -1,11 +1,5 @@ #!/bin/bash -# Determine if it is Mac OS and switch to use gxargs instead -CMD=xargs -if [ $(which system_profiler) ]; then - CMD=gxargs -fi - # Bring in the variables from SOLUTION file, supporting # nested substitution . SOLUTION @@ -23,19 +17,25 @@ RELEASE_INFO=${RELEASE_INFO:-N/A} } || { TAG=${TAG:-test-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} } -TAG=${CI_PIPELINE_IID}-${TAG} docker pull ${REGISTRY_URL}/${CONTAINER}:${TAG} +echo "Downloading test content" + +function fail { + >&2 echo "$*" + exit -1 +} + mkdir $(pwd)/media -wget -O $(pwd)/media/AUD_MW_E.264 \ - https://fate-suite.libav.org/h264-conformance/AUD_MW_E.264 +wget -q -O $(pwd)/media/AUD_MW_E.264 \ + https://fate-suite.libav.org/h264-conformance/AUD_MW_E.264 || + fail "Unable to download test content." chmod -R 777 $(pwd)/media docker run \ --rm \ - -a STDOUT \ --device=/dev/dri \ -e QSV_DEVICE=${QSV_DEVICE:-/dev/dri/renderD128} \ --volume $(pwd)/media:/media \ ${REGISTRY_URL}/${CONTAINER}:${TAG} \ - test + test || fail "Unable to execute docker" diff --git a/scripts/trigger.sh b/scripts/trigger.sh new file mode 100755 index 0000000..463be66 --- /dev/null +++ b/scripts/trigger.sh @@ -0,0 +1,139 @@ +#!/bin/bash + +# 1. List project triggers, looking for 'sys_osgc CI trigger' +# 2. If not found, create the 'sys_osgc CI trigger' +# 3. Invoke the trigger + +# The following constants are used so we can run trigger.sh +# against a known build +BUILD=${BUILD:-7517585} +BRANCH=${BRANCH:-releases_19.4} +PROJECT=${PROJECT:-50552} +REF=${REF:-master} + +eval $(grep ^SYS_OSGC_TOKEN SECRETS) + +[[ "$SYS_OSGC_TOKEN" == "" ]] && { + >&2 echo "SECRETS needs to contain a GitLab token for SYS_OSGC_TOKEN." + exit -1 +} + +GITLAB="https://gitlab.devtools.intel.com" + +echo "Looking for CI trigger" + +RESULTS=$(curl --noproxy '*' -s -X GET \ + --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \ + --header "Content-Type: application/json" \ + ${GITLAB}/api/v4/projects/${PROJECT}/triggers) + +TOKEN=$(echo ${RESULTS} | sed -E 's#([{},])#\1\n#g' | { + STATE=0 + MATCH= + TOKEN= + # Parse the JSON looking for the trigger token and description of + # 'sys_osgc CI Trigger' + while read line; do + case ${STATE} in + 0) + [[ ${line} =~ ^[[:space:]]*\{ ]] && STATE=1 + ;; + 1) + [[ "${MATCH}" != "sys_osgc CI trigger" ]] && + MATCH=$(echo ${line} | + sed -nE 's,^.*"description":\s*"([^"]+)".*$,\1,p') + [[ "${TOKEN}" == "" ]] && + TOKEN=$(echo ${line} | + sed -nE 's,^.*"token":\s*"([^"]+)".*$,\1,p') + [[ "${TOKEN}" != "" ]] && [[ "${MATCH}" == "sys_osgc CI trigger" ]] && break + [[ "${line}" =~ .*\s*} ]] && { + MATCH= + TOKEN= + STATE=0 + } + ;; + esac + done + [[ "${TOKEN}" == "" ]] && echo -n "" + echo -n "${TOKEN}" +}) + +[[ "${TOKEN}" == "" ]] && { + echo "'sys_osgc CI trigger' does not exist on ${PROJECT}." + + while read -n1 -r -p "Create new trigger [y|N]?"; do + case $REPLY in + y|Y) + echo "" + break + ;; + n|N|"") + echo " Exiting." + exit 1 + ;; + *) + echo "" + ;; + esac + done + + + RESULTS=$(curl --noproxy '*' -s -X POST \ + --header "PRIVATE-TOKEN: ${SYS_OSGC_TOKEN}" \ + -F 'description="sys_osgc CI trigger"' \ + ${GITLAB}/api/v4/projects/${PROJECT}/triggers) + + TOKEN=$(echo ${RESULTS} | sed -nE 's,.*"token":\s*"([^"]*).*,\1,p') + + [[ "${TOKEN}" == "" ]] && { + >&2 echo "Unable to create trigger" + >&2 echo "${RESULTS}" + exit 1 + } + + echo "Created token: ${TOKEN}" +} + +echo "Using token: ${TOKEN}" + +function post { + RESULTS=$(curl --noproxy '*' -s -X POST \ + -F "token=${TOKEN}" \ + -F "ref=${REF}" \ + -F "variables[OS_DISTRO]=ubuntu" \ + -F "variables[OS_RELEASE]=disco" \ + -F "variables[PACKAGE_STREAM]=disco" \ + -F "variables[BRANCH]=${BRANCH}" \ + -F "variables[BUILD]=${BUILD}" \ + -F "variables[BUILDNUMBER]=${BUILD}" \ + ${GITLAB}/api/v4/projects/${PROJECT}/trigger/pipeline) + + PIPELINE=$(echo ${RESULTS} | sed -nE 's,^.*details_path":"([^"]*)".*,\1,p') + + [[ "${PIPELINE}" == "" ]] && { + >&2 echo "Unable to find pipeline details URL" + >&2 echo ${RESULTS} + return 1 + } + + echo -e "Pipeline URL:\n\n ${GITLAB}${PIPELINE}\n" + true +} + +echo "Triggering pipeline on project ${PROJECT} for BUILD=${BUILD} and BRANCH=${BRANCH}" +while read -n1 -r -p "Is this correct [y|N]?"; do + case $REPLY in + y|Y) + echo "" + post + break + ;; + n|N|"") + echo " Exiting." + exit 1 + ;; + *) + echo "" + ;; + esac +done