diff --git a/Dockerfile b/Dockerfile index 21edd02..67ad275 100644 --- a/Dockerfile +++ b/Dockerfile @@ -112,35 +112,25 @@ ENV LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8 # # Template from templates/ubuntu/20-repositories-intel-com.in # -# Once we have a signed repository: +# Update and install wget and gpg-agent as it isn't in the base Ubuntu +# image and is needed for apt-key # -# Update and install gpg-agent as it isn't in the base Ubuntu image and -# is needed for apt-key +RUN apt-get -q update \ + && DEBIAN_FRONTEND=noninteractive \ + apt-get --no-install-recommends -q -y install \ + wget \ + gpg-agent + +# Fetch and install the signing key for https://osgc.jf.intel.com/internal # -#RUN apt-get -q update \ -# && DEBIAN_FRONTEND=noninteractive \ -# apt-get --no-install-recommends -q -y install \ -# wget \ -# gpg-agent \ -# && apt-get clean \ -# && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} - -# Fetch and install the signing key for repositories.intel.com -# -#RUN wget --no-proxy --quiet -O /tmp/repositories.key ${PACKAGE_KEYFILE} \ -# && APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \ -# apt-key add /tmp/repositories.key \ -# && rm /tmp/repositories.key - -# Once the keys are being used, remove 'trusted=yes' from the repo line -# below: - -# Install repository as trusted until we have a signed repository: -RUN echo "deb [trusted=yes arch=amd64] https://osgc.jf.intel.com/internal/ubuntu focal main" > /etc/apt/sources.list.d/intel-graphics.list +RUN wget -qO - https://osgc.jf.intel.com/internal/intel-graphics.key | sudo apt-key add - +RUN sudo apt-add-repository \ + 'deb [arch=amd64] https://osgc.jf.intel.com/internal/ubuntu focal main' # Cleanup -#RUN apt-get remove -y wget - +#RUN apt-get remove -y wget \ +# && apt-get clean \ +# && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} # # Template from templates/ubuntu/25-create-user.in diff --git a/scripts/build-dockerfile.sh b/scripts/build-dockerfile.sh index bbef853..d1a3931 100755 --- a/scripts/build-dockerfile.sh +++ b/scripts/build-dockerfile.sh @@ -31,15 +31,47 @@ The following Dockerfile.solutions were checked, but do not exist: * Dockerfile.solution.${OS_DISTRO} * Dockerfile.solution.${OS_DISTRO}-${OS_RELEASE} -Using 'Dockerfile.solution' as default. - EOF } export DOCKERFILE="Dockerfile${SOLUTION_SUFFIX}" export SOLUTION="Dockerfile.solution${SOLUTION_SUFFIX}" -VARS+=("DOCKERFILE") -VARS+=("SOLUTION") +VARS+=(DOCKERFILE SOLUTION BUILD) + +if [[ "${BUILD}" != "" ]] && [[ "${BUILD}" != "N/A" ]]; then + # Override PACKAGE_REPOSITORY as it won't be used -- instead a file + # repository will be configured + PACKAGE_REPOSITORY=file:///repos/${BUILD} + PACKAGE_STREAM="N/A" + + # ARTIFACTORY_PATH is set by scripts/build-dockerfile.sh based on the value of + # BUILD. For example, agama-ci-releases_20.4 becomes agama-ci-releases/20.4 + # + # sh (the shell interpreter of Docker) does not support subsitution so + # this can't be done within the Dockerfile itself. + # + ARTIFACTORY_PATH=${BUILD%-*} # Prune off -405 + ARTIFACTORY_PATH=${ARTIFACTORY_PATH#agama-ci-} # Prune of agama-ci- + ARTIFACTORY_PATH=${ARTIFACTORY_PATH//_/\/} # releases_20.4 => releases/20.4 + + # Since we're already making special variables, scripts/build-dockerfile.sh + # will also set ARTIFACTORY_RELEASE based on the OS_RELEASE name if Ubuntu + # is being used (eg., 20.04 for focal, etc) + case ${OS_RELEASE} in + focal) ARTIFACTORY_RELEASE=20.04 ;; # Focal Fossa + groovy) ARTIFACTORY_RELEASE=20.10 ;; # Groovy Gorilla + esac + ARTIFACTORY_RELEASE=${ARTIFACTORY_RELEASE:-${OS_RELEASE}} + + VARS+=(ARTIFACTORY_PATH ARTIFACTORY_RELEASE) + + export ARTIFACTORY_RELEASE + export ARTIFACTORY_PATH + + TAG_SUFFIX=${BUILD} +else + TAG_SUFFIX=${PACKAGE_STREAM} +fi cat << EOF Using: @@ -49,6 +81,9 @@ Using: To generate: ${DOCKERFILE} + +Values used: + EOF # Build a SHELL-FORMAT value to pass to envsubst. @@ -56,9 +91,63 @@ EOF # be escaped by envsubst. ENV='' for var in ${VARS[@]}; do + echo " ${var} = ${!var}" ENV=${ENV}'$'${var} done +SNIPPETS=() +# Create a list of all the templates for this distro +for snippet in templates/??-*.in templates/${OS_DISTRO}/??-*.in; do + if [[ "${TEMPLATE_IGNORE}" != "" ]] && [[ ${snippet} =~ ${TEMPLATE_IGNORE} ]]; then + continue + fi + + SNIPPETS+=($snippet) +done + +# Add in the list of templates from the release sub-directory +# removing from the base distro if a name conflict +if [[ -d templates/${OS_DISTRO}/${OS_RELEASE} ]]; then + for snippet in templates/${OS_DISTRO}/${OS_RELEASE}/??-*.in; do + if [[ "${TEMPLATE_IGNORE}" != "" ]] && [[ ${snippet} =~ ${TEMPLATE_IGNORE} ]]; then + continue + fi + SNIPPETS=(${SNIPPETS[@]%%*$(basename ${snippet})}) + SNIPPETS+=(${snippet}) + done +fi + +# Filter to include either ??-repositories-intel.in or +# ??-local-file-intel-repo.in based on if a BUILD was specified. +# +TMP=() +for snippet in ${SNIPPETS[@]}; do + if [[ "${BUILD}" != "" ]] && [[ "${BUILD}" != "N/A" ]]; then + # If a build was specified, then do not include ??-repositories-intel.in and + if [[ ${snippet} =~ ..-repositories-intel-com.in ]]; then + continue + fi + else + # If a build was NOT specified, then do not include ??-local-file-intel-repo.in and + if [[ ${snippet} =~ ..-local-file-intel-repo.in ]]; then + continue + fi + fi + TMP+=(${snippet}) +done +SNIPPETS=(${TMP[@]}) + +# Sort the entries by the filename by rewriting +# the entries as FILENAME DIRNAME +# Then read the sorted information back out and +# re-create the filepath, storing the results in +# the TEMPLATES array +TEMPLATES=($(for snippet in ${SNIPPETS[@]}; do + echo $(basename $snippet) $(dirname $snippet) +done | sort | while read base dir; do + echo ${dir}/${base} +done)) + # Remove the Dockerfile if it exists; should check # if it is clean first, and abort if not. # @@ -79,41 +168,6 @@ cat << EOM > ${DOCKERFILE} # EOM - - -SNIPPETS=() - -# Create a list of all the templates for this distro -for snippet in templates/??-*.in templates/${OS_DISTRO}/??-*.in; do - if [[ "${TEMPLATE_IGNORE}" != "" ]] && [[ ${snippet} =~ ${TEMPLATE_IGNORE} ]]; then - continue - fi - SNIPPETS+=($snippet) -done - -# Add in the list of templates from the release sub-directory -# removing from the base distro if a name conflict -[[ -d templates/${OS_DISTRO}/${OS_RELEASE} ]] && { - for snippet in templates/${OS_DISTRO}/${OS_RELEASE}/??-*.in; do - if [[ "${TEMPLATE_IGNORE}" != "" ]] && [[ ${snippet} =~ ${TEMPLATE_IGNORE} ]]; then - continue - fi - SNIPPETS=(${SNIPPETS[@]%%*$(basename ${snippet})}) - SNIPPETS+=($snippet) - done -} - -# Sort the entries by the filename by rewriting -# the entries as FILENAME DIRNAME -# Then read the sorted information back out and -# re-create the filepath, storing the results in -# the TEMPLATES array -TEMPLATES=($(for snippet in ${SNIPPETS[@]}; do - echo $(basename $snippet) $(dirname $snippet) -done | sort | while read base dir; do - echo ${dir}/${base} -done)) - for snippet in ${TEMPLATES[@]}; do cat << EOM >> ${DOCKERFILE} @@ -144,10 +198,9 @@ envsubst ${ENV} < templates/ending.in >> ${DOCKERFILE} cat << EOM -${DOCKERFILE} has been updated. - To build the image, you can run: - OS_DISTRO=${OS_DISTRO} OS_RELEASE=${OS_RELEASE} scripts/build-image.sh + export TAG=${OS_DISTRO}-${TAG_SUFFIX} + scripts/build-image.sh EOM diff --git a/scripts/promote-image.sh b/scripts/promote-image.sh index 97729c0..b79f877 100755 --- a/scripts/promote-image.sh +++ b/scripts/promote-image.sh @@ -10,16 +10,17 @@ for var in ${VARS[@]}; do done RELEASE_INFO=${RELEASE_INFO:-N/A} -[[ "${RELEASE_INFO}" == "N/A" ]] && { +if [[ "${RELEASE_INFO}" == "N/A" ]]; then TAG=${TAG:-${OS_DISTRO}-${PACKAGE_STREAM}-$(date +%Y%m%d)} -} || { +else TAG=${TAG:-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}} -} +fi -[[ "${TARGET_TAG}" != "" ]] || { - echo >&2 "TARGET_TAG needs to be set to the tag to push to Harbor" - exit 1 -} +if [[ "${BUILD}" != "" ]] && [[ "${BUILD}" != "N/A" ]]; then + TARGET_TAG="${OS_DISTRO}-${OS_RELEASE}-${BUILD}" +else + TARGET_TAG="${OS_DISTRO}-${PACKAGE_STREAM}" +fi function fail { >&2 echo "$*" @@ -39,13 +40,16 @@ The following will publish latest rolling image: EOF +echo "Pulling ${REGISTRY_URL}/${CONTAINER}:${TAG}" docker pull ${REGISTRY_URL}/${CONTAINER}:${TAG} || fail "Unable to pull image" +echo "Tagging as ...:${TARGET_TAG}" docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \ ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG} || fail "Unable to tag image (1/2)" +echo "Tagging as ...:latest-${TARGET_TAG}" docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \ ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG} || fail "Unable to tag image (2/2)" @@ -55,29 +59,64 @@ docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \ FQDN=${REGISTRY_URL%%/*} PROJECT=$(echo ${REGISTRY_URL#*/}/${CONTAINER} | sed s,/,%2F,g) -echo -e "Deleting tag:\n ${CONTAINER}:${TAG}\nFrom:\n ${PROJECT}" - HEADER="Authorization:Basic $(echo -n "${HARBOR_USER}:${HARBOR_PASSWD}" | base64)" -RESULTS=$(curl --noproxy '*' -s -k \ - -i \ - -X DELETE \ - -H "${HEADER}" \ - -H "accept: application/json" \ - "https://${FQDN}/api/repositories/${PROJECT}/tags/${TAG}") +# Try and delete build tag 5 times, with a 10s delay between attempts +# +iter=5 +while (( iter )); do + echo -e "Deleting tag:\n ${CONTAINER}:${TAG}\nFrom:\n ${PROJECT}" -echo ${RESULTS} | grep -q "HTTP.*200" && { - echo "Tag deleted successfully." -} || { - >&2 echo "Error deleting tag:" - >&2 echo "${RESULTS}" - exit 1 -} + RESULTS=$(curl --noproxy '*' -s -k \ + -i \ + -X DELETE \ + -H "${HEADER}" \ + -H "accept: application/json" \ + "https://${FQDN}/api/repositories/${PROJECT}/tags/${TAG}") -docker push ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG} || - fail "Unable to push ${CONTAINER}:${TARGET_TAG}" + if echo ${RESULTS} | grep -q "HTTP.*200"; then + echo "Tag deleted successfully." + break + else + echo "Error deleting tag:" >&2 + echo "${RESULTS}" >&2 + iter=$((iter-1)) + if (( ! iter )) + fail "No more tries." + fi + echo "Waiting 10s before trying again..." + sleep 10 + fi +done -docker push ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG} || +iter=5 +while (( iter )); do + echo "Pushing as ...:${TARGET_TAG}" + if docker push ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG}; then + break + fi + echo "Unable to push ${CONTAINER}:${TARGET_TAG}" >&2 + iter=$((iter-1)) + if (( ! iter )) + fail "No more tries." + fi + echo "Waiting 10s before trying again..." + sleep 10 +done + +iter=5 +while (( iter )); do + echo "Pushing as ...:latest-${TARGET_TAG}" + if docker push ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG}; then + break + fi fail "Unable to push ${CONTAINER}:latest-${TARGET_TAG}" + iter=$((iter-1)) + if (( ! iter )) + fail "No more tries." + fi + echo "Waiting 10s before trying again..." + sleep 10 +done echo "Done tagging and pushing ${CONTAINER} as ${TARGET_TAG}" diff --git a/templates/sles/15sp2/00-from.in b/templates/sles/15sp2/00-from.in new file mode 100644 index 0000000..1f045fc --- /dev/null +++ b/templates/sles/15sp2/00-from.in @@ -0,0 +1,3 @@ +# Pull internal sles 15sp1 image. + +FROM dockerv2-gfx-build.gfx-assets.intel.com/upstream/sle15:15.2 AS xe-base-stage diff --git a/templates/ubuntu/20-local-file-intel-repo.in b/templates/ubuntu/20-local-file-intel-repo.in new file mode 100755 index 0000000..0f18f96 --- /dev/null +++ b/templates/ubuntu/20-local-file-intel-repo.in @@ -0,0 +1,46 @@ +# Update and install curl and dpkg-dev as they isn't in the base Ubuntu +# image and is needed for creating a local filesystem repo +# +# Adapted from https://agama.jf.intel.com/drivers/dg1/ubuntu/local + +# Prerequisites +# +RUN apt-get -q update \ + && DEBIAN_FRONTEND=noninteractive \ + apt-get --no-install-recommends -q -y install \ + dpkg-dev \ + curl + +# Make /repos/${BUILD} directory +# +WORKDIR /repos/${BUILD} + +# Download ${BUILD} artifacts +# +# ARTIFACTORY_PATH is set by scripts/build-dockerfile.sh based on the value of +# BUILD. For example, agama-ci-releases_20.4 becomes agama-ci-releases/20.4 +# +# sh (the shell interpreter of Docker) does not support subsitution so +# this can't be done within the Dockerfile itself. +# +# Since we're already making special variables, scripts/build-dockerfile.sh +# will also set ARTIFACTORY_RELEASE based on the OS_RELEASE name if Ubuntu +# is being used (eg., 20.04 for focal, etc) +# +RUN curl -s --noproxy '*' -L \ + https://gfx-assets-build.fm.intel.com/artifactory/api/archive/download/agama-builds/ci/${ARTIFACTORY_PATH}/${BUILD}/artifacts/linux/${OS_DISTRO}/${ARTIFACTORY_RELEASE}?archiveType=tgz \ + | tar -xvz --warning=no-timestamp + +# Turn /repos/${BUILD} into a local package repository +# +RUN dpkg-scanpackages . > Packages + +# Add the local package repository to the system +RUN echo "deb [trusted=yes arch=amd64] file:///repos/${BUILD} ./" \ + | tee /etc/apt/sources.list.d/intel-graphics-agama-ci-devel-dg1-95.list +RUN apt-get -y update + +# Cleanup +RUN apt-get remove -y curl dpkg-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} diff --git a/templates/ubuntu/20-repositories-intel-com.in b/templates/ubuntu/20-repositories-intel-com.in index 56fbb67..9b2a419 100644 --- a/templates/ubuntu/20-repositories-intel-com.in +++ b/templates/ubuntu/20-repositories-intel-com.in @@ -1,29 +1,19 @@ -# Once we have a signed repository: +# Update and install wget and gpg-agent as it isn't in the base Ubuntu +# image and is needed for apt-key # -# Update and install gpg-agent as it isn't in the base Ubuntu image and -# is needed for apt-key +RUN apt-get -q update \ + && DEBIAN_FRONTEND=noninteractive \ + apt-get --no-install-recommends -q -y install \ + wget \ + gpg-agent + +# Fetch and install the signing key for ${PACKAGE_REPOSITORY} # -#RUN apt-get -q update \ -# && DEBIAN_FRONTEND=noninteractive \ -# apt-get --no-install-recommends -q -y install \ -# wget \ -# gpg-agent \ -# && apt-get clean \ -# && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log} - -# Fetch and install the signing key for repositories.intel.com -# -#RUN wget --no-proxy --quiet -O /tmp/repositories.key ${PACKAGE_KEYFILE} \ -# && APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \ -# apt-key add /tmp/repositories.key \ -# && rm /tmp/repositories.key - -# Once the keys are being used, remove 'trusted=yes' from the repo line -# below: - -# Install repository as trusted until we have a signed repository: -RUN echo "deb [trusted=yes arch=amd64] ${PACKAGE_REPOSITORY}/${OS_DISTRO} ${PACKAGE_STREAM} main" > /etc/apt/sources.list.d/intel-graphics.list +RUN wget -qO - ${PACKAGE_REPOSITORY}/intel-graphics.key | sudo apt-key add - +RUN sudo apt-add-repository \ + 'deb [arch=amd64] ${PACKAGE_REPOSITORY}/${OS_DISTRO} ${PACKAGE_STREAM} main' # Cleanup -#RUN apt-get remove -y wget - +#RUN apt-get remove -y wget \ +# && apt-get clean \ +# && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log}