Updated to latest xe-solutions

Signed-off-by: James Ketrenos <james.p.ketrenos@intel.com>

Dockerfile

@@ -112,35 +112,25 @@ ENV LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 LC_ALL=en_US.UTF-8
 #
 # Template from templates/ubuntu/20-repositories-intel-com.in
 #
-# Once we have a signed repository:
+# Update and install wget and gpg-agent as it isn't in the base Ubuntu
+# image and is needed for apt-key
 #
-# Update and install gpg-agent as it isn't in the base Ubuntu image and 
+RUN apt-get -q update \
-# is needed for apt-key
+  && DEBIAN_FRONTEND=noninteractive \
+    apt-get --no-install-recommends -q -y install \
+      wget \
+      gpg-agent
+
+# Fetch and install the signing key for https://osgc.jf.intel.com/internal
 #
-#RUN apt-get -q update \
+RUN wget -qO - https://osgc.jf.intel.com/internal/intel-graphics.key | sudo apt-key add -
-#  && DEBIAN_FRONTEND=noninteractive \
+RUN sudo apt-add-repository \
-#    apt-get --no-install-recommends -q -y install \
+  'deb [arch=amd64] https://osgc.jf.intel.com/internal/ubuntu focal main'
-#      wget \
-#      gpg-agent \
-#  && apt-get clean \
-#  && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log}
-
-# Fetch and install the signing key for repositories.intel.com
-#
-#RUN wget --no-proxy --quiet -O /tmp/repositories.key ${PACKAGE_KEYFILE} \
-#  && APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \
-#      apt-key add /tmp/repositories.key \
-#  && rm /tmp/repositories.key
-
-# Once the keys are being used, remove 'trusted=yes' from the repo line 
-# below:
-
-# Install repository as trusted until we have a signed repository:
-RUN echo "deb [trusted=yes arch=amd64] https://osgc.jf.intel.com/internal/ubuntu focal main" > /etc/apt/sources.list.d/intel-graphics.list
 
 # Cleanup
-#RUN apt-get remove -y wget
+#RUN apt-get remove -y wget \
-
+#  && apt-get clean \
+#  && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log}
 
 #
 # Template from templates/ubuntu/25-create-user.in

scripts/build-dockerfile.sh

@@ -31,15 +31,47 @@ The following Dockerfile.solutions were checked, but do not exist:
 * Dockerfile.solution.${OS_DISTRO}
 * Dockerfile.solution.${OS_DISTRO}-${OS_RELEASE}
 
-Using 'Dockerfile.solution' as default.
-
 EOF
 }
 
 export DOCKERFILE="Dockerfile${SOLUTION_SUFFIX}"
 export SOLUTION="Dockerfile.solution${SOLUTION_SUFFIX}"
-VARS+=("DOCKERFILE")
+VARS+=(DOCKERFILE SOLUTION BUILD)
-VARS+=("SOLUTION")
+
+if [[ "${BUILD}" != "" ]] && [[ "${BUILD}" != "N/A" ]]; then
+  # Override PACKAGE_REPOSITORY as it won't be used -- instead a file
+  # repository will be configured
+  PACKAGE_REPOSITORY=file:///repos/${BUILD}
+  PACKAGE_STREAM="N/A"
+
+  # ARTIFACTORY_PATH is set by scripts/build-dockerfile.sh based on the value of
+  # BUILD. For example, agama-ci-releases_20.4 becomes agama-ci-releases/20.4
+  #
+  # sh (the shell interpreter of Docker) does not support subsitution so
+  # this can't be done within the Dockerfile itself.
+  #
+  ARTIFACTORY_PATH=${BUILD%-*}                   # Prune off -405
+  ARTIFACTORY_PATH=${ARTIFACTORY_PATH#agama-ci-} # Prune of agama-ci-
+  ARTIFACTORY_PATH=${ARTIFACTORY_PATH//_/\/}     # releases_20.4 => releases/20.4
+
+  # Since we're already making special variables, scripts/build-dockerfile.sh
+  # will also set ARTIFACTORY_RELEASE based on the OS_RELEASE name if Ubuntu
+  # is being used (eg., 20.04 for focal, etc)
+  case ${OS_RELEASE} in
+  focal) ARTIFACTORY_RELEASE=20.04 ;;  # Focal Fossa
+  groovy) ARTIFACTORY_RELEASE=20.10 ;; # Groovy Gorilla
+  esac
+  ARTIFACTORY_RELEASE=${ARTIFACTORY_RELEASE:-${OS_RELEASE}}
+  
+  VARS+=(ARTIFACTORY_PATH ARTIFACTORY_RELEASE)
+
+  export ARTIFACTORY_RELEASE
+  export ARTIFACTORY_PATH
+  
+  TAG_SUFFIX=${BUILD}
+else
+  TAG_SUFFIX=${PACKAGE_STREAM}
+fi
 
 cat << EOF
 Using:
@@ -49,6 +81,9 @@ Using:
 To generate:
 
   ${DOCKERFILE}
+
+Values used:
+
 EOF
 
 # Build a SHELL-FORMAT value to pass to envsubst.
@@ -56,9 +91,63 @@ EOF
 # be escaped by envsubst.
 ENV=''
 for var in ${VARS[@]}; do
+  echo "  ${var} = ${!var}"
   ENV=${ENV}'$'${var}
 done
 
+SNIPPETS=()
+# Create a list of all the templates for this distro
+for snippet in templates/??-*.in templates/${OS_DISTRO}/??-*.in; do
+  if [[ "${TEMPLATE_IGNORE}" != "" ]] && [[ ${snippet} =~ ${TEMPLATE_IGNORE} ]]; then
+    continue
+  fi
+  
+  SNIPPETS+=($snippet)
+done
+
+# Add in the list of templates from the release sub-directory
+# removing from the base distro if a name conflict
+if [[ -d templates/${OS_DISTRO}/${OS_RELEASE} ]]; then
+  for snippet in templates/${OS_DISTRO}/${OS_RELEASE}/??-*.in; do
+    if [[ "${TEMPLATE_IGNORE}" != "" ]] && [[ ${snippet} =~ ${TEMPLATE_IGNORE} ]]; then
+      continue
+    fi
+    SNIPPETS=(${SNIPPETS[@]%%*$(basename ${snippet})})
+    SNIPPETS+=(${snippet})
+  done
+fi
+
+# Filter to include either ??-repositories-intel.in or 
+# ??-local-file-intel-repo.in based on if a BUILD was specified.
+#
+TMP=()
+for snippet in ${SNIPPETS[@]}; do
+  if [[ "${BUILD}" != "" ]] && [[ "${BUILD}" != "N/A" ]]; then
+    # If a build was specified, then do not include ??-repositories-intel.in and
+    if [[ ${snippet} =~ ..-repositories-intel-com.in ]]; then
+      continue
+    fi
+  else 
+    # If a build was NOT specified, then do not include ??-local-file-intel-repo.in and
+    if [[ ${snippet} =~ ..-local-file-intel-repo.in ]]; then
+      continue
+    fi
+  fi
+  TMP+=(${snippet})
+done
+SNIPPETS=(${TMP[@]})
+
+# Sort the entries by the filename by rewriting
+# the entries as FILENAME DIRNAME
+# Then read the sorted information back out and
+# re-create the filepath, storing the results in
+# the TEMPLATES array
+TEMPLATES=($(for snippet in ${SNIPPETS[@]}; do
+  echo $(basename $snippet) $(dirname $snippet)
+done | sort | while read base dir; do
+  echo ${dir}/${base}
+done))
+
 # Remove the Dockerfile if it exists; should check
 # if it is clean first, and abort if not.
 #
@@ -79,41 +168,6 @@ cat << EOM > ${DOCKERFILE}
 #
 EOM
 
-
-
-SNIPPETS=()
-
-# Create a list of all the templates for this distro
-for snippet in templates/??-*.in templates/${OS_DISTRO}/??-*.in; do
-  if [[ "${TEMPLATE_IGNORE}" != "" ]] && [[ ${snippet} =~ ${TEMPLATE_IGNORE} ]]; then
-    continue
-  fi
-  SNIPPETS+=($snippet)
-done
-
-# Add in the list of templates from the release sub-directory
-# removing from the base distro if a name conflict
-[[ -d templates/${OS_DISTRO}/${OS_RELEASE} ]] && {
-  for snippet in templates/${OS_DISTRO}/${OS_RELEASE}/??-*.in; do
-    if [[ "${TEMPLATE_IGNORE}" != "" ]] && [[ ${snippet} =~ ${TEMPLATE_IGNORE} ]]; then
-      continue
-    fi
-    SNIPPETS=(${SNIPPETS[@]%%*$(basename ${snippet})})
-    SNIPPETS+=($snippet)
-  done
-}
-
-# Sort the entries by the filename by rewriting
-# the entries as FILENAME DIRNAME
-# Then read the sorted information back out and
-# re-create the filepath, storing the results in
-# the TEMPLATES array
-TEMPLATES=($(for snippet in ${SNIPPETS[@]}; do
-  echo $(basename $snippet) $(dirname $snippet)
-done | sort | while read base dir; do
-  echo ${dir}/${base}
-done))
-
 for snippet in ${TEMPLATES[@]}; do
   cat << EOM >> ${DOCKERFILE}
 
@@ -144,10 +198,9 @@ envsubst ${ENV} < templates/ending.in >> ${DOCKERFILE}
 
 cat << EOM
 
-${DOCKERFILE} has been updated.
-
 To build the image, you can run:
 
-  OS_DISTRO=${OS_DISTRO} OS_RELEASE=${OS_RELEASE} scripts/build-image.sh
+  export TAG=${OS_DISTRO}-${TAG_SUFFIX}
+  scripts/build-image.sh
 
 EOM

scripts/promote-image.sh

@@ -10,16 +10,17 @@ for var in ${VARS[@]}; do
 done
 
 RELEASE_INFO=${RELEASE_INFO:-N/A}
-[[ "${RELEASE_INFO}" == "N/A" ]] && {
+if [[ "${RELEASE_INFO}" == "N/A" ]]; then
   TAG=${TAG:-${OS_DISTRO}-${PACKAGE_STREAM}-$(date +%Y%m%d)}
-} || {
+else
   TAG=${TAG:-${OS_DISTRO}-${PACKAGE_STREAM}-${RELEASE_INFO}}
-}
+fi
 
-[[ "${TARGET_TAG}" != "" ]] || {
+if [[ "${BUILD}" != "" ]] && [[ "${BUILD}" != "N/A" ]]; then
-  echo >&2 "TARGET_TAG needs to be set to the tag to push to Harbor"
+  TARGET_TAG="${OS_DISTRO}-${OS_RELEASE}-${BUILD}"
-  exit 1
+else
-}
+  TARGET_TAG="${OS_DISTRO}-${PACKAGE_STREAM}"
+fi
 
 function fail {
   >&2 echo "$*"
@@ -39,13 +40,16 @@ The following will publish latest rolling image:
 
 EOF
 
+echo "Pulling ${REGISTRY_URL}/${CONTAINER}:${TAG}"
 docker pull ${REGISTRY_URL}/${CONTAINER}:${TAG} ||
   fail "Unable to pull image"
 
+echo "Tagging as ...:${TARGET_TAG}"
 docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \
   ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG} ||
   fail "Unable to tag image (1/2)"
 
+echo "Tagging as ...:latest-${TARGET_TAG}"
 docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \
   ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG} ||
   fail "Unable to tag image (2/2)"
@@ -55,10 +59,14 @@ docker tag ${REGISTRY_URL}/${CONTAINER}:${TAG} \
 FQDN=${REGISTRY_URL%%/*}
 PROJECT=$(echo ${REGISTRY_URL#*/}/${CONTAINER} | sed s,/,%2F,g)
 
-echo -e "Deleting tag:\n  ${CONTAINER}:${TAG}\nFrom:\n  ${PROJECT}"
-
 HEADER="Authorization:Basic $(echo -n "${HARBOR_USER}:${HARBOR_PASSWD}" | base64)"
 
+# Try and delete build tag 5 times, with a 10s delay between attempts
+#
+iter=5
+while (( iter )); do
+  echo -e "Deleting tag:\n  ${CONTAINER}:${TAG}\nFrom:\n  ${PROJECT}"
+
   RESULTS=$(curl --noproxy '*' -s -k \
     -i \
     -X DELETE \
@@ -66,18 +74,49 @@ RESULTS=$(curl --noproxy '*' -s -k \
     -H "accept: application/json" \
     "https://${FQDN}/api/repositories/${PROJECT}/tags/${TAG}")
 
-echo ${RESULTS} | grep -q "HTTP.*200" && {
+  if echo ${RESULTS} | grep -q "HTTP.*200"; then
     echo "Tag deleted successfully."
-} || {
+    break
-  >&2 echo "Error deleting tag:"
+  else
-  >&2 echo "${RESULTS}"
+    echo "Error deleting tag:" >&2
-  exit 1
+    echo "${RESULTS}" >&2
-}
+    iter=$((iter-1))
+    if (( ! iter ))
+      fail "No more tries."
+    fi
+    echo "Waiting 10s before trying again..."
+    sleep 10
+  fi
+done
 
-docker push ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG} ||
+iter=5
-  fail "Unable to push ${CONTAINER}:${TARGET_TAG}"
+while (( iter )); do
+  echo "Pushing as ...:${TARGET_TAG}"
+  if docker push ${REGISTRY_URL}/${CONTAINER}:${TARGET_TAG}; then
+    break
+  fi
+  echo "Unable to push ${CONTAINER}:${TARGET_TAG}" >&2
+  iter=$((iter-1))
+  if (( ! iter ))
+    fail "No more tries."
+  fi
+  echo "Waiting 10s before trying again..."
+  sleep 10
+done
 
-docker push ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG} ||
+iter=5
+while (( iter )); do
+  echo "Pushing as ...:latest-${TARGET_TAG}"
+  if docker push ${REGISTRY_URL}/${CONTAINER}:latest-${TARGET_TAG}; then
+    break
+  fi
   fail "Unable to push ${CONTAINER}:latest-${TARGET_TAG}"
+  iter=$((iter-1))
+  if (( ! iter ))
+    fail "No more tries."
+  fi
+  echo "Waiting 10s before trying again..."
+  sleep 10
+done
 
 echo "Done tagging and pushing ${CONTAINER} as ${TARGET_TAG}"

templates/sles/15sp2/00-from.in

@@ -0,0 +1,3 @@
+# Pull internal sles 15sp1 image.
+
+FROM dockerv2-gfx-build.gfx-assets.intel.com/upstream/sle15:15.2 AS xe-base-stage

templates/ubuntu/20-local-file-intel-repo.in

@@ -0,0 +1,46 @@
+# Update and install curl and dpkg-dev as they isn't in the base Ubuntu
+# image and is needed for creating a local filesystem repo
+#
+# Adapted from https://agama.jf.intel.com/drivers/dg1/ubuntu/local
+
+# Prerequisites
+#
+RUN apt-get -q update \
+  && DEBIAN_FRONTEND=noninteractive \
+    apt-get --no-install-recommends -q -y install \
+      dpkg-dev \
+      curl
+
+# Make /repos/${BUILD} directory
+#
+WORKDIR /repos/${BUILD}
+
+# Download ${BUILD} artifacts
+#
+# ARTIFACTORY_PATH is set by scripts/build-dockerfile.sh based on the value of
+# BUILD. For example, agama-ci-releases_20.4 becomes agama-ci-releases/20.4
+#
+# sh (the shell interpreter of Docker) does not support subsitution so
+# this can't be done within the Dockerfile itself.
+#
+# Since we're already making special variables, scripts/build-dockerfile.sh
+# will also set ARTIFACTORY_RELEASE based on the OS_RELEASE name if Ubuntu
+# is being used (eg., 20.04 for focal, etc)
+#
+RUN curl -s --noproxy '*' -L \
+  https://gfx-assets-build.fm.intel.com/artifactory/api/archive/download/agama-builds/ci/${ARTIFACTORY_PATH}/${BUILD}/artifacts/linux/${OS_DISTRO}/${ARTIFACTORY_RELEASE}?archiveType=tgz \
+  | tar -xvz --warning=no-timestamp
+
+# Turn /repos/${BUILD} into a local package repository
+#
+RUN dpkg-scanpackages . > Packages
+
+# Add the local package repository to the system
+RUN echo "deb [trusted=yes arch=amd64] file:///repos/${BUILD} ./" \
+  | tee /etc/apt/sources.list.d/intel-graphics-agama-ci-devel-dg1-95.list
+RUN apt-get -y update
+
+# Cleanup
+RUN apt-get remove -y curl dpkg-dev \
+  && apt-get clean \
+  && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log}

templates/ubuntu/20-repositories-intel-com.in

@@ -1,29 +1,19 @@
-# Once we have a signed repository:
+# Update and install wget and gpg-agent as it isn't in the base Ubuntu
+# image and is needed for apt-key
 #
-# Update and install gpg-agent as it isn't in the base Ubuntu image and 
+RUN apt-get -q update \
-# is needed for apt-key
+  && DEBIAN_FRONTEND=noninteractive \
+    apt-get --no-install-recommends -q -y install \
+      wget \
+      gpg-agent
+
+# Fetch and install the signing key for ${PACKAGE_REPOSITORY}
 #
-#RUN apt-get -q update \
+RUN wget -qO - ${PACKAGE_REPOSITORY}/intel-graphics.key | sudo apt-key add -
-#  && DEBIAN_FRONTEND=noninteractive \
+RUN sudo apt-add-repository \
-#    apt-get --no-install-recommends -q -y install \
+  'deb [arch=amd64] ${PACKAGE_REPOSITORY}/${OS_DISTRO} ${PACKAGE_STREAM} main'
-#      wget \
-#      gpg-agent \
-#  && apt-get clean \
-#  && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log}
-
-# Fetch and install the signing key for repositories.intel.com
-#
-#RUN wget --no-proxy --quiet -O /tmp/repositories.key ${PACKAGE_KEYFILE} \
-#  && APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=DontWarn \
-#      apt-key add /tmp/repositories.key \
-#  && rm /tmp/repositories.key
-
-# Once the keys are being used, remove 'trusted=yes' from the repo line 
-# below:
-
-# Install repository as trusted until we have a signed repository:
-RUN echo "deb [trusted=yes arch=amd64] ${PACKAGE_REPOSITORY}/${OS_DISTRO} ${PACKAGE_STREAM} main" > /etc/apt/sources.list.d/intel-graphics.list
 
 # Cleanup
-#RUN apt-get remove -y wget
+#RUN apt-get remove -y wget \
-
+#  && apt-get clean \
+#  && rm -rf /var/lib/apt/lists/{apt,dpkg,cache,log}