openstack/project-config
Code Issues Proposed changes

Move wheel build role/job outside project-config

Browse Source 
We can not speculatively test changes to the wheel build jobs while it
is project config.  These jobs are in the gate for the requirements
project, which run them when bindep.txt changes.

https://review.opendev.org/731394 moved the build-wheel-mirror-base
job/playbook/role into openstack-zuul-jobs, renamed as
build-wheel-cache-* to a) distinguish the name and b) make it a little
clearer that we're building the wheels that goes on the mirror, rather
than passing through to mirroring something existing.

This updates the job names in project-config to reference these new
jobs.

Note the publishing steps are kept here along with the AFS secret.

Depends-On: https://review.opendev.org/732085
Depends-On: https://review.opendev.org/732087
Depends-On: https://review.opendev.org/732083
Depends-On: https://review.opendev.org/732084
Depends-On: https://review.opendev.org/732086

Change-Id: Iac3a906803177e8365f4cfb611800b5ccaed4a6e
This commit is contained in:
Ian Wienand
2020-05-28 13:28:09 +10:00
parent f2e40a8f02
commit 6c001b528c
8 changed files with 48 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
playbooks/wheel/build.yaml
View File

@@ -1,10 +0,0 @@
- hosts: all
strategy: free
pre_tasks:
- name: Set fact for wheel dir
set_fact:
wheel_dir: /opt/wheel/workspace
roles:
- build-wheels

14
playbooks/wheel/post-base.yaml
View File

@@ -1,14 +0,0 @@
- hosts: all
tasks:
- name: Ensure build logs directory exists
file:
path: "{{ zuul.executor.log_root }}/{{ wheel_python }}"
state: directory
delegate_to: localhost
- name: Collect build logs
synchronize:
dest: "{{ zuul.executor.log_root }}/{{ wheel_python }}"
mode: pull
src: ~/logs/
verify_host: true

26
playbooks/wheel/pre.yaml
View File

@@ -1,26 +0,0 @@
- hosts: all
roles:
- bindep
pre_tasks:
# NOTE(pabelanger): Manually install EPEL dependencies as not to
# keep EPEL enabled.
- name: Manually install EPEL dependencies
become: yes
yum:
name: "{{ item }}"
enablerepo: epel
state: present
loop:
- parallel
when: ansible_distribution == "CentOS"
# NOTE(pabelanger): We need to install extra libraries, such as
# liberasurecode-devel
# which are only part of the RDO repositories.
- name: Add centos-release-openstack-queens support
yum:
name: centos-release-openstack-queens
state: present
become: yes
when: ansible_distribution == "CentOS"

1
roles/build-wheels/README.rst
View File

@@ -1 +0,0 @@
Run the build scripts on the wheel mirror builders

90
roles/build-wheels/files/wheel-build.sh
View File

@@ -1,90 +0,0 @@
#!/bin/bash -xe
# Working variables
WHEELHOUSE_DIR=$1
WORKING_DIR=$(pwd)/src/opendev.org/openstack/requirements
PYTHON_VERSION=$2
LOGS=$(pwd)/logs
FAIL_LOG=${LOGS}/failed.txt
# preclean logs
mkdir -p ${LOGS}
rm -rf ${LOGS}/*
# Extract and iterate over all the branch names.
BRANCHES=`git --git-dir=$WORKING_DIR/.git branch -a | grep '^ stable' | \
grep -Ev '(newton)'`
for BRANCH in master $BRANCHES; do
git --git-dir=$WORKING_DIR/.git show $BRANCH:upper-constraints.txt \
2>/dev/null > /tmp/upper-constraints.txt || true
# setup the building virtualenv. We want to freshen this for each
# branch.
rm -rf build_env
virtualenv -p $PYTHON_VERSION build_env
build_env/bin/pip install --upgrade pip
# SHORT_BRANCH is just "master","newton","kilo" etc. because this
# keeps the output log hierarchy much simpler.
SHORT_BRANCH=${BRANCH##origin/}
SHORT_BRANCH=${SHORT_BRANCH##stable/}
# Failed parallel jobs don't fail the whole job, we just report
# the issues for investigation.
set +e
# This runs all the jobs under "parallel". The stdout, stderr and
# exit status for each pip invocation will be captured into files
# kept in ${LOGS}/build/${SHORT_BRANCH}/1/[package]. The --joblog
# file keeps an overview of all run jobs, which we can probe to
# find failed jobs.
cat /tmp/upper-constraints.txt | \
parallel --files --progress --joblog ${LOGS}/$SHORT_BRANCH-job.log \
--results ${LOGS}/build/$SHORT_BRANCH \
build_env/bin/pip --verbose --exists-action=i wheel \
-c /tmp/upper-constraints.txt \
-w $WHEELHOUSE_DIR {}
set -e
# Column $7 is the exit status of the job, $NF is the last
# argument to pip, which is our package.
FAILED=$(awk -e '$7!=0 {print $NF}' ${LOGS}/$SHORT_BRANCH-job.log)
if [ -n "${FAILED}" ]; then
echo "*** FAILED BUILDS FOR BRANCH ${BRANCH}" >> ${FAIL_LOG}
echo "${FAILED}" >> ${FAIL_LOG}
echo -e "***\n\n" >> ${FAIL_LOG}
fi
done
# Parse stdout from all the build logs to build up a unique list of all
# wheels downloaded from PyPI and delete them from the wheelhouse, since
# this is only meant to provide built wheels which are absent from PyPI.
find ${LOGS}/build/ -name stdout -exec grep 'Downloading' {} \; \
| sed -n 's,.*Downloading.*[ /]\([^ /]*\.whl\)\([ #].*\|$\),\1,p' \
| sort -u > ${LOGS}/remove-wheels.txt
pushd ${WHEELHOUSE_DIR}
cat ${LOGS}/remove-wheels.txt | xargs rm
popd
if [ -f ${FAIL_LOG} ]; then
cat ${FAIL_LOG}
fi
# XXX This does make a lot of log files; about 80mb after compression.
# In theory we could correlate just the failed logs and keep those
# from the failure logs above. This is currently (2017-01) left as an
# exercise for when the job is stable :) bz2 gave about 20%
# improvement over gzip in testing.
pushd ${LOGS}
tar zcvf build-logs.tar.gz ./build
rm -rf ./build
popd
# Set the final exit status to 1 if remove-wheels.txt is empty so the
# job will fail.
if [ ! -s ${LOGS}/remove-wheels.txt ]; then
echo "*** EMPTY WHEEL REMOVAL LIST: THIS SHOULD NOT HAPPEN ***"
exit 1
fi

17
roles/build-wheels/tasks/main.yaml
View File

@@ -1,17 +0,0 @@
- name: Ensure wheel dir exists
file:
group: zuul
owner: zuul
path: "{{ wheel_dir }}"
state: directory
become: yes
- name: Prevent using existing wheel mirror
become: yes
lineinfile:
path: /etc/pip.conf
state: absent
regexp: '^extra-index-url.*$'
- name: Build the wheel mirror
script: wheel-build.sh {{ wheel_dir }} {{ wheel_python }}

224
zuul.d/jobs.yaml
View File

@@ -776,7 +776,8 @@
- job:
name: openstack-upload-github-mirror
parent: opendev-upload-git-mirror
description: Mirrors official projects to github's openstack org
description: |
Mirrors official projects to github's openstack org
final: true
protected: true
secrets:
@@ -817,31 +818,10 @@
protected: true
- job:
# NOTE(ianw) : We are in the process of removing the
# "wheel-mirror" jobs in project-config as we have moved them into
# openstack-zuul-jobs. This is just until requirements removes
# all references to the old jobs.
name: build-wheel-mirror-base
parent: build-wheel-cache-base
- job:
name: build-wheel-mirror-centos-7
parent: build-wheel-mirror-base
description: |
Build CentOS 7 wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-centos-7-python2
label: centos-7
host-vars:
wheel-mirror-centos-7-python2:
wheel_python: python2
- job:
name: publish-wheel-mirror-centos-7
name: publish-wheel-cache-centos-7
description: |
Publish CentOS 7 wheels for OpenStack CI mirrors.
parent: build-wheel-mirror-centos-7
parent: build-wheel-cache-centos-7
pre-run: playbooks/openafs-client/setup.yaml
post-run: playbooks/publish/wheel-mirror.yaml
allowed-projects:
@@ -852,23 +832,10 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-centos-8
parent: build-wheel-mirror-base
description: |
Build CentOS 8 wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-centos-8-python3
label: centos-8
host-vars:
wheel-mirror-centos-8-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-centos-8
name: publish-wheel-cache-centos-8
description: |
Publish CentOS 8 wheels for OpenStack CI mirrors.
parent: build-wheel-mirror-centos-8
parent: build-wheel-cache-centos-8
pre-run: playbooks/openafs-client/setup.yaml
post-run: playbooks/publish/wheel-mirror.yaml
allowed-projects:
@@ -879,23 +846,10 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-centos-8-arm64
parent: build-wheel-mirror-base
description: |
Build CentOS 8 wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-centos-8-arm64-python3
label: centos-8-arm64
host-vars:
wheel-mirror-centos-8-arm64-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-centos-8-arm64
name: publish-wheel-cache-centos-8-arm64
description: |
Publish CentOS 8 wheels for OpenStack CI mirrors.
parent: build-wheel-mirror-centos-8-arm64
parent: build-wheel-cache-centos-8-arm64
pre-run: playbooks/openafs-client/setup.yaml
post-run: playbooks/publish/wheel-mirror.yaml
allowed-projects:
@@ -906,27 +860,10 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-debian-buster
parent: build-wheel-mirror-base
description: |
Build Debian buster wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-debian-buster-python2
label: debian-buster
- name: wheel-mirror-debian-buster-python3
label: debian-buster
host-vars:
wheel-mirror-debian-buster-python2:
wheel_python: python2
wheel-mirror-debian-buster-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-debian-buster
name: publish-wheel-cache-debian-buster
description: |
Publish Debian buster wheels for OpenStack CI mirrors.
parent: build-wheel-mirror-debian-buster
parent: build-wheel-cache-debian-buster
pre-run: playbooks/openafs-client/setup.yaml
post-run: playbooks/publish/wheel-mirror.yaml
allowed-projects:
@@ -937,27 +874,10 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-debian-buster-arm64
parent: build-wheel-mirror-base
description: |
Build Debian buster wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-debian-buster-arm64-python2
label: debian-buster-arm64
- name: wheel-mirror-debian-buster-arm64-python3
label: debian-buster-arm64
host-vars:
wheel-mirror-debian-buster-arm64-python2:
wheel_python: python2
wheel-mirror-debian-buster-arm64-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-debian-buster-arm64
name: publish-wheel-cache-debian-buster-arm64
description: |
Publish Debian buster wheels for OpenStack CI mirrors.
parent: build-wheel-mirror-debian-buster-arm64
parent: build-wheel-cache-debian-buster-arm64
pre-run: playbooks/openafs-client/setup.yaml
post-run: playbooks/publish/wheel-mirror.yaml
allowed-projects:
@@ -968,21 +888,8 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-ubuntu-focal
parent: build-wheel-mirror-base
description: |
Build Ubuntu Focal wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-ubuntu-focal-python3
label: ubuntu-focal
host-vars:
wheel-mirror-ubuntu-focal-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-ubuntu-focal
parent: build-wheel-mirror-ubuntu-focal
name: publish-wheel-cache-ubuntu-focal
parent: build-wheel-cache-ubuntu-focal
description: |
Publish Ubuntu Focal wheels for OpenStack CI mirrors.
pre-run: playbooks/openafs-client/setup.yaml
@@ -995,21 +902,8 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-ubuntu-focal-arm64
parent: build-wheel-mirror-base
description: |
Build Ubuntu Focal wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-ubuntu-focal-arm64-python3
label: ubuntu-focal-arm64
host-vars:
wheel-mirror-ubuntu-focal-arm64-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-ubuntu-focal-arm64
parent: build-wheel-mirror-ubuntu-focal-arm64
name: publish-wheel-cache-ubuntu-focal-arm64
parent: build-wheel-cache-ubuntu-focal-arm64
description: |
Publish Ubuntu Focal wheels for OpenStack CI mirrors.
pre-run: playbooks/openafs-client/setup.yaml
@@ -1022,25 +916,8 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-ubuntu-bionic
parent: build-wheel-mirror-base
description: |
Build Ubuntu Bionic wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-ubuntu-bionic-python2
label: ubuntu-bionic
- name: wheel-mirror-ubuntu-bionic-python3
label: ubuntu-bionic
host-vars:
wheel-mirror-ubuntu-bionic-python2:
wheel_python: python2
wheel-mirror-ubuntu-bionic-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-ubuntu-bionic
parent: build-wheel-mirror-ubuntu-bionic
name: publish-wheel-cache-ubuntu-bionic
parent: build-wheel-cache-ubuntu-bionic
description: |
Publish Ubuntu Bionic wheels for OpenStack CI mirrors.
pre-run: playbooks/openafs-client/setup.yaml
@@ -1053,25 +930,8 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-ubuntu-bionic-arm64
parent: build-wheel-mirror-base
description: |
Build Ubuntu Bionic wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-ubuntu-bionic-arm64-python2
label: ubuntu-bionic-arm64
- name: wheel-mirror-ubuntu-bionic-arm64-python3
label: ubuntu-bionic-arm64
host-vars:
wheel-mirror-ubuntu-bionic-arm64-python2:
wheel_python: python2
wheel-mirror-ubuntu-bionic-arm64-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-ubuntu-bionic-arm64
parent: build-wheel-mirror-ubuntu-bionic-arm64
name: publish-wheel-cache-ubuntu-bionic-arm64
parent: build-wheel-cache-ubuntu-bionic-arm64
description: |
Publish Ubuntu Bionic wheels for OpenStack CI mirrors.
pre-run: playbooks/openafs-client/setup.yaml
@@ -1084,25 +944,8 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-ubuntu-xenial
parent: build-wheel-mirror-base
description: |
Build Ubuntu Xenial wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-ubuntu-xenial-python2
label: ubuntu-xenial
- name: wheel-mirror-ubuntu-xenial-python3
label: ubuntu-xenial
host-vars:
wheel-mirror-ubuntu-xenial-python2:
wheel_python: python2
wheel-mirror-ubuntu-xenial-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-ubuntu-xenial
parent: build-wheel-mirror-ubuntu-xenial
name: publish-wheel-cache-ubuntu-xenial
parent: build-wheel-cache-ubuntu-xenial
description: |
Publish Ubuntu Xenial wheels for OpenStack CI mirrors.
pre-run: playbooks/openafs-client/setup.yaml
@@ -1115,25 +958,8 @@
secret: wheel_keytab
- job:
name: build-wheel-mirror-ubuntu-xenial-arm64
parent: build-wheel-mirror-base
description: |
Build Ubuntu Xenial ARM64 wheels for OpenStack CI mirrors.
nodeset:
nodes:
- name: wheel-mirror-ubuntu-xenial-arm64-python2
label: ubuntu-xenial-arm64
- name: wheel-mirror-ubuntu-xenial-arm64-python3
label: ubuntu-xenial-arm64
host-vars:
wheel-mirror-ubuntu-xenial-arm64-python2:
wheel_python: python2
wheel-mirror-ubuntu-xenial-arm64-python3:
wheel_python: python3
- job:
name: publish-wheel-mirror-ubuntu-xenial-arm64
parent: build-wheel-mirror-ubuntu-xenial-arm64
name: publish-wheel-cache-ubuntu-xenial-arm64
parent: build-wheel-cache-ubuntu-xenial-arm64
description: |
Publish Ubuntu Xenial ARM64 wheels for OpenStack CI mirrors.
pre-run: playbooks/openafs-client/setup.yaml
@@ -1146,7 +972,7 @@
secret: wheel_keytab
- job:
name: release-wheel-mirror
name: release-wheel-cache
description: |
Release published wheels to OpenStack CI mirrors.
run: playbooks/wheel/release.yaml

46
zuul.d/projects.yaml
View File

@@ -5413,30 +5413,30 @@
branches: master
periodic:
jobs:
- publish-wheel-mirror-centos-7
- publish-wheel-mirror-centos-8
- publish-wheel-mirror-debian-buster
- publish-wheel-mirror-ubuntu-focal
- publish-wheel-mirror-ubuntu-bionic
- publish-wheel-mirror-ubuntu-xenial
- publish-wheel-mirror-centos-8-arm64
- publish-wheel-mirror-debian-buster-arm64
- publish-wheel-mirror-ubuntu-focal-arm64
- publish-wheel-mirror-ubuntu-bionic-arm64
- publish-wheel-mirror-ubuntu-xenial-arm64
- release-wheel-mirror:
- publish-wheel-cache-centos-7
- publish-wheel-cache-centos-8
- publish-wheel-cache-debian-buster
- publish-wheel-cache-ubuntu-focal
- publish-wheel-cache-ubuntu-bionic
- publish-wheel-cache-ubuntu-xenial
- publish-wheel-cache-centos-8-arm64
- publish-wheel-cache-debian-buster-arm64
- publish-wheel-cache-ubuntu-focal-arm64
- publish-wheel-cache-ubuntu-bionic-arm64
- publish-wheel-cache-ubuntu-xenial-arm64
- release-wheel-cache:
dependencies:
- publish-wheel-mirror-centos-7
- publish-wheel-mirror-centos-8
- publish-wheel-mirror-debian-buster
- publish-wheel-mirror-ubuntu-focal
- publish-wheel-mirror-ubuntu-bionic
- publish-wheel-mirror-ubuntu-xenial
- publish-wheel-mirror-centos-8-arm64
- publish-wheel-mirror-debian-buster-arm64
- publish-wheel-mirror-ubuntu-focal-arm64
- publish-wheel-mirror-ubuntu-bionic-arm64
- publish-wheel-mirror-ubuntu-xenial-arm64
- publish-wheel-cache-centos-7
- publish-wheel-cache-centos-8
- publish-wheel-cache-debian-buster
- publish-wheel-cache-ubuntu-focal
- publish-wheel-cache-ubuntu-bionic
- publish-wheel-cache-ubuntu-xenial
- publish-wheel-cache-centos-8-arm64
- publish-wheel-cache-debian-buster-arm64
- publish-wheel-cache-ubuntu-focal-arm64
- publish-wheel-cache-ubuntu-bionic-arm64
- publish-wheel-cache-ubuntu-xenial-arm64
- propose-updates:
nodeset: ubuntu-bionic
branches: master