diff --git a/playbooks/pti-python-tarball/check.yaml b/playbooks/pti-python-tarball/check.yaml
new file mode 100644
index 0000000000..9d0b684c7c
--- /dev/null
+++ b/playbooks/pti-python-tarball/check.yaml
@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - check-python-release
+    - build-python-release
diff --git a/roles/check-python-release/README.rst b/roles/check-python-release/README.rst
new file mode 100644
index 0000000000..5ff6866da9
--- /dev/null
+++ b/roles/check-python-release/README.rst
@@ -0,0 +1,10 @@
+Test building sdist and wheel for Python projects and verify their
+metadata.
+
+** Role Variables **
+
+.. zuul:rolevar:: release_python
+   :default: python
+
+   The python interpreter to use. Set it to "python3" to use python 3,
+   for example.
diff --git a/roles/check-python-release/defaults/main.yaml b/roles/check-python-release/defaults/main.yaml
new file mode 100644
index 0000000000..10203cd5f2
--- /dev/null
+++ b/roles/check-python-release/defaults/main.yaml
@@ -0,0 +1,2 @@
+zuul_work_dir: "{{ zuul.project.src_dir }}"
+release_python: "python"
diff --git a/roles/check-python-release/tasks/main.yaml b/roles/check-python-release/tasks/main.yaml
new file mode 100644
index 0000000000..160c660faa
--- /dev/null
+++ b/roles/check-python-release/tasks/main.yaml
@@ -0,0 +1,13 @@
+- name: Install docutils for the README check
+  # NOTE(dhellmann): We install this unconstrained because we expect
+  # setuptools to expect the most recent version anyway. We use a
+  # virtualenv to avoid conflicts with system packages due to pip10
+  # and later.
+  pip:
+    name: docutils
+    virtualenv: "{{ zuul_work_dir }}/venv"
+    virtualenv_python: "{{ release_python }}"
+- name: Check the package metadata and README format
+  command: "{{ zuul_work_dir }}/venv/bin/{{ release_python }} setup.py check --restructuredtext --strict"
+  args:
+    chdir: "{{ zuul_work_dir }}"
diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
index a35e852d7b..765a024e71 100644
--- a/zuul.d/jobs.yaml
+++ b/zuul.d/jobs.yaml
@@ -224,6 +224,22 @@
     vars:
       release_python: python3
 
+- job:
+    name: test-release-openstack-python3
+    parent: base
+    description: |
+      Test building python tarballs / wheels and the packaging metadata.
+    pre-run: playbooks/pti-python-tarball/pre.yaml
+    run: playbooks/pti-python-tarball/check.yaml
+    post-run:
+      - playbooks/pti-python-tarball/post.yaml
+    vars:
+      release_python: python3
+    files:
+      - setup.cfg
+      - setup.py
+      - README.rst
+
 - job:
     name: publish-openstack-sphinx-docs-base
     description: |