Switch python-builder/python-base to pip wheel

A recent change in pip wheel cache behavior had upstream pip indicating
that we really should be using pip wheel instead. The reason we weren't
using pip wheel appears to be that we wanted to infer what top level
wheel to install via contents of a dir separate from our wheel output
dir/wheel cache. Using pip wheel implies everything gets flattened into
one location. We deal with this by having the build tool write all of
the top level wheels we care about into a separate location. Later we
can install all of the top level wheels while pointing find links at the
larger set of deps in the dir created by pip wheel.

Change-Id: Id9c674c1ec6fe5e72534549082e3adda9e286fd5

docker/python-builder/scripts/assemble

@@ -48,29 +48,30 @@ function install_wheels {
     # that sdist which is exactly what we want. This triggers code
     # generation steps such as are found in zuul, since the sequencing
     # otherwise happens in a way that makes wheel content copying unhappy.
-    # `pip wheel` isn't used here because it puts all of the output
-    # in the output dir and not the wheel cache, so it's not
-    # possible to tell what is the wheel for the project and
-    # what is the wheel cache.
-    /tmp/venv/bin/python3 -m build -o /output/wheels ./
-
-    # Install everything so that the wheel cache is populated with
-    # transitive depends. If a requirements.txt file exists, install
-    # it directly so that people can use git url syntax to do things
-    # like pick up patched but unreleased versions of dependencies.
+    #
+    # This is a separate step before we use pip wheel so that we can write
+    # the top level wheels to install into a different location than all of
+    # our dependencies.
+    /tmp/venv/bin/python3 -m build -o /output/toplevel_wheels ./
+
+    # Pip wheel everything to build a collection of all wheels for all
+    # transitive depends. This ensures that the final install on the prod
+    # images can avoid installing any build depends. If a requirements.txt
+    # file exists, start there so that people can use git url syntax to do
+    # things like pick up patched but unreleased versions of dependencies.
     # Only do this for the main package (i.e. only write requirements
     # once).
     if [ -f /tmp/src/requirements.txt ] && [ ! -f /output/requirements.txt ] ; then
-        /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels -r /tmp/src/requirements.txt
+        /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels -r /tmp/src/requirements.txt
         cp /tmp/src/requirements.txt /output/requirements.txt
     fi
-    /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels /output/wheels/*whl
+    /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels /output/toplevel_wheels/*whl
 
-    # Install each of the extras so that we collect all possibly
-    # needed wheels in the wheel cache. get-extras-packages also
+    # Pip wheel each of the extras so that we collect all possibly
+    # needed wheels in the wheel output dir. get-extras-packages also
     # writes out the req files into /output/$extra/requirements.txt.
     for req in $(get-extras-packages) ; do
-        /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels "$req"
+        /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels "$req"
     done
 }
 
@@ -98,10 +99,10 @@ if [ -f /tmp/src/upper-constraints.txt ] ; then
     CONSTRAINTS="-c /tmp/src/upper-constraints.txt"
 fi
 
-# If we got a list of packages, install them, otherwise install the
-# main package.
+# If we got a list of packages, get wheels for them, otherwise do
+# this for the main package.
 if [[ $PACKAGES ]] ; then
-    /tmp/venv/bin/pip install $CONSTRAINTS --cache-dir=/output/wheels $PACKAGES
+    /tmp/venv/bin/pip wheel $CONSTRAINTS --wheel-dir=/output/wheels $PACKAGES
     for package in $PACKAGES ; do
       echo "$package" >> /output/packages.txt
     done
@@ -113,7 +114,7 @@ else
     install_wheels
 fi
 
-# go through ZUUL_SIBLINGS, if any, and build those wheels too
+# Go through ZUUL_SIBLINGS, if any, and build those wheels too
 for sibling in ${ZUUL_SIBLINGS:-}; do
     pushd .zuul-siblings/${sibling}
     install_wheels

docker/python-builder/scripts/install-from-bindep

@@ -29,7 +29,10 @@ fi
 # to do things like pick up patched but unreleased versions
 # of dependencies.
 if [ -f /output/requirements.txt ] ; then
-    pip install $CONSTRAINTS --cache-dir=/output/wheels -r /output/requirements.txt
+    # --find-links will point pip at all of the wheels the assemble script
+    # downloaded or built. --no-index ensures that pip will only refer to
+    # the local set of wheels when looking for packages to install.
+    pip install $CONSTRAINTS --no-index --find-links=/output/wheels -r /output/requirements.txt
 fi
 
 # Add any requested extras to the list of things to install
@@ -41,10 +44,10 @@ done
 if [ -f /output/packages.txt ] ; then
   # If a package list was passed to assemble, install that in the final
   # image.
-  pip install $CONSTRAINTS --cache-dir=/output/wheels -r /output/packages.txt $EXTRAS
+  pip install $CONSTRAINTS --no-index --find-links=/output/wheels -r /output/packages.txt $EXTRAS
 else
-  # Install the wheels.
-  pip install $CONSTRAINTS --cache-dir=/output/wheels /output/wheels/*.whl $EXTRAS
+  # Install the top level wheels.
+  pip install $CONSTRAINTS --no-index --find-links=/output/wheels /output/toplevel_wheels/*.whl $EXTRAS
 fi
 
 # clean up after ourselves