letsencrypt: selfsigned testing certs - use common CA, setup SAN

Some of our testing makes use of secure communication between testing
nodes; e.g. testing a load-balancer pass-through.  Other parts
"loop-back" but require flags like "curl --insecure" because the
self-signed certificates aren't trusted.

To make testing more realistic, create a CA that is distributed and
trusted by all testing nodes early in the Zuul playbook.  This then
allows us to sign local certificates created by the letsencrypt
playbooks with this trusted CA and have realistic peer-to-peer secure

The other thing this does is reworks the letsencrypt self-signed cert
path to correctly setup SAN records for the host.  This also improves
the "realism" of our testing environment.  This is so realistic that
it requires fixing the gitea playbook :).  The Apache service proxying
gitea currently has to override in testing to "localhost" because that
is all the old certificate covered; we can now just proxy to the
hostname directly for testing and production.

Change-Id: I3d49a7b683462a076263127018ec6a0f16735c94
Ian Wienand 7 months ago
parent 98938a029e
commit 0d83dd3ea0

@ -2,8 +2,3 @@ Install, configure, and run Gitea.
**Role Variables**
.. zuul:rolevar:: gitea_reverse_proxy_hostname
:default: inventory_hostname
The name of the hostname to reverse proxy to. Only necessary for
testing where we do not have a certificate for the hostname.

@ -1,2 +1 @@
gitea_no_log: true
gitea_reverse_proxy_hostname: '{{ inventory_hostname }}'

@ -38,8 +38,8 @@ Listen 3081
Use UserAgentFilter
ProxyPass /.well-known/ !
ProxyPass / https://{{ gitea_reverse_proxy_hostname }}:3000/ retry=0
ProxyPassReverse / https://{{ gitea_reverse_proxy_hostname }}:3000/
ProxyPass / https://{{ inventory_hostname }}:3000/ retry=0
ProxyPassReverse / https://{{ inventory_hostname }}:3000/

@ -2,6 +2,8 @@
# Common CA setup by Zuul test infrastructure
# Set to !0 to use letsencrypt staging rather than production requests
@ -94,8 +96,6 @@ elif [[ ${1} == "selfsign" ]]; then
# For testing, simulate the key generation
for arg in "$@"; do
# TODO(ianw): Set SAN names from the other "-d" arguments?;
# it's a pita to parse.
read -r -a domain_array <<< "$arg"
@ -104,19 +104,56 @@ elif [[ ${1} == "selfsign" ]]; then
echo "Creating certs in ${CERT_HOME}/${domain}"
# Create key for domain
openssl genrsa -out ${domain}.key 2048
# openssl makes this 0600; match the permissions acme.sh
# makes it with for general sanity
# openssl makes this 0600; match the permissions in acme.sh
chmod 0640 ${domain}.key
# Generate a fake CA key
openssl genrsa -out ca.key 2048
# Create fake CA root certificate
openssl req -x509 -new -nodes -key ca.key -sha256 -days 1024 -subj "/C=US/ST=CA/O=opendev" -out ca.cer
# Create localhost certificate signing request
openssl req -sha256 -new -key ${domain}.key -out ${domain}.csr -subj '/CN=localhost'
# Create localhost certificate signed by fake CA
openssl x509 -req -CA ca.cer -CAkey ca.key -CAcreateserial \
-sha256 -days 365 -in ${domain}.csr -out ${domain}.cer
cp ${domain}.cer fullchain.cer
# Create the certificate signing request
openssl req -new -sha256 \
-key ${domain}.key \
-subj "/C=US/ST=CA/O=OpenDev Infra/CN=${domain}" \
-out ${domain}.csr
# The argument is "-d domain -d alias -d alias" Thus when
# reading, odd numbered elements > 1 are the SAN names.
# Always add the first (which must exist)
if [[ ${len} -gt 2 ]]; then
for (( i=3; i < ${len}; i=i+2 )); do
echo "Adding SAN : ${domain_array[$i]}"
# Issue the certificate signed by the OpenDev CA that Zuul
# has pre-installed.
# NOTE(ianw) :
# * CA has to be ".crt" for update-ca-certificates but
# we've used ".cer" for certificates everywhere else
# just to make things confusing.
# * I've seen some guides add the SAN names to the CSR
# but I found x509 here requires it explicitly anyway
# to actually get it in the resulting certificate?
# Seems to be multiple ways to skin the cat with all
# these arguments and quite some variations across
# openssl versions.
openssl x509 -req -days 30 -sha256 \
-in ${domain}.csr \
-CA ${OPENDEV_CA_HOME}/ca.crt -CAkey ${OPENDEV_CA_HOME}/ca.key \
-CAcreateserial \
-out ${domain}.cer \
-extensions SAN -extfile <(printf "[SAN]\nsubjectAltName=${san}")
# Copy CA certificate for apache SSLCertificateChainFile
cp ${OPENDEV_CA_HOME}/ca.crt ca.cer
chown root:letsencrypt ca.cer
chmod 0640 ca.cer
# Save the fullchain (some apps like gitea require)
cat ${domain}.cer > fullchain.cer
cat ca.cer >> fullchain.cer
chown root:letsencyrpt fullchain.cer
chmod 0640 fullchain.cer
} 2>&1 | tee -a ${LOG_FILE}

@ -4,6 +4,69 @@
ansible_cron_disable_job: true
cloud_launcher_disable_job: true
# setup opendev CA
- hosts: bridge.openstack.org
become: true
- name: Make temporary dir for CA generation
state: directory
register: _ca_tempdir
- name: Create CA PEM/crt
shell: |
set -x
# Generate a CA key
openssl genrsa -out ca.key 2048
# Create fake CA root certificate
openssl req -x509 -new -nodes -key ca.key -sha256 -days 30 -subj "/C=US/ST=CA/O=OpenDev Infra" -out ca.crt
chdir: '{{ _ca_tempdir.path }}'
executable: /bin/bash
- name: Save key
src: '{{ _ca_tempdir.path }}/ca.key'
register: _opendev_ca_key
- name: Save certificate
src: '{{ _ca_tempdir.path }}//ca.crt'
register: _opendev_ca_certificate
- name: Cleanup tempdir
path: '{{ _ca_tempdir.path }}'
state: absent
when: _ca_tempdir.path is defined
- hosts: all
become: true
- name: Make CA directory
path: '/etc/opendev-ca'
state: directory
owner: root
group: root
mode: 0600
- name: Import files
shell: 'echo "{{ item.content }}" | base64 -d > {{ item.file }}'
creates: '{{ item.file }}'
- file: '/etc/opendev-ca/ca.key'
content: '{{ hostvars["bridge.openstack.org"]["_opendev_ca_key"]["content"] }}'
- file: '/etc/opendev-ca/ca.crt'
content: '{{ hostvars["bridge.openstack.org"]["_opendev_ca_certificate"]["content"] }}'
- name: Install and trust certificate
cmd: |
cp /etc/opendev-ca/ca.crt /usr/local/share/ca-certificates/opendev-infra-ca.crt
- hosts: bridge.openstack.org
become: true

@ -7,4 +7,3 @@ gitea_db_password: 5bfuOBKtltff0XZX
gitea_root_password: BUbBcpToMwR05ZCB
gitea_no_log: false
gitea_gerrit_password: yVpMWIUIvT7f6NwA
gitea_reverse_proxy_hostname: localhost