Do more robust checks against zuul-web with haproxy

Switch the port 80 and 443 endpoints over to doing http checks instead
of tcp checks. This ensures that both apache and the zuul-web backend
are functional before balancing to them.

The fingergw remains a tcp check.

Change-Id: Iabe2d7822c9ef7e4514b9a0eb627f15b93ad48e2

inventory/service/group_vars/zuul-lb.yaml

@@ -2,27 +2,36 @@ zuul_lb_listeners:
   - name: balance_zuul_http
     bind:
       - ':::80'
+    check_option: 'httpchk'
     servers:
       - name: 'zuul01.opendev.org'
         address: '104.130.246.57:80'
+        check_method: 'check'
       - name: 'zuul02.opendev.org'
         address: '104.130.246.31:80'
+        check_method: 'check'
   - name: balance_zuul_https
     bind:
       - ':::443'
+    check_option: 'httpchk'
     servers:
       - name: 'zuul01.opendev.org'
         address: '104.130.246.57:443'
+        check_method: 'check-ssl verify none'
       - name: 'zuul02.opendev.org'
         address: '104.130.246.31:443'
+        check_method: 'check-ssl verify none'
   - name: balance_zuul_finger
     bind:
       - ':::79'
+    check_option: 'tcp-check'
     servers:
       - name: 'zuul01.opendev.org'
         address: '104.130.246.57:79'
+        check_method: 'check'
       - name: 'zuul02.opendev.org'
         address: '104.130.246.31:79'
+        check_method: 'check'
 iptables_extra_public_tcp_ports:
   - 443
   - 80

playbooks/roles/zuul-lb/templates/zuul-haproxy.cfg.j2

@@ -27,10 +27,10 @@ listen {{ listener.name }}
   {% endfor %}
   mode tcp
   balance source
-  option tcp-check
+  option {{ listener.check_option }}
 
   {% for server in listener.servers %}
-  server {{ server.name }} {{ server.address }} check
+  server {{ server.name }} {{ server.address }} {{ server.check_method }}
   {% endfor %}
 
 {% endfor %}

playbooks/zuul/templates/group_vars/zuul-lb.yaml.j2

@@ -2,18 +2,24 @@ zuul_lb_listeners:
   - name: balance_zuul_http
     bind:
       - ":::80"
+    check_option: "httpchk"
     servers:
       - name: "zuul02.opendev.org"
         address: "{{ (hostvars['zuul02.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:80"
+        check_method: "check"
   - name: balance_zuul_https
     bind:
       - ":::443"
+    check_option: "httpchk"
     servers:
       - name: "zuul02.opendev.org"
         address: "{{ (hostvars['zuul02.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:443"
+        check_method: 'check-ssl verify none'
   - name: balance_zuul_finger
     bind:
       - ":::79"
+    check_option: "tcp-check"
     servers:
       - name: "zuul02.opendev.org"
         address: "{{ (hostvars['zuul02.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:79"
+        check_method: "check"