Add runtime reconfiguration of kubelet

This adds the CLI command 'system kube-config-kubelet'. This invokes
puppet runtime manifests to reconfigure kubelet-config ConfigMap
with new parameters, and to upgrade kubernetes nodes with new
parameters, and restart kubelet. This gives the ability to update
kubelet parameters with a software patch.

The specific kubelet-config parameters are provided within the puppet
manifests and its supporting parameters script. The specific settings
values and engineering are described in the puppet component.
Identical settings are also configured at install time in
ansible-playbooks.

TESTING:
PASS - manually fill /var/lib/docker to exceed imageGC and
       verify GC operates
PASS - AIO-DX fresh install gets updated kubelet config
PASS - AIO-DX apply/remove designer patch with updated kubelet config
PASS - 'system kube-config-kubelet' updates K8S nodes kubelet config
PASS - AIO-DX reinstall controller-1 has updated kubelet config
PASS - AIO-DX install new worker node gets updated kubelet config
PASS - build and view REST documentation

Partial-Bug: 1977754
Depends-On: https://review.opendev.org/c/starlingx/stx-puppet/+/844298
Depends-On: https://review.opendev.org/c/starlingx/ansible-playbooks/+/844305

Signed-off-by: Jim Gauld <james.gauld@windriver.com>
Change-Id: Iad32a724d3f681bc9854fa663299f8539f70fd2a

api-ref/source/api-ref-sysinv-v1-config.rst

@@ -655,6 +655,16 @@ itemNotFound (404)
                "href": "http://10.10.10.3:6385/kube_upgrade/",
                "rel": "bookmark"
          }
+      ],
+      "kube_config_kubelet": [
+         {
+               "href": "http://10.10.10.3:6385/v1/kube_config_kubelet/",
+               "rel": "self"
+         },
+         {
+               "href": "http://10.10.10.3:6385/kube_config_kubelet/",
+               "rel": "bookmark"
+         }
       ]
    }
 
@@ -12515,3 +12525,43 @@ forbidden (403), badMethod (405), overLimit (413)
    }
 
 This operation does not accept a request body.
+
+-------------------------
+Kubernetes config kubelet
+-------------------------
+
+These APIs allow the reconfiguration of kubelet-config parameters and restart of kubelet on each node.
+
+******************************************************************************
+Apply kubelet-config parameters reconfiguration and restart kubelet procedure
+******************************************************************************
+
+.. rest_method:: POST /v1/kube_config_kubelet/apply
+
+**Normal response codes**
+
+200
+
+**Error response codes**
+
+computeFault (400, 500, ...), serviceUnavailable (503), badRequest (400),
+unauthorized (401), forbidden (403), badMethod (405), overLimit (413)
+
+**Request parameters**
+
+
+**Response parameters**
+
+.. csv-table::
+   :header: "Parameter", "Style", "Type", "Description"
+   :widths: 20, 20, 20, 60
+
+   "success", "plain", "xsd:string", "The success message indicating start of the kube-config-kubelet apply"
+   "error", "plain", "xsd:string", "The error message in case something wrong happen on the API execution"
+
+::
+
+   {
+      "success": "kube-config-kubelet applied.",
+      "error": ""
+   }

sysinv/cgts-client/cgts-client/cgtsclient/v1/client.py

@@ -55,6 +55,7 @@ from cgtsclient.v1 import isystem
 from cgtsclient.v1 import iuser
 from cgtsclient.v1 import kube_cluster
 from cgtsclient.v1 import kube_cmd_version
+from cgtsclient.v1 import kube_config_kubelet
 from cgtsclient.v1 import kube_host_upgrade
 from cgtsclient.v1 import kube_rootca_update
 from cgtsclient.v1 import kube_upgrade
@@ -181,3 +182,5 @@ class Client(object):
         self.device_label = device_label.DeviceLabelManager(self.http_client)
         self.restore = restore.RestoreManager(self.http_client)
         self.kube_rootca_update = kube_rootca_update.KubeRootCAUpdateManager(self.http_client)
+        self.kube_config_kubelet = \
+            kube_config_kubelet.KubeConfigKubeletManager(self.http_client)

sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_config_kubelet.py

@@ -0,0 +1,26 @@
+#
+# Copyright (c) 2022 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+from cgtsclient.common import base
+
+
+class KubeConfigKubelet(base.Resource):
+    def __repr__(self):
+        return "<kube_config_kubelet %s>" % self._info
+
+
+class KubeConfigKubeletManager(base.Manager):
+    resource_class = KubeConfigKubelet
+
+    @staticmethod
+    def _path(id=None):
+        return '/v1/kube_config_kubelet/%s' % id if id \
+            else '/v1/kube_config_kubelet'
+
+    def apply(self):
+        path = self._path("apply")
+        _, body = self.api.json_request('POST', path)
+        return body

sysinv/cgts-client/cgts-client/cgtsclient/v1/kube_config_kubelet_shell.py

@@ -0,0 +1,29 @@
+#
+# Copyright (c) 2022 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+# All Rights Reserved.
+#
+
+from cgtsclient import exc
+
+
+def do_kube_config_kubelet(cc, args):
+    """Apply the kubelet config."""
+
+    try:
+        response = cc.kube_config_kubelet.apply()
+    except exc.HTTPNotFound:
+        raise exc.CommandError('Failed to apply kubelet config. No response.')
+    except Exception as e:
+        raise exc.CommandError('Failed to apply kubelet config: %s' % (e))
+    else:
+        success = response.get('success')
+        error = response.get('error')
+        if success:
+            print("Success: " + success)
+        if error:
+            print("Error: " + error)

sysinv/cgts-client/cgts-client/cgtsclient/v1/shell.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2013-2021 Wind River Systems, Inc.
+# Copyright (c) 2013-2022 Wind River Systems, Inc.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -42,6 +42,7 @@ from cgtsclient.v1 import isystem_shell
 from cgtsclient.v1 import iuser_shell
 
 from cgtsclient.v1 import kube_cluster_shell
+from cgtsclient.v1 import kube_config_kubelet_shell
 from cgtsclient.v1 import kube_rootca_update_shell
 from cgtsclient.v1 import kube_upgrade_shell
 from cgtsclient.v1 import kube_version_shell
@@ -128,6 +129,7 @@ COMMAND_MODULES = [
     app_shell,
     host_fs_shell,
     kube_cluster_shell,
+    kube_config_kubelet_shell,
     kube_version_shell,
     kube_upgrade_shell,
     kube_rootca_update_shell,

sysinv/sysinv/sysinv/sysinv/api/controllers/v1/__init__.py

@@ -47,6 +47,7 @@ from sysinv.api.controllers.v1 import kube_rootca_update
 from sysinv.api.controllers.v1 import kube_upgrade
 from sysinv.api.controllers.v1 import kube_version
 from sysinv.api.controllers.v1 import kube_cmd_version
+from sysinv.api.controllers.v1 import kube_config_kubelet
 from sysinv.api.controllers.v1 import label
 from sysinv.api.controllers.v1 import interface
 from sysinv.api.controllers.v1 import interface_network
@@ -281,6 +282,9 @@ class V1(base.APIBase):
     kube_host_upgrades = [link.Link]
     "Links to the kube_host_upgrade resource"
 
+    kube_config_kubelet = [link.Link]
+    "Links to the kube_config_kubelet resource "
+
     device_images = [link.Link]
     "Links to the device images resource"
 
@@ -870,6 +874,14 @@ class V1(base.APIBase):
                                                      'kube_host_upgrades', '',
                                                      bookmark=True)]
 
+        v1.kube_config_kubelet = [
+            link.Link.make_link('self', pecan.request.host_url,
+                                'kube_config_kubelet', ''),
+            link.Link.make_link('bookmark', pecan.request.host_url,
+                                'kube_config_kubelet', '',
+                                bookmark=True)
+        ]
+
         v1.device_images = [link.Link.make_link('self', pecan.request.host_url,
                                                 'device_images', ''),
                             link.Link.make_link('bookmark',
@@ -974,6 +986,7 @@ class Controller(rest.RestController):
     kube_upgrade = kube_upgrade.KubeUpgradeController()
     kube_rootca_update = kube_rootca_update.KubeRootCAUpdateController()
     kube_host_upgrades = kube_host_upgrade.KubeHostUpgradeController()
+    kube_config_kubelet = kube_config_kubelet.KubeConfigKubeletController()
     device_images = device_image.DeviceImageController()
     device_image_state = device_image_state.DeviceImageStateController()
     device_labels = device_label.DeviceLabelController()

sysinv/sysinv/sysinv/sysinv/api/controllers/v1/kube_config_kubelet.py

@@ -0,0 +1,36 @@
+########################################################################
+#
+# Copyright (c) 2022 Wind River Systems, Inc.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+########################################################################
+
+import pecan
+from pecan import expose
+from pecan import rest
+
+from sysinv.common import utils as cutils
+from sysinv.openstack.common.rpc.common import RemoteError
+
+LOCK_NAME = 'KubeConfigKubeletController'
+
+
+class KubeConfigKubeletController(rest.RestController):
+    """REST controller for kube_config_kubelet."""
+
+    _custom_actions = {
+        'apply': ['POST'],
+    }
+
+    @expose('json')
+    @cutils.synchronized(LOCK_NAME)
+    def apply(self):
+        try:
+            pecan.request.rpcapi.kube_config_kubelet(pecan.request.context)
+        except RemoteError as e:
+            return dict(success="", error=e.value)
+        except Exception as ex:
+            return dict(success="", error=str(ex))
+
+        return dict(success="kube-config-kubelet applied.", error="")

sysinv/sysinv/sysinv/sysinv/conductor/manager.py

@@ -1703,6 +1703,30 @@ class ConductorManager(service.PeriodicService):
 
         return False
 
+    def kube_config_kubelet(self, context):
+        """Update kubernetes nodes kubelet configuration ConfigMap.
+
+        This method updates kubelet parameters in configmaps/kubelet-config.
+        This leverages puppet report status so we can wait for completion
+        of the runtime manifest and trigger subsequent per-node configuration.
+
+        :param context: request context
+        """
+        active_controller = utils.HostHelper.get_active_controller(self.dbapi)
+        personalities = [constants.CONTROLLER]
+        config_uuid = self._config_update_hosts(context, personalities,
+            [active_controller.uuid])
+        config_dict = {
+            "personalities": personalities,
+            "host_uuids": [active_controller.uuid],
+            "classes": [
+                'platform::kubernetes::master::update_kubelet_params::runtime'],
+            puppet_common.REPORT_STATUS_CFG:
+                puppet_common.REPORT_KUBE_UPDATE_KUBELET_PARAMS
+        }
+        self._config_apply_runtime_manifest(
+            context, config_uuid=config_uuid, config_dict=config_dict)
+
     def update_keystone_password(self, context):
         """This method calls a puppet class
            'openstack::keystone::password::runtime'
@@ -8488,6 +8512,26 @@ class ConductorManager(service.PeriodicService):
                 self.report_sysparam_http_update_success, [],
                 self.report_sysparam_http_update_failure, [error]
             )
+        # Kubernetes kubelet parameters update and per node configuration
+        elif reported_cfg == puppet_common.REPORT_KUBE_UPDATE_KUBELET_PARAMS:
+            # The agent is reporting the runtime update_kubelet_params has been applied.
+            host_uuid = iconfig['host_uuid']
+            if status == puppet_common.REPORT_SUCCESS:
+                # Update action was successful.
+                # Invoke per-node kubelet upgrade runtime configuration.
+                success = True
+                self.handle_kube_update_params_success(context, host_uuid)
+            elif status == puppet_common.REPORT_FAILURE:
+                # Update action has failed
+                args = {'cfg': reported_cfg, 'status': status, 'iconfig': iconfig}
+                LOG.error("config runtime failure, "
+                          "reported_cfg: %(cfg)s status: %(status)s "
+                          "iconfig: %(iconfig)s" % args)
+            else:
+                args = {'cfg': reported_cfg, 'status': status, 'iconfig': iconfig}
+                LOG.error("No match for sysinv-agent manifest application reported! "
+                          "reported_cfg: %(cfg)s status: %(status)s "
+                          "iconfig: %(iconfig)s" % args)
         else:
             LOG.error("Reported configuration '%(cfg)s' is not handled by"
                       " report_config_status! iconfig: %(iconfig)s" %
@@ -9266,6 +9310,37 @@ class ConductorManager(service.PeriodicService):
             upgrade.uuid,
             {'state': constants.UPGRADE_ACTIVATION_FAILED})
 
+    def handle_kube_update_params_success(self, context, host_uuid):
+        """
+           Callback for Sysinv Agent on kube update params success.
+
+           This is invoked after kubelet-config ConfigMap is updated,
+           and does per-node kubernetes configuration.
+
+           This will download the current kubelet-config ConfigMap,
+           regenerate the configuration file /var/lib/kubelet/config.yaml,
+           and restart kubelet per node.
+
+        :param context: request context
+        :param host_uuid: host unique id
+        """
+        LOG.info("Kube update params phase succeeded on host: %s"
+                % (host_uuid))
+
+        personalities = [constants.CONTROLLER, constants.WORKER]
+        hosts = self.dbapi.ihost_get_list()
+        host_uuids = [x.uuid for x in hosts if x.personality in personalities]
+        config_uuid = self._config_update_hosts(context, personalities,
+                                                host_uuids=host_uuids)
+        config_dict = {
+            "personalities": personalities,
+            "host_uuids": host_uuids,
+            "classes": [
+                'platform::kubernetes::update_kubelet_config::runtime']
+        }
+        self._config_apply_runtime_manifest(
+            context, config_uuid=config_uuid, config_dict=config_dict)
+
     def report_kube_rootca_update_success(self, host_uuid, reported_cfg):
         """
            Callback for Sysinv Agent on kube root CA update success
@@ -11165,7 +11240,7 @@ class ConductorManager(service.PeriodicService):
                                              config_dict=config_dict)
 
         if filter_classes and config_dict['classes'] in filter_classes:
-            LOG.info("config runtime filter_clasess add %s %s" %
+            LOG.info("config runtime filter_classes add %s %s" %
                      (filter_classes, config_dict))
             self._add_runtime_class_apply_in_progress(filter_classes,
                                                       host_uuids=config_dict.get('host_uuids', None))

sysinv/sysinv/sysinv/sysinv/conductor/rpcapi.py

@@ -2088,6 +2088,13 @@ class ConductorAPI(sysinv.openstack.common.rpc.proxy.RpcProxy):
         return self.cast(context, self.make_msg('kube_upgrade_networking',
                                                 kube_version=kube_version))
 
+    def kube_config_kubelet(self, context):
+        """Sychronously, have the conductor configure kubelet.
+
+        :param context: request context.
+        """
+        return self.call(context, self.make_msg('kube_config_kubelet'))
+
     def store_bitstream_file(self, context, filename):
         """Asynchronously, have the conductor store the device image
         on this host.

sysinv/sysinv/sysinv/sysinv/puppet/common.py

@@ -51,6 +51,7 @@ REPORT_KUBE_CERT_UPDATE_PODS_TRUSTBOTHCAS = \
     'pods_' + constants.KUBE_CERT_UPDATE_TRUSTBOTHCAS
 REPORT_KUBE_CERT_UPDATE_PODS_TRUSTNEWCA = \
     'pods_' + constants.KUBE_CERT_UPDATE_TRUSTNEWCA
+REPORT_KUBE_UPDATE_KUBELET_PARAMS = 'update_kubelet_params'
 REPORT_HTTP_CONFIG = 'http_config'