K8s core arguments

core/config.go

@@ -129,6 +129,9 @@ func setDefaults() error {
 	viper.SetDefault("enableKafka", true)
 	viper.SetDefault("logAllIL", false)
 	viper.SetDefault("metricsEndpoint", "8088/ecsmetrics")
+	viper.SetDefault("enableKubernetes", false)
+	viper.SetDefault("kubeNamespace", "alice-tasks")
+	viper.SetDefault("kubeConfig", "/home/aliecs/.kube/config")
 	return nil
 }
 
@@ -200,6 +203,9 @@ func setFlags() error {
 	pflag.Bool("enableKafka", viper.GetBool("enableKafka"), "Turn on the kafka messaging")
 	pflag.Bool("logAllIL", viper.GetBool("logAllIL"), "Send all the logs into IL, including Debug and Trace messages")
 	pflag.String("metricsEndpoint", viper.GetString("metricsEndpoint"), "Http endpoint from which metrics can be scraped: [port/endpoint]")
+	pflag.Bool("enableKubernetes", viper.GetBool("enableKubernetes"), "Turn on Kubernetes support (requires valid kubeNamespace and kubeConfig)")
+	pflag.String("kubeNamespace", viper.GetString("kubeNamespace"), "Kubernetes namespace in which ECS creates Task and Environment objects")
+	pflag.String("kubeConfig", viper.GetString("kubeConfig"), "Path to the kubeconfig used to access the Kubernetes cluster (empty uses in-cluster config)")
 
 	pflag.Parse()
 	return viper.BindPFlags(pflag.CommandLine)

core/task/manager.go

@@ -150,11 +150,15 @@ func NewManager(shutdown func(), internalEventCh chan<- event.Event) (taskman *M
 	taskman.reviveOffersTrg = taskman.schedulerState.reviveOffersTrg
 	taskman.ackKilledTasks = safeacks.NewAcks()
 	taskman.k8sEnvs = newK8sEnvRegistry()
-	if k8sClient, k8sErr := newK8sClientFromViper(); k8sErr != nil {
-		log.WithField("level", infologger.IL_Devel).WithError(k8sErr).Warn("K8s client init failed, K8s tasks disabled")
-	} else {
+	if viper.GetBool("enableKubernetes") {
+		k8sClient, k8sErr := newK8sClientFromViper()
+		if k8sErr != nil {
+			return nil, fmt.Errorf("failed to initialize K8s client: %w", k8sErr)
+		}
 		log.Info("K8s client initiated")
 		taskman.k8sClient = k8sClient
+	} else {
+		log.Info("enableKubernetes not set, K8s tasks disabled")
 	}
 
 	schedState.setupCli()

core/task/managerk8s.go

@@ -84,13 +84,12 @@ func (r *k8sEnvRegistry) delete(envId uid.ID) {
 }
 
 // newK8sClientFromViper creates a K8s client from viper config.
-// Returns nil, nil if kubeNamespace is not configured (K8s disabled).
 func newK8sClientFromViper() (*k8sclient.Client, error) {
 	namespace := viper.GetString("kubeNamespace")
 	if namespace == "" {
-		return nil, nil
+		return nil, fmt.Errorf("kubeNamespace is empty")
 	}
-	kubeconfig := viper.GetString("kubeconfig")
+	kubeconfig := viper.GetString("kubeConfig")
 	return k8sclient.New(kubeconfig, namespace)
 }
 

docs/kubernetes_ecs.md

@@ -47,9 +47,16 @@ The bridge works as follows:
 This design lets ECS reuse its existing environment lifecycle, configuration generation, and monitoring logic while outsourcing Pod scheduling and OCC communication to the in-cluster operators. The relevant code can be found in `task/manager.go` and `task/managerk8s.go`. 
 The manager.go implements the split between Mesos and K8s tasks while managerk8s.go implements actual creation, deployment, transitioning and updating ECS structure. 
 
-To deploy a Kubernetes task you can in principle reuse existing control-workflow task YAML manifests by changing the control mode to `kubernetes_direct` or `kubernetes_fairmq`. But in practice there can be issues
-with reusing existing manifests, for example bad quotation of arguments: Mesos runs tasks via shell, which interpreted and stripped quotation from commands. For this reason you can find ready-to-use manifests with all necessary changes
-inside `control-operator/ecs-manifests/control-workflows/*kube-direct*`.
+To deploy a Kubernetes task you can in principle reuse existing control-workflow task YAML manifests by changing the control mode to `kubernetes_direct` or `kubernetes_fairmq`. But in practice there can be issues reusing existing manifests, for example issues with argument quotation: Mesos runs tasks via shell, which interpreted and stripped quotation from commands. But K8s uses arguments as they are passed. For this reason you can find ready-to-use manifests with all necessary changes inside `control-operator/ecs-manifests/control-workflows/*kube-direct*` and you can just copy these files to be used instead of Workflow yaml templates.
+
+### Core configuration
+
+The Kubernetes client in ECS core is configured with the following parameters, passed as CLI arguments or in placed in `o2/components/aliecs/ANY/any/settings` consul file:
+| Parameter          | Default                     | Description |
+|--------------------|-----------------------------|-------------|
+| `enableKubernetes` | `false`                     | Turns on Kubernetes support in core. When disabled, the other parameters are ignored and core runs with Mesos only. |
+| `kubeNamespace`    | `alice-tasks`               | Namespace in which ECS creates custom Task and Environment objects. Must not be empty when `enableKubernetes` is set. |
+| `kubeConfig`       | `/home/aliecs/.kube/config` | Path to the kubeconfig used to access the cluster. If empty, core tries to use the in-cluster config, which works only when core itself runs inside the cluster. |
 
 ## Running tasks (`KubectlTask`)