From 8622f1e6a0bfd9c42e0a80af16f5ae50fd6b74e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Tich=C3=A1k?= Date: Fri, 16 Oct 2026 17:02:57 +0200 Subject: [PATCH 1/2] fixing k8s core arguments --- core/config.go | 4 ++++ core/task/manager.go | 2 ++ core/task/managerk8s.go | 2 +- docs/kubernetes_ecs.md | 25 ++++++++++++++++++++++--- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/core/config.go b/core/config.go index 0f64cbac9..acef70e4f 100644 --- a/core/config.go +++ b/core/config.go @@ -129,6 +129,8 @@ func setDefaults() error { viper.SetDefault("enableKafka", true) viper.SetDefault("logAllIL", false) viper.SetDefault("metricsEndpoint", "8088/ecsmetrics") + viper.SetDefault("kubeNamespace", "alice-tasks") + viper.SetDefault("kubeConfig", "/home/aliecs/.kube/config") return nil } @@ -200,6 +202,8 @@ func setFlags() error { pflag.Bool("enableKafka", viper.GetBool("enableKafka"), "Turn on the kafka messaging") pflag.Bool("logAllIL", viper.GetBool("logAllIL"), "Send all the logs into IL, including Debug and Trace messages") pflag.String("metricsEndpoint", viper.GetString("metricsEndpoint"), "Http endpoint from which metrics can be scraped: [port/endpoint]") + pflag.String("kubeNamespace", viper.GetString("kubeNamespace"), "Kubernetes namespace in which ECS creates Task and Environment objects (empty disables Kubernetes support)") + pflag.String("kubeConfig", viper.GetString("kubeConfig"), "Path to the kubeconfig used to access the Kubernetes cluster (empty uses in-cluster config)") pflag.Parse() return viper.BindPFlags(pflag.CommandLine) diff --git a/core/task/manager.go b/core/task/manager.go index 18815e934..31cfeafdb 100644 --- a/core/task/manager.go +++ b/core/task/manager.go @@ -152,6 +152,8 @@ func NewManager(shutdown func(), internalEventCh chan<- event.Event) (taskman *M taskman.k8sEnvs = newK8sEnvRegistry() if k8sClient, k8sErr := newK8sClientFromViper(); k8sErr != nil { log.WithField("level", infologger.IL_Devel).WithError(k8sErr).Warn("K8s client init failed, K8s tasks disabled") + } else if k8sClient == nil { + log.Info("kubeNamespace not set, K8s tasks disabled") } else { log.Info("K8s client initiated") taskman.k8sClient = k8sClient diff --git a/core/task/managerk8s.go b/core/task/managerk8s.go index 6600d15dc..36d2cf514 100644 --- a/core/task/managerk8s.go +++ b/core/task/managerk8s.go @@ -90,7 +90,7 @@ func newK8sClientFromViper() (*k8sclient.Client, error) { if namespace == "" { return nil, nil } - kubeconfig := viper.GetString("kubeconfig") + kubeconfig := viper.GetString("kubeConfig") return k8sclient.New(kubeconfig, namespace) } diff --git a/docs/kubernetes_ecs.md b/docs/kubernetes_ecs.md index 1452dd8c9..f41490217 100644 --- a/docs/kubernetes_ecs.md +++ b/docs/kubernetes_ecs.md @@ -47,9 +47,28 @@ The bridge works as follows: This design lets ECS reuse its existing environment lifecycle, configuration generation, and monitoring logic while outsourcing Pod scheduling and OCC communication to the in-cluster operators. The relevant code can be found in `task/manager.go` and `task/managerk8s.go`. The manager.go implements the split between Mesos and K8s tasks while managerk8s.go implements actual creation, deployment, transitioning and updating ECS structure. -To deploy a Kubernetes task you can in principle reuse existing control-workflow task YAML manifests by changing the control mode to `kubernetes_direct` or `kubernetes_fairmq`. But in practice there can be issues -with reusing existing manifests, for example bad quotation of arguments: Mesos runs tasks via shell, which interpreted and stripped quotation from commands. For this reason you can find ready-to-use manifests with all necessary changes -inside `control-operator/ecs-manifests/control-workflows/*kube-direct*`. +To deploy a Kubernetes task you can in principle reuse existing control-workflow task YAML manifests by changing the control mode to `kubernetes_direct` or `kubernetes_fairmq`. But in practice there can be issues reusing existing manifests, for example issues with argument quotation: Mesos runs tasks via shell, which interpreted and stripped quotation from commands. But K8s uses arguments as they are passed. For this reason you can find ready-to-use manifests with all necessary changes inside `control-operator/ecs-manifests/control-workflows/*kube-direct*` and you can just copy these files to be used instead of Workflow yaml templates. + +### Core configuration + +The Kubernetes client in ECS core is configured with the following parameters. Like any other core parameter they can be passed as CLI arguments +| Parameter | Default | Description | +|-----------------|-----------------------------|-------------| +| `kubeNamespace` | `alice-tasks` | Namespace in which ECS creates custom Task and Environment objects. Setting it to an empty string disables Kubernetes support in core. | +| `kubeConfig` | `/home/aliecs/.kube/config` | Path to the kubeconfig used to access the cluster. If empty, core tries to use the in-cluster config, which works only when core itself runs inside the cluster. | + +If the Kubernetes client cannot be created (e.g. the kubeconfig file doesn't exist), core logs a warning `K8s client init failed, K8s tasks disabled` +and continues running with Mesos only. Deploying a task with a Kubernetes control mode then fails. + +### TaskTemplates + +Since some tasks are deployed with known values ahead of time (image, env vars, ...), we added the `TaskTemplate` CR, which stores this data in the cluster so the user doesn't have to define everything every time. You can find all the `TaskTemplate`s in the `ecs-manifests/task-templates` folder. Currently we have templates for DPL, Readout, and StfBuilder/Sender. These templates must be applied to the cluster for the ECS bridge to work. + +### DPL and JIT + +DPL tasks work similarly to those mentioned before: they use `kubernetes_fairmq` in their workflow YAML manifests. However, there is a problem: JIT tasks' manifests are created the moment they are run. So for now you need to first run the workflow to generate those tasks, then change the control mode after they've been generated. This will be changed in the future. + +### Worflow templates ## Running tasks (`KubectlTask`) From ae89c439e8b1c97cc4d217fa0a7f9eb674a87bf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Tich=C3=A1k?= Date: Mon, 19 Oct 2026 16:04:05 +0200 Subject: [PATCH 2/2] fixup! fixing k8s core arguments --- core/config.go | 4 +++- core/task/manager.go | 12 +++++++----- core/task/managerk8s.go | 3 +-- docs/kubernetes_ecs.md | 24 ++++++------------------ 4 files changed, 17 insertions(+), 26 deletions(-) diff --git a/core/config.go b/core/config.go index acef70e4f..7c116d312 100644 --- a/core/config.go +++ b/core/config.go @@ -129,6 +129,7 @@ func setDefaults() error { viper.SetDefault("enableKafka", true) viper.SetDefault("logAllIL", false) viper.SetDefault("metricsEndpoint", "8088/ecsmetrics") + viper.SetDefault("enableKubernetes", false) viper.SetDefault("kubeNamespace", "alice-tasks") viper.SetDefault("kubeConfig", "/home/aliecs/.kube/config") return nil @@ -202,7 +203,8 @@ func setFlags() error { pflag.Bool("enableKafka", viper.GetBool("enableKafka"), "Turn on the kafka messaging") pflag.Bool("logAllIL", viper.GetBool("logAllIL"), "Send all the logs into IL, including Debug and Trace messages") pflag.String("metricsEndpoint", viper.GetString("metricsEndpoint"), "Http endpoint from which metrics can be scraped: [port/endpoint]") - pflag.String("kubeNamespace", viper.GetString("kubeNamespace"), "Kubernetes namespace in which ECS creates Task and Environment objects (empty disables Kubernetes support)") + pflag.Bool("enableKubernetes", viper.GetBool("enableKubernetes"), "Turn on Kubernetes support (requires valid kubeNamespace and kubeConfig)") + pflag.String("kubeNamespace", viper.GetString("kubeNamespace"), "Kubernetes namespace in which ECS creates Task and Environment objects") pflag.String("kubeConfig", viper.GetString("kubeConfig"), "Path to the kubeconfig used to access the Kubernetes cluster (empty uses in-cluster config)") pflag.Parse() diff --git a/core/task/manager.go b/core/task/manager.go index 31cfeafdb..268f9e88c 100644 --- a/core/task/manager.go +++ b/core/task/manager.go @@ -150,13 +150,15 @@ func NewManager(shutdown func(), internalEventCh chan<- event.Event) (taskman *M taskman.reviveOffersTrg = taskman.schedulerState.reviveOffersTrg taskman.ackKilledTasks = safeacks.NewAcks() taskman.k8sEnvs = newK8sEnvRegistry() - if k8sClient, k8sErr := newK8sClientFromViper(); k8sErr != nil { - log.WithField("level", infologger.IL_Devel).WithError(k8sErr).Warn("K8s client init failed, K8s tasks disabled") - } else if k8sClient == nil { - log.Info("kubeNamespace not set, K8s tasks disabled") - } else { + if viper.GetBool("enableKubernetes") { + k8sClient, k8sErr := newK8sClientFromViper() + if k8sErr != nil { + return nil, fmt.Errorf("failed to initialize K8s client: %w", k8sErr) + } log.Info("K8s client initiated") taskman.k8sClient = k8sClient + } else { + log.Info("enableKubernetes not set, K8s tasks disabled") } schedState.setupCli() diff --git a/core/task/managerk8s.go b/core/task/managerk8s.go index 36d2cf514..9a6039b2f 100644 --- a/core/task/managerk8s.go +++ b/core/task/managerk8s.go @@ -84,11 +84,10 @@ func (r *k8sEnvRegistry) delete(envId uid.ID) { } // newK8sClientFromViper creates a K8s client from viper config. -// Returns nil, nil if kubeNamespace is not configured (K8s disabled). func newK8sClientFromViper() (*k8sclient.Client, error) { namespace := viper.GetString("kubeNamespace") if namespace == "" { - return nil, nil + return nil, fmt.Errorf("kubeNamespace is empty") } kubeconfig := viper.GetString("kubeConfig") return k8sclient.New(kubeconfig, namespace) diff --git a/docs/kubernetes_ecs.md b/docs/kubernetes_ecs.md index f41490217..14ce6517b 100644 --- a/docs/kubernetes_ecs.md +++ b/docs/kubernetes_ecs.md @@ -51,24 +51,12 @@ To deploy a Kubernetes task you can in principle reuse existing control-workflow ### Core configuration -The Kubernetes client in ECS core is configured with the following parameters. Like any other core parameter they can be passed as CLI arguments -| Parameter | Default | Description | -|-----------------|-----------------------------|-------------| -| `kubeNamespace` | `alice-tasks` | Namespace in which ECS creates custom Task and Environment objects. Setting it to an empty string disables Kubernetes support in core. | -| `kubeConfig` | `/home/aliecs/.kube/config` | Path to the kubeconfig used to access the cluster. If empty, core tries to use the in-cluster config, which works only when core itself runs inside the cluster. | - -If the Kubernetes client cannot be created (e.g. the kubeconfig file doesn't exist), core logs a warning `K8s client init failed, K8s tasks disabled` -and continues running with Mesos only. Deploying a task with a Kubernetes control mode then fails. - -### TaskTemplates - -Since some tasks are deployed with known values ahead of time (image, env vars, ...), we added the `TaskTemplate` CR, which stores this data in the cluster so the user doesn't have to define everything every time. You can find all the `TaskTemplate`s in the `ecs-manifests/task-templates` folder. Currently we have templates for DPL, Readout, and StfBuilder/Sender. These templates must be applied to the cluster for the ECS bridge to work. - -### DPL and JIT - -DPL tasks work similarly to those mentioned before: they use `kubernetes_fairmq` in their workflow YAML manifests. However, there is a problem: JIT tasks' manifests are created the moment they are run. So for now you need to first run the workflow to generate those tasks, then change the control mode after they've been generated. This will be changed in the future. - -### Worflow templates +The Kubernetes client in ECS core is configured with the following parameters, passed as CLI arguments or in placed in `o2/components/aliecs/ANY/any/settings` consul file: +| Parameter | Default | Description | +|--------------------|-----------------------------|-------------| +| `enableKubernetes` | `false` | Turns on Kubernetes support in core. When disabled, the other parameters are ignored and core runs with Mesos only. | +| `kubeNamespace` | `alice-tasks` | Namespace in which ECS creates custom Task and Environment objects. Must not be empty when `enableKubernetes` is set. | +| `kubeConfig` | `/home/aliecs/.kube/config` | Path to the kubeconfig used to access the cluster. If empty, core tries to use the in-cluster config, which works only when core itself runs inside the cluster. | ## Running tasks (`KubectlTask`)