diff --git a/core/config.go b/core/config.go index 0f64cbac..7c116d31 100644 --- a/core/config.go +++ b/core/config.go @@ -129,6 +129,9 @@ func setDefaults() error { viper.SetDefault("enableKafka", true) viper.SetDefault("logAllIL", false) viper.SetDefault("metricsEndpoint", "8088/ecsmetrics") + viper.SetDefault("enableKubernetes", false) + viper.SetDefault("kubeNamespace", "alice-tasks") + viper.SetDefault("kubeConfig", "/home/aliecs/.kube/config") return nil } @@ -200,6 +203,9 @@ func setFlags() error { pflag.Bool("enableKafka", viper.GetBool("enableKafka"), "Turn on the kafka messaging") pflag.Bool("logAllIL", viper.GetBool("logAllIL"), "Send all the logs into IL, including Debug and Trace messages") pflag.String("metricsEndpoint", viper.GetString("metricsEndpoint"), "Http endpoint from which metrics can be scraped: [port/endpoint]") + pflag.Bool("enableKubernetes", viper.GetBool("enableKubernetes"), "Turn on Kubernetes support (requires valid kubeNamespace and kubeConfig)") + pflag.String("kubeNamespace", viper.GetString("kubeNamespace"), "Kubernetes namespace in which ECS creates Task and Environment objects") + pflag.String("kubeConfig", viper.GetString("kubeConfig"), "Path to the kubeconfig used to access the Kubernetes cluster (empty uses in-cluster config)") pflag.Parse() return viper.BindPFlags(pflag.CommandLine) diff --git a/core/task/manager.go b/core/task/manager.go index 18815e93..268f9e88 100644 --- a/core/task/manager.go +++ b/core/task/manager.go @@ -150,11 +150,15 @@ func NewManager(shutdown func(), internalEventCh chan<- event.Event) (taskman *M taskman.reviveOffersTrg = taskman.schedulerState.reviveOffersTrg taskman.ackKilledTasks = safeacks.NewAcks() taskman.k8sEnvs = newK8sEnvRegistry() - if k8sClient, k8sErr := newK8sClientFromViper(); k8sErr != nil { - log.WithField("level", infologger.IL_Devel).WithError(k8sErr).Warn("K8s client init failed, K8s tasks disabled") - } else { + if viper.GetBool("enableKubernetes") { + k8sClient, k8sErr := newK8sClientFromViper() + if k8sErr != nil { + return nil, fmt.Errorf("failed to initialize K8s client: %w", k8sErr) + } log.Info("K8s client initiated") taskman.k8sClient = k8sClient + } else { + log.Info("enableKubernetes not set, K8s tasks disabled") } schedState.setupCli() diff --git a/core/task/managerk8s.go b/core/task/managerk8s.go index 6600d15d..9a6039b2 100644 --- a/core/task/managerk8s.go +++ b/core/task/managerk8s.go @@ -84,13 +84,12 @@ func (r *k8sEnvRegistry) delete(envId uid.ID) { } // newK8sClientFromViper creates a K8s client from viper config. -// Returns nil, nil if kubeNamespace is not configured (K8s disabled). func newK8sClientFromViper() (*k8sclient.Client, error) { namespace := viper.GetString("kubeNamespace") if namespace == "" { - return nil, nil + return nil, fmt.Errorf("kubeNamespace is empty") } - kubeconfig := viper.GetString("kubeconfig") + kubeconfig := viper.GetString("kubeConfig") return k8sclient.New(kubeconfig, namespace) } diff --git a/docs/kubernetes_ecs.md b/docs/kubernetes_ecs.md index 1452dd8c..14ce6517 100644 --- a/docs/kubernetes_ecs.md +++ b/docs/kubernetes_ecs.md @@ -47,9 +47,16 @@ The bridge works as follows: This design lets ECS reuse its existing environment lifecycle, configuration generation, and monitoring logic while outsourcing Pod scheduling and OCC communication to the in-cluster operators. The relevant code can be found in `task/manager.go` and `task/managerk8s.go`. The manager.go implements the split between Mesos and K8s tasks while managerk8s.go implements actual creation, deployment, transitioning and updating ECS structure. -To deploy a Kubernetes task you can in principle reuse existing control-workflow task YAML manifests by changing the control mode to `kubernetes_direct` or `kubernetes_fairmq`. But in practice there can be issues -with reusing existing manifests, for example bad quotation of arguments: Mesos runs tasks via shell, which interpreted and stripped quotation from commands. For this reason you can find ready-to-use manifests with all necessary changes -inside `control-operator/ecs-manifests/control-workflows/*kube-direct*`. +To deploy a Kubernetes task you can in principle reuse existing control-workflow task YAML manifests by changing the control mode to `kubernetes_direct` or `kubernetes_fairmq`. But in practice there can be issues reusing existing manifests, for example issues with argument quotation: Mesos runs tasks via shell, which interpreted and stripped quotation from commands. But K8s uses arguments as they are passed. For this reason you can find ready-to-use manifests with all necessary changes inside `control-operator/ecs-manifests/control-workflows/*kube-direct*` and you can just copy these files to be used instead of Workflow yaml templates. + +### Core configuration + +The Kubernetes client in ECS core is configured with the following parameters, passed as CLI arguments or in placed in `o2/components/aliecs/ANY/any/settings` consul file: +| Parameter | Default | Description | +|--------------------|-----------------------------|-------------| +| `enableKubernetes` | `false` | Turns on Kubernetes support in core. When disabled, the other parameters are ignored and core runs with Mesos only. | +| `kubeNamespace` | `alice-tasks` | Namespace in which ECS creates custom Task and Environment objects. Must not be empty when `enableKubernetes` is set. | +| `kubeConfig` | `/home/aliecs/.kube/config` | Path to the kubeconfig used to access the cluster. If empty, core tries to use the in-cluster config, which works only when core itself runs inside the cluster. | ## Running tasks (`KubectlTask`)