Skip to content

feat: kube-rbac-proxy sidecar container for eval-hub #716

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
nbs-rh wants to merge 8 commits into
base: main
Choose a base branch
from
Open

feat: kube-rbac-proxy sidecar container for eval-hub #716

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
202f58c
feat: kube-rbac-proxy sidecar container for eval-hub
nbs-rh Apr 23, 2026
442cd36
feat: enable user header from kube-rbac-proxy
nbs-rh Apr 27, 2026
43c9e81
test: updated for latest changes
nbs-rh Apr 27, 2026
598898b
fix: for coderabbit comments
nbs-rh Apr 28, 2026
25b1a89
fix: for coderabbit comments
nbs-rh Apr 28, 2026
7cf9852
fix: for coderabbit comments
nbs-rh Apr 28, 2026
83592cc
fix: for hardcoded resource requests/limits
nbs-rh Apr 28, 2026
7872c03
fix: for coderabbit comments
nbs-rh Apr 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
597 changes: 294 additions & 303 deletions controllers/evalhub/build_test.go
View file
Open in desktop

Large diffs are not rendered by default.

5 changes: 4 additions & 1 deletion controllers/evalhub/configmap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
type ServiceConfig struct {
Port int `json:"port"`
Host string `json:"host,omitempty"`
DisableAuth bool `json:"disable_auth,omitempty"`
ReadyFile string `json:"ready_file"`
TerminationFile string `json:"termination_file"`
EvalInitImage string `json:"eval_init_image,omitempty"`
Expand Down Expand Up @@ -153,7 +154,8 @@ func (r *EvalHubReconciler) generateConfigData(ctx context.Context, instance *ev

config := EvalHubConfig{
Service: ServiceConfig{
Port: containerPort,
Port: evalHubAppPort,
DisableAuth: true,
ReadyFile: "/tmp/repo-ready",
TerminationFile: "/tmp/termination-log",
EvalInitImage: evalHubImage,
Expand All @@ -162,6 +164,7 @@ func (r *EvalHubReconciler) generateConfigData(ctx context.Context, instance *ev
EnvMappings: EnvMappings{
"PORT": "service.port",
"API_HOST": "service.host",
"DISABLE_AUTH": "service.disable_auth",
"TLS_CERT_FILE": "service.tls_cert_file",
"TLS_KEY_FILE": "service.tls_key_file",
"DB_URL": "database.url",
Expand Down
50 changes: 44 additions & 6 deletions controllers/evalhub/constants.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,15 +14,41 @@ const (

// Container configuration
containerName = "evalhub"
containerPort = 8443

// Service configuration
// evalHubAppPort is the eval-hub container listen port on loopback (API_HOST=127.0.0.1, PORT in deployment env).
// kube-rbac-proxy upstream is http://127.0.0.1:<evalHubAppPort>/ on the pod network (TLS is terminated on servicePort).
evalHubAppPort = 8444
// evalHubHealthPath is the application health check path. kube-rbac-proxy serves HTTPS on servicePort and forwards
// this path to the upstream; the proxy is started with --ignore-paths so kubelet HTTPS probes against the proxy
// can use this path without going through normal authn/z on every request.
evalHubHealthPath = "/api/v1/health"

// Service configuration (public HTTPS targets kube-rbac-proxy on this port)
serviceName = "evalhub"
servicePort = 8443

// kube-rbac-proxy sidecar
kubeRBACProxyContainerName = "kube-rbac-proxy"
kubeRBACProxyConfigMountPath = "/etc/kube-rbac-proxy/auth.yaml"
evalHubAuthConfigMapKey = "auth.yaml"
kubeRBACProxyUpstreamCAMountPath = "/etc/kube-rbac-proxy/upstream-ca"
kubeRBACProxyHealthPort = 9443

// Configuration constants
configMapName = "trustyai-service-operator-config"
configMapEvalHubImageKey = "evalHubImage"
configMapName = "trustyai-service-operator-config"
configMapEvalHubImageKey = "evalHubImage"
configMapKubeRBACProxyImageKey = "kube-rbac-proxy"
defaultKubeRBACProxyImage = "quay.io/openshift/origin-kube-rbac-proxy:4.19"
Copy link
Copy Markdown
Collaborator

@julpayne julpayne Apr 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@nbs-rh hard-coded version?

Copy link
Copy Markdown
Contributor Author

@nbs-rh nbs-rh Apr 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is the fallback if a version is not provided through the operator configmap. Since we do not own kube-rbac-proxy repo, it is better to stick to a default image that we know works well rather than use ":latest"? Btw, this version 4.19 does not have our changes with endpoint-specific SARs - it needs to be added to the configmap and here in code as default once we have an image.


// Operator ConfigMap keys — optional EvalHub / kube-rbac-proxy container CPU and memory.
configMapEvalHubCPURequestKey = "evalHubCPURequest"
configMapEvalHubMemoryRequestKey = "evalHubMemoryRequest"
configMapEvalHubCPULimitKey = "evalHubCPULimit"
configMapEvalHubMemoryLimitKey = "evalHubMemoryLimit"

configMapKubeRBACProxyCPURequestKey = "kubeRBACProxyCPURequest"
configMapKubeRBACProxyMemoryRequestKey = "kubeRBACProxyMemoryRequest"
configMapKubeRBACProxyCPULimitKey = "kubeRBACProxyCPULimit"
configMapKubeRBACProxyMemoryLimitKey = "kubeRBACProxyMemoryLimit"

// TLS configuration (OpenShift service serving certificates)
tlsSecretMountPath = "/etc/tls/private"
Expand Down Expand Up @@ -71,7 +97,7 @@ const (
)

var (
// Default resource requirements based on k8s examples
// Default resource requirements for the eval-hub app container (overridable via operator ConfigMap).
defaultResourceRequirements = corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("500m"),
Expand All @@ -83,6 +109,18 @@ var (
},
}

// defaultKubeRBACProxyResourceRequirements are defaults for the kube-rbac-proxy sidecar (overridable via operator ConfigMap).
defaultKubeRBACProxyResourceRequirements = corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("100m"),
corev1.ResourceMemory: resource.MustParse("128Mi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("500m"),
corev1.ResourceMemory: resource.MustParse("512Mi"),
},
}

// Default security context
allowPrivilegeEscalation = false
runAsNonRoot = true
Expand Down
133 changes: 106 additions & 27 deletions controllers/evalhub/deployment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,17 +74,24 @@ func (r *EvalHubReconciler) buildDeploymentSpec(ctx context.Context, instance *e
evalHubImage = defaultEvalHubImage
}

kubeRBACProxyImage, err := r.getKubeRBACProxyImage(ctx)
if err != nil {
log.FromContext(ctx).Error(err, "Error getting kube-rbac-proxy image from ConfigMap. Using the default image value of "+defaultKubeRBACProxyImage)
kubeRBACProxyImage = defaultKubeRBACProxyImage
}

settings := mergeEvalHubDeploymentOperatorSettings(ctx, r.readOperatorConfigMapData(ctx))

// Build default environment variables
// EvalHub serves TLS directly using OpenShift service serving certificates.
// Auth is handled internally via SAR checks.
// EvalHub listens on loopback only; kube-rbac-proxy terminates TLS on servicePort and enforces SAR (auth.yaml).
defaultEnvVars := []corev1.EnvVar{
{
Name: "API_HOST",
Value: "0.0.0.0",
Value: "127.0.0.1",
},
{
Name: "PORT",
Value: fmt.Sprintf("%d", containerPort),
Value: fmt.Sprintf("%d", evalHubAppPort),
},
{
Name: "TLS_CERT_FILE",
Expand Down Expand Up @@ -116,7 +123,7 @@ func (r *EvalHubReconciler) buildDeploymentSpec(ctx context.Context, instance *e
},
{
Name: "SERVICE_URL",
Value: fmt.Sprintf("https://%s.%s.svc.cluster.local:8443", instance.Name, instance.Namespace),
Value: fmt.Sprintf("https://%s.%s.svc.cluster.local:%d", instance.Name, instance.Namespace, servicePort),
},
{
Name: "EVALHUB_INSTANCE_NAME",
Expand All @@ -136,8 +143,9 @@ func (r *EvalHubReconciler) buildDeploymentSpec(ctx context.Context, instance *e
},
}

// Merge environment variables with CR values taking precedence
env := mergeEnvVars(defaultEnvVars, instance.Spec.Env)
// Merge environment variables with CR values taking precedence.
// API_HOST and PORT are fixed for the loopback listener and kube-rbac-proxy upstream; CR cannot override them.
env := mergeEnvVars(defaultEnvVars, instance.Spec.Env, "API_HOST", "PORT")

// Build volume mounts for the evalhub container
volumeMounts := []corev1.VolumeMount{
Expand Down Expand Up @@ -191,40 +199,93 @@ func (r *EvalHubReconciler) buildDeploymentSpec(ctx context.Context, instance *e
ImagePullPolicy: corev1.PullAlways,
Ports: []corev1.ContainerPort{
{
Name: "https",
ContainerPort: containerPort,
Name: "evalhub",
ContainerPort: evalHubAppPort,
Protocol: corev1.ProtocolTCP,
},
},
Env: env,
Resources: defaultResourceRequirements,
Resources: settings.EvalHubResources,
SecurityContext: defaultSecurityContext,
VolumeMounts: volumeMounts,
// HTTPGet probes with HTTPS scheme — kubelet skips TLS verification for probe requests.
LivenessProbe: &corev1.Probe{
}

upstreamURL := fmt.Sprintf("http://127.0.0.1:%d/", evalHubAppPort)
upstreamCAPath := kubeRBACProxyUpstreamCAMountPath + "/" + serviceCACertFile

kubeRBACProxyContainer := corev1.Container{
Name: kubeRBACProxyContainerName,
Image: kubeRBACProxyImage,
ImagePullPolicy: corev1.PullIfNotPresent,
Args: []string{
"--secure-listen-address=0.0.0.0:" + fmt.Sprintf("%d", servicePort),
"--upstream=" + upstreamURL,
"--upstream-ca-file=" + upstreamCAPath,
"--config-file=" + kubeRBACProxyConfigMountPath,
"--tls-cert-file=" + tlsSecretMountPath + "/" + tlsCertFile,
"--tls-private-key-file=" + tlsSecretMountPath + "/" + tlsKeyFile,
"--proxy-endpoints-port=" + fmt.Sprintf("%d", kubeRBACProxyHealthPort),
"--ignore-paths=" + evalHubHealthPath,
Comment thread
coderabbitai[bot] marked this conversation as resolved.
"--auth-header-fields-enabled",
"--auth-header-user-field-name=X-User",
"--v=0",
},
Ports: []corev1.ContainerPort{
{
Name: "https",
ContainerPort: servicePort,
Protocol: corev1.ProtocolTCP,
},
{
Name: "proxy-healthz",
ContainerPort: kubeRBACProxyHealthPort,
Protocol: corev1.ProtocolTCP,
},
},
Resources: settings.KubeRBACProxyResources,
SecurityContext: defaultSecurityContext,
VolumeMounts: []corev1.VolumeMount{
{
Name: "evalhub-config",
MountPath: kubeRBACProxyConfigMountPath,
SubPath: evalHubAuthConfigMapKey,
ReadOnly: true,
},
{
Name: instance.Name + "-tls",
MountPath: tlsSecretMountPath,
ReadOnly: true,
},
{
Name: serviceCAVolumeName,
MountPath: kubeRBACProxyUpstreamCAMountPath,
ReadOnly: true,
},
},
ReadinessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/api/v1/health",
Port: intstr.FromInt(containerPort),
Path: evalHubHealthPath,
Port: intstr.FromInt(servicePort),
Scheme: corev1.URISchemeHTTPS,
},
},
InitialDelaySeconds: 30,
PeriodSeconds: 10,
InitialDelaySeconds: 10,
PeriodSeconds: 5,
TimeoutSeconds: 5,
FailureThreshold: 3,
},
ReadinessProbe: &corev1.Probe{
LivenessProbe: &corev1.Probe{
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Path: "/api/v1/health",
Port: intstr.FromInt(containerPort),
Path: evalHubHealthPath,
Port: intstr.FromInt(servicePort),
Scheme: corev1.URISchemeHTTPS,
},
},
InitialDelaySeconds: 10,
PeriodSeconds: 5,
TimeoutSeconds: 3,
InitialDelaySeconds: 30,
PeriodSeconds: 10,
TimeoutSeconds: 5,
FailureThreshold: 3,
},
}
Expand Down Expand Up @@ -309,14 +370,14 @@ func (r *EvalHubReconciler) buildDeploymentSpec(ctx context.Context, instance *e
},
})
}
// Pod template with EvalHub container and required volumes
// Pod template with EvalHub + kube-rbac-proxy and required volumes
podTemplate := corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: labels,
},
Spec: corev1.PodSpec{
ServiceAccountName: generateServiceAccountName(instance),
Containers: []corev1.Container{container},
Containers: []corev1.Container{container, kubeRBACProxyContainer},
SecurityContext: defaultPodSecurityContext,
RestartPolicy: corev1.RestartPolicyAlways,
Volumes: volumes,
Expand Down Expand Up @@ -347,6 +408,15 @@ func (r *EvalHubReconciler) buildDeploymentSpec(ctx context.Context, instance *e
}, nil
}

// getKubeRBACProxyImage retrieves the kube-rbac-proxy image from the operator ConfigMap.
func (r *EvalHubReconciler) getKubeRBACProxyImage(ctx context.Context) (string, error) {
namespace := r.Namespace
if namespace == "" {
namespace = "trustyai-service-operator-system"
}
return utils.GetImageFromConfigMapWithFallback(ctx, r.Client, configMapKubeRBACProxyImageKey, r.effectiveOperatorConfigMapName(), namespace, defaultKubeRBACProxyImage)
}

// getEvalHubImage retrieves the EvalHub image from ConfigMap with fallback to default
func (r *EvalHubReconciler) getEvalHubImage(ctx context.Context) (string, error) {
// Get the namespace where the operator is deployed (where the ConfigMap should be)
Expand All @@ -356,12 +426,18 @@ func (r *EvalHubReconciler) getEvalHubImage(ctx context.Context) (string, error)
namespace = "trustyai-service-operator-system"
}

return utils.GetImageFromConfigMapWithFallback(ctx, r.Client, configMapEvalHubImageKey, configMapName, namespace, defaultEvalHubImage)
return utils.GetImageFromConfigMapWithFallback(ctx, r.Client, configMapEvalHubImageKey, r.effectiveOperatorConfigMapName(), namespace, defaultEvalHubImage)
}

// mergeEnvVars merges default environment variables with CR-specified ones,
// with CR values taking precedence over defaults when names conflict
func mergeEnvVars(defaults, overrides []corev1.EnvVar) []corev1.EnvVar {
// with CR values taking precedence over defaults when names conflict.
// protectedKeys names are never taken from overrides — defaults always win for those keys.
func mergeEnvVars(defaults, overrides []corev1.EnvVar, protectedKeys ...string) []corev1.EnvVar {
protected := make(map[string]struct{}, len(protectedKeys))
for _, k := range protectedKeys {
protected[k] = struct{}{}
}

// Build map of environment variables starting with defaults
envMap := make(map[string]corev1.EnvVar)
for _, env := range defaults {
Expand All @@ -370,6 +446,9 @@ func mergeEnvVars(defaults, overrides []corev1.EnvVar) []corev1.EnvVar {

// Overlay CR-specified values (they win over defaults)
for _, env := range overrides {
if _, locked := protected[env.Name]; locked {
continue
}
envMap[env.Name] = env
}

Expand Down
Loading
Loading