Migrate flyteconnector chart wiring and runtime defaults for Flyte Connectors

charts/flyte-binary/Chart.yaml

@@ -9,3 +9,9 @@ type: application
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
 
 version: v0.2.0  # VERSION
+
+dependencies:
+  - name: flyteconnector
+    version: v0.1.10
+    repository: file://../flyteconnector
+    condition: flyteconnector.enabled

charts/flyteconnector/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/flyteconnector/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v2
+name: flyteconnector
+description: A Helm chart for Flyte connector
+type: application
+version: v0.1.10  # VERSION

charts/flyteconnector/README.md

@@ -0,0 +1,45 @@
+# flyteconnector
+
+![Version: v0.1.10](https://img.shields.io/badge/Version-v0.1.10-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square)
+
+A Helm chart for Flyte connector
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| additionalContainers | list | `[]` | Appends additional containers to the deployment spec. May include template values. |
+| additionalVolumeMounts | list | `[]` | Appends additional volume mounts to the main container's spec. May include template values. |
+| additionalVolumes | list | `[]` | Appends additional volumes to the deployment spec. May include template values. |
+| affinity | object | `{}` | affinity for flyteconnector deployment |
+| commonAnnotations | object | `{}` |  |
+| commonLabels | object | `{}` |  |
+| configPath | string | `"/etc/flyteconnector/config/*.yaml"` | Default regex string for searching configuration files |
+| connectorSecret.secretData | object | `{}` | Specify your Secret (with sensitive data) or pseudo-manifest (without sensitive data). |
+| extraArgs | object | `{}` | Appends extra command line arguments to the main command |
+| fullnameOverride | string | `""` |  |
+| image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
+| image.repository | string | `"ghcr.io/flyteorg/flyte-connectors"` | Docker image for flyteconnector deployment |
+| image.tag | string | `"latest"` | Docker image tag |
+| nameOverride | string | `""` |  |
+| nodeSelector | object | `{}` | nodeSelector for flyteconnector deployment |
+| podAnnotations | object | `{}` | Annotations for flyteconnector pods |
+| podEnv | object | `{}` | Additional flyteconnector pod container environment variables |
+| podLabels | object | `{}` | Labels for flyteconnector pods |
+| podSecurityContext | object | `{}` |  |
+| ports.containerPort | int | `8000` |  |
+| ports.name | string | `"connector-grpc"` |  |
+| priorityClassName | string | `""` | Sets priorityClassName for datacatalog pod(s). |
+| rbac | object | `{"enabled":false,"rules":[{"apiGroups":[""],"resources":["secrets"],"verbs":["get"]}]}` | RBAC configuration for flyteconnector Enable to allow the connector to read secrets across namespaces (required for per-project Databricks token authentication) |
+| rbac.enabled | bool | `false` | Should RBAC resources (ClusterRole/ClusterRoleBinding) be created for flyteconnector |
+| rbac.rules | list | `[{"apiGroups":[""],"resources":["secrets"],"verbs":["get"]}]` | Rules for the ClusterRole created for flyteconnector |
+| readinessProbe | object | `{"grpc":{"port":8000},"initialDelaySeconds":1,"periodSeconds":3}` | https://kubernetes.io/blog/2022/05/13/grpc-probes-now-in-beta/#trying-the-feature-out |
+| replicaCount | int | `1` | Replicas count for flyteconnector deployment |
+| resources | object | `{"limits":{"cpu":"500m","ephemeral-storage":"200Mi","memory":"300Mi"},"requests":{"cpu":"500m","ephemeral-storage":"200Mi","memory":"200Mi"}}` | Default resources requests and limits for flyteconnector deployment |
+| securityContext | object | `{"allowPrivilegeEscalation":false}` | Security context for container |
+| service | object | `{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"}` | Service settings for flyteconnector |
+| serviceAccount | object | `{"annotations":{},"create":true,"imagePullSecrets":[]}` | Configuration for service accounts for flyteconnector |
+| serviceAccount.annotations | object | `{}` | Annotations for ServiceAccount attached to flyteconnector pods |
+| serviceAccount.create | bool | `true` | Should a service account be created for flyteconnector |
+| serviceAccount.imagePullSecrets | list | `[]` | ImagePullSecrets to automatically assign to the service account |
+| tolerations | list | `[]` | tolerations for flyteconnector deployment |

charts/flyteconnector/templates/_helpers.tpl

@@ -0,0 +1,53 @@
+{{/* vim: set filetype=mustache: */}}
+
+{{- define "flyte.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "flyte.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "flyte.namespace" -}}
+{{- default .Release.Namespace .Values.forceNamespace | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+
+{{- define "flyteconnector.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "flyteconnector.selectorLabels" -}}
+app.kubernetes.io/name: {{ template "flyteconnector.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{- define "flyteconnector.labels" -}}
+{{ include "flyteconnector.selectorLabels" . }}
+helm.sh/chart: {{ include "flyte.chart" . }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{- define "flyteconnector.podLabels" -}}
+{{ include "flyteconnector.labels" . }}
+{{- with .Values.podLabels }}
+{{ toYaml . }}
+{{- end }}
+{{- end -}}
+
+# Optional blocks for secret mount
+
+{{- define "connectorSecret.volume" -}}
+- name: {{ include "flyte.name" . }}
+  secret:
+    secretName: {{ include "flyte.name" . }}
+{{- end }}
+
+{{- define "connectorSecret.volumeMount" -}}
+- mountPath: /etc/secrets
+  name: {{ include "flyte.name" . }}
+{{- end }}
+
+{{- define "flyteconnector.servicePort" -}}
+{{ include .Values.ports.containerPort}}
+{{- end }}

charts/flyteconnector/templates/connector/deployment.yaml

@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ template "flyteconnector.name" . }}
+  namespace: {{ template "flyte.namespace" . }}
+  labels: {{ include "flyteconnector.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels: {{ include "flyteconnector.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      annotations:
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      labels: {{ include "flyteconnector.podLabels" . | nindent 8 }}
+    spec:
+      {{- if .Values.priorityClassName }}
+      priorityClassName: {{ .Values.priorityClassName }}
+      {{- end }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+      - command:
+        - flyte
+        - serve
+        - connector
+        {{- if .Values.podEnv }}
+        env:
+        {{- with .Values.podEnv }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+        {{- end }}
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        imagePullPolicy: "{{ .Values.image.pullPolicy }}"
+        name: flyteconnector
+        volumeMounts:
+        {{- include "connectorSecret.volumeMount" . | nindent 8 }}
+        {{- with .Values.additionalVolumeMounts -}}
+        {{ tpl (toYaml .) $ | nindent 8 }}
+        {{- end }}
+        ports:
+        - containerPort: {{ .Values.ports.containerPort }}
+          name: {{ .Values.ports.name }}
+        readinessProbe:
+        {{- with .Values.readinessProbe -}}
+          {{ tpl (toYaml .) $ | nindent 10 }}
+        {{- end }}
+        securityContext:
+          {{- toYaml .Values.securityContext | nindent 12 }}
+        resources: {{- toYaml .Values.resources | nindent 10 }}
+      {{- with .Values.additionalContainers -}}
+      {{- tpl (toYaml .) $ | nindent 6}}
+      {{- end }}
+      serviceAccountName: {{ template "flyteconnector.name" . }}
+      volumes: {{- include "connectorSecret.volume" . | nindent 6 }}
+      {{- with .Values.additionalVolumes -}}
+      {{ tpl (toYaml .) $ | nindent 6 }}
+      {{- end }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector: {{ tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity: {{ tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations: {{ tpl (toYaml .) $ | nindent 8 }}
+      {{- end }}

charts/flyteconnector/templates/connector/rbac.yaml

@@ -0,0 +1,31 @@
+{{- if .Values.rbac.enabled }}
+{{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }}
+apiVersion: rbac.authorization.k8s.io/v1
+{{- else }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+{{- end }}
+kind: ClusterRole
+metadata:
+  name: {{ template "flyte.namespace" . -}}-{{- template "flyteconnector.name" . }}
+  labels: {{ include "flyteconnector.labels" . | nindent 4 }}
+rules:
+  {{- toYaml .Values.rbac.rules | nindent 2 }}
+---
+{{- if $.Capabilities.APIVersions.Has "rbac.authorization.k8s.io/v1" }}
+apiVersion: rbac.authorization.k8s.io/v1
+{{- else }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+{{- end }}
+kind: ClusterRoleBinding
+metadata:
+  name: {{ template "flyte.namespace" . -}}-{{- template "flyteconnector.name" . }}
+  labels: {{ include "flyteconnector.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "flyte.namespace" . -}}-{{- template "flyteconnector.name" . }}
+subjects:
+- kind: ServiceAccount
+  name: {{ template "flyteconnector.name" . }}
+  namespace: {{ template "flyte.namespace" . }}
+{{- end }}

charts/flyteconnector/templates/connector/secret.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "flyteconnector.name" . }}
+  namespace: {{ template "flyte.namespace" . }}
+type: Opaque
+{{- with .Values.connectorSecret.secretData -}}
+{{ tpl (toYaml .) $ | nindent 0 }}
+{{- end }}

charts/flyteconnector/templates/connector/service.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "flyteconnector.name" . }}
+  namespace: {{ template "flyte.namespace" . }}
+  labels: {{ include "flyteconnector.labels" . | nindent 4 }}
+  {{- with .Values.service.annotations }}
+  annotations: {{ tpl (toYaml .) $ | nindent 4 }}
+  {{- end }}
+spec:
+  {{- with .Values.service.type}}
+  type: {{ . }}
+  {{- end }}
+  ports:
+  - name: {{ .Values.ports.name }}
+    port: {{ .Values.ports.containerPort }}
+    protocol: TCP
+    appProtocol: TCP
+    targetPort: {{ .Values.ports.name }}
+  selector: {{ include "flyteconnector.selectorLabels" . | nindent 4 }}

charts/flyteconnector/templates/connector/serviceaccount.yaml

@@ -0,0 +1,15 @@
+---
+{{- if .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "flyteconnector.name" . }}
+  namespace: {{ template "flyte.namespace" . }}
+  labels: {{ include "flyteconnector.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations: {{ tpl (toYaml .) $ | nindent 4 }}
+  {{- end}}
+{{- with .Values.serviceAccount.imagePullSecrets }}
+imagePullSecrets: {{ tpl (toYaml .) $ | nindent 2 }}
+{{- end }}
+{{- end }}

charts/flyteconnector/values.yaml

@@ -0,0 +1,102 @@
+# ---------------------------------------------------------------------
+# FLYTE_CONNCETOR SETTINGS
+# ---------------------------------------------------------------------
+
+# nameOverride String to override flyteconnector.name template
+nameOverride: ""
+# fullnameOverride String to override flyteconnector.fullname template
+fullnameOverride: ""
+# commonLabels Add labels to all the deployed resources
+commonLabels: {}
+# commonAnnotations Add annotations to all the deployed resources
+commonAnnotations: {}
+
+connectorSecret:
+  # -- Specify your Secret (with sensitive data) or pseudo-manifest (without sensitive data).
+  secretData: {}
+
+# -- Replicas count for flyteconnector deployment
+replicaCount: 1
+image:
+  # -- Docker image for flyteconnector deployment
+  repository: ghcr.io/flyteorg/flyte-connectors  # FLYTECONNECTOR_IMAGE
+  # -- Docker image tag
+  tag: latest  # FLYTECONNECTOR_TAG
+  # -- Docker image pull policy
+  pullPolicy: IfNotPresent
+ports:
+  containerPort: 8000
+  name: connector-grpc
+# -- Default resources requests and limits for flyteconnector deployment
+resources:
+  limits:
+    cpu: 500m
+    ephemeral-storage: 200Mi
+    memory: 300Mi
+  requests:
+    cpu: 500m
+    ephemeral-storage: 200Mi
+    memory: 200Mi
+# -- Default regex string for searching configuration files
+configPath: /etc/flyteconnector/config/*.yaml
+# -- Service settings for flyteconnector
+service:
+  annotations:
+    projectcontour.io/upstream-protocol.h2c: grpc
+  type: ClusterIP
+# -- Configuration for service accounts for flyteconnector
+serviceAccount:
+  # -- Should a service account be created for flyteconnector
+  create: true
+  # -- Annotations for ServiceAccount attached to flyteconnector pods
+  annotations: {}
+  # -- ImagePullSecrets to automatically assign to the service account
+  imagePullSecrets: []
+# -- Security context for pod
+# -- RBAC configuration for flyteconnector
+# Enable to allow the connector to read secrets across namespaces
+# (required for per-project Databricks token authentication)
+rbac:
+  # -- Should RBAC resources (ClusterRole/ClusterRoleBinding) be created for flyteconnector
+  enabled: false
+  # -- Rules for the ClusterRole created for flyteconnector
+  rules:
+    - apiGroups:
+        - ""
+      resources:
+        - secrets
+      verbs:
+        - get
+podSecurityContext: {}
+# -- Readiness probe for flyteconnector. Use readinessProbe: {} if connector doesn't implement grpc-health-checking service.
+# -- https://kubernetes.io/blog/2022/05/13/grpc-probes-now-in-beta/#trying-the-feature-out
+readinessProbe:
+  grpc:
+    port: 8000
+  initialDelaySeconds: 1
+  periodSeconds: 3
+# -- Security context for container
+securityContext:
+  allowPrivilegeEscalation: false
+# -- Annotations for flyteconnector pods
+podAnnotations: {}
+# -- Additional flyteconnector pod container environment variables
+podEnv: {}
+# -- Labels for flyteconnector pods
+podLabels: {}
+# -- nodeSelector for flyteconnector deployment
+nodeSelector: {}
+# -- tolerations for flyteconnector deployment
+tolerations: []
+# -- affinity for flyteconnector deployment
+affinity: {}
+# -- Appends additional volumes to the deployment spec. May include template values.
+additionalVolumes: []
+# -- Appends additional volume mounts to the main container's spec. May include template values.
+additionalVolumeMounts: []
+# -- Appends additional containers to the deployment spec. May include template values.
+additionalContainers: []
+# -- Appends extra command line arguments to the main command
+extraArgs: {}
+# -- Sets priorityClassName for datacatalog pod(s).
+priorityClassName: ""