Skip to content

Commit bec38d0

Browse files
authored
Merge pull request #20 from danehans/be_proto
Adds backend-protocol Ingress-Nginx Annotation Support
2 parents 6bdb48c + c7e008f commit bec38d0

14 files changed

Lines changed: 771 additions & 49 deletions

pkg/i2gw/implementations/kgateway/README.md

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,15 @@ The command should generate Gateway API and Kgateway resources.
8585
- `nginx.ingress.kubernetes.io/session-cookie-max-age`: Sets the TTL/expiration time for the cookie (takes precedence over `session-cookie-expires`). Maps to `BackendConfigPolicy.spec.loadBalancer.ringHash.hashPolicies[].cookie.ttl`.
8686
- `nginx.ingress.kubernetes.io/session-cookie-secure`: Sets the Secure flag on the cookie. Maps to `BackendConfigPolicy.spec.loadBalancer.ringHash.hashPolicies[].cookie.secure`.
8787
- `nginx.ingress.kubernetes.io/service-upstream`: When set to `"true"`, configures Kgateway to route to the Service’s cluster IP (or equivalent static host) instead of individual Pod IPs. For each covered Service, the emitter creates a `Backend` resource with `spec.type: Static` and rewrites the corresponding `HTTPRoute.spec.rules[].backendRefs[]` to reference that `Backend` (group `gateway.kgateway.dev`, kind `Backend`).
88+
- `nginx.ingress.kubernetes.io/backend-protocol`: Indicates the L7 protocol that is used to communicate with the proxied backend.
89+
- **Supported values (mapped):** `GRPC`, `GRPCS`
90+
- If `service-upstream: "true"` is also set for the same Service backend, the emitter sets `spec.static.appProtocol: grpc` on the generated `Backend`.
91+
- Otherwise, the emitter does **not** create or modify Kubernetes `Service` resources. Instead, it emits an **INFO** notification with a `kubectl patch`
92+
command to update the existing Service port with `appProtocol: grpc`.
93+
- **Values treated as default HTTP/1.x (no-op):** `HTTP`, `HTTPS`, `AUTO_HTTP`
94+
- **Unsupported values (rejected by provider):** `FCGI` (and others)
95+
- **Safety note:** Because emitting Service manifests could overwrite user-managed Service configuration, ingress2gateway intentionally avoids generating
96+
Service resources for this annotation.
8897

8998
### External Auth
9099

@@ -186,14 +195,19 @@ Currently supported:
186195
- `spec.type: Static`
187196
- `spec.static.hosts` containing a single `{host, port}` entry derived from the Service (e.g. `myservice.default.svc.cluster.local:80`).
188197
- Matching `HTTPRoute.spec.rules[].backendRefs[]` are rewritten to reference this `Backend` instead of the core Service.
198+
- `nginx.ingress.kubernetes.io/backend-protocol`:
199+
- When set to `GRPC` or `GRPCS` **and** `service-upstream: "true"` is set for the same backend, the emitter stamps `spec.static.appProtocol: grpc` on the generated `Backend`.
200+
- When set to `GRPC` or `GRPCS` **without** `service-upstream: "true"`, the emitter emits an **INFO** notification that includes a `kubectl patch service ...`
201+
command to set `spec.ports[].appProtocol` on the existing Service.
202+
- `HTTP`, `HTTPS`, and `AUTO_HTTP` are treated as default HTTP/1.x behavior and do not emit additional config.
189203

190204
### Summary of Policy Types
191205

192206
| Annotation Type | Kgateway Resource |
193207
|------------------------------------|-----------------------|
194208
| Request/response behavior | `TrafficPolicy` |
195209
| Upstream connection behavior | `BackendConfigPolicy` |
196-
| Upstream representation (static IP)| `Backend` |
210+
| Upstream representation. | `Backend` |
197211
| TLS passthrough | `TLSRoute` |
198212

199213
## Limitations
Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
Copyright 2023 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package kgateway
18+
19+
import (
20+
"github.com/kgateway-dev/ingress2gateway/pkg/i2gw/intermediate"
21+
kgw "github.com/kgateway-dev/kgateway/v2/api/v1alpha1/kgateway"
22+
23+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
"k8s.io/apimachinery/pkg/types"
25+
gwv1 "sigs.k8s.io/gateway-api/apis/v1"
26+
)
27+
28+
// sourceIngressAnnotation is used to label Backend CRs with the source Ingress name.
29+
const sourceIngressAnnotation = "ingress2gateway.kubernetes.io/source-ingress"
30+
31+
func backendNameForService(svcName string) string {
32+
return svcName + "-service-upstream"
33+
}
34+
35+
func backendKeyForService(ns, svcName string) types.NamespacedName {
36+
return types.NamespacedName{
37+
Namespace: ns,
38+
Name: backendNameForService(svcName),
39+
}
40+
}
41+
42+
// ensureStaticBackendForService ensures there is a Static kgateway.Backend for the given
43+
// service in the provided map. If it already exists, it is reused and optionally
44+
// updated with the given protocol. If not, a new Backend is created.
45+
//
46+
// ingressName is only used for the source-ingress label.
47+
func ensureStaticBackendForService(
48+
ingressName string,
49+
httpRouteKey types.NamespacedName,
50+
svcName string,
51+
host string,
52+
port int32,
53+
protocol *intermediate.BackendProtocol,
54+
backends map[types.NamespacedName]*kgw.Backend,
55+
) *kgw.Backend {
56+
backendKey := backendKeyForService(httpRouteKey.Namespace, svcName)
57+
58+
// Reuse existing Backend CR if present.
59+
if kb, ok := backends[backendKey]; ok {
60+
if protocol != nil && kb.Spec.Static != nil && kb.Spec.Static.AppProtocol == nil {
61+
switch *protocol {
62+
case intermediate.BackendProtocolGRPC:
63+
ap := kgw.AppProtocolGrpc
64+
kb.Spec.Static.AppProtocol = &ap
65+
}
66+
}
67+
return kb
68+
}
69+
70+
kb := &kgw.Backend{
71+
TypeMeta: metav1.TypeMeta{
72+
Kind: BackendGVK.Kind,
73+
APIVersion: BackendGVK.GroupVersion().String(),
74+
},
75+
ObjectMeta: metav1.ObjectMeta{
76+
Name: backendKey.Name,
77+
Namespace: backendKey.Namespace,
78+
Labels: map[string]string{
79+
sourceIngressAnnotation: ingressName,
80+
},
81+
},
82+
Spec: kgw.BackendSpec{
83+
Type: kgw.BackendTypeStatic,
84+
Static: &kgw.StaticBackend{
85+
Hosts: []kgw.Host{
86+
{
87+
Host: host,
88+
Port: gwv1.PortNumber(port),
89+
},
90+
},
91+
},
92+
},
93+
}
94+
95+
if protocol != nil {
96+
switch *protocol {
97+
case intermediate.BackendProtocolGRPC:
98+
ap := kgw.AppProtocolGrpc
99+
kb.Spec.Static.AppProtocol = &ap
100+
}
101+
}
102+
103+
backends[backendKey] = kb
104+
return kb
105+
}
Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
/*
2+
Copyright 2023 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package kgateway
18+
19+
import (
20+
"github.com/kgateway-dev/ingress2gateway/pkg/i2gw/intermediate"
21+
kgw "github.com/kgateway-dev/kgateway/v2/api/v1alpha1/kgateway"
22+
23+
"k8s.io/apimachinery/pkg/types"
24+
gwv1 "sigs.k8s.io/gateway-api/apis/v1"
25+
)
26+
27+
// applyBackendProtocol projects backend protocol metadata on IR Backends into
28+
// typed Kgateway Backend CRs and rewrites HTTPRoute backendRefs to reference
29+
// those Backends.
30+
//
31+
// Semantics:
32+
// - For each Policy.Backends entry produced by the ingress-nginx provider,
33+
// we ensure there is a Static Backend CR with the correct host / port and
34+
// (if set) appProtocol (currently only gRPC).
35+
// - HTTPRoute backendRefs that were originally core Services and are covered
36+
// by this Policy are rewritten to:
37+
// group: gateway.kgateway.dev
38+
// kind: Backend
39+
// name: <svc>-service-upstream
40+
// - This works regardless of whether service-upstream was also used.
41+
func applyBackendProtocol(
42+
pol intermediate.Policy,
43+
ingressName string,
44+
httpRouteKey types.NamespacedName,
45+
httpRouteCtx *intermediate.HTTPRouteContext,
46+
backends map[types.NamespacedName]*kgw.Backend,
47+
) {
48+
if len(pol.Backends) == 0 || len(pol.RuleBackendSources) == 0 {
49+
return
50+
}
51+
52+
for _, idx := range pol.RuleBackendSources {
53+
if idx.Rule >= len(httpRouteCtx.Spec.Rules) {
54+
continue
55+
}
56+
rule := &httpRouteCtx.Spec.Rules[idx.Rule]
57+
if idx.Backend >= len(rule.BackendRefs) {
58+
continue
59+
}
60+
61+
br := &rule.BackendRefs[idx.Backend]
62+
63+
// Only core Service backends.
64+
if br.BackendRef.Group != nil && *br.BackendRef.Group != "" {
65+
continue
66+
}
67+
if br.BackendRef.Kind != nil && *br.BackendRef.Kind != "Service" {
68+
continue
69+
}
70+
if br.BackendRef.Name == "" {
71+
continue
72+
}
73+
74+
svcName := string(br.BackendRef.Name)
75+
if svcName == "" {
76+
continue
77+
}
78+
79+
backendKey := backendKeyForService(httpRouteKey.Namespace, svcName)
80+
81+
be, ok := pol.Backends[backendKey]
82+
if !ok {
83+
// Provider did not produce metadata for this Service backend.
84+
continue
85+
}
86+
87+
// Ensure / create the typed Backend CR (host, port, protocol).
88+
kb := ensureStaticBackendForService(
89+
ingressName,
90+
httpRouteKey,
91+
svcName,
92+
be.Host,
93+
be.Port,
94+
be.Protocol,
95+
backends,
96+
)
97+
98+
// Rewrite BackendRef to point to this Backend.
99+
group := gwv1.Group(BackendGVK.Group)
100+
kind := gwv1.Kind(BackendGVK.Kind)
101+
102+
br.BackendRef.Group = &group
103+
br.BackendRef.Kind = &kind
104+
br.BackendRef.Name = gwv1.ObjectName(kb.Name)
105+
// Backend controls the port.
106+
br.BackendRef.Port = nil
107+
}
108+
}
Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
/*
2+
Copyright 2023 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package kgateway
18+
19+
import (
20+
"fmt"
21+
22+
"github.com/kgateway-dev/ingress2gateway/pkg/i2gw/intermediate"
23+
"github.com/kgateway-dev/ingress2gateway/pkg/i2gw/notifications"
24+
"k8s.io/apimachinery/pkg/types"
25+
)
26+
27+
// backendProtoPatchKey de-dupes notifications across rules/routes/policies.
28+
type backendProtoPatchKey struct {
29+
Namespace string
30+
Service string
31+
Port int32
32+
AppProtocol string
33+
}
34+
35+
// emitBackendProtocolPatchNotifications emits an INFO notification with the correct kubectl patch
36+
// command to set ServicePort.appProtocol on an existing Service.
37+
//
38+
// IMPORTANT:
39+
// - We intentionally do NOT emit a Service object to avoid overwriting user-managed Service config.
40+
// - We also skip backends that have been rewritten to a kgateway Backend (service-upstream case),
41+
// because the generated Backend will carry appProtocol instead.
42+
func emitBackendProtocolPatchNotifications(
43+
pol intermediate.Policy,
44+
sourceIngressName string,
45+
httpRouteKey types.NamespacedName,
46+
httpCtx intermediate.HTTPRouteContext,
47+
seen map[backendProtoPatchKey]struct{},
48+
) {
49+
if pol.BackendProtocol == nil {
50+
return
51+
}
52+
53+
// Map ingress-nginx backend-protocol → ServicePort.appProtocol
54+
var appProto string
55+
switch *pol.BackendProtocol {
56+
case intermediate.BackendProtocolGRPC:
57+
appProto = "grpc"
58+
default:
59+
// Nothing to do for unsupported/unknown mappings.
60+
return
61+
}
62+
63+
for _, idx := range pol.RuleBackendSources {
64+
if idx.Rule >= len(httpCtx.Spec.Rules) {
65+
continue
66+
}
67+
rule := httpCtx.Spec.Rules[idx.Rule]
68+
if idx.Backend >= len(rule.BackendRefs) {
69+
continue
70+
}
71+
72+
br := rule.BackendRefs[idx.Backend]
73+
74+
// If already rewritten to a kgateway Backend, skip; appProtocol will be applied there.
75+
if br.BackendRef.Group != nil && *br.BackendRef.Group != "" {
76+
continue
77+
}
78+
if br.BackendRef.Kind != nil && *br.BackendRef.Kind != "" && *br.BackendRef.Kind != "Service" {
79+
continue
80+
}
81+
if br.BackendRef.Name == "" || br.BackendRef.Port == nil {
82+
continue
83+
}
84+
85+
svcName := string(br.BackendRef.Name)
86+
// If service-upstream is enabled for this backend, the emitter will generate a
87+
// kgateway Backend with appProtocol, so do not suggest patching the Service.
88+
if len(pol.Backends) > 0 {
89+
if _, ok := pol.Backends[backendKeyForService(httpRouteKey.Namespace, svcName)]; ok {
90+
continue
91+
}
92+
}
93+
94+
port := int32(*br.BackendRef.Port)
95+
96+
key := backendProtoPatchKey{
97+
Namespace: httpRouteKey.Namespace,
98+
Service: svcName,
99+
Port: port,
100+
AppProtocol: appProto,
101+
}
102+
if _, ok := seen[key]; ok {
103+
continue
104+
}
105+
seen[key] = struct{}{}
106+
107+
// Use strategic merge patch (safe for core types) so only the matching port entry is updated.
108+
// This is far safer than emitting a Service manifest that users might blindly apply.
109+
patch := fmt.Sprintf(`{"spec":{"ports":[{"port":%d,"appProtocol":"%s"}]}}`, port, appProto)
110+
cmd := fmt.Sprintf(
111+
"kubectl patch service %s -n %s --type=strategic -p '%s'",
112+
svcName,
113+
httpRouteKey.Namespace,
114+
patch,
115+
)
116+
117+
msg := fmt.Sprintf(
118+
`Ingress %q uses nginx.ingress.kubernetes.io/backend-protocol=%q for Service %s/%s port %d.
119+
120+
To avoid overwriting existing Service configuration, ingress2gateway does not emit a Service for this annotation.
121+
Apply the equivalent behavior by patching your existing Service port's appProtocol:
122+
123+
%s`,
124+
sourceIngressName,
125+
*pol.BackendProtocol,
126+
httpRouteKey.Namespace,
127+
svcName,
128+
port,
129+
cmd,
130+
)
131+
132+
notifications.NotificationAggr.DispatchNotification(
133+
notifications.NewNotification(notifications.InfoNotification, msg),
134+
"ingress-nginx",
135+
)
136+
}
137+
}

0 commit comments

Comments
 (0)