diff --git a/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/ConfigSourceLifecycleObserverTest.java b/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/ConfigSourceLifecycleObserverTest.java index ff09ce9b86e..b32dc5d4359 100644 --- a/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/ConfigSourceLifecycleObserverTest.java +++ b/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/ConfigSourceLifecycleObserverTest.java @@ -88,6 +88,34 @@ protected void configure(ServerBuilder sb) { ads: {} cds_config: ads: {} + static_resources: + clusters: + - name: bootstrap-cluster + type: STATIC + load_assignment: + cluster_name: bootstrap-cluster + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: %s + """; + + //language=YAML + private static final String deltaBootstrapYaml = + """ + dynamic_resources: + ads_config: + api_type: DELTA_GRPC + grpc_services: + - envoy_grpc: + cluster_name: bootstrap-cluster + lds_config: + ads: {} + cds_config: + ads: {} static_resources: clusters: - name: bootstrap-cluster @@ -190,25 +218,93 @@ void basicAdsStream() throws Exception { final var metrics = measureAll(meterRegistry); // Basic stream metrics for bootstrap cluster ADS stream - assertThat(metrics).containsKey("armeria.xds.configsource.stream.active#" + - "value{name=bootstrap-cluster,type=ads,xdsType=ads}"); - assertThat(ensureExistsAndGet(metrics, "armeria.xds.configsource.stream.opened#" + - "count{name=bootstrap-cluster,type=ads,xdsType=ads}")) + final String grpcAds = "apiType=grpc,name=bootstrap-cluster,type=ads,xdsType=ads"; + assertThat(metrics).containsKey( + "armeria.xds.configsource.stream.active#value{" + grpcAds + '}'); + assertThat(ensureExistsAndGet( + metrics, "armeria.xds.configsource.stream.opened#count{" + grpcAds + '}')) + .isGreaterThan(0.0); + assertThat(ensureExistsAndGet( + metrics, "armeria.xds.configsource.stream.request#count{" + grpcAds + '}')) + .isGreaterThan(0.0); + assertThat(ensureExistsAndGet( + metrics, "armeria.xds.configsource.stream.response#count{" + grpcAds + '}')) + .isGreaterThan(0.0); + assertThat(ensureExistsAndGet( + metrics, "armeria.xds.configsource.resource.parse.success#count{" + grpcAds + '}')) + .isGreaterThan(0); + + // No parse rejections should occur for valid resources + final Double parseRejectedCount = + metrics.getOrDefault( + "armeria.xds.configsource.resource.parse.rejected#count{" + grpcAds + '}', + 0.0); + assertThat(parseRejectedCount).isEqualTo(0.0); + }); + + listenerRoot.close(); + } + + // Verify configsource metrics are cleaned up after XdsBootstrap closure + await().untilAsserted(() -> { + final var metrics = measureAll(meterRegistry); + assertThat(metrics).isEmpty(); + }); + } + + @Test + void basicAdsStreamDelta() throws Exception { + final MeterRegistry meterRegistry = new SimpleMeterRegistry(); + final Bootstrap bootstrap = + XdsResourceReader.fromYaml(deltaBootstrapYaml.formatted(server.httpPort()), + Bootstrap.class); + + try (XdsBootstrap xdsBootstrap = XdsBootstrap.builder(bootstrap) + .meterRegistry(meterRegistry) + .build()) { + + // Set up resources to trigger ADS stream activity + final Listener listener = XdsResourceReader.fromYaml(listenerYaml, Listener.class); + final RouteConfiguration route = XdsResourceReader.fromYaml(routeYaml, RouteConfiguration.class); + final Cluster cluster = XdsResourceReader.fromYaml(clusterYaml, Cluster.class); + final ClusterLoadAssignment loadAssignment = + XdsResourceReader.fromYaml(endpointYaml, ClusterLoadAssignment.class); + + version.incrementAndGet(); + cache.setSnapshot(GROUP, Snapshot.create(ImmutableList.of(cluster), + ImmutableList.of(loadAssignment), + ImmutableList.of(listener), + ImmutableList.of(route), + ImmutableList.of(), version.toString())); + + final ListenerRoot listenerRoot = xdsBootstrap.listenerRoot("my-listener"); + + // Wait for ADS stream to fetch resources and verify metrics + await().untilAsserted(() -> { + final var metrics = measureAll(meterRegistry); + + // Basic stream metrics for bootstrap cluster ADS stream + final String deltaGrpcAds = "apiType=delta_grpc,name=bootstrap-cluster,type=ads,xdsType=ads"; + assertThat(metrics).containsKey( + "armeria.xds.configsource.stream.active#value{" + deltaGrpcAds + '}'); + assertThat(ensureExistsAndGet( + metrics, "armeria.xds.configsource.stream.opened#count{" + deltaGrpcAds + '}')) .isGreaterThan(0.0); - assertThat(ensureExistsAndGet(metrics, "armeria.xds.configsource.stream.request#" + - "count{name=bootstrap-cluster,type=ads,xdsType=ads}")) + assertThat(ensureExistsAndGet( + metrics, "armeria.xds.configsource.stream.request#count{" + deltaGrpcAds + '}')) .isGreaterThan(0.0); - assertThat(ensureExistsAndGet(metrics, "armeria.xds.configsource.stream.response#" + - "count{name=bootstrap-cluster,type=ads,xdsType=ads}")) + assertThat(ensureExistsAndGet( + metrics, "armeria.xds.configsource.stream.response#count{" + deltaGrpcAds + '}')) .isGreaterThan(0.0); - assertThat(ensureExistsAndGet(metrics, "armeria.xds.configsource.resource.parse.success#" + - "count{name=bootstrap-cluster,type=ads,xdsType=ads}")) + assertThat(ensureExistsAndGet( + metrics, "armeria.xds.configsource.resource.parse.success#count{" + deltaGrpcAds + '}')) .isGreaterThan(0); // No parse rejections should occur for valid resources final Double parseRejectedCount = - metrics.getOrDefault("armeria.xds.configsource.resource.parse.rejected#" + - "count{name=bootstrap-cluster,type=ads,xdsType=ads}", 0.0); + metrics.getOrDefault( + "armeria.xds.configsource.resource.parse.rejected#count{" + deltaGrpcAds + '}', + 0.0); assertThat(parseRejectedCount).isEqualTo(0.0); }); @@ -357,27 +453,27 @@ void separateConfigSources() throws Exception { // Verify stream is active and opened assertThat(metrics).containsKey(String.format( "armeria.xds.configsource.stream.active#value" + - "{name=bootstrap-cluster,type=api_config_source,xdsType=%s}", + "{apiType=grpc,name=bootstrap-cluster,type=api_config_source,xdsType=%s}", xdsType)); assertThat(ensureExistsAndGet(metrics, String.format( "armeria.xds.configsource.stream.opened#count" + - "{name=bootstrap-cluster,type=api_config_source,xdsType=%s}", + "{apiType=grpc,name=bootstrap-cluster,type=api_config_source,xdsType=%s}", xdsType))).isGreaterThan(0.0); // Verify request/response activity assertThat(ensureExistsAndGet(metrics, String.format( "armeria.xds.configsource.stream.request#count" + - "{name=bootstrap-cluster,type=api_config_source,xdsType=%s}", + "{apiType=grpc,name=bootstrap-cluster,type=api_config_source,xdsType=%s}", xdsType))).isGreaterThan(0.0); assertThat(ensureExistsAndGet(metrics, String.format( "armeria.xds.configsource.stream.response#count" + - "{name=bootstrap-cluster,type=api_config_source,xdsType=%s}", + "{apiType=grpc,name=bootstrap-cluster,type=api_config_source,xdsType=%s}", xdsType))).isGreaterThan(0.0); // Verify successful resource parsing assertThat(ensureExistsAndGet(metrics, String.format( "armeria.xds.configsource.resource.parse.success#count" + - "{name=bootstrap-cluster,type=api_config_source,xdsType=%s}", + "{apiType=grpc,name=bootstrap-cluster,type=api_config_source,xdsType=%s}", xdsType))).isGreaterThan(0.0); } }); @@ -506,7 +602,7 @@ void resourceRejection(XdsType xdsType, Snapshot snapshot) throws Exception { // Verify that the malformed resource type has rejection count > 0 final String rejectionMetricKey = String.format( "armeria.xds.configsource.resource.parse.rejected#count" + - "{name=bootstrap-cluster,type=api_config_source,xdsType=%s}", + "{apiType=grpc,name=bootstrap-cluster,type=api_config_source,xdsType=%s}", metricXdsType); assertThat(ensureExistsAndGet(metrics, rejectionMetricKey)).isGreaterThan(0.0); }); @@ -521,6 +617,55 @@ void resourceRejection(XdsType xdsType, Snapshot snapshot) throws Exception { }); } + @Test + void resourceRejectionDelta() throws Exception { + final MeterRegistry meterRegistry = new SimpleMeterRegistry(); + final Bootstrap bootstrap = + XdsResourceReader.fromYaml(deltaBootstrapYaml.formatted(server.httpPort()), + Bootstrap.class); + + try (XdsBootstrap xdsBootstrap = XdsBootstrap.builder(bootstrap) + .meterRegistry(meterRegistry) + .build()) { + + final Listener validListener = XdsResourceReader.fromYaml(listenerYaml, Listener.class); + final RouteConfiguration validRoute = + XdsResourceReader.fromYaml(routeYaml, RouteConfiguration.class); + final Cluster validCluster = XdsResourceReader.fromYaml(clusterYaml, Cluster.class); + final ClusterLoadAssignment validEndpoint = + XdsResourceReader.fromYaml(endpointYaml, ClusterLoadAssignment.class); + final Listener malformedListener = + XdsResourceReader.fromYaml(malformedListenerYaml, Listener.class); + + version.incrementAndGet(); + cache.setSnapshot(GROUP, Snapshot.create(ImmutableList.of(validCluster), + ImmutableList.of(validEndpoint), + ImmutableList.of(malformedListener), + ImmutableList.of(validRoute), + ImmutableList.of(), version.toString())); + + final ListenerRoot listenerRoot = xdsBootstrap.listenerRoot("my-listener"); + + // Wait for the malformed resource to be rejected and verify rejection metrics + await().untilAsserted(() -> { + final var metrics = measureAll(meterRegistry); + + final String rejectionMetricKey = + "armeria.xds.configsource.resource.parse.rejected#count" + + "{apiType=delta_grpc,name=bootstrap-cluster,type=ads,xdsType=ads}"; + assertThat(ensureExistsAndGet(metrics, rejectionMetricKey)).isGreaterThan(0.0); + }); + + listenerRoot.close(); + } + + // Verify configsource metrics are cleaned up after XdsBootstrap closure + await().untilAsserted(() -> { + final var metrics = measureAll(meterRegistry); + assertThat(metrics).isEmpty(); + }); + } + private static Double ensureExistsAndGet(Map meters, String key) { assertThat(meters).containsKey(key); return meters.get(key); diff --git a/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/DeltaXdsPreprocessorTest.java b/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/DeltaXdsPreprocessorTest.java new file mode 100644 index 00000000000..9eecdf930a5 --- /dev/null +++ b/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/DeltaXdsPreprocessorTest.java @@ -0,0 +1,274 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds.it; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.google.common.collect.ImmutableList; + +import com.linecorp.armeria.client.BlockingWebClient; +import com.linecorp.armeria.client.WebClient; +import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.testing.junit5.common.EventLoopExtension; +import com.linecorp.armeria.testing.junit5.server.ServerExtension; +import com.linecorp.armeria.xds.XdsBootstrap; +import com.linecorp.armeria.xds.client.endpoint.XdsHttpPreprocessor; + +import io.envoyproxy.controlplane.cache.v3.SimpleCache; +import io.envoyproxy.controlplane.cache.v3.Snapshot; +import io.envoyproxy.controlplane.server.V3DiscoveryServer; +import io.envoyproxy.envoy.config.bootstrap.v3.Bootstrap; +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; + +class DeltaXdsPreprocessorTest { + + private static final String GROUP = "key"; + private static final SimpleCache cache = new SimpleCache<>(node -> GROUP); + private static final String CLUSTER_NAME = "cluster1"; + private static final String LISTENER_NAME = "listener1"; + private static final String ROUTE_NAME = "route1"; + private static final String BOOTSTRAP_CLUSTER_NAME = "bootstrap-cluster"; + + @RegisterExtension + @Order(0) + static final ServerExtension server = new ServerExtension() { + @Override + protected void configure(ServerBuilder sb) { + final V3DiscoveryServer v3DiscoveryServer = new V3DiscoveryServer(cache); + sb.service(GrpcService.builder() + .addService(v3DiscoveryServer.getAggregatedDiscoveryServiceImpl()) + .build()); + sb.http(0); + } + }; + + @RegisterExtension + @Order(1) + static final ServerExtension helloServer = new ServerExtension() { + @Override + protected void configure(ServerBuilder sb) { + sb.service("/hello", (ctx, req) -> HttpResponse.of("world")); + sb.http(0); + } + }; + + @RegisterExtension + @Order(2) + static final ServerExtension helloServer2 = new ServerExtension() { + @Override + protected void configure(ServerBuilder sb) { + sb.service("/hello", (ctx, req) -> HttpResponse.of("world")); + sb.http(0); + } + }; + + @RegisterExtension + @Order(3) + static EventLoopExtension eventLoop = new EventLoopExtension(); + + @BeforeEach + void beforeEach() { + final Cluster cluster = clusterYaml(CLUSTER_NAME); + final ClusterLoadAssignment assignment = + endpointYaml(CLUSTER_NAME, + helloServer.httpSocketAddress().getHostString(), + helloServer.httpPort()); + final Listener listener = listenerYaml(LISTENER_NAME, ROUTE_NAME); + final RouteConfiguration route = routeYaml(ROUTE_NAME, CLUSTER_NAME); + cache.setSnapshot( + GROUP, + Snapshot.create( + ImmutableList.of(cluster), + ImmutableList.of(assignment), + ImmutableList.of(listener), + ImmutableList.of(route), + ImmutableList.of(), + "1")); + } + + @Test + void testWithListener() { + final Bootstrap bootstrap = + bootstrapYaml(BOOTSTRAP_CLUSTER_NAME, + server.httpSocketAddress().getHostString(), + server.httpPort()); + try (XdsBootstrap xdsBootstrap = XdsBootstrap.of(bootstrap, eventLoop.get()); + XdsHttpPreprocessor xdsPreprocessor = + XdsHttpPreprocessor.ofListener(LISTENER_NAME, xdsBootstrap)) { + final BlockingWebClient blockingClient = WebClient.of(xdsPreprocessor).blocking(); + assertThat(blockingClient.get("/hello").contentUtf8()).isEqualTo("world"); + } + } + + @Test + void resourceUpdate() { + final Bootstrap bootstrap = + bootstrapYaml(BOOTSTRAP_CLUSTER_NAME, + server.httpSocketAddress().getHostString(), + server.httpPort()); + try (XdsBootstrap xdsBootstrap = XdsBootstrap.of(bootstrap, eventLoop.get()); + XdsHttpPreprocessor xdsPreprocessor = + XdsHttpPreprocessor.ofListener(LISTENER_NAME, xdsBootstrap)) { + final BlockingQueue ports = new LinkedBlockingQueue<>(); + final BlockingWebClient blockingClient = + WebClient.builder(xdsPreprocessor) + .decorator((delegate, ctx, req) -> { + if (ctx.endpoint() != null) { + ports.add(ctx.endpoint().port()); + } + return delegate.execute(ctx, req); + }) + .build() + .blocking(); + + // Confirm initial endpoint works + assertThat(blockingClient.get("/hello").contentUtf8()).isEqualTo("world"); + + // Update snapshot to point to helloServer2 + cache.setSnapshot( + GROUP, + Snapshot.create( + ImmutableList.of(clusterYaml(CLUSTER_NAME)), + ImmutableList.of(endpointYaml(CLUSTER_NAME, + helloServer2.httpSocketAddress().getHostString(), + helloServer2.httpPort())), + ImmutableList.of(listenerYaml(LISTENER_NAME, ROUTE_NAME)), + ImmutableList.of(routeYaml(ROUTE_NAME, CLUSTER_NAME)), + ImmutableList.of(), + "2")); + + // Wait until requests reach helloServer2 + final int newPort = helloServer2.httpPort(); + await().untilAsserted(() -> { + assertThat(blockingClient.get("/hello").contentUtf8()).isEqualTo("world"); + assertThat(ports).contains(newPort); + }); + } + } + + private static Cluster clusterYaml(String name) { + //language=YAML + final String yaml = """ + name: %s + type: EDS + connect_timeout: 1s + eds_cluster_config: + eds_config: + ads: {} + """.formatted(name); + return XdsResourceReader.fromYaml(yaml, Cluster.class); + } + + private static ClusterLoadAssignment endpointYaml(String clusterName, String address, int port) { + //language=YAML + final String yaml = + """ + cluster_name: %s + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: %s + port_value: %s + """.formatted(clusterName, address, port); + return XdsResourceReader.fromYaml(yaml, ClusterLoadAssignment.class); + } + + private static Listener listenerYaml(String name, String routeName) { + //language=YAML + final String yaml = + """ + name: %s + api_listener: + api_listener: + "@type": type.googleapis.com/envoy.extensions.filters.network\ + .http_connection_manager.v3.HttpConnectionManager + stat_prefix: http + rds: + route_config_name: %s + config_source: + ads: {} + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + """.formatted(name, routeName); + return XdsResourceReader.fromYaml(yaml, Listener.class); + } + + private static RouteConfiguration routeYaml(String name, String clusterName) { + //language=YAML + final String yaml = + """ + name: %s + virtual_hosts: + - name: local_service1 + domains: [ "*" ] + routes: + - match: + prefix: / + route: + cluster: %s + """.formatted(name, clusterName); + return XdsResourceReader.fromYaml(yaml, RouteConfiguration.class); + } + + private static Bootstrap bootstrapYaml(String clusterName, String address, int port) { + //language=YAML + final String yaml = """ + dynamic_resources: + ads_config: + api_type: AGGREGATED_DELTA_GRPC + grpc_services: + - envoy_grpc: + cluster_name: %s + cds_config: + ads: {} + lds_config: + ads: {} + static_resources: + clusters: + - name: %s + type: STATIC + load_assignment: + cluster_name: %s + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: %s + port_value: %s + """.formatted(clusterName, clusterName, clusterName, address, port); + return XdsResourceReader.fromYaml(yaml, Bootstrap.class); + } +} diff --git a/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/DeltaXdsResourceWatcherTest.java b/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/DeltaXdsResourceWatcherTest.java new file mode 100644 index 00000000000..ad0e7114490 --- /dev/null +++ b/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/DeltaXdsResourceWatcherTest.java @@ -0,0 +1,405 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds.it; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.google.common.collect.ImmutableList; + +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.testing.junit5.common.EventLoopExtension; +import com.linecorp.armeria.testing.junit5.server.ServerExtension; +import com.linecorp.armeria.xds.ClusterRoot; +import com.linecorp.armeria.xds.ClusterSnapshot; +import com.linecorp.armeria.xds.ListenerRoot; +import com.linecorp.armeria.xds.ListenerSnapshot; +import com.linecorp.armeria.xds.MissingXdsResourceException; +import com.linecorp.armeria.xds.XdsBootstrap; +import com.linecorp.armeria.xds.XdsType; + +import io.envoyproxy.controlplane.cache.v3.SimpleCache; +import io.envoyproxy.controlplane.cache.v3.Snapshot; +import io.envoyproxy.controlplane.server.V3DiscoveryServer; +import io.envoyproxy.envoy.config.bootstrap.v3.Bootstrap; +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; + +class DeltaXdsResourceWatcherTest { + + private static final String GROUP = "key"; + private static final String CLUSTER_NAME = "cluster1"; + private static final String CLUSTER_NAME_2 = "cluster2"; + private static final String LISTENER_NAME = "listener1"; + private static final String ROUTE_NAME = "route1"; + private static final String BOOTSTRAP_CLUSTER_NAME = "bootstrap-cluster"; + + private static final SimpleCache cache = new SimpleCache<>(node -> GROUP); + + @RegisterExtension + @Order(0) + static final ServerExtension server = new ServerExtension() { + @Override + protected void configure(ServerBuilder sb) { + final V3DiscoveryServer v3DiscoveryServer = new V3DiscoveryServer(cache); + sb.service(GrpcService.builder() + .addService(v3DiscoveryServer.getAggregatedDiscoveryServiceImpl()) + .build()); + sb.http(0); + } + + @Override + protected boolean runForEachTest() { + return true; + } + }; + + @RegisterExtension + @Order(1) + static final EventLoopExtension eventLoop = new EventLoopExtension(); + + @Test + void listenerAndRouteLifecycle() { + // Seed an empty snapshot so the control plane accepts delta subscriptions immediately. + cache.setSnapshot(GROUP, Snapshot.create( + ImmutableList.of(), ImmutableList.of(), + ImmutableList.of(), ImmutableList.of(), + ImmutableList.of(), "0")); + + final Bootstrap bootstrap = + bootstrapYaml(BOOTSTRAP_CLUSTER_NAME, + server.httpSocketAddress().getHostString(), + server.httpPort()); + try (XdsBootstrap xdsBootstrap = XdsBootstrap.of(bootstrap, eventLoop.get()); + ListenerRoot listenerRoot = xdsBootstrap.listenerRoot(LISTENER_NAME)) { + + final List snapshots = new CopyOnWriteArrayList<>(); + final List errors = new CopyOnWriteArrayList<>(); + + listenerRoot.addSnapshotWatcher((snapshot, t) -> { + if (snapshot != null) { + snapshots.add(snapshot); + } + if (t != null) { + errors.add(t); + } + }); + + // Step 1: Add listener1 → route1 → cluster1 (STATIC cluster with inline endpoints) + cache.setSnapshot(GROUP, Snapshot.create( + ImmutableList.of(staticClusterYaml(CLUSTER_NAME)), + ImmutableList.of(), + ImmutableList.of(listenerYaml(LISTENER_NAME, ROUTE_NAME)), + ImmutableList.of(routeYaml(ROUTE_NAME, CLUSTER_NAME)), + ImmutableList.of(), + "1")); + await().untilAsserted(() -> { + assertThat(snapshots).hasSize(1); + final ListenerSnapshot s = snapshots.get(0); + assertThat(s.xdsResource().resource().getName()).isEqualTo(LISTENER_NAME); + final RouteConfiguration routeConfig = s.routeSnapshot().xdsResource().resource(); + assertThat(routeConfig.getName()).isEqualTo(ROUTE_NAME); + assertThat(routeConfig.getVirtualHosts(0).getRoutes(0).getRoute().getCluster()) + .isEqualTo(CLUSTER_NAME); + }); + + // Step 2: Update route to reference cluster2 + cache.setSnapshot(GROUP, Snapshot.create( + ImmutableList.of(staticClusterYaml(CLUSTER_NAME_2)), + ImmutableList.of(), + ImmutableList.of(listenerYaml(LISTENER_NAME, ROUTE_NAME)), + ImmutableList.of(routeYaml(ROUTE_NAME, CLUSTER_NAME_2)), + ImmutableList.of(), + "2")); + await().untilAsserted(() -> { + assertThat(snapshots).hasSize(2); + final RouteConfiguration routeConfig = + snapshots.get(1).routeSnapshot().xdsResource().resource(); + assertThat(routeConfig.getVirtualHosts(0).getRoutes(0).getRoute().getCluster()) + .isEqualTo(CLUSTER_NAME_2); + }); + + // Step 3: Delete listener — errors queue gets a throwable; no new snapshot + cache.setSnapshot(GROUP, Snapshot.create( + ImmutableList.of(staticClusterYaml(CLUSTER_NAME)), + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(routeYaml(ROUTE_NAME, CLUSTER_NAME)), + ImmutableList.of(), + "3")); + await().untilAsserted(() -> assertThat(errors).anyMatch(error -> { + if (!(error instanceof MissingXdsResourceException)) { + return false; + } + final MissingXdsResourceException exception = (MissingXdsResourceException) error; + return exception.type() == XdsType.LISTENER && + exception.name().equals(LISTENER_NAME); + })); + final int sizeAfterRemoval = snapshots.size(); + + // Step 4: Re-add listener → route1 → cluster1 + cache.setSnapshot(GROUP, Snapshot.create( + ImmutableList.of(staticClusterYaml(CLUSTER_NAME)), + ImmutableList.of(), + ImmutableList.of(listenerYaml(LISTENER_NAME, ROUTE_NAME)), + ImmutableList.of(routeYaml(ROUTE_NAME, CLUSTER_NAME)), + ImmutableList.of(), + "4")); + await().untilAsserted(() -> { + assertThat(snapshots.size()).isGreaterThan(sizeAfterRemoval); + final ListenerSnapshot s = snapshots.get(snapshots.size() - 1); + assertThat(s.xdsResource().resource().getName()).isEqualTo(LISTENER_NAME); + assertThat(s.routeSnapshot().xdsResource().resource() + .getVirtualHosts(0).getRoutes(0).getRoute().getCluster()) + .isEqualTo(CLUSTER_NAME); + }); + } + } + + @Test + void clusterAndEndpointLifecycle() { + // Seed an empty snapshot so the control plane accepts delta subscriptions immediately. + cache.setSnapshot(GROUP, Snapshot.create( + ImmutableList.of(), ImmutableList.of(), + ImmutableList.of(), ImmutableList.of(), + ImmutableList.of(), "0")); + + final Bootstrap bootstrap = + bootstrapYaml(BOOTSTRAP_CLUSTER_NAME, + server.httpSocketAddress().getHostString(), + server.httpPort()); + try (XdsBootstrap xdsBootstrap = XdsBootstrap.of(bootstrap, eventLoop.get()); + ClusterRoot clusterRoot = xdsBootstrap.clusterRoot(CLUSTER_NAME)) { + + final List snapshots = new CopyOnWriteArrayList<>(); + final List errors = new CopyOnWriteArrayList<>(); + + clusterRoot.addSnapshotWatcher((snapshot, t) -> { + if (snapshot != null) { + snapshots.add(snapshot); + } + if (t != null) { + errors.add(t); + } + }); + + // Step 1: Add cluster1 + endpoint at port 1234 + cache.setSnapshot(GROUP, Snapshot.create( + ImmutableList.of(clusterYaml(CLUSTER_NAME)), + ImmutableList.of(endpointYaml(CLUSTER_NAME, "127.0.0.1", 1234)), + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(), + "1")); + await().untilAsserted(() -> { + assertThat(snapshots).hasSize(1); + final ClusterLoadAssignment cla = + snapshots.get(0).endpointSnapshot().xdsResource().resource(); + assertThat(cla.getEndpoints(0).getLbEndpoints(0) + .getEndpoint().getAddress().getSocketAddress().getPortValue()) + .isEqualTo(1234); + }); + + // Step 2: Update endpoint port to 5678 + cache.setSnapshot(GROUP, Snapshot.create( + ImmutableList.of(clusterYaml(CLUSTER_NAME)), + ImmutableList.of(endpointYaml(CLUSTER_NAME, "127.0.0.1", 5678)), + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(), + "2")); + await().untilAsserted(() -> { + assertThat(snapshots).hasSize(2); + final ClusterLoadAssignment cla = + snapshots.get(1).endpointSnapshot().xdsResource().resource(); + assertThat(cla.getEndpoints(0).getLbEndpoints(0) + .getEndpoint().getAddress().getSocketAddress().getPortValue()) + .isEqualTo(5678); + }); + + // Step 3: Add a second lb_endpoint (ports 5678, 9090) + cache.setSnapshot(GROUP, Snapshot.create( + ImmutableList.of(clusterYaml(CLUSTER_NAME)), + ImmutableList.of(endpointYamlMulti(CLUSTER_NAME, "127.0.0.1", 5678, 9090)), + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(), + "3")); + await().untilAsserted(() -> { + assertThat(snapshots).hasSize(3); + final ClusterLoadAssignment cla = + snapshots.get(2).endpointSnapshot().xdsResource().resource(); + assertThat(cla.getEndpoints(0).getLbEndpointsList()).hasSize(2); + }); + + // Step 4: Delete cluster — both CLUSTER and ENDPOINT deletions fire, + // so errors gets at least one throwable. + cache.setSnapshot(GROUP, Snapshot.create( + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(), + ImmutableList.of(), + "4")); + await().untilAsserted(() -> assertThat(errors).isNotEmpty()); + } + } + + private static Cluster staticClusterYaml(String name) { + //language=YAML + final String yaml = """ + name: %s + type: STATIC + load_assignment: + cluster_name: %s + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 1 + """.formatted(name, name); + return XdsResourceReader.fromYaml(yaml, Cluster.class); + } + + private static Cluster clusterYaml(String name) { + //language=YAML + final String yaml = """ + name: %s + type: EDS + connect_timeout: 1s + eds_cluster_config: + eds_config: + ads: {} + """.formatted(name); + return XdsResourceReader.fromYaml(yaml, Cluster.class); + } + + private static ClusterLoadAssignment endpointYaml(String clusterName, String address, int port) { + //language=YAML + final String yaml = """ + cluster_name: %s + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: %s + port_value: %s + """.formatted(clusterName, address, port); + return XdsResourceReader.fromYaml(yaml, ClusterLoadAssignment.class); + } + + private static ClusterLoadAssignment endpointYamlMulti(String clusterName, String address, + int port1, int port2) { + //language=YAML + final String yaml = """ + cluster_name: %s + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: %s + port_value: %s + - endpoint: + address: + socket_address: + address: %s + port_value: %s + """.formatted(clusterName, address, port1, address, port2); + return XdsResourceReader.fromYaml(yaml, ClusterLoadAssignment.class); + } + + private static Listener listenerYaml(String name, String routeName) { + //language=YAML + final String yaml = """ + name: %s + api_listener: + api_listener: + "@type": type.googleapis.com/envoy.extensions.filters.network\ + .http_connection_manager.v3.HttpConnectionManager + stat_prefix: http + rds: + route_config_name: %s + config_source: + ads: {} + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + """.formatted(name, routeName); + return XdsResourceReader.fromYaml(yaml, Listener.class); + } + + private static RouteConfiguration routeYaml(String name, String clusterName) { + //language=YAML + final String yaml = """ + name: %s + virtual_hosts: + - name: local_service1 + domains: [ "*" ] + routes: + - match: + prefix: / + route: + cluster: %s + """.formatted(name, clusterName); + return XdsResourceReader.fromYaml(yaml, RouteConfiguration.class); + } + + private static Bootstrap bootstrapYaml(String clusterName, String address, int port) { + //language=YAML + final String yaml = """ + dynamic_resources: + ads_config: + api_type: AGGREGATED_DELTA_GRPC + grpc_services: + - envoy_grpc: + cluster_name: %s + cds_config: + ads: {} + lds_config: + ads: {} + static_resources: + clusters: + - name: %s + type: STATIC + load_assignment: + cluster_name: %s + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: %s + port_value: %s + """.formatted(clusterName, clusterName, clusterName, address, port); + return XdsResourceReader.fromYaml(yaml, Bootstrap.class); + } +} diff --git a/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/XdsControlPlaneErrorHandlingTest.java b/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/XdsControlPlaneErrorHandlingTest.java new file mode 100644 index 00000000000..3f80a3d061b --- /dev/null +++ b/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/XdsControlPlaneErrorHandlingTest.java @@ -0,0 +1,635 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds.it; + +import static io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType.AGGREGATED_DELTA_GRPC; +import static io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType.AGGREGATED_GRPC; +import static io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType.DELTA_GRPC; +import static io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType.GRPC; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Any; + +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.ServiceRequestContext; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.testing.junit5.common.EventLoopExtension; +import com.linecorp.armeria.testing.junit5.server.ServerExtension; +import com.linecorp.armeria.testing.server.ServiceRequestContextCaptor; +import com.linecorp.armeria.xds.ClusterSnapshot; +import com.linecorp.armeria.xds.EndpointSnapshot; +import com.linecorp.armeria.xds.ListenerRoot; +import com.linecorp.armeria.xds.ListenerSnapshot; +import com.linecorp.armeria.xds.RouteEntry; +import com.linecorp.armeria.xds.RouteSnapshot; +import com.linecorp.armeria.xds.SnapshotWatcher; +import com.linecorp.armeria.xds.XdsBootstrap; +import com.linecorp.armeria.xds.XdsType; + +import io.envoyproxy.controlplane.cache.v3.SimpleCache; +import io.envoyproxy.controlplane.cache.v3.Snapshot; +import io.envoyproxy.controlplane.server.DiscoveryServerCallbacks; +import io.envoyproxy.controlplane.server.V3DiscoveryServer; +import io.envoyproxy.controlplane.server.exception.RequestException; +import io.envoyproxy.envoy.config.bootstrap.v3.Bootstrap; +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; + +class XdsControlPlaneErrorHandlingTest { + + private static final String GROUP = "key"; + private static final String LISTENER_NAME = "listener"; + private static final String ROUTE_NAME = "route"; + private static final String CLUSTER_NAME = "cluster"; + private static final String BOOTSTRAP_CLUSTER_NAME = "bootstrap-cluster"; + private static final int PORT_V1 = 8080; + private static final long TIMEOUT_V1 = 1; + private static final String STAT_PREFIX_V1 = "http1"; + private static final String ROUTE_PREFIX_V1 = "/"; + + private static final int PORT_V2 = 9090; + private static final long TIMEOUT_V2 = 2; + private static final String STAT_PREFIX_V2 = "http2"; + private static final String ROUTE_PREFIX_V2 = "/v2"; + + private static final AtomicLong version = new AtomicLong(); + private static final SimpleCache cache = new SimpleCache<>(node -> GROUP); + private static final NackTracker nackTracker = new NackTracker(); + + private static final List PROTOCOLS = ImmutableList.of( + AGGREGATED_GRPC, GRPC, AGGREGATED_DELTA_GRPC, DELTA_GRPC); + + private static final List TARGETS = ImmutableList.of( + XdsType.LISTENER, XdsType.ROUTE, XdsType.CLUSTER, XdsType.ENDPOINT); + + @RegisterExtension + @Order(0) + static final ServerExtension server = new ServerExtension() { + @Override + protected void configure(ServerBuilder sb) { + final V3DiscoveryServer v3DiscoveryServer = new V3DiscoveryServer(nackTracker, cache); + sb.service(GrpcService.builder() + .addService(v3DiscoveryServer.getAggregatedDiscoveryServiceImpl()) + .addService(v3DiscoveryServer.getListenerDiscoveryServiceImpl()) + .addService(v3DiscoveryServer.getRouteDiscoveryServiceImpl()) + .addService(v3DiscoveryServer.getClusterDiscoveryServiceImpl()) + .addService(v3DiscoveryServer.getEndpointDiscoveryServiceImpl()) + .build()); + sb.http(0); + } + }; + + @RegisterExtension + @Order(1) + static final EventLoopExtension eventLoop = new EventLoopExtension(); + + @BeforeEach + void setUp() { + nackTracker.reset(); + } + + static Stream nackRecoveryCases() { + return PROTOCOLS.stream().flatMap(proto -> + TARGETS.stream().map(target -> Arguments.of(proto, target))); + } + + @ParameterizedTest + @MethodSource("nackRecoveryCases") + void nackAndRecovery(ApiType apiType, XdsType malformedTarget) throws Exception { + cache.setSnapshot(GROUP, emptySnapshot()); + final Bootstrap bootstrap = XdsResourceReader.fromYaml( + bootstrapYaml(apiType).formatted(server.httpPort()), Bootstrap.class); + + try (XdsBootstrap xdsBootstrap = XdsBootstrap.of(bootstrap, eventLoop.get()); + ListenerRoot listenerRoot = xdsBootstrap.listenerRoot(LISTENER_NAME)) { + final RecordingWatcher watcher = new RecordingWatcher<>(); + listenerRoot.addSnapshotWatcher(watcher); + + // Step 1: send a snapshot with one PGV-invalid resource + final int nacksBefore = nackTracker.nackCount(); + cache.setSnapshot(GROUP, malformedSnapshot(apiType, malformedTarget)); + + // Step 2: verify NACK arrives at the server + // (Armeria applies a 3 s backoff before sending, so give Awaitility enough time) + await().untilAsserted(() -> + assertThat(nackTracker.nackCount()).isGreaterThan(nacksBefore)); + + // Step 3: set valid resources + cache.setSnapshot(GROUP, validSnapshot(apiType)); + + // Step 4: verify the client recovered and delivered the new snapshot + awaitExpectedState(watcher); + } + } + + static Stream protocols() { + return PROTOCOLS.stream().map(Arguments::of); + } + + @ParameterizedTest + @MethodSource("protocols") + void connectionClosureAndRecovery(ApiType apiType) throws Exception { + cache.setSnapshot(GROUP, emptySnapshot()); + final Bootstrap bootstrap = XdsResourceReader.fromYaml( + bootstrapYaml(apiType).formatted(server.httpPort()), Bootstrap.class); + + try (XdsBootstrap xdsBootstrap = XdsBootstrap.of(bootstrap, eventLoop.get()); + ListenerRoot listenerRoot = xdsBootstrap.listenerRoot(LISTENER_NAME)) { + final RecordingWatcher watcher = new RecordingWatcher<>(); + listenerRoot.addSnapshotWatcher(watcher); + + // Step 1: set V1 snapshot and verify it is received + cache.setSnapshot(GROUP, validSnapshot(apiType)); + awaitExpectedState(watcher); + + // Step 2: drain and cancel all open gRPC streams + final ServiceRequestContextCaptor captor = server.requestContextCaptor(); + await().untilAsserted(() -> assertThat(captor.isEmpty()).isFalse()); + for (ServiceRequestContext ctx : captor.all()) { + ctx.cancel(); + } + + // Step 3: push V2 snapshot + cache.setSnapshot(GROUP, validSnapshotV2(apiType)); + + // Step 4: verify the client reconnected and received the V2 snapshot + awaitExpectedState(watcher, STAT_PREFIX_V2, ROUTE_PREFIX_V2, TIMEOUT_V2, PORT_V2); + } + } + + private static Snapshot malformedSnapshot(ApiType apiType, XdsType malformedTarget) { + final Listener validListener = listenerYaml(apiType); + final RouteConfiguration validRoute = routeYaml(); + final Cluster validEdsCluster = clusterYaml(apiType); + final ClusterLoadAssignment validEndpoint = endpointYaml(); + + switch (malformedTarget) { + case LISTENER: { + final Listener malformedListener = XdsResourceReader.fromYaml( + malformedListenerYaml(), Listener.class); + return Snapshot.create( + ImmutableList.of(validEdsCluster), + ImmutableList.of(validEndpoint), + ImmutableList.of(malformedListener), + ImmutableList.of(validRoute), + ImmutableList.of(), + String.valueOf(version.incrementAndGet())); + } + case ROUTE: { + final RouteConfiguration malformedRoute = XdsResourceReader.fromYaml( + malformedRouteYaml(), RouteConfiguration.class); + return Snapshot.create( + ImmutableList.of(validEdsCluster), + ImmutableList.of(validEndpoint), + ImmutableList.of(validListener), + ImmutableList.of(malformedRoute), + ImmutableList.of(), + String.valueOf(version.incrementAndGet())); + } + case CLUSTER: { + final Cluster malformedCluster = XdsResourceReader.fromYaml( + malformedClusterYaml(), Cluster.class); + return Snapshot.create( + ImmutableList.of(malformedCluster), + ImmutableList.of(), + ImmutableList.of(validListener), + ImmutableList.of(validRoute), + ImmutableList.of(), + String.valueOf(version.incrementAndGet())); + } + case ENDPOINT: { + final ClusterLoadAssignment malformedEndpoint = XdsResourceReader.fromYaml( + malformedEndpointYaml(), ClusterLoadAssignment.class); + return Snapshot.create( + ImmutableList.of(validEdsCluster), + ImmutableList.of(malformedEndpoint), + ImmutableList.of(validListener), + ImmutableList.of(validRoute), + ImmutableList.of(), + String.valueOf(version.incrementAndGet())); + } + default: + throw new IllegalArgumentException("Unexpected target: " + malformedTarget); + } + } + + private static Snapshot validSnapshot(ApiType apiType) { + return Snapshot.create( + ImmutableList.of(clusterYaml(apiType)), + ImmutableList.of(endpointYaml()), + ImmutableList.of(listenerYaml(apiType)), + ImmutableList.of(routeYaml()), + ImmutableList.of(), + String.valueOf(version.incrementAndGet())); + } + + private static Snapshot validSnapshotV2(ApiType apiType) { + return Snapshot.create( + ImmutableList.of(XdsResourceReader.fromYaml(clusterYamlString(apiType, TIMEOUT_V2), + Cluster.class)), + ImmutableList.of(endpointYaml(PORT_V2)), + ImmutableList.of(XdsResourceReader.fromYaml(listenerYamlString(apiType, STAT_PREFIX_V2), + Listener.class)), + ImmutableList.of(routeYaml(ROUTE_PREFIX_V2)), + ImmutableList.of(), + String.valueOf(version.incrementAndGet())); + } + + private static void awaitExpectedState(RecordingWatcher listenerWatcher) + throws Exception { + awaitExpectedState(listenerWatcher, STAT_PREFIX_V1, ROUTE_PREFIX_V1, TIMEOUT_V1, PORT_V1); + } + + private static void awaitExpectedState(RecordingWatcher listenerWatcher, + String statPrefix, String routePrefix, + long timeout, int port) throws Exception { + await().untilAsserted(() -> { + final ListenerSnapshot listenerSnapshot = listenerWatcher.lastSnapshot(); + assertThat(listenerSnapshot).isNotNull(); + final Listener listener = listenerSnapshot.xdsResource().resource(); + assertThat(statPrefix(listener)).isEqualTo(statPrefix); + + final RouteSnapshot routeSnapshot = listenerSnapshot.routeSnapshot(); + assertThat(routeSnapshot).isNotNull(); + final RouteConfiguration route = routeSnapshot.xdsResource().resource(); + assertThat(route.getVirtualHosts(0).getRoutes(0).getMatch().getPrefix()) + .isEqualTo(routePrefix); + + final RouteEntry routeEntry = routeSnapshot.virtualHostSnapshots().get(0).routeEntries().get(0); + final ClusterSnapshot clusterSnapshot = routeEntry.clusterSnapshot(); + assertThat(clusterSnapshot).isNotNull(); + final Cluster cluster = clusterSnapshot.xdsResource().resource(); + assertThat(cluster.getConnectTimeout().getSeconds()).isEqualTo(timeout); + + final EndpointSnapshot endpointSnapshot = clusterSnapshot.endpointSnapshot(); + assertThat(endpointSnapshot).isNotNull(); + final ClusterLoadAssignment loadAssignment = endpointSnapshot.xdsResource().resource(); + assertThat(endpointPort(loadAssignment)).isEqualTo(port); + }); + } + + private static boolean isAds(ApiType t) { + return t == AGGREGATED_GRPC || t == AGGREGATED_DELTA_GRPC; + } + + private static String statPrefix(Listener listener) throws Exception { + final Any apiListener = listener.getApiListener().getApiListener(); + final HttpConnectionManager hcm = apiListener.unpack(HttpConnectionManager.class); + return hcm.getStatPrefix(); + } + + private static int endpointPort(ClusterLoadAssignment loadAssignment) { + return loadAssignment.getEndpoints(0) + .getLbEndpoints(0) + .getEndpoint() + .getAddress() + .getSocketAddress() + .getPortValue(); + } + + private static Listener listenerYaml(ApiType apiType) { + return XdsResourceReader.fromYaml(listenerYamlString(apiType), Listener.class); + } + + private static String listenerYamlString(ApiType apiType) { + return listenerYamlString(apiType, STAT_PREFIX_V1); + } + + private static String listenerYamlString(ApiType apiType, String statPrefix) { + if (isAds(apiType)) { + //language=YAML + return """ + name: %s + api_listener: + api_listener: + "@type": type.googleapis.com/envoy.extensions.filters.network\ + .http_connection_manager.v3.HttpConnectionManager + stat_prefix: %s + rds: + route_config_name: %s + config_source: + ads: {} + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + """.formatted(LISTENER_NAME, statPrefix, ROUTE_NAME); + } + //language=YAML + return """ + name: %s + api_listener: + api_listener: + "@type": type.googleapis.com/envoy.extensions.filters.network\ + .http_connection_manager.v3.HttpConnectionManager + stat_prefix: %s + rds: + route_config_name: %s + config_source: + api_config_source: + api_type: %s + grpc_services: + - envoy_grpc: + cluster_name: %s + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + """.formatted(LISTENER_NAME, statPrefix, ROUTE_NAME, + apiType.name(), BOOTSTRAP_CLUSTER_NAME); + } + + private static RouteConfiguration routeYaml() { + return routeYaml(ROUTE_PREFIX_V1); + } + + private static RouteConfiguration routeYaml(String routePrefix) { + //language=YAML + final String yaml = """ + name: %s + virtual_hosts: + - name: local_service1 + domains: [ "*" ] + routes: + - match: + prefix: %s + route: + cluster: %s + """.formatted(ROUTE_NAME, routePrefix, CLUSTER_NAME); + return XdsResourceReader.fromYaml(yaml, RouteConfiguration.class); + } + + private static Cluster clusterYaml(ApiType apiType) { + return XdsResourceReader.fromYaml(clusterYamlString(apiType), Cluster.class); + } + + private static String clusterYamlString(ApiType apiType) { + return clusterYamlString(apiType, TIMEOUT_V1); + } + + private static String clusterYamlString(ApiType apiType, long timeoutSeconds) { + if (isAds(apiType)) { + //language=YAML + return """ + name: %s + type: EDS + connect_timeout: %ss + eds_cluster_config: + eds_config: + ads: {} + """.formatted(CLUSTER_NAME, timeoutSeconds); + } + //language=YAML + return """ + name: %s + type: EDS + connect_timeout: %ss + eds_cluster_config: + eds_config: + api_config_source: + api_type: %s + grpc_services: + - envoy_grpc: + cluster_name: %s + """.formatted(CLUSTER_NAME, timeoutSeconds, apiType.name(), BOOTSTRAP_CLUSTER_NAME); + } + + private static ClusterLoadAssignment endpointYaml() { + return endpointYaml(PORT_V1); + } + + private static ClusterLoadAssignment endpointYaml(int port) { + //language=YAML + final String yaml = """ + cluster_name: %s + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: %s + """.formatted(CLUSTER_NAME, port); + return XdsResourceReader.fromYaml(yaml, ClusterLoadAssignment.class); + } + + private static String malformedListenerYaml() { + // PGV-invalid: HCM Any has correct @type but no stat_prefix or http_filters + //language=YAML + return """ + name: %s + api_listener: + api_listener: + "@type": type.googleapis.com/envoy.extensions.filters.network\ + .http_connection_manager.v3.HttpConnectionManager + """.formatted(LISTENER_NAME); + } + + private static String malformedRouteYaml() { + // PGV-invalid: domains must be non-empty + //language=YAML + return """ + name: %s + virtual_hosts: + - name: local_service1 + domains: [] + """.formatted(ROUTE_NAME); + } + + private static String malformedClusterYaml() { + // PGV-invalid: connect_timeout must be non-negative + //language=YAML + return """ + name: %s + connect_timeout: -1s + """.formatted(CLUSTER_NAME); + } + + private static String malformedEndpointYaml() { + // PGV-invalid: priority must be in range [0, 128) + //language=YAML + return """ + cluster_name: %s + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 1234 + priority: 1234 + """.formatted(CLUSTER_NAME); + } + + private static String bootstrapYaml(ApiType apiType) { + final String dynamicResources; + if (!isAds(apiType)) { + //language=YAML + dynamicResources = """ + lds_config: + api_config_source: + api_type: %s + grpc_services: + - envoy_grpc: + cluster_name: %s + cds_config: + api_config_source: + api_type: %s + grpc_services: + - envoy_grpc: + cluster_name: %s + """.formatted(apiType.name(), BOOTSTRAP_CLUSTER_NAME, + apiType.name(), BOOTSTRAP_CLUSTER_NAME); + } else { + //language=YAML + dynamicResources = """ + ads_config: + api_type: %s + grpc_services: + - envoy_grpc: + cluster_name: %s + lds_config: + ads: {} + cds_config: + ads: {} + """.formatted(apiType.name(), BOOTSTRAP_CLUSTER_NAME); + } + + final StringBuilder staticResources = new StringBuilder(); + staticResources.append(" clusters:\n"); + appendListItem(staticResources, bootstrapClusterYaml(), 2); + + //language=YAML + return """ + dynamic_resources: + %s + static_resources: + %s + """.formatted(dynamicResources.stripTrailing(), + staticResources.toString().stripTrailing()); + } + + private static String bootstrapClusterYaml() { + //language=YAML + return """ + name: %s + type: STATIC + load_assignment: + cluster_name: %s + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: %%s + """.formatted(BOOTSTRAP_CLUSTER_NAME, BOOTSTRAP_CLUSTER_NAME); + } + + private static void appendListItem(StringBuilder sb, String yaml, int indent) { + final String trimmed = yaml.stripTrailing(); + final String[] lines = trimmed.split("\\n"); + if (lines.length == 0) { + return; + } + final String padding = " ".repeat(indent); + sb.append(padding).append("- ").append(lines[0]).append('\n'); + for (int i = 1; i < lines.length; i++) { + if (lines[i].isEmpty()) { + sb.append('\n'); + } else { + sb.append(padding).append(" ").append(lines[i]).append('\n'); + } + } + } + + private static Snapshot emptySnapshot() { + return Snapshot.create(ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), + ImmutableList.of(), ImmutableList.of(), "0"); + } + + private static final class NackTracker implements DiscoveryServerCallbacks { + private final AtomicInteger nackCount = new AtomicInteger(); + + @Override + public void onV3StreamRequest(long streamId, DiscoveryRequest request) throws RequestException { + if (request.hasErrorDetail() && request.getErrorDetail().getCode() != 0) { + nackCount.incrementAndGet(); + } + } + + @Override + public void onV3StreamDeltaRequest(long streamId, + DeltaDiscoveryRequest request) throws RequestException { + if (request.hasErrorDetail() && request.getErrorDetail().getCode() != 0) { + nackCount.incrementAndGet(); + } + } + + int nackCount() { + return nackCount.get(); + } + + void reset() { + nackCount.set(0); + } + } + + private static final class RecordingWatcher implements SnapshotWatcher { + + private final List snapshots = new CopyOnWriteArrayList<>(); + private final List errors = new CopyOnWriteArrayList<>(); + + @Override + public void onUpdate(T snapshot, Throwable t) { + if (snapshot != null) { + snapshots.add(snapshot); + } + if (t != null) { + errors.add(t); + } + } + + T lastSnapshot() { + if (snapshots.isEmpty()) { + return null; + } + return snapshots.get(snapshots.size() - 1); + } + } +} diff --git a/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/XdsControlPlaneMatrixTest.java b/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/XdsControlPlaneMatrixTest.java new file mode 100644 index 00000000000..5de28bdaee2 --- /dev/null +++ b/it/xds-client/src/test/java/com/linecorp/armeria/xds/it/XdsControlPlaneMatrixTest.java @@ -0,0 +1,661 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds.it; + +import static io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType.AGGREGATED_DELTA_GRPC; +import static io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType.AGGREGATED_GRPC; +import static io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType.DELTA_GRPC; +import static io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType.GRPC; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.EnumSet; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Any; + +import com.linecorp.armeria.server.ServerBuilder; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.testing.junit5.common.EventLoopExtension; +import com.linecorp.armeria.testing.junit5.server.ServerExtension; +import com.linecorp.armeria.xds.ClusterSnapshot; +import com.linecorp.armeria.xds.EndpointSnapshot; +import com.linecorp.armeria.xds.ListenerRoot; +import com.linecorp.armeria.xds.ListenerSnapshot; +import com.linecorp.armeria.xds.MissingXdsResourceException; +import com.linecorp.armeria.xds.RouteEntry; +import com.linecorp.armeria.xds.RouteSnapshot; +import com.linecorp.armeria.xds.SnapshotWatcher; +import com.linecorp.armeria.xds.XdsBootstrap; +import com.linecorp.armeria.xds.XdsType; + +import io.envoyproxy.controlplane.cache.v3.SimpleCache; +import io.envoyproxy.controlplane.cache.v3.Snapshot; +import io.envoyproxy.controlplane.server.V3DiscoveryServer; +import io.envoyproxy.envoy.config.bootstrap.v3.Bootstrap; +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType; +import io.envoyproxy.envoy.config.endpoint.v3.ClusterLoadAssignment; +import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; + +class XdsControlPlaneMatrixTest { + + private static final String GROUP = "key"; + private static final String LISTENER_NAME = "listener"; + private static final String ROUTE_NAME = "route"; + private static final String CLUSTER_NAME = "cluster"; + private static final String BOOTSTRAP_CLUSTER_NAME = "bootstrap-cluster"; + private static final int PORT_V1 = 8080; + private static final int PORT_V2 = 9090; + private static final long TIMEOUT_V1 = 1; + private static final long TIMEOUT_V2 = 2; + private static final String STAT_PREFIX_V1 = "http1"; + private static final String STAT_PREFIX_V2 = "http2"; + private static final String ROUTE_PREFIX_V1 = "/"; + private static final String ROUTE_PREFIX_V2 = "/v2"; + + private static final AtomicLong version = new AtomicLong(); + private static final SimpleCache cache = new SimpleCache<>(node -> GROUP); + + private static final List PROTOCOLS = ImmutableList.of( + AGGREGATED_GRPC, GRPC, AGGREGATED_DELTA_GRPC, DELTA_GRPC); + + private static final List TARGETS = ImmutableList.of( + XdsType.LISTENER, XdsType.ROUTE, XdsType.CLUSTER, XdsType.ENDPOINT); + + private static final List SOTW_TYPES = ImmutableList.of(XdsType.LISTENER, XdsType.CLUSTER); + + @RegisterExtension + @Order(0) + static final ServerExtension server = new ServerExtension() { + @Override + protected void configure(ServerBuilder sb) { + final V3DiscoveryServer v3DiscoveryServer = new V3DiscoveryServer(cache); + sb.service(GrpcService.builder() + .addService(v3DiscoveryServer.getAggregatedDiscoveryServiceImpl()) + .addService(v3DiscoveryServer.getListenerDiscoveryServiceImpl()) + .addService(v3DiscoveryServer.getRouteDiscoveryServiceImpl()) + .addService(v3DiscoveryServer.getClusterDiscoveryServiceImpl()) + .addService(v3DiscoveryServer.getEndpointDiscoveryServiceImpl()) + .build()); + sb.http(0); + } + }; + + @RegisterExtension + @Order(1) + static final EventLoopExtension eventLoop = new EventLoopExtension(); + + private static List scenarios() { + final List result = new ArrayList<>(); + for (int mask = 0; mask < (1 << TARGETS.size()); mask++) { + final List statics = new ArrayList<>(); + for (int i = 0; i < TARGETS.size(); i++) { + if ((mask & (1 << i)) != 0) { + statics.add(TARGETS.get(i)); + } + } + result.add(new Scenario(statics.toArray(new XdsType[0]))); + } + return result; + } + + static Stream matrixCases() { + final List arguments = new ArrayList<>(); + for (ApiType apiType : PROTOCOLS) { + for (Scenario scenario : scenarios()) { + for (XdsType target : TARGETS) { + if (scenario.isStatic(target)) { + // can't modify/remove static resources + continue; + } + for (Operation operation : Operation.values()) { + arguments.add(Arguments.of(apiType, scenario, target, operation)); + } + } + } + } + return arguments.stream(); + } + + @ParameterizedTest + @MethodSource("matrixCases") + void controlPlaneMatrix(ApiType apiType, Scenario scenario, XdsType target, Operation operation) + throws Exception { + cache.setSnapshot(GROUP, emptySnapshot()); + final Bootstrap bootstrap = XdsResourceReader.fromYaml( + bootstrapYaml(apiType, scenario).formatted(server.httpPort()), Bootstrap.class); + + try (XdsBootstrap xdsBootstrap = XdsBootstrap.of(bootstrap, eventLoop.get()); + ListenerRoot listenerRoot = xdsBootstrap.listenerRoot(LISTENER_NAME)) { + final RecordingWatcher watcher = new RecordingWatcher<>(); + listenerRoot.addSnapshotWatcher(watcher); + + applyOperation(apiType, scenario, target, operation, watcher); + } + } + + private static void applyOperation(ApiType apiType, Scenario scenario, XdsType target, + Operation operation, + RecordingWatcher listenerWatcher) throws Exception { + final ResourceVariants baseline = ResourceVariants.v1(); + cache.setSnapshot(GROUP, snapshotFor(apiType, scenario, baseline, null)); + awaitExpectedState(baseline, listenerWatcher); + + switch (operation) { + case MODIFY: + cache.setSnapshot(GROUP, snapshotFor(apiType, scenario, + baseline.with(target, Variant.V2), null)); + awaitExpectedState(baseline.with(target, Variant.V2), listenerWatcher); + return; + case DELETE: + if (apiType == AGGREGATED_GRPC) { + // java-control-plane discards watchers for missing resources for ads grpc + return; + } + cache.setSnapshot(GROUP, snapshotFor(apiType, scenario, baseline, target)); + if (isAds(apiType) && SOTW_TYPES.contains(target)) { + // only sotw types report missing resource on deletion + awaitMissing(target, listenerWatcher); + } else if (isDelta(apiType)) { + awaitMissing(target, listenerWatcher); + } + cache.setSnapshot(GROUP, snapshotFor(apiType, scenario, + baseline.with(target, Variant.V2), null)); + awaitExpectedState(baseline.with(target, Variant.V2), listenerWatcher); + } + } + + private static boolean isAds(ApiType t) { + return t == AGGREGATED_GRPC || t == AGGREGATED_DELTA_GRPC; + } + + private static boolean isDelta(ApiType t) { + return t == DELTA_GRPC || t == AGGREGATED_DELTA_GRPC; + } + + private static void awaitExpectedState(ResourceVariants variants, + RecordingWatcher listenerWatcher) + throws Exception { + await().untilAsserted(() -> { + final ListenerSnapshot listenerSnapshot = listenerWatcher.lastSnapshot(); + assertThat(listenerSnapshot).isNotNull(); + final Listener listener = listenerSnapshot.xdsResource().resource(); + assertThat(statPrefix(listener)).isEqualTo(expectedStatPrefix(variants.listener)); + + final RouteSnapshot routeSnapshot = listenerSnapshot.routeSnapshot(); + assertThat(routeSnapshot).isNotNull(); + final RouteConfiguration route = routeSnapshot.xdsResource().resource(); + assertThat(route.getVirtualHosts(0).getRoutes(0).getMatch().getPrefix()) + .isEqualTo(expectedRoutePrefix(variants.route)); + + final RouteEntry routeEntry = routeSnapshot.virtualHostSnapshots().get(0).routeEntries().get(0); + final ClusterSnapshot clusterSnapshot = routeEntry.clusterSnapshot(); + assertThat(clusterSnapshot).isNotNull(); + final Cluster cluster = clusterSnapshot.xdsResource().resource(); + assertThat(cluster.getConnectTimeout().getSeconds()).isEqualTo(expectedTimeout(variants.cluster)); + + final EndpointSnapshot endpointSnapshot = clusterSnapshot.endpointSnapshot(); + assertThat(endpointSnapshot).isNotNull(); + final ClusterLoadAssignment loadAssignment = endpointSnapshot.xdsResource().resource(); + assertThat(endpointPort(loadAssignment)).isEqualTo(expectedEndpointPort(variants.endpoint)); + }); + } + + private static void awaitMissing(XdsType target, RecordingWatcher listenerWatcher) { + final String expectedName = switch (target) { + case LISTENER -> LISTENER_NAME; + case ROUTE -> ROUTE_NAME; + default -> CLUSTER_NAME; // CLUSTER and ENDPOINT share CLUSTER_NAME + }; + await().untilAsserted(() -> assertThat(listenerWatcher.errors()) + .anyMatch(error -> isMissingResource(error, target, expectedName))); + } + + private static boolean isMissingResource(Throwable error, XdsType type, String name) { + if (!(error instanceof MissingXdsResourceException exception)) { + return false; + } + return exception.type() == type && exception.name().equals(name); + } + + private static String expectedStatPrefix(Variant variant) { + return variant == Variant.V1 ? STAT_PREFIX_V1 : STAT_PREFIX_V2; + } + + private static String expectedRoutePrefix(Variant variant) { + return variant == Variant.V1 ? ROUTE_PREFIX_V1 : ROUTE_PREFIX_V2; + } + + private static long expectedTimeout(Variant variant) { + return variant == Variant.V1 ? TIMEOUT_V1 : TIMEOUT_V2; + } + + private static int expectedEndpointPort(Variant variant) { + return variant == Variant.V1 ? PORT_V1 : PORT_V2; + } + + private static int endpointPort(ClusterLoadAssignment loadAssignment) { + return loadAssignment.getEndpoints(0) + .getLbEndpoints(0) + .getEndpoint() + .getAddress() + .getSocketAddress() + .getPortValue(); + } + + private static Snapshot snapshotFor(ApiType apiType, Scenario scenario, ResourceVariants variants, + XdsType removedTarget) { + final List listeners = new ArrayList<>(); + if (!scenario.isStatic(XdsType.LISTENER) && removedTarget != XdsType.LISTENER) { + listeners.add(listenerYaml(apiType, scenario, variants.listener, variants.route)); + } + + final List routes = new ArrayList<>(); + if (!scenario.isStatic(XdsType.ROUTE) && removedTarget != XdsType.ROUTE) { + routes.add(routeYaml(variants.route)); + } + + final List clusters = new ArrayList<>(); + if (!scenario.isStatic(XdsType.CLUSTER) && removedTarget != XdsType.CLUSTER) { + clusters.add(clusterYaml(apiType, scenario, variants.cluster, variants.endpoint)); + } + + final List endpoints = new ArrayList<>(); + if (!scenario.isStatic(XdsType.ENDPOINT) && removedTarget != XdsType.ENDPOINT) { + endpoints.add(endpointYaml(variants.endpoint)); + } + + return Snapshot.create(clusters, endpoints, listeners, routes, ImmutableList.of(), + String.valueOf(version.incrementAndGet())); + } + + private static Snapshot emptySnapshot() { + return Snapshot.create(ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), + ImmutableList.of(), ImmutableList.of(), "0"); + } + + private static String statPrefix(Listener listener) throws Exception { + final Any apiListener = listener.getApiListener().getApiListener(); + final HttpConnectionManager hcm = apiListener.unpack(HttpConnectionManager.class); + return hcm.getStatPrefix(); + } + + private static Listener listenerYaml(ApiType apiType, Scenario scenario, Variant listenerVariant, + Variant routeVariant) { + return XdsResourceReader.fromYaml(listenerYamlString(apiType, scenario, listenerVariant, routeVariant), + Listener.class); + } + + private static String listenerYamlString(ApiType apiType, Scenario scenario, Variant listenerVariant, + Variant routeVariant) { + final String statPrefix = expectedStatPrefix(listenerVariant); + if (scenario.isStatic(XdsType.ROUTE)) { + //language=YAML + return """ + name: %s + api_listener: + api_listener: + "@type": type.googleapis.com/envoy.extensions.filters.network\ + .http_connection_manager.v3.HttpConnectionManager + stat_prefix: %s + route_config: + name: local_route + virtual_hosts: + - name: local_service1 + domains: [ "*" ] + routes: + - match: + prefix: %s + route: + cluster: %s + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + """.formatted(LISTENER_NAME, statPrefix, expectedRoutePrefix(routeVariant), CLUSTER_NAME); + } + if (isAds(apiType)) { + //language=YAML + return """ + name: %s + api_listener: + api_listener: + "@type": type.googleapis.com/envoy.extensions.filters.network\ + .http_connection_manager.v3.HttpConnectionManager + stat_prefix: %s + rds: + route_config_name: %s + config_source: + ads: {} + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + """.formatted(LISTENER_NAME, statPrefix, ROUTE_NAME); + } + //language=YAML + return """ + name: %s + api_listener: + api_listener: + "@type": type.googleapis.com/envoy.extensions.filters.network\ + .http_connection_manager.v3.HttpConnectionManager + stat_prefix: %s + rds: + route_config_name: %s + config_source: + api_config_source: + api_type: %s + grpc_services: + - envoy_grpc: + cluster_name: %s + http_filters: + - name: envoy.filters.http.router + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + """.formatted(LISTENER_NAME, statPrefix, ROUTE_NAME, apiType.name(), BOOTSTRAP_CLUSTER_NAME); + } + + private static RouteConfiguration routeYaml(Variant routeVariant) { + //language=YAML + final String yaml = """ + name: %s + virtual_hosts: + - name: local_service1 + domains: [ "*" ] + routes: + - match: + prefix: %s + route: + cluster: %s + """.formatted(ROUTE_NAME, expectedRoutePrefix(routeVariant), CLUSTER_NAME); + return XdsResourceReader.fromYaml(yaml, RouteConfiguration.class); + } + + private static Cluster clusterYaml(ApiType apiType, Scenario scenario, Variant clusterVariant, + Variant endpointVariant) { + return XdsResourceReader.fromYaml( + clusterYamlString(apiType, scenario, clusterVariant, endpointVariant), + Cluster.class); + } + + private static String clusterYamlString(ApiType apiType, Scenario scenario, Variant clusterVariant, + Variant endpointVariant) { + final long timeoutSeconds = expectedTimeout(clusterVariant); + if (scenario.isStatic(XdsType.ENDPOINT)) { + //language=YAML + return """ + name: %s + type: STATIC + connect_timeout: %ss + load_assignment: + cluster_name: %s + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: %s + """.formatted(CLUSTER_NAME, timeoutSeconds, CLUSTER_NAME, + expectedEndpointPort(endpointVariant)); + } + if (isAds(apiType)) { + //language=YAML + return """ + name: %s + type: EDS + connect_timeout: %ss + eds_cluster_config: + eds_config: + ads: {} + """.formatted(CLUSTER_NAME, timeoutSeconds); + } + //language=YAML + return """ + name: %s + type: EDS + connect_timeout: %ss + eds_cluster_config: + eds_config: + api_config_source: + api_type: %s + grpc_services: + - envoy_grpc: + cluster_name: %s + """.formatted(CLUSTER_NAME, timeoutSeconds, apiType.name(), BOOTSTRAP_CLUSTER_NAME); + } + + private static ClusterLoadAssignment endpointYaml(Variant endpointVariant) { + //language=YAML + final String yaml = """ + cluster_name: %s + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: %s + """.formatted(CLUSTER_NAME, expectedEndpointPort(endpointVariant)); + return XdsResourceReader.fromYaml(yaml, ClusterLoadAssignment.class); + } + + private static String bootstrapYaml(ApiType apiType, Scenario scenario) { + final String dynamicResources; + if (!isAds(apiType)) { + //language=YAML + dynamicResources = """ + lds_config: + api_config_source: + api_type: %s + grpc_services: + - envoy_grpc: + cluster_name: %s + cds_config: + api_config_source: + api_type: %s + grpc_services: + - envoy_grpc: + cluster_name: %s + """.formatted(apiType.name(), BOOTSTRAP_CLUSTER_NAME, + apiType.name(), BOOTSTRAP_CLUSTER_NAME); + } else { + //language=YAML + dynamicResources = """ + ads_config: + api_type: %s + grpc_services: + - envoy_grpc: + cluster_name: %s + lds_config: + ads: {} + cds_config: + ads: {} + """.formatted(apiType.name(), BOOTSTRAP_CLUSTER_NAME); + } + + final StringBuilder staticResources = new StringBuilder(); + staticResources.append(" clusters:\n"); + appendListItem(staticResources, bootstrapClusterYaml(), 2); + if (scenario.isStatic(XdsType.CLUSTER)) { + appendListItem(staticResources, clusterYamlString(apiType, scenario, Variant.V1, Variant.V1), 2); + } + if (scenario.isStatic(XdsType.LISTENER)) { + staticResources.append(" listeners:\n"); + appendListItem(staticResources, listenerYamlString(apiType, scenario, Variant.V1, Variant.V1), 2); + } + + //language=YAML + return """ + dynamic_resources: + %s + static_resources: + %s + """.formatted(dynamicResources.stripTrailing(), + staticResources.toString().stripTrailing()); + } + + private static String bootstrapClusterYaml() { + //language=YAML + return """ + name: %s + type: STATIC + load_assignment: + cluster_name: %s + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: %%s + """.formatted(BOOTSTRAP_CLUSTER_NAME, BOOTSTRAP_CLUSTER_NAME); + } + + private static void appendListItem(StringBuilder sb, String yaml, int indent) { + final String trimmed = yaml.stripTrailing(); + final String[] lines = trimmed.split("\\n"); + if (lines.length == 0) { + return; + } + final String padding = " ".repeat(indent); + sb.append(padding).append("- ").append(lines[0]).append('\n'); + for (int i = 1; i < lines.length; i++) { + if (lines[i].isEmpty()) { + sb.append('\n'); + } else { + sb.append(padding).append(" ").append(lines[i]).append('\n'); + } + } + } + + private enum Operation { + MODIFY, + DELETE, + } + + private enum Variant { + V1, + V2 + } + + private static final class Scenario { + private final EnumSet staticTypes; + + Scenario(XdsType... staticTypes) { + this.staticTypes = staticTypes.length == 0 ? + EnumSet.noneOf(XdsType.class) + : EnumSet.copyOf(Arrays.asList(staticTypes)); + } + + boolean isStatic(XdsType type) { + return staticTypes.contains(type); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Scenario)) { + return false; + } + return staticTypes.equals(((Scenario) o).staticTypes); + } + + @Override + public int hashCode() { + return staticTypes.hashCode(); + } + + @Override + public String toString() { + return "Scenario{static=" + staticTypes + '}'; + } + } + + private static final class ResourceVariants { + private final Variant listener; + private final Variant route; + private final Variant cluster; + private final Variant endpoint; + + private ResourceVariants(Variant listener, Variant route, Variant cluster, Variant endpoint) { + this.listener = listener; + this.route = route; + this.cluster = cluster; + this.endpoint = endpoint; + } + + private static ResourceVariants v1() { + return new ResourceVariants(Variant.V1, Variant.V1, Variant.V1, Variant.V1); + } + + private ResourceVariants with(XdsType target, Variant variant) { + return switch (target) { + case LISTENER -> new ResourceVariants(variant, route, cluster, endpoint); + case ROUTE -> new ResourceVariants(listener, variant, cluster, endpoint); + case CLUSTER -> new ResourceVariants(listener, route, variant, endpoint); + case ENDPOINT -> new ResourceVariants(listener, route, cluster, variant); + default -> throw new IllegalStateException("Unexpected target: " + target); + }; + } + } + + private static final class RecordingWatcher implements SnapshotWatcher { + + private final List snapshots = new CopyOnWriteArrayList<>(); + private final List errors = new CopyOnWriteArrayList<>(); + + @Override + public void onUpdate(T snapshot, Throwable t) { + if (snapshot != null) { + snapshots.add(snapshot); + } + if (t != null) { + errors.add(t); + } + } + + T lastSnapshot() { + if (snapshots.isEmpty()) { + return null; + } + return snapshots.get(snapshots.size() - 1); + } + + List errors() { + return errors; + } + } +} diff --git a/testing-internal/src/main/java/com/linecorp/armeria/internal/testing/InternalTestingBlockHoundIntegration.java b/testing-internal/src/main/java/com/linecorp/armeria/internal/testing/InternalTestingBlockHoundIntegration.java index 3d62508a2b6..dbdcf81eaf5 100644 --- a/testing-internal/src/main/java/com/linecorp/armeria/internal/testing/InternalTestingBlockHoundIntegration.java +++ b/testing-internal/src/main/java/com/linecorp/armeria/internal/testing/InternalTestingBlockHoundIntegration.java @@ -75,6 +75,7 @@ public void applyTo(Builder builder) { "writeBlockingMethod"); builder.allowBlockingCallsInside("com.linecorp.armeria.client.ClientFactory", "ofDefault"); builder.allowBlockingCallsInside("io.envoyproxy.controlplane.cache.SimpleCache", "createWatch"); + builder.allowBlockingCallsInside("io.envoyproxy.controlplane.cache.SimpleCache", "createDeltaWatch"); builder.allowBlockingCallsInside("io.grpc.netty.shaded.io.netty.util.Version", "identify"); // prints the exception which makes it easier to debug issues diff --git a/xds/src/main/java/com/linecorp/armeria/xds/AbstractXdsResource.java b/xds/src/main/java/com/linecorp/armeria/xds/AbstractXdsResource.java index 0836acd7585..47a52a355b4 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/AbstractXdsResource.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/AbstractXdsResource.java @@ -39,6 +39,8 @@ public final long revision() { return revision; } + abstract AbstractXdsResource withRevision(long revision); + @Override public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) { diff --git a/xds/src/main/java/com/linecorp/armeria/xds/AdsXdsStream.java b/xds/src/main/java/com/linecorp/armeria/xds/AdsXdsStream.java new file mode 100644 index 00000000000..d2aaa82913d --- /dev/null +++ b/xds/src/main/java/com/linecorp/armeria/xds/AdsXdsStream.java @@ -0,0 +1,157 @@ +/* + * Copyright 2023 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import static java.util.Objects.requireNonNull; + +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import com.google.common.annotations.VisibleForTesting; + +import com.linecorp.armeria.client.retry.Backoff; +import com.linecorp.armeria.common.annotation.Nullable; + +import io.grpc.Status; +import io.netty.util.concurrent.EventExecutor; + +final class AdsXdsStream implements XdsStream { + + interface ActualStream { + void closeStream(); + + void resourcesUpdated(XdsType type); + } + + @FunctionalInterface + interface ActualStreamFactory { + ActualStream create(AdsXdsStream owner); + } + + private final ActualStreamFactory factory; + private final Backoff backoff; + private final EventExecutor eventLoop; + private final StateCoordinator stateCoordinator; + private final ConfigSourceLifecycleObserver lifecycleObserver; + private final Set targetTypes; + + private int connBackoffAttempts = 1; + private boolean stopped; + @Nullable + private ActualStream actualStream; + + AdsXdsStream(ActualStreamFactory factory, Backoff backoff, EventExecutor eventLoop, + StateCoordinator stateCoordinator, ConfigSourceLifecycleObserver lifecycleObserver, + Set targetTypes) { + this.factory = requireNonNull(factory, "factory"); + this.backoff = requireNonNull(backoff, "backoff"); + this.eventLoop = requireNonNull(eventLoop, "eventLoop"); + this.stateCoordinator = requireNonNull(stateCoordinator, "stateCoordinator"); + this.lifecycleObserver = requireNonNull(lifecycleObserver, "lifecycleObserver"); + this.targetTypes = requireNonNull(targetTypes, "targetTypes"); + } + + @VisibleForTesting + static AdsXdsStream of(ActualStreamFactory factory, Backoff backoff, EventExecutor eventLoop, + StateCoordinator stateCoordinator, + ConfigSourceLifecycleObserver lifecycleObserver, + Set targetTypes) { + return new AdsXdsStream(factory, backoff, eventLoop, stateCoordinator, lifecycleObserver, + targetTypes); + } + + @VisibleForTesting + @Nullable + ActualStream actualStream() { + return actualStream; + } + + @VisibleForTesting + void start() { + if (!eventLoop.inEventLoop()) { + eventLoop.execute(this::start); + return; + } + stopped = false; + reset(); + } + + void stop() { + stop(Status.CANCELLED.withDescription("shutdown").asException()); + } + + void stop(Throwable throwable) { + requireNonNull(throwable, "throwable"); + if (!eventLoop.inEventLoop()) { + eventLoop.execute(() -> stop(throwable)); + return; + } + stopped = true; + if (actualStream == null) { + return; + } + actualStream.closeStream(); + actualStream = null; + } + + @Override + public void close() { + stop(); + lifecycleObserver.close(); + } + + @Override + public void resourcesUpdated(XdsType type) { + ensureStream().resourcesUpdated(type); + } + + void retryOrClose(boolean closedByError) { + if (!eventLoop.inEventLoop()) { + eventLoop.execute(() -> retryOrClose(closedByError)); + return; + } + if (stopped) { + return; + } + actualStream = null; + if (closedByError) { + connBackoffAttempts++; + } else { + connBackoffAttempts = 1; + } + final long nextDelayMillis = backoff.nextDelayMillis(connBackoffAttempts); + eventLoop.schedule(this::reset, Math.max(nextDelayMillis, 1_000L), TimeUnit.MILLISECONDS); + } + + private ActualStream ensureStream() { + if (actualStream == null) { + actualStream = factory.create(this); + } + return actualStream; + } + + private void reset() { + if (stopped) { + return; + } + for (XdsType targetType : targetTypes) { + if (!stateCoordinator.interestedResources(targetType).isEmpty()) { + resourcesUpdated(targetType); + } + } + } +} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/ClusterResourceParser.java b/xds/src/main/java/com/linecorp/armeria/xds/ClusterResourceParser.java index d5357441aba..8d98bd9530d 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/ClusterResourceParser.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/ClusterResourceParser.java @@ -17,7 +17,6 @@ package com.linecorp.armeria.xds; import io.envoyproxy.envoy.config.cluster.v3.Cluster; -import io.envoyproxy.envoy.config.cluster.v3.Cluster.EdsClusterConfig; final class ClusterResourceParser extends ResourceParser { @@ -26,13 +25,8 @@ final class ClusterResourceParser extends ResourceParser streamMap = new EnumMap<>(XdsType.class); - - CompositeXdsStream(GrpcClientBuilder clientBuilder, Node node, Backoff backoff, - EventExecutor eventLoop, XdsResponseHandler handler, - SubscriberStorage subscriberStorage, - Function metersFunction) { - for (XdsType type: XdsType.discoverableTypes()) { - final SotwXdsStream stream = new SotwXdsStream( - SotwDiscoveryStub.basic(type, clientBuilder), node, backoff, eventLoop, - handler, subscriberStorage, EnumSet.of(type), - metersFunction.apply(type.name().toLowerCase(Locale.ROOT))); - streamMap.put(type, stream); + private final Map streamMap; + + CompositeXdsStream(Function streamSupplier) { + final ImmutableMap.Builder streamMapBuilder = ImmutableMap.builder(); + for (XdsType type : XdsType.discoverableTypes()) { + streamMapBuilder.put(type, streamSupplier.apply(type)); } + streamMap = streamMapBuilder.build(); } @Override public void close() { - streamMap.values().forEach(SafeCloseable::close); + streamMap.values().forEach(XdsStream::close); } @Override diff --git a/xds/src/main/java/com/linecorp/armeria/xds/ConfigSourceClient.java b/xds/src/main/java/com/linecorp/armeria/xds/ConfigSourceClient.java index 66ac55282ac..fd41845e01f 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/ConfigSourceClient.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/ConfigSourceClient.java @@ -19,7 +19,9 @@ import static com.google.common.base.Preconditions.checkArgument; import java.time.Instant; +import java.util.EnumSet; import java.util.List; +import java.util.Locale; import java.util.function.Function; import com.google.protobuf.Duration; @@ -31,6 +33,7 @@ import com.linecorp.armeria.common.util.SafeCloseable; import io.envoyproxy.envoy.config.core.v3.ApiConfigSource; +import io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType; import io.envoyproxy.envoy.config.core.v3.ConfigSource; import io.envoyproxy.envoy.config.core.v3.GrpcService; import io.envoyproxy.envoy.config.core.v3.GrpcService.EnvoyGrpc; @@ -40,14 +43,15 @@ final class ConfigSourceClient implements SafeCloseable { - private final SubscriberStorage subscriberStorage; + private final StateCoordinator stateCoordinator; private final XdsStream stream; ConfigSourceClient(ConfigSource configSource, EventExecutor eventLoop, Node node, BootstrapClusters bootstrapClusters, ConfigSourceMapper configSourceMapper, MeterRegistry meterRegistry, - MeterIdPrefix meterIdPrefix) { + MeterIdPrefix meterIdPrefix, + XdsExtensionRegistry extensionRegistry) { final ApiConfigSource apiConfigSource; if (configSource.hasAds()) { apiConfigSource = configSourceMapper.bootstrapAdsConfig(); @@ -57,10 +61,6 @@ final class ConfigSourceClient implements SafeCloseable { throw new IllegalArgumentException("Unsupported config source: " + configSource); } - final long fetchTimeoutMillis = initialFetchTimeoutMillis(configSource); - subscriberStorage = new SubscriberStorage(eventLoop, fetchTimeoutMillis); - final XdsResponseHandler handler = new DefaultResponseHandler(subscriberStorage); - final List grpcServices = apiConfigSource.getGrpcServicesList(); checkArgument(!grpcServices.isEmpty(), "At least one GrpcService should be specified for '%s'", configSource); @@ -74,20 +74,60 @@ final class ConfigSourceClient implements SafeCloseable { builder.responseTimeoutMillis(Long.MAX_VALUE); builder.maxResponseLength(0); + final ApiType apiType = apiConfigSource.getApiType(); + checkArgument(apiType == ApiType.GRPC || apiType == ApiType.DELTA_GRPC || + apiType == ApiType.AGGREGATED_GRPC || apiType == ApiType.AGGREGATED_DELTA_GRPC, + "Unsupported api_type: %s", apiType); final Function metersFunction = xdsType -> new DefaultConfigSourceLifecycleObserver( meterRegistry, meterIdPrefix, configSource.getConfigSourceSpecifierCase(), - envoyGrpc.getClusterName(), xdsType); + envoyGrpc.getClusterName(), xdsType, apiType); - final boolean ads = configSource.hasAds(); - if (ads) { - final SotwDiscoveryStub stub = SotwDiscoveryStub.ads(builder); - stream = new SotwXdsStream(stub, node, Backoff.ofDefault(), - eventLoop, handler, subscriberStorage, metersFunction.apply("ads")); + final boolean isDelta = apiType == ApiType.AGGREGATED_DELTA_GRPC || apiType == ApiType.DELTA_GRPC; + final boolean isAds = configSource.hasAds() || apiType == ApiType.AGGREGATED_GRPC || + apiType == ApiType.AGGREGATED_DELTA_GRPC; + + final long fetchTimeoutMillis = initialFetchTimeoutMillis(configSource); + stateCoordinator = new StateCoordinator(eventLoop, fetchTimeoutMillis, isDelta, + extensionRegistry); + final Backoff backoff = Backoff.ofDefault(); + if (isAds) { + final ConfigSourceLifecycleObserver lifecycleObserver = metersFunction.apply("ads"); + if (isDelta) { + final DeltaDiscoveryStub stub = DeltaDiscoveryStub.ads(builder); + stream = new AdsXdsStream( + owner -> new DeltaActualStream(stub, owner, stateCoordinator, eventLoop, + lifecycleObserver, node), + backoff, eventLoop, stateCoordinator, lifecycleObserver, + XdsType.discoverableTypes()); + } else { + final SotwDiscoveryStub stub = SotwDiscoveryStub.ads(builder); + stream = new AdsXdsStream( + owner -> new SotwActualStream(stub, owner, stateCoordinator, eventLoop, + lifecycleObserver, node), + backoff, eventLoop, stateCoordinator, lifecycleObserver, + XdsType.discoverableTypes()); + } + } else if (isDelta) { + stream = new CompositeXdsStream(type -> { + final DeltaDiscoveryStub stub = DeltaDiscoveryStub.basic(type, builder); + final ConfigSourceLifecycleObserver lifecycleObserver = + metersFunction.apply(type.name().toLowerCase(Locale.ROOT)); + return new AdsXdsStream( + owner -> new DeltaActualStream(stub, owner, stateCoordinator, eventLoop, + lifecycleObserver, node), + backoff, eventLoop, stateCoordinator, lifecycleObserver, EnumSet.of(type)); + }); } else { - stream = new CompositeXdsStream(builder, node, Backoff.ofDefault(), - eventLoop, handler, subscriberStorage, - metersFunction); + stream = new CompositeXdsStream(type -> { + final SotwDiscoveryStub stub = SotwDiscoveryStub.basic(type, builder); + final ConfigSourceLifecycleObserver lifecycleObserver = + metersFunction.apply(type.name().toLowerCase(Locale.ROOT)); + return new AdsXdsStream( + owner -> new SotwActualStream(stub, owner, stateCoordinator, eventLoop, + lifecycleObserver, node), + backoff, eventLoop, stateCoordinator, lifecycleObserver, EnumSet.of(type)); + }); } } @@ -97,23 +137,23 @@ void updateResources(XdsType type) { void addSubscriber(XdsType type, String resourceName, ResourceWatcher watcher) { - if (subscriberStorage.register(type, resourceName, watcher)) { + if (stateCoordinator.register(type, resourceName, watcher)) { updateResources(type); } } boolean removeSubscriber(XdsType type, String resourceName, ResourceWatcher watcher) { - if (subscriberStorage.unregister(type, resourceName, watcher)) { + if (stateCoordinator.unregister(type, resourceName, watcher)) { updateResources(type); } - return subscriberStorage.allSubscribers().isEmpty(); + return stateCoordinator.hasNoSubscribers(); } @Override public void close() { stream.close(); - subscriberStorage.close(); + stateCoordinator.close(); } private static long initialFetchTimeoutMillis(ConfigSource configSource) { diff --git a/xds/src/main/java/com/linecorp/armeria/xds/ConfigSourceLifecycleObserver.java b/xds/src/main/java/com/linecorp/armeria/xds/ConfigSourceLifecycleObserver.java index cce90d3ee19..88c390566f5 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/ConfigSourceLifecycleObserver.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/ConfigSourceLifecycleObserver.java @@ -20,6 +20,8 @@ import com.linecorp.armeria.common.util.SafeCloseable; +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryResponse; import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; @@ -27,8 +29,12 @@ interface ConfigSourceLifecycleObserver extends SafeCloseable { default void requestSent(DiscoveryRequest request) {} + default void requestSent(DeltaDiscoveryRequest request) {} + default void responseReceived(DiscoveryResponse value) {} + default void responseReceived(DeltaDiscoveryResponse value) {} + default void streamOpened() {} default void streamError(Throwable throwable) {} @@ -38,9 +44,15 @@ default void streamCompleted() {} default void resourceUpdated(XdsType type, DiscoveryResponse response, Map updatedResources) {} + default void resourceUpdated(XdsType type, DeltaDiscoveryResponse response, + Map updatedResources) {} + default void resourceRejected(XdsType type, DiscoveryResponse response, Map rejectedResources) {} + default void resourceRejected(XdsType type, DeltaDiscoveryResponse response, + Map rejectedResources) {} + @Override default void close() { } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/ControlPlaneClientManager.java b/xds/src/main/java/com/linecorp/armeria/xds/ControlPlaneClientManager.java index aa46a7c6f60..c74274e1f01 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/ControlPlaneClientManager.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/ControlPlaneClientManager.java @@ -37,19 +37,22 @@ final class ControlPlaneClientManager implements SafeCloseable { private final ConfigSourceMapper configSourceMapper; private final MeterRegistry meterRegistry; private final MeterIdPrefix meterIdPrefix; + private final XdsExtensionRegistry extensionRegistry; private final Map clientMap = new HashMap<>(); private boolean closed; ControlPlaneClientManager(Bootstrap bootstrap, EventExecutor eventLoop, BootstrapClusters bootstrapClusters, ConfigSourceMapper configSourceMapper, - MeterRegistry meterRegistry, MeterIdPrefix meterIdPrefix) { + MeterRegistry meterRegistry, MeterIdPrefix meterIdPrefix, + XdsExtensionRegistry extensionRegistry) { bootstrapNode = bootstrap.getNode(); this.eventLoop = eventLoop; this.bootstrapClusters = bootstrapClusters; this.configSourceMapper = configSourceMapper; this.meterRegistry = meterRegistry; this.meterIdPrefix = meterIdPrefix; + this.extensionRegistry = extensionRegistry; } void subscribe(ResourceNode node) { @@ -65,7 +68,7 @@ void subscribe(ResourceNode node) { final ConfigSourceClient client = clientMap.computeIfAbsent( configSource, ignored -> new ConfigSourceClient( configSource, eventLoop, bootstrapNode, bootstrapClusters, configSourceMapper, - meterRegistry, meterIdPrefix)); + meterRegistry, meterIdPrefix, extensionRegistry)); client.addSubscriber(type, name, node); } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/DefaultConfigSourceLifecycleObserver.java b/xds/src/main/java/com/linecorp/armeria/xds/DefaultConfigSourceLifecycleObserver.java index 5e529f0c35b..4c1880072ea 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/DefaultConfigSourceLifecycleObserver.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/DefaultConfigSourceLifecycleObserver.java @@ -24,7 +24,10 @@ import com.linecorp.armeria.common.metric.MeterIdPrefix; +import io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType; import io.envoyproxy.envoy.config.core.v3.ConfigSource.ConfigSourceSpecifierCase; +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryResponse; import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; import io.micrometer.core.instrument.Counter; @@ -51,11 +54,12 @@ final class DefaultConfigSourceLifecycleObserver implements ConfigSourceLifecycl DefaultConfigSourceLifecycleObserver(MeterRegistry meterRegistry, MeterIdPrefix meterIdPrefix, ConfigSourceSpecifierCase specifierCase, - String reprName, String xdsType) { + String reprName, String xdsType, ApiType apiType) { this.meterRegistry = meterRegistry; loggingIdentifier = String.format("[%d.%s](%s)", specifierCase.getNumber(), xdsType, reprName); meterIdPrefix = meterIdPrefix.withTags("type", specifierCase.name().toLowerCase(Locale.ROOT), - "name", reprName, "xdsType", xdsType); + "name", reprName, "xdsType", xdsType, + "apiType", apiType.name().toLowerCase(Locale.ROOT)); streamOpenedCounter = meterRegistry.counter(meterIdPrefix.name("configsource.stream.opened"), meterIdPrefix.tags()); streamErrorCounter = meterRegistry.counter(meterIdPrefix.name("configsource.stream.error"), @@ -104,7 +108,31 @@ public void streamCompleted() { @Override public void requestSent(DiscoveryRequest request) { - logger.debug("{} Sending discovery request: {}", loggingIdentifier, request); + if (logger.isTraceEnabled()) { + logger.trace("{} Sending discovery request: {}", loggingIdentifier, request); + } else if (logger.isDebugEnabled()) { + logger.debug("{} Sending discovery request: type_url={}, " + + "subscribe_count={}, version_info={}, nonce={}", + loggingIdentifier, request.getTypeUrl(), + request.getResourceNamesList().size(), + request.getVersionInfo(), request.getResponseNonce()); + } + streamRequestCounter.increment(); + } + + @Override + public void requestSent(DeltaDiscoveryRequest request) { + if (logger.isTraceEnabled()) { + logger.trace("{} Sending discovery request: {}", loggingIdentifier, request); + } else if (logger.isDebugEnabled()) { + logger.debug("{} Sending discovery request: type_url={}, subscribe_count={}, " + + "unsubscribe_count={}, initial_versions_count={}, nonce={}", + loggingIdentifier, request.getTypeUrl(), + request.getResourceNamesSubscribeCount(), + request.getResourceNamesUnsubscribeCount(), + request.getInitialResourceVersionsCount(), + request.getResponseNonce()); + } streamRequestCounter.increment(); } @@ -112,6 +140,25 @@ public void requestSent(DiscoveryRequest request) { public void responseReceived(DiscoveryResponse value) { if (logger.isTraceEnabled()) { logger.trace("{} Received discovery response: {}", loggingIdentifier, value); + } else if (logger.isDebugEnabled()) { + logger.debug("{} Received discovery response: type_url={}, " + + "resources_count={}, version_info={}, nonce={}", + loggingIdentifier, value.getTypeUrl(), value.getResourcesCount(), + value.getVersionInfo(), value.getNonce()); + } + streamResponseCounter.increment(); + } + + @Override + public void responseReceived(DeltaDiscoveryResponse value) { + if (logger.isTraceEnabled()) { + logger.trace("{} Received discovery response: {}", loggingIdentifier, value); + } else if (logger.isDebugEnabled()) { + logger.debug("{} Received discovery response: type_url={}, resources_count={}, removed_count={}, " + + "system_version_info={}, nonce={}", + loggingIdentifier, value.getTypeUrl(), value.getResourcesCount(), + value.getRemovedResourcesCount(), value.getSystemVersionInfo(), + value.getNonce()); } streamResponseCounter.increment(); } @@ -125,6 +172,15 @@ public void resourceUpdated(XdsType type, DiscoveryResponse response, resourceParseSuccessCounter.increment(updatedResources.size()); } + @Override + public void resourceUpdated(XdsType type, DeltaDiscoveryResponse response, + Map updatedResources) { + if (!updatedResources.isEmpty()) { + logger.debug("{} Updating resources: {}", loggingIdentifier, updatedResources); + } + resourceParseSuccessCounter.increment(updatedResources.size()); + } + @Override public void resourceRejected(XdsType type, DiscoveryResponse response, Map rejectedResources) { @@ -134,6 +190,15 @@ public void resourceRejected(XdsType type, DiscoveryResponse response, resourceParseRejectedCounter.increment(rejectedResources.size()); } + @Override + public void resourceRejected(XdsType type, DeltaDiscoveryResponse response, + Map rejectedResources) { + if (!rejectedResources.isEmpty()) { + logger.warn("{} Rejected resources: {}", loggingIdentifier, rejectedResources); + } + resourceParseRejectedCounter.increment(rejectedResources.size()); + } + @Override public void close() { meterRegistry.remove(streamOpenedCounter); diff --git a/xds/src/main/java/com/linecorp/armeria/xds/DefaultResponseHandler.java b/xds/src/main/java/com/linecorp/armeria/xds/DefaultResponseHandler.java deleted file mode 100644 index 0ae4973402f..00000000000 --- a/xds/src/main/java/com/linecorp/armeria/xds/DefaultResponseHandler.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright 2023 LINE Corporation - * - * LINE Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.xds; - -import java.util.Map; - -import com.google.common.base.Joiner; -import com.google.protobuf.Message; - -import com.linecorp.armeria.xds.SotwXdsStream.ActualStream; - -import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; - -final class DefaultResponseHandler implements XdsResponseHandler { - - private final SubscriberStorage storage; - private static final Joiner errorMessageJoiner = Joiner.on('\n'); - - DefaultResponseHandler(SubscriberStorage storage) { - this.storage = storage; - } - - @Override - public void handleResponse( - ResourceParser resourceParser, DiscoveryResponse response, ActualStream sender, - ConfigSourceLifecycleObserver observer) { - final long nextRevision = sender.versionManager() - .nextRevision(resourceParser.type(), response.getVersionInfo()); - final ParsedResourcesHolder holder = - resourceParser.parseResources(response.getResourcesList(), response.getVersionInfo(), - nextRevision); - final String errorDetails; - if (holder.errors().isEmpty()) { - sender.ackResponse(resourceParser.type(), response.getVersionInfo(), response.getNonce()); - } else { - errorDetails = errorMessageJoiner.join(holder.errors()); - sender.nackResponse(resourceParser.type(), response.getNonce(), errorDetails); - } - observer.resourceUpdated(resourceParser.type(), response, holder.parsedResources()); - observer.resourceRejected(resourceParser.type(), response, holder.invalidResources()); - - final Map> subscribedResources = - storage.subscribers(resourceParser.type()); - for (Map.Entry> entry : subscribedResources.entrySet()) { - final String resourceName = entry.getKey(); - final XdsStreamSubscriber subscriber = entry.getValue(); - - if (holder.parsedResources().containsKey(resourceName)) { - // Happy path: the resource updated successfully. Notify the watchers of the update. - notifyOnData(subscriber, holder, resourceName); - continue; - } - - final Throwable errorCause = holder.invalidResources().get(resourceName); - if (errorCause != null) { - subscriber.onError(resourceName, errorCause); - continue; - } - - // Handle State of the World ADS - if (!resourceParser.isFullStateOfTheWorld()) { - continue; - } - - // For State of the World services, notify watchers when their watched resource is missing - // from the ADS update. Note that we can only do this if the resource update is coming from - // the same xDS server that the ResourceSubscriber is subscribed to. - subscriber.onAbsent(); - } - } - - @SuppressWarnings("unchecked") - private static void notifyOnData(XdsStreamSubscriber subscriber, - ParsedResourcesHolder holder, - String resourceName) { - final O data = (O) holder.parsedResources().get(resourceName); - assert data != null; - subscriber.onData(data); - } -} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/DefaultSubscriptionContext.java b/xds/src/main/java/com/linecorp/armeria/xds/DefaultSubscriptionContext.java index fd685797c12..4606e688b51 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/DefaultSubscriptionContext.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/DefaultSubscriptionContext.java @@ -33,12 +33,14 @@ final class DefaultSubscriptionContext implements SubscriptionContext { private final DirectoryWatchService watchService; private final BootstrapSecrets bootstrapSecrets; private final ResourceNodeMeterBinderFactory meterBinderFactory; + private final XdsExtensionRegistry extensionRegistry; DefaultSubscriptionContext(EventExecutor eventLoop, XdsClusterManager clusterManager, ConfigSourceMapper configSourceMapper, ControlPlaneClientManager controlPlaneClientManager, MeterRegistry meterRegistry, MeterIdPrefix meterIdPrefix, - DirectoryWatchService watchService, BootstrapSecrets bootstrapSecrets) { + DirectoryWatchService watchService, BootstrapSecrets bootstrapSecrets, + XdsExtensionRegistry extensionRegistry) { this.eventLoop = eventLoop; this.clusterManager = clusterManager; this.configSourceMapper = configSourceMapper; @@ -47,6 +49,7 @@ final class DefaultSubscriptionContext implements SubscriptionContext { this.meterIdPrefix = meterIdPrefix; this.watchService = watchService; this.bootstrapSecrets = bootstrapSecrets; + this.extensionRegistry = extensionRegistry; meterBinderFactory = new ResourceNodeMeterBinderFactory(meterRegistry, meterIdPrefix); } @@ -99,4 +102,9 @@ public BootstrapSecrets bootstrapSecrets() { public ResourceNodeMeterBinderFactory meterBinderFactory() { return meterBinderFactory; } + + @Override + public XdsExtensionRegistry extensionRegistry() { + return extensionRegistry; + } } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/DeltaActualStream.java b/xds/src/main/java/com/linecorp/armeria/xds/DeltaActualStream.java new file mode 100644 index 00000000000..f982797aa8b --- /dev/null +++ b/xds/src/main/java/com/linecorp/armeria/xds/DeltaActualStream.java @@ -0,0 +1,277 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import static com.linecorp.armeria.xds.XdsResourceParserUtil.fromTypeUrl; + +import java.util.ArrayDeque; +import java.util.EnumSet; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableSet; +import com.google.rpc.Code; +import com.google.rpc.Status; + +import com.linecorp.armeria.common.annotation.Nullable; + +import io.envoyproxy.envoy.config.core.v3.Node; +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryResponse; +import io.envoyproxy.envoy.service.discovery.v3.Resource; +import io.grpc.stub.StreamObserver; +import io.netty.util.concurrent.EventExecutor; + +final class DeltaActualStream implements StreamObserver, AdsXdsStream.ActualStream { + + private static final Logger logger = LoggerFactory.getLogger(DeltaActualStream.class); + + private final StreamObserver requestObserver; + private final AdsXdsStream owner; + private final StateCoordinator stateCoordinator; + private final EventExecutor eventLoop; + private final ConfigSourceLifecycleObserver lifecycleObserver; + private final Node node; + + private final ArrayDeque ackQueue = new ArrayDeque<>(); + private final EnumSet pendingUpdates = EnumSet.noneOf(XdsType.class); + // Types for which initial_resource_versions has already been sent on this stream. + private final Set initialVersionsSent = EnumSet.noneOf(XdsType.class); + private boolean completed; + private boolean draining; + + DeltaActualStream(DeltaDiscoveryStub stub, AdsXdsStream owner, StateCoordinator stateCoordinator, + EventExecutor eventLoop, ConfigSourceLifecycleObserver lifecycleObserver, Node node) { + this.owner = owner; + this.stateCoordinator = stateCoordinator; + this.eventLoop = eventLoop; + this.lifecycleObserver = lifecycleObserver; + this.node = node; + requestObserver = stub.stream(this); + lifecycleObserver.streamOpened(); + } + + void ackResponse(XdsType type, String nonce) { + enqueueAck(type, nonce, null); + } + + void nackResponse(XdsType type, String nonce, String errorDetail) { + if (completed) { + return; + } + eventLoop.schedule(() -> enqueueAck(type, nonce, errorDetail), SotwActualStream.NACK_BACKOFF_MILLIS, + TimeUnit.MILLISECONDS); + } + + @Override + public void closeStream() { + if (completed) { + return; + } + completed = true; + requestObserver.onCompleted(); + } + + @Override + public void resourcesUpdated(XdsType type) { + enqueueDeltaRequest(type); + } + + private void enqueueDeltaRequest(XdsType type) { + if (completed) { + return; + } + pendingUpdates.add(type); + drainRequests(); + } + + private void enqueueAck(XdsType type, String nonce, @Nullable String errorDetail) { + if (completed) { + return; + } + ackQueue.add(new PendingAck(type, nonce, errorDetail)); + drainRequests(); + } + + private void drainRequests() { + if (draining || completed) { + return; + } + draining = true; + try { + while (!completed) { + final PendingAck ack = ackQueue.poll(); + if (ack != null) { + sendDeltaRequest(ack.type, ack.nonce, ack.errorDetail); + continue; + } + final XdsType pendingType = nextPendingUpdate(); + if (pendingType == null) { + return; + } + pendingUpdates.remove(pendingType); + sendDeltaRequest(pendingType, null, null); + } + } finally { + draining = false; + } + } + + @Nullable + private XdsType nextPendingUpdate() { + for (XdsType type : XdsType.discoverableTypes()) { + if (pendingUpdates.contains(type)) { + return type; + } + } + return null; + } + + private void sendDeltaRequest(XdsType type, @Nullable String nonce, @Nullable String errorDetail) { + if (completed) { + return; + } + final Set current = stateCoordinator.interestedResources(type); + + final Set subscribe; + final Set unsubscribe; + final boolean isFirstOnStream = !initialVersionsSent.contains(type); + if (isFirstOnStream) { + subscribe = current; + unsubscribe = ImmutableSet.of(); + } else { + final Set previous = stateCoordinator.activeResources(type); + subscribe = new HashSet<>(current); + subscribe.removeAll(previous); + unsubscribe = new HashSet<>(previous); + unsubscribe.removeAll(current); + } + + final DeltaDiscoveryRequest.Builder builder = + DeltaDiscoveryRequest.newBuilder() + .setTypeUrl(type.typeUrl()) + .setNode(node) + .addAllResourceNamesSubscribe(subscribe) + .addAllResourceNamesUnsubscribe(unsubscribe); + if (nonce != null) { + builder.setResponseNonce(nonce); + } + if (errorDetail != null) { + builder.setErrorDetail(Status.newBuilder() + .setCode(Code.INVALID_ARGUMENT_VALUE) + .setMessage(errorDetail) + .build()); + } + if (isFirstOnStream) { + builder.putAllInitialResourceVersions(stateCoordinator.resourceVersions(type)); + initialVersionsSent.add(type); + } + final DeltaDiscoveryRequest request = builder.build(); + lifecycleObserver.requestSent(request); + requestObserver.onNext(request); + } + + private static final class PendingAck { + + private final XdsType type; + private final String nonce; + @Nullable + private final String errorDetail; + + private PendingAck(XdsType type, String nonce, @Nullable String errorDetail) { + this.type = type; + this.nonce = nonce; + this.errorDetail = errorDetail; + } + } + + @Override + public void onNext(DeltaDiscoveryResponse value) { + if (!eventLoop.inEventLoop()) { + eventLoop.execute(() -> onNext(value)); + return; + } + if (completed) { + return; + } + lifecycleObserver.responseReceived(value); + + final ResourceParser resourceParser = fromTypeUrl(value.getTypeUrl()); + if (resourceParser == null) { + logger.warn("Delta XDS stream received unexpected type: {}", value.getTypeUrl()); + return; + } + handleResponse(resourceParser, value); + } + + @Override + public void onError(Throwable throwable) { + if (!eventLoop.inEventLoop()) { + eventLoop.execute(() -> onError(throwable)); + return; + } + completed = true; + lifecycleObserver.streamError(throwable); + owner.retryOrClose(true); + } + + @Override + public void onCompleted() { + if (!eventLoop.inEventLoop()) { + eventLoop.execute(this::onCompleted); + return; + } + completed = true; + lifecycleObserver.streamCompleted(); + owner.retryOrClose(false); + } + + private void handleResponse(ResourceParser resourceParser, DeltaDiscoveryResponse response) { + final XdsType type = resourceParser.type(); + final List deltaResources = response.getResourcesList(); + final ParsedResourcesHolder holder = + resourceParser.parseDeltaResources(deltaResources, + stateCoordinator.extensionRegistry()); + + if (!holder.errors().isEmpty()) { + holder.invalidResources().forEach((name, error) -> + stateCoordinator.onResourceError(type, name, error)); + lifecycleObserver.resourceRejected(type, response, holder.invalidResources()); + nackResponse(type, response.getNonce(), String.join("\n", holder.errors())); + return; + } + lifecycleObserver.resourceUpdated(type, response, holder.parsedResources()); + + holder.parsedResources().forEach((name, resource) -> { + if (resource instanceof XdsResource) { + stateCoordinator.onResourceUpdated(type, name, (XdsResource) resource); + } + }); + + for (String removedName : response.getRemovedResourcesList()) { + stateCoordinator.onResourceMissing(type, removedName); + } + + // ack after processing so that the diff between interested - state is computed correctly + ackResponse(type, response.getNonce()); + } +} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/DeltaDiscoveryStub.java b/xds/src/main/java/com/linecorp/armeria/xds/DeltaDiscoveryStub.java new file mode 100644 index 00000000000..f90098348e3 --- /dev/null +++ b/xds/src/main/java/com/linecorp/armeria/xds/DeltaDiscoveryStub.java @@ -0,0 +1,67 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import com.linecorp.armeria.client.grpc.GrpcClientBuilder; + +import io.envoyproxy.envoy.service.cluster.v3.ClusterDiscoveryServiceGrpc.ClusterDiscoveryServiceStub; +import io.envoyproxy.envoy.service.discovery.v3.AggregatedDiscoveryServiceGrpc.AggregatedDiscoveryServiceStub; +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DeltaDiscoveryResponse; +import io.envoyproxy.envoy.service.endpoint.v3.EndpointDiscoveryServiceGrpc.EndpointDiscoveryServiceStub; +import io.envoyproxy.envoy.service.listener.v3.ListenerDiscoveryServiceGrpc.ListenerDiscoveryServiceStub; +import io.envoyproxy.envoy.service.route.v3.RouteDiscoveryServiceGrpc.RouteDiscoveryServiceStub; +import io.envoyproxy.envoy.service.secret.v3.SecretDiscoveryServiceGrpc.SecretDiscoveryServiceStub; +import io.grpc.stub.StreamObserver; + +@FunctionalInterface +interface DeltaDiscoveryStub { + + StreamObserver stream(StreamObserver responseObserver); + + static DeltaDiscoveryStub ads(GrpcClientBuilder builder) { + final AggregatedDiscoveryServiceStub stub = builder.build(AggregatedDiscoveryServiceStub.class); + return stub::deltaAggregatedResources; + } + + static DeltaDiscoveryStub basic(XdsType type, GrpcClientBuilder builder) { + switch (type) { + case LISTENER: + final ListenerDiscoveryServiceStub listenerStub = + builder.build(ListenerDiscoveryServiceStub.class); + return listenerStub::deltaListeners; + case ROUTE: + final RouteDiscoveryServiceStub routeStub = + builder.build(RouteDiscoveryServiceStub.class); + return routeStub::deltaRoutes; + case CLUSTER: + final ClusterDiscoveryServiceStub clusterStub = + builder.build(ClusterDiscoveryServiceStub.class); + return clusterStub::deltaClusters; + case ENDPOINT: + final EndpointDiscoveryServiceStub endpointStub = + builder.build(EndpointDiscoveryServiceStub.class); + return endpointStub::deltaEndpoints; + case SECRET: + final SecretDiscoveryServiceStub secretStub = + builder.build(SecretDiscoveryServiceStub.class); + return secretStub::deltaSecrets; + default: + throw new Error("Unexpected value: " + type); + } + } +} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/EndpointResourceParser.java b/xds/src/main/java/com/linecorp/armeria/xds/EndpointResourceParser.java index 0d3811ce0f6..19b553a8cda 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/EndpointResourceParser.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/EndpointResourceParser.java @@ -25,8 +25,9 @@ final class EndpointResourceParser extends ResourceParser mergeFilterConfigs( .buildKeepingLast(); } - static ClientPreprocessors buildDownstreamFilter( - @Nullable HttpConnectionManager connectionManager) { - if (connectionManager == null) { + static ClientPreprocessors buildDownstreamFilter(List downstreamFilters) { + if (downstreamFilters.isEmpty()) { return ClientPreprocessors.of(); } - final List httpFilters = connectionManager.getHttpFiltersList(); final ClientPreprocessorsBuilder builder = ClientPreprocessors.builder(); - for (int i = httpFilters.size() - 1; i >= 0; i--) { - final HttpFilter httpFilter = httpFilters.get(i); - final XdsHttpFilter instance = resolveInstance(httpFilter, null); - if (instance == null) { - continue; - } + for (int i = downstreamFilters.size() - 1; i >= 0; i--) { + final XdsHttpFilter instance = downstreamFilters.get(i); builder.add(instance.httpPreprocessor()); builder.addRpc(instance.rpcPreprocessor()); } @@ -69,13 +61,14 @@ static ClientPreprocessors buildDownstreamFilter( } static ClientDecoration buildUpstreamFilter( + XdsExtensionRegistry extensionRegistry, List httpFilters, Map filterConfigs, @Nullable RetryPolicy retryPolicy) { final ClientDecorationBuilder builder = ClientDecoration.builder(); for (int i = httpFilters.size() - 1; i >= 0; i--) { final HttpFilter httpFilter = httpFilters.get(i); final Any perRouteConfig = filterConfigs.get(httpFilter.getName()); - final XdsHttpFilter instance = resolveInstance(httpFilter, perRouteConfig); + final XdsHttpFilter instance = resolveInstance(extensionRegistry, httpFilter, perRouteConfig); if (instance == null) { continue; } @@ -90,15 +83,17 @@ static ClientDecoration buildUpstreamFilter( } @Nullable - private static XdsHttpFilter resolveInstance( + static XdsHttpFilter resolveInstance( + XdsExtensionRegistry extensionRegistry, HttpFilter httpFilter, @Nullable Any perRouteConfig) { - final HttpFilterFactory filterFactory = - HttpFilterFactoryRegistry.filterFactory(httpFilter.getName()); - if (filterFactory == null) { + final Any typedConfig = httpFilter.getTypedConfig(); + final HttpFilterFactory factory = extensionRegistry.query( + typedConfig, httpFilter.getName(), HttpFilterFactory.class); + if (factory == null) { if (!httpFilter.getIsOptional()) { throw new IllegalArgumentException( "Unknown HTTP filter '" + httpFilter.getName() + - "': no HttpFilterFactory registered. Register an SPI " + + "': no HttpFilterFactory registered. Register an " + "HttpFilterFactory implementation to handle this filter."); } return null; @@ -107,9 +102,8 @@ private static XdsHttpFilter resolveInstance( httpFilter.getConfigTypeCase() == ConfigTypeCase.CONFIGTYPE_NOT_SET, "Only 'typed_config' is supported, but '%s' was supplied", httpFilter.getConfigTypeCase()); - final Any effectiveConfig = - perRouteConfig != null ? perRouteConfig : httpFilter.getTypedConfig(); - return filterFactory.create(httpFilter, effectiveConfig); + final Any effectiveConfig = perRouteConfig != null ? perRouteConfig : typedConfig; + return factory.create(httpFilter, effectiveConfig, extensionRegistry.validator()); } private FilterUtil() {} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/HttpConnectionManagerFactory.java b/xds/src/main/java/com/linecorp/armeria/xds/HttpConnectionManagerFactory.java new file mode 100644 index 00000000000..2dab8886477 --- /dev/null +++ b/xds/src/main/java/com/linecorp/armeria/xds/HttpConnectionManagerFactory.java @@ -0,0 +1,49 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import java.util.List; + +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Any; + +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; + +final class HttpConnectionManagerFactory implements XdsExtensionFactory { + + static final HttpConnectionManagerFactory INSTANCE = new HttpConnectionManagerFactory(); + private static final String NAME = "envoy.http_connection_manager"; + private static final String TYPE_URL = + "type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3" + + ".HttpConnectionManager"; + + private HttpConnectionManagerFactory() {} + + @Override + public String name() { + return NAME; + } + + @Override + public List typeUrls() { + return ImmutableList.of(TYPE_URL); + } + + HttpConnectionManager create(Any config, XdsResourceValidator validator) { + return validator.unpack(config, HttpConnectionManager.class); + } +} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/ListenerManager.java b/xds/src/main/java/com/linecorp/armeria/xds/ListenerManager.java index 231110ee746..1e83c7eb60d 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/ListenerManager.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/ListenerManager.java @@ -31,6 +31,7 @@ import io.envoyproxy.envoy.config.bootstrap.v3.Bootstrap; import io.envoyproxy.envoy.config.bootstrap.v3.Bootstrap.StaticResources; import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; import io.netty.util.concurrent.EventExecutor; final class ListenerManager implements SafeCloseable { @@ -61,7 +62,14 @@ private void initializeBootstrap(Bootstrap bootstrap, SubscriptionContext bootst void register(Listener listener, SubscriptionContext context, SnapshotWatcher watcher) { checkArgument(!nodes.containsKey(listener.getName()), "Static listener with name '%s' already registered", listener.getName()); - final ListenerStream node = new ListenerStream(new ListenerXdsResource(listener), context); + final XdsExtensionRegistry registry = context.extensionRegistry(); + final HttpConnectionManager connectionManager = + XdsUnpackUtil.unpackConnectionManager(listener, registry); + final ListenerXdsResource listenerResource = + new ListenerXdsResource(listener, connectionManager, + XdsUnpackUtil.resolveDownstreamFilters(connectionManager, registry), + ""); + final ListenerStream node = new ListenerStream(listenerResource, context); nodes.put(listener.getName(), node); eventLoop.execute(safeRunnable(() -> { final Subscription subscription = node.subscribe(watcher); diff --git a/xds/src/main/java/com/linecorp/armeria/xds/ListenerResourceParser.java b/xds/src/main/java/com/linecorp/armeria/xds/ListenerResourceParser.java index fffd5126611..32bde5d075c 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/ListenerResourceParser.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/ListenerResourceParser.java @@ -16,9 +16,12 @@ package com.linecorp.armeria.xds; +import java.util.List; + +import com.linecorp.armeria.xds.filter.XdsHttpFilter; + import io.envoyproxy.envoy.config.listener.v3.Listener; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; -import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.Rds; final class ListenerResourceParser extends ResourceParser { @@ -27,16 +30,12 @@ final class ListenerResourceParser extends ResourceParser downstreamFilters = + XdsUnpackUtil.resolveDownstreamFilters(connectionManager, registry); + return new ListenerXdsResource(message, connectionManager, downstreamFilters, version); } @Override diff --git a/xds/src/main/java/com/linecorp/armeria/xds/ListenerSnapshot.java b/xds/src/main/java/com/linecorp/armeria/xds/ListenerSnapshot.java index 3b5d587d81b..4b00677679a 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/ListenerSnapshot.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/ListenerSnapshot.java @@ -43,7 +43,7 @@ public final class ListenerSnapshot implements Snapshot { ListenerSnapshot(ListenerXdsResource listenerXdsResource, @Nullable RouteSnapshot routeSnapshot) { this.listenerXdsResource = listenerXdsResource; this.routeSnapshot = routeSnapshot; - downstreamFilter = FilterUtil.buildDownstreamFilter(listenerXdsResource.connectionManager()); + downstreamFilter = FilterUtil.buildDownstreamFilter(listenerXdsResource.downstreamFilters()); } @Override diff --git a/xds/src/main/java/com/linecorp/armeria/xds/ListenerXdsResource.java b/xds/src/main/java/com/linecorp/armeria/xds/ListenerXdsResource.java index 550a85cafe4..749e57a3e67 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/ListenerXdsResource.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/ListenerXdsResource.java @@ -16,19 +16,18 @@ package com.linecorp.armeria.xds; -import static com.google.common.base.Preconditions.checkArgument; - import java.util.List; -import com.google.protobuf.Any; +import com.google.common.collect.ImmutableList; import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.annotation.UnstableApi; +import com.linecorp.armeria.xds.client.endpoint.RouterFilterFactory.RouterXdsHttpFilter; +import com.linecorp.armeria.xds.filter.XdsHttpFilter; import io.envoyproxy.envoy.config.listener.v3.Listener; import io.envoyproxy.envoy.extensions.filters.http.router.v3.Router; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; -import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter; /** * A resource object for a {@link Listener}. @@ -36,38 +35,42 @@ @UnstableApi public final class ListenerXdsResource extends AbstractXdsResource { - private static final String HTTP_CONNECTION_MANAGER_TYPE_URL = - "type.googleapis.com/" + - "envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager"; - private static final String ROUTER_TYPE_URL = - "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"; - private final Listener listener; @Nullable private final HttpConnectionManager connectionManager; + private final List downstreamFilters; @Nullable private final Router router; - ListenerXdsResource(Listener listener) { - this(listener, "", 0); + ListenerXdsResource(Listener listener, @Nullable HttpConnectionManager connectionManager) { + this(listener, connectionManager, ImmutableList.of(), "", 0); + } + + ListenerXdsResource(Listener listener, @Nullable HttpConnectionManager connectionManager, + List downstreamFilters, String version) { + this(listener, connectionManager, downstreamFilters, version, 0); } - ListenerXdsResource(Listener listener, String version, long revision) { + private ListenerXdsResource(Listener listener, @Nullable HttpConnectionManager connectionManager, + List downstreamFilters, + String version, long revision) { super(version, revision); - XdsValidatorIndexRegistry.assertValid(listener); this.listener = listener; + this.connectionManager = connectionManager; + this.downstreamFilters = downstreamFilters; + router = findRouter(downstreamFilters); + } - if (listener.getApiListener().hasApiListener()) { - final Any apiListener = listener.getApiListener().getApiListener(); - if (HTTP_CONNECTION_MANAGER_TYPE_URL.equals(apiListener.getTypeUrl())) { - connectionManager = XdsValidatorIndexRegistry.unpack(apiListener, HttpConnectionManager.class); - } else { - throw new IllegalArgumentException("Unsupported api listener: " + apiListener); - } - } else { - connectionManager = null; + @Nullable + private static Router findRouter(List filters) { + if (filters.isEmpty()) { + return null; + } + final XdsHttpFilter last = filters.get(filters.size() - 1); + if (last instanceof RouterXdsHttpFilter) { + return ((RouterXdsHttpFilter) last).router(); } - router = router(connectionManager); + return null; } @Override @@ -93,6 +96,15 @@ public String name() { return listener.getName(); } + @Override + ListenerXdsResource withRevision(long revision) { + if (revision == revision()) { + return this; + } + return new ListenerXdsResource(listener, connectionManager, downstreamFilters, + version(), revision); + } + /** * The {@link Router} contained in the {@link Listener}. */ @@ -101,21 +113,10 @@ public Router router() { return router; } - @Nullable - private static Router router(@Nullable HttpConnectionManager connectionManager) { - if (connectionManager == null) { - return null; - } - final List httpFilters = connectionManager.getHttpFiltersList(); - if (httpFilters.isEmpty()) { - return null; - } - final HttpFilter lastHttpFilter = httpFilters.get(httpFilters.size() - 1); - if (!ROUTER_TYPE_URL.equals(lastHttpFilter.getTypedConfig().getTypeUrl())) { - // the router should be the last/terminal filter - return null; - } - checkArgument(lastHttpFilter.hasTypedConfig(), "Only typedConfig is supported for 'Router'."); - return XdsValidatorIndexRegistry.unpack(lastHttpFilter.getTypedConfig(), Router.class); + /** + * The pre-resolved downstream {@link XdsHttpFilter} instances. + */ + List downstreamFilters() { + return downstreamFilters; } } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/RawBufferTransportSocketFactory.java b/xds/src/main/java/com/linecorp/armeria/xds/RawBufferTransportSocketFactory.java new file mode 100644 index 00000000000..5f70e344e42 --- /dev/null +++ b/xds/src/main/java/com/linecorp/armeria/xds/RawBufferTransportSocketFactory.java @@ -0,0 +1,53 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import java.util.List; + +import com.google.common.collect.ImmutableList; + +import com.linecorp.armeria.common.annotation.Nullable; + +import io.envoyproxy.envoy.config.core.v3.ConfigSource; +import io.envoyproxy.envoy.config.core.v3.TransportSocket; + +final class RawBufferTransportSocketFactory implements TransportSocketFactory { + + static final RawBufferTransportSocketFactory INSTANCE = new RawBufferTransportSocketFactory(); + private static final String NAME = "envoy.transport_sockets.raw_buffer"; + private static final String TYPE_URL = + "type.googleapis.com/envoy.extensions.transport_sockets.raw_buffer.v3.RawBuffer"; + + private RawBufferTransportSocketFactory() {} + + @Override + public String name() { + return NAME; + } + + @Override + public List typeUrls() { + return ImmutableList.of(TYPE_URL); + } + + @Override + public SnapshotStream create( + SubscriptionContext context, @Nullable ConfigSource configSource, + TransportSocket transportSocket) { + return SnapshotStream.just(new TransportSocketSnapshot(transportSocket)); + } +} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/ResourceParser.java b/xds/src/main/java/com/linecorp/armeria/xds/ResourceParser.java index 55e66d4838c..47942a5fe15 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/ResourceParser.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/ResourceParser.java @@ -23,15 +23,18 @@ import com.google.protobuf.Any; import com.google.protobuf.Message; +import io.envoyproxy.envoy.service.discovery.v3.Resource; + abstract class ResourceParser { abstract String name(I message); abstract Class clazz(); - abstract O parse(I message, String version, long revision); + abstract O parse(I message, XdsExtensionRegistry extensionRegistry, String version); - ParsedResourcesHolder parseResources(List resources, String version, long revision) { + ParsedResourcesHolder parseResources(List resources, XdsExtensionRegistry extensionRegistry, + String version) { final ImmutableMap.Builder parsedResources = ImmutableMap.builder(); final ImmutableMap.Builder invalidResources = ImmutableMap.builder(); @@ -49,7 +52,8 @@ ParsedResourcesHolder parseResources(List resources, String version, long r final String name = name(unpackedMessage); final O resourceUpdate; try { - resourceUpdate = parse(unpackedMessage, version, revision); + extensionRegistry.assertValid(unpackedMessage); + resourceUpdate = parse(unpackedMessage, extensionRegistry, version); } catch (Exception e) { invalidResources.put(name, e); continue; @@ -63,6 +67,39 @@ ParsedResourcesHolder parseResources(List resources, String version, long r invalidResources.buildKeepingLast()); } + ParsedResourcesHolder parseDeltaResources(List resources, + XdsExtensionRegistry extensionRegistry) { + final ImmutableMap.Builder parsedResources = ImmutableMap.builder(); + final ImmutableMap.Builder invalidResources = ImmutableMap.builder(); + + for (int i = 0; i < resources.size(); i++) { + final Resource resource = resources.get(i); + + final I unpackedMessage; + try { + unpackedMessage = resource.getResource().unpack(clazz()); + } catch (Exception e) { + final String genName = String.format("generated_%s_%s", i, clazz().getSimpleName()); + invalidResources.put(genName, e); + continue; + } + final String name = resource.getName(); + final O resourceUpdate; + try { + extensionRegistry.assertValid(unpackedMessage); + resourceUpdate = parse(unpackedMessage, extensionRegistry, resource.getVersion()); + } catch (Exception e) { + invalidResources.put(name, e); + continue; + } + + parsedResources.put(name, resourceUpdate); + } + + return new ParsedResourcesHolder(parsedResources.buildKeepingLast(), + invalidResources.buildKeepingLast()); + } + // Do not confuse with the SotW approach: it is the mechanism in which the client must specify all // resource names it is interested in with each request. Different resource types may behave // differently in this approach. For LDS and CDS resources, the server must return all resources diff --git a/xds/src/main/java/com/linecorp/armeria/xds/ResourceStateStore.java b/xds/src/main/java/com/linecorp/armeria/xds/ResourceStateStore.java new file mode 100644 index 00000000000..94b4adb2c48 --- /dev/null +++ b/xds/src/main/java/com/linecorp/armeria/xds/ResourceStateStore.java @@ -0,0 +1,168 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import com.linecorp.armeria.common.annotation.Nullable; + +final class ResourceStateStore { + + private final Map> states = new EnumMap<>(XdsType.class); + + @Nullable + ResourceState state(XdsType type, String resourceName) { + final Map perType = states.get(type); + if (perType == null) { + return null; + } + return perType.get(resourceName); + } + + ImmutableSet activeResources(XdsType type) { + final Map perType = states.get(type); + if (perType == null) { + return ImmutableSet.of(); + } + final ImmutableSet.Builder names = ImmutableSet.builder(); + for (Map.Entry entry : perType.entrySet()) { + if (entry.getValue().status != ResourceStatus.ABSENT) { + names.add(entry.getKey()); + } + } + return names.build(); + } + + ImmutableMap resourceVersions(XdsType type) { + final Map perType = states.get(type); + if (perType == null) { + return ImmutableMap.of(); + } + final ImmutableMap.Builder versions = ImmutableMap.builder(); + for (Map.Entry entry : perType.entrySet()) { + final ResourceState state = entry.getValue(); + if (state.status == ResourceStatus.VERSIONED && state.resource != null) { + versions.put(entry.getKey(), state.resource.version()); + } + } + return versions.build(); + } + + void putWaiting(XdsType type, String resourceName) { + statesFor(type).put(resourceName, ResourceState.waiting()); + } + + @Nullable + XdsResource putVersioned(XdsType type, String resourceName, XdsResource resource) { + final ResourceState prev = state(type, resourceName); + if (isDuplicateEntry(resource, prev)) { + return null; + } + final long revision = prev != null ? prev.revision + 1 : 1; + final XdsResource revised = resource instanceof AbstractXdsResource ? + ((AbstractXdsResource) resource).withRevision(revision) : resource; + statesFor(type).put(resourceName, new ResourceState(ResourceStatus.VERSIONED, revised, revision)); + return revised; + } + + private boolean isDuplicateEntry(XdsResource resource, @Nullable ResourceState prev) { + return prev != null && prev.status == ResourceStatus.VERSIONED && prev.resource != null && + Objects.equals(prev.resource.version(), resource.version()) && + prev.resource.resource().equals(resource.resource()); + } + + boolean putAbsent(XdsType type, String resourceName) { + final ResourceState prev = state(type, resourceName); + if (prev != null && prev.status == ResourceStatus.ABSENT) { + return false; + } + statesFor(type).put(resourceName, ResourceState.absent()); + return true; + } + + void removeIfWaiting(XdsType type, String resourceName) { + final Map perType = states.get(type); + if (perType == null) { + return; + } + final ResourceState state = perType.get(resourceName); + if (state == null || state.status != ResourceStatus.WAITING_FOR_SERVER) { + return; + } + perType.remove(resourceName); + if (perType.isEmpty()) { + states.remove(type); + } + } + + void remove(XdsType type, String resourceName) { + final Map perType = states.get(type); + if (perType == null) { + return; + } + perType.remove(resourceName); + if (perType.isEmpty()) { + states.remove(type); + } + } + + private Map statesFor(XdsType type) { + return states.computeIfAbsent(type, key -> new HashMap<>()); + } + + enum ResourceStatus { + WAITING_FOR_SERVER, + VERSIONED, + ABSENT + } + + static final class ResourceState { + private final ResourceStatus status; + @Nullable + private final XdsResource resource; + private final long revision; + + private ResourceState(ResourceStatus status, @Nullable XdsResource resource, long revision) { + this.status = status; + this.resource = resource; + this.revision = revision; + } + + ResourceStatus status() { + return status; + } + + @Nullable + XdsResource resource() { + return resource; + } + + private static ResourceState waiting() { + return new ResourceState(ResourceStatus.WAITING_FOR_SERVER, null, 0); + } + + private static ResourceState absent() { + return new ResourceState(ResourceStatus.ABSENT, null, 0); + } + } +} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/RouteEntry.java b/xds/src/main/java/com/linecorp/armeria/xds/RouteEntry.java index b3e0811212e..42a5c975a0f 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/RouteEntry.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/RouteEntry.java @@ -56,7 +56,7 @@ public final class RouteEntry { RouteEntry(Route route, @Nullable ClusterSnapshot clusterSnapshot, int index, @Nullable ListenerXdsResource listenerResource, RouteXdsResource routeResource, - VirtualHostXdsResource vhostResource) { + VirtualHostXdsResource vhostResource, XdsExtensionRegistry extensionRegistry) { this.route = route; this.clusterSnapshot = clusterSnapshot; this.index = index; @@ -86,7 +86,7 @@ public final class RouteEntry { final RetryPolicy effectiveRetryPolicy = retryPolicy == RetryPolicy.getDefaultInstance() ? null : retryPolicy; final ClientDecoration clientDecoration = FilterUtil.buildUpstreamFilter( - upstreamFilters, filterConfigs, effectiveRetryPolicy); + extensionRegistry, upstreamFilters, filterConfigs, effectiveRetryPolicy); httpClient = clientDecoration.decorate(DelegatingHttpClient.of()); rpcClient = clientDecoration.rpcDecorate(DelegatingRpcClient.of()); } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/RouteResourceParser.java b/xds/src/main/java/com/linecorp/armeria/xds/RouteResourceParser.java index 4482f1d2d72..4abc3759c74 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/RouteResourceParser.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/RouteResourceParser.java @@ -20,13 +20,14 @@ final class RouteResourceParser extends ResourceParser { - public static final RouteResourceParser INSTANCE = new RouteResourceParser(); + static final RouteResourceParser INSTANCE = new RouteResourceParser(); private RouteResourceParser() {} @Override - RouteXdsResource parse(RouteConfiguration message, String version, long revision) { - return new RouteXdsResource(message, version, revision); + RouteXdsResource parse(RouteConfiguration message, XdsExtensionRegistry registry, + String version) { + return new RouteXdsResource(message, version); } @Override diff --git a/xds/src/main/java/com/linecorp/armeria/xds/RouteStream.java b/xds/src/main/java/com/linecorp/armeria/xds/RouteStream.java index 5ffee74f89a..4d96116769c 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/RouteStream.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/RouteStream.java @@ -163,9 +163,12 @@ private static class RouteEntryStream extends RefCountedStream { @Override protected Subscription onStart(SnapshotWatcher watcher) { + final XdsExtensionRegistry extensionRegistry = + context.extensionRegistry(); if (!route.getRoute().hasCluster()) { return SnapshotStream.just(new RouteEntry(route, null, index, - listenerResource, routeResource, vhostResource)) + listenerResource, routeResource, vhostResource, + extensionRegistry)) .subscribe(watcher); } final SnapshotWatcher mapped = (snapshot, t) -> { @@ -174,7 +177,8 @@ protected Subscription onStart(SnapshotWatcher watcher) { return; } watcher.onUpdate(new RouteEntry(route, snapshot, index, - listenerResource, routeResource, vhostResource), null); + listenerResource, routeResource, vhostResource, + extensionRegistry), null); }; return context.clusterManager().register(clusterName, context, mapped); } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/RouteXdsResource.java b/xds/src/main/java/com/linecorp/armeria/xds/RouteXdsResource.java index dbf61bdd91c..38d2b33091d 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/RouteXdsResource.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/RouteXdsResource.java @@ -28,14 +28,17 @@ public final class RouteXdsResource extends AbstractXdsResource { private final RouteConfiguration routeConfiguration; - RouteXdsResource(RouteConfiguration routeConfiguration, String version, long revision) { + RouteXdsResource(RouteConfiguration routeConfiguration, String version) { + this(routeConfiguration, version, 0); + } + + private RouteXdsResource(RouteConfiguration routeConfiguration, String version, long revision) { super(version, revision); - XdsValidatorIndexRegistry.assertValid(routeConfiguration); this.routeConfiguration = routeConfiguration; } RouteXdsResource(RouteConfiguration routeConfiguration) { - this(routeConfiguration, "", 0); + this(routeConfiguration, ""); } @Override @@ -52,4 +55,12 @@ public RouteConfiguration resource() { public String name() { return routeConfiguration.getName(); } + + @Override + RouteXdsResource withRevision(long revision) { + if (revision == revision()) { + return this; + } + return new RouteXdsResource(routeConfiguration, version(), revision); + } } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/SecretResourceParser.java b/xds/src/main/java/com/linecorp/armeria/xds/SecretResourceParser.java index 92b45eca98f..420daedc613 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/SecretResourceParser.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/SecretResourceParser.java @@ -35,8 +35,8 @@ Class clazz() { } @Override - SecretXdsResource parse(Secret message, String version, long revision) { - return new SecretXdsResource(message, version, revision); + SecretXdsResource parse(Secret message, XdsExtensionRegistry registry, String version) { + return new SecretXdsResource(message, version); } @Override diff --git a/xds/src/main/java/com/linecorp/armeria/xds/SecretXdsResource.java b/xds/src/main/java/com/linecorp/armeria/xds/SecretXdsResource.java index 61bfaeb5f11..2b61fa73b39 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/SecretXdsResource.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/SecretXdsResource.java @@ -26,21 +26,21 @@ * used for securing transport sockets in xDS configurations. */ @UnstableApi -public final class SecretXdsResource implements XdsResource { +public final class SecretXdsResource extends AbstractXdsResource { private final Secret secret; - private final String version; - private final long revision; SecretXdsResource(Secret secret) { - this(secret, "", 0); + this(secret, ""); } - SecretXdsResource(Secret secret, String version, long revision) { - XdsValidatorIndexRegistry.assertValid(secret); + SecretXdsResource(Secret secret, String version) { + this(secret, version, 0); + } + + private SecretXdsResource(Secret secret, String version, long revision) { + super(version, revision); this.secret = secret; - this.version = version; - this.revision = revision; } @Override @@ -59,12 +59,10 @@ public String name() { } @Override - public String version() { - return version; - } - - @Override - public long revision() { - return revision; + SecretXdsResource withRevision(long revision) { + if (revision == revision()) { + return this; + } + return new SecretXdsResource(secret, version(), revision); } } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/SotwActualStream.java b/xds/src/main/java/com/linecorp/armeria/xds/SotwActualStream.java new file mode 100644 index 00000000000..14bec48485b --- /dev/null +++ b/xds/src/main/java/com/linecorp/armeria/xds/SotwActualStream.java @@ -0,0 +1,215 @@ +/* + * Copyright 2023 LINE Corporation + * + * LINE Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import static com.linecorp.armeria.xds.XdsResourceParserUtil.fromTypeUrl; + +import java.util.Collection; +import java.util.EnumMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.rpc.Code; +import com.google.rpc.Status; + +import com.linecorp.armeria.common.annotation.Nullable; + +import io.envoyproxy.envoy.config.core.v3.Node; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest.Builder; +import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; +import io.grpc.stub.StreamObserver; +import io.netty.util.concurrent.EventExecutor; + +final class SotwActualStream implements StreamObserver, AdsXdsStream.ActualStream { + + private static final Logger logger = LoggerFactory.getLogger(SotwActualStream.class); + + // NACK backoff to prevent hot loops when the server keeps sending bad responses + static final long NACK_BACKOFF_MILLIS = 3_000L; + + private final StreamObserver requestObserver; + private final AdsXdsStream owner; + private final StateCoordinator stateCoordinator; + private final EventExecutor eventLoop; + private final ConfigSourceLifecycleObserver lifecycleObserver; + private final Node node; + + private final Map noncesMap = new EnumMap<>(XdsType.class); + private final Map lastAckedVersions = new EnumMap<>(XdsType.class); + private boolean completed; + + SotwActualStream(SotwDiscoveryStub stub, AdsXdsStream owner, + StateCoordinator stateCoordinator, + EventExecutor eventLoop, ConfigSourceLifecycleObserver lifecycleObserver, + Node node) { + this.owner = owner; + this.stateCoordinator = stateCoordinator; + this.eventLoop = eventLoop; + this.lifecycleObserver = lifecycleObserver; + this.node = node; + requestObserver = stub.stream(this); + lifecycleObserver.streamOpened(); + } + + void ackResponse(XdsType type, String versionInfo, String nonce) { + noncesMap.put(type, nonce); + lastAckedVersions.put(type, versionInfo); + sendDiscoveryRequest(type, versionInfo, stateCoordinator.interestedResources(type), + nonce, null); + } + + void nackResponse(XdsType type, String nonce, String errorDetail) { + noncesMap.put(type, nonce); + eventLoop.schedule(() -> sendDiscoveryRequest(type, lastAckedVersions.get(type), + stateCoordinator.interestedResources(type), nonce, + errorDetail), + NACK_BACKOFF_MILLIS, TimeUnit.MILLISECONDS); + } + + @Override + public void closeStream() { + if (completed) { + return; + } + completed = true; + requestObserver.onCompleted(); + } + + @Override + public void resourcesUpdated(XdsType type) { + sendDiscoveryRequest(type); + } + + private void sendDiscoveryRequest(XdsType type) { + sendDiscoveryRequest(type, lastAckedVersions.get(type), + stateCoordinator.interestedResources(type), noncesMap.get(type), null); + } + + private void sendDiscoveryRequest(XdsType type, @Nullable String version, Collection resources, + @Nullable String nonce, @Nullable String errorDetail) { + if (completed) { + return; + } + final Builder builder = DiscoveryRequest.newBuilder() + .setTypeUrl(type.typeUrl()) + .setNode(node) + .addAllResourceNames(resources); + if (version != null) { + builder.setVersionInfo(version); + } + if (nonce != null) { + builder.setResponseNonce(nonce); + } + if (errorDetail != null) { + builder.setErrorDetail(Status.newBuilder() + .setCode(Code.INVALID_ARGUMENT_VALUE) + .setMessage(errorDetail) + .build()); + } + final DiscoveryRequest request = builder.build(); + lifecycleObserver.requestSent(request); + requestObserver.onNext(request); + } + + @Override + public void onNext(DiscoveryResponse value) { + if (!eventLoop.inEventLoop()) { + eventLoop.execute(() -> onNext(value)); + return; + } + if (completed) { + return; + } + lifecycleObserver.responseReceived(value); + + final ResourceParser resourceParser = fromTypeUrl(value.getTypeUrl()); + if (resourceParser == null) { + logger.warn("XDS stream Received unexpected type: {}", value.getTypeUrl()); + return; + } + handleResponse(resourceParser, value); + } + + @Override + public void onError(Throwable throwable) { + if (!eventLoop.inEventLoop()) { + eventLoop.execute(() -> onError(throwable)); + return; + } + completed = true; + lifecycleObserver.streamError(throwable); + owner.retryOrClose(true); + } + + @Override + public void onCompleted() { + if (!eventLoop.inEventLoop()) { + eventLoop.execute(this::onCompleted); + return; + } + completed = true; + lifecycleObserver.streamCompleted(); + owner.retryOrClose(false); + } + + private void handleResponse(ResourceParser resourceParser, DiscoveryResponse response) { + final ParsedResourcesHolder holder = + resourceParser.parseResources(response.getResourcesList(), + stateCoordinator.extensionRegistry(), response.getVersionInfo()); + + if (!holder.errors().isEmpty()) { + holder.invalidResources().forEach((name, error) -> stateCoordinator.onResourceError( + resourceParser.type(), name, error)); + lifecycleObserver.resourceRejected(resourceParser.type(), response, holder.invalidResources()); + + nackResponse(resourceParser.type(), response.getNonce(), + String.join("\n", holder.errors())); + return; + } + + // first save data + holder.parsedResources().forEach((name, resource) -> { + if (resource instanceof XdsResource) { + stateCoordinator.onResourceUpdated(resourceParser.type(), name, (XdsResource) resource); + } + }); + + final boolean fullStateOfTheWorld = resourceParser.isFullStateOfTheWorld(); + if (fullStateOfTheWorld && + (resourceParser.type() == XdsType.LISTENER || resourceParser.type() == XdsType.CLUSTER)) { + final Set currentSubscribers = + stateCoordinator.interestedResources(resourceParser.type()); + if (!holder.parsedResources().isEmpty() || !currentSubscribers.isEmpty()) { + for (String name : currentSubscribers) { + if (holder.parsedResources().containsKey(name) || + holder.invalidResources().containsKey(name)) { + continue; + } + stateCoordinator.onResourceMissing(resourceParser.type(), name); + } + } + } + lifecycleObserver.resourceUpdated(resourceParser.type(), response, holder.parsedResources()); + // send the ack + ackResponse(resourceParser.type(), response.getVersionInfo(), response.getNonce()); + } +} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/SotwXdsStream.java b/xds/src/main/java/com/linecorp/armeria/xds/SotwXdsStream.java deleted file mode 100644 index e1046310c61..00000000000 --- a/xds/src/main/java/com/linecorp/armeria/xds/SotwXdsStream.java +++ /dev/null @@ -1,374 +0,0 @@ -/* - * Copyright 2023 LINE Corporation - * - * LINE Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.xds; - -import static com.linecorp.armeria.xds.XdsResourceParserUtil.fromTypeUrl; -import static java.util.Objects.requireNonNull; - -import java.util.Collection; -import java.util.EnumMap; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.MoreObjects; -import com.google.common.base.Objects; -import com.google.rpc.Code; - -import com.linecorp.armeria.client.retry.Backoff; -import com.linecorp.armeria.common.annotation.Nullable; - -import io.envoyproxy.envoy.config.core.v3.Node; -import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest; -import io.envoyproxy.envoy.service.discovery.v3.DiscoveryRequest.Builder; -import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; -import io.grpc.Status; -import io.grpc.stub.StreamObserver; -import io.netty.util.concurrent.EventExecutor; - -final class SotwXdsStream implements XdsStream, XdsStreamState { - - private static final Logger logger = LoggerFactory.getLogger(SotwXdsStream.class); - - private final VersionManager versionManager = new VersionManager(); - private final SotwDiscoveryStub stub; - private final Node node; - private final Backoff backoff; - private final EventExecutor eventLoop; - private final XdsResponseHandler responseHandler; - private final SubscriberStorage subscriberStorage; - private int connBackoffAttempts = 1; - - // whether the stream is stopped explicitly by the user - private boolean stopped; - @Nullable - @VisibleForTesting - ActualStream actualStream; - - private final Set targetTypes; - private final ConfigSourceLifecycleObserver lifecycleObserver; - - SotwXdsStream(SotwDiscoveryStub stub, Node node, Backoff backoff, - EventExecutor eventLoop, XdsResponseHandler responseHandler, - SubscriberStorage subscriberStorage, - ConfigSourceLifecycleObserver lifecycleObserver) { - this(stub, node, backoff, eventLoop, responseHandler, subscriberStorage, - XdsType.discoverableTypes(), lifecycleObserver); - } - - SotwXdsStream(SotwDiscoveryStub stub, Node node, Backoff backoff, - EventExecutor eventLoop, XdsResponseHandler responseHandler, - SubscriberStorage subscriberStorage, Set targetTypes, - ConfigSourceLifecycleObserver lifecycleObserver) { - this.stub = requireNonNull(stub, "stub"); - this.node = requireNonNull(node, "node"); - this.backoff = requireNonNull(backoff, "backoff"); - this.eventLoop = requireNonNull(eventLoop, "eventLoop"); - this.responseHandler = requireNonNull(responseHandler, "responseHandler"); - this.subscriberStorage = requireNonNull(subscriberStorage, "subscriberStorage"); - this.targetTypes = targetTypes; - this.lifecycleObserver = requireNonNull(lifecycleObserver, "lifecycleObserver"); - } - - @VisibleForTesting - void start() { - if (!eventLoop.inEventLoop()) { - eventLoop.execute(this::start); - return; - } - stopped = false; - reset(); - } - - private void reset() { - if (stopped) { - return; - } - - for (XdsType targetType : targetTypes) { - // check the resource type actually has subscriptions. - // otherwise, an unintentional onMissing callback may be received - if (!subscriberStorage.resources(targetType).isEmpty()) { - resourcesUpdated(targetType); - } - } - } - - void stop() { - stop(Status.CANCELLED.withDescription("shutdown").asException()); - } - - void stop(Throwable throwable) { - requireNonNull(throwable, "throwable"); - if (!eventLoop.inEventLoop()) { - eventLoop.execute(() -> stop(throwable)); - return; - } - stopped = true; - if (actualStream == null) { - return; - } - actualStream.closeStream(); - actualStream = null; - } - - @Override - public void close() { - stop(); - lifecycleObserver.close(); - } - - @Override - public void resourcesUpdated(XdsType type) { - actualStream().sendDiscoveryRequest(type); - } - - private ActualStream actualStream() { - if (actualStream == null) { - actualStream = new ActualStream(stub, this, versionManager, eventLoop, - lifecycleObserver, responseHandler, backoff, node); - } - return actualStream; - } - - @Override - public void retryOrClose(boolean closedByError) { - if (!eventLoop.inEventLoop()) { - eventLoop.execute(() -> retryOrClose(closedByError)); - return; - } - if (stopped) { - // don't reschedule automatically since the user explicitly closed the stream - return; - } - actualStream = null; - // wait backoff - if (closedByError) { - connBackoffAttempts++; - } else { - connBackoffAttempts = 1; - } - final long nextDelayMillis = backoff.nextDelayMillis(connBackoffAttempts); - if (nextDelayMillis < 0) { - return; - } - eventLoop.schedule(this::reset, nextDelayMillis, TimeUnit.MILLISECONDS); - } - - @Override - public Collection watchedResources(XdsType type) { - return subscriberStorage.resources(type); - } - - static class ActualStream implements StreamObserver { - - private final StreamObserver requestObserver; - private final XdsStreamState xdsStreamState; - private final VersionManager versionManager; - private final EventExecutor eventLoop; - private final ConfigSourceLifecycleObserver lifecycleObserver; - private final XdsResponseHandler responseHandler; - private final Backoff backoff; - private final Node node; - - private int ackBackoffAttempts; - private final Map noncesMap = new EnumMap<>(XdsType.class); - boolean completed; - - ActualStream(SotwDiscoveryStub stub, XdsStreamState xdsStreamState, VersionManager versionManager, - EventExecutor eventLoop, ConfigSourceLifecycleObserver lifecycleObserver, - XdsResponseHandler responseHandler, Backoff backoff, Node node) { - this.xdsStreamState = xdsStreamState; - this.versionManager = versionManager; - this.eventLoop = eventLoop; - this.lifecycleObserver = lifecycleObserver; - this.responseHandler = responseHandler; - this.backoff = backoff; - this.node = node; - requestObserver = stub.stream(this); - lifecycleObserver.streamOpened(); - } - - void ackResponse(XdsType type, String versionInfo, String nonce) { - ackBackoffAttempts = 0; - versionManager.updateVersion(type, versionInfo); - sendDiscoveryRequest(type, versionInfo, xdsStreamState.watchedResources(type), - nonce, null); - } - - void nackResponse(XdsType type, String nonce, String errorDetail) { - ackBackoffAttempts++; - eventLoop.schedule(() -> sendDiscoveryRequest(type, versionManager.getVersion(type), - xdsStreamState.watchedResources(type), nonce, - errorDetail), - backoff.nextDelayMillis(ackBackoffAttempts), TimeUnit.MILLISECONDS); - } - - VersionManager versionManager() { - return versionManager; - } - - void closeStream() { - if (completed) { - return; - } - completed = true; - requestObserver.onCompleted(); - } - - void sendDiscoveryRequest(XdsType type) { - sendDiscoveryRequest(type, versionManager.getVersion(type), - xdsStreamState.watchedResources(type), noncesMap.get(type), null); - } - - private void sendDiscoveryRequest(XdsType type, @Nullable String version, Collection resources, - @Nullable String nonce, @Nullable String errorDetail) { - if (completed) { - return; - } - final Builder builder = DiscoveryRequest.newBuilder() - .setTypeUrl(type.typeUrl()) - .setNode(node) - .addAllResourceNames(resources); - if (version != null) { - builder.setVersionInfo(version); - } - if (nonce != null) { - builder.setResponseNonce(nonce); - } - if (errorDetail != null) { - builder.setErrorDetail(com.google.rpc.Status.newBuilder() - .setCode(Code.INVALID_ARGUMENT_VALUE) - .setMessage(errorDetail) - .build()); - } - final DiscoveryRequest request = builder.build(); - lifecycleObserver.requestSent(request); - requestObserver.onNext(request); - } - - @Override - public void onNext(DiscoveryResponse value) { - if (!eventLoop.inEventLoop()) { - eventLoop.execute(() -> onNext(value)); - return; - } - if (completed) { - return; - } - lifecycleObserver.responseReceived(value); - - final ResourceParser resourceParser = fromTypeUrl(value.getTypeUrl()); - if (resourceParser == null) { - logger.warn("XDS stream Received unexpected type: {}", value.getTypeUrl()); - return; - } - noncesMap.put(resourceParser.type(), value.getNonce()); - responseHandler.handleResponse(resourceParser, value, this, lifecycleObserver); - } - - @Override - public void onError(Throwable throwable) { - if (!eventLoop.inEventLoop()) { - eventLoop.execute(() -> onError(throwable)); - return; - } - completed = true; - lifecycleObserver.streamError(throwable); - xdsStreamState.retryOrClose(true); - } - - @Override - public void onCompleted() { - if (!eventLoop.inEventLoop()) { - eventLoop.execute(this::onCompleted); - return; - } - completed = true; - lifecycleObserver.streamCompleted(); - xdsStreamState.retryOrClose(false); - } - } - - static class VersionManager { - - private final Map versionMap = new EnumMap<>(XdsType.class); - - void updateVersion(XdsType type, String version) { - final VersionInfo prevVersion = versionMap.get(type); - if (prevVersion != null && prevVersion.version.equals(version)) { - return; - } - final long revision = prevVersion != null ? prevVersion.revision + 1 : 1; - versionMap.put(type, new VersionInfo(version, revision)); - } - - @Nullable - String getVersion(XdsType type) { - final VersionInfo versionInfo = versionMap.get(type); - if (versionInfo == null) { - return null; - } - return versionInfo.version; - } - - long nextRevision(XdsType type, String version) { - final VersionInfo prevVersion = versionMap.get(type); - if (prevVersion != null && Objects.equal(prevVersion.version, version)) { - return prevVersion.revision; - } - return prevVersion != null ? prevVersion.revision + 1 : 1; - } - - private static final class VersionInfo { - - private final String version; - private final long revision; - - private VersionInfo(String version, long revision) { - this.version = version; - this.revision = revision; - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) { - return false; - } - final VersionInfo that = (VersionInfo) o; - return revision == that.revision && Objects.equal(version, that.version); - } - - @Override - public int hashCode() { - return Objects.hashCode(version, revision); - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("version", version) - .add("revision", revision) - .toString(); - } - } - } -} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/StateCoordinator.java b/xds/src/main/java/com/linecorp/armeria/xds/StateCoordinator.java new file mode 100644 index 00000000000..e26f0bb316b --- /dev/null +++ b/xds/src/main/java/com/linecorp/armeria/xds/StateCoordinator.java @@ -0,0 +1,145 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import java.util.EnumSet; +import java.util.Set; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.common.util.SafeCloseable; + +import io.netty.util.concurrent.EventExecutor; + +final class StateCoordinator implements SafeCloseable { + + private static final Set FULL_SOTW_TYPES = EnumSet.of(XdsType.LISTENER, XdsType.CLUSTER); + + private final SubscriberStorage subscriberStorage; + private final ResourceStateStore stateStore; + private final boolean delta; + private final XdsExtensionRegistry extensionRegistry; + + StateCoordinator(EventExecutor eventLoop, long timeoutMillis, boolean delta, + XdsExtensionRegistry extensionRegistry) { + this.delta = delta; + this.extensionRegistry = extensionRegistry; + subscriberStorage = new SubscriberStorage(eventLoop, timeoutMillis, delta); + stateStore = new ResourceStateStore(); + } + + XdsExtensionRegistry extensionRegistry() { + return extensionRegistry; + } + + boolean register(XdsType type, String resourceName, ResourceWatcher watcher) { + final boolean updated = subscriberStorage.register(type, resourceName, watcher); + replayToWatcher(type, resourceName, watcher); + return updated; + } + + boolean unregister(XdsType type, String resourceName, ResourceWatcher watcher) { + final boolean removed = subscriberStorage.unregister(type, resourceName, watcher); + if (removed) { + if (!delta && !FULL_SOTW_TYPES.contains(type)) { + // For sotw, we don't receive missing signals for non-sotw types, + // so removal is done when no subscribers are left. + stateStore.remove(type, resourceName); + } else { + stateStore.removeIfWaiting(type, resourceName); + } + } + return removed; + } + + ImmutableSet interestedResources(XdsType type) { + return subscriberStorage.resources(type); + } + + boolean hasNoSubscribers() { + return subscriberStorage.hasNoSubscribers(); + } + + ImmutableSet activeResources(XdsType type) { + return stateStore.activeResources(type); + } + + ImmutableMap resourceVersions(XdsType type) { + return stateStore.resourceVersions(type); + } + + void onResourceUpdated(XdsType type, String resourceName, XdsResource resource) { + final XdsResource revised = stateStore.putVersioned(type, resourceName, resource); + if (revised == null) { + return; + } + final XdsStreamSubscriber subscriber = subscriber(type, resourceName); + if (subscriber != null) { + subscriber.onData(revised); + } + } + + void onResourceMissing(XdsType type, String resourceName) { + if (!stateStore.putAbsent(type, resourceName)) { + return; + } + final XdsStreamSubscriber subscriber = subscriber(type, resourceName); + if (subscriber != null) { + subscriber.onAbsent(); + } + } + + void onResourceError(XdsType type, String resourceName, Throwable cause) { + final XdsStreamSubscriber subscriber = subscriber(type, resourceName); + if (subscriber != null) { + subscriber.onError(resourceName, cause); + } + } + + @Nullable + private XdsStreamSubscriber subscriber(XdsType type, String resourceName) { + return subscriberStorage.subscriber(type, resourceName); + } + + private void replayToWatcher(XdsType type, String resourceName, + ResourceWatcher watcher) { + final ResourceStateStore.ResourceState state = stateStore.state(type, resourceName); + if (state == null) { + stateStore.putWaiting(type, resourceName); + return; + } + switch (state.status()) { + case VERSIONED: + assert state.resource() != null; + //noinspection unchecked + watcher.onChanged((T) state.resource()); + break; + case ABSENT: + watcher.onResourceDoesNotExist(type, resourceName); + break; + case WAITING_FOR_SERVER: + break; + } + } + + @Override + public void close() { + subscriberStorage.close(); + } +} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/SubscriberStorage.java b/xds/src/main/java/com/linecorp/armeria/xds/SubscriberStorage.java index fc030d61e83..e5806a9a471 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/SubscriberStorage.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/SubscriberStorage.java @@ -16,12 +16,14 @@ package com.linecorp.armeria.xds; -import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.Map; -import java.util.Set; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.util.SafeCloseable; import io.netty.util.concurrent.EventExecutor; @@ -30,13 +32,14 @@ final class SubscriberStorage implements SafeCloseable { private final EventExecutor eventLoop; private final long timeoutMillis; - + private final boolean delta; private final Map>> subscriberMap = new EnumMap<>(XdsType.class); - SubscriberStorage(EventExecutor eventLoop, long timeoutMillis) { + SubscriberStorage(EventExecutor eventLoop, long timeoutMillis, boolean delta) { this.eventLoop = eventLoop; this.timeoutMillis = timeoutMillis; + this.delta = delta; } /** @@ -48,7 +51,9 @@ boolean register(XdsType type, String resourceName, Reso type, key -> new HashMap<>()).get(resourceName); boolean updated = false; if (subscriber == null) { - subscriber = new XdsStreamSubscriber<>(type, resourceName, eventLoop, timeoutMillis); + final boolean enableAbsentOnTimeout = !delta && timeoutMillis > 0; + subscriber = new XdsStreamSubscriber<>(type, resourceName, eventLoop, timeoutMillis, + enableAbsentOnTimeout); subscriberMap.get(type).put(resourceName, subscriber); updated = true; } @@ -82,21 +87,23 @@ boolean unregister(XdsType type, String resourceName, Re return false; } - Map> subscribers(XdsType type) { - return unsafeCast(subscriberMap.getOrDefault(type, Collections.emptyMap())); + @Nullable + XdsStreamSubscriber subscriber(XdsType type, String resourceName) { + return unsafeCast(subscriberMap.getOrDefault(type, ImmutableMap.of()).get(resourceName)); } - static T unsafeCast(Object obj) { + @Nullable + private static T unsafeCast(@Nullable Object obj) { //noinspection unchecked return (T) obj; } - Set resources(XdsType type) { - return subscriberMap.getOrDefault(type, Collections.emptyMap()).keySet(); + ImmutableSet resources(XdsType type) { + return ImmutableSet.copyOf(subscriberMap.getOrDefault(type, ImmutableMap.of()).keySet()); } - Map>> allSubscribers() { - return subscriberMap; + boolean hasNoSubscribers() { + return subscriberMap.isEmpty(); } @Override diff --git a/xds/src/main/java/com/linecorp/armeria/xds/SubscriptionContext.java b/xds/src/main/java/com/linecorp/armeria/xds/SubscriptionContext.java index 2e9199d2aa0..bb912ff1ce6 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/SubscriptionContext.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/SubscriptionContext.java @@ -43,4 +43,6 @@ interface SubscriptionContext { BootstrapSecrets bootstrapSecrets(); ResourceNodeMeterBinderFactory meterBinderFactory(); + + XdsExtensionRegistry extensionRegistry(); } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/XdsStreamState.java b/xds/src/main/java/com/linecorp/armeria/xds/TransportSocketFactory.java similarity index 55% rename from xds/src/main/java/com/linecorp/armeria/xds/XdsStreamState.java rename to xds/src/main/java/com/linecorp/armeria/xds/TransportSocketFactory.java index 5842732df57..09ede15b0cf 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/XdsStreamState.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/TransportSocketFactory.java @@ -1,5 +1,5 @@ /* - * Copyright 2025 LY Corporation + * Copyright 2026 LY Corporation * * LY Corporation licenses this file to you under the Apache License, * version 2.0 (the "License"); you may not use this file except in compliance @@ -16,11 +16,14 @@ package com.linecorp.armeria.xds; -import java.util.Collection; +import com.linecorp.armeria.common.annotation.Nullable; -interface XdsStreamState { +import io.envoyproxy.envoy.config.core.v3.ConfigSource; +import io.envoyproxy.envoy.config.core.v3.TransportSocket; - void retryOrClose(boolean closedByError); +interface TransportSocketFactory extends XdsExtensionFactory { - Collection watchedResources(XdsType type); + SnapshotStream create(SubscriptionContext context, + @Nullable ConfigSource configSource, + TransportSocket transportSocket); } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/TransportSocketStream.java b/xds/src/main/java/com/linecorp/armeria/xds/TransportSocketStream.java index bc901956a1c..fe6634047e5 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/TransportSocketStream.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/TransportSocketStream.java @@ -16,18 +16,10 @@ package com.linecorp.armeria.xds; -import java.util.Optional; - import com.linecorp.armeria.common.annotation.Nullable; import io.envoyproxy.envoy.config.core.v3.ConfigSource; import io.envoyproxy.envoy.config.core.v3.TransportSocket; -import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; -import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext.CombinedCertificateValidationContext; -import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.SdsSecretConfig; -import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret; -import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.TlsCertificate; -import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext; final class TransportSocketStream extends RefCountedStream { @@ -45,68 +37,18 @@ final class TransportSocketStream extends RefCountedStream watcher) { - if (!"envoy.transport_sockets.tls".equals(transportSocket.getName())) { + if (transportSocket.equals(TransportSocket.getDefaultInstance())) { return SnapshotStream.just(new TransportSocketSnapshot(transportSocket)) .subscribe(watcher); } - if (!transportSocket.hasTypedConfig()) { - return SnapshotStream.just(new TransportSocketSnapshot(TransportSocket.getDefaultInstance())) - .subscribe(watcher); - } - final UpstreamTlsContext tlsContext = XdsValidatorIndexRegistry.unpack(transportSocket.getTypedConfig(), - UpstreamTlsContext.class); - final CommonTlsContext commonTlsContext = tlsContext.getCommonTlsContext(); - - final SnapshotStream> validationStream; - - if (commonTlsContext.hasValidationContext()) { - final Secret secret = Secret.newBuilder() - .setValidationContext(commonTlsContext.getValidationContext()) - .build(); - final SecretStream secretStream = new SecretStream(secret, context); - validationStream = secretStream - .switchMapEager(resource -> new CertificateValidationContextStream(context, resource)) - .map(Optional::of); - } else if (commonTlsContext.hasValidationContextSdsSecretConfig()) { - final SdsSecretConfig sdsConfig = commonTlsContext.getValidationContextSdsSecretConfig(); - final SecretStream secretStream = new SecretStream(sdsConfig, configSource, context); - validationStream = secretStream - .switchMapEager(resource -> new CertificateValidationContextStream(context, resource)) - .map(Optional::of); - } else if (commonTlsContext.hasCombinedValidationContext()) { - final CombinedCertificateValidationContext combined = - commonTlsContext.getCombinedValidationContext(); - final SdsSecretConfig sdsConfig = combined.getValidationContextSdsSecretConfig(); - final SecretStream secretStream = new SecretStream(sdsConfig, configSource, context); - validationStream = secretStream.switchMapEager(resource -> new CertificateValidationContextStream( - context, resource, combined.getDefaultValidationContext())) - .map(Optional::of); - } else { - validationStream = SnapshotStream.empty(); - } - - final SnapshotStream> tlsCertStream; - if (!commonTlsContext.getTlsCertificatesList().isEmpty()) { - final TlsCertificate tlsCertificate = commonTlsContext.getTlsCertificatesList().get(0); - final Secret secret = Secret.newBuilder().setTlsCertificate(tlsCertificate).build(); - final SecretStream secretStream = new SecretStream(secret, context); - tlsCertStream = secretStream.switchMapEager(resource -> new TlsCertificateStream(context, resource)) - .map(Optional::of); - } else if (!commonTlsContext.getTlsCertificateSdsSecretConfigsList().isEmpty()) { - final SdsSecretConfig sdsConfig = - commonTlsContext.getTlsCertificateSdsSecretConfigsList().get(0); - final SecretStream secretStream = new SecretStream(sdsConfig, configSource, context); - tlsCertStream = secretStream.switchMapEager(resource -> new TlsCertificateStream(context, resource)) - .map(Optional::of); - } else { - // static - tlsCertStream = SnapshotStream.empty(); + final TransportSocketFactory factory = context.extensionRegistry().query( + transportSocket.getTypedConfig(), transportSocket.getName(), + TransportSocketFactory.class); + if (factory == null) { + throw new IllegalArgumentException( + "No TransportSocketFactory registered for transport socket: " + + transportSocket.getName()); } - - final SnapshotStream stream = - SnapshotStream.combineLatest(tlsCertStream, validationStream, (cert, validation) -> { - return new TransportSocketSnapshot(transportSocket, cert, validation); - }); - return stream.subscribe(watcher); + return factory.create(context, configSource, transportSocket).subscribe(watcher); } } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/UpstreamTlsTransportSocketFactory.java b/xds/src/main/java/com/linecorp/armeria/xds/UpstreamTlsTransportSocketFactory.java new file mode 100644 index 00000000000..cc175beccda --- /dev/null +++ b/xds/src/main/java/com/linecorp/armeria/xds/UpstreamTlsTransportSocketFactory.java @@ -0,0 +1,115 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import java.util.List; +import java.util.Optional; + +import com.google.common.collect.ImmutableList; + +import com.linecorp.armeria.common.annotation.Nullable; + +import io.envoyproxy.envoy.config.core.v3.ConfigSource; +import io.envoyproxy.envoy.config.core.v3.TransportSocket; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.CommonTlsContext.CombinedCertificateValidationContext; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.SdsSecretConfig; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.Secret; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.TlsCertificate; +import io.envoyproxy.envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext; + +final class UpstreamTlsTransportSocketFactory implements TransportSocketFactory { + + static final UpstreamTlsTransportSocketFactory INSTANCE = new UpstreamTlsTransportSocketFactory(); + private static final String NAME = "envoy.transport_sockets.tls"; + private static final String TYPE_URL = + "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext"; + + private UpstreamTlsTransportSocketFactory() {} + + @Override + public String name() { + return NAME; + } + + @Override + public List typeUrls() { + return ImmutableList.of(TYPE_URL); + } + + @Override + public SnapshotStream create( + SubscriptionContext context, @Nullable ConfigSource configSource, + TransportSocket transportSocket) { + if (!transportSocket.hasTypedConfig()) { + return SnapshotStream.just(new TransportSocketSnapshot(TransportSocket.getDefaultInstance())); + } + final UpstreamTlsContext tlsContext = context.extensionRegistry().unpack( + transportSocket.getTypedConfig(), UpstreamTlsContext.class); + final CommonTlsContext commonTlsContext = tlsContext.getCommonTlsContext(); + + final SnapshotStream> validationStream; + + if (commonTlsContext.hasValidationContext()) { + final Secret secret = Secret.newBuilder() + .setValidationContext(commonTlsContext.getValidationContext()) + .build(); + final SecretStream secretStream = new SecretStream(secret, context); + validationStream = secretStream + .switchMapEager(resource -> new CertificateValidationContextStream(context, resource)) + .map(Optional::of); + } else if (commonTlsContext.hasValidationContextSdsSecretConfig()) { + final SdsSecretConfig sdsConfig = commonTlsContext.getValidationContextSdsSecretConfig(); + final SecretStream secretStream = new SecretStream(sdsConfig, configSource, context); + validationStream = secretStream + .switchMapEager(resource -> new CertificateValidationContextStream(context, resource)) + .map(Optional::of); + } else if (commonTlsContext.hasCombinedValidationContext()) { + final CombinedCertificateValidationContext combined = + commonTlsContext.getCombinedValidationContext(); + final SdsSecretConfig sdsConfig = combined.getValidationContextSdsSecretConfig(); + final SecretStream secretStream = new SecretStream(sdsConfig, configSource, context); + validationStream = secretStream.switchMapEager(resource -> new CertificateValidationContextStream( + context, resource, combined.getDefaultValidationContext())) + .map(Optional::of); + } else { + validationStream = SnapshotStream.empty(); + } + + final SnapshotStream> tlsCertStream; + if (!commonTlsContext.getTlsCertificatesList().isEmpty()) { + final TlsCertificate tlsCertificate = commonTlsContext.getTlsCertificatesList().get(0); + final Secret secret = Secret.newBuilder().setTlsCertificate(tlsCertificate).build(); + final SecretStream secretStream = new SecretStream(secret, context); + tlsCertStream = secretStream.switchMapEager(resource -> new TlsCertificateStream(context, resource)) + .map(Optional::of); + } else if (!commonTlsContext.getTlsCertificateSdsSecretConfigsList().isEmpty()) { + final SdsSecretConfig sdsConfig = + commonTlsContext.getTlsCertificateSdsSecretConfigsList().get(0); + final SecretStream secretStream = new SecretStream(sdsConfig, configSource, context); + tlsCertStream = secretStream.switchMapEager(resource -> new TlsCertificateStream(context, resource)) + .map(Optional::of); + } else { + // static + tlsCertStream = SnapshotStream.empty(); + } + + return SnapshotStream.combineLatest(tlsCertStream, validationStream, (cert, validation) -> { + return new TransportSocketSnapshot(transportSocket, cert, validation); + }); + } +} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/VirtualHostXdsResource.java b/xds/src/main/java/com/linecorp/armeria/xds/VirtualHostXdsResource.java index 12200bcb43e..cb6e5900005 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/VirtualHostXdsResource.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/VirtualHostXdsResource.java @@ -25,9 +25,12 @@ public final class VirtualHostXdsResource extends AbstractXdsResource { private final VirtualHost virtualHost; + VirtualHostXdsResource(VirtualHost virtualHost, String version) { + this(virtualHost, version, 0); + } + VirtualHostXdsResource(VirtualHost virtualHost, String version, long revision) { super(version, revision); - XdsValidatorIndexRegistry.assertValid(virtualHost); this.virtualHost = virtualHost; } @@ -45,4 +48,12 @@ public VirtualHost resource() { public String name() { return virtualHost.getName(); } + + @Override + VirtualHostXdsResource withRevision(long revision) { + if (revision == revision()) { + return this; + } + return new VirtualHostXdsResource(virtualHost, version(), revision); + } } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/XdsBootstrapImpl.java b/xds/src/main/java/com/linecorp/armeria/xds/XdsBootstrapImpl.java index 9b6204195f4..4b40dfecbae 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/XdsBootstrapImpl.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/XdsBootstrapImpl.java @@ -47,8 +47,11 @@ final class XdsBootstrapImpl implements XdsBootstrap { this.bootstrap = bootstrap; this.defaultWatcher = defaultWatcher; this.eventLoop = requireNonNull(eventLoop, "eventLoop"); - clusterManager = new XdsClusterManager(eventLoop, bootstrap, meterIdPrefix, meterRegistry); watchService = new DirectoryWatchService(); + final XdsResourceValidator resourceValidator = new XdsResourceValidator(); + final XdsExtensionRegistry extensionRegistry = XdsExtensionRegistry.of(resourceValidator); + extensionRegistry.assertValid(bootstrap); + clusterManager = new XdsClusterManager(eventLoop, bootstrap, meterIdPrefix, meterRegistry); final BootstrapClusters bootstrapClusters = new BootstrapClusters(bootstrap, clusterManager, defaultWatcher); final BootstrapSecrets bootstrapSecrets = new BootstrapSecrets(bootstrap); @@ -56,10 +59,10 @@ final class XdsBootstrapImpl implements XdsBootstrap { final ConfigSourceMapper configSourceMapper = new ConfigSourceMapper(bootstrap); controlPlaneClientManager = new ControlPlaneClientManager( bootstrap, eventLoop, bootstrapClusters, - configSourceMapper, meterRegistry, meterIdPrefix); + configSourceMapper, meterRegistry, meterIdPrefix, extensionRegistry); subscriptionContext = new DefaultSubscriptionContext( eventLoop, clusterManager, configSourceMapper, controlPlaneClientManager, - meterRegistry, meterIdPrefix, watchService, bootstrapSecrets); + meterRegistry, meterIdPrefix, watchService, bootstrapSecrets, extensionRegistry); bootstrapClusters.initializeStaticClusters(subscriptionContext); listenerManager = new ListenerManager(eventLoop, bootstrap, subscriptionContext, defaultWatcher); } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/XdsConverterUtil.java b/xds/src/main/java/com/linecorp/armeria/xds/XdsConverterUtil.java deleted file mode 100644 index 28d71f69792..00000000000 --- a/xds/src/main/java/com/linecorp/armeria/xds/XdsConverterUtil.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2023 LINE Corporation - * - * LINE Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.xds; - -import static com.google.common.base.Preconditions.checkArgument; - -import com.linecorp.armeria.common.annotation.Nullable; - -import io.envoyproxy.envoy.config.core.v3.ApiConfigSource; -import io.envoyproxy.envoy.config.core.v3.ApiConfigSource.ApiType; -import io.envoyproxy.envoy.config.core.v3.ConfigSource; - -final class XdsConverterUtil { - - private XdsConverterUtil() {} - - static void validateConfigSource(@Nullable ConfigSource configSource) { - if (configSource == null || configSource.equals(ConfigSource.getDefaultInstance())) { - return; - } - checkArgument(configSource.hasAds() || configSource.hasApiConfigSource() || configSource.hasSelf(), - "Only one of (Ads, ApiConfigSource, or Self) type ConfigSource is supported for %s", - configSource); - if (configSource.hasApiConfigSource()) { - final ApiConfigSource apiConfigSource = configSource.getApiConfigSource(); - final ApiType apiType = apiConfigSource.getApiType(); - checkArgument(apiType == ApiType.GRPC || apiType == ApiType.AGGREGATED_GRPC, - "Unsupported apiType %s. Only GRPC and AGGREGATED_GRPC are supported.", configSource); - checkArgument(apiConfigSource.getGrpcServicesCount() > 0, - "At least once GrpcService is required for ApiConfigSource for %s", configSource); - apiConfigSource.getGrpcServicesList().forEach( - grpcService -> checkArgument(grpcService.hasEnvoyGrpc(), - "Only envoyGrpc is supported for %s", grpcService)); - } - } -} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/XdsExtensionFactory.java b/xds/src/main/java/com/linecorp/armeria/xds/XdsExtensionFactory.java new file mode 100644 index 00000000000..e41bad60ad8 --- /dev/null +++ b/xds/src/main/java/com/linecorp/armeria/xds/XdsExtensionFactory.java @@ -0,0 +1,49 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import java.util.List; + +import com.google.common.collect.ImmutableList; + +import com.linecorp.armeria.common.annotation.UnstableApi; + +/** + * Base interface for xDS extension factories, resolved by the {@code XdsExtensionRegistry}. + * + *

Subtype-specific behavior (e.g. HTTP filter creation) lives on sub-interfaces + * such as {@link com.linecorp.armeria.xds.filter.HttpFilterFactory}. + */ +@UnstableApi +public interface XdsExtensionFactory { + + /** + * Returns the extension name for registry resolution (required, non-nullable). + * For example, {@code "envoy.filters.http.router"} or {@code "envoy.transport_sockets.tls"}. + */ + String name(); + + /** + * Returns the type URLs for registry resolution. + * For example, + * {@code List.of("type.googleapis.com/envoy.extensions.filters.http.router.v3.Router")}. + * Returns an empty list if this factory has no type-URL-based registration. + */ + default List typeUrls() { + return ImmutableList.of(); + } +} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/XdsExtensionRegistry.java b/xds/src/main/java/com/linecorp/armeria/xds/XdsExtensionRegistry.java new file mode 100644 index 00000000000..0a498903f9a --- /dev/null +++ b/xds/src/main/java/com/linecorp/armeria/xds/XdsExtensionRegistry.java @@ -0,0 +1,158 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import java.util.Map; +import java.util.ServiceLoader; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import com.google.protobuf.Any; +import com.google.protobuf.Message; + +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.xds.filter.HttpFilterFactory; + +/** + * A dual-key registry for {@link XdsExtensionFactory} instances. + * Factories are resolved by type URL (primary) or extension name (fallback). + * + *

Also serves as the single entry point for {@link Any}-related operations: + * factory lookup ({@link #query}) and proto decode ({@link #unpack}). + */ +final class XdsExtensionRegistry { + + private final Map byTypeUrl; + private final Map byName; + private final XdsResourceValidator validator; + + private XdsExtensionRegistry(Map byTypeUrl, + Map byName, + XdsResourceValidator validator) { + this.byTypeUrl = byTypeUrl; + this.byName = byName; + this.validator = validator; + } + + @VisibleForTesting + static XdsExtensionRegistry of(XdsResourceValidator validator) { + final ImmutableMap.Builder byName = ImmutableMap.builder(); + final ImmutableMap.Builder byTypeUrl = ImmutableMap.builder(); + + // Load SPI-discovered HttpFilterFactory instances as base factories + ServiceLoader.load(HttpFilterFactory.class).forEach(factory -> { + register(factory, byName, byTypeUrl); + }); + + // Built-in network filter factories + register(HttpConnectionManagerFactory.INSTANCE, byName, byTypeUrl); + + // Built-in transport socket factories + register(UpstreamTlsTransportSocketFactory.INSTANCE, byName, byTypeUrl); + register(RawBufferTransportSocketFactory.INSTANCE, byName, byTypeUrl); + + return new XdsExtensionRegistry(byTypeUrl.build(), + byName.build(), validator); + } + + private static void register(XdsExtensionFactory factory, + ImmutableMap.Builder byName, + ImmutableMap.Builder byTypeUrl) { + byName.put(factory.name(), factory); + for (String typeUrl : factory.typeUrls()) { + byTypeUrl.put(typeUrl, factory); + } + } + + XdsResourceValidator validator() { + return validator; + } + + /** + * Validates the given message using both pgv structural validation and supported-field + * validation. + */ + void assertValid(Object message) { + validator.assertValid(message); + } + + /** + * Unpacks an {@link Any} into the expected proto type using the validator. + */ + T unpack(Any any, Class expectedType) { + return validator.unpack(any, expectedType); + } + + /** + * Looks up a factory by typeUrl and validates it implements the expected type. + * Returns {@code null} if no factory is registered. + * + * @throws IllegalArgumentException if the factory does not implement the expected interface + */ + @Nullable + @VisibleForTesting + T queryByTypeUrl(String typeUrl, Class expectedType) { + final XdsExtensionFactory factory = byTypeUrl.get(typeUrl); + if (factory == null) { + return null; + } + if (!expectedType.isInstance(factory)) { + throw new IllegalArgumentException( + "Factory for typeUrl '" + typeUrl + "' is " + factory.getClass().getName() + + ", expected " + expectedType.getName()); + } + return expectedType.cast(factory); + } + + /** + * Looks up a factory by name and validates it implements the expected type. + * Returns {@code null} if no factory is registered. + * + * @throws IllegalArgumentException if the factory does not implement the expected interface + */ + @Nullable + T queryByName(String name, Class expectedType) { + final XdsExtensionFactory factory = byName.get(name); + if (factory == null) { + return null; + } + if (!expectedType.isInstance(factory)) { + throw new IllegalArgumentException( + "Factory for name '" + name + "' is " + factory.getClass().getName() + + ", expected " + expectedType.getName()); + } + return expectedType.cast(factory); + } + + /** + * Resolves a factory by {@link Any}'s typeUrl first, then by name. + * Returns {@code null} if no factory is found. + * + * @throws IllegalArgumentException if a found factory does not implement the expected interface + */ + @Nullable + T query(Any any, @Nullable String name, Class expectedType) { + final T factory = queryByTypeUrl(any.getTypeUrl(), expectedType); + if (factory != null) { + return factory; + } + if (name != null) { + return queryByName(name, expectedType); + } + return null; + } +} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/XdsResource.java b/xds/src/main/java/com/linecorp/armeria/xds/XdsResource.java index 612e2896a65..e7fed90310c 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/XdsResource.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/XdsResource.java @@ -45,8 +45,9 @@ public interface XdsResource { String name(); /** - * The version of this resource. - * An empty string is returned if this resource is not associated with a version. + * The version assigned by the control plane for this resource. + * For SotW, this is the response version_info. For delta, this is the per-resource version. + * An empty string is returned if the control plane does not assign a version. */ String version(); diff --git a/xds/src/main/java/com/linecorp/armeria/xds/XdsResourceException.java b/xds/src/main/java/com/linecorp/armeria/xds/XdsResourceException.java index 22b760810bf..be025621987 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/XdsResourceException.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/XdsResourceException.java @@ -73,6 +73,13 @@ static XdsResourceException maybeWrap(XdsType type, String name, Throwable t) { return new XdsResourceException(type, name, t); } + @Override + public String getMessage() { + final String superMsg = super.getMessage(); + final String typeAndName = "[type=" + type + ", name='" + name + "']"; + return superMsg != null ? typeAndName + ": " + superMsg : typeAndName; + } + @Override public String toString() { return MoreObjects.toStringHelper(this) diff --git a/xds/src/main/java/com/linecorp/armeria/xds/XdsResourceValidator.java b/xds/src/main/java/com/linecorp/armeria/xds/XdsResourceValidator.java new file mode 100644 index 00000000000..a5597d1de6b --- /dev/null +++ b/xds/src/main/java/com/linecorp/armeria/xds/XdsResourceValidator.java @@ -0,0 +1,86 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import java.util.Comparator; +import java.util.ServiceLoader; + +import com.google.common.collect.Streams; +import com.google.protobuf.Any; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Message; + +import com.linecorp.armeria.common.annotation.UnstableApi; +import com.linecorp.armeria.xds.validator.XdsValidatorIndex; + +/** + * Per-bootstrap validator that delegates to the highest-priority {@link XdsValidatorIndex} loaded via SPI. + * Created once in {@link XdsBootstrapBuilder#build()} and threaded through the entire resource pipeline. + * + *

Validation is performed at exactly two levels: + *

    + *
  • Static resources — {@link XdsBootstrapImpl} calls {@link #assertValid(Object)} once + * on the entire {@code Bootstrap} message at construction time. Both pgv and supported-field + * validators recurse into nested messages, so this single call covers all static clusters, + * listeners, secrets, and their sub-messages. Inline sub-resources (e.g. {@code VirtualHost} + * within a {@code RouteConfiguration}, {@code ClusterLoadAssignment} within a {@code Cluster}) + * are covered by parent validation and do not need separate calls.
    • + *
  • Dynamic resources — calls + * {@link #assertValid(Object)} on each top-level resource unpacked from a + * {@code DiscoveryResponse}. Validation failures are caught and reported as invalid + * resources (NACK'd back to the control plane).
    • + *
+ * + *

In addition, {@link #unpack(Any, Class)} is used for {@code google.protobuf.Any}-typed + * fields that cannot be validated by parent recursion (since {@code Any} is opaque to protobuf + * field traversal). + */ +@UnstableApi +public final class XdsResourceValidator { + + private static final XdsValidatorIndex spiValidator = + Streams.stream(ServiceLoader.load(XdsValidatorIndex.class, + XdsValidatorIndex.class.getClassLoader())) + .max(Comparator.comparingInt(XdsValidatorIndex::priority)) + .orElse(XdsValidatorIndex.noop()); + + XdsResourceValidator() { + } + + /** + * Validates the given message using the SPI-loaded {@link XdsValidatorIndex}. + */ + void assertValid(Object message) { + spiValidator.assertValid(message); + } + + /** + * Unpacks an {@link Any} message into the given class and validates the result. + * This is necessary for {@code Any}-typed fields because protobuf treats {@code Any} + * as an opaque blob — parent-level validation cannot recurse into it. + */ + public T unpack(Any message, Class clazz) { + final T unpacked; + try { + unpacked = message.unpack(clazz); + } catch (InvalidProtocolBufferException e) { + throw new IllegalArgumentException("Error unpacking: " + clazz.getName(), e); + } + assertValid(unpacked); + return unpacked; + } +} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/XdsResponseHandler.java b/xds/src/main/java/com/linecorp/armeria/xds/XdsResponseHandler.java deleted file mode 100644 index 2d381571b4c..00000000000 --- a/xds/src/main/java/com/linecorp/armeria/xds/XdsResponseHandler.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2023 LINE Corporation - * - * LINE Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.xds; - -import com.google.protobuf.Message; - -import com.linecorp.armeria.xds.SotwXdsStream.ActualStream; - -import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; - -/** - * Handles callbacks for {@link SotwXdsStream}. - */ -interface XdsResponseHandler { - - void handleResponse( - ResourceParser resourceParser, DiscoveryResponse value, ActualStream sender, - ConfigSourceLifecycleObserver lifecycleObserver); -} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/XdsStreamSubscriber.java b/xds/src/main/java/com/linecorp/armeria/xds/XdsStreamSubscriber.java index ea1af5a7144..61b208e3117 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/XdsStreamSubscriber.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/XdsStreamSubscriber.java @@ -17,7 +17,6 @@ package com.linecorp.armeria.xds; import java.util.HashSet; -import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -38,25 +37,24 @@ class XdsStreamSubscriber implements SafeCloseable { private final String resource; private final long timeoutMillis; private final EventExecutor eventLoop; - - @Nullable - private T data; - private boolean absent; + private final boolean enableAbsentOnTimeout; @Nullable private ScheduledFuture initialAbsentFuture; private final Set> resourceWatchers = new HashSet<>(); - XdsStreamSubscriber(XdsType type, String resource, EventExecutor eventLoop, long timeoutMillis) { + XdsStreamSubscriber(XdsType type, String resource, EventExecutor eventLoop, long timeoutMillis, + boolean enableAbsentOnTimeout) { this.type = type; this.resource = resource; this.eventLoop = eventLoop; this.timeoutMillis = timeoutMillis; + this.enableAbsentOnTimeout = enableAbsentOnTimeout; restartTimer(); } void restartTimer() { - if (data != null || absent) { // resource already resolved + if (!enableAbsentOnTimeout) { return; } @@ -80,18 +78,12 @@ public void close() { void onData(T data) { maybeCancelAbsentTimer(); - - final T oldData = this.data; - this.data = data; - absent = false; - if (!Objects.equals(oldData, data)) { - for (ResourceWatcher watcher: resourceWatchers) { - try { - watcher.onChanged(data); - } catch (Exception e) { - logger.warn("Unexpected exception while invoking {}.onChanged() with ({}, {}) for ({}).", - getClass().getSimpleName(), type, resource, data, e); - } + for (ResourceWatcher watcher: resourceWatchers) { + try { + watcher.onChanged(data); + } catch (Exception e) { + logger.warn("Unexpected exception while invoking {}.onChanged() with ({}, {}) for ({}).", + getClass().getSimpleName(), type, resource, data, e); } } } @@ -110,18 +102,13 @@ void onError(String resourceName, Throwable t) { void onAbsent() { maybeCancelAbsentTimer(); - - if (!absent) { - data = null; - absent = true; - for (ResourceWatcher watcher: resourceWatchers) { - try { - watcher.onResourceDoesNotExist(type, resource); - } catch (Exception e) { - logger.warn("Unexpected exception while invoking" + - " {}.onResourceDoesNotExist() with ({}, {}).", - getClass().getSimpleName(), type, resource, e); - } + for (ResourceWatcher watcher: resourceWatchers) { + try { + watcher.onResourceDoesNotExist(type, resource); + } catch (Exception e) { + logger.warn("Unexpected exception while invoking" + + " {}.onResourceDoesNotExist() with ({}, {}).", + getClass().getSimpleName(), type, resource, e); } } } @@ -132,12 +119,6 @@ boolean isEmpty() { void registerWatcher(ResourceWatcher watcher) { resourceWatchers.add(watcher); - final T cached = data; - if (cached != null) { - watcher.onChanged(cached); - } else if (absent) { - watcher.onResourceDoesNotExist(type, resource); - } } void unregisterWatcher(ResourceWatcher watcher) { diff --git a/xds/src/main/java/com/linecorp/armeria/xds/XdsUnpackUtil.java b/xds/src/main/java/com/linecorp/armeria/xds/XdsUnpackUtil.java new file mode 100644 index 00000000000..cfbf93939f6 --- /dev/null +++ b/xds/src/main/java/com/linecorp/armeria/xds/XdsUnpackUtil.java @@ -0,0 +1,65 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import java.util.List; + +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Any; + +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.xds.filter.XdsHttpFilter; + +import io.envoyproxy.envoy.config.listener.v3.Listener; +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; +import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter; + +final class XdsUnpackUtil { + + @Nullable + static HttpConnectionManager unpackConnectionManager(Listener listener, + XdsExtensionRegistry registry) { + if (listener.getApiListener().hasApiListener()) { + final Any apiListener = listener.getApiListener().getApiListener(); + final HttpConnectionManagerFactory factory = + registry.queryByTypeUrl(apiListener.getTypeUrl(), + HttpConnectionManagerFactory.class); + assert factory != null; + return factory.create(apiListener, registry.validator()); + } + return null; + } + + static List resolveDownstreamFilters( + @Nullable HttpConnectionManager connectionManager, + XdsExtensionRegistry registry) { + if (connectionManager == null) { + return ImmutableList.of(); + } + final List httpFilters = connectionManager.getHttpFiltersList(); + final ImmutableList.Builder builder = ImmutableList.builder(); + for (HttpFilter httpFilter : httpFilters) { + final XdsHttpFilter instance = FilterUtil.resolveInstance(registry, httpFilter, null); + if (instance != null) { + builder.add(instance); + } + } + return builder.build(); + } + + private XdsUnpackUtil() {} +} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/XdsValidatorIndexRegistry.java b/xds/src/main/java/com/linecorp/armeria/xds/XdsValidatorIndexRegistry.java deleted file mode 100644 index f82221f9cc5..00000000000 --- a/xds/src/main/java/com/linecorp/armeria/xds/XdsValidatorIndexRegistry.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2025 LY Corporation - * - * LY Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.xds; - -import java.util.Comparator; -import java.util.ServiceLoader; - -import com.google.common.collect.Streams; -import com.google.protobuf.Any; -import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Message; - -import com.linecorp.armeria.xds.validator.XdsValidatorIndex; - -final class XdsValidatorIndexRegistry { - - private static final XdsValidatorIndex xdsValidatorIndex = - Streams.stream(ServiceLoader.load(XdsValidatorIndex.class, - XdsValidatorIndex.class.getClassLoader())) - .max(Comparator.comparingInt(XdsValidatorIndex::priority)) - .orElse(XdsValidatorIndex.noop()); - - static void assertValid(Object message) { - xdsValidatorIndex.assertValid(message); - } - - static T unpack(Any message, Class clazz) { - final T unpacked; - try { - unpacked = message.unpack(clazz); - } catch (InvalidProtocolBufferException e) { - throw new IllegalArgumentException("Error unpacking: " + clazz.getName(), e); - } - xdsValidatorIndex.assertValid(unpacked); - return unpacked; - } - - private XdsValidatorIndexRegistry() { - } -} diff --git a/xds/src/main/java/com/linecorp/armeria/xds/client/endpoint/RouterFilterFactory.java b/xds/src/main/java/com/linecorp/armeria/xds/client/endpoint/RouterFilterFactory.java index f4f3aa28694..cac828cd94a 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/client/endpoint/RouterFilterFactory.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/client/endpoint/RouterFilterFactory.java @@ -16,6 +16,9 @@ package com.linecorp.armeria.xds.client.endpoint; +import java.util.List; + +import com.google.common.collect.ImmutableList; import com.google.protobuf.Any; import com.linecorp.armeria.client.HttpPreprocessor; @@ -25,6 +28,7 @@ import com.linecorp.armeria.common.RpcRequest; import com.linecorp.armeria.common.RpcResponse; import com.linecorp.armeria.common.annotation.UnstableApi; +import com.linecorp.armeria.xds.XdsResourceValidator; import com.linecorp.armeria.xds.filter.HttpFilterFactory; import com.linecorp.armeria.xds.filter.XdsHttpFilter; @@ -38,10 +42,51 @@ public final class RouterFilterFactory implements HttpFilterFactory { private static final String NAME = "envoy.filters.http.router"; + private static final String TYPE_URL = + "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"; private static final RouterFilter rpcFilter = new RouterFilter<>(); private static final RouterFilter httpFilter = new RouterFilter<>(); - private static final XdsHttpFilter ROUTER_FILTER = new XdsHttpFilter() { + @Override + public String name() { + return NAME; + } + + @Override + public List typeUrls() { + return ImmutableList.of(TYPE_URL); + } + + @Override + public XdsHttpFilter create(HttpFilter filter, Any config, XdsResourceValidator validator) { + final Router router; + if (config == Any.getDefaultInstance()) { + router = Router.getDefaultInstance(); + } else { + router = validator.unpack(config, Router.class); + } + return new RouterXdsHttpFilter(router); + } + + /** + * An {@link XdsHttpFilter} that holds the parsed {@link Router} config. + */ + @UnstableApi + public static final class RouterXdsHttpFilter implements XdsHttpFilter { + + private final Router router; + + RouterXdsHttpFilter(Router router) { + this.router = router; + } + + /** + * Returns the {@link Router} config. + */ + public Router router() { + return router; + } + @Override public HttpPreprocessor httpPreprocessor() { return httpFilter::execute; @@ -51,15 +96,5 @@ public HttpPreprocessor httpPreprocessor() { public RpcPreprocessor rpcPreprocessor() { return rpcFilter::execute; } - }; - - @Override - public String filterName() { - return NAME; - } - - @Override - public XdsHttpFilter create(HttpFilter filter, Any config) { - return ROUTER_FILTER; } } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/filter/HttpFilterFactory.java b/xds/src/main/java/com/linecorp/armeria/xds/filter/HttpFilterFactory.java index f513a737845..ffc0328d87d 100644 --- a/xds/src/main/java/com/linecorp/armeria/xds/filter/HttpFilterFactory.java +++ b/xds/src/main/java/com/linecorp/armeria/xds/filter/HttpFilterFactory.java @@ -20,6 +20,8 @@ import com.linecorp.armeria.common.annotation.Nullable; import com.linecorp.armeria.common.annotation.UnstableApi; +import com.linecorp.armeria.xds.XdsExtensionFactory; +import com.linecorp.armeria.xds.XdsResourceValidator; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager; import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter; @@ -33,12 +35,7 @@ * Returning {@code null} from {@link #create} causes the filter to be silently skipped. */ @UnstableApi -public interface HttpFilterFactory { - - /** - * The filter name that should be equivalent to {@link HttpFilter#getName()}. - */ - String filterName(); +public interface HttpFilterFactory extends XdsExtensionFactory { /** * Creates an {@link XdsHttpFilter} for the given filter and its raw typed config. @@ -54,7 +51,8 @@ public interface HttpFilterFactory { * @param httpFilter the filter descriptor from {@link HttpConnectionManager#getHttpFiltersList()} * @param config the raw typed config {@link Any}; may be {@link Any#getDefaultInstance()} * if no config was provided + * @param validator the {@link XdsResourceValidator} for validating and unpacking {@link Any} protos */ @Nullable - XdsHttpFilter create(HttpFilter httpFilter, Any config); + XdsHttpFilter create(HttpFilter httpFilter, Any config, XdsResourceValidator validator); } diff --git a/xds/src/main/java/com/linecorp/armeria/xds/filter/HttpFilterFactoryRegistry.java b/xds/src/main/java/com/linecorp/armeria/xds/filter/HttpFilterFactoryRegistry.java deleted file mode 100644 index bc8371044fa..00000000000 --- a/xds/src/main/java/com/linecorp/armeria/xds/filter/HttpFilterFactoryRegistry.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2025 LY Corporation - * - * LY Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.xds.filter; - -import java.util.Map; -import java.util.ServiceLoader; - -import com.google.common.collect.ImmutableMap; - -import com.linecorp.armeria.common.annotation.Nullable; -import com.linecorp.armeria.common.annotation.UnstableApi; - -import io.envoyproxy.envoy.extensions.filters.network.http_connection_manager.v3.HttpFilter; - -/** - * A registry for {@link HttpFilterFactory} implementations. - */ -@UnstableApi -public final class HttpFilterFactoryRegistry { - - private static final Map factories; - - static { - final ImmutableMap.Builder factoriesBuilder = ImmutableMap.builder(); - ServiceLoader.load(HttpFilterFactory.class).forEach(factory -> { - factoriesBuilder.put(factory.filterName(), factory); - }); - factories = factoriesBuilder.build(); - } - - /** - * Returns the registered {@link HttpFilterFactory}. - * - * @param name the name of the filter represented by {@link HttpFilter#getName()} - */ - @Nullable - public static HttpFilterFactory filterFactory(String name) { - return factories.get(name); - } - - private HttpFilterFactoryRegistry() { - } -} diff --git a/xds/src/test/java/com/linecorp/armeria/xds/SotwXdsStreamTest.java b/xds/src/test/java/com/linecorp/armeria/xds/SotwXdsStreamTest.java deleted file mode 100644 index 06c73ff4693..00000000000 --- a/xds/src/test/java/com/linecorp/armeria/xds/SotwXdsStreamTest.java +++ /dev/null @@ -1,312 +0,0 @@ -/* - * Copyright 2023 LINE Corporation - * - * LINE Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.xds; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.awaitility.Awaitility.await; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import org.hamcrest.Matchers; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import com.google.common.collect.ImmutableList; -import com.google.protobuf.Duration; -import com.google.protobuf.Message; - -import com.linecorp.armeria.client.grpc.GrpcClients; -import com.linecorp.armeria.client.retry.Backoff; -import com.linecorp.armeria.server.ServerBuilder; -import com.linecorp.armeria.server.grpc.GrpcService; -import com.linecorp.armeria.testing.junit5.common.EventLoopExtension; -import com.linecorp.armeria.testing.junit5.server.ServerExtension; -import com.linecorp.armeria.xds.SotwXdsStream.ActualStream; - -import io.envoyproxy.controlplane.cache.v3.SimpleCache; -import io.envoyproxy.controlplane.cache.v3.Snapshot; -import io.envoyproxy.controlplane.server.V3DiscoveryServer; -import io.envoyproxy.envoy.config.cluster.v3.Cluster; -import io.envoyproxy.envoy.config.core.v3.Node; -import io.envoyproxy.envoy.service.discovery.v3.DiscoveryResponse; - -class SotwXdsStreamTest { - - private static final Node SERVER_INFO = Node.getDefaultInstance(); - - private static final String GROUP = "key"; - private static final SimpleCache cache = new SimpleCache<>(node -> GROUP); - private static final String clusterName = "cluster1"; - private static final ConfigSourceLifecycleObserver lifecycleObserver = - new ConfigSourceLifecycleObserver() {}; - - @RegisterExtension - static final ServerExtension server = new ServerExtension() { - @Override - protected void configure(ServerBuilder sb) throws Exception { - final V3DiscoveryServer v3DiscoveryServer = new V3DiscoveryServer(cache); - sb.service(GrpcService.builder() - .addService(v3DiscoveryServer.getAggregatedDiscoveryServiceImpl()) - .build()); - } - }; - - @BeforeEach - void beforeEach() { - cache.setSnapshot( - GROUP, - Snapshot.create( - ImmutableList.of(createCluster(clusterName, 1)), - ImmutableList.of(), - ImmutableList.of(), - ImmutableList.of(), - ImmutableList.of(), - "1")); - } - - @RegisterExtension - static EventLoopExtension eventLoop = new EventLoopExtension(); - - static class TestResponseHandler implements XdsResponseHandler { - - private final List responses = new ArrayList<>(); - private final List resets = new ArrayList<>(); - - public List getResponses() { - return responses; - } - - public void clear() { - responses.clear(); - resets.clear(); - } - - @Override - public void handleResponse( - ResourceParser resourceParser, DiscoveryResponse value, ActualStream sender, - ConfigSourceLifecycleObserver observer) { - responses.add(value); - sender.ackResponse(resourceParser.type(), value.getVersionInfo(), value.getNonce()); - } - } - - @Test - void basicCase() throws Exception { - final SotwDiscoveryStub stub = SotwDiscoveryStub.ads(GrpcClients.builder(server.httpUri())); - final DummyResourceWatcher watcher = new DummyResourceWatcher(); - final SubscriberStorage subscriberStorage = new SubscriberStorage(eventLoop.get(), 15_000); - final TestResponseHandler responseHandler = new TestResponseHandler(); - try (SotwXdsStream stream = new SotwXdsStream(stub, SERVER_INFO, Backoff.ofDefault(), eventLoop.get(), - responseHandler, subscriberStorage, - lifecycleObserver)) { - - await().pollDelay(100, TimeUnit.MILLISECONDS) - .untilAsserted(() -> assertThat(responseHandler.getResponses()).isEmpty()); - - subscriberStorage.register(XdsType.CLUSTER, clusterName, watcher); - stream.start(); - - // check if the initial cache update is done - await().until(() -> !responseHandler.getResponses().isEmpty()); - assertThat(responseHandler.getResponses()).allSatisfy(res -> { - final Cluster expected = cache.getSnapshot(GROUP).clusters().resources().get(clusterName); - assertThat(res.getResources(0).unpack(Cluster.class)).isEqualTo(expected); - }); - responseHandler.clear(); - - // check if a cache update is propagated to the handler - cache.setSnapshot( - GROUP, - Snapshot.create( - ImmutableList.of(createCluster(clusterName, 1)), - ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), - ImmutableList.of(), "2")); - - await().until(() -> !responseHandler.getResponses().isEmpty()); - assertThat(responseHandler.getResponses()).allSatisfy(res -> { - final Cluster expected = cache.getSnapshot(GROUP).clusters().resources().get(clusterName); - assertThat(res.getResources(0).unpack(Cluster.class)).isEqualTo(expected); - }); - responseHandler.clear(); - - // now the stream is stopped, so no more updates - stream.stop(); - await().until(() -> stream.actualStream == null); - - cache.setSnapshot( - GROUP, - Snapshot.create( - ImmutableList.of(createCluster(clusterName, 2)), - ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), - ImmutableList.of(), "3")); - - await().pollDelay(100, TimeUnit.MILLISECONDS) - .untilAsserted(() -> assertThat(responseHandler.getResponses()).isEmpty()); - } - } - - @Test - void restart() throws Exception { - final SotwDiscoveryStub stub = SotwDiscoveryStub.ads(GrpcClients.builder(server.httpUri())); - final DummyResourceWatcher watcher = new DummyResourceWatcher(); - final SubscriberStorage subscriberStorage = new SubscriberStorage(eventLoop.get(), 15_000); - final TestResponseHandler responseHandler = new TestResponseHandler(); - - try (SotwXdsStream stream = new SotwXdsStream(stub, SERVER_INFO, Backoff.ofDefault(), eventLoop.get(), - responseHandler, subscriberStorage, lifecycleObserver)) { - - await().pollDelay(100, TimeUnit.MILLISECONDS) - .untilAsserted(() -> assertThat(responseHandler.getResponses()).isEmpty()); - - subscriberStorage.register(XdsType.CLUSTER, clusterName, watcher); - stream.start(); - - // check if the initial cache update is done - await().until(() -> !responseHandler.getResponses().isEmpty()); - assertThat(responseHandler.getResponses()).allSatisfy(res -> { - final Cluster expected = cache.getSnapshot(GROUP).clusters().resources().get(clusterName); - assertThat(res.getResources(0).unpack(Cluster.class)).isEqualTo(expected); - }); - responseHandler.clear(); - - // stop the stream and verify there are no updates - stream.stop(); - await().until(() -> stream.actualStream == null); - - cache.setSnapshot( - GROUP, - Snapshot.create( - ImmutableList.of(createCluster(clusterName, 1)), - ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), - ImmutableList.of(), "2")); - await().pollDelay(100, TimeUnit.MILLISECONDS) - .untilAsserted(() -> assertThat(responseHandler.getResponses()).isEmpty()); - - // restart the thread and verify that the handle receives the update - stream.start(); - await().until(() -> !responseHandler.getResponses().isEmpty()); - assertThat(responseHandler.getResponses()).allSatisfy(res -> { - final Cluster expected = cache.getSnapshot(GROUP).clusters().resources().get(clusterName); - assertThat(res.getResources(0).unpack(Cluster.class)).isEqualTo(expected); - }); - } - } - - @Test - void errorHandling() throws Exception { - final SotwDiscoveryStub stub = SotwDiscoveryStub.ads(GrpcClients.builder(server.httpUri())); - final DummyResourceWatcher watcher = new DummyResourceWatcher(); - final SubscriberStorage subscriberStorage = new SubscriberStorage(eventLoop.get(), 15_000); - final AtomicInteger cntRef = new AtomicInteger(); - final CountDownLatch latch = new CountDownLatch(1); - final TestResponseHandler responseHandler = new TestResponseHandler() { - @Override - public void handleResponse( - ResourceParser resourceParser, DiscoveryResponse value, ActualStream sender, - ConfigSourceLifecycleObserver observer) { - if (cntRef.getAndIncrement() < 3) { - sender.onError(new Exception("error")); - return; - } - try { - latch.await(); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - super.handleResponse(resourceParser, value, sender, observer); - } - }; - - try (SotwXdsStream stream = new SotwXdsStream(stub, SERVER_INFO, Backoff.ofDefault(), eventLoop.get(), - responseHandler, subscriberStorage, lifecycleObserver)) { - - await().pollDelay(100, TimeUnit.MILLISECONDS) - .untilAsserted(() -> assertThat(responseHandler.getResponses()).isEmpty()); - - subscriberStorage.register(XdsType.CLUSTER, clusterName, watcher); - stream.start(); - - await().untilAtomic(cntRef, Matchers.greaterThanOrEqualTo(3)); - assertThat(responseHandler.getResponses()).isEmpty(); - - latch.countDown(); - // Once an update is done, the handler will eventually receive the new update - await().until(() -> !responseHandler.getResponses().isEmpty()); - assertThat(responseHandler.getResponses()).allSatisfy(res -> { - final Cluster expected = cache.getSnapshot(GROUP).clusters().resources().get(clusterName); - assertThat(res.getResources(0).unpack(Cluster.class)).isEqualTo(expected); - }); - } - } - - @Test - void nackResponse() throws Exception { - final SotwDiscoveryStub stub = SotwDiscoveryStub.ads(GrpcClients.builder(server.httpUri())); - final DummyResourceWatcher watcher = new DummyResourceWatcher(); - final SubscriberStorage subscriberStorage = new SubscriberStorage(eventLoop.get(), 15_000); - final AtomicBoolean ackRef = new AtomicBoolean(); - final AtomicInteger nackResponses = new AtomicInteger(); - final TestResponseHandler responseHandler = new TestResponseHandler() { - @Override - public void handleResponse( - ResourceParser resourceParser, DiscoveryResponse value, ActualStream sender, - ConfigSourceLifecycleObserver observer) { - if (ackRef.get()) { - super.handleResponse(resourceParser, value, sender, observer); - } else { - nackResponses.incrementAndGet(); - sender.nackResponse(XdsType.CLUSTER, value.getNonce(), "temporarily unavailable"); - } - } - }; - - try (SotwXdsStream stream = new SotwXdsStream( - stub, SERVER_INFO, Backoff.ofDefault(), eventLoop.get(), responseHandler, - subscriberStorage, lifecycleObserver)) { - - await().pollDelay(100, TimeUnit.MILLISECONDS) - .untilAsserted(() -> assertThat(responseHandler.getResponses()).isEmpty()); - - subscriberStorage.register(XdsType.CLUSTER, clusterName, watcher); - stream.start(); - - await().untilAtomic(nackResponses, Matchers.greaterThan(2)); - assertThat(responseHandler.getResponses()).isEmpty(); - ackRef.set(true); - - // Once an update is done, the handler will eventually receive the new update - await().until(() -> !responseHandler.getResponses().isEmpty()); - assertThat(responseHandler.getResponses()).allSatisfy(res -> { - final Cluster expected = cache.getSnapshot(GROUP).clusters().resources().get(clusterName); - assertThat(res.getResources(0).unpack(Cluster.class)).isEqualTo(expected); - }); - } - } - - static Cluster createCluster(String clusterName, long connectTimeout) { - return Cluster.newBuilder() - .setName(clusterName) - .setConnectTimeout(Duration.newBuilder().setSeconds(connectTimeout)) - .build(); - } -} diff --git a/xds/src/test/java/com/linecorp/armeria/xds/StateCoordinatorTest.java b/xds/src/test/java/com/linecorp/armeria/xds/StateCoordinatorTest.java new file mode 100644 index 00000000000..385d2fa50f5 --- /dev/null +++ b/xds/src/test/java/com/linecorp/armeria/xds/StateCoordinatorTest.java @@ -0,0 +1,117 @@ +/* + * Copyright 2026 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.linecorp.armeria.common.annotation.Nullable; +import com.linecorp.armeria.testing.junit5.common.EventLoopExtension; + +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.route.v3.RouteConfiguration; + +class StateCoordinatorTest { + + private static final String CLUSTER_NAME = "cluster1"; + private static final String ROUTE_NAME = "route1"; + + @RegisterExtension + static EventLoopExtension eventLoop = new EventLoopExtension(); + + @Test + void lateSubscriberReceivesCachedResource() { + final StateCoordinator coordinator = new StateCoordinator(eventLoop.get(), 15_000, false, + XdsExtensionRegistry.of(new XdsResourceValidator())); + final ClusterXdsResource resource = + new ClusterXdsResource(createCluster(CLUSTER_NAME), "1").withRevision(1); + coordinator.onResourceUpdated(XdsType.CLUSTER, CLUSTER_NAME, resource); + + final CapturingWatcher watcher = new CapturingWatcher(); + coordinator.register(XdsType.CLUSTER, CLUSTER_NAME, watcher); + + assertThat(watcher.changed).isSameAs(resource); + assertThat(watcher.missingType).isNull(); + } + + @Test + void missingResourceCachedAsAbsent() { + final StateCoordinator coordinator = new StateCoordinator(eventLoop.get(), 15_000, false, + XdsExtensionRegistry.of(new XdsResourceValidator())); + final CapturingWatcher watcher1 = new CapturingWatcher(); + coordinator.register(XdsType.CLUSTER, CLUSTER_NAME, watcher1); + + coordinator.onResourceMissing(XdsType.CLUSTER, CLUSTER_NAME); + coordinator.unregister(XdsType.CLUSTER, CLUSTER_NAME, watcher1); + + final CapturingWatcher watcher2 = new CapturingWatcher(); + coordinator.register(XdsType.CLUSTER, CLUSTER_NAME, watcher2); + + assertThat(watcher2.changed).isNull(); + assertThat(watcher2.missingType).isEqualTo(XdsType.CLUSTER); + assertThat(watcher2.missingName).isEqualTo(CLUSTER_NAME); + } + + @Test + void nonFullStateDroppedOnLastUnsubscribe() { + final StateCoordinator coordinator = new StateCoordinator(eventLoop.get(), 15_000, false, + XdsExtensionRegistry.of(new XdsResourceValidator())); + final RouteXdsResource resource = + new RouteXdsResource(RouteConfiguration.newBuilder().setName(ROUTE_NAME).build(), "1") + .withRevision(1); + coordinator.onResourceUpdated(XdsType.ROUTE, ROUTE_NAME, resource); + + final CapturingWatcher watcher1 = new CapturingWatcher(); + coordinator.register(XdsType.ROUTE, ROUTE_NAME, watcher1); + assertThat(watcher1.changed).isSameAs(resource); + + coordinator.unregister(XdsType.ROUTE, ROUTE_NAME, watcher1); + + final CapturingWatcher watcher2 = new CapturingWatcher(); + coordinator.register(XdsType.ROUTE, ROUTE_NAME, watcher2); + + assertThat(watcher2.changed).isNull(); + assertThat(watcher2.missingType).isNull(); + assertThat(watcher2.missingName).isNull(); + } + + private static Cluster createCluster(String name) { + return Cluster.newBuilder().setName(name).build(); + } + + private static final class CapturingWatcher implements ResourceWatcher { + @Nullable + private XdsResource changed; + @Nullable + private XdsType missingType; + @Nullable + private String missingName; + + @Override + public void onChanged(XdsResource update) { + changed = update; + } + + @Override + public void onResourceDoesNotExist(XdsType type, String resourceName) { + missingType = type; + missingName = resourceName; + } + } +} diff --git a/xds/src/test/java/com/linecorp/armeria/xds/SubscriberStorageTest.java b/xds/src/test/java/com/linecorp/armeria/xds/SubscriberStorageTest.java index 538bd6f3d31..694548405fc 100644 --- a/xds/src/test/java/com/linecorp/armeria/xds/SubscriberStorageTest.java +++ b/xds/src/test/java/com/linecorp/armeria/xds/SubscriberStorageTest.java @@ -17,6 +17,9 @@ package com.linecorp.armeria.xds; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -25,31 +28,59 @@ class SubscriberStorageTest { + private static final String CLUSTER_NAME = "cluster1"; + private static final String ROUTE_NAME = "route1"; + @RegisterExtension static EventLoopExtension eventLoop = new EventLoopExtension(); @Test void registerAndUnregister() throws Exception { final DummyResourceWatcher watcher = new DummyResourceWatcher(); - final SubscriberStorage storage = new SubscriberStorage(eventLoop.get(), 15_000); - storage.register(XdsType.CLUSTER, "cluster1", watcher); - assertThat(storage.subscribers(XdsType.CLUSTER)).hasSize(1); - storage.unregister(XdsType.CLUSTER, "cluster1", watcher); - assertThat(storage.subscribers(XdsType.CLUSTER)).isEmpty(); - assertThat(storage.allSubscribers()).isEmpty(); + final SubscriberStorage storage = new SubscriberStorage(eventLoop.get(), 15_000, false); + storage.register(XdsType.CLUSTER, CLUSTER_NAME, watcher); + assertThat(storage.resources(XdsType.CLUSTER)).hasSize(1); + storage.unregister(XdsType.CLUSTER, CLUSTER_NAME, watcher); + assertThat(storage.resources(XdsType.CLUSTER)).isEmpty(); + assertThat(storage.hasNoSubscribers()).isTrue(); } @Test void identityBasedUnregister() { final DummyResourceWatcher watcher1 = new DummyResourceWatcher(); - final SubscriberStorage storage = new SubscriberStorage(eventLoop.get(), 15_000); - storage.register(XdsType.CLUSTER, "cluster1", watcher1); - assertThat(storage.subscribers(XdsType.CLUSTER)).hasSize(1); - storage.register(XdsType.CLUSTER, "cluster1", watcher1); - assertThat(storage.subscribers(XdsType.CLUSTER)).hasSize(1); - - storage.unregister(XdsType.CLUSTER, "cluster1", watcher1); - assertThat(storage.subscribers(XdsType.CLUSTER)).isEmpty(); - assertThat(storage.allSubscribers()).isEmpty(); + final SubscriberStorage storage = new SubscriberStorage(eventLoop.get(), 15_000, false); + storage.register(XdsType.CLUSTER, CLUSTER_NAME, watcher1); + assertThat(storage.resources(XdsType.CLUSTER)).hasSize(1); + storage.register(XdsType.CLUSTER, CLUSTER_NAME, watcher1); + assertThat(storage.resources(XdsType.CLUSTER)).hasSize(1); + + storage.unregister(XdsType.CLUSTER, CLUSTER_NAME, watcher1); + assertThat(storage.resources(XdsType.CLUSTER)).isEmpty(); + assertThat(storage.hasNoSubscribers()).isTrue(); + } + + @Test + void nonClusterListenerTimeout() { + final CapturingWatcher watcher = new CapturingWatcher(); + final SubscriberStorage storage = new SubscriberStorage(eventLoop.get(), 50, false); + storage.register(XdsType.ROUTE, ROUTE_NAME, watcher); + + await().atMost(1, TimeUnit.SECONDS) + .untilAsserted(() -> assertThat(watcher.missingType).isEqualTo(XdsType.ROUTE)); + assertThat(watcher.missingName).isEqualTo(ROUTE_NAME); + } + + private static final class CapturingWatcher implements ResourceWatcher { + private XdsType missingType; + private String missingName; + + @Override + public void onChanged(XdsResource update) {} + + @Override + public void onResourceDoesNotExist(XdsType type, String resourceName) { + missingType = type; + missingName = resourceName; + } } } diff --git a/xds/src/test/java/com/linecorp/armeria/xds/TestResourceWatcher.java b/xds/src/test/java/com/linecorp/armeria/xds/TestResourceWatcher.java index a68353f4715..e30b4149e8a 100644 --- a/xds/src/test/java/com/linecorp/armeria/xds/TestResourceWatcher.java +++ b/xds/src/test/java/com/linecorp/armeria/xds/TestResourceWatcher.java @@ -20,7 +20,6 @@ import java.util.concurrent.BlockingDeque; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,14 +58,16 @@ List blockingMissing() { return blockingFirst("onMissing", List.class); } - void batchAssertChanged(Class clazz, Consumer> assertion) throws Exception { - Thread.sleep(1000); - final ImmutableList.Builder builder = ImmutableList.builder(); - while (!events.isEmpty()) { - final T obj = blockingFirst("snapshotUpdated", clazz); - builder.add(obj); + @Nullable + T pollChanged(Class clazz) { + T last = null; + List event; + while ((event = events.poll()) != null) { + if ("snapshotUpdated".equals(event.get(0)) && clazz.isInstance(event.get(1))) { + last = clazz.cast(event.get(1)); + } } - assertion.accept(builder.build()); + return last; } T blockingChanged(Class clazz) { diff --git a/xds/src/test/java/com/linecorp/armeria/xds/XdsClientIntegrationTest.java b/xds/src/test/java/com/linecorp/armeria/xds/XdsClientIntegrationTest.java index 3c93b17975d..a9bf65a160a 100644 --- a/xds/src/test/java/com/linecorp/armeria/xds/XdsClientIntegrationTest.java +++ b/xds/src/test/java/com/linecorp/armeria/xds/XdsClientIntegrationTest.java @@ -91,12 +91,12 @@ void basicCase() throws Exception { ImmutableList.of(XdsTestResources.loadAssignment("cluster1", URI.create("http://a.b"))), ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), "2")); - watcher.batchAssertChanged(ClusterSnapshot.class, snapshots -> { - assertThat(snapshots.size()).isBetween(1, 2); - final ClusterSnapshot last = snapshots.get(snapshots.size() - 1); + await().untilAsserted(() -> { + final ClusterSnapshot snapshot = watcher.pollChanged(ClusterSnapshot.class); + assertThat(snapshot).isNotNull(); final Cluster expectedCluster2 = cache.getSnapshot(GROUP).clusters() .resources().get(clusterName); - assertThat(last.xdsResource().resource()).isEqualTo(expectedCluster2); + assertThat(snapshot.xdsResource().resource()).isEqualTo(expectedCluster2); }); // Updates aren't propagated after the watch is removed @@ -135,12 +135,12 @@ void multipleResources() throws Exception { ImmutableList.of(XdsTestResources.loadAssignment("cluster1", URI.create("http://a.b")), XdsTestResources.loadAssignment("cluster2", URI.create("http://c.d"))), ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), "2")); - watcher.batchAssertChanged(ClusterSnapshot.class, snapshots -> { - assertThat(snapshots.size()).isBetween(1, 2); - final ClusterSnapshot last = snapshots.get(snapshots.size() - 1); + await().untilAsserted(() -> { + final ClusterSnapshot snapshot = watcher.pollChanged(ClusterSnapshot.class); + assertThat(snapshot).isNotNull(); final Cluster expectedCluster2 = cache.getSnapshot(GROUP).clusters().resources().get("cluster1"); - assertThat(last.xdsResource().resource()).isEqualTo(expectedCluster2); + assertThat(snapshot.xdsResource().resource()).isEqualTo(expectedCluster2); }); final ClusterRoot clusterRoot2 = xdsBootstrap.clusterRoot("cluster2"); @@ -161,14 +161,14 @@ void multipleResources() throws Exception { ImmutableList.of(XdsTestResources.loadAssignment("cluster1", URI.create("http://a.b")), XdsTestResources.loadAssignment("cluster2", URI.create("http://c.d"))), ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), "3")); - watcher.batchAssertChanged(ClusterSnapshot.class, snapshots -> { - assertThat(snapshots.size()).isBetween(1, 2); - final ClusterSnapshot last = snapshots.get(snapshots.size() - 1); - assertThat(last.xdsResource().version()).isEqualTo("3"); - assertThat(last.endpointSnapshot().xdsResource().version()).isEqualTo("3"); + await().untilAsserted(() -> { + final ClusterSnapshot snapshot = watcher.pollChanged(ClusterSnapshot.class); + assertThat(snapshot).isNotNull(); + assertThat(snapshot.xdsResource().version()).isEqualTo("3"); + assertThat(snapshot.endpointSnapshot().xdsResource().version()).isEqualTo("3"); final Cluster expectedCluster4 = cache.getSnapshot(GROUP).clusters().resources().get("cluster2"); - assertThat(last.xdsResource().resource()).isEqualTo(expectedCluster4); + assertThat(snapshot.xdsResource().resource()).isEqualTo(expectedCluster4); }); await().pollDelay(100, TimeUnit.MILLISECONDS) .untilAsserted(() -> assertThat(watcher.events()).isEmpty()); @@ -226,14 +226,14 @@ void errorHandling() throws Exception { ImmutableList.of(XdsTestResources.loadAssignment("cluster1", URI.create("http://a.b"))), ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), "2")); - watcher.batchAssertChanged(ClusterSnapshot.class, snapshots -> { + await().untilAsserted(() -> { + final ClusterSnapshot snapshot = watcher.pollChanged(ClusterSnapshot.class); + assertThat(snapshot).isNotNull(); final Cluster expectedCluster2 = cache.getSnapshot(GROUP).clusters() .resources().get("cluster1"); - final ClusterSnapshot last = snapshots.get(snapshots.size() - 1); - assertThat(snapshots.size()).isBetween(1, 2); - assertThat(last.xdsResource().version()).isEqualTo("2"); - assertThat(last.endpointSnapshot().xdsResource().version()).isEqualTo("2"); - assertThat(last.xdsResource().resource()).isEqualTo(expectedCluster2); + assertThat(snapshot.xdsResource().version()).isEqualTo("2"); + assertThat(snapshot.endpointSnapshot().xdsResource().version()).isEqualTo("2"); + assertThat(snapshot.xdsResource().resource()).isEqualTo(expectedCluster2); }); await().pollDelay(100, TimeUnit.MILLISECONDS) diff --git a/xds/src/test/java/com/linecorp/armeria/xds/XdsExtensionRegistryTest.java b/xds/src/test/java/com/linecorp/armeria/xds/XdsExtensionRegistryTest.java new file mode 100644 index 00000000000..162d131e9bf --- /dev/null +++ b/xds/src/test/java/com/linecorp/armeria/xds/XdsExtensionRegistryTest.java @@ -0,0 +1,103 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.junit.jupiter.api.Test; + +import com.google.protobuf.Any; +import com.google.protobuf.Duration; + +import com.linecorp.armeria.xds.filter.HttpFilterFactory; + +class XdsExtensionRegistryTest { + + private static final XdsResourceValidator VALIDATOR = new XdsResourceValidator(); + + private static XdsExtensionRegistry createRegistry() { + return XdsExtensionRegistry.of(VALIDATOR); + } + + @Test + void queryWithTypeMismatch() { + // HttpConnectionManagerFactory is registered by default and is not an HttpFilterFactory + final XdsExtensionRegistry registry = createRegistry(); + assertThatThrownBy(() -> registry.queryByTypeUrl( + "type.googleapis.com/envoy.extensions.filters.network" + + ".http_connection_manager.v3.HttpConnectionManager", + HttpFilterFactory.class)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("expected"); + } + + @Test + void spiFactoriesLoadedByDefault() { + // SPI should load RouterFilterFactory + final XdsExtensionRegistry registry = createRegistry(); + final HttpFilterFactory resolved = registry.queryByName( + "envoy.filters.http.router", HttpFilterFactory.class); + assertThat(resolved).isNotNull(); + assertThat(resolved).isInstanceOf(HttpFilterFactory.class); + } + + @Test + void emptyRegistryReturnsNull() { + final XdsExtensionRegistry registry = createRegistry(); + assertThat(registry.queryByName("nonexistent.filter", HttpFilterFactory.class)).isNull(); + assertThat(registry.queryByTypeUrl("type.googleapis.com/nonexistent", + HttpFilterFactory.class)).isNull(); + } + + @Test + void assertValidDelegatesToValidator() { + final XdsExtensionRegistry registry = createRegistry(); + // Should not throw for a valid message + final Duration valid = Duration.newBuilder().setSeconds(42).build(); + registry.assertValid(valid); + } + + @Test + void unpackDelegatesToValidator() { + final XdsExtensionRegistry registry = createRegistry(); + final Duration original = Duration.newBuilder().setSeconds(42).build(); + final Any packed = Any.pack(original); + final Duration unpacked = registry.unpack(packed, Duration.class); + assertThat(unpacked).isEqualTo(original); + } + + @Test + void queryPreferTypeUrl() { + final XdsExtensionRegistry registry = createRegistry(); + // RouterFilterFactory is registered by both name and type URL via SPI + final String routerTypeUrl = + "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router"; + final Any any = Any.newBuilder().setTypeUrl(routerTypeUrl).build(); + + // query() checks type URL first + assertThat(registry.query(any, "envoy.filters.http.router", HttpFilterFactory.class)) + .isNotNull(); + // falls back to name when type URL doesn't match + final Any unknownAny = Any.newBuilder().setTypeUrl("unknown").build(); + assertThat(registry.query(unknownAny, "envoy.filters.http.router", + HttpFilterFactory.class)) + .isNotNull(); + // returns null when neither matches + assertThat(registry.query(unknownAny, "unknown", HttpFilterFactory.class)).isNull(); + } +} diff --git a/xds/src/test/java/com/linecorp/armeria/xds/XdsResourceValidatorTest.java b/xds/src/test/java/com/linecorp/armeria/xds/XdsResourceValidatorTest.java new file mode 100644 index 00000000000..e2907ae8161 --- /dev/null +++ b/xds/src/test/java/com/linecorp/armeria/xds/XdsResourceValidatorTest.java @@ -0,0 +1,71 @@ +/* + * Copyright 2025 LY Corporation + * + * LY Corporation licenses this file to you under the Apache License, + * version 2.0 (the "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at: + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.linecorp.armeria.xds; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.junit.jupiter.api.Test; + +import com.google.protobuf.Any; + +import io.envoyproxy.envoy.config.cluster.v3.Cluster; +import io.envoyproxy.envoy.config.route.v3.VirtualHost; +import io.envoyproxy.pgv.ValidationException; + +class XdsResourceValidatorTest { + + @Test + void pgvValidationIsEnabled() { + final XdsResourceValidator validator = new XdsResourceValidator(); + final VirtualHost virtualHost = VirtualHost.getDefaultInstance(); + assertThatThrownBy(() -> validator.assertValid(virtualHost)) + .isInstanceOf(IllegalArgumentException.class) + .cause() + .isInstanceOf(ValidationException.class) + .hasMessageContaining("length must be at least 1 but got: 0"); + } + + @Test + void supportedFieldValidationRuns() { + // SPI loads DefaultXdsValidatorIndex which includes supported-field validation + final XdsResourceValidator validator = new XdsResourceValidator(); + // Cluster with name set (passes pgv) — should not throw + final Cluster cluster = Cluster.newBuilder().setName("test").build(); + validator.assertValid(cluster); + } + + @Test + void unpackValidatesUnpackedMessage() { + final XdsResourceValidator validator = new XdsResourceValidator(); + // pack a valid cluster + final Cluster cluster = Cluster.newBuilder().setName("test").build(); + final Any packed = Any.pack(cluster); + final Cluster unpacked = validator.unpack(packed, Cluster.class); + assertThat(unpacked.getName()).isEqualTo("test"); + } + + @Test + void unpackFailsOnInvalidMessage() { + final XdsResourceValidator validator = new XdsResourceValidator(); + // Pack a default VirtualHost (will fail pgv) + final VirtualHost vhost = VirtualHost.getDefaultInstance(); + final Any packed = Any.pack(vhost); + assertThatThrownBy(() -> validator.unpack(packed, VirtualHost.class)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/xds/src/test/java/com/linecorp/armeria/xds/XdsValidatorIndexRegistryTest.java b/xds/src/test/java/com/linecorp/armeria/xds/XdsValidatorIndexRegistryTest.java deleted file mode 100644 index 4fce41eed71..00000000000 --- a/xds/src/test/java/com/linecorp/armeria/xds/XdsValidatorIndexRegistryTest.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2025 LY Corporation - * - * LY Corporation licenses this file to you under the Apache License, - * version 2.0 (the "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at: - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -package com.linecorp.armeria.xds; - -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -import org.junit.jupiter.api.Test; - -import io.envoyproxy.envoy.config.route.v3.VirtualHost; -import io.envoyproxy.pgv.ValidationException; - -class XdsValidatorIndexRegistryTest { - - @Test - void validationIsEnabled() throws Exception { - final VirtualHost virtualHost = VirtualHost.getDefaultInstance(); - assertThatThrownBy(() -> XdsValidatorIndexRegistry.assertValid(virtualHost)) - .isInstanceOf(IllegalArgumentException.class) - .cause() - .isInstanceOf(ValidationException.class) - .hasMessageContaining("length must be at least 1 but got: 0"); - } -}