Mark HTTPClient as deprecated (#2020)

* Mark HTTPClient as deprecated

Signed-off-by: Julien Nioche <julien@digitalpebble.com>

* Formatting!

Signed-off-by: Julien Nioche <julien@digitalpebble.com>

---------

Signed-off-by: Julien Nioche <julien@digitalpebble.com>

Author: jnioche

core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java

@@ -38,7 +38,7 @@
  *
  * <pre>
  * protocol.delegator.config:
- * - className: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
+ * - className: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
  *   filters:
  *     domain: "example.com"
  *     depth: "3"
@@ -50,7 +50,7 @@
  *   regex:
  *    - \.pdf
  *    - \.doc
- * - className: "org.apache.stormcrawler.protocol.selenium.SeleniumProtocol"
+ * - className: "org.apache.stormcrawler.protocol.playwright.HttpProtocol"
  * </pre>
  *
  * Typically, the last one in the list must not have filters as it is used as a default value. The

core/src/main/java/org/apache/stormcrawler/protocol/ProtocolFactory.java

@@ -71,11 +71,9 @@ private void configure(Config conf) {
             if (StringUtils.isBlank(protocolimplementation)) {
                 // set the default values
                 if (protocol.equalsIgnoreCase("http")) {
-                    protocolimplementation =
-                            "org.apache.stormcrawler.protocol.httpclient.HttpProtocol";
+                    protocolimplementation = "org.apache.stormcrawler.protocol.okhttp.HttpProtocol";
                 } else if (protocol.equalsIgnoreCase("https")) {
-                    protocolimplementation =
-                            "org.apache.stormcrawler.protocol.httpclient.HttpProtocol";
+                    protocolimplementation = "org.apache.stormcrawler.protocol.okhttp.HttpProtocol";
                 } else {
                     throw new RuntimeException(paramName + "should not have an empty value");
                 }

core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java

@@ -68,7 +68,14 @@
 import org.apache.stormcrawler.util.CookieConverter;
 import org.slf4j.LoggerFactory;
 
-/** Uses Apache httpclient to handle http and https. */
+/**
+ * Uses Apache httpclient to handle http and https.
+ *
+ * @deprecated The Apache HttpClient protocol implementation is deprecated and will be removed in
+ *     the next major release of StormCrawler. Use the OkHttp implementation ({@link
+ *     org.apache.stormcrawler.protocol.okhttp.HttpProtocol}) instead.
+ */
+@Deprecated
 public class HttpProtocol extends AbstractHttpProtocol
         implements ResponseHandler<ProtocolResponse> {
 

core/src/main/java/org/apache/stormcrawler/proxy/MultiProxyManager.java

@@ -30,7 +30,6 @@
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.storm.Config;
 import org.apache.stormcrawler.Metadata;
-import org.apache.stormcrawler.protocol.httpclient.HttpProtocol;
 import org.apache.stormcrawler.util.ConfUtils;
 import org.slf4j.LoggerFactory;
 
@@ -48,7 +47,7 @@ public enum ProxyRotation {
     private final AtomicInteger lastAccessedIndex = new AtomicInteger(0);
     private Map<SCProxy, SCProxy> proxyLookupMap;
 
-    private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(HttpProtocol.class);
+    private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MultiProxyManager.class);
 
     /** Default constructor for setting up the proxy manager. */
     private void init(ProxyRotation rotation) {

core/src/main/resources/crawler-default.yaml

@@ -194,8 +194,11 @@ config:
   robots.error.cache.spec: "maximumSize=10000,expireAfterWrite=1h"
 
   protocols: "http,https,file"
-  http.protocol.implementation: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
-  https.protocol.implementation: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
+  # NOTE: the Apache HttpClient protocol implementation
+  # (org.apache.stormcrawler.protocol.httpclient.HttpProtocol) is deprecated and will be removed in
+  # the next major release of StormCrawler. OkHttp is now the default.
+  http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
+  https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
   file.protocol.implementation: "org.apache.stormcrawler.protocol.file.FileProtocol"
 
   # number of instances for each protocol implementation

core/src/test/java/org/apache/stormcrawler/protocol/httpclient/HttpClientProtocolProxyManagerTest.java

@@ -26,6 +26,7 @@
 import org.apache.stormcrawler.proxy.SingleProxyManager;
 import org.junit.jupiter.api.Test;
 
+@SuppressWarnings("deprecation")
 class HttpClientProtocolProxyManagerTest {
 
     @Test

docs/src/main/asciidoc/configuration.adoc

@@ -201,8 +201,10 @@ is defined.
 | http.basicauth.password | - | Password for http.basicauth.user.
 | http.basicauth.user | - | Username for Basic Authentication.
 | http.content.limit | -1 | Maximum HTTP response body size (bytes). Default: no limit.
-| http.protocol.implementation | org.apache.stormcrawler.protocol.httpclient.HttpProtocol | HTTP Protocol
-implementation.
+| http.protocol.implementation | org.apache.stormcrawler.protocol.okhttp.HttpProtocol | HTTP Protocol
+implementation. Note: the Apache HttpClient implementation
+(`org.apache.stormcrawler.protocol.httpclient.HttpProtocol`) is *deprecated* and will be removed in
+the next major release.
 | http.proxy | - | Full **SCProxy** connection string read by SingleProxyManager when that manager is selected, e.g. `http://user:pass@proxy.example.com:8080`.
 | http.proxy.file | - | Proxy connection-string file read by MultiProxyManager when that manager is selected.
 | http.proxy.host | - | SingleProxyManager component proxy host.
@@ -225,8 +227,10 @@ implementation.
 | http.store.headers | false | Whether to store response headers.
 | http.timeout | 10000 | Connection timeout (ms).
 | http.use.cookies | false | Use cookies in subsequent requests.
-| https.protocol.implementation | org.apache.stormcrawler.protocol.httpclient.HttpProtocol | HTTPS Protocol
-implementation.
+| https.protocol.implementation | org.apache.stormcrawler.protocol.okhttp.HttpProtocol | HTTPS Protocol
+implementation. Note: the Apache HttpClient implementation
+(`org.apache.stormcrawler.protocol.httpclient.HttpProtocol`) is *deprecated* and will be removed in
+the next major release.
 | partition.url.mode | byHost | Defines how URLs are partitioned: byHost, byDomain, or byIP.
 | protocols | http,https,file | Supported protocols.
 | redirections.allowed | true | If true, emit redirect target URLs as "outlinks" to the status stream. If false, do not follow redirects. See also `http.allow.redirects`.

docs/src/main/asciidoc/internals.adoc

@@ -395,13 +395,13 @@ The following network protocols are implemented in StormCrawler:
 * link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/file/FileProtocol.java[FileProtocol]
 
 ===== DelegatorProtocol
-The link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java[DelegatorProtocol] enables selection from a collection of sub-protocols using filters based on metadata and URL patterns. The last protocol in the list acts as the default. This is useful when certain URLs need to be fetched with a browser-based protocol (e.g., Selenium) while the majority use a standard HTTP client.
+The link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/DelegatorProtocol.java[DelegatorProtocol] enables selection from a collection of sub-protocols using filters based on metadata and URL patterns. The last protocol in the list acts as the default. This is useful when certain URLs need to be fetched with a browser-based protocol (e.g., Playwright) while the majority use a standard HTTP client.
 
 [source,yaml]
 ----
 http.protocol.implementation: "org.apache.stormcrawler.protocol.DelegatorProtocol"
 protocol.delegator.config:
-  - className: "org.apache.stormcrawler.protocol.httpclient.HttpProtocol"
+  - className: "org.apache.stormcrawler.protocol.playwright.HttpProtocol"
     filters:
       domain: "example.com"
   - className: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
@@ -421,7 +421,7 @@ http.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProto
 https.protocol.implementation: "org.apache.stormcrawler.protocol.okhttp.HttpProtocol"
 ----
 
-* link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java[HttpClient]
+* link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/httpclient/HttpProtocol.java[HttpClient] (*deprecated* — will be removed in the next major release; use OKHttp instead)
 * link:https://github.com/apache/stormcrawler/blob/main/external/selenium/src/main/java/org/apache/stormcrawler/protocol/selenium/SeleniumProtocol.java[Selenium]
 * link:https://github.com/apache/stormcrawler/blob/main/core/src/main/java/org/apache/stormcrawler/protocol/okhttp/HttpProtocol.java[OKHttp]