Pluggable robot rule parser (#1987)

* Don't trigger Mvn workflow for changes to the docs

Signed-off-by: Julien Nioche <julien@digitalpebble.com>

* Pluggable Robots Rule Parser

Signed-off-by: Julien Nioche <julien@digitalpebble.com>

* Format fix

Signed-off-by: Julien Nioche <julien@digitalpebble.com>

* Revert "Don't trigger Mvn workflow for changes to the docs"

This reverts commit 9152167.

* Added config to documentation

Signed-off-by: Julien Nioche <julien@digitalpebble.com>

---------

Signed-off-by: Julien Nioche <julien@digitalpebble.com>

Author: jnioche

core/src/main/java/org/apache/stormcrawler/protocol/AbstractHttpProtocol.java

@@ -58,7 +58,7 @@ public abstract class AbstractHttpProtocol implements Protocol {
     private static final DateTimeFormatter ISO_INSTANT_FORMATTER =
             DateTimeFormatter.ISO_INSTANT.withZone(ZoneId.of(ZoneOffset.UTC.toString()));
 
-    private org.apache.stormcrawler.protocol.HttpRobotRulesParser robots;
+    private org.apache.stormcrawler.protocol.RobotRulesParser robots;
 
     protected boolean skipRobots = false;
 
@@ -121,7 +121,15 @@ public void configure(Config conf) {
             customHeaders.add(KeyValue.build(h));
         }
 
-        robots = new HttpRobotRulesParser(conf);
+        String robotsParserImplementation =
+                ConfUtils.getString(
+                        conf,
+                        "http.robots.parser.class",
+                        "org.apache.stormcrawler.protocol.HttpRobotRulesParser");
+        robots =
+                InitialisationUtil.initializeFromQualifiedName(
+                        robotsParserImplementation, RobotRulesParser.class);
+        robots.setConf(conf);
         protocolMetadataPrefix =
                 ConfUtils.getString(
                         conf, ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, protocolMetadataPrefix);

core/src/main/java/org/apache/stormcrawler/protocol/HttpRobotRulesParser.java

@@ -48,7 +48,7 @@ public class HttpRobotRulesParser extends RobotRulesParser {
 
     private static final int MAX_NUM_REDIRECTS = 5;
 
-    HttpRobotRulesParser() {}
+    public HttpRobotRulesParser() {}
 
     public HttpRobotRulesParser(Config conf) {
         setConf(conf);

core/src/main/java/org/apache/stormcrawler/protocol/RobotRules.java

@@ -35,6 +35,17 @@ public RobotRules(BaseRobotRules base) {
         this.base = base;
     }
 
+    /**
+     * Returns the {@link BaseRobotRules} wrapped by this instance. {@link HttpRobotRulesParser}
+     * always wraps the rules returned by {@link RobotRulesParser#parseRules} in a plain {@code
+     * RobotRules} for content-length tracking (see {@link #getContentLengthFetched()}), so a custom
+     * {@link RobotRulesParser} returning its own {@code BaseRobotRules} subclass (e.g. to expose
+     * additional robots.txt directives) needs this accessor to unwrap it again.
+     */
+    public BaseRobotRules getWrapped() {
+        return base;
+    }
+
     @Override
     public boolean isAllowed(String url) {
         return base.isAllowed(url);

core/src/main/resources/crawler-default.yaml

@@ -182,6 +182,13 @@ config:
   # http.robots.content.limit: 524288  # 512 kiB
   http.robots.content.limit: -1  # default same as http.content.limit
 
+  # Implementation of RobotRulesParser used by the HTTP protocol implementations
+  # to fetch and parse robots.txt. Override to plug in custom robots.txt
+  # directive handling (e.g. parsing extensions not supported by crawler-commons).
+  # The class must extend org.apache.stormcrawler.protocol.RobotRulesParser and
+  # have a public no-arg constructor.
+  http.robots.parser.class: "org.apache.stormcrawler.protocol.HttpRobotRulesParser"
+
   # Guava caches used for the robots.txt directives
   robots.cache.spec: "maximumSize=10000,expireAfterWrite=6h"
   robots.error.cache.spec: "maximumSize=10000,expireAfterWrite=1h"

docs/src/main/asciidoc/configuration.adoc

@@ -245,6 +245,7 @@ implementation.
 | file.encoding | UTF-8 | Encoding for FileProtocol.
 | protocol.instances.num | 1 | Number of instances per protocol implementation.
 | http.protocol.versions | - | HTTP protocol versions in order of preference (h2, http/1.1, http/1.0, h2c). If empty, uses implementation defaults.
+| http.robots.parser.class | org.apache.stormcrawler.protocol.HttpRobotRulesParser | Implementation to use for parsing robots.txt files.
 | robots.cache.spec | maximumSize=10000,expireAfterWrite=6h | CacheBuilder configuration for robots cache.
 | robots.error.cache.spec | maximumSize=10000,expireAfterWrite=1h | CacheBuilder configuration for error cache.
 | okhttp.protocol.connection.pool.max.idle.connections | 5 | OkHttp maximum number of idle connections.