#108 Replace custom HttpHeaders constants with the org.apache.http.HttpHeaders ones (#1587)

* Replace custom HttpHeaders constants with the org.apache.http.HttpHeaders ones

* Remove org.apache.httpcomponents.core5.httpcore5 explicit dependency and maintain consistency using the implicit httpcore version came from org.apache.httpcomponents.httpclient

* Replace hardcoded header keys to constants

* Fix code formatting

* Use ROOT Locale for toLowerCase()

Author: sigee

core/src/main/java/org/apache/stormcrawler/Metadata.java

@@ -25,6 +25,7 @@
 import java.util.ConcurrentModificationException;
 import java.util.HashMap;
 import java.util.Iterator;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Set;
@@ -89,7 +90,7 @@ public void putAll(Metadata m, String prefix) {
      * @return the first value for the key or null if it does not exist *
      */
     public String getFirstValue(String key) {
-        String[] values = md.get(key);
+        String[] values = getValues(key);
         if (values == null) return null;
         if (values.length == 0) return null;
         return values[0];
@@ -109,14 +110,15 @@ public String[] getValues(String key, String prefix) {
     }
 
     public String[] getValues(String key) {
-        String[] values = md.get(key);
+        if (key == null || key.isEmpty()) return null;
+        String[] values = md.getOrDefault(key, md.get(key.toLowerCase(Locale.ROOT)));
         if (values == null) return null;
         if (values.length == 0) return null;
         return values;
     }
 
     public boolean containsKey(String key) {
-        return md.containsKey(key);
+        return md.containsKey(key) || md.containsKey(key.toLowerCase(Locale.ROOT));
     }
 
     public boolean containsKeyWithValue(String key, String value) {

core/src/main/java/org/apache/stormcrawler/bolt/FeedParserBolt.java

@@ -35,6 +35,7 @@
 import java.util.Map;
 import java.util.TimeZone;
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpHeaders;
 import org.apache.storm.task.OutputCollector;
 import org.apache.storm.task.TopologyContext;
 import org.apache.storm.topology.OutputFieldsDeclarer;
@@ -48,7 +49,6 @@
 import org.apache.stormcrawler.parse.ParseFilters;
 import org.apache.stormcrawler.parse.ParseResult;
 import org.apache.stormcrawler.persistence.Status;
-import org.apache.stormcrawler.protocol.HttpHeaders;
 import org.apache.stormcrawler.protocol.ProtocolResponse;
 import org.apache.stormcrawler.util.ConfUtils;
 import org.slf4j.LoggerFactory;

core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java

@@ -38,6 +38,7 @@
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.regex.Pattern;
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpHeaders;
 import org.apache.storm.Config;
 import org.apache.storm.metric.api.MeanReducer;
 import org.apache.storm.metric.api.MultiCountMetric;
@@ -53,7 +54,6 @@
 import org.apache.stormcrawler.Constants;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.persistence.Status;
-import org.apache.stormcrawler.protocol.HttpHeaders;
 import org.apache.stormcrawler.protocol.Protocol;
 import org.apache.stormcrawler.protocol.ProtocolFactory;
 import org.apache.stormcrawler.protocol.ProtocolResponse;

core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java

@@ -33,6 +33,7 @@
 import java.util.Map;
 import java.util.stream.Stream;
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpHeaders;
 import org.apache.storm.metric.api.MultiCountMetric;
 import org.apache.storm.task.OutputCollector;
 import org.apache.storm.task.TopologyContext;
@@ -53,7 +54,6 @@
 import org.apache.stormcrawler.parse.ParseResult;
 import org.apache.stormcrawler.parse.TextExtractor;
 import org.apache.stormcrawler.persistence.Status;
-import org.apache.stormcrawler.protocol.HttpHeaders;
 import org.apache.stormcrawler.protocol.ProtocolResponse;
 import org.apache.stormcrawler.util.CharsetIdentification;
 import org.apache.stormcrawler.util.ConfUtils;

core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java

@@ -30,6 +30,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpHeaders;
 import org.apache.storm.Config;
 import org.apache.storm.metric.api.IMetric;
 import org.apache.storm.metric.api.MeanReducer;
@@ -45,7 +46,6 @@
 import org.apache.stormcrawler.Constants;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.persistence.Status;
-import org.apache.stormcrawler.protocol.HttpHeaders;
 import org.apache.stormcrawler.protocol.Protocol;
 import org.apache.stormcrawler.protocol.ProtocolFactory;
 import org.apache.stormcrawler.protocol.ProtocolResponse;

core/src/main/java/org/apache/stormcrawler/bolt/SiteMapParserBolt.java

@@ -43,6 +43,7 @@
 import java.util.Map;
 import java.util.TimeZone;
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpHeaders;
 import org.apache.storm.metric.api.MeanReducer;
 import org.apache.storm.metric.api.ReducedMetric;
 import org.apache.storm.task.OutputCollector;
@@ -59,7 +60,6 @@
 import org.apache.stormcrawler.parse.ParseResult;
 import org.apache.stormcrawler.persistence.DefaultScheduler;
 import org.apache.stormcrawler.persistence.Status;
-import org.apache.stormcrawler.protocol.HttpHeaders;
 import org.apache.stormcrawler.util.ConfUtils;
 import org.slf4j.LoggerFactory;
 

core/src/main/java/org/apache/stormcrawler/persistence/AdaptiveScheduler.java

@@ -23,10 +23,10 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.TimeZone;
+import org.apache.http.HttpHeaders;
 import org.apache.stormcrawler.Constants;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.parse.filter.MD5SignatureParseFilter;
-import org.apache.stormcrawler.protocol.HttpHeaders;
 import org.apache.stormcrawler.util.ConfUtils;
 import org.slf4j.LoggerFactory;
 

core/src/main/java/org/apache/stormcrawler/protocol/AbstractHttpProtocol.java

@@ -17,8 +17,14 @@
 package org.apache.stormcrawler.protocol;
 
 import crawlercommons.robots.BaseRobotRules;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Locale;
 import org.apache.commons.lang.StringUtils;
 import org.apache.storm.Config;
 import org.apache.stormcrawler.proxy.ProxyManager;
@@ -30,6 +36,27 @@ public abstract class AbstractHttpProtocol implements Protocol {
 
     private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(AbstractHttpProtocol.class);
 
+    /**
+     * Formatter for dates in HTTP headers, used to fill the "If-Modified-Since" request
+     * header field, e.g.
+     *
+     * <pre>
+     * Sun, 06 Nov 1994 08:49:37 GMT
+     * </pre>
+     *
+     * See <a href= "https://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.3.1">sec. 3.3.1 in
+     * RFC 2616</a> and <a href="https://tools.ietf.org/html/rfc7231#section-7.1.1.1">sec. 7.1.1.1
+     * in RFC 7231</a>. The latter specifies the format defined in RFC 1123 as the
+     * &quot;preferred&quot; format.
+     */
+    private static final DateTimeFormatter HTTP_DATE_FORMATTER =
+            DateTimeFormatter.ofPattern("EEE, dd MMM yyyy HH:mm:ss 'GMT'", Locale.ROOT)
+                    .withZone(ZoneId.of(ZoneOffset.UTC.toString()));
+
+    /** Formatter to parse ISO-formatted dates persisted in status index */
+    private static final DateTimeFormatter ISO_INSTANT_FORMATTER =
+            DateTimeFormatter.ISO_INSTANT.withZone(ZoneId.of(ZoneOffset.UTC.toString()));
+
     private org.apache.stormcrawler.protocol.HttpRobotRulesParser robots;
 
     protected boolean skipRobots = false;
@@ -189,4 +216,29 @@ private static String getAgentString(
 
         return buf.toString();
     }
+
+    /**
+     * Format an ISO date string as HTTP date used in HTTP headers, e.g.,
+     *
+     * <pre>
+     * 1994-11-06T08:49:37.000Z
+     * </pre>
+     *
+     * is formatted to
+     *
+     * <pre>
+     * Sun, 06 Nov 1994 08:49:37 GMT
+     * </pre>
+     *
+     * See {@link #HTTP_DATE_FORMATTER}
+     */
+    protected static String formatHttpDate(String isoDate) {
+        try {
+            ZonedDateTime date = ISO_INSTANT_FORMATTER.parse(isoDate, ZonedDateTime::from);
+            return HTTP_DATE_FORMATTER.format(date);
+        } catch (DateTimeParseException e) {
+            // not an ISO date
+            return "";
+        }
+    }
 }

core/src/main/java/org/apache/stormcrawler/protocol/HttpHeaders.java

@@ -26,6 +26,7 @@
 import java.util.Locale;
 import java.util.Set;
 import org.apache.commons.lang.StringUtils;
+import org.apache.http.HttpHeaders;
 import org.apache.storm.Config;
 import org.apache.stormcrawler.Metadata;
 import org.apache.stormcrawler.util.ConfUtils;