Merge branch 'main' of https://github.com/Ranchero-Software/RSParser

Author: vincode-io

README.md

@@ -1,6 +1,6 @@
 # RSParser
 
-This framework was developed for [NetNewsWire](https://github.com/brentsimmons/NetNewsWire) and is made available here for developers who just need the parsing code. It has no depencies that aren’t provided by the system.
+This framework was developed for [NetNewsWire](https://github.com/brentsimmons/NetNewsWire) and is made available here for developers who just need the parsing code. It has no dependencies that aren’t provided by the system.
 
 _Update 6 Feb. 2018_: RSParser is now a CocoaPod, with the much-appreciated help of [Silver Fox](https://github.com/dcilia). (We _think_ it worked, anyway. Looked like it did.)
 

Sources/ObjC/NSData+RSParser.m

@@ -58,13 +58,6 @@ - (BOOL)isProbablyRSSInJSON {
 
 - (BOOL)isProbablyRSS {
 
-	if (bytesStartWithRSS(self.bytes, self.length) || bytesStartWithRDF(self.bytes, self.length)) { // Macworld’s RSS feed does not start with xml header.
-		return YES;
-	}
-	if (![self isProbablyXML]) {
-		return NO;
-	}
-
 	if (didFindString("<rss", self.bytes, self.length) || didFindString("<rdf:RDF", self.bytes, self.length)) {
 		return YES;
 	}
@@ -75,13 +68,6 @@ - (BOOL)isProbablyRSS {
 
 - (BOOL)isProbablyAtom {
 
-	if (bytesStartWithAtom(self.bytes, self.length)) { // https://research.swtch.com/feed.atom does not start with xml header.
-		return YES;
-	}
-	if (![self isProbablyXML]) {
-		return NO;
-	}
-
 	return didFindString("<feed", self.bytes, self.length);
 }
 
@@ -151,18 +137,3 @@ static BOOL bytesAreProbablyXML(const char *bytes, NSUInteger numberOfBytes) {
 
 	return bytesStartWithStringIgnoringWhitespace("<?xml", bytes, numberOfBytes);
 }
-
-static BOOL bytesStartWithRSS(const char *bytes, NSUInteger numberOfBytes) {
-
-	return bytesStartWithStringIgnoringWhitespace("<rss", bytes, numberOfBytes);
-}
-
-static BOOL bytesStartWithRDF(const char *bytes, NSUInteger numberOfBytes) {
-
-	return bytesStartWithStringIgnoringWhitespace("<rdf:RDF", bytes, numberOfBytes);
-}
-
-static BOOL bytesStartWithAtom(const char *bytes, NSUInteger numberOfBytes) {
-
-	return bytesStartWithStringIgnoringWhitespace("<feed", bytes, numberOfBytes);
-}

Sources/ObjC/RSHTMLMetadataParser.m

@@ -24,6 +24,7 @@ @interface RSHTMLMetadataParser () <RSSAXHTMLParserDelegate>
 @property (nonatomic, readwrite) RSHTMLMetadata *metadata;
 @property (nonatomic) NSMutableArray *tags;
 @property (nonatomic) BOOL didFinishParsing;
+@property (nonatomic) BOOL shouldScanPastHeadSection;
 
 @end
 
@@ -55,6 +56,11 @@ - (instancetype)initWithParserData:(ParserData *)parserData {
 	_parserData = parserData;
 	_tags = [NSMutableArray new];
 
+	// YouTube has a weird bug where, on some pages, it puts the feed link tag after the head section, in the body section.
+	// This allows for a special case where we continue to scan after the head section.
+	// (Yes, this match could yield false positives, but it’s harmless.)
+	_shouldScanPastHeadSection = [parserData.url rangeOfString:@"youtube" options:NSCaseInsensitiveSearch].location != NSNotFound;
+
 	[self parse];
 
 	return self;
@@ -121,7 +127,7 @@ - (void)saxParser:(RSSAXHTMLParser *)SAXParser XMLStartElement:(const xmlChar *)
 		return;
 	}
 
-	if (RSSAXEqualTags(localName, kBody, kBodyLength)) {
+	if (RSSAXEqualTags(localName, kBody, kBodyLength) && !self.shouldScanPastHeadSection) {
 		self.didFinishParsing = YES;
 		return;
 	}

Tests/RSParserTests/AtomParserTests.swift

@@ -44,12 +44,7 @@ class AtomParserTests: XCTestCase {
 
 		for article in parsedFeed.items {
 
-			XCTAssertNotNil(article.externalURL)
-
-			if !article.title!.hasPrefix("★ ") {
-				XCTAssertNotNil(article.url)
-				XCTAssert(article.url!.hasPrefix("https://daringfireball.net/"))
-			}
+			XCTAssertNotNil(article.url)
 
 			XCTAssertTrue(article.uniqueID.hasPrefix("tag:daringfireball.net,2017:/"))
 

Tests/RSParserTests/FeedParserTypeTests.swift

@@ -106,6 +106,12 @@ class FeedParserTypeTests: XCTestCase {
 		XCTAssertTrue(type == .rss)
 	}
 
+	func testPHPXML() {
+		let d = parserData("phpxml", "rss", "https://www.fcutrecht.net/")
+		let type = feedType(d)
+		XCTAssertTrue(type == .rss)
+	}
+
 	// MARK: Atom
 
 	func testDaringFireballAtomType() {

Tests/RSParserTests/HTMLMetadataTests.swift

@@ -139,4 +139,16 @@ class HTMLMetadataTests: XCTestCase {
 		let imageURL = twitterData.imageURL!
 		XCTAssert(imageURL == "https://cdn.theatlantic.com/assets/media/img/mt/2017/11/1033101_first_full_length_trailer_arrives_pixars_coco/facebook.jpg?1511382177")
 	}
+
+	func testYouTube() {
+		// YouTube is a special case — the feed links appear after the head section, in the body section.
+		let d = parserData("YouTubeTheVolvoRocks", "html", "https://www.youtube.com/user/TheVolvorocks")
+		let metadata = RSHTMLMetadataParser.htmlMetadata(with: d)
+
+		XCTAssertEqual(metadata.feedLinks.count, 1);
+		let feedLink = metadata.feedLinks.first!
+		XCTAssertEqual(feedLink.title, "RSS");
+		XCTAssertEqual(feedLink.type, "application/rss+xml");
+		XCTAssertEqual(feedLink.urlString, "https://www.youtube.com/feeds/videos.xml?channel_id=UCct7QF2jcWRY6dhXWMSq9LQ");
+	}
 }

Tests/RSParserTests/RSDateParserTests.swift

@@ -100,10 +100,9 @@ class RSDateParserTests: XCTestCase {
 		XCTAssertEqual(d, expectedDateResult)
 	}
 
-	func testHighMillisecondDate() {
-		let expectedDateResult = Self.dateWithValues(2021, 03, 29, 10, 46, 56)
-		let d = RSDateWithString("2021-03-29T10:46:56.516941+00:00")
-		XCTAssertEqual(d, expectedDateResult)
-	}
-	
+//	func testHighMillisecondDate() {
+//		let expectedDateResult = Self.dateWithValues(2021, 03, 29, 10, 46, 56)
+//		let d = RSDateWithString("2021-03-29T10:46:56.516941+00:00")
+//		XCTAssertEqual(d, expectedDateResult)
+//	}
 }