Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replaced deprecated rome-fetcher with java 11 HttpClient #92

Merged
merged 1 commit into from
Jul 4, 2021
Merged

Replaced deprecated rome-fetcher with java 11 HttpClient #92

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion app/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -506,7 +506,7 @@ limitations under the License.

<dependency>
<groupId>com.rometools</groupId>
<artifactId>rome-fetcher</artifactId>
<artifactId>rome</artifactId>
<version>${rome.version}</version>
<scope>compile</scope>
<exclusions>
Expand Down
138 changes: 35 additions & 103 deletions app/src/main/java/org/apache/roller/planet/business/fetcher/RomeFeedFetcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,16 @@
import com.rometools.rome.feed.synd.SyndContent;
import com.rometools.rome.feed.synd.SyndEntry;
import com.rometools.rome.feed.synd.SyndFeed;
import com.rometools.fetcher.FeedFetcher;
import com.rometools.fetcher.impl.FeedFetcherCache;
import com.rometools.fetcher.impl.HttpURLFeedFetcher;
import com.rometools.fetcher.impl.SyndFeedInfo;
import com.rometools.fetcher.impl.DiskFeedInfoCache;
import com.rometools.rome.io.FeedException;
import com.rometools.rome.io.SyndFeedInput;
import com.rometools.rome.io.XmlReader;

import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.io.IOException;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.sql.Timestamp;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
Expand All @@ -43,28 +43,35 @@
import org.apache.commons.logging.LogFactory;
import org.apache.roller.planet.pojos.SubscriptionEntry;
import org.apache.roller.planet.pojos.Subscription;
import org.apache.roller.weblogger.config.WebloggerConfig;

import static java.net.http.HttpResponse.BodyHandlers.ofInputStream;


/**
* A FeedFetcher based on the ROME RSS/Atom feed parser (http://rome.dev.java.net).
* A FeedFetcher based on Apache ROME and {@link java.net.http.HttpClient}.
*/
public class RomeFeedFetcher implements org.apache.roller.planet.business.fetcher.FeedFetcher {
public class RomeFeedFetcher implements FeedFetcher {

private static final Log log = LogFactory.getLog(RomeFeedFetcher.class);

private static Log log = LogFactory.getLog(RomeFeedFetcher.class);
// mutable, copy() first
private static final HttpRequest.Builder requestBuilder = HttpRequest.newBuilder()
.timeout(Duration.ofSeconds(3))
.header("User-Agent", "RollerPlanetAggregator");

private final HttpClient client;

public RomeFeedFetcher() {
// no-op
// immutable + thread safe, prefers HTTP/2, no redirects
this.client = HttpClient.newBuilder()
.connectTimeout(Duration.ofSeconds(3)).build();
}


/**
* @inheritDoc
*/
@Override
public Subscription fetchSubscription(String feedURL)
throws FetcherException {
public Subscription fetchSubscription(String feedURL) throws FetcherException {
return fetchSubscription(feedURL, null);
}

Expand All @@ -73,22 +80,18 @@ public Subscription fetchSubscription(String feedURL)
* @inheritDoc
*/
@Override
public Subscription fetchSubscription(String feedURL, Date lastModified)
throws FetcherException {

public Subscription fetchSubscription(String feedURL, Date lastModified) throws FetcherException {

if(feedURL == null) {
throw new IllegalArgumentException("feed url cannot be null");
}

// setup Rome feed fetcher
FeedFetcher feedFetcher = getRomeFetcher();

// fetch the feed
log.debug("Fetching feed: "+feedURL);
SyndFeed feed;
try {
feed = feedFetcher.retrieveFeed(new URL(feedURL));
} catch (Exception ex) {
feed = fetchFeed(feedURL);
} catch (FeedException | IOException | InterruptedException ex) {
throw new FetcherException("Error fetching subscription - "+feedURL, ex);
}

Expand All @@ -112,21 +115,6 @@ public Subscription fetchSubscription(String feedURL, Date lastModified)
// set the author to the title
newSub.setAuthor(newSub.getTitle());
}
if(newSub.getLastUpdated() == null) {
// no update time specified in feed, so try consulting feed info cache
FeedFetcherCache feedCache = getRomeFetcherCache();
try {
SyndFeedInfo feedInfo = feedCache.getFeedInfo(new URL(newSub.getFeedURL()));
if(feedInfo.getLastModified() != null) {
long lastUpdatedLong = (Long) feedInfo.getLastModified();
if (lastUpdatedLong != 0) {
newSub.setLastUpdated(new Date(lastUpdatedLong));
}
}
} catch (MalformedURLException ex) {
// should never happen since we check this above
}
}

// check if feed is unchanged and bail now if so
if(lastModified != null && newSub.getLastUpdated() != null &&
Expand Down Expand Up @@ -162,9 +150,7 @@ public Subscription fetchSubscription(String feedURL, Date lastModified)
cal.add(Calendar.DATE, -1);
}

if(newEntry != null) {
newSub.addEntry(newEntry);
}
newSub.addEntry(newEntry);
}

log.debug(feedEntries.size()+" entries included");
Expand Down Expand Up @@ -230,77 +216,23 @@ private SubscriptionEntry buildEntry(SyndEntry romeEntry) {
// copy categories
if (!romeEntry.getCategories().isEmpty()) {
List<String> list = new ArrayList<>();
for (Object cat : romeEntry.getCategories()) {
list.add(((SyndCategory) cat).getName());
for (SyndCategory cat : romeEntry.getCategories()) {
list.add(cat.getName());
}
newEntry.setCategoriesString(list);
}

return newEntry;
}


// get a feed fetcher cache, if possible
private FeedFetcherCache getRomeFetcherCache() {

String cacheDirPath = WebloggerConfig.getProperty("cache.dir");

// can't continue without cache dir
if (cacheDirPath == null) {
log.warn("Planet cache directory not set, feeds cannot be cached.");
return null;
}

// allow ${user.home} in cache dir property
String cacheDirName = cacheDirPath.replaceFirst(
"\\$\\{user.home}",System.getProperty("user.home"));

// allow ${catalina.home} in cache dir property
if (System.getProperty("catalina.home") != null) {
cacheDirName = cacheDirName.replaceFirst(
"\\$\\{catalina.home}",System.getProperty("catalina.home"));
}

// create cache dir if it does not exist
File cacheDir = null;
try {
cacheDir = new File(cacheDirName);
if (!cacheDir.exists()) {
cacheDir.mkdirs();
}
} catch (Exception e) {
log.error("Unable to create planet cache directory: " +
((cacheDir != null) ? cacheDir.getPath() : null), e);
return null;
}

// abort if cache dir is not writable
if (!cacheDir.canWrite()) {
log.error("Planet cache directory is not writable: " + cacheDir.getPath());
return null;
}

return new DiskFeedInfoCache(cacheDirName);
}


// get a feed fetcher
private FeedFetcher getRomeFetcher() {
private SyndFeed fetchFeed(String url) throws IOException, InterruptedException, FeedException {

FeedFetcherCache feedCache = getRomeFetcherCache();
HttpRequest request = requestBuilder.copy().uri(URI.create(url)).build();

FeedFetcher feedFetcher;
if(feedCache != null) {
feedFetcher = new HttpURLFeedFetcher(feedCache);
} else {
feedFetcher = new HttpURLFeedFetcher();
try(XmlReader reader = new XmlReader(client.send(request, ofInputStream()).body())) {
return new SyndFeedInput().build(reader);
}

// set options
feedFetcher.setUsingDeltaEncoding(false);
feedFetcher.setUserAgent("RollerPlanetAggregator");

return feedFetcher;

}

}
4 changes: 2 additions & 2 deletions app/src/main/java/org/apache/roller/weblogger/planet/business/WebloggerRomeFeedFetcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
*/
public class WebloggerRomeFeedFetcher extends RomeFeedFetcher {

private static Log log = LogFactory.getLog(WebloggerRomeFeedFetcher.class);
private static final Log log = LogFactory.getLog(WebloggerRomeFeedFetcher.class);


/**
Expand Down Expand Up @@ -135,7 +135,7 @@ public Subscription fetchSubscription(String feedURL, Date lastModified)
Map pagePlugins = ppmgr.getWeblogEntryPlugins(localWeblog);
for ( WeblogEntry rollerEntry : entries ) {
SubscriptionEntry entry = new SubscriptionEntry();
String content = "";
String content;
if (!StringUtils.isEmpty(rollerEntry.getText())) {
content = rollerEntry.getText();
} else {
Expand Down
56 changes: 0 additions & 56 deletions app/src/test/java/org/apache/roller/planet/util/rome/DiskFeedInfoCacheTest.java
View file
Open in desktop

This file was deleted.