Merge pull request #251 from kris-sigur/Issue-116

Issue 116 and 250

Author: Kristinn Sigurðsson

pom.xml

@@ -388,6 +388,12 @@
 	</developers>
 
 	<contributors>
+		<contributor>
+			<name>Mohamed Elsayed</name>
+			<url>https://github.com/MohammedElsayyed</url>
+			<organization>The New Library of Alexandria</organization>
+			<organizationUrl>http://bibalex.org/</organizationUrl>
+		</contributor>
 	</contributors>
 
 	<reporting>

src/site/xdoc/release_notes.xml

@@ -125,6 +125,12 @@
       <version>2.5.1</version>
       <scope>test</scope>
     </dependency>
+    <dependency>  
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <version>4.3.5</version>
+      <type>jar</type>
+    </dependency>
   </dependencies>
 
 </project>

wayback-core/pom.xml

@@ -0,0 +1,240 @@
+/*
+ *  This file is part of the Wayback archival access software
+ *   (http://archive-access.sourceforge.net/projects/wayback/).
+ *
+ *  Licensed to the Internet Archive (IA) by one or more individual 
+ *  contributors. 
+ *
+ *  The IA licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.archive.wayback.liveweb;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.net.ConnectException;
+import java.net.SocketException;
+import java.net.SocketTimeoutException;
+import java.net.URL;
+import java.util.logging.Logger;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.commons.httpclient.ConnectTimeoutException;
+import org.apache.commons.httpclient.HostConfiguration;
+import org.apache.commons.httpclient.HttpClient;
+import org.apache.commons.httpclient.HttpMethod;
+import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
+import org.apache.commons.httpclient.NoHttpResponseException;
+import org.apache.commons.httpclient.methods.GetMethod;
+import org.apache.commons.httpclient.params.HttpClientParams;
+import org.archive.io.arc.ARCRecord;
+import org.archive.wayback.core.Resource;
+import org.archive.wayback.exception.LiveDocumentNotAvailableException;
+import org.archive.wayback.exception.LiveWebCacheUnavailableException;
+import org.archive.wayback.exception.LiveWebTimeoutException;
+import org.archive.wayback.exception.ResourceNotAvailableException;
+import org.archive.wayback.resourcestore.resourcefile.ArcResource;
+import org.archive.wayback.resourcestore.resourcefile.ResourceFactory;
+
+/**
+ * This class fetches resource from live web. 
+ * It works with {@link ARCRecordingProxy} not standard proxy servers.
+ * 
+ * @author brad
+ * @see LiveWebCache
+ * @see StdRemoteLiveWebCache
+ *
+ */
+public class ArcRemoteLiveWebCache implements LiveWebCache {
+	private static final Logger LOGGER = Logger.getLogger(
+			ArcRemoteLiveWebCache.class.getName());
+
+    protected MultiThreadedHttpConnectionManager connectionManager = null;
+    protected HostConfiguration hostConfiguration = null;
+    protected HttpClient http = null; 
+    protected String requestPrefix = null;
+
+    /**
+     * 
+     */
+    public ArcRemoteLiveWebCache() {
+    	connectionManager = new MultiThreadedHttpConnectionManager();
+    	hostConfiguration = new HostConfiguration();
+		HttpClientParams params = new HttpClientParams();
+        params.setParameter(HttpClientParams.RETRY_HANDLER, new NoRetryHandler());
+    	http = new HttpClient(params,connectionManager);
+    	http.setHostConfiguration(hostConfiguration);
+    }
+
+    /* (non-Javadoc)
+	 * @see org.archive.wayback.liveweb.LiveWebCache#getCachedResource(java.net.URL, long, boolean)
+	 */
+	public Resource getCachedResource(URL url, long maxCacheMS,
+			boolean bUseOlder) throws LiveDocumentNotAvailableException,
+			LiveWebCacheUnavailableException, LiveWebTimeoutException, IOException {
+		String urlString = url.toExternalForm();
+		
+		if (requestPrefix != null) {
+			urlString = requestPrefix + urlString;
+		}
+		
+		HttpMethod method = null;
+		try {
+			method = new GetMethod(urlString);
+		} catch(IllegalArgumentException e) {
+			LOGGER.warning("Bad URL for live web fetch:" + urlString);
+			throw new LiveDocumentNotAvailableException("Url:" + urlString +
+					"does not look like an URL?");
+		}
+		boolean success = false;
+	    try {
+	    	int status = http.executeMethod(method);
+	    	if(status == 200) {
+
+	    		ByteArrayInputStream bais = new ByteArrayInputStream(method.getResponseBody());
+	    		ARCRecord r = new ARCRecord(
+	    				new GZIPInputStream(bais),
+	    				"id",0L,false,false,true);
+	    		ArcResource ar = (ArcResource) 
+	    			ResourceFactory.ARCArchiveRecordToResource(r, null);
+	    		if(ar.getStatusCode() == 502) {
+	    			throw new LiveDocumentNotAvailableException(urlString);
+	    		} else if(ar.getStatusCode() == 504) {
+	    			throw new LiveWebTimeoutException("Timeout:" + urlString);
+	    		}
+	    		success = true;
+	    		return ar;
+	    		
+	    	} else {
+	    		throw new LiveWebCacheUnavailableException(urlString);
+	    	}
+
+	    } catch (ResourceNotAvailableException e) {
+    		throw new LiveDocumentNotAvailableException(urlString);
+
+	    } catch (NoHttpResponseException e) {
+
+	    	throw new LiveWebCacheUnavailableException("No Http Response for "
+	    			+ urlString);
+
+	    } catch (ConnectException e) {
+    		throw new LiveWebCacheUnavailableException(e.getLocalizedMessage() 
+    				+ " : " + urlString);
+	    } catch (SocketException e) {
+    		throw new LiveWebCacheUnavailableException(e.getLocalizedMessage() 
+    				+ " : " + urlString);
+	    } catch (SocketTimeoutException e) {
+    		throw new LiveWebTimeoutException(e.getLocalizedMessage() 
+    				+ " : " + urlString);
+	    } catch(ConnectTimeoutException e) {
+    		throw new LiveWebTimeoutException(e.getLocalizedMessage() 
+    				+ " : " + urlString);	    	
+		} finally {
+			if (!success) {
+				method.abort();
+			}
+	    	method.releaseConnection();
+	    }
+	}
+
+	/* (non-Javadoc)
+	 * @see org.archive.wayback.liveweb.LiveWebCache#shutdown()
+	 */
+	public void shutdown() {
+		// TODO Auto-generated method stub
+	}
+    
+
+    /**
+     * @param hostPort to proxy requests through - ex. "localhost:3128"
+     */
+    public void setProxyHostPort(String hostPort) {
+    	int colonIdx = hostPort.indexOf(':');
+    	if(colonIdx > 0) {
+    		String host = hostPort.substring(0,colonIdx);
+    		int port = Integer.valueOf(hostPort.substring(colonIdx+1));
+    		
+//            http.getHostConfiguration().setProxy(host, port);
+    		hostConfiguration.setProxy(host, port);
+    	}
+    }
+    /**
+     * @param maxTotalConnections the HttpConnectionManagerParams config
+     */
+    public void setMaxTotalConnections(int maxTotalConnections) {
+    	connectionManager.getParams().
+    		setMaxTotalConnections(maxTotalConnections);
+    }
+    /**
+     * @return the HttpConnectionManagerParams maxTotalConnections config
+     */
+    public int getMaxTotalConnections() {
+    	return connectionManager.getParams().getMaxTotalConnections();
+    }
+ 
+    /**
+     * @param maxHostConnections the HttpConnectionManagerParams config 
+     */
+    public void setMaxHostConnections(int maxHostConnections) {
+    	connectionManager.getParams().
+    		setMaxConnectionsPerHost(hostConfiguration, maxHostConnections);
+    }
+
+    /**
+     * @return the HttpConnectionManagerParams maxHostConnections config 
+     */
+    public int getMaxHostConnections() {
+    	return connectionManager.getParams().
+    		getMaxConnectionsPerHost(hostConfiguration);
+    }
+
+    /**
+	 * @return the connectionTimeoutMS
+	 */
+	public int getConnectionTimeoutMS() {
+		return connectionManager.getParams().getConnectionTimeout();
+	}
+
+	/**
+	 * @param connectionTimeoutMS the connectionTimeoutMS to set
+	 */
+	public void setConnectionTimeoutMS(int connectionTimeoutMS) {
+    	connectionManager.getParams().setConnectionTimeout(connectionTimeoutMS);
+	}
+
+	/**
+	 * @return the socketTimeoutMS
+	 */
+	public int getSocketTimeoutMS() {
+		return connectionManager.getParams().getSoTimeout();
+	}
+
+	/**
+	 * @param socketTimeoutMS the socketTimeoutMS to set
+	 */
+	public void setSocketTimeoutMS(int socketTimeoutMS) {
+    	connectionManager.getParams().setSoTimeout(socketTimeoutMS);
+	}
+
+	public String getRequestPrefix() {
+		return requestPrefix;
+	}
+
+	public void setRequestPrefix(String requestPrefix) {
+		this.requestPrefix = requestPrefix;
+	}
+	
+	public HttpClient getHttpClient()
+	{
+		return http;
+	}
+}

wayback-core/src/main/java/org/archive/wayback/liveweb/ArcRemoteLiveWebCache.java

@@ -16,7 +16,7 @@
 
 import com.google.common.io.ByteStreams;
 
-public class LiveRobotsNoCache extends RemoteLiveWebCache {
+public class LiveRobotsNoCache extends ArcRemoteLiveWebCache {
 
 	protected int maxRobotsSize = 512000;