diff --git a/src/main/scala/com/gravity/goose/network/HtmlFetcher.scala b/src/main/scala/com/gravity/goose/network/HtmlFetcher.scala index 34ebf44a7..78eb0ed8a 100644 --- a/src/main/scala/com/gravity/goose/network/HtmlFetcher.scala +++ b/src/main/scala/com/gravity/goose/network/HtmlFetcher.scala @@ -19,6 +19,7 @@ package com.gravity.goose.network import org.apache.http.HttpEntity +import org.apache.http.HttpHost import org.apache.http.HttpResponse import org.apache.http.HttpVersion import org.apache.http.client.CookieStore @@ -26,6 +27,7 @@ import org.apache.http.client.HttpClient import org.apache.http.client.methods.HttpGet import org.apache.http.client.params.CookiePolicy import org.apache.http.client.protocol.ClientContext +import org.apache.http.conn.params.ConnRoutePNames import org.apache.http.conn.scheme.PlainSocketFactory import org.apache.http.conn.ssl.SSLSocketFactory import org.apache.http.conn.scheme.Scheme @@ -42,6 +44,7 @@ import org.apache.http.util.EntityUtils import java.io._ import java.net.SocketException import java.net.SocketTimeoutException +import java.net.URL import java.net.URLConnection import java.util.ArrayList import java.util.Date @@ -278,6 +281,19 @@ object HtmlFetcher extends AbstractHtmlFetcher with Logging { httpClient.getParams.setParameter("http.conn-manager.timeout", 120000L) httpClient.getParams.setParameter("http.protocol.wait-for-continue", 10000L) httpClient.getParams.setParameter("http.tcp.nodelay", true) + + // First check proxy configured from java properties, otherwise use env var if set + if (scala.sys.props.isDefinedAt("http.proxyHost")) { + val host = scala.sys.props.getOrElse("http.proxyHost", "") + val port = scala.sys.props.getOrElse("http.proxyPort", "80").toInt + httpClient.getParams.setParameter(ConnRoutePNames.DEFAULT_PROXY, new HttpHost(host, port)) + } else if (sys.env.isDefinedAt("http_proxy")) { + val url = new URL(sys.env.getOrElse("http_proxy", "")) + val host = url.getHost + val port = url.getPort + httpClient.getParams.setParameter(ConnRoutePNames.DEFAULT_PROXY, new HttpHost(host, port)) + } + } /**