commit 3cfa8ac9b9e4b07a78ad0daa16e1f640eba36ff5 Author: Simon Legner Date: Sat Dec 26 00:34:55 2015 +0100 Uniform access to HTTP resources diff --git a/src/org/openstreetmap/josm/gui/help/HelpContentReader.java b/src/org/openstreetmap/josm/gui/help/HelpContentReader.java index 14c569c..29c886f 100644 --- a/src/org/openstreetmap/josm/gui/help/HelpContentReader.java +++ b/src/org/openstreetmap/josm/gui/help/HelpContentReader.java @@ -3,14 +3,10 @@ import java.io.BufferedReader; import java.io.IOException; -import java.io.InputStreamReader; -import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; -import java.nio.charset.StandardCharsets; -import org.openstreetmap.josm.Main; -import org.openstreetmap.josm.tools.Utils; +import org.openstreetmap.josm.tools.HttpClient; import org.openstreetmap.josm.tools.WikiReader; /** @@ -45,12 +41,11 @@ public HelpContentReader(String baseUrl) { public String fetchHelpTopicContent(String helpTopicUrl, boolean dotest) throws HelpContentReaderException { if (helpTopicUrl == null) throw new MissingHelpContentException("helpTopicUrl is null"); - HttpURLConnection con = null; + HttpClient.Response con = null; try { URL u = new URL(helpTopicUrl); - con = Utils.openHttpConnection(u); - con.connect(); - try (BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream(), StandardCharsets.UTF_8))) { + con = HttpClient.create(u).connect(); + try (BufferedReader in = con.getContentReader()) { return prepareHelpContent(in, dotest, u); } } catch (MalformedURLException e) { @@ -58,14 +53,7 @@ public String fetchHelpTopicContent(String helpTopicUrl, boolean dotest) throws } catch (IOException e) { HelpContentReaderException ex = new HelpContentReaderException(e); if (con != null) { - try { - ex.setResponseCode(con.getResponseCode()); - } catch (IOException e1) { - // ignore - if (Main.isTraceEnabled()) { - Main.trace(e1.getMessage()); - } - } + ex.setResponseCode(con.getResponseCode()); } throw ex; } diff --git a/src/org/openstreetmap/josm/io/CachedFile.java b/src/org/openstreetmap/josm/io/CachedFile.java index e8090a7..7cb490a 100644 --- a/src/org/openstreetmap/josm/io/CachedFile.java +++ b/src/org/openstreetmap/josm/io/CachedFile.java @@ -20,13 +20,12 @@ import java.util.Enumeration; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.concurrent.ConcurrentHashMap; import java.util.zip.ZipEntry; import java.util.zip.ZipFile; import org.openstreetmap.josm.Main; -import org.openstreetmap.josm.tools.CheckParameterUtil; +import org.openstreetmap.josm.tools.HttpClient; import org.openstreetmap.josm.tools.Pair; import org.openstreetmap.josm.tools.Utils; @@ -414,7 +413,11 @@ private File checkLocal(URL url) throws IOException { String localPath = "mirror_" + a; destDirFile = new File(destDir, localPath + ".tmp"); try { - HttpURLConnection con = connectFollowingRedirect(url, httpAccept, ifModifiedSince, httpHeaders); + final HttpClient.Response con = HttpClient.create(url) + .setAccept(httpAccept) + .setIfModifiedSince(ifModifiedSince == null ? 0L : ifModifiedSince) + .setHeaders(httpHeaders) + .connect(); if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { if (Main.isDebugEnabled()) { Main.debug("304 Not Modified ("+urlStr+')'); @@ -426,7 +429,7 @@ private File checkLocal(URL url) throws IOException { return localFile; } try ( - InputStream bis = new BufferedInputStream(con.getInputStream()); + InputStream bis = new BufferedInputStream(con.getContent()); OutputStream fos = new FileOutputStream(destDirFile); OutputStream bos = new BufferedOutputStream(fos) ) { @@ -461,115 +464,4 @@ private static void checkOfflineAccess(String urlString) { OnlineResource.OSM_API.checkOfflineAccess(urlString, Main.pref.get("osm-server.url", OsmApi.DEFAULT_API_URL)); } - /** - * Opens a connection for downloading a resource. - *

- * Manually follows redirects because - * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect - * is going from a http to a https URL, see bug report. - *

- * This can cause problems when downloading from certain GitHub URLs. - * - * @param downloadUrl The resource URL to download - * @param httpAccept The accepted MIME types sent in the HTTP Accept header. Can be {@code null} - * @param ifModifiedSince The download time of the cache file, optional - * @return The HTTP connection effectively linked to the resource, after all potential redirections - * @throws MalformedURLException If a redirected URL is wrong - * @throws IOException If any I/O operation goes wrong - * @throws OfflineAccessException if resource is accessed in offline mode, in any protocol - * @since 6867 - */ - public static HttpURLConnection connectFollowingRedirect(URL downloadUrl, String httpAccept, Long ifModifiedSince) - throws MalformedURLException, IOException { - return connectFollowingRedirect(downloadUrl, httpAccept, ifModifiedSince, null); - } - - /** - * Opens a connection for downloading a resource. - *

- * Manually follows redirects because - * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect - * is going from a http to a https URL, see bug report. - *

- * This can cause problems when downloading from certain GitHub URLs. - * - * @param downloadUrl The resource URL to download - * @param httpAccept The accepted MIME types sent in the HTTP Accept header. Can be {@code null} - * @param ifModifiedSince The download time of the cache file, optional - * @param headers http headers to be sent together with http request - * @return The HTTP connection effectively linked to the resource, after all potential redirections - * @throws MalformedURLException If a redirected URL is wrong - * @throws IOException If any I/O operation goes wrong - * @throws OfflineAccessException if resource is accessed in offline mode, in any protocol - * @since TODO - */ - public static HttpURLConnection connectFollowingRedirect(URL downloadUrl, String httpAccept, Long ifModifiedSince, - Map headers) throws MalformedURLException, IOException { - CheckParameterUtil.ensureParameterNotNull(downloadUrl, "downloadUrl"); - String downloadString = downloadUrl.toExternalForm(); - - checkOfflineAccess(downloadString); - - int numRedirects = 0; - while (true) { - HttpURLConnection con = Utils.openHttpConnection(downloadUrl); - if (ifModifiedSince != null) { - con.setIfModifiedSince(ifModifiedSince); - } - if (headers != null) { - for (Entry header: headers.entrySet()) { - con.setRequestProperty(header.getKey(), header.getValue()); - } - } - con.setInstanceFollowRedirects(false); - con.setConnectTimeout(Main.pref.getInteger("socket.timeout.connect", 15)*1000); - con.setReadTimeout(Main.pref.getInteger("socket.timeout.read", 30)*1000); - if (Main.isDebugEnabled()) { - Main.debug("GET "+downloadString); - } - if (httpAccept != null) { - if (Main.isTraceEnabled()) { - Main.trace("Accept: "+httpAccept); - } - con.setRequestProperty("Accept", httpAccept); - } - try { - con.connect(); - } catch (IOException e) { - Main.addNetworkError(downloadUrl, Utils.getRootCause(e)); - throw e; - } - switch(con.getResponseCode()) { - case HttpURLConnection.HTTP_OK: - return con; - case HttpURLConnection.HTTP_NOT_MODIFIED: - if (ifModifiedSince != null) - return con; - case HttpURLConnection.HTTP_MOVED_PERM: - case HttpURLConnection.HTTP_MOVED_TEMP: - case HttpURLConnection.HTTP_SEE_OTHER: - String redirectLocation = con.getHeaderField("Location"); - if (redirectLocation == null) { - /* I18n: argument is HTTP response code */ - String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header."+ - " Can''t redirect. Aborting.", con.getResponseCode()); - throw new IOException(msg); - } - downloadUrl = new URL(redirectLocation); - downloadString = downloadUrl.toExternalForm(); - // keep track of redirect attempts to break a redirect loops if it happens - // to occur for whatever reason - numRedirects++; - if (numRedirects >= Main.pref.getInteger("socket.maxredirects", 5)) { - String msg = tr("Too many redirects to the download URL detected. Aborting."); - throw new IOException(msg); - } - Main.info(tr("Download redirected to ''{0}''", downloadString)); - break; - default: - String msg = tr("Failed to read from ''{0}''. Server responded with status code {1}.", downloadString, con.getResponseCode()); - throw new IOException(msg); - } - } - } } diff --git a/src/org/openstreetmap/josm/plugins/PluginDownloadTask.java b/src/org/openstreetmap/josm/plugins/PluginDownloadTask.java index 33e411b..3dfe19a 100644 --- a/src/org/openstreetmap/josm/plugins/PluginDownloadTask.java +++ b/src/org/openstreetmap/josm/plugins/PluginDownloadTask.java @@ -9,7 +9,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; import java.util.Collection; @@ -21,8 +20,8 @@ import org.openstreetmap.josm.gui.PleaseWaitRunnable; import org.openstreetmap.josm.gui.progress.NullProgressMonitor; import org.openstreetmap.josm.gui.progress.ProgressMonitor; -import org.openstreetmap.josm.io.CachedFile; import org.openstreetmap.josm.tools.CheckParameterUtil; +import org.openstreetmap.josm.tools.HttpClient; import org.xml.sax.SAXException; /** @@ -44,7 +43,7 @@ private final Collection failed = new LinkedList<>(); private final Collection downloaded = new LinkedList<>(); private boolean canceled; - private HttpURLConnection downloadConnection; + private HttpClient.Response downloadConnection; /** * Creates the download task @@ -123,10 +122,12 @@ protected void download(PluginInformation pi, File file) throws PluginDownloadEx } URL url = new URL(pi.downloadlink); synchronized (this) { - downloadConnection = CachedFile.connectFollowingRedirect(url, PLUGIN_MIME_TYPES, null); + downloadConnection = HttpClient.create(url) + .setAccept(PLUGIN_MIME_TYPES) + .connect(); } try ( - InputStream in = downloadConnection.getInputStream(); + InputStream in = downloadConnection.getContent(); OutputStream out = new FileOutputStream(file) ) { byte[] buffer = new byte[8192]; diff --git a/src/org/openstreetmap/josm/tools/HttpClient.java b/src/org/openstreetmap/josm/tools/HttpClient.java new file mode 100644 index 0000000..ab44a17 --- /dev/null +++ b/src/org/openstreetmap/josm/tools/HttpClient.java @@ -0,0 +1,321 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.tools; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.io.BufferedOutputStream; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.zip.GZIPInputStream; + +import org.openstreetmap.josm.Main; +import org.openstreetmap.josm.data.Version; + +/** + * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}. + */ +public class HttpClient { + + private URL url; + private final String requestMethod; + private int connectTimeout = Main.pref.getInteger("socket.timeout.connect", 15) * 1000; + private int readTimeout = Main.pref.getInteger("socket.timeout.read", 30) * 1000; + private String accept; + private String contentType; + private String acceptEncoding = "gzip"; + private long contentLength; + private byte[] requestBody; + private long ifModifiedSince; + private final Map headers = new ConcurrentHashMap<>(); + private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5); + + private HttpClient(URL url, String requestMethod) { + this.url = url; + this.requestMethod = requestMethod; + } + + public Response connect() throws IOException { + final HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString()); + connection.setConnectTimeout(connectTimeout); + connection.setReadTimeout(readTimeout); + if (accept != null) { + connection.setRequestProperty("Accept", accept); + } + if (contentType != null) { + connection.setRequestProperty("Content-Type", contentType); + } + if (acceptEncoding != null) { + connection.setRequestProperty("Accept-Encoding", acceptEncoding); + } + if (contentLength > 0) { + connection.setRequestProperty("Content-Length", String.valueOf(contentLength)); + } + if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) { + connection.setDoOutput(true); + try (OutputStream out = new BufferedOutputStream(connection.getOutputStream())) { + out.write(requestBody); + } + } + if (ifModifiedSince > 0) { + connection.setIfModifiedSince(ifModifiedSince); + } + for (Map.Entry header : headers.entrySet()) { + connection.setRequestProperty(header.getKey(), header.getValue()); + } + + boolean successfulConnection = false; + try { + try { + connection.connect(); + } catch (IOException e) { + //noinspection ThrowableResultOfMethodCallIgnored + Main.addNetworkError(url, Utils.getRootCause(e)); + throw e; + } + if (isRedirect(connection.getResponseCode())) { + final String redirectLocation = connection.getHeaderField("Location"); + if (redirectLocation == null) { + /* I18n: argument is HTTP response code */ + String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." + + " Can''t redirect. Aborting.", connection.getResponseCode()); + throw new IOException(msg); + } else if (maxRedirects > 0) { + url = new URL(redirectLocation); + maxRedirects--; + Main.info(tr("Download redirected to ''{0}''", redirectLocation)); + return connect(); + } else { + String msg = tr("Too many redirects to the download URL detected. Aborting."); + throw new IOException(msg); + } + } + Response response = new Response(connection); + successfulConnection = true; + return response; + } finally { + if (!successfulConnection) { + connection.disconnect(); + } + } + } + + /** + * A wrapper for the HTTP response. + */ + public static class Response { + private final HttpURLConnection connection; + private final int responseCode; + + private Response(HttpURLConnection connection) throws IOException { + this.connection = connection; + this.responseCode = connection.getResponseCode(); + } + + /** + * Returns an input stream that reads from this HTTP connection, or, + * error stream if the connection failed but the server sent useful data. + * + * @see HttpURLConnection#getInputStream() + * @see HttpURLConnection#getErrorStream() + */ + public InputStream getContent() throws IOException { + InputStream in; + try { + in = connection.getInputStream(); + } catch (IOException ioe) { + in = connection.getErrorStream(); + } + return "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in; + } + + /** + * Returns {@link #getContent()} wrapped in a buffered reader + */ + public BufferedReader getContentReader() throws IOException { + return new BufferedReader(new InputStreamReader(getContent(), StandardCharsets.UTF_8)); + } + + /** + * Gets the response code from this HTTP connection. + * + * @see HttpURLConnection#getResponseCode() + */ + public int getResponseCode() { + return responseCode; + } + + /** + * Returns the {@code Content-Encoding} header. + */ + public String getContentEncoding() { + return connection.getContentEncoding(); + } + + /** + * Returns the {@code Content-Type} header. + */ + public String getContentType() { + return connection.getHeaderField("Content-Type"); + } + + /** + * @see HttpURLConnection#disconnect() + */ + public void disconnect() { + connection.disconnect(); + } + } + + /** + * Creates a new instance for the given URL and a {@code GET} request + * + * @param url the URL + * @return a new instance + */ + public static HttpClient create(URL url) { + return create(url, "GET"); + } + + /** + * Creates a new instance for the given URL and a {@code GET} request + * + * @param url the URL + * @param requestMethod the HTTP request method to perform when calling + * @return a new instance + */ + public static HttpClient create(URL url, String requestMethod) { + return new HttpClient(url, requestMethod); + } + + /** + * @return {@code this} + * @see HttpURLConnection#setConnectTimeout(int) + */ + public HttpClient setConnectTimeout(int connectTimeout) { + this.connectTimeout = connectTimeout; + return this; + } + + /** + * @return {@code this} + * @see HttpURLConnection#setReadTimeout(int) (int) + */ + + public HttpClient setReadTimeout(int readTimeout) { + this.readTimeout = readTimeout; + return this; + } + + /** + * Sets the {@code Accept} header. + * + * @return {@code this} + */ + public HttpClient setAccept(String accept) { + this.accept = accept; + return this; + } + + /** + * Sets the {@code Content-Type} header. + * + * @return {@code this} + */ + public HttpClient setContentType(String contentType) { + this.contentType = contentType; + return this; + } + + /** + * Sets the {@code Accept-Encoding} header. + * + * @return {@code this} + */ + public HttpClient setAcceptEncoding(String acceptEncoding) { + this.acceptEncoding = acceptEncoding; + return this; + } + + /** + * Sets the {@code Content-Length} header for {@code PUT}/{@code POST} requests. + * + * @return {@code this} + */ + public HttpClient setContentLength(long contentLength) { + this.contentLength = contentLength; + return this; + } + + /** + * Sets the request body for {@code PUT}/{@code POST} requests. + * + * @return {@code this} + */ + public HttpClient setRequestBody(byte[] requestBody) { + this.requestBody = requestBody; + return this; + } + + /** + * Sets the {@code If-Modified-Since} header. + * + * @return {@code this} + */ + public HttpClient setIfModifiedSince(long ifModifiedSince) { + this.ifModifiedSince = ifModifiedSince; + return this; + } + + /** + * Sets the maximum number of redirections to follow. + * + * @return {@code this} + */ + public HttpClient setMaxRedirects(int maxRedirects) { + this.maxRedirects = maxRedirects; + return this; + } + + /** + * Sets an arbitrary HTTP header. + * + * @return {@code this} + */ + public HttpClient setHeader(String key, String value) { + this.headers.put(key, value); + return this; + } + + /** + * Sets arbitrary HTTP headers. + * + * @return {@code this} + */ + public HttpClient setHeaders(Map headers) { + this.headers.putAll(headers); + return this; + } + + private static boolean isRedirect(final int statusCode) { + switch (statusCode) { + case HttpURLConnection.HTTP_MOVED_PERM: // 301 + case HttpURLConnection.HTTP_MOVED_TEMP: // 302 + case HttpURLConnection.HTTP_SEE_OTHER: // 303 + case 307: // TEMPORARY_REDIRECT: + case 308: // PERMANENT_REDIRECT: + return true; + default: + return false; + } + } + +} diff --git a/src/org/openstreetmap/josm/tools/WikiReader.java b/src/org/openstreetmap/josm/tools/WikiReader.java index 3e6527a..97d7c7e 100644 --- a/src/org/openstreetmap/josm/tools/WikiReader.java +++ b/src/org/openstreetmap/josm/tools/WikiReader.java @@ -52,7 +52,7 @@ public final String getBaseUrlWiki() { */ public String read(String url) throws IOException { URL u = new URL(url); - try (BufferedReader in = Utils.openURLReader(u)) { + try (BufferedReader in = HttpClient.create(u).connect().getContentReader()) { boolean txt = url.endsWith("?format=txt"); if (url.startsWith(getBaseUrlWiki()) && !txt) return readFromTrac(in, u); @@ -97,11 +97,8 @@ public String readLang(String text) throws IOException { } private String readLang(URL url) throws IOException { - try (BufferedReader in = Utils.openURLReader(url)) { + try (BufferedReader in = HttpClient.create(url).connect().getContentReader()) { return readFromTrac(in, url); - } catch (IOException e) { - Main.addNetworkError(url, Utils.getRootCause(e)); - throw e; } }