001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.tools; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedOutputStream; 007import java.io.BufferedReader; 008import java.io.IOException; 009import java.io.InputStream; 010import java.io.OutputStream; 011import java.net.HttpRetryException; 012import java.net.HttpURLConnection; 013import java.net.URL; 014import java.util.List; 015import java.util.Locale; 016import java.util.Map; 017import java.util.Scanner; 018import java.util.TreeMap; 019import java.util.regex.Matcher; 020import java.util.regex.Pattern; 021import java.util.zip.GZIPInputStream; 022 023import org.openstreetmap.josm.Main; 024import org.openstreetmap.josm.data.Version; 025import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 026import org.openstreetmap.josm.gui.progress.ProgressMonitor; 027import org.openstreetmap.josm.io.Compression; 028import org.openstreetmap.josm.io.ProgressInputStream; 029import org.openstreetmap.josm.io.ProgressOutputStream; 030import org.openstreetmap.josm.io.UTFInputStreamReader; 031 032/** 033 * Provides a uniform access for a HTTP/HTTPS server. This class should be used in favour of {@link HttpURLConnection}. 034 * @since 9168 035 */ 036public final class HttpClient { 037 038 private URL url; 039 private final String requestMethod; 040 private int connectTimeout = Main.pref.getInteger("socket.timeout.connect", 15) * 1000; 041 private int readTimeout = Main.pref.getInteger("socket.timeout.read", 30) * 1000; 042 private byte[] requestBody; 043 private long ifModifiedSince; 044 private final Map<String, String> headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); 045 private int maxRedirects = Main.pref.getInteger("socket.maxredirects", 5); 046 private boolean useCache; 047 private String reasonForRequest; 048 private transient HttpURLConnection connection; // to allow disconnecting before `response` is set 049 private transient Response response; 050 051 private HttpClient(URL url, String requestMethod) { 052 this.url = url; 053 this.requestMethod = requestMethod; 054 this.headers.put("Accept-Encoding", "gzip"); 055 } 056 057 /** 058 * Opens the HTTP connection. 059 * @return HTTP response 060 * @throws IOException if any I/O error occurs 061 */ 062 public Response connect() throws IOException { 063 return connect(null); 064 } 065 066 /** 067 * Opens the HTTP connection. 068 * @param progressMonitor progress monitor 069 * @return HTTP response 070 * @throws IOException if any I/O error occurs 071 * @since 9179 072 */ 073 public Response connect(ProgressMonitor progressMonitor) throws IOException { 074 if (progressMonitor == null) { 075 progressMonitor = NullProgressMonitor.INSTANCE; 076 } 077 final HttpURLConnection connection = (HttpURLConnection) url.openConnection(); 078 this.connection = connection; 079 connection.setRequestMethod(requestMethod); 080 connection.setRequestProperty("User-Agent", Version.getInstance().getFullAgentString()); 081 connection.setConnectTimeout(connectTimeout); 082 connection.setReadTimeout(readTimeout); 083 connection.setInstanceFollowRedirects(false); // we do that ourselves 084 if (ifModifiedSince > 0) { 085 connection.setIfModifiedSince(ifModifiedSince); 086 } 087 connection.setUseCaches(useCache); 088 if (!useCache) { 089 connection.setRequestProperty("Cache-Control", "no-cache"); 090 } 091 for (Map.Entry<String, String> header : headers.entrySet()) { 092 if (header.getValue() != null) { 093 connection.setRequestProperty(header.getKey(), header.getValue()); 094 } 095 } 096 097 progressMonitor.beginTask(tr("Contacting Server..."), 1); 098 progressMonitor.indeterminateSubTask(null); 099 100 if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) { 101 Main.info("{0} {1} ({2}) ...", requestMethod, url, Utils.getSizeString(requestBody.length, Locale.getDefault())); 102 connection.setFixedLengthStreamingMode(requestBody.length); 103 connection.setDoOutput(true); 104 try (OutputStream out = new BufferedOutputStream( 105 new ProgressOutputStream(connection.getOutputStream(), requestBody.length, progressMonitor))) { 106 out.write(requestBody); 107 } 108 } 109 110 boolean successfulConnection = false; 111 try { 112 try { 113 connection.connect(); 114 final boolean hasReason = reasonForRequest != null && !reasonForRequest.isEmpty(); 115 Main.info("{0} {1}{2} -> {3}{4}", 116 requestMethod, url, hasReason ? " (" + reasonForRequest + ")" : "", 117 connection.getResponseCode(), 118 connection.getContentLengthLong() > 0 119 ? " (" + Utils.getSizeString(connection.getContentLengthLong(), Locale.getDefault()) + ")" 120 : "" 121 ); 122 if (Main.isDebugEnabled()) { 123 Main.debug("RESPONSE: " + connection.getHeaderFields()); 124 } 125 } catch (IOException e) { 126 Main.info("{0} {1} -> !!!", requestMethod, url); 127 Main.warn(e); 128 //noinspection ThrowableResultOfMethodCallIgnored 129 Main.addNetworkError(url, Utils.getRootCause(e)); 130 throw e; 131 } 132 if (isRedirect(connection.getResponseCode())) { 133 final String redirectLocation = connection.getHeaderField("Location"); 134 if (redirectLocation == null) { 135 /* I18n: argument is HTTP response code */ 136 String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header." + 137 " Can''t redirect. Aborting.", connection.getResponseCode()); 138 throw new IOException(msg); 139 } else if (maxRedirects > 0) { 140 url = new URL(url, redirectLocation); 141 maxRedirects--; 142 Main.info(tr("Download redirected to ''{0}''", redirectLocation)); 143 return connect(); 144 } else if (maxRedirects == 0) { 145 String msg = tr("Too many redirects to the download URL detected. Aborting."); 146 throw new IOException(msg); 147 } 148 } 149 response = new Response(connection, progressMonitor); 150 successfulConnection = true; 151 return response; 152 } finally { 153 if (!successfulConnection) { 154 connection.disconnect(); 155 } 156 } 157 } 158 159 /** 160 * Returns the HTTP response which is set only after calling {@link #connect()}. 161 * Calling this method again, returns the identical object (unless another {@link #connect()} is performed). 162 * 163 * @return the HTTP response 164 * @since 9309 165 */ 166 public Response getResponse() { 167 return response; 168 } 169 170 /** 171 * A wrapper for the HTTP response. 172 */ 173 public static final class Response { 174 private final HttpURLConnection connection; 175 private final ProgressMonitor monitor; 176 private final int responseCode; 177 private final String responseMessage; 178 private boolean uncompress; 179 private boolean uncompressAccordingToContentDisposition; 180 181 private Response(HttpURLConnection connection, ProgressMonitor monitor) throws IOException { 182 CheckParameterUtil.ensureParameterNotNull(connection, "connection"); 183 CheckParameterUtil.ensureParameterNotNull(monitor, "monitor"); 184 this.connection = connection; 185 this.monitor = monitor; 186 this.responseCode = connection.getResponseCode(); 187 this.responseMessage = connection.getResponseMessage(); 188 } 189 190 /** 191 * Sets whether {@link #getContent()} should uncompress the input stream if necessary. 192 * 193 * @param uncompress whether the input stream should be uncompressed if necessary 194 * @return {@code this} 195 */ 196 public Response uncompress(boolean uncompress) { 197 this.uncompress = uncompress; 198 return this; 199 } 200 201 /** 202 * Sets whether {@link #getContent()} should uncompress the input stream according to {@code Content-Disposition} 203 * HTTP header. 204 * @param uncompressAccordingToContentDisposition whether the input stream should be uncompressed according to 205 * {@code Content-Disposition} 206 * @return {@code this} 207 * @since 9172 208 */ 209 public Response uncompressAccordingToContentDisposition(boolean uncompressAccordingToContentDisposition) { 210 this.uncompressAccordingToContentDisposition = uncompressAccordingToContentDisposition; 211 return this; 212 } 213 214 /** 215 * Returns the URL. 216 * @return the URL 217 * @see HttpURLConnection#getURL() 218 * @since 9172 219 */ 220 public URL getURL() { 221 return connection.getURL(); 222 } 223 224 /** 225 * Returns the request method. 226 * @return the HTTP request method 227 * @see HttpURLConnection#getRequestMethod() 228 * @since 9172 229 */ 230 public String getRequestMethod() { 231 return connection.getRequestMethod(); 232 } 233 234 /** 235 * Returns an input stream that reads from this HTTP connection, or, 236 * error stream if the connection failed but the server sent useful data. 237 * <p> 238 * Note: the return value can be null, if both the input and the error stream are null. 239 * Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887 240 * @return input or error stream 241 * @throws IOException if any I/O error occurs 242 * 243 * @see HttpURLConnection#getInputStream() 244 * @see HttpURLConnection#getErrorStream() 245 */ 246 @SuppressWarnings("resource") 247 public InputStream getContent() throws IOException { 248 InputStream in; 249 try { 250 in = connection.getInputStream(); 251 } catch (IOException ioe) { 252 in = connection.getErrorStream(); 253 } 254 if (in != null) { 255 in = new ProgressInputStream(in, getContentLength(), monitor); 256 in = "gzip".equalsIgnoreCase(getContentEncoding()) ? new GZIPInputStream(in) : in; 257 Compression compression = Compression.NONE; 258 if (uncompress) { 259 final String contentType = getContentType(); 260 Main.debug("Uncompressing input stream according to Content-Type header: {0}", contentType); 261 compression = Compression.forContentType(contentType); 262 } 263 if (uncompressAccordingToContentDisposition && Compression.NONE.equals(compression)) { 264 final String contentDisposition = getHeaderField("Content-Disposition"); 265 final Matcher matcher = Pattern.compile("filename=\"([^\"]+)\"").matcher(contentDisposition); 266 if (matcher.find()) { 267 Main.debug("Uncompressing input stream according to Content-Disposition header: {0}", contentDisposition); 268 compression = Compression.byExtension(matcher.group(1)); 269 } 270 } 271 in = compression.getUncompressedInputStream(in); 272 } 273 return in; 274 } 275 276 /** 277 * Returns {@link #getContent()} wrapped in a buffered reader. 278 * 279 * Detects Unicode charset in use utilizing {@link UTFInputStreamReader}. 280 * @return buffered reader 281 * @throws IOException if any I/O error occurs 282 */ 283 public BufferedReader getContentReader() throws IOException { 284 return new BufferedReader( 285 UTFInputStreamReader.create(getContent()) 286 ); 287 } 288 289 /** 290 * Fetches the HTTP response as String. 291 * @return the response 292 * @throws IOException if any I/O error occurs 293 */ 294 @SuppressWarnings("resource") 295 public String fetchContent() throws IOException { 296 try (Scanner scanner = new Scanner(getContentReader()).useDelimiter("\\A")) { 297 return scanner.hasNext() ? scanner.next() : ""; 298 } 299 } 300 301 /** 302 * Gets the response code from this HTTP connection. 303 * @return HTTP response code 304 * 305 * @see HttpURLConnection#getResponseCode() 306 */ 307 public int getResponseCode() { 308 return responseCode; 309 } 310 311 /** 312 * Gets the response message from this HTTP connection. 313 * @return HTTP response message 314 * 315 * @see HttpURLConnection#getResponseMessage() 316 * @since 9172 317 */ 318 public String getResponseMessage() { 319 return responseMessage; 320 } 321 322 /** 323 * Returns the {@code Content-Encoding} header. 324 * @return {@code Content-Encoding} HTTP header 325 * @see HttpURLConnection#getContentEncoding() 326 */ 327 public String getContentEncoding() { 328 return connection.getContentEncoding(); 329 } 330 331 /** 332 * Returns the {@code Content-Type} header. 333 * @return {@code Content-Type} HTTP header 334 */ 335 public String getContentType() { 336 return connection.getHeaderField("Content-Type"); 337 } 338 339 /** 340 * Returns the {@code Expire} header. 341 * @return {@code Expire} HTTP header 342 * @see HttpURLConnection#getExpiration() 343 * @since 9232 344 */ 345 public long getExpiration() { 346 return connection.getExpiration(); 347 } 348 349 /** 350 * Returns the {@code Last-Modified} header. 351 * @return {@code Last-Modified} HTTP header 352 * @see HttpURLConnection#getLastModified() 353 * @since 9232 354 */ 355 public long getLastModified() { 356 return connection.getLastModified(); 357 } 358 359 /** 360 * Returns the {@code Content-Length} header. 361 * @return {@code Content-Length} HTTP header 362 * @see HttpURLConnection#getContentLengthLong() 363 */ 364 public long getContentLength() { 365 return connection.getContentLengthLong(); 366 } 367 368 /** 369 * Returns the value of the named header field. 370 * @param name the name of a header field 371 * @return the value of the named header field, or {@code null} if there is no such field in the header 372 * @see HttpURLConnection#getHeaderField(String) 373 * @since 9172 374 */ 375 public String getHeaderField(String name) { 376 return connection.getHeaderField(name); 377 } 378 379 /** 380 * Returns an unmodifiable Map mapping header keys to a List of header values. 381 * @return unmodifiable Map mapping header keys to a List of header values 382 * @see HttpURLConnection#getHeaderFields() 383 * @since 9232 384 */ 385 public Map<String, List<String>> getHeaderFields() { 386 return connection.getHeaderFields(); 387 } 388 389 /** 390 * @see HttpURLConnection#disconnect() 391 */ 392 public void disconnect() { 393 HttpClient.disconnect(connection); 394 } 395 } 396 397 /** 398 * Creates a new instance for the given URL and a {@code GET} request 399 * 400 * @param url the URL 401 * @return a new instance 402 */ 403 public static HttpClient create(URL url) { 404 return create(url, "GET"); 405 } 406 407 /** 408 * Creates a new instance for the given URL and a {@code GET} request 409 * 410 * @param url the URL 411 * @param requestMethod the HTTP request method to perform when calling 412 * @return a new instance 413 */ 414 public static HttpClient create(URL url, String requestMethod) { 415 return new HttpClient(url, requestMethod); 416 } 417 418 /** 419 * Returns the URL set for this connection. 420 * @return the URL 421 * @see #create(URL) 422 * @see #create(URL, String) 423 * @since 9172 424 */ 425 public URL getURL() { 426 return url; 427 } 428 429 /** 430 * Returns the request method set for this connection. 431 * @return the HTTP request method 432 * @see #create(URL, String) 433 * @since 9172 434 */ 435 public String getRequestMethod() { 436 return requestMethod; 437 } 438 439 /** 440 * Returns the set value for the given {@code header}. 441 * @param header HTTP header name 442 * @return HTTP header value 443 * @since 9172 444 */ 445 public String getRequestHeader(String header) { 446 return headers.get(header); 447 } 448 449 /** 450 * Sets whether not to set header {@code Cache-Control=no-cache} 451 * 452 * @param useCache whether not to set header {@code Cache-Control=no-cache} 453 * @return {@code this} 454 * @see HttpURLConnection#setUseCaches(boolean) 455 */ 456 public HttpClient useCache(boolean useCache) { 457 this.useCache = useCache; 458 return this; 459 } 460 461 /** 462 * Sets whether not to set header {@code Connection=close} 463 * <p> 464 * This might fix #7640, see 465 * <a href='https://web.archive.org/web/20140118201501/http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive'>here</a>. 466 * 467 * @param keepAlive whether not to set header {@code Connection=close} 468 * @return {@code this} 469 */ 470 public HttpClient keepAlive(boolean keepAlive) { 471 return setHeader("Connection", keepAlive ? null : "close"); 472 } 473 474 /** 475 * Sets a specified timeout value, in milliseconds, to be used when opening a communications link to the resource referenced 476 * by this URLConnection. If the timeout expires before the connection can be established, a 477 * {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout. 478 * @param connectTimeout an {@code int} that specifies the connect timeout value in milliseconds 479 * @return {@code this} 480 * @see HttpURLConnection#setConnectTimeout(int) 481 */ 482 public HttpClient setConnectTimeout(int connectTimeout) { 483 this.connectTimeout = connectTimeout; 484 return this; 485 } 486 487 /** 488 * Sets the read timeout to a specified timeout, in milliseconds. A non-zero value specifies the timeout when reading from 489 * input stream when a connection is established to a resource. If the timeout expires before there is data available for 490 * read, a {@link java.net.SocketTimeoutException} is raised. A timeout of zero is interpreted as an infinite timeout. 491 * @param readTimeout an {@code int} that specifies the read timeout value in milliseconds 492 * @return {@code this} 493 * @see HttpURLConnection#setReadTimeout(int) 494 */ 495 public HttpClient setReadTimeout(int readTimeout) { 496 this.readTimeout = readTimeout; 497 return this; 498 } 499 500 /** 501 * This method is used to enable streaming of a HTTP request body without internal buffering, 502 * when the content length is known in advance. 503 * <p> 504 * An exception will be thrown if the application attempts to write more data than the indicated content-length, 505 * or if the application closes the OutputStream before writing the indicated amount. 506 * <p> 507 * When output streaming is enabled, authentication and redirection cannot be handled automatically. 508 * A {@linkplain HttpRetryException} will be thrown when reading the response if authentication or redirection 509 * are required. This exception can be queried for the details of the error. 510 * 511 * @param contentLength The number of bytes which will be written to the OutputStream 512 * @return {@code this} 513 * @see HttpURLConnection#setFixedLengthStreamingMode(long) 514 * @since 9178 515 * @deprecated Submitting data via POST, PUT, DELETE automatically sets this property on the connection 516 */ 517 @Deprecated 518 public HttpClient setFixedLengthStreamingMode(long contentLength) { 519 return this; 520 } 521 522 /** 523 * Sets the {@code Accept} header. 524 * @param accept header value 525 * 526 * @return {@code this} 527 */ 528 public HttpClient setAccept(String accept) { 529 return setHeader("Accept", accept); 530 } 531 532 /** 533 * Sets the request body for {@code PUT}/{@code POST} requests. 534 * @param requestBody request body 535 * 536 * @return {@code this} 537 */ 538 public HttpClient setRequestBody(byte[] requestBody) { 539 this.requestBody = requestBody; 540 return this; 541 } 542 543 /** 544 * Sets the {@code If-Modified-Since} header. 545 * @param ifModifiedSince header value 546 * 547 * @return {@code this} 548 */ 549 public HttpClient setIfModifiedSince(long ifModifiedSince) { 550 this.ifModifiedSince = ifModifiedSince; 551 return this; 552 } 553 554 /** 555 * Sets the maximum number of redirections to follow. 556 * 557 * Set {@code maxRedirects} to {@code -1} in order to ignore redirects, i.e., 558 * to not throw an {@link IOException} in {@link #connect()}. 559 * @param maxRedirects header value 560 * 561 * @return {@code this} 562 */ 563 public HttpClient setMaxRedirects(int maxRedirects) { 564 this.maxRedirects = maxRedirects; 565 return this; 566 } 567 568 /** 569 * Sets an arbitrary HTTP header. 570 * @param key header name 571 * @param value header value 572 * 573 * @return {@code this} 574 */ 575 public HttpClient setHeader(String key, String value) { 576 this.headers.put(key, value); 577 return this; 578 } 579 580 /** 581 * Sets arbitrary HTTP headers. 582 * @param headers HTTP headers 583 * 584 * @return {@code this} 585 */ 586 public HttpClient setHeaders(Map<String, String> headers) { 587 this.headers.putAll(headers); 588 return this; 589 } 590 591 /** 592 * Sets a reason to show on console. Can be {@code null} if no reason is given. 593 * @param reasonForRequest Reason to show 594 * @return {@code this} 595 * @since 9172 596 */ 597 public HttpClient setReasonForRequest(String reasonForRequest) { 598 this.reasonForRequest = reasonForRequest; 599 return this; 600 } 601 602 private static boolean isRedirect(final int statusCode) { 603 switch (statusCode) { 604 case HttpURLConnection.HTTP_MOVED_PERM: // 301 605 case HttpURLConnection.HTTP_MOVED_TEMP: // 302 606 case HttpURLConnection.HTTP_SEE_OTHER: // 303 607 case 307: // TEMPORARY_REDIRECT: 608 case 308: // PERMANENT_REDIRECT: 609 return true; 610 default: 611 return false; 612 } 613 } 614 615 /** 616 * @see HttpURLConnection#disconnect() 617 * @since 9309 618 */ 619 public void disconnect() { 620 HttpClient.disconnect(connection); 621 } 622 623 private static void disconnect(final HttpURLConnection connection) { 624 // Fix upload aborts - see #263 625 connection.setConnectTimeout(100); 626 connection.setReadTimeout(100); 627 try { 628 Thread.sleep(100); 629 } catch (InterruptedException ex) { 630 Main.warn("InterruptedException in " + HttpClient.class + " during cancel"); 631 } 632 connection.disconnect(); 633 } 634}