001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedInputStream; 007import java.io.BufferedOutputStream; 008import java.io.File; 009import java.io.FileInputStream; 010import java.io.FileOutputStream; 011import java.io.IOException; 012import java.io.InputStream; 013import java.io.OutputStream; 014import java.net.HttpURLConnection; 015import java.net.MalformedURLException; 016import java.net.URL; 017import java.nio.charset.StandardCharsets; 018import java.util.ArrayList; 019import java.util.Arrays; 020import java.util.Enumeration; 021import java.util.List; 022import java.util.Map; 023import java.util.Map.Entry; 024import java.util.concurrent.ConcurrentHashMap; 025import java.util.zip.ZipEntry; 026import java.util.zip.ZipFile; 027 028import org.openstreetmap.josm.Main; 029import org.openstreetmap.josm.tools.CheckParameterUtil; 030import org.openstreetmap.josm.tools.Pair; 031import org.openstreetmap.josm.tools.Utils; 032 033/** 034 * Downloads a file and caches it on disk in order to reduce network load. 035 * 036 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get 037 * resources from the current *.jar file. (Local caching is only done for URLs.) 038 * <p> 039 * The mirrored file is only downloaded if it has been more than 7 days since 040 * last download. (Time can be configured.) 041 * <p> 042 * The file content is normally accessed with {@link #getInputStream()}, but 043 * you can also get the mirrored copy with {@link #getFile()}. 044 */ 045public class CachedFile { 046 047 /** 048 * Caching strategy. 049 */ 050 public enum CachingStrategy { 051 /** 052 * If cached file on disk is older than a certain time (7 days by default), 053 * consider the cache stale and try to download the file again. 054 */ 055 MaxAge, 056 /** 057 * Similar to MaxAge, considers the cache stale when a certain age is 058 * exceeded. In addition, a If-Modified-Since HTTP header is added. 059 * When the server replies "304 Not Modified", this is considered the same 060 * as a full download. 061 */ 062 IfModifiedSince 063 } 064 065 protected String name; 066 protected long maxAge; 067 protected String destDir; 068 protected String httpAccept; 069 protected CachingStrategy cachingStrategy; 070 071 protected File cacheFile; 072 protected boolean initialized; 073 074 public static final long DEFAULT_MAXTIME = -1L; 075 public static final long DAYS = 24*60*60; // factor to get caching time in days 076 077 private Map<String, String> httpHeaders = new ConcurrentHashMap<>(); 078 079 /** 080 * Constructs a CachedFile object from a given filename, URL or internal resource. 081 * 082 * @param name can be:<ul> 083 * <li>relative or absolute file name</li> 084 * <li>{@code file:///SOME/FILE} the same as above</li> 085 * <li>{@code http://...} a URL. It will be cached on disk.</li> 086 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 087 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 088 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 089 */ 090 public CachedFile(String name) { 091 this.name = name; 092 } 093 094 /** 095 * Set the name of the resource. 096 * @param name can be:<ul> 097 * <li>relative or absolute file name</li> 098 * <li>{@code file:///SOME/FILE} the same as above</li> 099 * <li>{@code http://...} a URL. It will be cached on disk.</li> 100 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 101 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li> 102 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 103 * @return this object 104 */ 105 public CachedFile setName(String name) { 106 this.name = name; 107 return this; 108 } 109 110 /** 111 * Set maximum age of cache file. Only applies to URLs. 112 * When this time has passed after the last download of the file, the 113 * cache is considered stale and a new download will be attempted. 114 * @param maxAge the maximum cache age in seconds 115 * @return this object 116 */ 117 public CachedFile setMaxAge(long maxAge) { 118 this.maxAge = maxAge; 119 return this; 120 } 121 122 /** 123 * Set the destination directory for the cache file. Only applies to URLs. 124 * @param destDir the destination directory 125 * @return this object 126 */ 127 public CachedFile setDestDir(String destDir) { 128 this.destDir = destDir; 129 return this; 130 } 131 132 /** 133 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs. 134 * @param httpAccept the accepted MIME types 135 * @return this object 136 */ 137 public CachedFile setHttpAccept(String httpAccept) { 138 this.httpAccept = httpAccept; 139 return this; 140 } 141 142 /** 143 * Set the caching strategy. Only applies to URLs. 144 * @param cachingStrategy caching strategy 145 * @return this object 146 */ 147 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) { 148 this.cachingStrategy = cachingStrategy; 149 return this; 150 } 151 152 /** 153 * Sets the http headers. Only applies to URL pointing to http or https resources 154 * @param headers that should be sent together with request 155 * @return this object 156 */ 157 public CachedFile setHttpHeaders(Map<String, String> headers) { 158 this.httpHeaders.putAll(headers); 159 return this; 160 } 161 162 public String getName() { 163 return name; 164 } 165 166 public long getMaxAge() { 167 return maxAge; 168 } 169 170 public String getDestDir() { 171 return destDir; 172 } 173 174 public String getHttpAccept() { 175 return httpAccept; 176 } 177 178 public CachingStrategy getCachingStrategy() { 179 return cachingStrategy; 180 } 181 182 /** 183 * Get InputStream to the requested resource. 184 * @return the InputStream 185 * @throws IOException when the resource with the given name could not be retrieved 186 */ 187 public InputStream getInputStream() throws IOException { 188 File file = getFile(); 189 if (file == null) { 190 if (name.startsWith("resource://")) { 191 InputStream is = getClass().getResourceAsStream( 192 name.substring("resource:/".length())); 193 if (is == null) 194 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name)); 195 return is; 196 } else { 197 throw new IOException("No file found for: "+name); 198 } 199 } 200 return new FileInputStream(file); 201 } 202 203 /** 204 * Get local file for the requested resource. 205 * @return The local cache file for URLs. If the resource is a local file, 206 * returns just that file. 207 * @throws IOException when the resource with the given name could not be retrieved 208 */ 209 public synchronized File getFile() throws IOException { 210 if (initialized) 211 return cacheFile; 212 initialized = true; 213 URL url; 214 try { 215 url = new URL(name); 216 if ("file".equals(url.getProtocol())) { 217 cacheFile = new File(name.substring("file:/".length() - 1)); 218 if (!cacheFile.exists()) { 219 cacheFile = new File(name.substring("file://".length() - 1)); 220 } 221 } else { 222 cacheFile = checkLocal(url); 223 } 224 } catch (MalformedURLException e) { 225 if (name.startsWith("resource://")) { 226 return null; 227 } else if (name.startsWith("josmdir://")) { 228 cacheFile = new File(Main.pref.getUserDataDirectory(), name.substring("josmdir://".length())); 229 } else if (name.startsWith("josmplugindir://")) { 230 cacheFile = new File(Main.pref.getPluginsDirectory(), name.substring("josmplugindir://".length())); 231 } else { 232 cacheFile = new File(name); 233 } 234 } 235 if (cacheFile == null) 236 throw new IOException("Unable to get cache file for "+name); 237 return cacheFile; 238 } 239 240 /** 241 * Looks for a certain entry inside a zip file and returns the entry path. 242 * 243 * Replies a file in the top level directory of the ZIP file which has an 244 * extension <code>extension</code>. If more than one files have this 245 * extension, the last file whose name includes <code>namepart</code> 246 * is opened. 247 * 248 * @param extension the extension of the file we're looking for 249 * @param namepart the name part 250 * @return The zip entry path of the matching file. Null if this cached file 251 * doesn't represent a zip file or if there was no matching 252 * file in the ZIP file. 253 */ 254 public String findZipEntryPath(String extension, String namepart) { 255 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 256 if (ze == null) return null; 257 return ze.a; 258 } 259 260 /** 261 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream. 262 * @param extension the extension of the file we're looking for 263 * @param namepart the name part 264 * @return InputStream to the matching file. Null if this cached file 265 * doesn't represent a zip file or if there was no matching 266 * file in the ZIP file. 267 * @since 6148 268 */ 269 public InputStream findZipEntryInputStream(String extension, String namepart) { 270 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 271 if (ze == null) return null; 272 return ze.b; 273 } 274 275 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) { 276 File file = null; 277 try { 278 file = getFile(); 279 } catch (IOException ex) { 280 Main.warn(ex, false); 281 } 282 if (file == null) 283 return null; 284 Pair<String, InputStream> res = null; 285 try { 286 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8); 287 ZipEntry resentry = null; 288 Enumeration<? extends ZipEntry> entries = zipFile.entries(); 289 while (entries.hasMoreElements()) { 290 ZipEntry entry = entries.nextElement(); 291 if (entry.getName().endsWith('.' + extension)) { 292 /* choose any file with correct extension. When more than 293 one file, prefer the one which matches namepart */ 294 if (resentry == null || entry.getName().indexOf(namepart) >= 0) { 295 resentry = entry; 296 } 297 } 298 } 299 if (resentry != null) { 300 InputStream is = zipFile.getInputStream(resentry); 301 res = Pair.create(resentry.getName(), is); 302 } else { 303 Utils.close(zipFile); 304 } 305 } catch (Exception e) { 306 if (file.getName().endsWith(".zip")) { 307 Main.warn(tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}", 308 file.getName(), e.toString(), extension, namepart)); 309 } 310 } 311 return res; 312 } 313 314 /** 315 * Clear the cache for the given resource. 316 * This forces a fresh download. 317 * @param name the URL 318 */ 319 public static void cleanup(String name) { 320 cleanup(name, null); 321 } 322 323 /** 324 * Clear the cache for the given resource. 325 * This forces a fresh download. 326 * @param name the URL 327 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)}) 328 */ 329 public static void cleanup(String name, String destDir) { 330 URL url; 331 try { 332 url = new URL(name); 333 if (!"file".equals(url.getProtocol())) { 334 String prefKey = getPrefKey(url, destDir); 335 List<String> localPath = new ArrayList<>(Main.pref.getCollection(prefKey)); 336 if (localPath.size() == 2) { 337 File lfile = new File(localPath.get(1)); 338 if (lfile.exists()) { 339 lfile.delete(); 340 } 341 } 342 Main.pref.putCollection(prefKey, null); 343 } 344 } catch (MalformedURLException e) { 345 Main.warn(e); 346 } 347 } 348 349 /** 350 * Get preference key to store the location and age of the cached file. 351 * 2 resources that point to the same url, but that are to be stored in different 352 * directories will not share a cache file. 353 * @param url URL 354 * @param destDir destination directory 355 * @return Preference key 356 */ 357 private static String getPrefKey(URL url, String destDir) { 358 StringBuilder prefKey = new StringBuilder("mirror."); 359 if (destDir != null) { 360 prefKey.append(destDir).append('.'); 361 } 362 prefKey.append(url.toString()); 363 return prefKey.toString().replaceAll("=", "_"); 364 } 365 366 private File checkLocal(URL url) throws IOException { 367 String prefKey = getPrefKey(url, destDir); 368 String urlStr = url.toExternalForm(); 369 long age = 0L; 370 long lMaxAge = maxAge; 371 Long ifModifiedSince = null; 372 File localFile = null; 373 List<String> localPathEntry = new ArrayList<>(Main.pref.getCollection(prefKey)); 374 boolean offline = false; 375 try { 376 checkOfflineAccess(urlStr); 377 } catch (OfflineAccessException e) { 378 offline = true; 379 } 380 if (localPathEntry.size() == 2) { 381 localFile = new File(localPathEntry.get(1)); 382 if (!localFile.exists()) { 383 localFile = null; 384 } else { 385 if (maxAge == DEFAULT_MAXTIME 386 || maxAge <= 0 // arbitrary value <= 0 is deprecated 387 ) { 388 lMaxAge = Main.pref.getInteger("mirror.maxtime", 7*24*60*60); // one week 389 } 390 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0)); 391 if (offline || age < lMaxAge*1000) { 392 return localFile; 393 } 394 if (cachingStrategy == CachingStrategy.IfModifiedSince) { 395 ifModifiedSince = Long.valueOf(localPathEntry.get(0)); 396 } 397 } 398 } 399 if (destDir == null) { 400 destDir = Main.pref.getCacheDirectory().getPath(); 401 } 402 403 File destDirFile = new File(destDir); 404 if (!destDirFile.exists()) { 405 destDirFile.mkdirs(); 406 } 407 408 // No local file + offline => nothing to do 409 if (offline) { 410 return null; 411 } 412 413 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_"); 414 String localPath = "mirror_" + a; 415 destDirFile = new File(destDir, localPath + ".tmp"); 416 try { 417 HttpURLConnection con = connectFollowingRedirect(url, httpAccept, ifModifiedSince, httpHeaders); 418 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { 419 if (Main.isDebugEnabled()) { 420 Main.debug("304 Not Modified ("+urlStr+')'); 421 } 422 if (localFile == null) 423 throw new AssertionError(); 424 Main.pref.putCollection(prefKey, 425 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1))); 426 return localFile; 427 } 428 try ( 429 InputStream bis = new BufferedInputStream(con.getInputStream()); 430 OutputStream fos = new FileOutputStream(destDirFile); 431 OutputStream bos = new BufferedOutputStream(fos) 432 ) { 433 byte[] buffer = new byte[4096]; 434 int length; 435 while ((length = bis.read(buffer)) > -1) { 436 bos.write(buffer, 0, length); 437 } 438 } 439 localFile = new File(destDir, localPath); 440 if (Main.platform.rename(destDirFile, localFile)) { 441 Main.pref.putCollection(prefKey, 442 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString())); 443 } else { 444 Main.warn(tr("Failed to rename file {0} to {1}.", 445 destDirFile.getPath(), localFile.getPath())); 446 } 447 } catch (IOException e) { 448 if (age >= lMaxAge*1000 && age < lMaxAge*1000*2) { 449 Main.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e)); 450 return localFile; 451 } else { 452 throw e; 453 } 454 } 455 456 return localFile; 457 } 458 459 private static void checkOfflineAccess(String urlString) { 460 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Main.getJOSMWebsite()); 461 OnlineResource.OSM_API.checkOfflineAccess(urlString, Main.pref.get("osm-server.url", OsmApi.DEFAULT_API_URL)); 462 } 463 464 /** 465 * Opens a connection for downloading a resource. 466 * <p> 467 * Manually follows redirects because 468 * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect 469 * is going from a http to a https URL, see <a href="https://bugs.openjdk.java.net/browse/JDK-4620571">bug report</a>. 470 * <p> 471 * This can cause problems when downloading from certain GitHub URLs. 472 * 473 * @param downloadUrl The resource URL to download 474 * @param httpAccept The accepted MIME types sent in the HTTP Accept header. Can be {@code null} 475 * @param ifModifiedSince The download time of the cache file, optional 476 * @return The HTTP connection effectively linked to the resource, after all potential redirections 477 * @throws MalformedURLException If a redirected URL is wrong 478 * @throws IOException If any I/O operation goes wrong 479 * @throws OfflineAccessException if resource is accessed in offline mode, in any protocol 480 * @since 6867 481 */ 482 public static HttpURLConnection connectFollowingRedirect(URL downloadUrl, String httpAccept, Long ifModifiedSince) 483 throws MalformedURLException, IOException { 484 return connectFollowingRedirect(downloadUrl, httpAccept, ifModifiedSince, null); 485 } 486 487 /** 488 * Opens a connection for downloading a resource. 489 * <p> 490 * Manually follows redirects because 491 * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect 492 * is going from a http to a https URL, see <a href="https://bugs.openjdk.java.net/browse/JDK-4620571">bug report</a>. 493 * <p> 494 * This can cause problems when downloading from certain GitHub URLs. 495 * 496 * @param downloadUrl The resource URL to download 497 * @param httpAccept The accepted MIME types sent in the HTTP Accept header. Can be {@code null} 498 * @param ifModifiedSince The download time of the cache file, optional 499 * @param headers http headers to be sent together with http request 500 * @return The HTTP connection effectively linked to the resource, after all potential redirections 501 * @throws MalformedURLException If a redirected URL is wrong 502 * @throws IOException If any I/O operation goes wrong 503 * @throws OfflineAccessException if resource is accessed in offline mode, in any protocol 504 * @since TODO 505 */ 506 public static HttpURLConnection connectFollowingRedirect(URL downloadUrl, String httpAccept, Long ifModifiedSince, 507 Map<String, String> headers) throws MalformedURLException, IOException { 508 CheckParameterUtil.ensureParameterNotNull(downloadUrl, "downloadUrl"); 509 String downloadString = downloadUrl.toExternalForm(); 510 511 checkOfflineAccess(downloadString); 512 513 int numRedirects = 0; 514 while (true) { 515 HttpURLConnection con = Utils.openHttpConnection(downloadUrl); 516 if (ifModifiedSince != null) { 517 con.setIfModifiedSince(ifModifiedSince); 518 } 519 if (headers != null) { 520 for (Entry<String, String> header: headers.entrySet()) { 521 con.setRequestProperty(header.getKey(), header.getValue()); 522 } 523 } 524 con.setInstanceFollowRedirects(false); 525 con.setConnectTimeout(Main.pref.getInteger("socket.timeout.connect", 15)*1000); 526 con.setReadTimeout(Main.pref.getInteger("socket.timeout.read", 30)*1000); 527 if (Main.isDebugEnabled()) { 528 Main.debug("GET "+downloadString); 529 } 530 if (httpAccept != null) { 531 if (Main.isTraceEnabled()) { 532 Main.trace("Accept: "+httpAccept); 533 } 534 con.setRequestProperty("Accept", httpAccept); 535 } 536 try { 537 con.connect(); 538 } catch (IOException e) { 539 Main.addNetworkError(downloadUrl, Utils.getRootCause(e)); 540 throw e; 541 } 542 switch(con.getResponseCode()) { 543 case HttpURLConnection.HTTP_OK: 544 return con; 545 case HttpURLConnection.HTTP_NOT_MODIFIED: 546 if (ifModifiedSince != null) 547 return con; 548 case HttpURLConnection.HTTP_MOVED_PERM: 549 case HttpURLConnection.HTTP_MOVED_TEMP: 550 case HttpURLConnection.HTTP_SEE_OTHER: 551 String redirectLocation = con.getHeaderField("Location"); 552 if (redirectLocation == null) { 553 /* I18n: argument is HTTP response code */ 554 String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header."+ 555 " Can''t redirect. Aborting.", con.getResponseCode()); 556 throw new IOException(msg); 557 } 558 downloadUrl = new URL(redirectLocation); 559 downloadString = downloadUrl.toExternalForm(); 560 // keep track of redirect attempts to break a redirect loops if it happens 561 // to occur for whatever reason 562 numRedirects++; 563 if (numRedirects >= Main.pref.getInteger("socket.maxredirects", 5)) { 564 String msg = tr("Too many redirects to the download URL detected. Aborting."); 565 throw new IOException(msg); 566 } 567 Main.info(tr("Download redirected to ''{0}''", downloadString)); 568 break; 569 default: 570 String msg = tr("Failed to read from ''{0}''. Server responded with status code {1}.", downloadString, con.getResponseCode()); 571 throw new IOException(msg); 572 } 573 } 574 } 575}