001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedInputStream; 007import java.io.BufferedOutputStream; 008import java.io.File; 009import java.io.FileInputStream; 010import java.io.FileOutputStream; 011import java.io.IOException; 012import java.io.InputStream; 013import java.io.OutputStream; 014import java.net.HttpURLConnection; 015import java.net.MalformedURLException; 016import java.net.URL; 017import java.nio.charset.StandardCharsets; 018import java.util.ArrayList; 019import java.util.Arrays; 020import java.util.Enumeration; 021import java.util.List; 022import java.util.zip.ZipEntry; 023import java.util.zip.ZipFile; 024 025import org.openstreetmap.josm.Main; 026import org.openstreetmap.josm.tools.CheckParameterUtil; 027import org.openstreetmap.josm.tools.Pair; 028import org.openstreetmap.josm.tools.Utils; 029 030/** 031 * Downloads a file and caches it on disk in order to reduce network load. 032 * 033 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get 034 * resources from the current *.jar file. (Local caching is only done for URLs.) 035 * <p> 036 * The mirrored file is only downloaded if it has been more than 7 days since 037 * last download. (Time can be configured.) 038 * <p> 039 * The file content is normally accessed with {@link #getInputStream()}, but 040 * you can also get the mirrored copy with {@link #getFile()}. 041 */ 042public class CachedFile { 043 044 /** 045 * Caching strategy. 046 */ 047 public enum CachingStrategy { 048 /** 049 * If cached file on disk is older than a certain time (7 days by default), 050 * consider the cache stale and try to download the file again. 051 */ 052 MaxAge, 053 /** 054 * Similar to MaxAge, considers the cache stale when a certain age is 055 * exceeded. In addition, a If-Modified-Since HTTP header is added. 056 * When the server replies "304 Not Modified", this is considered the same 057 * as a full download. 058 */ 059 IfModifiedSince 060 } 061 protected String name; 062 protected long maxAge; 063 protected String destDir; 064 protected String httpAccept; 065 protected CachingStrategy cachingStrategy; 066 067 protected File cacheFile = null; 068 protected boolean initialized = false; 069 070 public static final long DEFAULT_MAXTIME = -1L; 071 public static final long DAYS = 24*60*60; // factor to get caching time in days 072 073 /** 074 * Constructs a CachedFile object from a given filename, URL or internal resource. 075 * 076 * @param name can be:<ul> 077 * <li>relative or absolute file name</li> 078 * <li>{@code file:///SOME/FILE} the same as above</li> 079 * <li>{@code http://...} a URL. It will be cached on disk.</li></ul> 080 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 081 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li></ul> 082 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 083 */ 084 public CachedFile(String name) { 085 this.name = name; 086 } 087 088 /** 089 * Set the name of the resource. 090 * @param name can be:<ul> 091 * <li>relative or absolute file name</li> 092 * <li>{@code file:///SOME/FILE} the same as above</li> 093 * <li>{@code http://...} a URL. It will be cached on disk.</li></ul> 094 * <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li> 095 * <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li></ul> 096 * <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul> 097 * @return this object 098 */ 099 public CachedFile setName(String name) { 100 this.name = name; 101 return this; 102 } 103 104 /** 105 * Set maximum age of cache file. Only applies to URLs. 106 * When this time has passed after the last download of the file, the 107 * cache is considered stale and a new download will be attempted. 108 * @param maxAge the maximum cache age in seconds 109 * @return this object 110 */ 111 public CachedFile setMaxAge(long maxAge) { 112 this.maxAge = maxAge; 113 return this; 114 } 115 116 /** 117 * Set the destination directory for the cache file. Only applies to URLs. 118 * @param destDir the destination directory 119 * @return this object 120 */ 121 public CachedFile setDestDir(String destDir) { 122 this.destDir = destDir; 123 return this; 124 } 125 126 /** 127 * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs. 128 * @param httpAccept the accepted MIME types 129 * @return this object 130 */ 131 public CachedFile setHttpAccept(String httpAccept) { 132 this.httpAccept = httpAccept; 133 return this; 134 } 135 136 /** 137 * Set the caching strategy. Only applies to URLs. 138 * @param cachingStrategy 139 * @return this object 140 */ 141 public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) { 142 this.cachingStrategy = cachingStrategy; 143 return this; 144 } 145 146 public String getName() { 147 return name; 148 } 149 150 public long getMaxAge() { 151 return maxAge; 152 } 153 154 public String getDestDir() { 155 return destDir; 156 } 157 158 public String getHttpAccept() { 159 return httpAccept; 160 } 161 162 public CachingStrategy getCachingStrategy() { 163 return cachingStrategy; 164 } 165 166 /** 167 * Get InputStream to the requested resource. 168 * @return the InputStream 169 * @throws IOException when the resource with the given name could not be retrieved 170 */ 171 public InputStream getInputStream() throws IOException { 172 File file = getFile(); 173 if (file == null) { 174 if (name.startsWith("resource://")) { 175 InputStream is = getClass().getResourceAsStream( 176 name.substring("resource:/".length())); 177 if (is == null) 178 throw new IOException(tr("Failed to open input stream for resource ''{0}''", name)); 179 return is; 180 } else { 181 throw new IOException("No file found for: "+name); 182 } 183 } 184 return new FileInputStream(file); 185 } 186 187 /** 188 * Get local file for the requested resource. 189 * @return The local cache file for URLs. If the resource is a local file, 190 * returns just that file. 191 * @throws IOException when the resource with the given name could not be retrieved 192 */ 193 public synchronized File getFile() throws IOException { 194 if (initialized) 195 return cacheFile; 196 initialized = true; 197 URL url; 198 try { 199 url = new URL(name); 200 if ("file".equals(url.getProtocol())) { 201 cacheFile = new File(name.substring("file:/".length())); 202 if (!cacheFile.exists()) { 203 cacheFile = new File(name.substring("file://".length())); 204 } 205 } else { 206 cacheFile = checkLocal(url); 207 } 208 } catch (MalformedURLException e) { 209 if (name.startsWith("resource://")) { 210 return null; 211 } else if (name.startsWith("josmdir://")) { 212 cacheFile = new File(Main.pref.getUserDataDirectory(), name.substring("josmdir://".length())); 213 } else if (name.startsWith("josmplugindir://")) { 214 cacheFile = new File(Main.pref.getPluginsDirectory(), name.substring("josmplugindir://".length())); 215 } else { 216 cacheFile = new File(name); 217 } 218 } 219 if (cacheFile == null) 220 throw new IOException("Unable to get cache file for "+name); 221 return cacheFile; 222 } 223 224 /** 225 * Looks for a certain entry inside a zip file and returns the entry path. 226 * 227 * Replies a file in the top level directory of the ZIP file which has an 228 * extension <code>extension</code>. If more than one files have this 229 * extension, the last file whose name includes <code>namepart</code> 230 * is opened. 231 * 232 * @param extension the extension of the file we're looking for 233 * @param namepart the name part 234 * @return The zip entry path of the matching file. Null if this cached file 235 * doesn't represent a zip file or if there was no matching 236 * file in the ZIP file. 237 */ 238 public String findZipEntryPath(String extension, String namepart) { 239 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 240 if (ze == null) return null; 241 return ze.a; 242 } 243 244 /** 245 * Like {@link #findZipEntryPath}, but returns the corresponding InputStream. 246 * @param extension the extension of the file we're looking for 247 * @param namepart the name part 248 * @return InputStream to the matching file. Null if this cached file 249 * doesn't represent a zip file or if there was no matching 250 * file in the ZIP file. 251 * @since 6148 252 */ 253 public InputStream findZipEntryInputStream(String extension, String namepart) { 254 Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart); 255 if (ze == null) return null; 256 return ze.b; 257 } 258 259 private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) { 260 File file = null; 261 try { 262 file = getFile(); 263 } catch (IOException ex) { 264 Main.warn(ex, false); 265 } 266 if (file == null) 267 return null; 268 Pair<String, InputStream> res = null; 269 try { 270 ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8); 271 ZipEntry resentry = null; 272 Enumeration<? extends ZipEntry> entries = zipFile.entries(); 273 while (entries.hasMoreElements()) { 274 ZipEntry entry = entries.nextElement(); 275 if (entry.getName().endsWith("." + extension)) { 276 /* choose any file with correct extension. When more than 277 one file, prefer the one which matches namepart */ 278 if (resentry == null || entry.getName().indexOf(namepart) >= 0) { 279 resentry = entry; 280 } 281 } 282 } 283 if (resentry != null) { 284 InputStream is = zipFile.getInputStream(resentry); 285 res = Pair.create(resentry.getName(), is); 286 } else { 287 Utils.close(zipFile); 288 } 289 } catch (Exception e) { 290 if (file.getName().endsWith(".zip")) { 291 Main.warn(tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}", 292 file.getName(), e.toString(), extension, namepart)); 293 } 294 } 295 return res; 296 } 297 298 /** 299 * Clear the cache for the given resource. 300 * This forces a fresh download. 301 * @param name the URL 302 */ 303 public static void cleanup(String name) { 304 cleanup(name, null); 305 } 306 307 /** 308 * Clear the cache for the given resource. 309 * This forces a fresh download. 310 * @param name the URL 311 * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)}) 312 */ 313 public static void cleanup(String name, String destDir) { 314 URL url; 315 try { 316 url = new URL(name); 317 if (!"file".equals(url.getProtocol())) { 318 String prefKey = getPrefKey(url, destDir); 319 List<String> localPath = new ArrayList<>(Main.pref.getCollection(prefKey)); 320 if (localPath.size() == 2) { 321 File lfile = new File(localPath.get(1)); 322 if(lfile.exists()) { 323 lfile.delete(); 324 } 325 } 326 Main.pref.putCollection(prefKey, null); 327 } 328 } catch (MalformedURLException e) { 329 Main.warn(e); 330 } 331 } 332 333 /** 334 * Get preference key to store the location and age of the cached file. 335 * 2 resources that point to the same url, but that are to be stored in different 336 * directories will not share a cache file. 337 */ 338 private static String getPrefKey(URL url, String destDir) { 339 StringBuilder prefKey = new StringBuilder("mirror."); 340 if (destDir != null) { 341 prefKey.append(destDir); 342 prefKey.append("."); 343 } 344 prefKey.append(url.toString()); 345 return prefKey.toString().replaceAll("=","_"); 346 } 347 348 private File checkLocal(URL url) throws IOException { 349 String prefKey = getPrefKey(url, destDir); 350 String urlStr = url.toExternalForm(); 351 long age = 0L; 352 long lMaxAge = maxAge; 353 Long ifModifiedSince = null; 354 File localFile = null; 355 List<String> localPathEntry = new ArrayList<>(Main.pref.getCollection(prefKey)); 356 boolean offline = false; 357 try { 358 checkOfflineAccess(urlStr); 359 } catch (OfflineAccessException e) { 360 offline = true; 361 } 362 if (localPathEntry.size() == 2) { 363 localFile = new File(localPathEntry.get(1)); 364 if (!localFile.exists()) { 365 localFile = null; 366 } else { 367 if ( maxAge == DEFAULT_MAXTIME 368 || maxAge <= 0 // arbitrary value <= 0 is deprecated 369 ) { 370 lMaxAge = Main.pref.getInteger("mirror.maxtime", 7*24*60*60); // one week 371 } 372 age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0)); 373 if (offline || age < lMaxAge*1000) { 374 return localFile; 375 } 376 if (cachingStrategy == CachingStrategy.IfModifiedSince) { 377 ifModifiedSince = Long.parseLong(localPathEntry.get(0)); 378 } 379 } 380 } 381 if (destDir == null) { 382 destDir = Main.pref.getCacheDirectory().getPath(); 383 } 384 385 File destDirFile = new File(destDir); 386 if (!destDirFile.exists()) { 387 destDirFile.mkdirs(); 388 } 389 390 // No local file + offline => nothing to do 391 if (offline) { 392 return null; 393 } 394 395 String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_"); 396 String localPath = "mirror_" + a; 397 destDirFile = new File(destDir, localPath + ".tmp"); 398 try { 399 HttpURLConnection con = connectFollowingRedirect(url, httpAccept, ifModifiedSince); 400 if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) { 401 if (Main.isDebugEnabled()) { 402 Main.debug("304 Not Modified ("+urlStr+")"); 403 } 404 if (localFile == null) 405 throw new AssertionError(); 406 Main.pref.putCollection(prefKey, 407 Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1))); 408 return localFile; 409 } 410 try ( 411 InputStream bis = new BufferedInputStream(con.getInputStream()); 412 OutputStream fos = new FileOutputStream(destDirFile); 413 OutputStream bos = new BufferedOutputStream(fos) 414 ) { 415 byte[] buffer = new byte[4096]; 416 int length; 417 while ((length = bis.read(buffer)) > -1) { 418 bos.write(buffer, 0, length); 419 } 420 } 421 localFile = new File(destDir, localPath); 422 if (Main.platform.rename(destDirFile, localFile)) { 423 Main.pref.putCollection(prefKey, 424 Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString())); 425 } else { 426 Main.warn(tr("Failed to rename file {0} to {1}.", 427 destDirFile.getPath(), localFile.getPath())); 428 } 429 } catch (IOException e) { 430 if (age >= lMaxAge*1000 && age < lMaxAge*1000*2) { 431 Main.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e)); 432 return localFile; 433 } else { 434 throw e; 435 } 436 } 437 438 return localFile; 439 } 440 441 private static void checkOfflineAccess(String urlString) { 442 OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Main.getJOSMWebsite()); 443 OnlineResource.OSM_API.checkOfflineAccess(urlString, Main.pref.get("osm-server.url", OsmApi.DEFAULT_API_URL)); 444 } 445 446 /** 447 * Opens a connection for downloading a resource. 448 * <p> 449 * Manually follows redirects because 450 * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect 451 * is going from a http to a https URL, see <a href="https://bugs.openjdk.java.net/browse/JDK-4620571">bug report</a>. 452 * <p> 453 * This can cause problems when downloading from certain GitHub URLs. 454 * 455 * @param downloadUrl The resource URL to download 456 * @param httpAccept The accepted MIME types sent in the HTTP Accept header. Can be {@code null} 457 * @param ifModifiedSince The download time of the cache file, optional 458 * @return The HTTP connection effectively linked to the resource, after all potential redirections 459 * @throws MalformedURLException If a redirected URL is wrong 460 * @throws IOException If any I/O operation goes wrong 461 * @throws OfflineAccessException if resource is accessed in offline mode, in any protocol 462 * @since 6867 463 */ 464 public static HttpURLConnection connectFollowingRedirect(URL downloadUrl, String httpAccept, Long ifModifiedSince) throws MalformedURLException, IOException { 465 CheckParameterUtil.ensureParameterNotNull(downloadUrl, "downloadUrl"); 466 String downloadString = downloadUrl.toExternalForm(); 467 468 checkOfflineAccess(downloadString); 469 470 HttpURLConnection con = null; 471 int numRedirects = 0; 472 while(true) { 473 con = Utils.openHttpConnection(downloadUrl); 474 if (con == null) { 475 throw new IOException("Cannot open http connection to "+downloadString); 476 } 477 if (ifModifiedSince != null) { 478 con.setIfModifiedSince(ifModifiedSince); 479 } 480 con.setInstanceFollowRedirects(false); 481 con.setConnectTimeout(Main.pref.getInteger("socket.timeout.connect",15)*1000); 482 con.setReadTimeout(Main.pref.getInteger("socket.timeout.read",30)*1000); 483 if (Main.isDebugEnabled()) { 484 Main.debug("GET "+downloadString); 485 } 486 if (httpAccept != null) { 487 if (Main.isTraceEnabled()) { 488 Main.trace("Accept: "+httpAccept); 489 } 490 con.setRequestProperty("Accept", httpAccept); 491 } 492 try { 493 con.connect(); 494 } catch (IOException e) { 495 Main.addNetworkError(downloadUrl, Utils.getRootCause(e)); 496 throw e; 497 } 498 switch(con.getResponseCode()) { 499 case HttpURLConnection.HTTP_OK: 500 return con; 501 case HttpURLConnection.HTTP_NOT_MODIFIED: 502 if (ifModifiedSince != null) 503 return con; 504 case HttpURLConnection.HTTP_MOVED_PERM: 505 case HttpURLConnection.HTTP_MOVED_TEMP: 506 case HttpURLConnection.HTTP_SEE_OTHER: 507 String redirectLocation = con.getHeaderField("Location"); 508 if (redirectLocation == null) { 509 /* I18n: argument is HTTP response code */ 510 String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header."+ 511 " Can''t redirect. Aborting.", con.getResponseCode()); 512 throw new IOException(msg); 513 } 514 downloadUrl = new URL(redirectLocation); 515 downloadString = downloadUrl.toExternalForm(); 516 // keep track of redirect attempts to break a redirect loops if it happens 517 // to occur for whatever reason 518 numRedirects++; 519 if (numRedirects >= Main.pref.getInteger("socket.maxredirects", 5)) { 520 String msg = tr("Too many redirects to the download URL detected. Aborting."); 521 throw new IOException(msg); 522 } 523 Main.info(tr("Download redirected to ''{0}''", downloadString)); 524 break; 525 default: 526 String msg = tr("Failed to read from ''{0}''. Server responded with status code {1}.", downloadString, con.getResponseCode()); 527 throw new IOException(msg); 528 } 529 } 530 } 531}