001// License: GPL. For details, see Readme.txt file. 002package org.openstreetmap.gui.jmapviewer; 003 004import java.io.BufferedReader; 005import java.io.ByteArrayInputStream; 006import java.io.ByteArrayOutputStream; 007import java.io.File; 008import java.io.FileInputStream; 009import java.io.FileNotFoundException; 010import java.io.FileOutputStream; 011import java.io.IOException; 012import java.io.InputStream; 013import java.io.InputStreamReader; 014import java.io.OutputStreamWriter; 015import java.io.PrintWriter; 016import java.net.HttpURLConnection; 017import java.net.URL; 018import java.net.URLConnection; 019import java.nio.charset.Charset; 020import java.util.HashMap; 021import java.util.Map; 022import java.util.Map.Entry; 023import java.util.Random; 024import java.util.logging.Level; 025import java.util.logging.Logger; 026 027import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader; 028import org.openstreetmap.gui.jmapviewer.interfaces.TileClearController; 029import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; 030import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; 031import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; 032import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; 033import org.openstreetmap.gui.jmapviewer.interfaces.TileSource.TileUpdate; 034 035/** 036 * A {@link TileLoader} implementation that loads tiles from OSM via HTTP and 037 * saves all loaded files in a directory located in the temporary directory. 038 * If a tile is present in this file cache it will not be loaded from OSM again. 039 * 040 * @author Jan Peter Stotz 041 * @author Stefan Zeller 042 */ 043public class OsmFileCacheTileLoader extends OsmTileLoader implements CachedTileLoader { 044 045 private static final Logger log = Logger.getLogger(OsmFileCacheTileLoader.class.getName()); 046 047 private static final String ETAG_FILE_EXT = ".etag"; 048 private static final String TAGS_FILE_EXT = ".tags"; 049 050 private static final Charset TAGS_CHARSET = Charset.forName("UTF-8"); 051 052 public static final long FILE_AGE_ONE_DAY = 1000 * 60 * 60 * 24; 053 public static final long FILE_AGE_ONE_WEEK = FILE_AGE_ONE_DAY * 7; 054 055 protected String cacheDirBase; 056 057 protected final Map<TileSource, File> sourceCacheDirMap; 058 059 protected long maxCacheFileAge = FILE_AGE_ONE_WEEK; 060 protected long recheckAfter = FILE_AGE_ONE_DAY; 061 062 public static File getDefaultCacheDir() throws SecurityException { 063 String tempDir = null; 064 String userName = System.getProperty("user.name"); 065 try { 066 tempDir = System.getProperty("java.io.tmpdir"); 067 } catch (SecurityException e) { 068 log.log(Level.WARNING, 069 "Failed to access system property ''java.io.tmpdir'' for security reasons. Exception was: " 070 + e.toString()); 071 throw e; // rethrow 072 } 073 try { 074 if (tempDir == null) 075 throw new IOException("No temp directory set"); 076 String subDirName = "JMapViewerTiles"; 077 // On Linux/Unix systems we do not have a per user tmp directory. 078 // Therefore we add the user name for getting a unique dir name. 079 if (userName != null && userName.length() > 0) { 080 subDirName += "_" + userName; 081 } 082 File cacheDir = new File(tempDir, subDirName); 083 return cacheDir; 084 } catch (Exception e) { 085 } 086 return null; 087 } 088 089 /** 090 * Create a OSMFileCacheTileLoader with given cache directory. 091 * If cacheDir is not set or invalid, IOException will be thrown. 092 * @param map the listener checking for tile load events (usually the map for display) 093 * @param cacheDir directory to store cached tiles 094 */ 095 public OsmFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException { 096 super(map); 097 if (cacheDir == null || (!cacheDir.exists() && !cacheDir.mkdirs())) 098 throw new IOException("Cannot access cache directory"); 099 100 log.finest("Tile cache directory: " + cacheDir); 101 cacheDirBase = cacheDir.getAbsolutePath(); 102 sourceCacheDirMap = new HashMap<>(); 103 } 104 105 /** 106 * Create a OSMFileCacheTileLoader with system property temp dir. 107 * If not set an IOException will be thrown. 108 * @param map the listener checking for tile load events (usually the map for display) 109 */ 110 public OsmFileCacheTileLoader(TileLoaderListener map) throws SecurityException, IOException { 111 this(map, getDefaultCacheDir()); 112 } 113 114 @Override 115 public TileJob createTileLoaderJob(final Tile tile) { 116 return new FileLoadJob(tile); 117 } 118 119 protected File getSourceCacheDir(TileSource source) { 120 File dir = sourceCacheDirMap.get(source); 121 if (dir == null) { 122 dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_")); 123 if (!dir.exists()) { 124 dir.mkdirs(); 125 } 126 } 127 return dir; 128 } 129 130 protected class FileLoadJob implements TileJob { 131 InputStream input = null; 132 133 Tile tile; 134 File tileCacheDir; 135 File tileFile = null; 136 long fileAge = 0; 137 boolean fileTilePainted = false; 138 139 public FileLoadJob(Tile tile) { 140 this.tile = tile; 141 } 142 143 @Override 144 public Tile getTile() { 145 return tile; 146 } 147 148 @Override 149 public void run() { 150 synchronized (tile) { 151 if ((tile.isLoaded() && !tile.hasError()) || tile.isLoading()) 152 return; 153 tile.loaded = false; 154 tile.error = false; 155 tile.loading = true; 156 } 157 tileCacheDir = getSourceCacheDir(tile.getSource()); 158 if (loadTileFromFile()) { 159 return; 160 } 161 if (fileTilePainted) { 162 TileJob job = new TileJob() { 163 164 @Override 165 public void run() { 166 loadOrUpdateTile(); 167 } 168 @Override 169 public Tile getTile() { 170 return tile; 171 } 172 }; 173 JobDispatcher.getInstance().addJob(job); 174 } else { 175 loadOrUpdateTile(); 176 } 177 } 178 179 protected void loadOrUpdateTile() { 180 try { 181 URLConnection urlConn = loadTileFromOsm(tile); 182 if (tileFile != null) { 183 switch (tile.getSource().getTileUpdate()) { 184 case IfModifiedSince: 185 urlConn.setIfModifiedSince(fileAge); 186 break; 187 case LastModified: 188 if (!isOsmTileNewer(fileAge)) { 189 log.finest("LastModified test: local version is up to date: " + tile); 190 tile.setLoaded(true); 191 tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter); 192 return; 193 } 194 break; 195 } 196 } 197 if (tile.getSource().getTileUpdate() == TileUpdate.ETag || tile.getSource().getTileUpdate() == TileUpdate.IfNoneMatch) { 198 String fileETag = tile.getValue("etag"); 199 if (fileETag != null) { 200 switch (tile.getSource().getTileUpdate()) { 201 case IfNoneMatch: 202 urlConn.addRequestProperty("If-None-Match", fileETag); 203 break; 204 case ETag: 205 if (hasOsmTileETag(fileETag)) { 206 tile.setLoaded(true); 207 tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge 208 + recheckAfter); 209 return; 210 } 211 } 212 } 213 tile.putValue("etag", urlConn.getHeaderField("ETag")); 214 } 215 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) { 216 // If we are isModifiedSince or If-None-Match has been set 217 // and the server answers with a HTTP 304 = "Not Modified" 218 log.finest("ETag test: local version is up to date: " + tile); 219 tile.setLoaded(true); 220 tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter); 221 return; 222 } 223 224 loadTileMetadata(tile, urlConn); 225 saveTagsToFile(); 226 227 if ("no-tile".equals(tile.getValue("tile-info"))) 228 { 229 tile.setError("No tile at this zoom level"); 230 listener.tileLoadingFinished(tile, true); 231 } else { 232 for(int i = 0; i < 5; ++i) { 233 if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) { 234 Thread.sleep(5000+(new Random()).nextInt(5000)); 235 continue; 236 } 237 byte[] buffer = loadTileInBuffer(urlConn); 238 if (buffer != null) { 239 tile.loadImage(new ByteArrayInputStream(buffer)); 240 tile.setLoaded(true); 241 listener.tileLoadingFinished(tile, true); 242 saveTileToFile(buffer); 243 break; 244 } 245 } 246 } 247 } catch (Exception e) { 248 tile.setError(e.getMessage()); 249 listener.tileLoadingFinished(tile, false); 250 if (input == null) { 251 try { 252 System.err.println("Failed loading " + tile.getUrl() +": " + e.getMessage()); 253 } catch(IOException i) { 254 } 255 } 256 } finally { 257 tile.loading = false; 258 tile.setLoaded(true); 259 } 260 } 261 262 protected boolean loadTileFromFile() { 263 try { 264 tileFile = getTileFile(); 265 if (!tileFile.exists()) 266 return false; 267 268 loadTagsFromFile(); 269 if ("no-tile".equals(tile.getValue("tile-info"))) { 270 tile.setError("No tile at this zoom level"); 271 if (tileFile.exists()) { 272 tileFile.delete(); 273 } 274 tileFile = getTagsFile(); 275 } else { 276 try (FileInputStream fin = new FileInputStream(tileFile)) { 277 if (fin.available() == 0) 278 throw new IOException("File empty"); 279 tile.loadImage(fin); 280 } 281 } 282 283 fileAge = tileFile.lastModified(); 284 boolean oldTile = System.currentTimeMillis() - fileAge > maxCacheFileAge; 285 if (!oldTile) { 286 tile.setLoaded(true); 287 listener.tileLoadingFinished(tile, true); 288 fileTilePainted = true; 289 return true; 290 } 291 listener.tileLoadingFinished(tile, true); 292 fileTilePainted = true; 293 } catch (Exception e) { 294 tileFile.delete(); 295 tileFile = null; 296 fileAge = 0; 297 } 298 return false; 299 } 300 301 protected byte[] loadTileInBuffer(URLConnection urlConn) throws IOException { 302 input = urlConn.getInputStream(); 303 try { 304 ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available()); 305 byte[] buffer = new byte[2048]; 306 boolean finished = false; 307 do { 308 int read = input.read(buffer); 309 if (read >= 0) { 310 bout.write(buffer, 0, read); 311 } else { 312 finished = true; 313 } 314 } while (!finished); 315 if (bout.size() == 0) 316 return null; 317 return bout.toByteArray(); 318 } finally { 319 input.close(); 320 input = null; 321 } 322 } 323 324 /** 325 * Performs a <code>HEAD</code> request for retrieving the 326 * <code>LastModified</code> header value. 327 * 328 * Note: This does only work with servers providing the 329 * <code>LastModified</code> header: 330 * <ul> 331 * <li>{@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.CycleMap} - supported</li> 332 * <li>{@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.Mapnik} - not supported</li> 333 * </ul> 334 * 335 * @param fileAge time of the 336 * @return <code>true</code> if the tile on the server is newer than the 337 * file 338 * @throws IOException 339 */ 340 protected boolean isOsmTileNewer(long fileAge) throws IOException { 341 URL url; 342 url = new URL(tile.getUrl()); 343 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection(); 344 prepareHttpUrlConnection(urlConn); 345 urlConn.setRequestMethod("HEAD"); 346 urlConn.setReadTimeout(30000); // 30 seconds read timeout 347 // System.out.println("Tile age: " + new 348 // Date(urlConn.getLastModified()) + " / " 349 // + new Date(fileAge)); 350 long lastModified = urlConn.getLastModified(); 351 if (lastModified == 0) 352 return true; // no LastModified time returned 353 return (lastModified > fileAge); 354 } 355 356 protected boolean hasOsmTileETag(String eTag) throws IOException { 357 URL url; 358 url = new URL(tile.getUrl()); 359 HttpURLConnection urlConn = (HttpURLConnection) url.openConnection(); 360 prepareHttpUrlConnection(urlConn); 361 urlConn.setRequestMethod("HEAD"); 362 urlConn.setReadTimeout(30000); // 30 seconds read timeout 363 // System.out.println("Tile age: " + new 364 // Date(urlConn.getLastModified()) + " / " 365 // + new Date(fileAge)); 366 String osmETag = urlConn.getHeaderField("ETag"); 367 if (osmETag == null) 368 return true; 369 return (osmETag.equals(eTag)); 370 } 371 372 protected File getTileFile() { 373 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "." 374 + tile.getSource().getTileType()); 375 } 376 377 protected File getTagsFile() { 378 return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() 379 + TAGS_FILE_EXT); 380 } 381 382 protected void saveTileToFile(byte[] rawData) { 383 try ( 384 FileOutputStream f = new FileOutputStream(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() 385 + "_" + tile.getYtile() + "." + tile.getSource().getTileType()) 386 ) { 387 f.write(rawData); 388 } catch (Exception e) { 389 System.err.println("Failed to save tile content: " + e.getLocalizedMessage()); 390 } 391 } 392 393 protected void saveTagsToFile() { 394 File tagsFile = getTagsFile(); 395 if (tile.getMetadata() == null) { 396 tagsFile.delete(); 397 return; 398 } 399 try (PrintWriter f = new PrintWriter(new OutputStreamWriter(new FileOutputStream(tagsFile), TAGS_CHARSET))) { 400 for (Entry<String, String> entry : tile.getMetadata().entrySet()) { 401 f.println(entry.getKey() + "=" + entry.getValue()); 402 } 403 } catch (Exception e) { 404 System.err.println("Failed to save tile tags: " + e.getLocalizedMessage()); 405 } 406 } 407 408 /** Load backward-compatiblity .etag file and if it exists move it to new .tags file*/ 409 private void loadOldETagfromFile() { 410 File etagFile = new File(tileCacheDir, tile.getZoom() + "_" 411 + tile.getXtile() + "_" + tile.getYtile() + ETAG_FILE_EXT); 412 if (!etagFile.exists()) return; 413 try (FileInputStream f = new FileInputStream(etagFile)) { 414 byte[] buf = new byte[f.available()]; 415 f.read(buf); 416 String etag = new String(buf, TAGS_CHARSET.name()); 417 tile.putValue("etag", etag); 418 if (etagFile.delete()) { 419 saveTagsToFile(); 420 } 421 } catch (IOException e) { 422 System.err.println("Failed to load compatiblity etag: " + e.getLocalizedMessage()); 423 } 424 } 425 426 protected void loadTagsFromFile() { 427 loadOldETagfromFile(); 428 File tagsFile = getTagsFile(); 429 try (BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(tagsFile), TAGS_CHARSET))) { 430 for (String line = f.readLine(); line != null; line = f.readLine()) { 431 final int i = line.indexOf('='); 432 if (i == -1 || i == 0) { 433 System.err.println("Malformed tile tag in file '" + tagsFile.getName() + "':" + line); 434 continue; 435 } 436 tile.putValue(line.substring(0,i),line.substring(i+1)); 437 } 438 } catch (FileNotFoundException e) { 439 } catch (Exception e) { 440 System.err.println("Failed to load tile tags: " + e.getLocalizedMessage()); 441 } 442 } 443 } 444 445 public long getMaxFileAge() { 446 return maxCacheFileAge; 447 } 448 449 /** 450 * Sets the maximum age of the local cached tile in the file system. If a 451 * local tile is older than the specified file age 452 * {@link OsmFileCacheTileLoader} will connect to the tile server and check 453 * if a newer tile is available using the mechanism specified for the 454 * selected tile source/server. 455 * 456 * @param maxFileAge 457 * maximum age in milliseconds 458 * @see #FILE_AGE_ONE_DAY 459 * @see #FILE_AGE_ONE_WEEK 460 * @see TileSource#getTileUpdate() 461 */ 462 public void setCacheMaxFileAge(long maxFileAge) { 463 this.maxCacheFileAge = maxFileAge; 464 } 465 466 public String getCacheDirBase() { 467 return cacheDirBase; 468 } 469 470 public void setTileCacheDir(String tileCacheDir) { 471 File dir = new File(tileCacheDir); 472 dir.mkdirs(); 473 this.cacheDirBase = dir.getAbsolutePath(); 474 } 475 476 @Override 477 public void clearCache(TileSource source) { 478 clearCache(source, null); 479 } 480 481 @Override 482 public void clearCache(TileSource source, TileClearController controller) { 483 File dir = getSourceCacheDir(source); 484 if (dir != null) { 485 if (controller != null) controller.initClearDir(dir); 486 if (dir.isDirectory()) { 487 File[] files = dir.listFiles(); 488 if (controller != null) controller.initClearFiles(files); 489 for (File file : files) { 490 if (controller != null && controller.cancel()) return; 491 file.delete(); 492 if (controller != null) controller.fileDeleted(file); 493 } 494 } 495 dir.delete(); 496 } 497 if (controller != null) controller.clearFinished(); 498 } 499}