001// License: GPL. For details, see Readme.txt file.
002package org.openstreetmap.gui.jmapviewer;
003
004import java.io.BufferedReader;
005import java.io.ByteArrayInputStream;
006import java.io.ByteArrayOutputStream;
007import java.io.File;
008import java.io.FileInputStream;
009import java.io.FileNotFoundException;
010import java.io.FileOutputStream;
011import java.io.IOException;
012import java.io.InputStream;
013import java.io.InputStreamReader;
014import java.io.OutputStreamWriter;
015import java.io.PrintWriter;
016import java.net.HttpURLConnection;
017import java.net.URL;
018import java.net.URLConnection;
019import java.nio.charset.Charset;
020import java.util.HashMap;
021import java.util.Map;
022import java.util.Map.Entry;
023import java.util.Random;
024import java.util.logging.Level;
025import java.util.logging.Logger;
026
027import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader;
028import org.openstreetmap.gui.jmapviewer.interfaces.TileClearController;
029import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
030import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader;
031import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
032import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
033import org.openstreetmap.gui.jmapviewer.interfaces.TileSource.TileUpdate;
034
035/**
036 * A {@link TileLoader} implementation that loads tiles from OSM via HTTP and
037 * saves all loaded files in a directory located in the temporary directory.
038 * If a tile is present in this file cache it will not be loaded from OSM again.
039 *
040 * @author Jan Peter Stotz
041 * @author Stefan Zeller
042 */
043public class OsmFileCacheTileLoader extends OsmTileLoader implements CachedTileLoader {
044
045    private static final Logger log = Logger.getLogger(OsmFileCacheTileLoader.class.getName());
046
047    private static final String ETAG_FILE_EXT = ".etag";
048    private static final String TAGS_FILE_EXT = ".tags";
049
050    private static final Charset TAGS_CHARSET = Charset.forName("UTF-8");
051
052    public static final long FILE_AGE_ONE_DAY = 1000 * 60 * 60 * 24;
053    public static final long FILE_AGE_ONE_WEEK = FILE_AGE_ONE_DAY * 7;
054
055    protected String cacheDirBase;
056
057    protected final Map<TileSource, File> sourceCacheDirMap;
058
059    protected long maxCacheFileAge = FILE_AGE_ONE_WEEK;
060    protected long recheckAfter = FILE_AGE_ONE_DAY;
061
062    public static File getDefaultCacheDir() throws SecurityException {
063        String tempDir = null;
064        String userName = System.getProperty("user.name");
065        try {
066            tempDir = System.getProperty("java.io.tmpdir");
067        } catch (SecurityException e) {
068            log.log(Level.WARNING,
069                    "Failed to access system property ''java.io.tmpdir'' for security reasons. Exception was: "
070                    + e.toString());
071            throw e; // rethrow
072        }
073        try {
074            if (tempDir == null)
075                throw new IOException("No temp directory set");
076            String subDirName = "JMapViewerTiles";
077            // On Linux/Unix systems we do not have a per user tmp directory.
078            // Therefore we add the user name for getting a unique dir name.
079            if (userName != null && userName.length() > 0) {
080                subDirName += "_" + userName;
081            }
082            File cacheDir = new File(tempDir, subDirName);
083            return cacheDir;
084        } catch (Exception e) {
085        }
086        return null;
087    }
088
089    /**
090     * Create a OSMFileCacheTileLoader with given cache directory.
091     * If cacheDir is not set or invalid, IOException will be thrown.
092     * @param map the listener checking for tile load events (usually the map for display)
093     * @param cacheDir directory to store cached tiles
094     */
095    public OsmFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException  {
096        super(map);
097        if (cacheDir == null || (!cacheDir.exists() && !cacheDir.mkdirs()))
098            throw new IOException("Cannot access cache directory");
099
100        log.finest("Tile cache directory: " + cacheDir);
101        cacheDirBase = cacheDir.getAbsolutePath();
102        sourceCacheDirMap = new HashMap<>();
103    }
104
105    /**
106     * Create a OSMFileCacheTileLoader with system property temp dir.
107     * If not set an IOException will be thrown.
108     * @param map the listener checking for tile load events (usually the map for display)
109     */
110    public OsmFileCacheTileLoader(TileLoaderListener map) throws SecurityException, IOException {
111        this(map, getDefaultCacheDir());
112    }
113
114    @Override
115    public TileJob createTileLoaderJob(final Tile tile) {
116        return new FileLoadJob(tile);
117    }
118
119    protected File getSourceCacheDir(TileSource source) {
120        File dir = sourceCacheDirMap.get(source);
121        if (dir == null) {
122            dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_"));
123            if (!dir.exists()) {
124                dir.mkdirs();
125            }
126        }
127        return dir;
128    }
129
130    protected class FileLoadJob implements TileJob {
131        InputStream input = null;
132
133        Tile tile;
134        File tileCacheDir;
135        File tileFile = null;
136        long fileAge = 0;
137        boolean fileTilePainted = false;
138
139        public FileLoadJob(Tile tile) {
140            this.tile = tile;
141        }
142
143        @Override
144        public Tile getTile() {
145            return tile;
146        }
147
148        @Override
149        public void run() {
150            synchronized (tile) {
151                if ((tile.isLoaded() && !tile.hasError()) || tile.isLoading())
152                    return;
153                tile.loaded = false;
154                tile.error = false;
155                tile.loading = true;
156            }
157            tileCacheDir = getSourceCacheDir(tile.getSource());
158            if (loadTileFromFile()) {
159                return;
160            }
161            if (fileTilePainted) {
162                TileJob job = new TileJob() {
163
164                    @Override
165                    public void run() {
166                        loadOrUpdateTile();
167                    }
168                    @Override
169                    public Tile getTile() {
170                        return tile;
171                    }
172                };
173                JobDispatcher.getInstance().addJob(job);
174            } else {
175                loadOrUpdateTile();
176            }
177        }
178
179        protected void loadOrUpdateTile() {
180            try {
181                URLConnection urlConn = loadTileFromOsm(tile);
182                if (tileFile != null) {
183                    switch (tile.getSource().getTileUpdate()) {
184                    case IfModifiedSince:
185                        urlConn.setIfModifiedSince(fileAge);
186                        break;
187                    case LastModified:
188                        if (!isOsmTileNewer(fileAge)) {
189                            log.finest("LastModified test: local version is up to date: " + tile);
190                            tile.setLoaded(true);
191                            tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter);
192                            return;
193                        }
194                        break;
195                    }
196                }
197                if (tile.getSource().getTileUpdate() == TileUpdate.ETag || tile.getSource().getTileUpdate() == TileUpdate.IfNoneMatch) {
198                    String fileETag = tile.getValue("etag");
199                    if (fileETag != null) {
200                        switch (tile.getSource().getTileUpdate()) {
201                        case IfNoneMatch:
202                            urlConn.addRequestProperty("If-None-Match", fileETag);
203                            break;
204                        case ETag:
205                            if (hasOsmTileETag(fileETag)) {
206                                tile.setLoaded(true);
207                                tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge
208                                        + recheckAfter);
209                                return;
210                            }
211                        }
212                    }
213                    tile.putValue("etag", urlConn.getHeaderField("ETag"));
214                }
215                if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) {
216                    // If we are isModifiedSince or If-None-Match has been set
217                    // and the server answers with a HTTP 304 = "Not Modified"
218                    log.finest("ETag test: local version is up to date: " + tile);
219                    tile.setLoaded(true);
220                    tileFile.setLastModified(System.currentTimeMillis() - maxCacheFileAge + recheckAfter);
221                    return;
222                }
223
224                loadTileMetadata(tile, urlConn);
225                saveTagsToFile();
226
227                if ("no-tile".equals(tile.getValue("tile-info")))
228                {
229                    tile.setError("No tile at this zoom level");
230                    listener.tileLoadingFinished(tile, true);
231                } else {
232                    for(int i = 0; i < 5; ++i) {
233                        if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) {
234                            Thread.sleep(5000+(new Random()).nextInt(5000));
235                            continue;
236                        }
237                        byte[] buffer = loadTileInBuffer(urlConn);
238                        if (buffer != null) {
239                            tile.loadImage(new ByteArrayInputStream(buffer));
240                            tile.setLoaded(true);
241                            listener.tileLoadingFinished(tile, true);
242                            saveTileToFile(buffer);
243                            break;
244                        }
245                    }
246                }
247            } catch (Exception e) {
248                tile.setError(e.getMessage());
249                listener.tileLoadingFinished(tile, false);
250                if (input == null) {
251                    try {
252                        System.err.println("Failed loading " + tile.getUrl() +": " + e.getMessage());
253                    } catch(IOException i) {
254                    }
255                }
256            } finally {
257                tile.loading = false;
258                tile.setLoaded(true);
259            }
260        }
261
262        protected boolean loadTileFromFile() {
263            try {
264                tileFile = getTileFile();
265                if (!tileFile.exists())
266                    return false;
267
268                loadTagsFromFile();
269                if ("no-tile".equals(tile.getValue("tile-info"))) {
270                    tile.setError("No tile at this zoom level");
271                    if (tileFile.exists()) {
272                        tileFile.delete();
273                    }
274                    tileFile = getTagsFile();
275                } else {
276                    try (FileInputStream fin = new FileInputStream(tileFile)) {
277                        if (fin.available() == 0)
278                            throw new IOException("File empty");
279                        tile.loadImage(fin);
280                    }
281                }
282
283                fileAge = tileFile.lastModified();
284                boolean oldTile = System.currentTimeMillis() - fileAge > maxCacheFileAge;
285                if (!oldTile) {
286                    tile.setLoaded(true);
287                    listener.tileLoadingFinished(tile, true);
288                    fileTilePainted = true;
289                    return true;
290                }
291                listener.tileLoadingFinished(tile, true);
292                fileTilePainted = true;
293            } catch (Exception e) {
294                tileFile.delete();
295                tileFile = null;
296                fileAge = 0;
297            }
298            return false;
299        }
300
301        protected byte[] loadTileInBuffer(URLConnection urlConn) throws IOException {
302            input = urlConn.getInputStream();
303            try {
304                ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available());
305                byte[] buffer = new byte[2048];
306                boolean finished = false;
307                do {
308                    int read = input.read(buffer);
309                    if (read >= 0) {
310                        bout.write(buffer, 0, read);
311                    } else {
312                        finished = true;
313                    }
314                } while (!finished);
315                if (bout.size() == 0)
316                    return null;
317                return bout.toByteArray();
318            } finally {
319                input.close();
320                input = null;
321            }
322        }
323
324        /**
325         * Performs a <code>HEAD</code> request for retrieving the
326         * <code>LastModified</code> header value.
327         *
328         * Note: This does only work with servers providing the
329         * <code>LastModified</code> header:
330         * <ul>
331         * <li>{@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.CycleMap} - supported</li>
332         * <li>{@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.Mapnik} - not supported</li>
333         * </ul>
334         *
335         * @param fileAge time of the
336         * @return <code>true</code> if the tile on the server is newer than the
337         *         file
338         * @throws IOException
339         */
340        protected boolean isOsmTileNewer(long fileAge) throws IOException {
341            URL url;
342            url = new URL(tile.getUrl());
343            HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
344            prepareHttpUrlConnection(urlConn);
345            urlConn.setRequestMethod("HEAD");
346            urlConn.setReadTimeout(30000); // 30 seconds read timeout
347            // System.out.println("Tile age: " + new
348            // Date(urlConn.getLastModified()) + " / "
349            // + new Date(fileAge));
350            long lastModified = urlConn.getLastModified();
351            if (lastModified == 0)
352                return true; // no LastModified time returned
353            return (lastModified > fileAge);
354        }
355
356        protected boolean hasOsmTileETag(String eTag) throws IOException {
357            URL url;
358            url = new URL(tile.getUrl());
359            HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
360            prepareHttpUrlConnection(urlConn);
361            urlConn.setRequestMethod("HEAD");
362            urlConn.setReadTimeout(30000); // 30 seconds read timeout
363            // System.out.println("Tile age: " + new
364            // Date(urlConn.getLastModified()) + " / "
365            // + new Date(fileAge));
366            String osmETag = urlConn.getHeaderField("ETag");
367            if (osmETag == null)
368                return true;
369            return (osmETag.equals(eTag));
370        }
371
372        protected File getTileFile() {
373            return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "."
374                    + tile.getSource().getTileType());
375        }
376
377        protected File getTagsFile() {
378            return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile()
379                    + TAGS_FILE_EXT);
380        }
381
382        protected void saveTileToFile(byte[] rawData) {
383            try (
384                FileOutputStream f = new FileOutputStream(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile()
385                        + "_" + tile.getYtile() + "." + tile.getSource().getTileType())
386            ) {
387                f.write(rawData);
388            } catch (Exception e) {
389                System.err.println("Failed to save tile content: " + e.getLocalizedMessage());
390            }
391        }
392
393        protected void saveTagsToFile() {
394            File tagsFile = getTagsFile();
395            if (tile.getMetadata() == null) {
396                tagsFile.delete();
397                return;
398            }
399            try (PrintWriter f = new PrintWriter(new OutputStreamWriter(new FileOutputStream(tagsFile), TAGS_CHARSET))) {
400                for (Entry<String, String> entry : tile.getMetadata().entrySet()) {
401                    f.println(entry.getKey() + "=" + entry.getValue());
402                }
403            } catch (Exception e) {
404                System.err.println("Failed to save tile tags: " + e.getLocalizedMessage());
405            }
406        }
407
408        /** Load backward-compatiblity .etag file and if it exists move it to new .tags file*/
409        private void loadOldETagfromFile() {
410            File etagFile = new File(tileCacheDir, tile.getZoom() + "_"
411                    + tile.getXtile() + "_" + tile.getYtile() + ETAG_FILE_EXT);
412            if (!etagFile.exists()) return;
413            try (FileInputStream f = new FileInputStream(etagFile)) {
414                byte[] buf = new byte[f.available()];
415                f.read(buf);
416                String etag = new String(buf, TAGS_CHARSET.name());
417                tile.putValue("etag", etag);
418                if (etagFile.delete()) {
419                    saveTagsToFile();
420                }
421            } catch (IOException e) {
422                System.err.println("Failed to load compatiblity etag: " + e.getLocalizedMessage());
423            }
424        }
425
426        protected void loadTagsFromFile() {
427            loadOldETagfromFile();
428            File tagsFile = getTagsFile();
429            try (BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(tagsFile), TAGS_CHARSET))) {
430                for (String line = f.readLine(); line != null; line = f.readLine()) {
431                    final int i = line.indexOf('=');
432                    if (i == -1 || i == 0) {
433                        System.err.println("Malformed tile tag in file '" + tagsFile.getName() + "':" + line);
434                        continue;
435                    }
436                    tile.putValue(line.substring(0,i),line.substring(i+1));
437                }
438            } catch (FileNotFoundException e) {
439            } catch (Exception e) {
440                System.err.println("Failed to load tile tags: " + e.getLocalizedMessage());
441            }
442        }
443    }
444
445    public long getMaxFileAge() {
446        return maxCacheFileAge;
447    }
448
449    /**
450     * Sets the maximum age of the local cached tile in the file system. If a
451     * local tile is older than the specified file age
452     * {@link OsmFileCacheTileLoader} will connect to the tile server and check
453     * if a newer tile is available using the mechanism specified for the
454     * selected tile source/server.
455     *
456     * @param maxFileAge
457     *            maximum age in milliseconds
458     * @see #FILE_AGE_ONE_DAY
459     * @see #FILE_AGE_ONE_WEEK
460     * @see TileSource#getTileUpdate()
461     */
462    public void setCacheMaxFileAge(long maxFileAge) {
463        this.maxCacheFileAge = maxFileAge;
464    }
465
466    public String getCacheDirBase() {
467        return cacheDirBase;
468    }
469
470    public void setTileCacheDir(String tileCacheDir) {
471        File dir = new File(tileCacheDir);
472        dir.mkdirs();
473        this.cacheDirBase = dir.getAbsolutePath();
474    }
475
476    @Override
477    public void clearCache(TileSource source) {
478        clearCache(source, null);
479    }
480
481    @Override
482    public void clearCache(TileSource source, TileClearController controller) {
483        File dir = getSourceCacheDir(source);
484        if (dir != null) {
485            if (controller != null) controller.initClearDir(dir);
486            if (dir.isDirectory()) {
487                File[] files = dir.listFiles();
488                if (controller != null) controller.initClearFiles(files);
489                for (File file : files) {
490                    if (controller != null && controller.cancel()) return;
491                    file.delete();
492                    if (controller != null) controller.fileDeleted(file);
493                }
494            }
495            dir.delete();
496        }
497        if (controller != null) controller.clearFinished();
498    }
499}