001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.BufferedInputStream;
007import java.io.BufferedOutputStream;
008import java.io.File;
009import java.io.FileInputStream;
010import java.io.FileOutputStream;
011import java.io.IOException;
012import java.io.InputStream;
013import java.io.OutputStream;
014import java.net.HttpURLConnection;
015import java.net.MalformedURLException;
016import java.net.URL;
017import java.nio.charset.StandardCharsets;
018import java.util.ArrayList;
019import java.util.Arrays;
020import java.util.Enumeration;
021import java.util.List;
022import java.util.zip.ZipEntry;
023import java.util.zip.ZipFile;
024
025import org.openstreetmap.josm.Main;
026import org.openstreetmap.josm.tools.CheckParameterUtil;
027import org.openstreetmap.josm.tools.Pair;
028import org.openstreetmap.josm.tools.Utils;
029
030/**
031 * Downloads a file and caches it on disk in order to reduce network load.
032 *
033 * Supports URLs, local files, and a custom scheme (<code>resource:</code>) to get
034 * resources from the current *.jar file. (Local caching is only done for URLs.)
035 * <p>
036 * The mirrored file is only downloaded if it has been more than 7 days since
037 * last download. (Time can be configured.)
038 * <p>
039 * The file content is normally accessed with {@link #getInputStream()}, but
040 * you can also get the mirrored copy with {@link #getFile()}.
041 */
042public class CachedFile {
043
044    /**
045     * Caching strategy.
046     */
047    public enum CachingStrategy {
048        /**
049         * If cached file on disk is older than a certain time (7 days by default),
050         * consider the cache stale and try to download the file again.
051         */
052        MaxAge,
053        /**
054         * Similar to MaxAge, considers the cache stale when a certain age is
055         * exceeded. In addition, a If-Modified-Since HTTP header is added.
056         * When the server replies "304 Not Modified", this is considered the same
057         * as a full download.
058         */
059        IfModifiedSince
060    }
061    protected String name;
062    protected long maxAge;
063    protected String destDir;
064    protected String httpAccept;
065    protected CachingStrategy cachingStrategy;
066
067    protected File cacheFile = null;
068    protected boolean initialized = false;
069
070    public static final long DEFAULT_MAXTIME = -1L;
071    public static final long DAYS = 24*60*60; // factor to get caching time in days
072
073    /**
074     * Constructs a CachedFile object from a given filename, URL or internal resource.
075     *
076     * @param name can be:<ul>
077     *  <li>relative or absolute file name</li>
078     *  <li>{@code file:///SOME/FILE} the same as above</li>
079     *  <li>{@code http://...} a URL. It will be cached on disk.</li></ul>
080     *  <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
081     *  <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li></ul>
082     *  <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul>
083     */
084    public CachedFile(String name) {
085        this.name = name;
086    }
087
088    /**
089     * Set the name of the resource.
090     * @param name can be:<ul>
091     *  <li>relative or absolute file name</li>
092     *  <li>{@code file:///SOME/FILE} the same as above</li>
093     *  <li>{@code http://...} a URL. It will be cached on disk.</li></ul>
094     *  <li>{@code resource://SOME/FILE} file from the classpath (usually in the current *.jar)</li>
095     *  <li>{@code josmdir://SOME/FILE} file inside josm user data directory (since r7058)</li></ul>
096     *  <li>{@code josmplugindir://SOME/FILE} file inside josm plugin directory (since r7834)</li></ul>
097     * @return this object
098     */
099    public CachedFile setName(String name) {
100        this.name = name;
101        return this;
102    }
103
104    /**
105     * Set maximum age of cache file. Only applies to URLs.
106     * When this time has passed after the last download of the file, the
107     * cache is considered stale and a new download will be attempted.
108     * @param maxAge the maximum cache age in seconds
109     * @return this object
110     */
111    public CachedFile setMaxAge(long maxAge) {
112        this.maxAge = maxAge;
113        return this;
114    }
115
116    /**
117     * Set the destination directory for the cache file. Only applies to URLs.
118     * @param destDir the destination directory
119     * @return this object
120     */
121    public CachedFile setDestDir(String destDir) {
122        this.destDir = destDir;
123        return this;
124    }
125
126    /**
127     * Set the accepted MIME types sent in the HTTP Accept header. Only applies to URLs.
128     * @param httpAccept the accepted MIME types
129     * @return this object
130     */
131    public CachedFile setHttpAccept(String httpAccept) {
132        this.httpAccept = httpAccept;
133        return this;
134    }
135
136    /**
137     * Set the caching strategy. Only applies to URLs.
138     * @param cachingStrategy
139     * @return this object
140     */
141    public CachedFile setCachingStrategy(CachingStrategy cachingStrategy) {
142        this.cachingStrategy = cachingStrategy;
143        return this;
144    }
145
146    public String getName() {
147        return name;
148    }
149
150    public long getMaxAge() {
151        return maxAge;
152    }
153
154    public String getDestDir() {
155        return destDir;
156    }
157
158    public String getHttpAccept() {
159        return httpAccept;
160    }
161
162    public CachingStrategy getCachingStrategy() {
163        return cachingStrategy;
164    }
165
166    /**
167     * Get InputStream to the requested resource.
168     * @return the InputStream
169     * @throws IOException when the resource with the given name could not be retrieved
170     */
171    public InputStream getInputStream() throws IOException {
172        File file = getFile();
173        if (file == null) {
174            if (name.startsWith("resource://")) {
175                InputStream is = getClass().getResourceAsStream(
176                        name.substring("resource:/".length()));
177                if (is == null)
178                    throw new IOException(tr("Failed to open input stream for resource ''{0}''", name));
179                return is;
180            } else {
181                throw new IOException("No file found for: "+name);
182            }
183        }
184        return new FileInputStream(file);
185    }
186
187    /**
188     * Get local file for the requested resource.
189     * @return The local cache file for URLs. If the resource is a local file,
190     * returns just that file.
191     * @throws IOException when the resource with the given name could not be retrieved
192     */
193    public synchronized File getFile() throws IOException {
194        if (initialized)
195            return cacheFile;
196        initialized = true;
197        URL url;
198        try {
199            url = new URL(name);
200            if ("file".equals(url.getProtocol())) {
201                cacheFile = new File(name.substring("file:/".length()));
202                if (!cacheFile.exists()) {
203                    cacheFile = new File(name.substring("file://".length()));
204                }
205            } else {
206                cacheFile = checkLocal(url);
207            }
208        } catch (MalformedURLException e) {
209            if (name.startsWith("resource://")) {
210                return null;
211            } else if (name.startsWith("josmdir://")) {
212                cacheFile = new File(Main.pref.getUserDataDirectory(), name.substring("josmdir://".length()));
213            } else if (name.startsWith("josmplugindir://")) {
214                cacheFile = new File(Main.pref.getPluginsDirectory(), name.substring("josmplugindir://".length()));
215            } else {
216                cacheFile = new File(name);
217            }
218        }
219        if (cacheFile == null)
220            throw new IOException("Unable to get cache file for "+name);
221        return cacheFile;
222    }
223
224    /**
225     * Looks for a certain entry inside a zip file and returns the entry path.
226     *
227     * Replies a file in the top level directory of the ZIP file which has an
228     * extension <code>extension</code>. If more than one files have this
229     * extension, the last file whose name includes <code>namepart</code>
230     * is opened.
231     *
232     * @param extension  the extension of the file we're looking for
233     * @param namepart the name part
234     * @return The zip entry path of the matching file. Null if this cached file
235     * doesn't represent a zip file or if there was no matching
236     * file in the ZIP file.
237     */
238    public String findZipEntryPath(String extension, String namepart) {
239        Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
240        if (ze == null) return null;
241        return ze.a;
242    }
243
244    /**
245     * Like {@link #findZipEntryPath}, but returns the corresponding InputStream.
246     * @param extension  the extension of the file we're looking for
247     * @param namepart the name part
248     * @return InputStream to the matching file. Null if this cached file
249     * doesn't represent a zip file or if there was no matching
250     * file in the ZIP file.
251     * @since 6148
252     */
253    public InputStream findZipEntryInputStream(String extension, String namepart) {
254        Pair<String, InputStream> ze = findZipEntryImpl(extension, namepart);
255        if (ze == null) return null;
256        return ze.b;
257    }
258
259    @SuppressWarnings("resource")
260    private Pair<String, InputStream> findZipEntryImpl(String extension, String namepart) {
261        File file = null;
262        try {
263            file = getFile();
264        } catch (IOException ex) {
265        }
266        if (file == null)
267            return null;
268        Pair<String, InputStream> res = null;
269        try {
270            ZipFile zipFile = new ZipFile(file, StandardCharsets.UTF_8);
271            ZipEntry resentry = null;
272            Enumeration<? extends ZipEntry> entries = zipFile.entries();
273            while (entries.hasMoreElements()) {
274                ZipEntry entry = entries.nextElement();
275                if (entry.getName().endsWith("." + extension)) {
276                    /* choose any file with correct extension. When more than
277                        one file, prefer the one which matches namepart */
278                    if (resentry == null || entry.getName().indexOf(namepart) >= 0) {
279                        resentry = entry;
280                    }
281                }
282            }
283            if (resentry != null) {
284                InputStream is = zipFile.getInputStream(resentry);
285                res = Pair.create(resentry.getName(), is);
286            } else {
287                Utils.close(zipFile);
288            }
289        } catch (Exception e) {
290            if (file.getName().endsWith(".zip")) {
291                Main.warn(tr("Failed to open file with extension ''{2}'' and namepart ''{3}'' in zip file ''{0}''. Exception was: {1}",
292                        file.getName(), e.toString(), extension, namepart));
293            }
294        }
295        return res;
296    }
297
298    /**
299     * Clear the cache for the given resource.
300     * This forces a fresh download.
301     * @param name the URL
302     */
303    public static void cleanup(String name) {
304        cleanup(name, null);
305    }
306
307    /**
308     * Clear the cache for the given resource.
309     * This forces a fresh download.
310     * @param name the URL
311     * @param destDir the destination directory (see {@link #setDestDir(java.lang.String)})
312     */
313    public static void cleanup(String name, String destDir) {
314        URL url;
315        try {
316            url = new URL(name);
317            if (!"file".equals(url.getProtocol())) {
318                String prefKey = getPrefKey(url, destDir);
319                List<String> localPath = new ArrayList<>(Main.pref.getCollection(prefKey));
320                if (localPath.size() == 2) {
321                    File lfile = new File(localPath.get(1));
322                    if(lfile.exists()) {
323                        lfile.delete();
324                    }
325                }
326                Main.pref.putCollection(prefKey, null);
327            }
328        } catch (MalformedURLException e) {
329            Main.warn(e);
330        }
331    }
332
333    /**
334     * Get preference key to store the location and age of the cached file.
335     * 2 resources that point to the same url, but that are to be stored in different
336     * directories will not share a cache file.
337     */
338    private static String getPrefKey(URL url, String destDir) {
339        StringBuilder prefKey = new StringBuilder("mirror.");
340        if (destDir != null) {
341            prefKey.append(destDir);
342            prefKey.append(".");
343        }
344        prefKey.append(url.toString());
345        return prefKey.toString().replaceAll("=","_");
346    }
347
348    private File checkLocal(URL url) throws IOException {
349        String prefKey = getPrefKey(url, destDir);
350        String urlStr = url.toExternalForm();
351        long age = 0L;
352        long lMaxAge = maxAge;
353        Long ifModifiedSince = null;
354        File localFile = null;
355        List<String> localPathEntry = new ArrayList<>(Main.pref.getCollection(prefKey));
356        boolean offline = false;
357        try {
358            checkOfflineAccess(urlStr);
359        } catch (OfflineAccessException e) {
360            offline = true;
361        }
362        if (localPathEntry.size() == 2) {
363            localFile = new File(localPathEntry.get(1));
364            if (!localFile.exists()) {
365                localFile = null;
366            } else {
367                if ( maxAge == DEFAULT_MAXTIME
368                        || maxAge <= 0 // arbitrary value <= 0 is deprecated
369                ) {
370                    lMaxAge = Main.pref.getInteger("mirror.maxtime", 7*24*60*60); // one week
371                }
372                age = System.currentTimeMillis() - Long.parseLong(localPathEntry.get(0));
373                if (offline || age < lMaxAge*1000) {
374                    return localFile;
375                }
376                if (cachingStrategy == CachingStrategy.IfModifiedSince) {
377                    ifModifiedSince = Long.parseLong(localPathEntry.get(0));
378                }
379            }
380        }
381        if (destDir == null) {
382            destDir = Main.pref.getCacheDirectory().getPath();
383        }
384
385        File destDirFile = new File(destDir);
386        if (!destDirFile.exists()) {
387            destDirFile.mkdirs();
388        }
389
390        // No local file + offline => nothing to do
391        if (offline) {
392            return null;
393        }
394
395        String a = urlStr.replaceAll("[^A-Za-z0-9_.-]", "_");
396        String localPath = "mirror_" + a;
397        destDirFile = new File(destDir, localPath + ".tmp");
398        try {
399            HttpURLConnection con = connectFollowingRedirect(url, httpAccept, ifModifiedSince);
400            if (ifModifiedSince != null && con.getResponseCode() == HttpURLConnection.HTTP_NOT_MODIFIED) {
401                if (Main.isDebugEnabled()) {
402                    Main.debug("304 Not Modified ("+urlStr+")");
403                }
404                if (localFile == null)
405                    throw new AssertionError();
406                Main.pref.putCollection(prefKey,
407                        Arrays.asList(Long.toString(System.currentTimeMillis()), localPathEntry.get(1)));
408                return localFile;
409            }
410            try (
411                InputStream bis = new BufferedInputStream(con.getInputStream());
412                OutputStream fos = new FileOutputStream(destDirFile);
413                OutputStream bos = new BufferedOutputStream(fos)
414            ) {
415                byte[] buffer = new byte[4096];
416                int length;
417                while ((length = bis.read(buffer)) > -1) {
418                    bos.write(buffer, 0, length);
419                }
420            }
421            localFile = new File(destDir, localPath);
422            if (Main.platform.rename(destDirFile, localFile)) {
423                Main.pref.putCollection(prefKey,
424                        Arrays.asList(Long.toString(System.currentTimeMillis()), localFile.toString()));
425            } else {
426                Main.warn(tr("Failed to rename file {0} to {1}.",
427                destDirFile.getPath(), localFile.getPath()));
428            }
429        } catch (IOException e) {
430            if (age >= lMaxAge*1000 && age < lMaxAge*1000*2) {
431                Main.warn(tr("Failed to load {0}, use cached file and retry next time: {1}", urlStr, e));
432                return localFile;
433            } else {
434                throw e;
435            }
436        }
437
438        return localFile;
439    }
440
441    private static void checkOfflineAccess(String urlString) {
442        OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlString, Main.getJOSMWebsite());
443        OnlineResource.OSM_API.checkOfflineAccess(urlString, Main.pref.get("osm-server.url", OsmApi.DEFAULT_API_URL));
444    }
445
446    /**
447     * Opens a connection for downloading a resource.
448     * <p>
449     * Manually follows redirects because
450     * {@link HttpURLConnection#setFollowRedirects(boolean)} fails if the redirect
451     * is going from a http to a https URL, see <a href="https://bugs.openjdk.java.net/browse/JDK-4620571">bug report</a>.
452     * <p>
453     * This can cause problems when downloading from certain GitHub URLs.
454     *
455     * @param downloadUrl The resource URL to download
456     * @param httpAccept The accepted MIME types sent in the HTTP Accept header. Can be {@code null}
457     * @param ifModifiedSince The download time of the cache file, optional
458     * @return The HTTP connection effectively linked to the resource, after all potential redirections
459     * @throws MalformedURLException If a redirected URL is wrong
460     * @throws IOException If any I/O operation goes wrong
461     * @throws OfflineAccessException if resource is accessed in offline mode, in any protocol
462     * @since 6867
463     */
464    public static HttpURLConnection connectFollowingRedirect(URL downloadUrl, String httpAccept, Long ifModifiedSince) throws MalformedURLException, IOException {
465        CheckParameterUtil.ensureParameterNotNull(downloadUrl, "downloadUrl");
466        String downloadString = downloadUrl.toExternalForm();
467
468        checkOfflineAccess(downloadString);
469
470        HttpURLConnection con = null;
471        int numRedirects = 0;
472        while(true) {
473            con = Utils.openHttpConnection(downloadUrl);
474            if (con == null) {
475                throw new IOException("Cannot open http connection to "+downloadString);
476            }
477            if (ifModifiedSince != null) {
478                con.setIfModifiedSince(ifModifiedSince);
479            }
480            con.setInstanceFollowRedirects(false);
481            con.setConnectTimeout(Main.pref.getInteger("socket.timeout.connect",15)*1000);
482            con.setReadTimeout(Main.pref.getInteger("socket.timeout.read",30)*1000);
483            if (Main.isDebugEnabled()) {
484                Main.debug("GET "+downloadString);
485            }
486            if (httpAccept != null) {
487                if (Main.isTraceEnabled()) {
488                    Main.trace("Accept: "+httpAccept);
489                }
490                con.setRequestProperty("Accept", httpAccept);
491            }
492            try {
493                con.connect();
494            } catch (IOException e) {
495                Main.addNetworkError(downloadUrl, Utils.getRootCause(e));
496                throw e;
497            }
498            switch(con.getResponseCode()) {
499            case HttpURLConnection.HTTP_OK:
500                return con;
501            case HttpURLConnection.HTTP_NOT_MODIFIED:
502                if (ifModifiedSince != null)
503                    return con;
504            case HttpURLConnection.HTTP_MOVED_PERM:
505            case HttpURLConnection.HTTP_MOVED_TEMP:
506            case HttpURLConnection.HTTP_SEE_OTHER:
507                String redirectLocation = con.getHeaderField("Location");
508                if (redirectLocation == null) {
509                    /* I18n: argument is HTTP response code */
510                    String msg = tr("Unexpected response from HTTP server. Got {0} response without ''Location'' header."+
511                            " Can''t redirect. Aborting.", con.getResponseCode());
512                    throw new IOException(msg);
513                }
514                downloadUrl = new URL(redirectLocation);
515                downloadString = downloadUrl.toExternalForm();
516                // keep track of redirect attempts to break a redirect loops if it happens
517                // to occur for whatever reason
518                numRedirects++;
519                if (numRedirects >= Main.pref.getInteger("socket.maxredirects", 5)) {
520                    String msg = tr("Too many redirects to the download URL detected. Aborting.");
521                    throw new IOException(msg);
522                }
523                Main.info(tr("Download redirected to ''{0}''", downloadString));
524                break;
525            default:
526                String msg = tr("Failed to read from ''{0}''. Server responded with status code {1}.", downloadString, con.getResponseCode());
527                throw new IOException(msg);
528            }
529        }
530    }
531}