001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.cache;
003
004import java.io.FileNotFoundException;
005import java.io.IOException;
006import java.net.HttpURLConnection;
007import java.net.URL;
008import java.net.URLConnection;
009import java.util.HashSet;
010import java.util.List;
011import java.util.Map;
012import java.util.Random;
013import java.util.Set;
014import java.util.concurrent.ConcurrentHashMap;
015import java.util.concurrent.ConcurrentMap;
016import java.util.concurrent.LinkedBlockingDeque;
017import java.util.concurrent.ThreadPoolExecutor;
018import java.util.concurrent.TimeUnit;
019import java.util.logging.Level;
020import java.util.logging.Logger;
021
022import org.apache.commons.jcs.access.behavior.ICacheAccess;
023import org.apache.commons.jcs.engine.behavior.ICacheElement;
024import org.openstreetmap.gui.jmapviewer.FeatureAdapter;
025import org.openstreetmap.josm.Main;
026import org.openstreetmap.josm.data.cache.ICachedLoaderListener.LoadResult;
027import org.openstreetmap.josm.data.preferences.IntegerProperty;
028import org.openstreetmap.josm.tools.Utils;
029
030/**
031 * @author Wiktor Niesiobędzki
032 *
033 * @param <K> cache entry key type
034 * @param <V> cache value type
035 *
036 * Generic loader for HTTP based tiles. Uses custom attribute, to check, if entry has expired
037 * according to HTTP headers sent with tile. If so, it tries to verify using Etags
038 * or If-Modified-Since / Last-Modified.
039 *
040 * If the tile is not valid, it will try to download it from remote service and put it
041 * to cache. If remote server will fail it will try to use stale entry.
042 *
043 * This class will keep only one Job running for specified tile. All others will just finish, but
044 * listeners will be gathered and notified, once download job will be finished
045 *
046 * @since 8168
047 */
048public abstract class JCSCachedTileLoaderJob<K, V extends CacheEntry> implements ICachedLoaderJob<K>, Runnable {
049    private static final Logger log = FeatureAdapter.getLogger(JCSCachedTileLoaderJob.class.getCanonicalName());
050    protected static final long DEFAULT_EXPIRE_TIME = 1000L * 60 * 60 * 24 * 7; // 7 days
051    // Limit for the max-age value send by the server.
052    protected static final long EXPIRE_TIME_SERVER_LIMIT = 1000L * 60 * 60 * 24 * 28; // 4 weeks
053    // Absolute expire time limit. Cached tiles that are older will not be used,
054    // even if the refresh from the server fails.
055    protected static final long ABSOLUTE_EXPIRE_TIME_LIMIT = 1000L * 60 * 60 * 24 * 365; // 1 year
056
057    /**
058     * maximum download threads that will be started
059     */
060    public static final IntegerProperty THREAD_LIMIT = new IntegerProperty("cache.jcs.max_threads", 10);
061
062    /*
063     * ThreadPoolExecutor starts new threads, until THREAD_LIMIT is reached. Then it puts tasks into LinkedBlockingDeque.
064     *
065     * The queue works FIFO, so one needs to take care about ordering of the entries submitted
066     *
067     * There is no point in canceling tasks, that are already taken by worker threads (if we made so much effort, we can at least cache
068     * the response, so later it could be used). We could actually cancel what is in LIFOQueue, but this is a tradeoff between simplicity
069     * and performance (we do want to have something to offer to worker threads before tasks will be resubmitted by class consumer)
070     */
071
072    private static ThreadPoolExecutor DEFAULT_DOWNLOAD_JOB_DISPATCHER = new ThreadPoolExecutor(
073            2, // we have a small queue, so threads will be quickly started (threads are started only, when queue is full)
074            THREAD_LIMIT.get().intValue(), // do not this number of threads
075            30, // keepalive for thread
076            TimeUnit.SECONDS,
077            // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see)
078            new LinkedBlockingDeque<Runnable>(),
079            Utils.newThreadFactory("JCS-downloader-%d", Thread.NORM_PRIORITY)
080            );
081
082
083
084    private static ConcurrentMap<String, Set<ICachedLoaderListener>> inProgress = new ConcurrentHashMap<>();
085    private static ConcurrentMap<String, Boolean> useHead = new ConcurrentHashMap<>();
086
087    protected long now; // when the job started
088
089    private ICacheAccess<K, V> cache;
090    private ICacheElement<K, V> cacheElement;
091    protected V cacheData;
092    protected CacheEntryAttributes attributes;
093
094    // HTTP connection parameters
095    private int connectTimeout;
096    private int readTimeout;
097    private Map<String, String> headers;
098    private ThreadPoolExecutor downloadJobExecutor;
099    private Runnable finishTask;
100    private boolean force;
101
102    /**
103     * @param cache cache instance that we will work on
104     * @param headers HTTP headers to be sent together with request
105     * @param readTimeout when connecting to remote resource
106     * @param connectTimeout when connecting to remote resource
107     * @param downloadJobExecutor that will be executing the jobs
108     */
109    public JCSCachedTileLoaderJob(ICacheAccess<K, V> cache,
110            int connectTimeout, int readTimeout,
111            Map<String, String> headers,
112            ThreadPoolExecutor downloadJobExecutor) {
113
114        this.cache = cache;
115        this.now = System.currentTimeMillis();
116        this.connectTimeout = connectTimeout;
117        this.readTimeout = readTimeout;
118        this.headers = headers;
119        this.downloadJobExecutor = downloadJobExecutor;
120    }
121
122    /**
123     * @param cache cache instance that we will work on
124     * @param headers HTTP headers to be sent together with request
125     * @param readTimeout when connecting to remote resource
126     * @param connectTimeout when connecting to remote resource
127     */
128    public JCSCachedTileLoaderJob(ICacheAccess<K, V> cache,
129            int connectTimeout, int readTimeout,
130            Map<String, String> headers) {
131        this(cache, connectTimeout, readTimeout,
132                headers, DEFAULT_DOWNLOAD_JOB_DISPATCHER);
133    }
134
135    private void ensureCacheElement() {
136        if (cacheElement == null && getCacheKey() != null) {
137            cacheElement = cache.getCacheElement(getCacheKey());
138            if (cacheElement != null) {
139                attributes = (CacheEntryAttributes) cacheElement.getElementAttributes();
140                cacheData = cacheElement.getVal();
141            }
142        }
143    }
144
145    public V get() {
146        ensureCacheElement();
147        return cacheData;
148    }
149
150    @Override
151    public void submit(ICachedLoaderListener listener, boolean force) throws IOException {
152        this.force = force;
153        boolean first = false;
154        URL url = getUrl();
155        String deduplicationKey = null;
156        if (url != null) {
157            // url might be null, for example when Bing Attribution is not loaded yet
158            deduplicationKey = url.toString();
159        }
160        if (deduplicationKey == null) {
161            log.log(Level.WARNING, "No url returned for: {0}, skipping", getCacheKey());
162            throw new IllegalArgumentException("No url returned");
163        }
164        synchronized (inProgress) {
165            Set<ICachedLoaderListener> newListeners = inProgress.get(deduplicationKey);
166            if (newListeners == null) {
167                newListeners = new HashSet<>();
168                inProgress.put(deduplicationKey, newListeners);
169                first = true;
170            }
171            newListeners.add(listener);
172        }
173
174        if (first || force) {
175            // submit all jobs to separate thread, so calling thread is not blocked with IO when loading from disk
176            log.log(Level.FINE, "JCS - Submitting job for execution for url: {0}", getUrlNoException());
177            downloadJobExecutor.execute(this);
178        }
179    }
180
181    /**
182     * This method is run when job has finished
183     */
184    protected void executionFinished() {
185        if (finishTask != null) {
186            finishTask.run();
187        }
188    }
189
190    /**
191     *
192     * @return checks if object from cache has sufficient data to be returned
193     */
194    protected boolean isObjectLoadable() {
195        if (cacheData == null) {
196            return false;
197        }
198        byte[] content = cacheData.getContent();
199        return content != null && content.length > 0;
200    }
201
202    /**
203     * Simple implementation. All errors should be cached as empty. Though some JDK (JDK8 on Windows for example)
204     * doesn't return 4xx error codes, instead they do throw an FileNotFoundException or IOException
205     *
206     * @return true if we should put empty object into cache, regardless of what remote resource has returned
207     */
208    protected boolean cacheAsEmpty() {
209        return attributes.getResponseCode() < 500;
210    }
211
212    /**
213     * @return key under which discovered server settings will be kept
214     */
215    protected String getServerKey() {
216        return getUrlNoException().getHost();
217    }
218
219    @Override
220    public void run() {
221        final Thread currentThread = Thread.currentThread();
222        final String oldName = currentThread.getName();
223        currentThread.setName("JCS Downloading: " + getUrlNoException());
224        log.log(Level.FINE, "JCS - starting fetch of url: {0} ", getUrlNoException());
225        ensureCacheElement();
226        try {
227            // try to fetch from cache
228            if (!force && cacheElement != null && isCacheElementValid() && isObjectLoadable()) {
229                // we got something in cache, and it's valid, so lets return it
230                log.log(Level.FINE, "JCS - Returning object from cache: {0}", getCacheKey());
231                finishLoading(LoadResult.SUCCESS);
232                return;
233            }
234
235            // try to load object from remote resource
236            if (loadObject()) {
237                finishLoading(LoadResult.SUCCESS);
238            } else {
239                // if loading failed - check if we can return stale entry
240                if (isObjectLoadable()) {
241                    // try to get stale entry in cache
242                    finishLoading(LoadResult.SUCCESS);
243                    log.log(Level.FINE, "JCS - found stale object in cache: {0}", getUrlNoException());
244                } else {
245                    // failed completely
246                    finishLoading(LoadResult.FAILURE);
247                }
248            }
249        } finally {
250            executionFinished();
251            currentThread.setName(oldName);
252        }
253    }
254
255    private void finishLoading(LoadResult result) {
256        Set<ICachedLoaderListener> listeners = null;
257        synchronized (inProgress) {
258            listeners = inProgress.remove(getUrlNoException().toString());
259        }
260        if (listeners == null) {
261            log.log(Level.WARNING, "Listener not found for URL: {0}. Listener not notified!", getUrlNoException());
262            return;
263        }
264        for (ICachedLoaderListener l: listeners) {
265            l.loadingFinished(cacheData, attributes, result);
266        }
267    }
268
269    protected boolean isCacheElementValid() {
270        long expires = attributes.getExpirationTime();
271
272        // check by expire date set by server
273        if (expires != 0L) {
274            // put a limit to the expire time (some servers send a value
275            // that is too large)
276            expires = Math.min(expires, attributes.getCreateTime() + EXPIRE_TIME_SERVER_LIMIT);
277            if (now > expires) {
278                log.log(Level.FINE, "JCS - Object {0} has expired -> valid to {1}, now is: {2}",
279                        new Object[]{getUrlNoException(), Long.toString(expires), Long.toString(now)});
280                return false;
281            }
282        } else if (attributes.getLastModification() > 0 &&
283                now - attributes.getLastModification() > DEFAULT_EXPIRE_TIME) {
284            // check by file modification date
285            log.log(Level.FINE, "JCS - Object has expired, maximum file age reached {0}", getUrlNoException());
286            return false;
287        } else if (now - attributes.getCreateTime() > DEFAULT_EXPIRE_TIME) {
288            log.log(Level.FINE, "JCS - Object has expired, maximum time since object creation reached {0}", getUrlNoException());
289            return false;
290        }
291        return true;
292    }
293
294    /**
295     * @return true if object was successfully downloaded, false, if there was a loading failure
296     */
297
298    private boolean loadObject() {
299        if (attributes == null) {
300            attributes = new CacheEntryAttributes();
301        }
302        try {
303            // if we have object in cache, and host doesn't support If-Modified-Since nor If-None-Match
304            // then just use HEAD request and check returned values
305            if (isObjectLoadable() &&
306                    Boolean.TRUE.equals(useHead.get(getServerKey())) &&
307                    isCacheValidUsingHead()) {
308                log.log(Level.FINE, "JCS - cache entry verified using HEAD request: {0}", getUrl());
309                return true;
310            }
311
312            HttpURLConnection urlConn = getURLConnection(getUrl());
313
314            if (isObjectLoadable()  &&
315                    (now - attributes.getLastModification()) <= ABSOLUTE_EXPIRE_TIME_LIMIT) {
316                urlConn.setIfModifiedSince(attributes.getLastModification());
317            }
318            if (isObjectLoadable() && attributes.getEtag() != null) {
319                urlConn.addRequestProperty("If-None-Match", attributes.getEtag());
320            }
321
322            log.log(Level.INFO, "GET {0} -> {1}", new Object[]{getUrl(), urlConn.getResponseCode()});
323
324            // follow redirects
325            for (int i = 0; i < 5; i++) {
326                if (urlConn.getResponseCode() == 302) {
327                    urlConn = getURLConnection(new URL(urlConn.getHeaderField("Location")));
328                } else {
329                    break;
330                }
331            }
332            if (urlConn.getResponseCode() == 304) {
333                // If isModifiedSince or If-None-Match has been set
334                // and the server answers with a HTTP 304 = "Not Modified"
335                log.log(Level.FINE, "JCS - IfModifiedSince/Etag test: local version is up to date: {0}", getUrl());
336                return true;
337            } else if (isObjectLoadable() // we have an object in cache, but we haven't received 304 resposne code
338                    && (
339                            (attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) ||
340                            attributes.getLastModification() == urlConn.getLastModified())
341                    ) {
342                // we sent ETag or If-Modified-Since, but didn't get 304 response code
343                // for further requests - use HEAD
344                String serverKey = getServerKey();
345                log.log(Level.INFO, "JCS - Host: {0} found not to return 304 codes for If-Modifed-Since or If-None-Match headers",
346                        serverKey);
347                useHead.put(serverKey, Boolean.TRUE);
348            }
349
350
351            attributes = parseHeaders(urlConn);
352
353            for (int i = 0; i < 5; ++i) {
354                if (urlConn.getResponseCode() == 503) {
355                    Thread.sleep(5000+(new Random()).nextInt(5000));
356                    continue;
357                }
358
359                attributes.setResponseCode(urlConn.getResponseCode());
360                byte[] raw;
361                if (urlConn.getResponseCode() == 200) {
362                    raw = Utils.readBytesFromStream(urlConn.getInputStream());
363                } else {
364                    raw = new byte[]{};
365                }
366
367                if (isResponseLoadable(urlConn.getHeaderFields(), urlConn.getResponseCode(), raw)) {
368                    // we need to check cacheEmpty, so for cases, when data is returned, but we want to store
369                    // as empty (eg. empty tile images) to save some space
370                    cacheData = createCacheEntry(raw);
371                    cache.put(getCacheKey(), cacheData, attributes);
372                    log.log(Level.FINE, "JCS - downloaded key: {0}, length: {1}, url: {2}",
373                            new Object[] {getCacheKey(), raw.length, getUrl()});
374                    return true;
375                } else if (cacheAsEmpty()) {
376                    cacheData = createCacheEntry(new byte[]{});
377                    cache.put(getCacheKey(), cacheData, attributes);
378                    log.log(Level.FINE, "JCS - Caching empty object {0}", getUrl());
379                    return true;
380                } else {
381                    log.log(Level.FINE, "JCS - failure during load - reponse is not loadable nor cached as empty");
382                    return false;
383                }
384            }
385        } catch (FileNotFoundException e) {
386            log.log(Level.FINE, "JCS - Caching empty object as server returned 404 for: {0}", getUrlNoException());
387            attributes.setResponseCode(404);
388            attributes.setErrorMessage(e.toString());
389            boolean doCache = isResponseLoadable(null, 404, null) || cacheAsEmpty();
390            if (doCache) {
391                cacheData = createCacheEntry(new byte[]{});
392                cache.put(getCacheKey(), cacheData, attributes);
393            }
394            return doCache;
395        } catch (IOException e) {
396            log.log(Level.FINE, "JCS - IOExecption during communication with server for: {0}", getUrlNoException());
397            attributes.setErrorMessage(e.toString());
398            attributes.setResponseCode(499); // set dummy error code
399            boolean doCache = isResponseLoadable(null, 499, null) || cacheAsEmpty(); //generic 499 error code returned
400            if (doCache) {
401                cacheData = createCacheEntry(new byte[]{});
402                cache.put(getCacheKey(), createCacheEntry(new byte[]{}), attributes);
403            }
404            return doCache;
405        } catch (Exception e) {
406            attributes.setErrorMessage(e.toString());
407            log.log(Level.WARNING, "JCS - Exception during download {0}",  getUrlNoException());
408            Main.warn(e);
409        }
410        log.log(Level.WARNING, "JCS - Silent failure during download: {0}", getUrlNoException());
411        return false;
412
413    }
414
415    /**
416     * Check if the object is loadable. This means, if the data will be parsed, and if this response
417     * will finish as successful retrieve.
418     *
419     * This simple implementation doesn't load empty response, nor client (4xx) and server (5xx) errors
420     *
421     * @param headerFields headers sent by server
422     * @param responseCode http status code
423     * @param raw data read from server
424     * @return true if object should be cached and returned to listener
425     */
426    protected boolean isResponseLoadable(Map<String, List<String>> headerFields, int responseCode, byte[] raw) {
427        if (raw == null || raw.length == 0 || responseCode >= 400) {
428            return false;
429        }
430        return true;
431    }
432
433    protected abstract V createCacheEntry(byte[] content);
434
435    protected CacheEntryAttributes parseHeaders(URLConnection urlConn) {
436        CacheEntryAttributes ret = new CacheEntryAttributes();
437
438        Long lng = urlConn.getExpiration();
439        if (lng.equals(0L)) {
440            try {
441                String str = urlConn.getHeaderField("Cache-Control");
442                if (str != null) {
443                    for (String token: str.split(",")) {
444                        if (token.startsWith("max-age=")) {
445                            lng = Long.parseLong(token.substring(8)) * 1000 +
446                                    System.currentTimeMillis();
447                        }
448                    }
449                }
450            } catch (NumberFormatException e) {
451                // ignore malformed Cache-Control headers
452                if (Main.isTraceEnabled()) {
453                    Main.trace(e.getMessage());
454                }
455            }
456        }
457
458        ret.setExpirationTime(lng);
459        ret.setLastModification(now);
460        ret.setEtag(urlConn.getHeaderField("ETag"));
461        return ret;
462    }
463
464    private HttpURLConnection getURLConnection(URL url) throws IOException {
465        HttpURLConnection urlConn = (HttpURLConnection) url.openConnection();
466        urlConn.setRequestProperty("Accept", "text/html, image/png, image/jpeg, image/gif, */*");
467        urlConn.setReadTimeout(readTimeout); // 30 seconds read timeout
468        urlConn.setConnectTimeout(connectTimeout);
469        if (headers != null) {
470            for (Map.Entry<String, String> e: headers.entrySet()) {
471                urlConn.setRequestProperty(e.getKey(), e.getValue());
472            }
473        }
474
475        if (force) {
476            urlConn.setUseCaches(false);
477        }
478        return urlConn;
479    }
480
481    private boolean isCacheValidUsingHead() throws IOException {
482        HttpURLConnection urlConn = getURLConnection(getUrl());
483        urlConn.setRequestMethod("HEAD");
484        for (int i = 0; i < 5; i++) {
485            if (urlConn.getResponseCode() == 302) {
486                urlConn = getURLConnection(new URL(urlConn.getHeaderField("Location")));
487            } else {
488                break;
489            }
490        }
491        long lastModified = urlConn.getLastModified();
492        return (attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) ||
493                (lastModified != 0 && lastModified <= attributes.getLastModification());
494    }
495
496    /**
497     * TODO: move to JobFactory
498     * cancels all outstanding tasks in the queue.
499     */
500    public void cancelOutstandingTasks() {
501        for (Runnable r: downloadJobExecutor.getQueue()) {
502            if (downloadJobExecutor.remove(r) && r instanceof JCSCachedTileLoaderJob) {
503                ((JCSCachedTileLoaderJob<?, ?>) r).handleJobCancellation();
504            }
505        }
506    }
507
508    /**
509     * Sets a job, that will be run, when job will finish execution
510     * @param runnable that will be executed
511     */
512    public void setFinishedTask(Runnable runnable) {
513        this.finishTask = runnable;
514
515    }
516
517    /**
518     * Marks this job as canceled
519     */
520    public void handleJobCancellation() {
521        finishLoading(LoadResult.CANCELED);
522    }
523
524    private URL getUrlNoException() {
525        try {
526            return getUrl();
527        } catch (IOException e) {
528            return null;
529        }
530    }
531}