001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.imagery;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.ByteArrayInputStream;
007import java.io.IOException;
008import java.net.SocketTimeoutException;
009import java.net.URL;
010import java.nio.charset.StandardCharsets;
011import java.util.HashSet;
012import java.util.List;
013import java.util.Map;
014import java.util.Map.Entry;
015import java.util.Optional;
016import java.util.Set;
017import java.util.concurrent.ConcurrentHashMap;
018import java.util.concurrent.ConcurrentMap;
019import java.util.concurrent.ThreadPoolExecutor;
020import java.util.concurrent.TimeUnit;
021import java.util.regex.Matcher;
022import java.util.regex.Pattern;
023
024import org.apache.commons.jcs.access.behavior.ICacheAccess;
025import org.openstreetmap.gui.jmapviewer.Tile;
026import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
027import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
028import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
029import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
030import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
031import org.openstreetmap.josm.data.cache.CacheEntry;
032import org.openstreetmap.josm.data.cache.CacheEntryAttributes;
033import org.openstreetmap.josm.data.cache.ICachedLoaderListener;
034import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob;
035import org.openstreetmap.josm.data.preferences.LongProperty;
036import org.openstreetmap.josm.tools.HttpClient;
037import org.openstreetmap.josm.tools.Logging;
038import org.openstreetmap.josm.tools.Utils;
039
040/**
041 * Class bridging TMS requests to JCS cache requests
042 *
043 * @author Wiktor Niesiobędzki
044 * @since 8168
045 */
046public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener {
047    /** General maximum expires for tiles. Might be overridden by imagery settings */
048    public static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30));
049    /** General minimum expires for tiles. Might be overridden by imagery settings */
050    public static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1));
051    static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException[^>]*>(.+)</ServiceException>.+");
052    static final Pattern CDATA_PATTERN = Pattern.compile("(?s)\\s*<!\\[CDATA\\[(.+)\\]\\]>\\s*");
053    static final Pattern JSON_PATTERN = Pattern.compile("\\{\"message\":\"(.+)\"\\}");
054    protected final Tile tile;
055    private volatile URL url;
056    private final TileJobOptions options;
057
058    // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created
059    // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints
060    private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>();
061
062    /**
063     * Constructor for creating a job, to get a specific tile from cache
064     * @param listener Tile loader listener
065     * @param tile to be fetched from cache
066     * @param cache object
067     * @param options for job (such as http headers, timeouts etc.)
068     * @param downloadExecutor that will be executing the jobs
069     */
070    public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile,
071            ICacheAccess<String, BufferedImageCacheEntry> cache,
072            TileJobOptions options,
073            ThreadPoolExecutor downloadExecutor) {
074        super(cache, options, downloadExecutor);
075        this.tile = tile;
076        this.options = options;
077        if (listener != null) {
078            String deduplicationKey = getCacheKey();
079            synchronized (inProgress) {
080                inProgress.computeIfAbsent(deduplicationKey, k -> new HashSet<>()).add(listener);
081            }
082        }
083    }
084
085    @Override
086    public String getCacheKey() {
087        if (tile != null) {
088            TileSource tileSource = tile.getTileSource();
089            return Optional.ofNullable(tileSource.getName()).orElse("").replace(':', '_') + ':'
090                    + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile());
091        }
092        return null;
093    }
094
095    /*
096     *  this doesn't needs to be synchronized, as it's not that costly to keep only one execution
097     *  in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching
098     *  data from cache, that's why URL creation is postponed until it's needed
099     *
100     *  We need to have static url value for TileLoaderJob, as for some TileSources we might get different
101     *  URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection
102     *
103     */
104    @Override
105    public URL getUrl() throws IOException {
106        if (url == null) {
107            synchronized (this) {
108                if (url == null) {
109                    String sUrl = tile.getUrl();
110                    if (!"".equals(sUrl)) {
111                        url = new URL(sUrl);
112                    }
113                }
114            }
115        }
116        return url;
117    }
118
119    @Override
120    public boolean isObjectLoadable() {
121        if (cacheData != null) {
122            byte[] content = cacheData.getContent();
123            try {
124                return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom();
125            } catch (IOException e) {
126                Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}",
127                        tile.getKey(), e.getMessage());
128            }
129        }
130        return false;
131    }
132
133    @Override
134    protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) {
135        attributes.setMetadata(tile.getTileSource().getMetadata(headers));
136        if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) {
137            attributes.setNoTileAtZoom(true);
138            return false; // do no try to load data from no-tile at zoom, cache empty object instead
139        }
140        return super.isResponseLoadable(headers, statusCode, content);
141    }
142
143    @Override
144    protected boolean cacheAsEmpty() {
145        return isNoTileAtZoom() || super.cacheAsEmpty();
146    }
147
148    @Override
149    public void submit(boolean force) {
150        tile.initLoading();
151        try {
152            super.submit(this, force);
153        } catch (IOException | IllegalArgumentException e) {
154            // if we fail to submit the job, mark tile as loaded and set error message
155            Logging.log(Logging.LEVEL_WARN, e);
156            tile.finishLoading();
157            tile.setError(e.getMessage());
158        }
159    }
160
161    @Override
162    public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) {
163        this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along
164        Set<TileLoaderListener> listeners;
165        synchronized (inProgress) {
166            listeners = inProgress.remove(getCacheKey());
167        }
168        boolean status = result == LoadResult.SUCCESS;
169
170        try {
171            tile.finishLoading(); // whatever happened set that loading has finished
172            // set tile metadata
173            if (this.attributes != null) {
174                for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) {
175                    tile.putValue(e.getKey(), e.getValue());
176                }
177            }
178
179            switch(result) {
180            case SUCCESS:
181                handleNoTileAtZoom();
182                if (attributes != null) {
183                    int httpStatusCode = attributes.getResponseCode();
184                    if (httpStatusCode >= 400 && !isNoTileAtZoom()) {
185                        status = false;
186                        handleError(attributes);
187                    }
188                }
189                status &= tryLoadTileImage(object); //try to keep returned image as background
190                break;
191            case FAILURE:
192                handleError(attributes);
193                tryLoadTileImage(object);
194                break;
195            case CANCELED:
196                tile.loadingCanceled();
197                // do nothing
198            }
199
200            // always check, if there is some listener interested in fact, that tile has finished loading
201            if (listeners != null) { // listeners might be null, if some other thread notified already about success
202                for (TileLoaderListener l: listeners) {
203                    l.tileLoadingFinished(tile, status);
204                }
205            }
206        } catch (IOException e) {
207            Logging.warn("JCS TMS - error loading object for tile {0}: {1}", tile.getKey(), e.getMessage());
208            tile.setError(e);
209            tile.setLoaded(false);
210            if (listeners != null) { // listeners might be null, if some other thread notified already about success
211                for (TileLoaderListener l: listeners) {
212                    l.tileLoadingFinished(tile, false);
213                }
214            }
215        }
216    }
217
218    private void handleError(CacheEntryAttributes attributes) {
219        if (attributes != null) {
220            int httpStatusCode = attributes.getResponseCode();
221            if (attributes.getErrorMessage() == null) {
222                tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode));
223            } else {
224                tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage()));
225            }
226            if (httpStatusCode >= 500 && httpStatusCode != 599) {
227                // httpStatusCode = 599 is set by JCSCachedTileLoaderJob on IOException
228                tile.setLoaded(false); // treat 500 errors as temporary and try to load it again
229            }
230            // treat SocketTimeoutException as transient error
231            attributes.getException()
232            .filter(x -> x.isAssignableFrom(SocketTimeoutException.class))
233            .ifPresent(x -> tile.setLoaded(false));
234        } else {
235            tile.setError(tr("Problem loading tile"));
236            // treat unknown errors as permanent and do not try to load tile again
237        }
238    }
239
240    /**
241     * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers)
242     *
243     * @return base URL of TMS or server url as defined in super class
244     */
245    @Override
246    protected String getServerKey() {
247        TileSource ts = tile.getSource();
248        if (ts instanceof AbstractTMSTileSource) {
249            return ((AbstractTMSTileSource) ts).getBaseUrl();
250        }
251        return super.getServerKey();
252    }
253
254    @Override
255    protected BufferedImageCacheEntry createCacheEntry(byte[] content) {
256        return new BufferedImageCacheEntry(content);
257    }
258
259    @Override
260    public void submit() {
261        submit(false);
262    }
263
264    @Override
265    protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) {
266        CacheEntryAttributes ret = super.parseHeaders(urlConn);
267        // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles
268        // at least for some short period of time, but not too long
269        if (ret.getExpirationTime() < now + Math.max(MINIMUM_EXPIRES.get(), options.getMinimumExpiryTime())) {
270            ret.setExpirationTime(now + Math.max(MINIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime())));
271        }
272        if (ret.getExpirationTime() > now + Math.max(MAXIMUM_EXPIRES.get(), options.getMinimumExpiryTime())) {
273            ret.setExpirationTime(now + Math.max(MAXIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime())));
274        }
275        return ret;
276    }
277
278    private boolean handleNoTileAtZoom() {
279        if (isNoTileAtZoom()) {
280            Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile);
281            tile.setError(tr("No tiles at this zoom level"));
282            tile.putValue("tile-info", "no-tile");
283            return true;
284        }
285        return false;
286    }
287
288    private boolean isNoTileAtZoom() {
289        if (attributes == null) {
290            Logging.warn("Cache attributes are null");
291        }
292        return attributes != null && attributes.isNoTileAtZoom();
293    }
294
295    private boolean tryLoadTileImage(CacheEntry object) throws IOException {
296        if (object != null) {
297            byte[] content = object.getContent();
298            if (content.length > 0) {
299                try (ByteArrayInputStream in = new ByteArrayInputStream(content)) {
300                    tile.loadImage(in);
301                    if (tile.getImage() == null) {
302                        String s = new String(content, StandardCharsets.UTF_8);
303                        Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s);
304                        if (m.matches()) {
305                            String message = Utils.strip(m.group(1));
306                            tile.setError(message);
307                            Logging.error(message);
308                            Logging.debug(s);
309                        } else {
310                            tile.setError(tr("Could not load image from tile server"));
311                        }
312                        return false;
313                    }
314                } catch (UnsatisfiedLinkError | SecurityException e) {
315                    throw new IOException(e);
316                }
317            }
318        }
319        return true;
320    }
321
322    @Override
323    public String detectErrorMessage(String data) {
324        Matcher xml = SERVICE_EXCEPTION_PATTERN.matcher(data);
325        Matcher json = JSON_PATTERN.matcher(data);
326        return xml.matches() ? removeCdata(Utils.strip(xml.group(1)))
327            : json.matches() ? Utils.strip(json.group(1))
328            : super.detectErrorMessage(data);
329    }
330
331    private static String removeCdata(String msg) {
332        Matcher m = CDATA_PATTERN.matcher(msg);
333        return m.matches() ? Utils.strip(m.group(1)) : msg;
334    }
335}