001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.imagery; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.ByteArrayInputStream; 007import java.io.IOException; 008import java.net.SocketTimeoutException; 009import java.net.URL; 010import java.nio.charset.StandardCharsets; 011import java.util.HashSet; 012import java.util.List; 013import java.util.Map; 014import java.util.Map.Entry; 015import java.util.Optional; 016import java.util.Set; 017import java.util.concurrent.ConcurrentHashMap; 018import java.util.concurrent.ConcurrentMap; 019import java.util.concurrent.ThreadPoolExecutor; 020import java.util.concurrent.TimeUnit; 021import java.util.regex.Matcher; 022import java.util.regex.Pattern; 023 024import org.apache.commons.jcs.access.behavior.ICacheAccess; 025import org.openstreetmap.gui.jmapviewer.Tile; 026import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; 027import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; 028import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; 029import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource; 030import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry; 031import org.openstreetmap.josm.data.cache.CacheEntry; 032import org.openstreetmap.josm.data.cache.CacheEntryAttributes; 033import org.openstreetmap.josm.data.cache.ICachedLoaderListener; 034import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob; 035import org.openstreetmap.josm.data.preferences.LongProperty; 036import org.openstreetmap.josm.tools.HttpClient; 037import org.openstreetmap.josm.tools.Logging; 038import org.openstreetmap.josm.tools.Utils; 039 040/** 041 * Class bridging TMS requests to JCS cache requests 042 * 043 * @author Wiktor Niesiobędzki 044 * @since 8168 045 */ 046public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob<String, BufferedImageCacheEntry> implements TileJob, ICachedLoaderListener { 047 /** General maximum expires for tiles. Might be overridden by imagery settings */ 048 public static final LongProperty MAXIMUM_EXPIRES = new LongProperty("imagery.generic.maximum_expires", TimeUnit.DAYS.toMillis(30)); 049 /** General minimum expires for tiles. Might be overridden by imagery settings */ 050 public static final LongProperty MINIMUM_EXPIRES = new LongProperty("imagery.generic.minimum_expires", TimeUnit.HOURS.toMillis(1)); 051 static final Pattern SERVICE_EXCEPTION_PATTERN = Pattern.compile("(?s).+<ServiceException[^>]*>(.+)</ServiceException>.+"); 052 static final Pattern CDATA_PATTERN = Pattern.compile("(?s)\\s*<!\\[CDATA\\[(.+)\\]\\]>\\s*"); 053 static final Pattern JSON_PATTERN = Pattern.compile("\\{\"message\":\"(.+)\"\\}"); 054 protected final Tile tile; 055 private volatile URL url; 056 private final TileJobOptions options; 057 058 // we need another deduplication of Tile Loader listeners, as for each submit, new TMSCachedTileLoaderJob was created 059 // that way, we reduce calls to tileLoadingFinished, and general CPU load due to surplus Map repaints 060 private static final ConcurrentMap<String, Set<TileLoaderListener>> inProgress = new ConcurrentHashMap<>(); 061 062 /** 063 * Constructor for creating a job, to get a specific tile from cache 064 * @param listener Tile loader listener 065 * @param tile to be fetched from cache 066 * @param cache object 067 * @param options for job (such as http headers, timeouts etc.) 068 * @param downloadExecutor that will be executing the jobs 069 */ 070 public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile, 071 ICacheAccess<String, BufferedImageCacheEntry> cache, 072 TileJobOptions options, 073 ThreadPoolExecutor downloadExecutor) { 074 super(cache, options, downloadExecutor); 075 this.tile = tile; 076 this.options = options; 077 if (listener != null) { 078 String deduplicationKey = getCacheKey(); 079 synchronized (inProgress) { 080 inProgress.computeIfAbsent(deduplicationKey, k -> new HashSet<>()).add(listener); 081 } 082 } 083 } 084 085 @Override 086 public String getCacheKey() { 087 if (tile != null) { 088 TileSource tileSource = tile.getTileSource(); 089 return Optional.ofNullable(tileSource.getName()).orElse("").replace(':', '_') + ':' 090 + tileSource.getTileId(tile.getZoom(), tile.getXtile(), tile.getYtile()); 091 } 092 return null; 093 } 094 095 /* 096 * this doesn't needs to be synchronized, as it's not that costly to keep only one execution 097 * in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching 098 * data from cache, that's why URL creation is postponed until it's needed 099 * 100 * We need to have static url value for TileLoaderJob, as for some TileSources we might get different 101 * URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection 102 * 103 */ 104 @Override 105 public URL getUrl() throws IOException { 106 if (url == null) { 107 synchronized (this) { 108 if (url == null) { 109 String sUrl = tile.getUrl(); 110 if (!"".equals(sUrl)) { 111 url = new URL(sUrl); 112 } 113 } 114 } 115 } 116 return url; 117 } 118 119 @Override 120 public boolean isObjectLoadable() { 121 if (cacheData != null) { 122 byte[] content = cacheData.getContent(); 123 try { 124 return content.length > 0 || cacheData.getImage() != null || isNoTileAtZoom(); 125 } catch (IOException e) { 126 Logging.logWithStackTrace(Logging.LEVEL_WARN, e, "JCS TMS - error loading from cache for tile {0}: {1}", 127 tile.getKey(), e.getMessage()); 128 } 129 } 130 return false; 131 } 132 133 @Override 134 protected boolean isResponseLoadable(Map<String, List<String>> headers, int statusCode, byte[] content) { 135 attributes.setMetadata(tile.getTileSource().getMetadata(headers)); 136 if (tile.getTileSource().isNoTileAtZoom(headers, statusCode, content)) { 137 attributes.setNoTileAtZoom(true); 138 return false; // do no try to load data from no-tile at zoom, cache empty object instead 139 } 140 return super.isResponseLoadable(headers, statusCode, content); 141 } 142 143 @Override 144 protected boolean cacheAsEmpty() { 145 return isNoTileAtZoom() || super.cacheAsEmpty(); 146 } 147 148 @Override 149 public void submit(boolean force) { 150 tile.initLoading(); 151 try { 152 super.submit(this, force); 153 } catch (IOException | IllegalArgumentException e) { 154 // if we fail to submit the job, mark tile as loaded and set error message 155 Logging.log(Logging.LEVEL_WARN, e); 156 tile.finishLoading(); 157 tile.setError(e.getMessage()); 158 } 159 } 160 161 @Override 162 public void loadingFinished(CacheEntry object, CacheEntryAttributes attributes, LoadResult result) { 163 this.attributes = attributes; // as we might get notification from other object than our selfs, pass attributes along 164 Set<TileLoaderListener> listeners; 165 synchronized (inProgress) { 166 listeners = inProgress.remove(getCacheKey()); 167 } 168 boolean status = result == LoadResult.SUCCESS; 169 170 try { 171 tile.finishLoading(); // whatever happened set that loading has finished 172 // set tile metadata 173 if (this.attributes != null) { 174 for (Entry<String, String> e: this.attributes.getMetadata().entrySet()) { 175 tile.putValue(e.getKey(), e.getValue()); 176 } 177 } 178 179 switch(result) { 180 case SUCCESS: 181 handleNoTileAtZoom(); 182 if (attributes != null) { 183 int httpStatusCode = attributes.getResponseCode(); 184 if (httpStatusCode >= 400 && !isNoTileAtZoom()) { 185 status = false; 186 handleError(attributes); 187 } 188 } 189 status &= tryLoadTileImage(object); //try to keep returned image as background 190 break; 191 case FAILURE: 192 handleError(attributes); 193 tryLoadTileImage(object); 194 break; 195 case CANCELED: 196 tile.loadingCanceled(); 197 // do nothing 198 } 199 200 // always check, if there is some listener interested in fact, that tile has finished loading 201 if (listeners != null) { // listeners might be null, if some other thread notified already about success 202 for (TileLoaderListener l: listeners) { 203 l.tileLoadingFinished(tile, status); 204 } 205 } 206 } catch (IOException e) { 207 Logging.warn("JCS TMS - error loading object for tile {0}: {1}", tile.getKey(), e.getMessage()); 208 tile.setError(e); 209 tile.setLoaded(false); 210 if (listeners != null) { // listeners might be null, if some other thread notified already about success 211 for (TileLoaderListener l: listeners) { 212 l.tileLoadingFinished(tile, false); 213 } 214 } 215 } 216 } 217 218 private void handleError(CacheEntryAttributes attributes) { 219 if (attributes != null) { 220 int httpStatusCode = attributes.getResponseCode(); 221 if (attributes.getErrorMessage() == null) { 222 tile.setError(tr("HTTP error {0} when loading tiles", httpStatusCode)); 223 } else { 224 tile.setError(tr("Error downloading tiles: {0}", attributes.getErrorMessage())); 225 } 226 if (httpStatusCode >= 500 && httpStatusCode != 599) { 227 // httpStatusCode = 599 is set by JCSCachedTileLoaderJob on IOException 228 tile.setLoaded(false); // treat 500 errors as temporary and try to load it again 229 } 230 // treat SocketTimeoutException as transient error 231 attributes.getException() 232 .filter(x -> x.isAssignableFrom(SocketTimeoutException.class)) 233 .ifPresent(x -> tile.setLoaded(false)); 234 } else { 235 tile.setError(tr("Problem loading tile")); 236 // treat unknown errors as permanent and do not try to load tile again 237 } 238 } 239 240 /** 241 * For TMS use BaseURL as settings discovery, so for different paths, we will have different settings (useful for developer servers) 242 * 243 * @return base URL of TMS or server url as defined in super class 244 */ 245 @Override 246 protected String getServerKey() { 247 TileSource ts = tile.getSource(); 248 if (ts instanceof AbstractTMSTileSource) { 249 return ((AbstractTMSTileSource) ts).getBaseUrl(); 250 } 251 return super.getServerKey(); 252 } 253 254 @Override 255 protected BufferedImageCacheEntry createCacheEntry(byte[] content) { 256 return new BufferedImageCacheEntry(content); 257 } 258 259 @Override 260 public void submit() { 261 submit(false); 262 } 263 264 @Override 265 protected CacheEntryAttributes parseHeaders(HttpClient.Response urlConn) { 266 CacheEntryAttributes ret = super.parseHeaders(urlConn); 267 // keep the expiration time between MINIMUM_EXPIRES and MAXIMUM_EXPIRES, so we will cache the tiles 268 // at least for some short period of time, but not too long 269 if (ret.getExpirationTime() < now + Math.max(MINIMUM_EXPIRES.get(), options.getMinimumExpiryTime())) { 270 ret.setExpirationTime(now + Math.max(MINIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime()))); 271 } 272 if (ret.getExpirationTime() > now + Math.max(MAXIMUM_EXPIRES.get(), options.getMinimumExpiryTime())) { 273 ret.setExpirationTime(now + Math.max(MAXIMUM_EXPIRES.get(), TimeUnit.SECONDS.toMillis(options.getMinimumExpiryTime()))); 274 } 275 return ret; 276 } 277 278 private boolean handleNoTileAtZoom() { 279 if (isNoTileAtZoom()) { 280 Logging.debug("JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile); 281 tile.setError(tr("No tiles at this zoom level")); 282 tile.putValue("tile-info", "no-tile"); 283 return true; 284 } 285 return false; 286 } 287 288 private boolean isNoTileAtZoom() { 289 if (attributes == null) { 290 Logging.warn("Cache attributes are null"); 291 } 292 return attributes != null && attributes.isNoTileAtZoom(); 293 } 294 295 private boolean tryLoadTileImage(CacheEntry object) throws IOException { 296 if (object != null) { 297 byte[] content = object.getContent(); 298 if (content.length > 0) { 299 try (ByteArrayInputStream in = new ByteArrayInputStream(content)) { 300 tile.loadImage(in); 301 if (tile.getImage() == null) { 302 String s = new String(content, StandardCharsets.UTF_8); 303 Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s); 304 if (m.matches()) { 305 String message = Utils.strip(m.group(1)); 306 tile.setError(message); 307 Logging.error(message); 308 Logging.debug(s); 309 } else { 310 tile.setError(tr("Could not load image from tile server")); 311 } 312 return false; 313 } 314 } catch (UnsatisfiedLinkError | SecurityException e) { 315 throw new IOException(e); 316 } 317 } 318 } 319 return true; 320 } 321 322 @Override 323 public String detectErrorMessage(String data) { 324 Matcher xml = SERVICE_EXCEPTION_PATTERN.matcher(data); 325 Matcher json = JSON_PATTERN.matcher(data); 326 return xml.matches() ? removeCdata(Utils.strip(xml.group(1))) 327 : json.matches() ? Utils.strip(json.group(1)) 328 : super.detectErrorMessage(data); 329 } 330 331 private static String removeCdata(String msg) { 332 Matcher m = CDATA_PATTERN.matcher(msg); 333 return m.matches() ? Utils.strip(m.group(1)) : msg; 334 } 335}