001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.IOException; 007import java.io.InputStream; 008import java.nio.charset.StandardCharsets; 009import java.time.Duration; 010import java.time.LocalDateTime; 011import java.time.Period; 012import java.time.ZoneOffset; 013import java.util.Arrays; 014import java.util.EnumMap; 015import java.util.List; 016import java.util.Locale; 017import java.util.Map; 018import java.util.NoSuchElementException; 019import java.util.Objects; 020import java.util.concurrent.ConcurrentHashMap; 021import java.util.concurrent.TimeUnit; 022import java.util.regex.Matcher; 023import java.util.regex.Pattern; 024 025import javax.xml.stream.XMLStreamConstants; 026import javax.xml.stream.XMLStreamException; 027 028import org.openstreetmap.josm.data.Bounds; 029import org.openstreetmap.josm.data.DataSource; 030import org.openstreetmap.josm.data.coor.LatLon; 031import org.openstreetmap.josm.data.osm.BBox; 032import org.openstreetmap.josm.data.osm.DataSet; 033import org.openstreetmap.josm.data.osm.DataSetMerger; 034import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 035import org.openstreetmap.josm.data.osm.PrimitiveId; 036import org.openstreetmap.josm.data.preferences.BooleanProperty; 037import org.openstreetmap.josm.data.preferences.ListProperty; 038import org.openstreetmap.josm.data.preferences.StringProperty; 039import org.openstreetmap.josm.gui.download.OverpassDownloadSource; 040import org.openstreetmap.josm.gui.progress.ProgressMonitor; 041import org.openstreetmap.josm.io.NameFinder.SearchResult; 042import org.openstreetmap.josm.tools.HttpClient; 043import org.openstreetmap.josm.tools.Logging; 044import org.openstreetmap.josm.tools.UncheckedParseException; 045import org.openstreetmap.josm.tools.Utils; 046 047/** 048 * Read content from an Overpass server. 049 * 050 * @since 8744 051 */ 052public class OverpassDownloadReader extends BoundingBoxDownloader { 053 054 /** 055 * Property for current Overpass server. 056 * @since 12816 057 */ 058 public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server", 059 "https://overpass-api.de/api/"); 060 /** 061 * Property for list of known Overpass servers. 062 * @since 12816 063 */ 064 public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers", 065 Arrays.asList("https://overpass-api.de/api/", "http://overpass.openstreetmap.ru/cgi/")); 066 /** 067 * Property to determine if Overpass API should be used for multi-fetch download. 068 * @since 12816 069 */ 070 public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false); 071 072 private static final String DATA_PREFIX = "?data="; 073 074 static final class OverpassOsmReader extends OsmReader { 075 @Override 076 protected void parseUnknown(boolean printWarning) throws XMLStreamException { 077 if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) { 078 final String text = parser.getElementText(); 079 if (text.contains("runtime error")) { 080 throw new XMLStreamException(text); 081 } 082 } 083 super.parseUnknown(printWarning); 084 } 085 } 086 087 static final class OverpassOsmJsonReader extends OsmJsonReader { 088 089 } 090 091 /** 092 * Possible Overpass API output format, with the {@code [out:<directive>]} statement. 093 * @since 11916 094 */ 095 public enum OverpassOutpoutFormat { 096 /** Default output format: plain OSM XML */ 097 OSM_XML("xml"), 098 /** OSM JSON format (not GeoJson) */ 099 OSM_JSON("json"), 100 /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */ 101 CSV("csv"), 102 /** Custom, see https://overpass-api.de/output_formats.html#custom */ 103 CUSTOM("custom"), 104 /** Popup, see https://overpass-api.de/output_formats.html#popup */ 105 POPUP("popup"), 106 /** PBF, see https://josm.openstreetmap.de/ticket/14653 */ 107 PBF("pbf"); 108 109 private final String directive; 110 111 OverpassOutpoutFormat(String directive) { 112 this.directive = directive; 113 } 114 115 /** 116 * Returns the directive used in {@code [out:<directive>]} statement. 117 * @return the directive used in {@code [out:<directive>]} statement 118 */ 119 public String getDirective() { 120 return directive; 121 } 122 123 /** 124 * Returns the {@code OverpassOutpoutFormat} matching the given directive. 125 * @param directive directive used in {@code [out:<directive>]} statement 126 * @return {@code OverpassOutpoutFormat} matching the given directive 127 * @throws IllegalArgumentException in case of invalid directive 128 */ 129 static OverpassOutpoutFormat from(String directive) { 130 for (OverpassOutpoutFormat oof : values()) { 131 if (oof.directive.equals(directive)) { 132 return oof; 133 } 134 } 135 throw new IllegalArgumentException(directive); 136 } 137 } 138 139 static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL); 140 141 static final Map<OverpassOutpoutFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>(); 142 143 final String overpassServer; 144 final String overpassQuery; 145 146 /** 147 * Constructs a new {@code OverpassDownloadReader}. 148 * 149 * @param downloadArea The area to download 150 * @param overpassServer The Overpass server to use 151 * @param overpassQuery The Overpass query 152 */ 153 public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) { 154 super(downloadArea); 155 setDoAuthenticate(false); 156 this.overpassServer = overpassServer; 157 this.overpassQuery = overpassQuery.trim(); 158 } 159 160 /** 161 * Registers an OSM reader for the given Overpass output format. 162 * @param format Overpass output format 163 * @param readerClass OSM reader class 164 * @return the previous value associated with {@code format}, or {@code null} if there was no mapping 165 */ 166 public static final Class<? extends AbstractReader> registerOverpassOutpoutFormatReader( 167 OverpassOutpoutFormat format, Class<? extends AbstractReader> readerClass) { 168 return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass)); 169 } 170 171 static { 172 registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class); 173 registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_JSON, OverpassOsmJsonReader.class); 174 } 175 176 @Override 177 protected String getBaseUrl() { 178 return overpassServer; 179 } 180 181 @Override 182 protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) { 183 if (overpassQuery.isEmpty()) 184 return super.getRequestForBbox(lon1, lat1, lon2, lat2); 185 else { 186 final String query = this.overpassQuery 187 .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2)) 188 .replace("{{center}}", center(lon1, lat1, lon2, lat2)); 189 final String expandedOverpassQuery = expandExtendedQueries(query); 190 return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery); 191 } 192 } 193 194 /** 195 * Evaluates some features of overpass turbo extended query syntax. 196 * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries 197 * @param query unexpanded query 198 * @return expanded query 199 */ 200 static String expandExtendedQueries(String query) { 201 final StringBuffer sb = new StringBuffer(); 202 final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query); 203 while (matcher.find()) { 204 try { 205 switch (matcher.group(1)) { 206 case "date": 207 matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now())); 208 break; 209 case "geocodeArea": 210 matcher.appendReplacement(sb, geocodeArea(matcher.group(2))); 211 break; 212 case "geocodeBbox": 213 matcher.appendReplacement(sb, geocodeBbox(matcher.group(2))); 214 break; 215 case "geocodeCoords": 216 matcher.appendReplacement(sb, geocodeCoords(matcher.group(2))); 217 break; 218 case "geocodeId": 219 matcher.appendReplacement(sb, geocodeId(matcher.group(2))); 220 break; 221 default: 222 Logging.warn("Unsupported syntax: " + matcher.group(1)); 223 } 224 } catch (UncheckedParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) { 225 final String msg = tr("Failed to evaluate {0}", matcher.group()); 226 Logging.log(Logging.LEVEL_WARN, msg, ex); 227 matcher.appendReplacement(sb, "// " + msg + "\n"); 228 } 229 } 230 matcher.appendTail(sb); 231 return sb.toString(); 232 } 233 234 static String bbox(double lon1, double lat1, double lon2, double lat2) { 235 return lat1 + "," + lon1 + "," + lat2 + "," + lon2; 236 } 237 238 static String center(double lon1, double lat1, double lon2, double lat2) { 239 LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter(); 240 return c.lat()+ "," + c.lon(); 241 } 242 243 static String date(String humanDuration, LocalDateTime from) { 244 // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes 245 String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "") 246 .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W") 247 .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S"); 248 Matcher matcher = Pattern.compile( 249 "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+ 250 "((?:[0-9]+D)?)" + 251 "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration); 252 boolean javaPer = false; 253 boolean javaDur = false; 254 if (matcher.matches()) { 255 javaPer = matcher.group(1) != null && !matcher.group(1).isEmpty(); 256 javaDur = matcher.group(3) != null && !matcher.group(3).isEmpty(); 257 duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2); 258 if (javaDur) { 259 duration += 'T' + matcher.group(3); 260 } 261 } 262 263 // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely. 264 // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds). 265 Period p = null; 266 Duration d = null; 267 int idx = duration.indexOf('T'); 268 if (javaPer) { 269 p = Period.parse(javaDur ? duration.substring(0, idx) : duration); 270 } 271 if (javaDur) { 272 d = Duration.parse(javaPer ? 'P' + duration.substring(idx, duration.length()) : duration); 273 } else if (!javaPer) { 274 d = Duration.parse(duration); 275 } 276 277 // Now that period and duration are known, compute the correct date/time 278 LocalDateTime dt = from; 279 if (p != null) { 280 dt = dt.minus(p); 281 } 282 if (d != null) { 283 dt = dt.minus(d); 284 } 285 286 // Returns the date/time formatted in ISO 8601 287 return dt.toInstant(ZoneOffset.UTC).toString(); 288 } 289 290 private static SearchResult searchName(String area) throws IOException { 291 return searchName(NameFinder.queryNominatim(area)); 292 } 293 294 static SearchResult searchName(List<SearchResult> results) { 295 return results.stream().filter( 296 x -> OsmPrimitiveType.NODE != x.getOsmId().getType()).iterator().next(); 297 } 298 299 static String geocodeArea(String area) throws IOException { 300 // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id 301 final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class); 302 idOffset.put(OsmPrimitiveType.NODE, 0L); 303 idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L); 304 idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L); 305 final PrimitiveId osmId = searchName(area).getOsmId(); 306 Logging.debug("Area ''{0}'' resolved to {1}", area, osmId); 307 return String.format("area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType())); 308 } 309 310 static String geocodeBbox(String area) throws IOException { 311 Bounds bounds = searchName(area).getBounds(); 312 return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon(); 313 } 314 315 static String geocodeCoords(String area) throws IOException { 316 SearchResult result = searchName(area); 317 return result.getLat() + "," + result.getLon(); 318 } 319 320 static String geocodeId(String area) throws IOException { 321 PrimitiveId osmId = searchName(area).getOsmId(); 322 return String.format("%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId()); 323 } 324 325 @Override 326 protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, 327 boolean uncompressAccordingToContentDisposition) throws OsmTransferException { 328 try { 329 int index = urlStr.indexOf(DATA_PREFIX); 330 // Make an HTTP POST request instead of a simple GET, allows more complex queries 331 return super.getInputStreamRaw(urlStr.substring(0, index), 332 progressMonitor, reason, uncompressAccordingToContentDisposition, 333 "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8)); 334 } catch (OsmApiException ex) { 335 final String errorIndicator = "Error</strong>: "; 336 if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) { 337 final String errorPlusRest = ex.getMessage().split(errorIndicator)[1]; 338 if (errorPlusRest != null) { 339 ex.setErrorHeader(errorPlusRest.split("</")[0].replaceAll(".*::request_read_and_idx::", "")); 340 } 341 } 342 throw ex; 343 } 344 } 345 346 @Override 347 protected void adaptRequest(HttpClient request) { 348 // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout 349 final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery); 350 final int timeout; 351 if (timeoutMatcher.find()) { 352 timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1))); 353 } else { 354 timeout = (int) TimeUnit.MINUTES.toMillis(3); 355 } 356 request.setConnectTimeout(timeout); 357 request.setReadTimeout(timeout); 358 } 359 360 @Override 361 protected String getTaskName() { 362 return tr("Contacting Server..."); 363 } 364 365 @Override 366 protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 367 AbstractReader reader = null; 368 Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery); 369 if (m.matches()) { 370 Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutpoutFormat.from(m.group(1))); 371 if (readerClass != null) { 372 try { 373 reader = readerClass.getDeclaredConstructor().newInstance(); 374 } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) { 375 Logging.error(e); 376 } 377 } 378 } 379 if (reader == null) { 380 reader = new OverpassOsmReader(); 381 } 382 return reader.doParseDataSet(source, progressMonitor); 383 } 384 385 @Override 386 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 387 388 DataSet ds = super.parseOsm(progressMonitor); 389 if (!considerAsFullDownload()) { 390 DataSet noBounds = new DataSet(); 391 DataSetMerger dsm = new DataSetMerger(noBounds, ds); 392 dsm.merge(null, false); 393 return dsm.getTargetDataSet(); 394 } else { 395 // add bounds if necessary (note that Overpass API does not return bounds in the response XML) 396 if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) { 397 if (crosses180th) { 398 Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0); 399 DataSource src = new DataSource(bounds, getBaseUrl()); 400 ds.addDataSource(src); 401 402 bounds = new Bounds(lat1, -180.0, lat2, lon2); 403 src = new DataSource(bounds, getBaseUrl()); 404 ds.addDataSource(src); 405 } else { 406 Bounds bounds = new Bounds(lat1, lon1, lat2, lon2); 407 DataSource src = new DataSource(bounds, getBaseUrl()); 408 ds.addDataSource(src); 409 } 410 } 411 return ds; 412 } 413 } 414 415 /** 416 * Fixes Overpass API query to make sure it will be accepted by JOSM. 417 * @param query Overpass query to check 418 * @return fixed query 419 * @since 13335 420 */ 421 public static String fixQuery(String query) { 422 return query == null ? query : query 423 .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;") 424 .replaceAll("(?s)\\[out:(csv)[^\\]]*\\]", "[out:xml]"); 425 } 426 427 @Override 428 public boolean considerAsFullDownload() { 429 return overpassQuery.equals(OverpassDownloadSource.FULL_DOWNLOAD_QUERY); 430 } 431}