001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.IOException;
007import java.io.InputStream;
008import java.nio.charset.StandardCharsets;
009import java.time.Duration;
010import java.time.LocalDateTime;
011import java.time.Period;
012import java.time.ZoneOffset;
013import java.util.Arrays;
014import java.util.EnumMap;
015import java.util.List;
016import java.util.Locale;
017import java.util.Map;
018import java.util.NoSuchElementException;
019import java.util.Objects;
020import java.util.concurrent.ConcurrentHashMap;
021import java.util.concurrent.TimeUnit;
022import java.util.regex.Matcher;
023import java.util.regex.Pattern;
024
025import javax.xml.stream.XMLStreamConstants;
026import javax.xml.stream.XMLStreamException;
027
028import org.openstreetmap.josm.data.Bounds;
029import org.openstreetmap.josm.data.DataSource;
030import org.openstreetmap.josm.data.coor.LatLon;
031import org.openstreetmap.josm.data.osm.BBox;
032import org.openstreetmap.josm.data.osm.DataSet;
033import org.openstreetmap.josm.data.osm.DataSetMerger;
034import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
035import org.openstreetmap.josm.data.osm.PrimitiveId;
036import org.openstreetmap.josm.data.preferences.BooleanProperty;
037import org.openstreetmap.josm.data.preferences.ListProperty;
038import org.openstreetmap.josm.data.preferences.StringProperty;
039import org.openstreetmap.josm.gui.download.OverpassDownloadSource;
040import org.openstreetmap.josm.gui.progress.ProgressMonitor;
041import org.openstreetmap.josm.io.NameFinder.SearchResult;
042import org.openstreetmap.josm.tools.HttpClient;
043import org.openstreetmap.josm.tools.Logging;
044import org.openstreetmap.josm.tools.UncheckedParseException;
045import org.openstreetmap.josm.tools.Utils;
046
047/**
048 * Read content from an Overpass server.
049 *
050 * @since 8744
051 */
052public class OverpassDownloadReader extends BoundingBoxDownloader {
053
054    /**
055     * Property for current Overpass server.
056     * @since 12816
057     */
058    public static final StringProperty OVERPASS_SERVER = new StringProperty("download.overpass.server",
059            "https://overpass-api.de/api/");
060    /**
061     * Property for list of known Overpass servers.
062     * @since 12816
063     */
064    public static final ListProperty OVERPASS_SERVER_HISTORY = new ListProperty("download.overpass.servers",
065            Arrays.asList("https://overpass-api.de/api/", "http://overpass.openstreetmap.ru/cgi/"));
066    /**
067     * Property to determine if Overpass API should be used for multi-fetch download.
068     * @since 12816
069     */
070    public static final BooleanProperty FOR_MULTI_FETCH = new BooleanProperty("download.overpass.for-multi-fetch", false);
071
072    private static final String DATA_PREFIX = "?data=";
073
074    static final class OverpassOsmReader extends OsmReader {
075        @Override
076        protected void parseUnknown(boolean printWarning) throws XMLStreamException {
077            if ("remark".equals(parser.getLocalName()) && parser.getEventType() == XMLStreamConstants.START_ELEMENT) {
078                final String text = parser.getElementText();
079                if (text.contains("runtime error")) {
080                    throw new XMLStreamException(text);
081                }
082            }
083            super.parseUnknown(printWarning);
084        }
085    }
086
087    static final class OverpassOsmJsonReader extends OsmJsonReader {
088
089    }
090
091    /**
092     * Possible Overpass API output format, with the {@code [out:<directive>]} statement.
093     * @since 11916
094     */
095    public enum OverpassOutpoutFormat {
096        /** Default output format: plain OSM XML */
097        OSM_XML("xml"),
098        /** OSM JSON format (not GeoJson) */
099        OSM_JSON("json"),
100        /** CSV, see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#Output_Format_.28out.29 */
101        CSV("csv"),
102        /** Custom, see https://overpass-api.de/output_formats.html#custom */
103        CUSTOM("custom"),
104        /** Popup, see https://overpass-api.de/output_formats.html#popup */
105        POPUP("popup"),
106        /** PBF, see https://josm.openstreetmap.de/ticket/14653 */
107        PBF("pbf");
108
109        private final String directive;
110
111        OverpassOutpoutFormat(String directive) {
112            this.directive = directive;
113        }
114
115        /**
116         * Returns the directive used in {@code [out:<directive>]} statement.
117         * @return the directive used in {@code [out:<directive>]} statement
118         */
119        public String getDirective() {
120            return directive;
121        }
122
123        /**
124         * Returns the {@code OverpassOutpoutFormat} matching the given directive.
125         * @param directive directive used in {@code [out:<directive>]} statement
126         * @return {@code OverpassOutpoutFormat} matching the given directive
127         * @throws IllegalArgumentException in case of invalid directive
128         */
129        static OverpassOutpoutFormat from(String directive) {
130            for (OverpassOutpoutFormat oof : values()) {
131                if (oof.directive.equals(directive)) {
132                    return oof;
133                }
134            }
135            throw new IllegalArgumentException(directive);
136        }
137    }
138
139    static final Pattern OUTPUT_FORMAT_STATEMENT = Pattern.compile(".*\\[out:([a-z]{3,})\\].*", Pattern.DOTALL);
140
141    static final Map<OverpassOutpoutFormat, Class<? extends AbstractReader>> outputFormatReaders = new ConcurrentHashMap<>();
142
143    final String overpassServer;
144    final String overpassQuery;
145
146    /**
147     * Constructs a new {@code OverpassDownloadReader}.
148     *
149     * @param downloadArea   The area to download
150     * @param overpassServer The Overpass server to use
151     * @param overpassQuery  The Overpass query
152     */
153    public OverpassDownloadReader(Bounds downloadArea, String overpassServer, String overpassQuery) {
154        super(downloadArea);
155        setDoAuthenticate(false);
156        this.overpassServer = overpassServer;
157        this.overpassQuery = overpassQuery.trim();
158    }
159
160    /**
161     * Registers an OSM reader for the given Overpass output format.
162     * @param format Overpass output format
163     * @param readerClass OSM reader class
164     * @return the previous value associated with {@code format}, or {@code null} if there was no mapping
165     */
166    public static final Class<? extends AbstractReader> registerOverpassOutpoutFormatReader(
167            OverpassOutpoutFormat format, Class<? extends AbstractReader> readerClass) {
168        return outputFormatReaders.put(Objects.requireNonNull(format), Objects.requireNonNull(readerClass));
169    }
170
171    static {
172        registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_XML, OverpassOsmReader.class);
173        registerOverpassOutpoutFormatReader(OverpassOutpoutFormat.OSM_JSON, OverpassOsmJsonReader.class);
174    }
175
176    @Override
177    protected String getBaseUrl() {
178        return overpassServer;
179    }
180
181    @Override
182    protected String getRequestForBbox(double lon1, double lat1, double lon2, double lat2) {
183        if (overpassQuery.isEmpty())
184            return super.getRequestForBbox(lon1, lat1, lon2, lat2);
185        else {
186            final String query = this.overpassQuery
187                    .replace("{{bbox}}", bbox(lon1, lat1, lon2, lat2))
188                    .replace("{{center}}", center(lon1, lat1, lon2, lat2));
189            final String expandedOverpassQuery = expandExtendedQueries(query);
190            return "interpreter" + DATA_PREFIX + Utils.encodeUrl(expandedOverpassQuery);
191        }
192    }
193
194    /**
195     * Evaluates some features of overpass turbo extended query syntax.
196     * See https://wiki.openstreetmap.org/wiki/Overpass_turbo/Extended_Overpass_Turbo_Queries
197     * @param query unexpanded query
198     * @return expanded query
199     */
200    static String expandExtendedQueries(String query) {
201        final StringBuffer sb = new StringBuffer();
202        final Matcher matcher = Pattern.compile("\\{\\{(date|geocodeArea|geocodeBbox|geocodeCoords|geocodeId):([^}]+)\\}\\}").matcher(query);
203        while (matcher.find()) {
204            try {
205                switch (matcher.group(1)) {
206                    case "date":
207                        matcher.appendReplacement(sb, date(matcher.group(2), LocalDateTime.now()));
208                        break;
209                    case "geocodeArea":
210                        matcher.appendReplacement(sb, geocodeArea(matcher.group(2)));
211                        break;
212                    case "geocodeBbox":
213                        matcher.appendReplacement(sb, geocodeBbox(matcher.group(2)));
214                        break;
215                    case "geocodeCoords":
216                        matcher.appendReplacement(sb, geocodeCoords(matcher.group(2)));
217                        break;
218                    case "geocodeId":
219                        matcher.appendReplacement(sb, geocodeId(matcher.group(2)));
220                        break;
221                    default:
222                        Logging.warn("Unsupported syntax: " + matcher.group(1));
223                }
224            } catch (UncheckedParseException | IOException | NoSuchElementException | IndexOutOfBoundsException ex) {
225                final String msg = tr("Failed to evaluate {0}", matcher.group());
226                Logging.log(Logging.LEVEL_WARN, msg, ex);
227                matcher.appendReplacement(sb, "// " + msg + "\n");
228            }
229        }
230        matcher.appendTail(sb);
231        return sb.toString();
232    }
233
234    static String bbox(double lon1, double lat1, double lon2, double lat2) {
235        return lat1 + "," + lon1 + "," + lat2 + "," + lon2;
236    }
237
238    static String center(double lon1, double lat1, double lon2, double lat2) {
239        LatLon c = new BBox(lon1, lat1, lon2, lat2).getCenter();
240        return c.lat()+ "," + c.lon();
241    }
242
243    static String date(String humanDuration, LocalDateTime from) {
244        // Convert to ISO 8601. Replace months by X temporarily to avoid conflict with minutes
245        String duration = humanDuration.toLowerCase(Locale.ENGLISH).replace(" ", "")
246                .replaceAll("years?", "Y").replaceAll("months?", "X").replaceAll("weeks?", "W")
247                .replaceAll("days?", "D").replaceAll("hours?", "H").replaceAll("minutes?", "M").replaceAll("seconds?", "S");
248        Matcher matcher = Pattern.compile(
249                "((?:[0-9]+Y)?(?:[0-9]+X)?(?:[0-9]+W)?)"+
250                "((?:[0-9]+D)?)" +
251                "((?:[0-9]+H)?(?:[0-9]+M)?(?:[0-9]+(?:[.,][0-9]{0,9})?S)?)?").matcher(duration);
252        boolean javaPer = false;
253        boolean javaDur = false;
254        if (matcher.matches()) {
255            javaPer = matcher.group(1) != null && !matcher.group(1).isEmpty();
256            javaDur = matcher.group(3) != null && !matcher.group(3).isEmpty();
257            duration = 'P' + matcher.group(1).replace('X', 'M') + matcher.group(2);
258            if (javaDur) {
259                duration += 'T' + matcher.group(3);
260            }
261        }
262
263        // Duration is now a full ISO 8601 duration string. Unfortunately Java does not allow to parse it entirely.
264        // We must split the "period" (years, months, weeks, days) from the "duration" (days, hours, minutes, seconds).
265        Period p = null;
266        Duration d = null;
267        int idx = duration.indexOf('T');
268        if (javaPer) {
269            p = Period.parse(javaDur ? duration.substring(0, idx) : duration);
270        }
271        if (javaDur) {
272            d = Duration.parse(javaPer ? 'P' + duration.substring(idx, duration.length()) : duration);
273        } else if (!javaPer) {
274            d = Duration.parse(duration);
275        }
276
277        // Now that period and duration are known, compute the correct date/time
278        LocalDateTime dt = from;
279        if (p != null) {
280            dt = dt.minus(p);
281        }
282        if (d != null) {
283            dt = dt.minus(d);
284        }
285
286        // Returns the date/time formatted in ISO 8601
287        return dt.toInstant(ZoneOffset.UTC).toString();
288    }
289
290    private static SearchResult searchName(String area) throws IOException {
291        return searchName(NameFinder.queryNominatim(area));
292    }
293
294    static SearchResult searchName(List<SearchResult> results) {
295        return results.stream().filter(
296                x -> OsmPrimitiveType.NODE != x.getOsmId().getType()).iterator().next();
297    }
298
299    static String geocodeArea(String area) throws IOException {
300        // Offsets defined in https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#By_element_id
301        final EnumMap<OsmPrimitiveType, Long> idOffset = new EnumMap<>(OsmPrimitiveType.class);
302        idOffset.put(OsmPrimitiveType.NODE, 0L);
303        idOffset.put(OsmPrimitiveType.WAY, 2_400_000_000L);
304        idOffset.put(OsmPrimitiveType.RELATION, 3_600_000_000L);
305        final PrimitiveId osmId = searchName(area).getOsmId();
306        Logging.debug("Area ''{0}'' resolved to {1}", area, osmId);
307        return String.format("area(%d)", osmId.getUniqueId() + idOffset.get(osmId.getType()));
308    }
309
310    static String geocodeBbox(String area) throws IOException {
311        Bounds bounds = searchName(area).getBounds();
312        return bounds.getMinLat() + "," + bounds.getMinLon() + "," + bounds.getMaxLat() + "," + bounds.getMaxLon();
313    }
314
315    static String geocodeCoords(String area) throws IOException {
316        SearchResult result = searchName(area);
317        return result.getLat() + "," + result.getLon();
318    }
319
320    static String geocodeId(String area) throws IOException {
321        PrimitiveId osmId = searchName(area).getOsmId();
322        return String.format("%s(%d)", osmId.getType().getAPIName(), osmId.getUniqueId());
323    }
324
325    @Override
326    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason,
327                                            boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
328        try {
329            int index = urlStr.indexOf(DATA_PREFIX);
330            // Make an HTTP POST request instead of a simple GET, allows more complex queries
331            return super.getInputStreamRaw(urlStr.substring(0, index),
332                    progressMonitor, reason, uncompressAccordingToContentDisposition,
333                    "POST", Utils.decodeUrl(urlStr.substring(index + DATA_PREFIX.length())).getBytes(StandardCharsets.UTF_8));
334        } catch (OsmApiException ex) {
335            final String errorIndicator = "Error</strong>: ";
336            if (ex.getMessage() != null && ex.getMessage().contains(errorIndicator)) {
337                final String errorPlusRest = ex.getMessage().split(errorIndicator)[1];
338                if (errorPlusRest != null) {
339                    ex.setErrorHeader(errorPlusRest.split("</")[0].replaceAll(".*::request_read_and_idx::", ""));
340                }
341            }
342            throw ex;
343        }
344    }
345
346    @Override
347    protected void adaptRequest(HttpClient request) {
348        // see https://wiki.openstreetmap.org/wiki/Overpass_API/Overpass_QL#timeout
349        final Matcher timeoutMatcher = Pattern.compile("\\[timeout:(\\d+)\\]").matcher(overpassQuery);
350        final int timeout;
351        if (timeoutMatcher.find()) {
352            timeout = (int) TimeUnit.SECONDS.toMillis(Integer.parseInt(timeoutMatcher.group(1)));
353        } else {
354            timeout = (int) TimeUnit.MINUTES.toMillis(3);
355        }
356        request.setConnectTimeout(timeout);
357        request.setReadTimeout(timeout);
358    }
359
360    @Override
361    protected String getTaskName() {
362        return tr("Contacting Server...");
363    }
364
365    @Override
366    protected DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
367        AbstractReader reader = null;
368        Matcher m = OUTPUT_FORMAT_STATEMENT.matcher(overpassQuery);
369        if (m.matches()) {
370            Class<? extends AbstractReader> readerClass = outputFormatReaders.get(OverpassOutpoutFormat.from(m.group(1)));
371            if (readerClass != null) {
372                try {
373                    reader = readerClass.getDeclaredConstructor().newInstance();
374                } catch (ReflectiveOperationException | IllegalArgumentException | SecurityException e) {
375                    Logging.error(e);
376                }
377            }
378        }
379        if (reader == null) {
380            reader = new OverpassOsmReader();
381        }
382        return reader.doParseDataSet(source, progressMonitor);
383    }
384
385    @Override
386    public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
387
388        DataSet ds = super.parseOsm(progressMonitor);
389        if (!considerAsFullDownload()) {
390            DataSet noBounds = new DataSet();
391            DataSetMerger dsm = new DataSetMerger(noBounds, ds);
392            dsm.merge(null, false);
393            return dsm.getTargetDataSet();
394        } else {
395            // add bounds if necessary (note that Overpass API does not return bounds in the response XML)
396            if (ds != null && ds.getDataSources().isEmpty() && overpassQuery.contains("{{bbox}}")) {
397                if (crosses180th) {
398                    Bounds bounds = new Bounds(lat1, lon1, lat2, 180.0);
399                    DataSource src = new DataSource(bounds, getBaseUrl());
400                    ds.addDataSource(src);
401
402                    bounds = new Bounds(lat1, -180.0, lat2, lon2);
403                    src = new DataSource(bounds, getBaseUrl());
404                    ds.addDataSource(src);
405                } else {
406                    Bounds bounds = new Bounds(lat1, lon1, lat2, lon2);
407                    DataSource src = new DataSource(bounds, getBaseUrl());
408                    ds.addDataSource(src);
409                }
410            }
411            return ds;
412        }
413    }
414
415    /**
416     * Fixes Overpass API query to make sure it will be accepted by JOSM.
417     * @param query Overpass query to check
418     * @return fixed query
419     * @since 13335
420     */
421    public static String fixQuery(String query) {
422        return query == null ? query : query
423                .replaceAll("out( body| skel| ids)?( id| qt)?;", "out meta$2;")
424                .replaceAll("(?s)\\[out:(csv)[^\\]]*\\]", "[out:xml]");
425    }
426
427    @Override
428    public boolean considerAsFullDownload() {
429        return overpassQuery.equals(OverpassDownloadSource.FULL_DOWNLOAD_QUERY);
430    }
431}