001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.actions.downloadtasks;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.IOException;
007import java.net.MalformedURLException;
008import java.net.URL;
009import java.util.ArrayList;
010import java.util.Arrays;
011import java.util.Collection;
012import java.util.Collections;
013import java.util.HashSet;
014import java.util.Objects;
015import java.util.Optional;
016import java.util.Set;
017import java.util.concurrent.Future;
018import java.util.regex.Matcher;
019import java.util.regex.Pattern;
020import java.util.stream.Stream;
021
022import org.openstreetmap.josm.data.Bounds;
023import org.openstreetmap.josm.data.DataSource;
024import org.openstreetmap.josm.data.ProjectionBounds;
025import org.openstreetmap.josm.data.ViewportData;
026import org.openstreetmap.josm.data.coor.LatLon;
027import org.openstreetmap.josm.data.osm.DataSet;
028import org.openstreetmap.josm.data.osm.OsmPrimitive;
029import org.openstreetmap.josm.data.osm.Relation;
030import org.openstreetmap.josm.data.osm.Way;
031import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor;
032import org.openstreetmap.josm.gui.MainApplication;
033import org.openstreetmap.josm.gui.MapFrame;
034import org.openstreetmap.josm.gui.PleaseWaitRunnable;
035import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask;
036import org.openstreetmap.josm.gui.layer.OsmDataLayer;
037import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
038import org.openstreetmap.josm.gui.progress.ProgressMonitor;
039import org.openstreetmap.josm.io.BoundingBoxDownloader;
040import org.openstreetmap.josm.io.Compression;
041import org.openstreetmap.josm.io.OsmServerLocationReader;
042import org.openstreetmap.josm.io.OsmServerReader;
043import org.openstreetmap.josm.io.OsmTransferCanceledException;
044import org.openstreetmap.josm.io.OsmTransferException;
045import org.openstreetmap.josm.io.OverpassDownloadReader;
046import org.openstreetmap.josm.io.UrlPatterns.OsmUrlPattern;
047import org.openstreetmap.josm.tools.Logging;
048import org.openstreetmap.josm.tools.Utils;
049import org.xml.sax.SAXException;
050
051/**
052 * Open the download dialog and download the data.
053 * Run in the worker thread.
054 */
055public class DownloadOsmTask extends AbstractDownloadTask<DataSet> {
056
057    protected Bounds currentBounds;
058    protected DownloadTask downloadTask;
059
060    protected String newLayerName;
061
062    /** This allows subclasses to ignore this warning */
063    protected boolean warnAboutEmptyArea = true;
064
065    protected static final String OVERPASS_INTERPRETER_DATA = "interpreter?data=";
066
067    private static final String NO_DATA_FOUND = tr("No data found in this area.");
068    static {
069        PostDownloadHandler.addNoDataErrorMessage(NO_DATA_FOUND);
070    }
071
072    @Override
073    public String[] getPatterns() {
074        if (this.getClass() == DownloadOsmTask.class) {
075            return patterns(OsmUrlPattern.class);
076        } else {
077            return super.getPatterns();
078        }
079    }
080
081    @Override
082    public String getTitle() {
083        if (this.getClass() == DownloadOsmTask.class) {
084            return tr("Download OSM");
085        } else {
086            return super.getTitle();
087        }
088    }
089
090    @Override
091    public Future<?> download(DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) {
092        return download(new BoundingBoxDownloader(downloadArea), settings, downloadArea, progressMonitor);
093    }
094
095    /**
096     * Asynchronously launches the download task for a given bounding box.
097     *
098     * Set <code>progressMonitor</code> to null, if the task should create, open, and close a progress monitor.
099     * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to
100     * be discarded.
101     *
102     * You can wait for the asynchronous download task to finish by synchronizing on the returned
103     * {@link Future}, but make sure not to freeze up JOSM. Example:
104     * <pre>
105     *    Future&lt;?&gt; future = task.download(...);
106     *    // DON'T run this on the Swing EDT or JOSM will freeze
107     *    future.get(); // waits for the dowload task to complete
108     * </pre>
109     *
110     * The following example uses a pattern which is better suited if a task is launched from
111     * the Swing EDT:
112     * <pre>
113     *    final Future&lt;?&gt; future = task.download(...);
114     *    Runnable runAfterTask = new Runnable() {
115     *       public void run() {
116     *           // this is not strictly necessary because of the type of executor service
117     *           // Main.worker is initialized with, but it doesn't harm either
118     *           //
119     *           future.get(); // wait for the download task to complete
120     *           doSomethingAfterTheTaskCompleted();
121     *       }
122     *    }
123     *    MainApplication.worker.submit(runAfterTask);
124     * </pre>
125     * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm})
126     * @param settings download settings
127     * @param downloadArea the area to download
128     * @param progressMonitor the progressMonitor
129     * @return the future representing the asynchronous task
130     * @since 13927
131     */
132    public Future<?> download(OsmServerReader reader, DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) {
133        return download(new DownloadTask(settings, reader, progressMonitor, zoomAfterDownload), downloadArea);
134    }
135
136    protected Future<?> download(DownloadTask downloadTask, Bounds downloadArea) {
137        this.downloadTask = downloadTask;
138        this.currentBounds = new Bounds(downloadArea);
139        // We need submit instead of execute so we can wait for it to finish and get the error
140        // message if necessary. If no one calls getErrorMessage() it just behaves like execute.
141        return MainApplication.worker.submit(downloadTask);
142    }
143
144    /**
145     * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed.
146     * @param url the original URL
147     * @return the modified URL
148     */
149    protected String modifyUrlBeforeLoad(String url) {
150        return url;
151    }
152
153    /**
154     * Loads a given URL from the OSM Server
155     * @param settings download settings
156     * @param url The URL as String
157     */
158    @Override
159    public Future<?> loadUrl(DownloadParams settings, String url, ProgressMonitor progressMonitor) {
160        String newUrl = modifyUrlBeforeLoad(url);
161        Optional<OsmUrlPattern> urlPattern = Arrays.stream(OsmUrlPattern.values()).filter(p -> p.matches(newUrl)).findFirst();
162        downloadTask = new DownloadTask(settings, getOsmServerReader(newUrl), progressMonitor, true, Compression.byExtension(newUrl));
163        currentBounds = null;
164        // Extract .osm filename from URL to set the new layer name
165        extractOsmFilename(settings, urlPattern.orElse(OsmUrlPattern.EXTERNAL_OSM_FILE).pattern(), newUrl);
166        return MainApplication.worker.submit(downloadTask);
167    }
168
169    protected OsmServerReader getOsmServerReader(String url) {
170        try {
171            String host = new URL(url).getHost();
172            for (String knownOverpassServer : OverpassDownloadReader.OVERPASS_SERVER_HISTORY.get()) {
173                if (host.equals(new URL(knownOverpassServer).getHost())) {
174                    int index = url.indexOf(OVERPASS_INTERPRETER_DATA);
175                    if (index > 0) {
176                        return new OverpassDownloadReader(new Bounds(LatLon.ZERO), knownOverpassServer,
177                                Utils.decodeUrl(url.substring(index + OVERPASS_INTERPRETER_DATA.length())));
178                    }
179                }
180            }
181        } catch (MalformedURLException e) {
182            Logging.error(e);
183        }
184        return new OsmServerLocationReader(url);
185    }
186
187    protected final void extractOsmFilename(DownloadParams settings, String pattern, String url) {
188        newLayerName = settings.getLayerName();
189        if (newLayerName == null || newLayerName.isEmpty()) {
190            Matcher matcher = Pattern.compile(pattern).matcher(url);
191            newLayerName = matcher.matches() && matcher.groupCount() > 0 ? matcher.group(1) : null;
192        }
193    }
194
195    @Override
196    public void cancel() {
197        if (downloadTask != null) {
198            downloadTask.cancel();
199        }
200    }
201
202    @Override
203    public boolean isSafeForRemotecontrolRequests() {
204        return true;
205    }
206
207    @Override
208    public ProjectionBounds getDownloadProjectionBounds() {
209        return downloadTask != null ? downloadTask.computeBbox(currentBounds).orElse(null) : null;
210    }
211
212    protected Collection<OsmPrimitive> searchPotentiallyDeletedPrimitives(DataSet ds) {
213        return downloadTask.searchPrimitivesToUpdate(currentBounds, ds);
214    }
215
216    /**
217     * Superclass of internal download task.
218     * @since 7636
219     */
220    public abstract static class AbstractInternalTask extends PleaseWaitRunnable {
221
222        protected final DownloadParams settings;
223        protected final boolean zoomAfterDownload;
224        protected DataSet dataSet;
225
226        /**
227         * Constructs a new {@code AbstractInternalTask}.
228         * @param settings download settings
229         * @param title message for the user
230         * @param ignoreException If true, exception will be propagated to calling code. If false then
231         * exception will be thrown directly in EDT. When this runnable is executed using executor framework
232         * then use false unless you read result of task (because exception will get lost if you don't)
233         * @param zoomAfterDownload If true, the map view will zoom to download area after download
234         */
235        public AbstractInternalTask(DownloadParams settings, String title, boolean ignoreException, boolean zoomAfterDownload) {
236            super(title, ignoreException);
237            this.settings = Objects.requireNonNull(settings);
238            this.zoomAfterDownload = zoomAfterDownload;
239        }
240
241        /**
242         * Constructs a new {@code AbstractInternalTask}.
243         * @param settings download settings
244         * @param title message for the user
245         * @param progressMonitor progress monitor
246         * @param ignoreException If true, exception will be propagated to calling code. If false then
247         * exception will be thrown directly in EDT. When this runnable is executed using executor framework
248         * then use false unless you read result of task (because exception will get lost if you don't)
249         * @param zoomAfterDownload If true, the map view will zoom to download area after download
250         */
251        public AbstractInternalTask(DownloadParams settings, String title, ProgressMonitor progressMonitor, boolean ignoreException,
252                boolean zoomAfterDownload) {
253            super(title, progressMonitor, ignoreException);
254            this.settings = Objects.requireNonNull(settings);
255            this.zoomAfterDownload = zoomAfterDownload;
256        }
257
258        protected OsmDataLayer getEditLayer() {
259            return MainApplication.getLayerManager().getEditLayer();
260        }
261
262        private static Stream<OsmDataLayer> getModifiableDataLayers() {
263            return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class)
264                    .stream().filter(OsmDataLayer::isDownloadable);
265        }
266
267        /**
268         * Returns the number of modifiable data layers
269         * @return number of modifiable data layers
270         * @since 13434
271         */
272        protected long getNumModifiableDataLayers() {
273            return getModifiableDataLayers().count();
274        }
275
276        /**
277         * Returns the first modifiable data layer
278         * @return the first modifiable data layer
279         * @since 13434
280         */
281        protected OsmDataLayer getFirstModifiableDataLayer() {
282            return getModifiableDataLayers().findFirst().orElse(null);
283        }
284
285        /**
286         * Creates a name for a new layer by utilizing the settings ({@link DownloadParams#getLayerName()}) or
287         * {@link OsmDataLayer#createNewName()} if the former option is {@code null}.
288         *
289         * @return a name for a new layer
290         * @since 14347
291         */
292        protected String generateLayerName() {
293            return Optional.ofNullable(settings.getLayerName())
294                .filter(layerName -> !Utils.isStripEmpty(layerName))
295                .orElse(OsmDataLayer.createNewName());
296        }
297
298        /**
299         * Can be overridden (e.g. by plugins) if a subclass of {@link OsmDataLayer} is needed.
300         * If you want to change how the name is determined, consider overriding
301         * {@link #generateLayerName()} instead.
302         *
303         * @param ds the dataset on which the layer is based, must be non-null
304         * @param layerName the name of the new layer, must be either non-blank or non-present
305         * @return a new instance of {@link OsmDataLayer} constructed with the given arguments
306         * @since 14347
307         */
308        protected OsmDataLayer createNewLayer(final DataSet ds, final Optional<String> layerName) {
309            if (layerName.filter(Utils::isStripEmpty).isPresent()) {
310                throw new IllegalArgumentException("Blank layer name!");
311            }
312            return new OsmDataLayer(
313                Objects.requireNonNull(ds, "dataset parameter"),
314                layerName.orElseGet(this::generateLayerName),
315                null
316            );
317        }
318
319        /**
320         * Convenience method for {@link #createNewLayer(DataSet, Optional)}, uses the dataset
321         * from field {@link #dataSet} and applies the settings from field {@link #settings}.
322         *
323         * @param layerName an optional layer name, must be non-blank if the [Optional] is present
324         * @return a newly constructed layer
325         * @since 14347
326         */
327        protected final OsmDataLayer createNewLayer(final Optional<String> layerName) {
328            Optional.ofNullable(settings.getDownloadPolicy())
329                .ifPresent(dataSet::setDownloadPolicy);
330            Optional.ofNullable(settings.getUploadPolicy())
331                .ifPresent(dataSet::setUploadPolicy);
332            if (dataSet.isLocked() && !settings.isLocked()) {
333                dataSet.unlock();
334            } else if (!dataSet.isLocked() && settings.isLocked()) {
335                dataSet.lock();
336            }
337            return createNewLayer(dataSet, layerName);
338        }
339
340        protected Optional<ProjectionBounds> computeBbox(Bounds bounds) {
341            BoundingXYVisitor v = new BoundingXYVisitor();
342            if (bounds != null) {
343                v.visit(bounds);
344            } else {
345                v.computeBoundingBox(dataSet.getNodes());
346            }
347            return Optional.ofNullable(v.getBounds());
348        }
349
350        protected OsmDataLayer addNewLayerIfRequired(String newLayerName) {
351            long numDataLayers = getNumModifiableDataLayers();
352            if (settings.isNewLayer() || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) {
353                // the user explicitly wants a new layer, we don't have any layer at all
354                // or it is not clear which layer to merge to
355                final OsmDataLayer layer = createNewLayer(Optional.ofNullable(newLayerName).filter(it -> !Utils.isStripEmpty(it)));
356                MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload);
357                return layer;
358            }
359            return null;
360        }
361
362        protected void loadData(String newLayerName, Bounds bounds) {
363            OsmDataLayer layer = addNewLayerIfRequired(newLayerName);
364            if (layer == null) {
365                layer = getEditLayer();
366                if (layer == null || !layer.isDownloadable()) {
367                    layer = getFirstModifiableDataLayer();
368                }
369                Collection<OsmPrimitive> primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.getDataSet());
370                layer.mergeFrom(dataSet);
371                MapFrame map = MainApplication.getMap();
372                if (map != null && zoomAfterDownload) {
373                    computeBbox(bounds).map(ViewportData::new).ifPresent(map.mapView::zoomTo);
374                }
375                if (!primitivesToUpdate.isEmpty()) {
376                    MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate));
377                }
378                layer.onPostDownloadFromServer();
379            }
380        }
381
382        /**
383         * Look for primitives deleted on server (thus absent from downloaded data)
384         * but still present in existing data layer
385         * @param bounds download bounds
386         * @param ds existing data set
387         * @return the primitives to update
388         */
389        protected Collection<OsmPrimitive> searchPrimitivesToUpdate(Bounds bounds, DataSet ds) {
390            if (bounds == null)
391                return Collections.emptySet();
392            Collection<OsmPrimitive> col = new ArrayList<>();
393            ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add);
394            if (!col.isEmpty()) {
395                Set<Way> ways = new HashSet<>();
396                Set<Relation> rels = new HashSet<>();
397                for (OsmPrimitive n : col) {
398                    for (OsmPrimitive ref : n.getReferrers()) {
399                        if (ref.isNew()) {
400                            continue;
401                        } else if (ref instanceof Way) {
402                            ways.add((Way) ref);
403                        } else if (ref instanceof Relation) {
404                            rels.add((Relation) ref);
405                        }
406                    }
407                }
408                ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add);
409                rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add);
410            }
411            return col;
412        }
413    }
414
415    protected class DownloadTask extends AbstractInternalTask {
416        protected final OsmServerReader reader;
417        protected final Compression compression;
418
419        /**
420         * Constructs a new {@code DownloadTask}.
421         * @param settings download settings
422         * @param reader OSM data reader
423         * @param progressMonitor progress monitor
424         * @since 13927
425         */
426        public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor) {
427            this(settings, reader, progressMonitor, true);
428        }
429
430        /**
431         * Constructs a new {@code DownloadTask}.
432         * @param settings download settings
433         * @param reader OSM data reader
434         * @param progressMonitor progress monitor
435         * @param zoomAfterDownload If true, the map view will zoom to download area after download
436         * @since 13927
437         */
438        public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) {
439            this(settings, reader, progressMonitor, zoomAfterDownload, Compression.NONE);
440        }
441
442        /**
443         * Constructs a new {@code DownloadTask}.
444         * @param settings download settings
445         * @param reader OSM data reader
446         * @param progressMonitor progress monitor
447         * @param zoomAfterDownload If true, the map view will zoom to download area after download
448         * @param compression compression to use
449         * @since 15784
450         */
451        public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload,
452                Compression compression) {
453            super(settings, tr("Downloading data"), progressMonitor, false, zoomAfterDownload);
454            this.reader = Objects.requireNonNull(reader);
455            this.compression = compression;
456        }
457
458        protected DataSet parseDataSet() throws OsmTransferException {
459            ProgressMonitor subTaskMonitor = progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false);
460            // Don't call parseOsm signature with compression if not needed, too many implementations to update before to avoid side effects
461            return compression != null && compression != Compression.NONE ?
462                    reader.parseOsm(subTaskMonitor, compression) : reader.parseOsm(subTaskMonitor);
463        }
464
465        @Override
466        public void realRun() throws IOException, SAXException, OsmTransferException {
467            try {
468                if (isCanceled())
469                    return;
470                dataSet = parseDataSet();
471            } catch (OsmTransferException e) {
472                if (isCanceled()) {
473                    Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString()));
474                    return;
475                }
476                if (e instanceof OsmTransferCanceledException) {
477                    setCanceled(true);
478                    return;
479                } else {
480                    rememberException(e);
481                }
482                DownloadOsmTask.this.setFailed(true);
483            }
484        }
485
486        @Override
487        protected void finish() {
488            if (isFailed() || isCanceled())
489                return;
490            if (dataSet == null)
491                return; // user canceled download or error occurred
492            if (dataSet.allPrimitives().isEmpty()) {
493                if (warnAboutEmptyArea) {
494                    rememberErrorMessage(NO_DATA_FOUND);
495                }
496                String remark = dataSet.getRemark();
497                if (remark != null && !remark.isEmpty()) {
498                    rememberErrorMessage(remark);
499                }
500                if (!(reader instanceof BoundingBoxDownloader)
501                        || ((BoundingBoxDownloader) reader).considerAsFullDownload()) {
502                    // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work
503                    dataSet.addDataSource(new DataSource(
504                            currentBounds != null ? currentBounds : new Bounds(LatLon.ZERO), "OpenStreetMap server"));
505                }
506            }
507
508            rememberDownloadedData(dataSet);
509            loadData(newLayerName, currentBounds);
510        }
511
512        @Override
513        protected void cancel() {
514            setCanceled(true);
515            if (reader != null) {
516                reader.cancel();
517            }
518        }
519    }
520
521    @Override
522    public String getConfirmationMessage(URL url) {
523        if (OsmUrlPattern.OSM_API_URL.matches(url)) {
524            Collection<String> items = new ArrayList<>();
525            items.add(tr("OSM Server URL:") + ' ' + url.getHost());
526            items.add(tr("Command")+": "+url.getPath());
527            if (url.getQuery() != null) {
528                items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", ")));
529            }
530            return Utils.joinAsHtmlUnorderedList(items);
531        }
532        return null;
533    }
534}