001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.actions.downloadtasks;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.IOException;
007import java.net.MalformedURLException;
008import java.net.URL;
009import java.util.ArrayList;
010import java.util.Arrays;
011import java.util.Collection;
012import java.util.Collections;
013import java.util.HashSet;
014import java.util.Objects;
015import java.util.Optional;
016import java.util.Set;
017import java.util.concurrent.Future;
018import java.util.regex.Matcher;
019import java.util.regex.Pattern;
020import java.util.stream.Stream;
021
022import org.openstreetmap.josm.data.Bounds;
023import org.openstreetmap.josm.data.DataSource;
024import org.openstreetmap.josm.data.ProjectionBounds;
025import org.openstreetmap.josm.data.ViewportData;
026import org.openstreetmap.josm.data.coor.LatLon;
027import org.openstreetmap.josm.data.osm.DataSet;
028import org.openstreetmap.josm.data.osm.OsmPrimitive;
029import org.openstreetmap.josm.data.osm.Relation;
030import org.openstreetmap.josm.data.osm.Way;
031import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor;
032import org.openstreetmap.josm.gui.MainApplication;
033import org.openstreetmap.josm.gui.MapFrame;
034import org.openstreetmap.josm.gui.PleaseWaitRunnable;
035import org.openstreetmap.josm.gui.io.UpdatePrimitivesTask;
036import org.openstreetmap.josm.gui.layer.OsmDataLayer;
037import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
038import org.openstreetmap.josm.gui.progress.ProgressMonitor;
039import org.openstreetmap.josm.io.BoundingBoxDownloader;
040import org.openstreetmap.josm.io.OsmServerLocationReader;
041import org.openstreetmap.josm.io.OsmServerLocationReader.OsmUrlPattern;
042import org.openstreetmap.josm.io.OsmServerReader;
043import org.openstreetmap.josm.io.OsmTransferCanceledException;
044import org.openstreetmap.josm.io.OsmTransferException;
045import org.openstreetmap.josm.io.OverpassDownloadReader;
046import org.openstreetmap.josm.tools.Logging;
047import org.openstreetmap.josm.tools.Utils;
048import org.xml.sax.SAXException;
049
050/**
051 * Open the download dialog and download the data.
052 * Run in the worker thread.
053 */
054public class DownloadOsmTask extends AbstractDownloadTask<DataSet> {
055
056    protected Bounds currentBounds;
057    protected DownloadTask downloadTask;
058
059    protected String newLayerName;
060
061    /** This allows subclasses to ignore this warning */
062    protected boolean warnAboutEmptyArea = true;
063
064    protected static final String OVERPASS_INTERPRETER_DATA = "interpreter?data=";
065
066    private static final String NO_DATA_FOUND = tr("No data found in this area.");
067    static {
068        PostDownloadHandler.addNoDataErrorMessage(NO_DATA_FOUND);
069    }
070
071    @Override
072    public String[] getPatterns() {
073        if (this.getClass() == DownloadOsmTask.class) {
074            return Arrays.stream(OsmUrlPattern.values()).map(OsmUrlPattern::pattern).toArray(String[]::new);
075        } else {
076            return super.getPatterns();
077        }
078    }
079
080    @Override
081    public String getTitle() {
082        if (this.getClass() == DownloadOsmTask.class) {
083            return tr("Download OSM");
084        } else {
085            return super.getTitle();
086        }
087    }
088
089    @Override
090    public Future<?> download(DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) {
091        return download(new BoundingBoxDownloader(downloadArea), settings, downloadArea, progressMonitor);
092    }
093
094    /**
095     * Asynchronously launches the download task for a given bounding box.
096     *
097     * Set <code>progressMonitor</code> to null, if the task should create, open, and close a progress monitor.
098     * Set progressMonitor to {@link NullProgressMonitor#INSTANCE} if progress information is to
099     * be discarded.
100     *
101     * You can wait for the asynchronous download task to finish by synchronizing on the returned
102     * {@link Future}, but make sure not to freeze up JOSM. Example:
103     * <pre>
104     *    Future&lt;?&gt; future = task.download(...);
105     *    // DON'T run this on the Swing EDT or JOSM will freeze
106     *    future.get(); // waits for the dowload task to complete
107     * </pre>
108     *
109     * The following example uses a pattern which is better suited if a task is launched from
110     * the Swing EDT:
111     * <pre>
112     *    final Future&lt;?&gt; future = task.download(...);
113     *    Runnable runAfterTask = new Runnable() {
114     *       public void run() {
115     *           // this is not strictly necessary because of the type of executor service
116     *           // Main.worker is initialized with, but it doesn't harm either
117     *           //
118     *           future.get(); // wait for the download task to complete
119     *           doSomethingAfterTheTaskCompleted();
120     *       }
121     *    }
122     *    MainApplication.worker.submit(runAfterTask);
123     * </pre>
124     * @param reader the reader used to parse OSM data (see {@link OsmServerReader#parseOsm})
125     * @param settings download settings
126     * @param downloadArea the area to download
127     * @param progressMonitor the progressMonitor
128     * @return the future representing the asynchronous task
129     * @since 13927
130     */
131    public Future<?> download(OsmServerReader reader, DownloadParams settings, Bounds downloadArea, ProgressMonitor progressMonitor) {
132        return download(new DownloadTask(settings, reader, progressMonitor, zoomAfterDownload), downloadArea);
133    }
134
135    protected Future<?> download(DownloadTask downloadTask, Bounds downloadArea) {
136        this.downloadTask = downloadTask;
137        this.currentBounds = new Bounds(downloadArea);
138        // We need submit instead of execute so we can wait for it to finish and get the error
139        // message if necessary. If no one calls getErrorMessage() it just behaves like execute.
140        return MainApplication.worker.submit(downloadTask);
141    }
142
143    /**
144     * This allows subclasses to perform operations on the URL before {@link #loadUrl} is performed.
145     * @param url the original URL
146     * @return the modified URL
147     */
148    protected String modifyUrlBeforeLoad(String url) {
149        return url;
150    }
151
152    /**
153     * Loads a given URL from the OSM Server
154     * @param settings download settings
155     * @param url The URL as String
156     */
157    @Override
158    public Future<?> loadUrl(DownloadParams settings, String url, ProgressMonitor progressMonitor) {
159        String newUrl = modifyUrlBeforeLoad(url);
160        downloadTask = new DownloadTask(settings, getOsmServerReader(newUrl), progressMonitor);
161        currentBounds = null;
162        // Extract .osm filename from URL to set the new layer name
163        extractOsmFilename(settings, "https?://.*/(.*\\.osm)", newUrl);
164        return MainApplication.worker.submit(downloadTask);
165    }
166
167    protected OsmServerReader getOsmServerReader(String url) {
168        try {
169            String host = new URL(url).getHost();
170            for (String knownOverpassServer : OverpassDownloadReader.OVERPASS_SERVER_HISTORY.get()) {
171                if (host.equals(new URL(knownOverpassServer).getHost())) {
172                    int index = url.indexOf(OVERPASS_INTERPRETER_DATA);
173                    if (index > 0) {
174                        return new OverpassDownloadReader(new Bounds(LatLon.ZERO), knownOverpassServer,
175                                Utils.decodeUrl(url.substring(index + OVERPASS_INTERPRETER_DATA.length())));
176                    }
177                }
178            }
179        } catch (MalformedURLException e) {
180            Logging.error(e);
181        }
182        return new OsmServerLocationReader(url);
183    }
184
185    protected final void extractOsmFilename(DownloadParams settings, String pattern, String url) {
186        newLayerName = settings.getLayerName();
187        if (newLayerName == null || newLayerName.isEmpty()) {
188            Matcher matcher = Pattern.compile(pattern).matcher(url);
189            newLayerName = matcher.matches() ? matcher.group(1) : null;
190        }
191    }
192
193    @Override
194    public void cancel() {
195        if (downloadTask != null) {
196            downloadTask.cancel();
197        }
198    }
199
200    @Override
201    public boolean isSafeForRemotecontrolRequests() {
202        return true;
203    }
204
205    @Override
206    public ProjectionBounds getDownloadProjectionBounds() {
207        return downloadTask != null ? downloadTask.computeBbox(currentBounds).orElse(null) : null;
208    }
209
210    protected Collection<OsmPrimitive> searchPotentiallyDeletedPrimitives(DataSet ds) {
211        return downloadTask.searchPrimitivesToUpdate(currentBounds, ds);
212    }
213
214    /**
215     * Superclass of internal download task.
216     * @since 7636
217     */
218    public abstract static class AbstractInternalTask extends PleaseWaitRunnable {
219
220        protected final DownloadParams settings;
221        protected final boolean zoomAfterDownload;
222        protected DataSet dataSet;
223
224        /**
225         * Constructs a new {@code AbstractInternalTask}.
226         * @param settings download settings
227         * @param title message for the user
228         * @param ignoreException If true, exception will be propagated to calling code. If false then
229         * exception will be thrown directly in EDT. When this runnable is executed using executor framework
230         * then use false unless you read result of task (because exception will get lost if you don't)
231         * @param zoomAfterDownload If true, the map view will zoom to download area after download
232         */
233        public AbstractInternalTask(DownloadParams settings, String title, boolean ignoreException, boolean zoomAfterDownload) {
234            super(title, ignoreException);
235            this.settings = Objects.requireNonNull(settings);
236            this.zoomAfterDownload = zoomAfterDownload;
237        }
238
239        /**
240         * Constructs a new {@code AbstractInternalTask}.
241         * @param settings download settings
242         * @param title message for the user
243         * @param progressMonitor progress monitor
244         * @param ignoreException If true, exception will be propagated to calling code. If false then
245         * exception will be thrown directly in EDT. When this runnable is executed using executor framework
246         * then use false unless you read result of task (because exception will get lost if you don't)
247         * @param zoomAfterDownload If true, the map view will zoom to download area after download
248         */
249        public AbstractInternalTask(DownloadParams settings, String title, ProgressMonitor progressMonitor, boolean ignoreException,
250                boolean zoomAfterDownload) {
251            super(title, progressMonitor, ignoreException);
252            this.settings = Objects.requireNonNull(settings);
253            this.zoomAfterDownload = zoomAfterDownload;
254        }
255
256        protected OsmDataLayer getEditLayer() {
257            return MainApplication.getLayerManager().getEditLayer();
258        }
259
260        private static Stream<OsmDataLayer> getModifiableDataLayers() {
261            return MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class)
262                    .stream().filter(OsmDataLayer::isDownloadable);
263        }
264
265        /**
266         * Returns the number of modifiable data layers
267         * @return number of modifiable data layers
268         * @since 13434
269         */
270        protected long getNumModifiableDataLayers() {
271            return getModifiableDataLayers().count();
272        }
273
274        /**
275         * Returns the first modifiable data layer
276         * @return the first modifiable data layer
277         * @since 13434
278         */
279        protected OsmDataLayer getFirstModifiableDataLayer() {
280            return getModifiableDataLayers().findFirst().orElse(null);
281        }
282
283        /**
284         * Creates a name for a new layer by utilizing the settings ({@link DownloadParams#getLayerName()}) or
285         * {@link OsmDataLayer#createNewName()} if the former option is {@code null}.
286         *
287         * @return a name for a new layer
288         * @since 14347
289         */
290        protected String generateLayerName() {
291            return Optional.ofNullable(settings.getLayerName())
292                .filter(layerName -> !Utils.isStripEmpty(layerName))
293                .orElse(OsmDataLayer.createNewName());
294        }
295
296        /**
297         * Can be overridden (e.g. by plugins) if a subclass of {@link OsmDataLayer} is needed.
298         * If you want to change how the name is determined, consider overriding
299         * {@link #generateLayerName()} instead.
300         *
301         * @param ds the dataset on which the layer is based, must be non-null
302         * @param layerName the name of the new layer, must be either non-blank or non-present
303         * @return a new instance of {@link OsmDataLayer} constructed with the given arguments
304         * @since 14347
305         */
306        protected OsmDataLayer createNewLayer(final DataSet ds, final Optional<String> layerName) {
307            if (layerName.filter(Utils::isStripEmpty).isPresent()) {
308                throw new IllegalArgumentException("Blank layer name!");
309            }
310            return new OsmDataLayer(
311                Objects.requireNonNull(ds, "dataset parameter"),
312                layerName.orElseGet(this::generateLayerName),
313                null
314            );
315        }
316
317        /**
318         * Convenience method for {@link #createNewLayer(DataSet, Optional)}, uses the dataset
319         * from field {@link #dataSet} and applies the settings from field {@link #settings}.
320         *
321         * @param layerName an optional layer name, must be non-blank if the [Optional] is present
322         * @return a newly constructed layer
323         * @since 14347
324         */
325        protected final OsmDataLayer createNewLayer(final Optional<String> layerName) {
326            Optional.ofNullable(settings.getDownloadPolicy())
327                .ifPresent(dataSet::setDownloadPolicy);
328            Optional.ofNullable(settings.getUploadPolicy())
329                .ifPresent(dataSet::setUploadPolicy);
330            if (dataSet.isLocked() && !settings.isLocked()) {
331                dataSet.unlock();
332            } else if (!dataSet.isLocked() && settings.isLocked()) {
333                dataSet.lock();
334            }
335            return createNewLayer(dataSet, layerName);
336        }
337
338        protected Optional<ProjectionBounds> computeBbox(Bounds bounds) {
339            BoundingXYVisitor v = new BoundingXYVisitor();
340            if (bounds != null) {
341                v.visit(bounds);
342            } else {
343                v.computeBoundingBox(dataSet.getNodes());
344            }
345            return Optional.ofNullable(v.getBounds());
346        }
347
348        protected OsmDataLayer addNewLayerIfRequired(String newLayerName) {
349            long numDataLayers = getNumModifiableDataLayers();
350            if (settings.isNewLayer() || numDataLayers == 0 || (numDataLayers > 1 && getEditLayer() == null)) {
351                // the user explicitly wants a new layer, we don't have any layer at all
352                // or it is not clear which layer to merge to
353                final OsmDataLayer layer = createNewLayer(Optional.ofNullable(newLayerName).filter(it -> !Utils.isStripEmpty(it)));
354                MainApplication.getLayerManager().addLayer(layer, zoomAfterDownload);
355                return layer;
356            }
357            return null;
358        }
359
360        protected void loadData(String newLayerName, Bounds bounds) {
361            OsmDataLayer layer = addNewLayerIfRequired(newLayerName);
362            if (layer == null) {
363                layer = getEditLayer();
364                if (layer == null || !layer.isDownloadable()) {
365                    layer = getFirstModifiableDataLayer();
366                }
367                Collection<OsmPrimitive> primitivesToUpdate = searchPrimitivesToUpdate(bounds, layer.getDataSet());
368                layer.mergeFrom(dataSet);
369                MapFrame map = MainApplication.getMap();
370                if (map != null && zoomAfterDownload) {
371                    computeBbox(bounds).map(ViewportData::new).ifPresent(map.mapView::zoomTo);
372                }
373                if (!primitivesToUpdate.isEmpty()) {
374                    MainApplication.worker.submit(new UpdatePrimitivesTask(layer, primitivesToUpdate));
375                }
376                layer.onPostDownloadFromServer();
377            }
378        }
379
380        /**
381         * Look for primitives deleted on server (thus absent from downloaded data)
382         * but still present in existing data layer
383         * @param bounds download bounds
384         * @param ds existing data set
385         * @return the primitives to update
386         */
387        protected Collection<OsmPrimitive> searchPrimitivesToUpdate(Bounds bounds, DataSet ds) {
388            if (bounds == null)
389                return Collections.emptySet();
390            Collection<OsmPrimitive> col = new ArrayList<>();
391            ds.searchNodes(bounds.toBBox()).stream().filter(n -> !n.isNew() && !dataSet.containsNode(n)).forEachOrdered(col::add);
392            if (!col.isEmpty()) {
393                Set<Way> ways = new HashSet<>();
394                Set<Relation> rels = new HashSet<>();
395                for (OsmPrimitive n : col) {
396                    for (OsmPrimitive ref : n.getReferrers()) {
397                        if (ref.isNew()) {
398                            continue;
399                        } else if (ref instanceof Way) {
400                            ways.add((Way) ref);
401                        } else if (ref instanceof Relation) {
402                            rels.add((Relation) ref);
403                        }
404                    }
405                }
406                ways.stream().filter(w -> !dataSet.containsWay(w)).forEachOrdered(col::add);
407                rels.stream().filter(r -> !dataSet.containsRelation(r)).forEachOrdered(col::add);
408            }
409            return col;
410        }
411    }
412
413    protected class DownloadTask extends AbstractInternalTask {
414        protected final OsmServerReader reader;
415
416        /**
417         * Constructs a new {@code DownloadTask}.
418         * @param settings download settings
419         * @param reader OSM data reader
420         * @param progressMonitor progress monitor
421         * @since 13927
422         */
423        public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor) {
424            this(settings, reader, progressMonitor, true);
425        }
426
427        /**
428         * Constructs a new {@code DownloadTask}.
429         * @param settings download settings
430         * @param reader OSM data reader
431         * @param progressMonitor progress monitor
432         * @param zoomAfterDownload If true, the map view will zoom to download area after download
433         * @since 13927
434         */
435        public DownloadTask(DownloadParams settings, OsmServerReader reader, ProgressMonitor progressMonitor, boolean zoomAfterDownload) {
436            super(settings, tr("Downloading data"), progressMonitor, false, zoomAfterDownload);
437            this.reader = reader;
438        }
439
440        protected DataSet parseDataSet() throws OsmTransferException {
441            return reader.parseOsm(progressMonitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false));
442        }
443
444        @Override
445        public void realRun() throws IOException, SAXException, OsmTransferException {
446            try {
447                if (isCanceled())
448                    return;
449                dataSet = parseDataSet();
450            } catch (OsmTransferException e) {
451                if (isCanceled()) {
452                    Logging.info(tr("Ignoring exception because download has been canceled. Exception was: {0}", e.toString()));
453                    return;
454                }
455                if (e instanceof OsmTransferCanceledException) {
456                    setCanceled(true);
457                    return;
458                } else {
459                    rememberException(e);
460                }
461                DownloadOsmTask.this.setFailed(true);
462            }
463        }
464
465        @Override
466        protected void finish() {
467            if (isFailed() || isCanceled())
468                return;
469            if (dataSet == null)
470                return; // user canceled download or error occurred
471            if (dataSet.allPrimitives().isEmpty()) {
472                if (warnAboutEmptyArea) {
473                    rememberErrorMessage(NO_DATA_FOUND);
474                }
475                String remark = dataSet.getRemark();
476                if (remark != null && !remark.isEmpty()) {
477                    rememberErrorMessage(remark);
478                }
479                if (!(reader instanceof BoundingBoxDownloader)
480                        || ((BoundingBoxDownloader) reader).considerAsFullDownload()) {
481                    // need to synthesize a download bounds lest the visual indication of downloaded area doesn't work
482                    dataSet.addDataSource(new DataSource(
483                            currentBounds != null ? currentBounds : new Bounds(LatLon.ZERO), "OpenStreetMap server"));
484                }
485            }
486
487            rememberDownloadedData(dataSet);
488            loadData(newLayerName, currentBounds);
489        }
490
491        @Override
492        protected void cancel() {
493            setCanceled(true);
494            if (reader != null) {
495                reader.cancel();
496            }
497        }
498    }
499
500    @Override
501    public String getConfirmationMessage(URL url) {
502        if (url != null) {
503            String urlString = url.toExternalForm();
504            if (urlString.matches(OsmUrlPattern.OSM_API_URL.pattern())) {
505                // TODO: proper i18n after stabilization
506                Collection<String> items = new ArrayList<>();
507                items.add(tr("OSM Server URL:") + ' ' + url.getHost());
508                items.add(tr("Command")+": "+url.getPath());
509                if (url.getQuery() != null) {
510                    items.add(tr("Request details: {0}", url.getQuery().replaceAll(",\\s*", ", ")));
511                }
512                return Utils.joinAsHtmlUnorderedList(items);
513            }
514            // TODO: other APIs
515        }
516        return null;
517    }
518}