001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005import static org.openstreetmap.josm.tools.I18n.trn;
006
007import java.io.IOException;
008import java.io.InputStream;
009import java.net.HttpURLConnection;
010import java.util.ArrayList;
011import java.util.Collection;
012import java.util.HashSet;
013import java.util.Iterator;
014import java.util.LinkedHashSet;
015import java.util.List;
016import java.util.Set;
017import java.util.concurrent.Callable;
018import java.util.concurrent.CompletionService;
019import java.util.concurrent.ExecutionException;
020import java.util.concurrent.Executor;
021import java.util.concurrent.ExecutorCompletionService;
022import java.util.concurrent.Executors;
023import java.util.concurrent.Future;
024
025import org.openstreetmap.josm.Main;
026import org.openstreetmap.josm.data.osm.DataSet;
027import org.openstreetmap.josm.data.osm.DataSetMerger;
028import org.openstreetmap.josm.data.osm.Node;
029import org.openstreetmap.josm.data.osm.OsmPrimitive;
030import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
031import org.openstreetmap.josm.data.osm.PrimitiveId;
032import org.openstreetmap.josm.data.osm.Relation;
033import org.openstreetmap.josm.data.osm.RelationMember;
034import org.openstreetmap.josm.data.osm.SimplePrimitiveId;
035import org.openstreetmap.josm.data.osm.Way;
036import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
037import org.openstreetmap.josm.gui.progress.ProgressMonitor;
038import org.openstreetmap.josm.tools.Utils;
039
040/**
041 * Retrieves a set of {@link OsmPrimitive}s from an OSM server using the so called
042 * Multi Fetch API.
043 *
044 * Usage:
045 * <pre>
046 *    MultiFetchServerObjectReader reader = MultiFetchServerObjectReader()
047 *         .append(2345,2334,4444)
048 *         .append(new Node(72343));
049 *    reader.parseOsm();
050 *    if (!reader.getMissingPrimitives().isEmpty()) {
051 *        Main.info("There are missing primitives: " + reader.getMissingPrimitives());
052 *    }
053 *    if (!reader.getSkippedWays().isEmpty()) {
054 *       Main.info("There are skipped ways: " + reader.getMissingPrimitives());
055 *    }
056 * </pre>
057 */
058public class MultiFetchServerObjectReader extends OsmServerReader {
059    /**
060     * the max. number of primitives retrieved in one step. Assuming IDs with 7 digits,
061     * this leads to a max. request URL of ~ 1600 Bytes ((7 digits +  1 Separator) * 200),
062     * which should be safe according to the
063     * <a href="http://www.boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>.
064     */
065    private static final int MAX_IDS_PER_REQUEST = 200;
066
067    private Set<Long> nodes;
068    private Set<Long> ways;
069    private Set<Long> relations;
070    private Set<PrimitiveId> missingPrimitives;
071    private DataSet outputDataSet;
072
073    /**
074     * Constructs a {@code MultiFetchServerObjectReader}.
075     */
076    public MultiFetchServerObjectReader() {
077        nodes = new LinkedHashSet<>();
078        ways = new LinkedHashSet<>();
079        relations = new LinkedHashSet<>();
080        this.outputDataSet = new DataSet();
081        this.missingPrimitives = new LinkedHashSet<>();
082    }
083
084    /**
085     * Remembers an {@link OsmPrimitive}'s id. The id will
086     * later be fetched as part of a Multi Get request.
087     *
088     * Ignore the id if it represents a new primitives.
089     *
090     * @param id  the id
091     */
092    protected void remember(PrimitiveId id) {
093        if (id.isNew()) return;
094        switch(id.getType()) {
095        case NODE: nodes.add(id.getUniqueId()); break;
096        case WAY: ways.add(id.getUniqueId()); break;
097        case RELATION: relations.add(id.getUniqueId()); break;
098        }
099    }
100
101    /**
102     * appends a {@link OsmPrimitive} id to the list of ids which will be fetched from the server.
103     *
104     * @param ds the {@link DataSet} to which the primitive belongs
105     * @param id the primitive id
106     * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
107     * {@link OsmPrimitiveType#RELATION RELATION}
108     * @return this
109     */
110    public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) {
111        OsmPrimitive p = ds.getPrimitiveById(id, type);
112        switch(type) {
113        case NODE:
114            return appendNode((Node) p);
115        case WAY:
116            return appendWay((Way) p);
117        case RELATION:
118            return appendRelation((Relation) p);
119        }
120        return this;
121    }
122
123    /**
124     * appends a {@link Node} id to the list of ids which will be fetched from the server.
125     *
126     * @param node  the node (ignored, if null)
127     * @return this
128     */
129    public MultiFetchServerObjectReader appendNode(Node node) {
130        if (node == null) return this;
131        remember(node.getPrimitiveId());
132        return this;
133    }
134
135    /**
136     * appends a {@link Way} id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server.
137     *
138     * @param way the way (ignored, if null)
139     * @return this
140     */
141    public MultiFetchServerObjectReader appendWay(Way way) {
142        if (way == null) return this;
143        if (way.isNew()) return this;
144        for (Node node: way.getNodes()) {
145            if (!node.isNew()) {
146                remember(node.getPrimitiveId());
147            }
148        }
149        remember(way.getPrimitiveId());
150        return this;
151    }
152
153    /**
154     * appends a {@link Relation} id to the list of ids which will be fetched from the server.
155     *
156     * @param relation  the relation (ignored, if null)
157     * @return this
158     */
159    protected MultiFetchServerObjectReader appendRelation(Relation relation) {
160        if (relation == null) return this;
161        if (relation.isNew()) return this;
162        remember(relation.getPrimitiveId());
163        for (RelationMember member : relation.getMembers()) {
164            if (OsmPrimitiveType.from(member.getMember()).equals(OsmPrimitiveType.RELATION)) {
165                // avoid infinite recursion in case of cyclic dependencies in relations
166                //
167                if (relations.contains(member.getMember().getId())) {
168                    continue;
169                }
170            }
171            if (!member.getMember().isIncomplete()) {
172                append(member.getMember());
173            }
174        }
175        return this;
176    }
177
178    /**
179     * appends an {@link OsmPrimitive} to the list of ids which will be fetched from the server.
180     * @param primitive the primitive
181     * @return this
182     */
183    public MultiFetchServerObjectReader append(OsmPrimitive primitive) {
184        if (primitive != null) {
185            switch (OsmPrimitiveType.from(primitive)) {
186                case NODE: return appendNode((Node) primitive);
187                case WAY: return appendWay((Way) primitive);
188                case RELATION: return appendRelation((Relation) primitive);
189            }
190        }
191        return this;
192    }
193
194    /**
195     * appends a list of {@link OsmPrimitive} to the list of ids which will be fetched from the server.
196     *
197     * @param primitives  the list of primitives (ignored, if null)
198     * @return this
199     *
200     * @see #append(OsmPrimitive)
201     */
202    public MultiFetchServerObjectReader append(Collection<? extends OsmPrimitive> primitives) {
203        if (primitives == null) return this;
204        for (OsmPrimitive primitive : primitives) {
205            append(primitive);
206        }
207        return this;
208    }
209
210    /**
211     * extracts a subset of max {@link #MAX_IDS_PER_REQUEST} ids from <code>ids</code> and
212     * replies the subset. The extracted subset is removed from <code>ids</code>.
213     *
214     * @param ids a set of ids
215     * @return the subset of ids
216     */
217    protected Set<Long> extractIdPackage(Set<Long> ids) {
218        Set<Long> pkg = new HashSet<>();
219        if (ids.isEmpty())
220            return pkg;
221        if (ids.size() > MAX_IDS_PER_REQUEST) {
222            Iterator<Long> it = ids.iterator();
223            for (int i = 0; i < MAX_IDS_PER_REQUEST; i++) {
224                pkg.add(it.next());
225            }
226            ids.removeAll(pkg);
227        } else {
228            pkg.addAll(ids);
229            ids.clear();
230        }
231        return pkg;
232    }
233
234    /**
235     * builds the Multi Get request string for a set of ids and a given {@link OsmPrimitiveType}.
236     *
237     * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
238     * {@link OsmPrimitiveType#RELATION RELATION}
239     * @param idPackage  the package of ids
240     * @return the request string
241     */
242    protected static String buildRequestString(OsmPrimitiveType type, Set<Long> idPackage) {
243        StringBuilder sb = new StringBuilder();
244        sb.append(type.getAPIName()).append("s?")
245        .append(type.getAPIName()).append("s=");
246
247        Iterator<Long> it = idPackage.iterator();
248        for (int i = 0; i < idPackage.size(); i++) {
249            sb.append(it.next());
250            if (i < idPackage.size()-1) {
251                sb.append(',');
252            }
253        }
254        return sb.toString();
255    }
256
257    /**
258     * builds the Multi Get request string for a single id and a given {@link OsmPrimitiveType}.
259     *
260     * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
261     * {@link OsmPrimitiveType#RELATION RELATION}
262     * @param id the id
263     * @return the request string
264     */
265    protected static String buildRequestString(OsmPrimitiveType type, long id) {
266        StringBuilder sb = new StringBuilder();
267        sb.append(type.getAPIName()).append("s?")
268        .append(type.getAPIName()).append("s=")
269        .append(id);
270        return sb.toString();
271    }
272
273    protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) {
274        for (Way w: from.getWays()) {
275            if (w.hasIncompleteNodes()) {
276                for (Node n: w.getNodes()) {
277                    if (n.isIncomplete()) {
278                        nodes.add(n.getId());
279                    }
280                }
281            }
282        }
283    }
284
285    /**
286     * merges the dataset <code>from</code> to {@link #outputDataSet}.
287     *
288     * @param from the other dataset
289     */
290    protected void merge(DataSet from) {
291        final DataSetMerger visitor = new DataSetMerger(outputDataSet, from);
292        visitor.merge();
293    }
294
295    /**
296     * fetches a set of ids of a given {@link OsmPrimitiveType} from the server
297     *
298     * @param ids the set of ids
299     * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
300     * {@link OsmPrimitiveType#RELATION RELATION}
301     * @throws OsmTransferException if an error occurs while communicating with the API server
302     */
303    protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException {
304        String msg = "";
305        String baseUrl = OsmApi.getOsmApi().getBaseUrl();
306        switch (type) {
307            case NODE:     msg = tr("Fetching a package of nodes from ''{0}''",     baseUrl); break;
308            case WAY:      msg = tr("Fetching a package of ways from ''{0}''",      baseUrl); break;
309            case RELATION: msg = tr("Fetching a package of relations from ''{0}''", baseUrl); break;
310        }
311        progressMonitor.setTicksCount(ids.size());
312        progressMonitor.setTicks(0);
313        // The complete set containing all primitives to fetch
314        Set<Long> toFetch = new HashSet<>(ids);
315        // Build a list of fetchers that will  download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each.
316        // we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers.
317        int threadsNumber = Main.pref.getInteger("osm.download.threads", OsmApi.MAX_DOWNLOAD_THREADS);
318        threadsNumber = Math.min(Math.max(threadsNumber, 1), OsmApi.MAX_DOWNLOAD_THREADS);
319        Executor exec = Executors.newFixedThreadPool(threadsNumber, Utils.newThreadFactory(getClass() + "-%d", Thread.NORM_PRIORITY));
320        CompletionService<FetchResult> ecs = new ExecutorCompletionService<>(exec);
321        List<Future<FetchResult>> jobs = new ArrayList<>();
322        while (!toFetch.isEmpty()) {
323            jobs.add(ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor)));
324        }
325        // Run the fetchers
326        for (int i = 0; i < jobs.size() && !isCanceled(); i++) {
327            progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + '/' + progressMonitor.getTicksCount());
328            try {
329                FetchResult result = ecs.take().get();
330                if (result.missingPrimitives != null) {
331                    missingPrimitives.addAll(result.missingPrimitives);
332                }
333                if (result.dataSet != null && !isCanceled()) {
334                    rememberNodesOfIncompleteWaysToLoad(result.dataSet);
335                    merge(result.dataSet);
336                }
337            } catch (InterruptedException | ExecutionException e) {
338                Main.error(e);
339            }
340        }
341        // Cancel requests if the user choosed to
342        if (isCanceled()) {
343            for (Future<FetchResult> job : jobs) {
344                job.cancel(true);
345            }
346        }
347    }
348
349    /**
350     * invokes one or more Multi Gets to fetch the {@link OsmPrimitive}s and replies
351     * the dataset of retrieved primitives. Note that the dataset includes non visible primitives too!
352     * In contrast to a simple Get for a node, a way, or a relation, a Multi Get always replies
353     * the latest version of the primitive (if any), even if the primitive is not visible (i.e. if
354     * visible==false).
355     *
356     * Invoke {@link #getMissingPrimitives()} to get a list of primitives which have not been
357     * found on  the server (the server response code was 404)
358     *
359     * @return the parsed data
360     * @throws OsmTransferException if an error occurs while communicating with the API server
361     * @see #getMissingPrimitives()
362     *
363     */
364    @Override
365    public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
366        int n = nodes.size() + ways.size() + relations.size();
367        progressMonitor.beginTask(trn("Downloading {0} object from ''{1}''",
368                "Downloading {0} objects from ''{1}''", n, n, OsmApi.getOsmApi().getBaseUrl()));
369        try {
370            missingPrimitives = new HashSet<>();
371            if (isCanceled()) return null;
372            fetchPrimitives(ways, OsmPrimitiveType.WAY, progressMonitor);
373            if (isCanceled()) return null;
374            fetchPrimitives(nodes, OsmPrimitiveType.NODE, progressMonitor);
375            if (isCanceled()) return null;
376            fetchPrimitives(relations, OsmPrimitiveType.RELATION, progressMonitor);
377            if (outputDataSet != null) {
378                outputDataSet.deleteInvisible();
379            }
380            return outputDataSet;
381        } finally {
382            progressMonitor.finishTask();
383        }
384    }
385
386    /**
387     * replies the set of ids of all primitives for which a fetch request to the
388     * server was submitted but which are not available from the server (the server
389     * replied a return code of 404)
390     *
391     * @return the set of ids of missing primitives
392     */
393    public Set<PrimitiveId> getMissingPrimitives() {
394        return missingPrimitives;
395    }
396
397    /**
398     * The class holding the results given by {@link Fetcher}.
399     * It is only a wrapper of the resulting {@link DataSet} and the collection of {@link PrimitiveId} that could not have been loaded.
400     */
401    protected static class FetchResult {
402
403        /**
404         * The resulting data set
405         */
406        public final DataSet dataSet;
407
408        /**
409         * The collection of primitive ids that could not have been loaded
410         */
411        public final Set<PrimitiveId> missingPrimitives;
412
413        /**
414         * Constructs a {@code FetchResult}
415         * @param dataSet The resulting data set
416         * @param missingPrimitives The collection of primitive ids that could not have been loaded
417         */
418        public FetchResult(DataSet dataSet, Set<PrimitiveId> missingPrimitives) {
419            this.dataSet = dataSet;
420            this.missingPrimitives = missingPrimitives;
421        }
422    }
423
424    /**
425     * The class that actually download data from OSM API.
426     * Several instances of this class are used by {@link MultiFetchServerObjectReader} (one per set of primitives to fetch).
427     * The inheritance of {@link OsmServerReader} is only explained by the need to have a distinct OSM connection by {@code Fetcher} instance.
428     * @see FetchResult
429     */
430    protected static class Fetcher extends OsmServerReader implements Callable<FetchResult> {
431
432        private final Set<Long> pkg;
433        private final OsmPrimitiveType type;
434        private final ProgressMonitor progressMonitor;
435
436        /**
437         * Constructs a {@code Fetcher}
438         * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
439         * {@link OsmPrimitiveType#RELATION RELATION}
440         * @param idsPackage The set of primitives ids to fetch
441         * @param progressMonitor The progress monitor
442         */
443        public Fetcher(OsmPrimitiveType type, Set<Long> idsPackage, ProgressMonitor progressMonitor) {
444            this.pkg = idsPackage;
445            this.type = type;
446            this.progressMonitor = progressMonitor;
447        }
448
449        @Override
450        public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
451            // This method is implemented because of the OsmServerReader inheritance, but not used,
452            // as the main target of this class is the call() method.
453            return fetch(progressMonitor).dataSet;
454        }
455
456        @Override
457        public FetchResult call() throws Exception {
458            return fetch(progressMonitor);
459        }
460
461        /**
462         * fetches the requested primitives and updates the specified progress monitor.
463         * @param progressMonitor the progress monitor
464         * @return the {@link FetchResult} of this operation
465         * @throws OsmTransferException if an error occurs while communicating with the API server
466         */
467        protected FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException {
468            try {
469                return multiGetIdPackage(type, pkg, progressMonitor);
470            } catch (OsmApiException e) {
471                if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
472                    Main.info(tr("Server replied with response code 404, retrying with an individual request for each object."));
473                    return singleGetIdPackage(type, pkg, progressMonitor);
474                } else {
475                    throw e;
476                }
477            }
478        }
479
480        /**
481         * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}.
482         * The retrieved primitives are merged to {@link #outputDataSet}.
483         *
484         * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
485         * {@link OsmPrimitiveType#RELATION RELATION}
486         * @param pkg the package of ids
487         * @return the {@link FetchResult} of this operation
488         * @throws OsmTransferException if an error occurs while communicating with the API server
489         */
490        protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor)
491                throws OsmTransferException {
492            String request = buildRequestString(type, pkg);
493            FetchResult result = null;
494            try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) {
495                if (in == null) return null;
496                progressMonitor.subTask(tr("Downloading OSM data..."));
497                try {
498                    result = new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null);
499                } catch (Exception e) {
500                    throw new OsmTransferException(e);
501                }
502            } catch (IOException ex) {
503                Main.warn(ex);
504            }
505            return result;
506        }
507
508        /**
509         * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}.
510         * The retrieved primitive is merged to {@link #outputDataSet}.
511         *
512         * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
513         * {@link OsmPrimitiveType#RELATION RELATION}
514         * @param id the id
515         * @return the {@link DataSet} resulting of this operation
516         * @throws OsmTransferException if an error occurs while communicating with the API server
517         */
518        protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException {
519            String request = buildRequestString(type, id);
520            DataSet result = null;
521            try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) {
522                if (in == null) return null;
523                progressMonitor.subTask(tr("Downloading OSM data..."));
524                try {
525                    result = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false));
526                } catch (Exception e) {
527                    throw new OsmTransferException(e);
528                }
529            } catch (IOException ex) {
530                Main.warn(ex);
531            }
532            return result;
533        }
534
535        /**
536         * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}.
537         * The retrieved primitives are merged to {@link #outputDataSet}.
538         *
539         * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404).
540         * If the set is fetched with this method it is possible to find out which of the ids doesn't exist.
541         * Unfortunately, the server does not provide an error header or an error body for a 404 reply.
542         *
543         * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY},
544         * {@link OsmPrimitiveType#RELATION RELATION}
545         * @param pkg the set of ids
546         * @return the {@link FetchResult} of this operation
547         * @throws OsmTransferException if an error occurs while communicating with the API server
548         */
549        protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor)
550                throws OsmTransferException {
551            FetchResult result = new FetchResult(new DataSet(), new HashSet<PrimitiveId>());
552            String baseUrl = OsmApi.getOsmApi().getBaseUrl();
553            for (long id : pkg) {
554                try {
555                    String msg = "";
556                    switch (type) {
557                        case NODE:     msg = tr("Fetching node with id {0} from ''{1}''",     id, baseUrl); break;
558                        case WAY:      msg = tr("Fetching way with id {0} from ''{1}''",      id, baseUrl); break;
559                        case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break;
560                    }
561                    progressMonitor.setCustomText(msg);
562                    result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor));
563                } catch (OsmApiException e) {
564                    if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
565                        Main.info(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id)));
566                        result.missingPrimitives.add(new SimplePrimitiveId(id, type));
567                    } else {
568                        throw e;
569                    }
570                }
571            }
572            return result;
573        }
574    }
575}