001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005import static org.openstreetmap.josm.tools.I18n.trn; 006 007import java.io.IOException; 008import java.io.InputStream; 009import java.net.HttpURLConnection; 010import java.util.ArrayList; 011import java.util.Collection; 012import java.util.Collections; 013import java.util.HashSet; 014import java.util.Iterator; 015import java.util.LinkedHashSet; 016import java.util.List; 017import java.util.Set; 018import java.util.concurrent.Callable; 019import java.util.concurrent.CompletionService; 020import java.util.concurrent.ExecutionException; 021import java.util.concurrent.ExecutorCompletionService; 022import java.util.concurrent.ExecutorService; 023import java.util.concurrent.Executors; 024import java.util.concurrent.Future; 025 026import org.openstreetmap.josm.Main; 027import org.openstreetmap.josm.data.osm.DataSet; 028import org.openstreetmap.josm.data.osm.DataSetMerger; 029import org.openstreetmap.josm.data.osm.Node; 030import org.openstreetmap.josm.data.osm.OsmPrimitive; 031import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 032import org.openstreetmap.josm.data.osm.PrimitiveId; 033import org.openstreetmap.josm.data.osm.Relation; 034import org.openstreetmap.josm.data.osm.RelationMember; 035import org.openstreetmap.josm.data.osm.SimplePrimitiveId; 036import org.openstreetmap.josm.data.osm.Way; 037import org.openstreetmap.josm.gui.preferences.server.OverpassServerPreference; 038import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 039import org.openstreetmap.josm.gui.progress.ProgressMonitor; 040import org.openstreetmap.josm.tools.Utils; 041 042/** 043 * Retrieves a set of {@link OsmPrimitive}s from an OSM server using the so called 044 * Multi Fetch API. 045 * 046 * Usage: 047 * <pre> 048 * MultiFetchServerObjectReader reader = MultiFetchServerObjectReader() 049 * .append(2345,2334,4444) 050 * .append(new Node(72343)); 051 * reader.parseOsm(); 052 * if (!reader.getMissingPrimitives().isEmpty()) { 053 * Main.info("There are missing primitives: " + reader.getMissingPrimitives()); 054 * } 055 * if (!reader.getSkippedWays().isEmpty()) { 056 * Main.info("There are skipped ways: " + reader.getMissingPrimitives()); 057 * } 058 * </pre> 059 */ 060public class MultiFetchServerObjectReader extends OsmServerReader { 061 /** 062 * the max. number of primitives retrieved in one step. Assuming IDs with 7 digits, 063 * this leads to a max. request URL of ~ 1600 Bytes ((7 digits + 1 Separator) * 200), 064 * which should be safe according to the 065 * <a href="http://www.boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>. 066 */ 067 private static final int MAX_IDS_PER_REQUEST = 200; 068 069 private final Set<Long> nodes; 070 private final Set<Long> ways; 071 private final Set<Long> relations; 072 private Set<PrimitiveId> missingPrimitives; 073 private final DataSet outputDataSet; 074 075 /** 076 * Constructs a {@code MultiFetchServerObjectReader}. 077 */ 078 protected MultiFetchServerObjectReader() { 079 nodes = new LinkedHashSet<>(); 080 ways = new LinkedHashSet<>(); 081 relations = new LinkedHashSet<>(); 082 this.outputDataSet = new DataSet(); 083 this.missingPrimitives = new LinkedHashSet<>(); 084 } 085 086 /** 087 * Creates a new instance of {@link MultiFetchServerObjectReader} or {@link MultiFetchOverpassObjectReader} 088 * depending on the {@link OverpassServerPreference#useForMultiFetch preference}. 089 * 090 * @return a new instance 091 * @since 9241 092 */ 093 public static MultiFetchServerObjectReader create() { 094 return create(OverpassServerPreference.useForMultiFetch()); 095 } 096 097 /** 098 * Creates a new instance of {@link MultiFetchServerObjectReader} or {@link MultiFetchOverpassObjectReader} 099 * depending on the {@code fromMirror} parameter. 100 * 101 * @param fromMirror {@code false} for {@link MultiFetchServerObjectReader}, {@code true} for {@link MultiFetchOverpassObjectReader} 102 * @return a new instance 103 * @since 9241 104 */ 105 static MultiFetchServerObjectReader create(final boolean fromMirror) { 106 if (fromMirror) { 107 return new MultiFetchOverpassObjectReader(); 108 } else { 109 return new MultiFetchServerObjectReader(); 110 } 111 } 112 113 /** 114 * Remembers an {@link OsmPrimitive}'s id. The id will 115 * later be fetched as part of a Multi Get request. 116 * 117 * Ignore the id if it represents a new primitives. 118 * 119 * @param id the id 120 */ 121 protected void remember(PrimitiveId id) { 122 if (id.isNew()) return; 123 switch(id.getType()) { 124 case NODE: nodes.add(id.getUniqueId()); break; 125 case WAY: ways.add(id.getUniqueId()); break; 126 case RELATION: relations.add(id.getUniqueId()); break; 127 } 128 } 129 130 /** 131 * appends a {@link OsmPrimitive} id to the list of ids which will be fetched from the server. 132 * 133 * @param ds the {@link DataSet} to which the primitive belongs 134 * @param id the primitive id 135 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 136 * {@link OsmPrimitiveType#RELATION RELATION} 137 * @return this 138 */ 139 public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) { 140 OsmPrimitive p = ds.getPrimitiveById(id, type); 141 switch(type) { 142 case NODE: 143 return appendNode((Node) p); 144 case WAY: 145 return appendWay((Way) p); 146 case RELATION: 147 return appendRelation((Relation) p); 148 } 149 return this; 150 } 151 152 /** 153 * appends a {@link Node} id to the list of ids which will be fetched from the server. 154 * 155 * @param node the node (ignored, if null) 156 * @return this 157 */ 158 public MultiFetchServerObjectReader appendNode(Node node) { 159 if (node == null) return this; 160 remember(node.getPrimitiveId()); 161 return this; 162 } 163 164 /** 165 * appends a {@link Way} id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server. 166 * 167 * @param way the way (ignored, if null) 168 * @return this 169 */ 170 public MultiFetchServerObjectReader appendWay(Way way) { 171 if (way == null) return this; 172 if (way.isNew()) return this; 173 for (Node node: !recursesDown() ? way.getNodes() : Collections.<Node>emptyList()) { 174 if (!node.isNew()) { 175 remember(node.getPrimitiveId()); 176 } 177 } 178 remember(way.getPrimitiveId()); 179 return this; 180 } 181 182 /** 183 * appends a {@link Relation} id to the list of ids which will be fetched from the server. 184 * 185 * @param relation the relation (ignored, if null) 186 * @return this 187 */ 188 protected MultiFetchServerObjectReader appendRelation(Relation relation) { 189 if (relation == null) return this; 190 if (relation.isNew()) return this; 191 remember(relation.getPrimitiveId()); 192 for (RelationMember member : !recursesDown() ? relation.getMembers() : Collections.<RelationMember>emptyList()) { 193 if (OsmPrimitiveType.from(member.getMember()).equals(OsmPrimitiveType.RELATION)) { 194 // avoid infinite recursion in case of cyclic dependencies in relations 195 // 196 if (relations.contains(member.getMember().getId())) { 197 continue; 198 } 199 } 200 if (!member.getMember().isIncomplete()) { 201 append(member.getMember()); 202 } 203 } 204 return this; 205 } 206 207 /** 208 * appends an {@link OsmPrimitive} to the list of ids which will be fetched from the server. 209 * @param primitive the primitive 210 * @return this 211 */ 212 public MultiFetchServerObjectReader append(OsmPrimitive primitive) { 213 if (primitive != null) { 214 switch (OsmPrimitiveType.from(primitive)) { 215 case NODE: return appendNode((Node) primitive); 216 case WAY: return appendWay((Way) primitive); 217 case RELATION: return appendRelation((Relation) primitive); 218 } 219 } 220 return this; 221 } 222 223 /** 224 * appends a list of {@link OsmPrimitive} to the list of ids which will be fetched from the server. 225 * 226 * @param primitives the list of primitives (ignored, if null) 227 * @return this 228 * 229 * @see #append(OsmPrimitive) 230 */ 231 public MultiFetchServerObjectReader append(Collection<? extends OsmPrimitive> primitives) { 232 if (primitives == null) return this; 233 for (OsmPrimitive primitive : primitives) { 234 append(primitive); 235 } 236 return this; 237 } 238 239 /** 240 * extracts a subset of max {@link #MAX_IDS_PER_REQUEST} ids from <code>ids</code> and 241 * replies the subset. The extracted subset is removed from <code>ids</code>. 242 * 243 * @param ids a set of ids 244 * @return the subset of ids 245 */ 246 protected Set<Long> extractIdPackage(Set<Long> ids) { 247 Set<Long> pkg = new HashSet<>(); 248 if (ids.isEmpty()) 249 return pkg; 250 if (ids.size() > MAX_IDS_PER_REQUEST) { 251 Iterator<Long> it = ids.iterator(); 252 for (int i = 0; i < MAX_IDS_PER_REQUEST; i++) { 253 pkg.add(it.next()); 254 } 255 ids.removeAll(pkg); 256 } else { 257 pkg.addAll(ids); 258 ids.clear(); 259 } 260 return pkg; 261 } 262 263 /** 264 * builds the Multi Get request string for a set of ids and a given {@link OsmPrimitiveType}. 265 * 266 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 267 * {@link OsmPrimitiveType#RELATION RELATION} 268 * @param idPackage the package of ids 269 * @return the request string 270 */ 271 protected String buildRequestString(final OsmPrimitiveType type, Set<Long> idPackage) { 272 return type.getAPIName() + "s?" + type.getAPIName() + "s=" + Utils.join(",", idPackage); 273 } 274 275 @Override 276 protected String getBaseUrl() { 277 return super.getBaseUrl(); 278 } 279 280 protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) { 281 for (Way w: from.getWays()) { 282 if (w.hasIncompleteNodes()) { 283 for (Node n: w.getNodes()) { 284 if (n.isIncomplete()) { 285 nodes.add(n.getId()); 286 } 287 } 288 } 289 } 290 } 291 292 /** 293 * merges the dataset <code>from</code> to {@link #outputDataSet}. 294 * 295 * @param from the other dataset 296 */ 297 protected void merge(DataSet from) { 298 final DataSetMerger visitor = new DataSetMerger(outputDataSet, from); 299 visitor.merge(); 300 } 301 302 /** 303 * fetches a set of ids of a given {@link OsmPrimitiveType} from the server 304 * 305 * @param ids the set of ids 306 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 307 * {@link OsmPrimitiveType#RELATION RELATION} 308 * @param progressMonitor progress monitor 309 * @throws OsmTransferException if an error occurs while communicating with the API server 310 */ 311 protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException { 312 String msg = ""; 313 final String baseUrl = getBaseUrl(); 314 switch (type) { 315 case NODE: msg = tr("Fetching a package of nodes from ''{0}''", baseUrl); break; 316 case WAY: msg = tr("Fetching a package of ways from ''{0}''", baseUrl); break; 317 case RELATION: msg = tr("Fetching a package of relations from ''{0}''", baseUrl); break; 318 } 319 progressMonitor.setTicksCount(ids.size()); 320 progressMonitor.setTicks(0); 321 // The complete set containing all primitives to fetch 322 Set<Long> toFetch = new HashSet<>(ids); 323 // Build a list of fetchers that will download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each. 324 // we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers. 325 int threadsNumber = Main.pref.getInteger("osm.download.threads", OsmApi.MAX_DOWNLOAD_THREADS); 326 threadsNumber = Math.min(Math.max(threadsNumber, 1), OsmApi.MAX_DOWNLOAD_THREADS); 327 final ExecutorService exec = Executors.newFixedThreadPool( 328 threadsNumber, Utils.newThreadFactory(getClass() + "-%d", Thread.NORM_PRIORITY)); 329 CompletionService<FetchResult> ecs = new ExecutorCompletionService<>(exec); 330 List<Future<FetchResult>> jobs = new ArrayList<>(); 331 while (!toFetch.isEmpty()) { 332 jobs.add(ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor))); 333 } 334 // Run the fetchers 335 for (int i = 0; i < jobs.size() && !isCanceled(); i++) { 336 progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + '/' + progressMonitor.getTicksCount()); 337 try { 338 FetchResult result = ecs.take().get(); 339 if (result.missingPrimitives != null) { 340 missingPrimitives.addAll(result.missingPrimitives); 341 } 342 if (result.dataSet != null && !isCanceled()) { 343 rememberNodesOfIncompleteWaysToLoad(result.dataSet); 344 merge(result.dataSet); 345 } 346 } catch (InterruptedException | ExecutionException e) { 347 Main.error(e); 348 } 349 } 350 exec.shutdown(); 351 // Cancel requests if the user chose to 352 if (isCanceled()) { 353 for (Future<FetchResult> job : jobs) { 354 job.cancel(true); 355 } 356 } 357 } 358 359 /** 360 * invokes one or more Multi Gets to fetch the {@link OsmPrimitive}s and replies 361 * the dataset of retrieved primitives. Note that the dataset includes non visible primitives too! 362 * In contrast to a simple Get for a node, a way, or a relation, a Multi Get always replies 363 * the latest version of the primitive (if any), even if the primitive is not visible (i.e. if 364 * visible==false). 365 * 366 * Invoke {@link #getMissingPrimitives()} to get a list of primitives which have not been 367 * found on the server (the server response code was 404) 368 * 369 * @return the parsed data 370 * @throws OsmTransferException if an error occurs while communicating with the API server 371 * @see #getMissingPrimitives() 372 * 373 */ 374 @Override 375 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 376 int n = nodes.size() + ways.size() + relations.size(); 377 progressMonitor.beginTask(trn("Downloading {0} object from ''{1}''", 378 "Downloading {0} objects from ''{1}''", n, n, OsmApi.getOsmApi().getBaseUrl())); 379 try { 380 missingPrimitives = new HashSet<>(); 381 if (isCanceled()) return null; 382 fetchPrimitives(ways, OsmPrimitiveType.WAY, progressMonitor); 383 if (isCanceled()) return null; 384 fetchPrimitives(nodes, OsmPrimitiveType.NODE, progressMonitor); 385 if (isCanceled()) return null; 386 fetchPrimitives(relations, OsmPrimitiveType.RELATION, progressMonitor); 387 if (outputDataSet != null) { 388 outputDataSet.deleteInvisible(); 389 } 390 return outputDataSet; 391 } finally { 392 progressMonitor.finishTask(); 393 } 394 } 395 396 /** 397 * replies the set of ids of all primitives for which a fetch request to the 398 * server was submitted but which are not available from the server (the server 399 * replied a return code of 404) 400 * 401 * @return the set of ids of missing primitives 402 */ 403 public Set<PrimitiveId> getMissingPrimitives() { 404 return missingPrimitives; 405 } 406 407 /** 408 * Whether this reader fetches nodes when loading ways, or members when loading relations. 409 * 410 * @return {@code true} if the reader recurses down 411 */ 412 protected boolean recursesDown() { 413 return false; 414 } 415 416 /** 417 * The class holding the results given by {@link Fetcher}. 418 * It is only a wrapper of the resulting {@link DataSet} and the collection of {@link PrimitiveId} that could not have been loaded. 419 */ 420 protected static class FetchResult { 421 422 /** 423 * The resulting data set 424 */ 425 public final DataSet dataSet; 426 427 /** 428 * The collection of primitive ids that could not have been loaded 429 */ 430 public final Set<PrimitiveId> missingPrimitives; 431 432 /** 433 * Constructs a {@code FetchResult} 434 * @param dataSet The resulting data set 435 * @param missingPrimitives The collection of primitive ids that could not have been loaded 436 */ 437 public FetchResult(DataSet dataSet, Set<PrimitiveId> missingPrimitives) { 438 this.dataSet = dataSet; 439 this.missingPrimitives = missingPrimitives; 440 } 441 } 442 443 /** 444 * The class that actually download data from OSM API. 445 * Several instances of this class are used by {@link MultiFetchServerObjectReader} (one per set of primitives to fetch). 446 * The inheritance of {@link OsmServerReader} is only explained by the need to have a distinct OSM connection by {@code Fetcher} instance. 447 * @see FetchResult 448 */ 449 protected class Fetcher extends OsmServerReader implements Callable<FetchResult> { 450 451 private final Set<Long> pkg; 452 private final OsmPrimitiveType type; 453 private final ProgressMonitor progressMonitor; 454 455 /** 456 * Constructs a {@code Fetcher} 457 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 458 * {@link OsmPrimitiveType#RELATION RELATION} 459 * @param idsPackage The set of primitives ids to fetch 460 * @param progressMonitor The progress monitor 461 */ 462 public Fetcher(OsmPrimitiveType type, Set<Long> idsPackage, ProgressMonitor progressMonitor) { 463 this.pkg = idsPackage; 464 this.type = type; 465 this.progressMonitor = progressMonitor; 466 } 467 468 @Override 469 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 470 // This method is implemented because of the OsmServerReader inheritance, but not used, 471 // as the main target of this class is the call() method. 472 return fetch(progressMonitor).dataSet; 473 } 474 475 @Override 476 public FetchResult call() throws Exception { 477 return fetch(progressMonitor); 478 } 479 480 /** 481 * fetches the requested primitives and updates the specified progress monitor. 482 * @param progressMonitor the progress monitor 483 * @return the {@link FetchResult} of this operation 484 * @throws OsmTransferException if an error occurs while communicating with the API server 485 */ 486 protected FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException { 487 try { 488 return multiGetIdPackage(type, pkg, progressMonitor); 489 } catch (OsmApiException e) { 490 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 491 Main.info(tr("Server replied with response code 404, retrying with an individual request for each object.")); 492 return singleGetIdPackage(type, pkg, progressMonitor); 493 } else { 494 throw e; 495 } 496 } 497 } 498 499 @Override 500 protected String getBaseUrl() { 501 return MultiFetchServerObjectReader.this.getBaseUrl(); 502 } 503 504 /** 505 * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}. 506 * The retrieved primitives are merged to {@link #outputDataSet}. 507 * 508 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 509 * {@link OsmPrimitiveType#RELATION RELATION} 510 * @param pkg the package of ids 511 * @param progressMonitor progress monitor 512 * @return the {@link FetchResult} of this operation 513 * @throws OsmTransferException if an error occurs while communicating with the API server 514 */ 515 protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) 516 throws OsmTransferException { 517 String request = buildRequestString(type, pkg); 518 FetchResult result = null; 519 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) { 520 if (in == null) return null; 521 progressMonitor.subTask(tr("Downloading OSM data...")); 522 try { 523 result = new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null); 524 } catch (Exception e) { 525 throw new OsmTransferException(e); 526 } 527 } catch (IOException ex) { 528 Main.warn(ex); 529 } 530 return result; 531 } 532 533 /** 534 * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}. 535 * The retrieved primitive is merged to {@link #outputDataSet}. 536 * 537 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 538 * {@link OsmPrimitiveType#RELATION RELATION} 539 * @param id the id 540 * @param progressMonitor progress monitor 541 * @return the {@link DataSet} resulting of this operation 542 * @throws OsmTransferException if an error occurs while communicating with the API server 543 */ 544 protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException { 545 String request = buildRequestString(type, Collections.singleton(id)); 546 DataSet result = null; 547 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) { 548 if (in == null) return null; 549 progressMonitor.subTask(tr("Downloading OSM data...")); 550 try { 551 result = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false)); 552 } catch (Exception e) { 553 throw new OsmTransferException(e); 554 } 555 } catch (IOException ex) { 556 Main.warn(ex); 557 } 558 return result; 559 } 560 561 /** 562 * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}. 563 * The retrieved primitives are merged to {@link #outputDataSet}. 564 * 565 * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404). 566 * If the set is fetched with this method it is possible to find out which of the ids doesn't exist. 567 * Unfortunately, the server does not provide an error header or an error body for a 404 reply. 568 * 569 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 570 * {@link OsmPrimitiveType#RELATION RELATION} 571 * @param pkg the set of ids 572 * @param progressMonitor progress monitor 573 * @return the {@link FetchResult} of this operation 574 * @throws OsmTransferException if an error occurs while communicating with the API server 575 */ 576 protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) 577 throws OsmTransferException { 578 FetchResult result = new FetchResult(new DataSet(), new HashSet<PrimitiveId>()); 579 String baseUrl = OsmApi.getOsmApi().getBaseUrl(); 580 for (long id : pkg) { 581 try { 582 String msg = ""; 583 switch (type) { 584 case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, baseUrl); break; 585 case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, baseUrl); break; 586 case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break; 587 } 588 progressMonitor.setCustomText(msg); 589 result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor)); 590 } catch (OsmApiException e) { 591 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 592 Main.info(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id))); 593 result.missingPrimitives.add(new SimplePrimitiveId(id, type)); 594 } else { 595 throw e; 596 } 597 } 598 } 599 return result; 600 } 601 } 602}