001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005import static org.openstreetmap.josm.tools.I18n.trn; 006 007import java.io.IOException; 008import java.io.InputStream; 009import java.net.HttpURLConnection; 010import java.util.ArrayList; 011import java.util.Collection; 012import java.util.HashSet; 013import java.util.Iterator; 014import java.util.LinkedHashSet; 015import java.util.List; 016import java.util.NoSuchElementException; 017import java.util.Set; 018import java.util.concurrent.Callable; 019import java.util.concurrent.CompletionService; 020import java.util.concurrent.ExecutionException; 021import java.util.concurrent.Executor; 022import java.util.concurrent.ExecutorCompletionService; 023import java.util.concurrent.Executors; 024import java.util.concurrent.Future; 025 026import org.openstreetmap.josm.Main; 027import org.openstreetmap.josm.data.osm.DataSet; 028import org.openstreetmap.josm.data.osm.DataSetMerger; 029import org.openstreetmap.josm.data.osm.Node; 030import org.openstreetmap.josm.data.osm.OsmPrimitive; 031import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 032import org.openstreetmap.josm.data.osm.PrimitiveId; 033import org.openstreetmap.josm.data.osm.Relation; 034import org.openstreetmap.josm.data.osm.RelationMember; 035import org.openstreetmap.josm.data.osm.SimplePrimitiveId; 036import org.openstreetmap.josm.data.osm.Way; 037import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 038import org.openstreetmap.josm.gui.progress.ProgressMonitor; 039import org.openstreetmap.josm.tools.CheckParameterUtil; 040 041/** 042 * Retrieves a set of {@link OsmPrimitive}s from an OSM server using the so called 043 * Multi Fetch API. 044 * 045 * Usage: 046 * <pre> 047 * MultiFetchServerObjectReader reader = MultiFetchServerObjectReader() 048 * .append(2345,2334,4444) 049 * .append(new Node(72343)); 050 * reader.parseOsm(); 051 * if (!reader.getMissingPrimitives().isEmpty()) { 052 * Main.info("There are missing primitives: " + reader.getMissingPrimitives()); 053 * } 054 * if (!reader.getSkippedWays().isEmpty()) { 055 * Main.info("There are skipped ways: " + reader.getMissingPrimitives()); 056 * } 057 * </pre> 058 */ 059public class MultiFetchServerObjectReader extends OsmServerReader{ 060 /** 061 * the max. number of primitives retrieved in one step. Assuming IDs with 7 digits, 062 * this leads to a max. request URL of ~ 1600 Bytes ((7 digits + 1 Separator) * 200), 063 * which should be safe according to the 064 * <a href="http://www.boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>. 065 */ 066 private static final int MAX_IDS_PER_REQUEST = 200; 067 068 private Set<Long> nodes; 069 private Set<Long> ways; 070 private Set<Long> relations; 071 private Set<PrimitiveId> missingPrimitives; 072 private DataSet outputDataSet; 073 074 /** 075 * Constructs a {@code MultiFetchServerObjectReader}. 076 */ 077 public MultiFetchServerObjectReader() { 078 nodes = new LinkedHashSet<>(); 079 ways = new LinkedHashSet<>(); 080 relations = new LinkedHashSet<>(); 081 this.outputDataSet = new DataSet(); 082 this.missingPrimitives = new LinkedHashSet<>(); 083 } 084 085 /** 086 * Remembers an {@link OsmPrimitive}'s id. The id will 087 * later be fetched as part of a Multi Get request. 088 * 089 * Ignore the id if it represents a new primitives. 090 * 091 * @param id the id 092 */ 093 protected void remember(PrimitiveId id) { 094 if (id.isNew()) return; 095 switch(id.getType()) { 096 case NODE: nodes.add(id.getUniqueId()); break; 097 case WAY: ways.add(id.getUniqueId()); break; 098 case RELATION: relations.add(id.getUniqueId()); break; 099 } 100 } 101 102 /** 103 * remembers an {@link OsmPrimitive}'s id. <code>ds</code> must include 104 * an {@link OsmPrimitive} with id=<code>id</code>. The id will 105 * later we fetched as part of a Multi Get request. 106 * 107 * Ignore the id if it id <= 0. 108 * 109 * @param ds the dataset (must not be null) 110 * @param id the primitive id 111 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 112 * @throws IllegalArgumentException if ds is null 113 * @throws NoSuchElementException if ds does not include an {@link OsmPrimitive} with id=<code>id</code> 114 */ 115 protected void remember(DataSet ds, long id, OsmPrimitiveType type) throws IllegalArgumentException, NoSuchElementException{ 116 CheckParameterUtil.ensureParameterNotNull(ds, "ds"); 117 if (id <= 0) return; 118 OsmPrimitive primitive = ds.getPrimitiveById(id, type); 119 if (primitive == null) 120 throw new NoSuchElementException(tr("No primitive with id {0} in local dataset. Cannot infer primitive type.", id)); 121 remember(primitive.getPrimitiveId()); 122 return; 123 } 124 125 /** 126 * appends a {@link OsmPrimitive} id to the list of ids which will be fetched from the server. 127 * 128 * @param ds the {@link DataSet} to which the primitive belongs 129 * @param id the primitive id 130 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 131 * @return this 132 */ 133 public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) { 134 OsmPrimitive p = ds.getPrimitiveById(id,type); 135 switch(type) { 136 case NODE: 137 return appendNode((Node)p); 138 case WAY: 139 return appendWay((Way)p); 140 case RELATION: 141 return appendRelation((Relation)p); 142 } 143 return this; 144 } 145 146 /** 147 * appends a {@link Node} id to the list of ids which will be fetched from the server. 148 * 149 * @param node the node (ignored, if null) 150 * @return this 151 */ 152 public MultiFetchServerObjectReader appendNode(Node node) { 153 if (node == null) return this; 154 remember(node.getPrimitiveId()); 155 return this; 156 } 157 158 /** 159 * appends a {@link Way} id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server. 160 * 161 * @param way the way (ignored, if null) 162 * @return this 163 */ 164 public MultiFetchServerObjectReader appendWay(Way way) { 165 if (way == null) return this; 166 if (way.isNew()) return this; 167 for (Node node: way.getNodes()) { 168 if (!node.isNew()) { 169 remember(node.getPrimitiveId()); 170 } 171 } 172 remember(way.getPrimitiveId()); 173 return this; 174 } 175 176 /** 177 * appends a {@link Relation} id to the list of ids which will be fetched from the server. 178 * 179 * @param relation the relation (ignored, if null) 180 * @return this 181 */ 182 protected MultiFetchServerObjectReader appendRelation(Relation relation) { 183 if (relation == null) return this; 184 if (relation.isNew()) return this; 185 remember(relation.getPrimitiveId()); 186 for (RelationMember member : relation.getMembers()) { 187 if (OsmPrimitiveType.from(member.getMember()).equals(OsmPrimitiveType.RELATION)) { 188 // avoid infinite recursion in case of cyclic dependencies in relations 189 // 190 if (relations.contains(member.getMember().getId())) { 191 continue; 192 } 193 } 194 if (!member.getMember().isIncomplete()) { 195 append(member.getMember()); 196 } 197 } 198 return this; 199 } 200 201 /** 202 * appends an {@link OsmPrimitive} to the list of ids which will be fetched from the server. 203 * @param primitive the primitive 204 * @return this 205 */ 206 public MultiFetchServerObjectReader append(OsmPrimitive primitive) { 207 if (primitive != null) { 208 switch (OsmPrimitiveType.from(primitive)) { 209 case NODE: return appendNode((Node)primitive); 210 case WAY: return appendWay((Way)primitive); 211 case RELATION: return appendRelation((Relation)primitive); 212 } 213 } 214 return this; 215 } 216 217 /** 218 * appends a list of {@link OsmPrimitive} to the list of ids which will be fetched from the server. 219 * 220 * @param primitives the list of primitives (ignored, if null) 221 * @return this 222 * 223 * @see #append(OsmPrimitive) 224 */ 225 public MultiFetchServerObjectReader append(Collection<? extends OsmPrimitive> primitives) { 226 if (primitives == null) return this; 227 for (OsmPrimitive primitive : primitives) { 228 append(primitive); 229 } 230 return this; 231 } 232 233 /** 234 * extracts a subset of max {@link #MAX_IDS_PER_REQUEST} ids from <code>ids</code> and 235 * replies the subset. The extracted subset is removed from <code>ids</code>. 236 * 237 * @param ids a set of ids 238 * @return the subset of ids 239 */ 240 protected Set<Long> extractIdPackage(Set<Long> ids) { 241 HashSet<Long> pkg = new HashSet<>(); 242 if (ids.isEmpty()) 243 return pkg; 244 if (ids.size() > MAX_IDS_PER_REQUEST) { 245 Iterator<Long> it = ids.iterator(); 246 for (int i=0; i<MAX_IDS_PER_REQUEST; i++) { 247 pkg.add(it.next()); 248 } 249 ids.removeAll(pkg); 250 } else { 251 pkg.addAll(ids); 252 ids.clear(); 253 } 254 return pkg; 255 } 256 257 /** 258 * builds the Multi Get request string for a set of ids and a given 259 * {@link OsmPrimitiveType}. 260 * 261 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 262 * @param idPackage the package of ids 263 * @return the request string 264 */ 265 protected static String buildRequestString(OsmPrimitiveType type, Set<Long> idPackage) { 266 StringBuilder sb = new StringBuilder(); 267 sb.append(type.getAPIName()).append("s?") 268 .append(type.getAPIName()).append("s="); 269 270 Iterator<Long> it = idPackage.iterator(); 271 for (int i=0; i<idPackage.size(); i++) { 272 sb.append(it.next()); 273 if (i < idPackage.size()-1) { 274 sb.append(","); 275 } 276 } 277 return sb.toString(); 278 } 279 280 /** 281 * builds the Multi Get request string for a single id and a given 282 * {@link OsmPrimitiveType}. 283 * 284 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 285 * @param id the id 286 * @return the request string 287 */ 288 protected static String buildRequestString(OsmPrimitiveType type, long id) { 289 StringBuilder sb = new StringBuilder(); 290 sb.append(type.getAPIName()).append("s?") 291 .append(type.getAPIName()).append("s=") 292 .append(id); 293 return sb.toString(); 294 } 295 296 protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) { 297 for (Way w: from.getWays()) { 298 if (w.hasIncompleteNodes()) { 299 for (Node n: w.getNodes()) { 300 if (n.isIncomplete()) { 301 nodes.add(n.getId()); 302 } 303 } 304 } 305 } 306 } 307 308 /** 309 * merges the dataset <code>from</code> to {@link #outputDataSet}. 310 * 311 * @param from the other dataset 312 */ 313 protected void merge(DataSet from) { 314 final DataSetMerger visitor = new DataSetMerger(outputDataSet,from); 315 visitor.merge(); 316 } 317 318 /** 319 * fetches a set of ids of a given {@link OsmPrimitiveType} from the server 320 * 321 * @param ids the set of ids 322 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 323 * @throws OsmTransferException if an error occurs while communicating with the API server 324 */ 325 protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException { 326 String msg = ""; 327 String baseUrl = OsmApi.getOsmApi().getBaseUrl(); 328 switch (type) { 329 case NODE: msg = tr("Fetching a package of nodes from ''{0}''", baseUrl); break; 330 case WAY: msg = tr("Fetching a package of ways from ''{0}''", baseUrl); break; 331 case RELATION: msg = tr("Fetching a package of relations from ''{0}''", baseUrl); break; 332 } 333 progressMonitor.setTicksCount(ids.size()); 334 progressMonitor.setTicks(0); 335 // The complete set containg all primitives to fetch 336 Set<Long> toFetch = new HashSet<>(ids); 337 // Build a list of fetchers that will download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each. 338 // we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers. 339 int threadsNumber = Main.pref.getInteger("osm.download.threads", OsmApi.MAX_DOWNLOAD_THREADS); 340 threadsNumber = Math.min(Math.max(threadsNumber, 1), OsmApi.MAX_DOWNLOAD_THREADS); 341 Executor exec = Executors.newFixedThreadPool(threadsNumber); 342 CompletionService<FetchResult> ecs = new ExecutorCompletionService<>(exec); 343 List<Future<FetchResult>> jobs = new ArrayList<>(); 344 while (!toFetch.isEmpty()) { 345 jobs.add(ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor))); 346 } 347 // Run the fetchers 348 for (int i = 0; i < jobs.size() && !isCanceled(); i++) { 349 progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + "/" + progressMonitor.getTicksCount()); 350 try { 351 FetchResult result = ecs.take().get(); 352 if (result.missingPrimitives != null) { 353 missingPrimitives.addAll(result.missingPrimitives); 354 } 355 if (result.dataSet != null && !isCanceled()) { 356 rememberNodesOfIncompleteWaysToLoad(result.dataSet); 357 merge(result.dataSet); 358 } 359 } catch (InterruptedException | ExecutionException e) { 360 Main.error(e); 361 } 362 } 363 // Cancel requests if the user choosed to 364 if (isCanceled()) { 365 for (Future<FetchResult> job : jobs) { 366 job.cancel(true); 367 } 368 } 369 } 370 371 /** 372 * invokes one or more Multi Gets to fetch the {@link OsmPrimitive}s and replies 373 * the dataset of retrieved primitives. Note that the dataset includes non visible primitives too! 374 * In contrast to a simple Get for a node, a way, or a relation, a Multi Get always replies 375 * the latest version of the primitive (if any), even if the primitive is not visible (i.e. if 376 * visible==false). 377 * 378 * Invoke {@link #getMissingPrimitives()} to get a list of primitives which have not been 379 * found on the server (the server response code was 404) 380 * 381 * @return the parsed data 382 * @throws OsmTransferException if an error occurs while communicating with the API server 383 * @see #getMissingPrimitives() 384 * 385 */ 386 @Override 387 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 388 int n = nodes.size() + ways.size() + relations.size(); 389 progressMonitor.beginTask(trn("Downloading {0} object from ''{1}''", "Downloading {0} objects from ''{1}''", n, n, OsmApi.getOsmApi().getBaseUrl())); 390 try { 391 missingPrimitives = new HashSet<>(); 392 if (isCanceled()) return null; 393 fetchPrimitives(ways,OsmPrimitiveType.WAY, progressMonitor); 394 if (isCanceled()) return null; 395 fetchPrimitives(nodes,OsmPrimitiveType.NODE, progressMonitor); 396 if (isCanceled()) return null; 397 fetchPrimitives(relations,OsmPrimitiveType.RELATION, progressMonitor); 398 if (outputDataSet != null) { 399 outputDataSet.deleteInvisible(); 400 } 401 return outputDataSet; 402 } finally { 403 progressMonitor.finishTask(); 404 } 405 } 406 407 /** 408 * replies the set of ids of all primitives for which a fetch request to the 409 * server was submitted but which are not available from the server (the server 410 * replied a return code of 404) 411 * 412 * @return the set of ids of missing primitives 413 */ 414 public Set<PrimitiveId> getMissingPrimitives() { 415 return missingPrimitives; 416 } 417 418 /** 419 * The class holding the results given by {@link Fetcher}. 420 * It is only a wrapper of the resulting {@link DataSet} and the collection of {@link PrimitiveId} that could not have been loaded. 421 */ 422 protected static class FetchResult { 423 424 /** 425 * The resulting data set 426 */ 427 public final DataSet dataSet; 428 429 /** 430 * The collection of primitive ids that could not have been loaded 431 */ 432 public final Set<PrimitiveId> missingPrimitives; 433 434 /** 435 * Constructs a {@code FetchResult} 436 * @param dataSet The resulting data set 437 * @param missingPrimitives The collection of primitive ids that could not have been loaded 438 */ 439 public FetchResult(DataSet dataSet, Set<PrimitiveId> missingPrimitives) { 440 this.dataSet = dataSet; 441 this.missingPrimitives = missingPrimitives; 442 } 443 } 444 445 /** 446 * The class that actually download data from OSM API. Several instances of this class are used by {@link MultiFetchServerObjectReader} (one per set of primitives to fetch). 447 * The inheritance of {@link OsmServerReader} is only explained by the need to have a distinct OSM connection by {@code Fetcher} instance. 448 * @see FetchResult 449 */ 450 protected static class Fetcher extends OsmServerReader implements Callable<FetchResult> { 451 452 private final Set<Long> pkg; 453 private final OsmPrimitiveType type; 454 private final ProgressMonitor progressMonitor; 455 456 /** 457 * Constructs a {@code Fetcher} 458 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 459 * @param idsPackage The set of primitives ids to fetch 460 * @param progressMonitor The progress monitor 461 */ 462 public Fetcher(OsmPrimitiveType type, Set<Long> idsPackage, ProgressMonitor progressMonitor) { 463 this.pkg = idsPackage; 464 this.type = type; 465 this.progressMonitor = progressMonitor; 466 } 467 468 @Override 469 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 470 // This method is implemented because of the OsmServerReader inheritance, but not used, as the main target of this class is the call() method. 471 return fetch(progressMonitor).dataSet; 472 } 473 474 @Override 475 public FetchResult call() throws Exception { 476 return fetch(progressMonitor); 477 } 478 479 /** 480 * fetches the requested primitives and updates the specified progress monitor. 481 * @param progressMonitor the progress monitor 482 * @return the {@link FetchResult} of this operation 483 * @throws OsmTransferException if an error occurs while communicating with the API server 484 */ 485 protected FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException { 486 try { 487 return multiGetIdPackage(type, pkg, progressMonitor); 488 } catch (OsmApiException e) { 489 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 490 Main.info(tr("Server replied with response code 404, retrying with an individual request for each object.")); 491 return singleGetIdPackage(type, pkg, progressMonitor); 492 } else { 493 throw e; 494 } 495 } 496 } 497 498 /** 499 * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}. 500 * The retrieved primitives are merged to {@link #outputDataSet}. 501 * 502 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 503 * @param pkg the package of ids 504 * @return the {@link FetchResult} of this operation 505 * @throws OsmTransferException if an error occurs while communicating with the API server 506 */ 507 protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException { 508 String request = buildRequestString(type, pkg); 509 FetchResult result = null; 510 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) { 511 if (in == null) return null; 512 progressMonitor.subTask(tr("Downloading OSM data...")); 513 try { 514 result = new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null); 515 } catch (Exception e) { 516 throw new OsmTransferException(e); 517 } 518 } catch (IOException ex) { 519 Main.warn(ex); 520 } 521 return result; 522 } 523 524 /** 525 * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}. 526 * The retrieved primitive is merged to {@link #outputDataSet}. 527 * 528 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 529 * @param id the id 530 * @return the {@link DataSet} resulting of this operation 531 * @throws OsmTransferException if an error occurs while communicating with the API server 532 */ 533 protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException { 534 String request = buildRequestString(type, id); 535 DataSet result = null; 536 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) { 537 if (in == null) return null; 538 progressMonitor.subTask(tr("Downloading OSM data...")); 539 try { 540 result = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false)); 541 } catch (Exception e) { 542 throw new OsmTransferException(e); 543 } 544 } catch (IOException ex) { 545 Main.warn(ex); 546 } 547 return result; 548 } 549 550 /** 551 * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}. 552 * The retrieved primitives are merged to {@link #outputDataSet}. 553 * 554 * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404). 555 * If the set is fetched with this method it is possible to find out which of the ids doesn't exist. 556 * Unfortunately, the server does not provide an error header or an error body for a 404 reply. 557 * 558 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, {@link OsmPrimitiveType#RELATION RELATION} 559 * @param pkg the set of ids 560 * @return the {@link FetchResult} of this operation 561 * @throws OsmTransferException if an error occurs while communicating with the API server 562 */ 563 protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException { 564 FetchResult result = new FetchResult(new DataSet(), new HashSet<PrimitiveId>()); 565 String baseUrl = OsmApi.getOsmApi().getBaseUrl(); 566 for (long id : pkg) { 567 try { 568 String msg = ""; 569 switch (type) { 570 case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, baseUrl); break; 571 case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, baseUrl); break; 572 case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break; 573 } 574 progressMonitor.setCustomText(msg); 575 result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor)); 576 } catch (OsmApiException e) { 577 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 578 Main.info(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id))); 579 result.missingPrimitives.add(new SimplePrimitiveId(id, type)); 580 } else { 581 throw e; 582 } 583 } 584 } 585 return result; 586 } 587 } 588}