001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005import static org.openstreetmap.josm.tools.I18n.trn; 006 007import java.io.IOException; 008import java.io.InputStream; 009import java.net.HttpURLConnection; 010import java.util.ArrayList; 011import java.util.Collection; 012import java.util.HashSet; 013import java.util.Iterator; 014import java.util.LinkedHashSet; 015import java.util.List; 016import java.util.Set; 017import java.util.concurrent.Callable; 018import java.util.concurrent.CompletionService; 019import java.util.concurrent.ExecutionException; 020import java.util.concurrent.Executor; 021import java.util.concurrent.ExecutorCompletionService; 022import java.util.concurrent.Executors; 023import java.util.concurrent.Future; 024 025import org.openstreetmap.josm.Main; 026import org.openstreetmap.josm.data.osm.DataSet; 027import org.openstreetmap.josm.data.osm.DataSetMerger; 028import org.openstreetmap.josm.data.osm.Node; 029import org.openstreetmap.josm.data.osm.OsmPrimitive; 030import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 031import org.openstreetmap.josm.data.osm.PrimitiveId; 032import org.openstreetmap.josm.data.osm.Relation; 033import org.openstreetmap.josm.data.osm.RelationMember; 034import org.openstreetmap.josm.data.osm.SimplePrimitiveId; 035import org.openstreetmap.josm.data.osm.Way; 036import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 037import org.openstreetmap.josm.gui.progress.ProgressMonitor; 038import org.openstreetmap.josm.tools.Utils; 039 040/** 041 * Retrieves a set of {@link OsmPrimitive}s from an OSM server using the so called 042 * Multi Fetch API. 043 * 044 * Usage: 045 * <pre> 046 * MultiFetchServerObjectReader reader = MultiFetchServerObjectReader() 047 * .append(2345,2334,4444) 048 * .append(new Node(72343)); 049 * reader.parseOsm(); 050 * if (!reader.getMissingPrimitives().isEmpty()) { 051 * Main.info("There are missing primitives: " + reader.getMissingPrimitives()); 052 * } 053 * if (!reader.getSkippedWays().isEmpty()) { 054 * Main.info("There are skipped ways: " + reader.getMissingPrimitives()); 055 * } 056 * </pre> 057 */ 058public class MultiFetchServerObjectReader extends OsmServerReader { 059 /** 060 * the max. number of primitives retrieved in one step. Assuming IDs with 7 digits, 061 * this leads to a max. request URL of ~ 1600 Bytes ((7 digits + 1 Separator) * 200), 062 * which should be safe according to the 063 * <a href="http://www.boutell.com/newfaq/misc/urllength.html">WWW FAQ</a>. 064 */ 065 private static final int MAX_IDS_PER_REQUEST = 200; 066 067 private Set<Long> nodes; 068 private Set<Long> ways; 069 private Set<Long> relations; 070 private Set<PrimitiveId> missingPrimitives; 071 private DataSet outputDataSet; 072 073 /** 074 * Constructs a {@code MultiFetchServerObjectReader}. 075 */ 076 public MultiFetchServerObjectReader() { 077 nodes = new LinkedHashSet<>(); 078 ways = new LinkedHashSet<>(); 079 relations = new LinkedHashSet<>(); 080 this.outputDataSet = new DataSet(); 081 this.missingPrimitives = new LinkedHashSet<>(); 082 } 083 084 /** 085 * Remembers an {@link OsmPrimitive}'s id. The id will 086 * later be fetched as part of a Multi Get request. 087 * 088 * Ignore the id if it represents a new primitives. 089 * 090 * @param id the id 091 */ 092 protected void remember(PrimitiveId id) { 093 if (id.isNew()) return; 094 switch(id.getType()) { 095 case NODE: nodes.add(id.getUniqueId()); break; 096 case WAY: ways.add(id.getUniqueId()); break; 097 case RELATION: relations.add(id.getUniqueId()); break; 098 } 099 } 100 101 /** 102 * appends a {@link OsmPrimitive} id to the list of ids which will be fetched from the server. 103 * 104 * @param ds the {@link DataSet} to which the primitive belongs 105 * @param id the primitive id 106 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 107 * {@link OsmPrimitiveType#RELATION RELATION} 108 * @return this 109 */ 110 public MultiFetchServerObjectReader append(DataSet ds, long id, OsmPrimitiveType type) { 111 OsmPrimitive p = ds.getPrimitiveById(id, type); 112 switch(type) { 113 case NODE: 114 return appendNode((Node) p); 115 case WAY: 116 return appendWay((Way) p); 117 case RELATION: 118 return appendRelation((Relation) p); 119 } 120 return this; 121 } 122 123 /** 124 * appends a {@link Node} id to the list of ids which will be fetched from the server. 125 * 126 * @param node the node (ignored, if null) 127 * @return this 128 */ 129 public MultiFetchServerObjectReader appendNode(Node node) { 130 if (node == null) return this; 131 remember(node.getPrimitiveId()); 132 return this; 133 } 134 135 /** 136 * appends a {@link Way} id and the list of ids of nodes the way refers to the list of ids which will be fetched from the server. 137 * 138 * @param way the way (ignored, if null) 139 * @return this 140 */ 141 public MultiFetchServerObjectReader appendWay(Way way) { 142 if (way == null) return this; 143 if (way.isNew()) return this; 144 for (Node node: way.getNodes()) { 145 if (!node.isNew()) { 146 remember(node.getPrimitiveId()); 147 } 148 } 149 remember(way.getPrimitiveId()); 150 return this; 151 } 152 153 /** 154 * appends a {@link Relation} id to the list of ids which will be fetched from the server. 155 * 156 * @param relation the relation (ignored, if null) 157 * @return this 158 */ 159 protected MultiFetchServerObjectReader appendRelation(Relation relation) { 160 if (relation == null) return this; 161 if (relation.isNew()) return this; 162 remember(relation.getPrimitiveId()); 163 for (RelationMember member : relation.getMembers()) { 164 if (OsmPrimitiveType.from(member.getMember()).equals(OsmPrimitiveType.RELATION)) { 165 // avoid infinite recursion in case of cyclic dependencies in relations 166 // 167 if (relations.contains(member.getMember().getId())) { 168 continue; 169 } 170 } 171 if (!member.getMember().isIncomplete()) { 172 append(member.getMember()); 173 } 174 } 175 return this; 176 } 177 178 /** 179 * appends an {@link OsmPrimitive} to the list of ids which will be fetched from the server. 180 * @param primitive the primitive 181 * @return this 182 */ 183 public MultiFetchServerObjectReader append(OsmPrimitive primitive) { 184 if (primitive != null) { 185 switch (OsmPrimitiveType.from(primitive)) { 186 case NODE: return appendNode((Node) primitive); 187 case WAY: return appendWay((Way) primitive); 188 case RELATION: return appendRelation((Relation) primitive); 189 } 190 } 191 return this; 192 } 193 194 /** 195 * appends a list of {@link OsmPrimitive} to the list of ids which will be fetched from the server. 196 * 197 * @param primitives the list of primitives (ignored, if null) 198 * @return this 199 * 200 * @see #append(OsmPrimitive) 201 */ 202 public MultiFetchServerObjectReader append(Collection<? extends OsmPrimitive> primitives) { 203 if (primitives == null) return this; 204 for (OsmPrimitive primitive : primitives) { 205 append(primitive); 206 } 207 return this; 208 } 209 210 /** 211 * extracts a subset of max {@link #MAX_IDS_PER_REQUEST} ids from <code>ids</code> and 212 * replies the subset. The extracted subset is removed from <code>ids</code>. 213 * 214 * @param ids a set of ids 215 * @return the subset of ids 216 */ 217 protected Set<Long> extractIdPackage(Set<Long> ids) { 218 Set<Long> pkg = new HashSet<>(); 219 if (ids.isEmpty()) 220 return pkg; 221 if (ids.size() > MAX_IDS_PER_REQUEST) { 222 Iterator<Long> it = ids.iterator(); 223 for (int i = 0; i < MAX_IDS_PER_REQUEST; i++) { 224 pkg.add(it.next()); 225 } 226 ids.removeAll(pkg); 227 } else { 228 pkg.addAll(ids); 229 ids.clear(); 230 } 231 return pkg; 232 } 233 234 /** 235 * builds the Multi Get request string for a set of ids and a given {@link OsmPrimitiveType}. 236 * 237 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 238 * {@link OsmPrimitiveType#RELATION RELATION} 239 * @param idPackage the package of ids 240 * @return the request string 241 */ 242 protected static String buildRequestString(OsmPrimitiveType type, Set<Long> idPackage) { 243 StringBuilder sb = new StringBuilder(); 244 sb.append(type.getAPIName()).append("s?") 245 .append(type.getAPIName()).append("s="); 246 247 Iterator<Long> it = idPackage.iterator(); 248 for (int i = 0; i < idPackage.size(); i++) { 249 sb.append(it.next()); 250 if (i < idPackage.size()-1) { 251 sb.append(','); 252 } 253 } 254 return sb.toString(); 255 } 256 257 /** 258 * builds the Multi Get request string for a single id and a given {@link OsmPrimitiveType}. 259 * 260 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 261 * {@link OsmPrimitiveType#RELATION RELATION} 262 * @param id the id 263 * @return the request string 264 */ 265 protected static String buildRequestString(OsmPrimitiveType type, long id) { 266 StringBuilder sb = new StringBuilder(); 267 sb.append(type.getAPIName()).append("s?") 268 .append(type.getAPIName()).append("s=") 269 .append(id); 270 return sb.toString(); 271 } 272 273 protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) { 274 for (Way w: from.getWays()) { 275 if (w.hasIncompleteNodes()) { 276 for (Node n: w.getNodes()) { 277 if (n.isIncomplete()) { 278 nodes.add(n.getId()); 279 } 280 } 281 } 282 } 283 } 284 285 /** 286 * merges the dataset <code>from</code> to {@link #outputDataSet}. 287 * 288 * @param from the other dataset 289 */ 290 protected void merge(DataSet from) { 291 final DataSetMerger visitor = new DataSetMerger(outputDataSet, from); 292 visitor.merge(); 293 } 294 295 /** 296 * fetches a set of ids of a given {@link OsmPrimitiveType} from the server 297 * 298 * @param ids the set of ids 299 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 300 * {@link OsmPrimitiveType#RELATION RELATION} 301 * @throws OsmTransferException if an error occurs while communicating with the API server 302 */ 303 protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException { 304 String msg = ""; 305 String baseUrl = OsmApi.getOsmApi().getBaseUrl(); 306 switch (type) { 307 case NODE: msg = tr("Fetching a package of nodes from ''{0}''", baseUrl); break; 308 case WAY: msg = tr("Fetching a package of ways from ''{0}''", baseUrl); break; 309 case RELATION: msg = tr("Fetching a package of relations from ''{0}''", baseUrl); break; 310 } 311 progressMonitor.setTicksCount(ids.size()); 312 progressMonitor.setTicks(0); 313 // The complete set containing all primitives to fetch 314 Set<Long> toFetch = new HashSet<>(ids); 315 // Build a list of fetchers that will download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each. 316 // we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers. 317 int threadsNumber = Main.pref.getInteger("osm.download.threads", OsmApi.MAX_DOWNLOAD_THREADS); 318 threadsNumber = Math.min(Math.max(threadsNumber, 1), OsmApi.MAX_DOWNLOAD_THREADS); 319 Executor exec = Executors.newFixedThreadPool(threadsNumber, Utils.newThreadFactory(getClass() + "-%d", Thread.NORM_PRIORITY)); 320 CompletionService<FetchResult> ecs = new ExecutorCompletionService<>(exec); 321 List<Future<FetchResult>> jobs = new ArrayList<>(); 322 while (!toFetch.isEmpty()) { 323 jobs.add(ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor))); 324 } 325 // Run the fetchers 326 for (int i = 0; i < jobs.size() && !isCanceled(); i++) { 327 progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + '/' + progressMonitor.getTicksCount()); 328 try { 329 FetchResult result = ecs.take().get(); 330 if (result.missingPrimitives != null) { 331 missingPrimitives.addAll(result.missingPrimitives); 332 } 333 if (result.dataSet != null && !isCanceled()) { 334 rememberNodesOfIncompleteWaysToLoad(result.dataSet); 335 merge(result.dataSet); 336 } 337 } catch (InterruptedException | ExecutionException e) { 338 Main.error(e); 339 } 340 } 341 // Cancel requests if the user choosed to 342 if (isCanceled()) { 343 for (Future<FetchResult> job : jobs) { 344 job.cancel(true); 345 } 346 } 347 } 348 349 /** 350 * invokes one or more Multi Gets to fetch the {@link OsmPrimitive}s and replies 351 * the dataset of retrieved primitives. Note that the dataset includes non visible primitives too! 352 * In contrast to a simple Get for a node, a way, or a relation, a Multi Get always replies 353 * the latest version of the primitive (if any), even if the primitive is not visible (i.e. if 354 * visible==false). 355 * 356 * Invoke {@link #getMissingPrimitives()} to get a list of primitives which have not been 357 * found on the server (the server response code was 404) 358 * 359 * @return the parsed data 360 * @throws OsmTransferException if an error occurs while communicating with the API server 361 * @see #getMissingPrimitives() 362 * 363 */ 364 @Override 365 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 366 int n = nodes.size() + ways.size() + relations.size(); 367 progressMonitor.beginTask(trn("Downloading {0} object from ''{1}''", 368 "Downloading {0} objects from ''{1}''", n, n, OsmApi.getOsmApi().getBaseUrl())); 369 try { 370 missingPrimitives = new HashSet<>(); 371 if (isCanceled()) return null; 372 fetchPrimitives(ways, OsmPrimitiveType.WAY, progressMonitor); 373 if (isCanceled()) return null; 374 fetchPrimitives(nodes, OsmPrimitiveType.NODE, progressMonitor); 375 if (isCanceled()) return null; 376 fetchPrimitives(relations, OsmPrimitiveType.RELATION, progressMonitor); 377 if (outputDataSet != null) { 378 outputDataSet.deleteInvisible(); 379 } 380 return outputDataSet; 381 } finally { 382 progressMonitor.finishTask(); 383 } 384 } 385 386 /** 387 * replies the set of ids of all primitives for which a fetch request to the 388 * server was submitted but which are not available from the server (the server 389 * replied a return code of 404) 390 * 391 * @return the set of ids of missing primitives 392 */ 393 public Set<PrimitiveId> getMissingPrimitives() { 394 return missingPrimitives; 395 } 396 397 /** 398 * The class holding the results given by {@link Fetcher}. 399 * It is only a wrapper of the resulting {@link DataSet} and the collection of {@link PrimitiveId} that could not have been loaded. 400 */ 401 protected static class FetchResult { 402 403 /** 404 * The resulting data set 405 */ 406 public final DataSet dataSet; 407 408 /** 409 * The collection of primitive ids that could not have been loaded 410 */ 411 public final Set<PrimitiveId> missingPrimitives; 412 413 /** 414 * Constructs a {@code FetchResult} 415 * @param dataSet The resulting data set 416 * @param missingPrimitives The collection of primitive ids that could not have been loaded 417 */ 418 public FetchResult(DataSet dataSet, Set<PrimitiveId> missingPrimitives) { 419 this.dataSet = dataSet; 420 this.missingPrimitives = missingPrimitives; 421 } 422 } 423 424 /** 425 * The class that actually download data from OSM API. 426 * Several instances of this class are used by {@link MultiFetchServerObjectReader} (one per set of primitives to fetch). 427 * The inheritance of {@link OsmServerReader} is only explained by the need to have a distinct OSM connection by {@code Fetcher} instance. 428 * @see FetchResult 429 */ 430 protected static class Fetcher extends OsmServerReader implements Callable<FetchResult> { 431 432 private final Set<Long> pkg; 433 private final OsmPrimitiveType type; 434 private final ProgressMonitor progressMonitor; 435 436 /** 437 * Constructs a {@code Fetcher} 438 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 439 * {@link OsmPrimitiveType#RELATION RELATION} 440 * @param idsPackage The set of primitives ids to fetch 441 * @param progressMonitor The progress monitor 442 */ 443 public Fetcher(OsmPrimitiveType type, Set<Long> idsPackage, ProgressMonitor progressMonitor) { 444 this.pkg = idsPackage; 445 this.type = type; 446 this.progressMonitor = progressMonitor; 447 } 448 449 @Override 450 public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException { 451 // This method is implemented because of the OsmServerReader inheritance, but not used, 452 // as the main target of this class is the call() method. 453 return fetch(progressMonitor).dataSet; 454 } 455 456 @Override 457 public FetchResult call() throws Exception { 458 return fetch(progressMonitor); 459 } 460 461 /** 462 * fetches the requested primitives and updates the specified progress monitor. 463 * @param progressMonitor the progress monitor 464 * @return the {@link FetchResult} of this operation 465 * @throws OsmTransferException if an error occurs while communicating with the API server 466 */ 467 protected FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException { 468 try { 469 return multiGetIdPackage(type, pkg, progressMonitor); 470 } catch (OsmApiException e) { 471 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 472 Main.info(tr("Server replied with response code 404, retrying with an individual request for each object.")); 473 return singleGetIdPackage(type, pkg, progressMonitor); 474 } else { 475 throw e; 476 } 477 } 478 } 479 480 /** 481 * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}. 482 * The retrieved primitives are merged to {@link #outputDataSet}. 483 * 484 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 485 * {@link OsmPrimitiveType#RELATION RELATION} 486 * @param pkg the package of ids 487 * @return the {@link FetchResult} of this operation 488 * @throws OsmTransferException if an error occurs while communicating with the API server 489 */ 490 protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) 491 throws OsmTransferException { 492 String request = buildRequestString(type, pkg); 493 FetchResult result = null; 494 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) { 495 if (in == null) return null; 496 progressMonitor.subTask(tr("Downloading OSM data...")); 497 try { 498 result = new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null); 499 } catch (Exception e) { 500 throw new OsmTransferException(e); 501 } 502 } catch (IOException ex) { 503 Main.warn(ex); 504 } 505 return result; 506 } 507 508 /** 509 * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}. 510 * The retrieved primitive is merged to {@link #outputDataSet}. 511 * 512 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 513 * {@link OsmPrimitiveType#RELATION RELATION} 514 * @param id the id 515 * @return the {@link DataSet} resulting of this operation 516 * @throws OsmTransferException if an error occurs while communicating with the API server 517 */ 518 protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException { 519 String request = buildRequestString(type, id); 520 DataSet result = null; 521 try (InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE)) { 522 if (in == null) return null; 523 progressMonitor.subTask(tr("Downloading OSM data...")); 524 try { 525 result = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false)); 526 } catch (Exception e) { 527 throw new OsmTransferException(e); 528 } 529 } catch (IOException ex) { 530 Main.warn(ex); 531 } 532 return result; 533 } 534 535 /** 536 * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}. 537 * The retrieved primitives are merged to {@link #outputDataSet}. 538 * 539 * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404). 540 * If the set is fetched with this method it is possible to find out which of the ids doesn't exist. 541 * Unfortunately, the server does not provide an error header or an error body for a 404 reply. 542 * 543 * @param type The primitive type. Must be one of {@link OsmPrimitiveType#NODE NODE}, {@link OsmPrimitiveType#WAY WAY}, 544 * {@link OsmPrimitiveType#RELATION RELATION} 545 * @param pkg the set of ids 546 * @return the {@link FetchResult} of this operation 547 * @throws OsmTransferException if an error occurs while communicating with the API server 548 */ 549 protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) 550 throws OsmTransferException { 551 FetchResult result = new FetchResult(new DataSet(), new HashSet<PrimitiveId>()); 552 String baseUrl = OsmApi.getOsmApi().getBaseUrl(); 553 for (long id : pkg) { 554 try { 555 String msg = ""; 556 switch (type) { 557 case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, baseUrl); break; 558 case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, baseUrl); break; 559 case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break; 560 } 561 progressMonitor.setCustomText(msg); 562 result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor)); 563 } catch (OsmApiException e) { 564 if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) { 565 Main.info(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id))); 566 result.missingPrimitives.add(new SimplePrimitiveId(id, type)); 567 } else { 568 throw e; 569 } 570 } 571 } 572 return result; 573 } 574 } 575}