001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.io; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.InputStream; 007import java.io.InputStreamReader; 008import java.text.MessageFormat; 009import java.util.ArrayList; 010import java.util.Collection; 011import java.util.List; 012import java.util.regex.Matcher; 013import java.util.regex.Pattern; 014 015import javax.xml.stream.Location; 016import javax.xml.stream.XMLInputFactory; 017import javax.xml.stream.XMLStreamConstants; 018import javax.xml.stream.XMLStreamException; 019import javax.xml.stream.XMLStreamReader; 020 021import org.openstreetmap.josm.Main; 022import org.openstreetmap.josm.data.Bounds; 023import org.openstreetmap.josm.data.DataSource; 024import org.openstreetmap.josm.data.coor.LatLon; 025import org.openstreetmap.josm.data.osm.Changeset; 026import org.openstreetmap.josm.data.osm.DataSet; 027import org.openstreetmap.josm.data.osm.Node; 028import org.openstreetmap.josm.data.osm.NodeData; 029import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 030import org.openstreetmap.josm.data.osm.PrimitiveData; 031import org.openstreetmap.josm.data.osm.Relation; 032import org.openstreetmap.josm.data.osm.RelationData; 033import org.openstreetmap.josm.data.osm.RelationMemberData; 034import org.openstreetmap.josm.data.osm.Tagged; 035import org.openstreetmap.josm.data.osm.User; 036import org.openstreetmap.josm.data.osm.Way; 037import org.openstreetmap.josm.data.osm.WayData; 038import org.openstreetmap.josm.gui.progress.NullProgressMonitor; 039import org.openstreetmap.josm.gui.progress.ProgressMonitor; 040import org.openstreetmap.josm.tools.CheckParameterUtil; 041import org.openstreetmap.josm.tools.date.DateUtils; 042 043/** 044 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it. 045 * 046 * For each xml element, there is a dedicated method. 047 * The XMLStreamReader cursor points to the start of the element, when the method is 048 * entered, and it must point to the end of the same element, when it is exited. 049 */ 050public class OsmReader extends AbstractReader { 051 052 protected XMLStreamReader parser; 053 054 protected boolean cancel; 055 056 /** Used by plugins to register themselves as data postprocessors. */ 057 private static volatile List<OsmServerReadPostprocessor> postprocessors; 058 059 /** register a new postprocessor */ 060 public static void registerPostprocessor(OsmServerReadPostprocessor pp) { 061 if (postprocessors == null) { 062 postprocessors = new ArrayList<>(); 063 } 064 postprocessors.add(pp); 065 } 066 067 /** deregister a postprocessor previously registered with registerPostprocessor */ 068 public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) { 069 if (postprocessors != null) { 070 postprocessors.remove(pp); 071 } 072 } 073 074 /** 075 * constructor (for private and subclasses use only) 076 * 077 * @see #parseDataSet(InputStream, ProgressMonitor) 078 */ 079 protected OsmReader() { 080 // Restricts visibility 081 } 082 083 protected void setParser(XMLStreamReader parser) { 084 this.parser = parser; 085 } 086 087 protected void throwException(String msg, Throwable th) throws XMLStreamException { 088 throw new OsmParsingException(msg, parser.getLocation(), th); 089 } 090 091 protected void throwException(String msg) throws XMLStreamException { 092 throw new OsmParsingException(msg, parser.getLocation()); 093 } 094 095 protected void parse() throws XMLStreamException { 096 int event = parser.getEventType(); 097 while (true) { 098 if (event == XMLStreamConstants.START_ELEMENT) { 099 parseRoot(); 100 } else if (event == XMLStreamConstants.END_ELEMENT) 101 return; 102 if (parser.hasNext()) { 103 event = parser.next(); 104 } else { 105 break; 106 } 107 } 108 parser.close(); 109 } 110 111 protected void parseRoot() throws XMLStreamException { 112 if ("osm".equals(parser.getLocalName())) { 113 parseOsm(); 114 } else { 115 parseUnknown(); 116 } 117 } 118 119 private void parseOsm() throws XMLStreamException { 120 String v = parser.getAttributeValue(null, "version"); 121 if (v == null) { 122 throwException(tr("Missing mandatory attribute ''{0}''.", "version")); 123 } 124 if (!"0.6".equals(v)) { 125 throwException(tr("Unsupported version: {0}", v)); 126 } 127 ds.setVersion(v); 128 String upload = parser.getAttributeValue(null, "upload"); 129 if (upload != null) { 130 ds.setUploadDiscouraged(!Boolean.parseBoolean(upload)); 131 } 132 String generator = parser.getAttributeValue(null, "generator"); 133 Long uploadChangesetId = null; 134 if (parser.getAttributeValue(null, "upload-changeset") != null) { 135 uploadChangesetId = getLong("upload-changeset"); 136 } 137 while (true) { 138 int event = parser.next(); 139 140 if (cancel) { 141 cancel = false; 142 throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation()); 143 } 144 145 if (event == XMLStreamConstants.START_ELEMENT) { 146 switch (parser.getLocalName()) { 147 case "bounds": 148 parseBounds(generator); 149 break; 150 case "node": 151 parseNode(); 152 break; 153 case "way": 154 parseWay(); 155 break; 156 case "relation": 157 parseRelation(); 158 break; 159 case "changeset": 160 parseChangeset(uploadChangesetId); 161 break; 162 default: 163 parseUnknown(); 164 } 165 } else if (event == XMLStreamConstants.END_ELEMENT) 166 return; 167 } 168 } 169 170 private void parseBounds(String generator) throws XMLStreamException { 171 String minlon = parser.getAttributeValue(null, "minlon"); 172 String minlat = parser.getAttributeValue(null, "minlat"); 173 String maxlon = parser.getAttributeValue(null, "maxlon"); 174 String maxlat = parser.getAttributeValue(null, "maxlat"); 175 String origin = parser.getAttributeValue(null, "origin"); 176 if (minlon != null && maxlon != null && minlat != null && maxlat != null) { 177 if (origin == null) { 178 origin = generator; 179 } 180 Bounds bounds = new Bounds( 181 Double.parseDouble(minlat), Double.parseDouble(minlon), 182 Double.parseDouble(maxlat), Double.parseDouble(maxlon)); 183 if (bounds.isOutOfTheWorld()) { 184 Bounds copy = new Bounds(bounds); 185 bounds.normalize(); 186 Main.info("Bbox " + copy + " is out of the world, normalized to " + bounds); 187 } 188 DataSource src = new DataSource(bounds, origin); 189 ds.dataSources.add(src); 190 } else { 191 throwException(tr("Missing mandatory attributes on element ''bounds''. " + 192 "Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.", 193 minlon, minlat, maxlon, maxlat, origin 194 )); 195 } 196 jumpToEnd(); 197 } 198 199 protected Node parseNode() throws XMLStreamException { 200 NodeData nd = new NodeData(); 201 String lat = parser.getAttributeValue(null, "lat"); 202 String lon = parser.getAttributeValue(null, "lon"); 203 if (lat != null && lon != null) { 204 nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon))); 205 } 206 readCommon(nd); 207 Node n = new Node(nd.getId(), nd.getVersion()); 208 n.setVisible(nd.isVisible()); 209 n.load(nd); 210 externalIdMap.put(nd.getPrimitiveId(), n); 211 while (true) { 212 int event = parser.next(); 213 if (event == XMLStreamConstants.START_ELEMENT) { 214 if ("tag".equals(parser.getLocalName())) { 215 parseTag(n); 216 } else { 217 parseUnknown(); 218 } 219 } else if (event == XMLStreamConstants.END_ELEMENT) 220 return n; 221 } 222 } 223 224 protected Way parseWay() throws XMLStreamException { 225 WayData wd = new WayData(); 226 readCommon(wd); 227 Way w = new Way(wd.getId(), wd.getVersion()); 228 w.setVisible(wd.isVisible()); 229 w.load(wd); 230 externalIdMap.put(wd.getPrimitiveId(), w); 231 232 Collection<Long> nodeIds = new ArrayList<>(); 233 while (true) { 234 int event = parser.next(); 235 if (event == XMLStreamConstants.START_ELEMENT) { 236 switch (parser.getLocalName()) { 237 case "nd": 238 nodeIds.add(parseWayNode(w)); 239 break; 240 case "tag": 241 parseTag(w); 242 break; 243 default: 244 parseUnknown(); 245 } 246 } else if (event == XMLStreamConstants.END_ELEMENT) { 247 break; 248 } 249 } 250 if (w.isDeleted() && !nodeIds.isEmpty()) { 251 Main.info(tr("Deleted way {0} contains nodes", w.getUniqueId())); 252 nodeIds = new ArrayList<>(); 253 } 254 ways.put(wd.getUniqueId(), nodeIds); 255 return w; 256 } 257 258 private long parseWayNode(Way w) throws XMLStreamException { 259 if (parser.getAttributeValue(null, "ref") == null) { 260 throwException( 261 tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId()) 262 ); 263 } 264 long id = getLong("ref"); 265 if (id == 0) { 266 throwException( 267 tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id) 268 ); 269 } 270 jumpToEnd(); 271 return id; 272 } 273 274 protected Relation parseRelation() throws XMLStreamException { 275 RelationData rd = new RelationData(); 276 readCommon(rd); 277 Relation r = new Relation(rd.getId(), rd.getVersion()); 278 r.setVisible(rd.isVisible()); 279 r.load(rd); 280 externalIdMap.put(rd.getPrimitiveId(), r); 281 282 Collection<RelationMemberData> members = new ArrayList<>(); 283 while (true) { 284 int event = parser.next(); 285 if (event == XMLStreamConstants.START_ELEMENT) { 286 switch (parser.getLocalName()) { 287 case "member": 288 members.add(parseRelationMember(r)); 289 break; 290 case "tag": 291 parseTag(r); 292 break; 293 default: 294 parseUnknown(); 295 } 296 } else if (event == XMLStreamConstants.END_ELEMENT) { 297 break; 298 } 299 } 300 if (r.isDeleted() && !members.isEmpty()) { 301 Main.info(tr("Deleted relation {0} contains members", r.getUniqueId())); 302 members = new ArrayList<>(); 303 } 304 relations.put(rd.getUniqueId(), members); 305 return r; 306 } 307 308 private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException { 309 String role = null; 310 OsmPrimitiveType type = null; 311 long id = 0; 312 String value = parser.getAttributeValue(null, "ref"); 313 if (value == null) { 314 throwException(tr("Missing attribute ''ref'' on member in relation {0}.", r.getUniqueId())); 315 } 316 try { 317 id = Long.parseLong(value); 318 } catch (NumberFormatException e) { 319 throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()), 320 value), e); 321 } 322 value = parser.getAttributeValue(null, "type"); 323 if (value == null) { 324 throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId()))); 325 } 326 try { 327 type = OsmPrimitiveType.fromApiTypeName(value); 328 } catch (IllegalArgumentException e) { 329 throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.", 330 Long.toString(id), Long.toString(r.getUniqueId()), value), e); 331 } 332 value = parser.getAttributeValue(null, "role"); 333 role = value; 334 335 if (id == 0) { 336 throwException(tr("Incomplete <member> specification with ref=0")); 337 } 338 jumpToEnd(); 339 return new RelationMemberData(role, type, id); 340 } 341 342 private void parseChangeset(Long uploadChangesetId) throws XMLStreamException { 343 344 Long id = null; 345 if (parser.getAttributeValue(null, "id") != null) { 346 id = getLong("id"); 347 } 348 // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value 349 if (id == uploadChangesetId || (id != null && id.equals(uploadChangesetId))) { 350 uploadChangeset = new Changeset(id != null ? id.intValue() : 0); 351 while (true) { 352 int event = parser.next(); 353 if (event == XMLStreamConstants.START_ELEMENT) { 354 if ("tag".equals(parser.getLocalName())) { 355 parseTag(uploadChangeset); 356 } else { 357 parseUnknown(); 358 } 359 } else if (event == XMLStreamConstants.END_ELEMENT) 360 return; 361 } 362 } else { 363 jumpToEnd(false); 364 } 365 } 366 367 private void parseTag(Tagged t) throws XMLStreamException { 368 String key = parser.getAttributeValue(null, "k"); 369 String value = parser.getAttributeValue(null, "v"); 370 if (key == null || value == null) { 371 throwException(tr("Missing key or value attribute in tag.")); 372 } 373 t.put(key.intern(), value.intern()); 374 jumpToEnd(); 375 } 376 377 protected void parseUnknown(boolean printWarning) throws XMLStreamException { 378 if (printWarning) { 379 Main.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", parser.getLocalName())); 380 } 381 while (true) { 382 int event = parser.next(); 383 if (event == XMLStreamConstants.START_ELEMENT) { 384 parseUnknown(false); /* no more warning for inner elements */ 385 } else if (event == XMLStreamConstants.END_ELEMENT) 386 return; 387 } 388 } 389 390 protected void parseUnknown() throws XMLStreamException { 391 parseUnknown(true); 392 } 393 394 /** 395 * When cursor is at the start of an element, moves it to the end tag of that element. 396 * Nested content is skipped. 397 * 398 * This is basically the same code as parseUnknown(), except for the warnings, which 399 * are displayed for inner elements and not at top level. 400 * @throws XMLStreamException if there is an error processing the underlying XML source 401 */ 402 private void jumpToEnd(boolean printWarning) throws XMLStreamException { 403 while (true) { 404 int event = parser.next(); 405 if (event == XMLStreamConstants.START_ELEMENT) { 406 parseUnknown(printWarning); 407 } else if (event == XMLStreamConstants.END_ELEMENT) 408 return; 409 } 410 } 411 412 private void jumpToEnd() throws XMLStreamException { 413 jumpToEnd(true); 414 } 415 416 private User createUser(String uid, String name) throws XMLStreamException { 417 if (uid == null) { 418 if (name == null) 419 return null; 420 return User.createLocalUser(name); 421 } 422 try { 423 long id = Long.parseLong(uid); 424 return User.createOsmUser(id, name); 425 } catch (NumberFormatException e) { 426 throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e); 427 } 428 return null; 429 } 430 431 /** 432 * Read out the common attributes and put them into current OsmPrimitive. 433 * @throws XMLStreamException if there is an error processing the underlying XML source 434 */ 435 private void readCommon(PrimitiveData current) throws XMLStreamException { 436 current.setId(getLong("id")); 437 if (current.getUniqueId() == 0) { 438 throwException(tr("Illegal object with ID=0.")); 439 } 440 441 String time = parser.getAttributeValue(null, "timestamp"); 442 if (time != null && !time.isEmpty()) { 443 current.setRawTimestamp((int) (DateUtils.tsFromString(time)/1000)); 444 } 445 446 String user = parser.getAttributeValue(null, "user"); 447 String uid = parser.getAttributeValue(null, "uid"); 448 current.setUser(createUser(uid, user)); 449 450 String visible = parser.getAttributeValue(null, "visible"); 451 if (visible != null) { 452 current.setVisible(Boolean.parseBoolean(visible)); 453 } 454 455 String versionString = parser.getAttributeValue(null, "version"); 456 int version = 0; 457 if (versionString != null) { 458 try { 459 version = Integer.parseInt(versionString); 460 } catch (NumberFormatException e) { 461 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", 462 Long.toString(current.getUniqueId()), versionString), e); 463 } 464 switch (ds.getVersion()) { 465 case "0.6": 466 if (version <= 0 && !current.isNew()) { 467 throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.", 468 Long.toString(current.getUniqueId()), versionString)); 469 } else if (version < 0 && current.isNew()) { 470 Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.", 471 current.getUniqueId(), version, 0, "0.6")); 472 version = 0; 473 } 474 break; 475 default: 476 // should not happen. API version has been checked before 477 throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion())); 478 } 479 } else { 480 // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6 481 if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) { 482 throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId()))); 483 } 484 } 485 current.setVersion(version); 486 487 String action = parser.getAttributeValue(null, "action"); 488 if (action == null) { 489 // do nothing 490 } else if ("delete".equals(action)) { 491 current.setDeleted(true); 492 current.setModified(current.isVisible()); 493 } else if ("modify".equals(action)) { 494 current.setModified(true); 495 } 496 497 String v = parser.getAttributeValue(null, "changeset"); 498 if (v == null) { 499 current.setChangesetId(0); 500 } else { 501 try { 502 current.setChangesetId(Integer.parseInt(v)); 503 } catch (IllegalArgumentException e) { 504 Main.debug(e.getMessage()); 505 if (current.isNew()) { 506 // for a new primitive we just log a warning 507 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", 508 v, current.getUniqueId())); 509 current.setChangesetId(0); 510 } else { 511 // for an existing primitive this is a problem 512 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e); 513 } 514 } catch (IllegalStateException e) { 515 // thrown for positive changeset id on new primitives 516 Main.info(e.getMessage()); 517 current.setChangesetId(0); 518 } 519 if (current.getChangesetId() <= 0) { 520 if (current.isNew()) { 521 // for a new primitive we just log a warning 522 Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.", 523 v, current.getUniqueId())); 524 current.setChangesetId(0); 525 } else { 526 // for an existing primitive this is a problem 527 throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v)); 528 } 529 } 530 } 531 } 532 533 private long getLong(String name) throws XMLStreamException { 534 String value = parser.getAttributeValue(null, name); 535 if (value == null) { 536 throwException(tr("Missing required attribute ''{0}''.", name)); 537 } 538 try { 539 return Long.parseLong(value); 540 } catch (NumberFormatException e) { 541 throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.", name, value), e); 542 } 543 return 0; // should not happen 544 } 545 546 private static class OsmParsingException extends XMLStreamException { 547 548 OsmParsingException(String msg, Location location) { 549 super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */ 550 this.location = location; 551 } 552 553 OsmParsingException(String msg, Location location, Throwable th) { 554 super(msg, th); 555 this.location = location; 556 } 557 558 @Override 559 public String getMessage() { 560 String msg = super.getMessage(); 561 if (msg == null) { 562 msg = getClass().getName(); 563 } 564 if (getLocation() == null) 565 return msg; 566 msg += ' ' + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber()); 567 int offset = getLocation().getCharacterOffset(); 568 if (offset > -1) { 569 msg += ". "+ tr("{0} bytes have been read", offset); 570 } 571 return msg; 572 } 573 } 574 575 /** 576 * Exception thrown after user cancelation. 577 */ 578 private static final class OsmParsingCanceledException extends OsmParsingException implements ImportCancelException { 579 /** 580 * Constructs a new {@code OsmParsingCanceledException}. 581 * @param msg The error message 582 * @param location The parser location 583 */ 584 OsmParsingCanceledException(String msg, Location location) { 585 super(msg, location); 586 } 587 } 588 589 protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 590 if (progressMonitor == null) { 591 progressMonitor = NullProgressMonitor.INSTANCE; 592 } 593 ProgressMonitor.CancelListener cancelListener = new ProgressMonitor.CancelListener() { 594 @Override public void operationCanceled() { 595 cancel = true; 596 } 597 }; 598 progressMonitor.addCancelListener(cancelListener); 599 CheckParameterUtil.ensureParameterNotNull(source, "source"); 600 try { 601 progressMonitor.beginTask(tr("Prepare OSM data...", 2)); 602 progressMonitor.indeterminateSubTask(tr("Parsing OSM data...")); 603 604 try (InputStreamReader ir = UTFInputStreamReader.create(source)) { 605 XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir); 606 setParser(parser); 607 parse(); 608 } 609 progressMonitor.worked(1); 610 611 progressMonitor.indeterminateSubTask(tr("Preparing data set...")); 612 prepareDataSet(); 613 progressMonitor.worked(1); 614 615 // iterate over registered postprocessors and give them each a chance 616 // to modify the dataset we have just loaded. 617 if (postprocessors != null) { 618 for (OsmServerReadPostprocessor pp : postprocessors) { 619 pp.postprocessDataSet(getDataSet(), progressMonitor); 620 } 621 } 622 return getDataSet(); 623 } catch (IllegalDataException e) { 624 throw e; 625 } catch (OsmParsingException e) { 626 throw new IllegalDataException(e.getMessage(), e); 627 } catch (XMLStreamException e) { 628 String msg = e.getMessage(); 629 Pattern p = Pattern.compile("Message: (.+)"); 630 Matcher m = p.matcher(msg); 631 if (m.find()) { 632 msg = m.group(1); 633 } 634 if (e.getLocation() != null) 635 throw new IllegalDataException(tr("Line {0} column {1}: ", 636 e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e); 637 else 638 throw new IllegalDataException(msg, e); 639 } catch (Exception e) { 640 throw new IllegalDataException(e); 641 } finally { 642 progressMonitor.finishTask(); 643 progressMonitor.removeCancelListener(cancelListener); 644 } 645 } 646 647 /** 648 * Parse the given input source and return the dataset. 649 * 650 * @param source the source input stream. Must not be null. 651 * @param progressMonitor the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed 652 * 653 * @return the dataset with the parsed data 654 * @throws IllegalDataException if an error was found while parsing the data from the source 655 * @throws IllegalArgumentException if source is null 656 */ 657 public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException { 658 return new OsmReader().doParseDataSet(source, progressMonitor); 659 } 660}