001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.InputStream;
007import java.io.InputStreamReader;
008import java.text.MessageFormat;
009import java.util.ArrayList;
010import java.util.Collection;
011import java.util.List;
012import java.util.regex.Matcher;
013import java.util.regex.Pattern;
014
015import javax.xml.stream.Location;
016import javax.xml.stream.XMLInputFactory;
017import javax.xml.stream.XMLStreamConstants;
018import javax.xml.stream.XMLStreamException;
019import javax.xml.stream.XMLStreamReader;
020
021import org.openstreetmap.josm.Main;
022import org.openstreetmap.josm.data.Bounds;
023import org.openstreetmap.josm.data.DataSource;
024import org.openstreetmap.josm.data.coor.LatLon;
025import org.openstreetmap.josm.data.osm.Changeset;
026import org.openstreetmap.josm.data.osm.DataSet;
027import org.openstreetmap.josm.data.osm.Node;
028import org.openstreetmap.josm.data.osm.NodeData;
029import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
030import org.openstreetmap.josm.data.osm.PrimitiveData;
031import org.openstreetmap.josm.data.osm.Relation;
032import org.openstreetmap.josm.data.osm.RelationData;
033import org.openstreetmap.josm.data.osm.RelationMemberData;
034import org.openstreetmap.josm.data.osm.Tagged;
035import org.openstreetmap.josm.data.osm.User;
036import org.openstreetmap.josm.data.osm.Way;
037import org.openstreetmap.josm.data.osm.WayData;
038import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
039import org.openstreetmap.josm.gui.progress.ProgressMonitor;
040import org.openstreetmap.josm.tools.CheckParameterUtil;
041import org.openstreetmap.josm.tools.date.DateUtils;
042
043/**
044 * Parser for the Osm Api. Read from an input stream and construct a dataset out of it.
045 *
046 * For each xml element, there is a dedicated method.
047 * The XMLStreamReader cursor points to the start of the element, when the method is
048 * entered, and it must point to the end of the same element, when it is exited.
049 */
050public class OsmReader extends AbstractReader {
051
052    protected XMLStreamReader parser;
053
054    protected boolean cancel;
055
056    /** Used by plugins to register themselves as data postprocessors. */
057    private static volatile List<OsmServerReadPostprocessor> postprocessors;
058
059    /** register a new postprocessor */
060    public static void registerPostprocessor(OsmServerReadPostprocessor pp) {
061        if (postprocessors == null) {
062            postprocessors = new ArrayList<>();
063        }
064        postprocessors.add(pp);
065    }
066
067    /** deregister a postprocessor previously registered with registerPostprocessor */
068    public static void deregisterPostprocessor(OsmServerReadPostprocessor pp) {
069        if (postprocessors != null) {
070            postprocessors.remove(pp);
071        }
072    }
073
074    /**
075     * constructor (for private and subclasses use only)
076     *
077     * @see #parseDataSet(InputStream, ProgressMonitor)
078     */
079    protected OsmReader() {
080        // Restricts visibility
081    }
082
083    protected void setParser(XMLStreamReader parser) {
084        this.parser = parser;
085    }
086
087    protected void throwException(String msg, Throwable th) throws XMLStreamException {
088        throw new OsmParsingException(msg, parser.getLocation(), th);
089    }
090
091    protected void throwException(String msg) throws XMLStreamException {
092        throw new OsmParsingException(msg, parser.getLocation());
093    }
094
095    protected void parse() throws XMLStreamException {
096        int event = parser.getEventType();
097        while (true) {
098            if (event == XMLStreamConstants.START_ELEMENT) {
099                parseRoot();
100            } else if (event == XMLStreamConstants.END_ELEMENT)
101                return;
102            if (parser.hasNext()) {
103                event = parser.next();
104            } else {
105                break;
106            }
107        }
108        parser.close();
109    }
110
111    protected void parseRoot() throws XMLStreamException {
112        if ("osm".equals(parser.getLocalName())) {
113            parseOsm();
114        } else {
115            parseUnknown();
116        }
117    }
118
119    private void parseOsm() throws XMLStreamException {
120        String v = parser.getAttributeValue(null, "version");
121        if (v == null) {
122            throwException(tr("Missing mandatory attribute ''{0}''.", "version"));
123        }
124        if (!"0.6".equals(v)) {
125            throwException(tr("Unsupported version: {0}", v));
126        }
127        ds.setVersion(v);
128        String upload = parser.getAttributeValue(null, "upload");
129        if (upload != null) {
130            ds.setUploadDiscouraged(!Boolean.parseBoolean(upload));
131        }
132        String generator = parser.getAttributeValue(null, "generator");
133        Long uploadChangesetId = null;
134        if (parser.getAttributeValue(null, "upload-changeset") != null) {
135            uploadChangesetId = getLong("upload-changeset");
136        }
137        while (true) {
138            int event = parser.next();
139
140            if (cancel) {
141                cancel = false;
142                throw new OsmParsingCanceledException(tr("Reading was canceled"), parser.getLocation());
143            }
144
145            if (event == XMLStreamConstants.START_ELEMENT) {
146                switch (parser.getLocalName()) {
147                case "bounds":
148                    parseBounds(generator);
149                    break;
150                case "node":
151                    parseNode();
152                    break;
153                case "way":
154                    parseWay();
155                    break;
156                case "relation":
157                    parseRelation();
158                    break;
159                case "changeset":
160                    parseChangeset(uploadChangesetId);
161                    break;
162                default:
163                    parseUnknown();
164                }
165            } else if (event == XMLStreamConstants.END_ELEMENT)
166                return;
167        }
168    }
169
170    private void parseBounds(String generator) throws XMLStreamException {
171        String minlon = parser.getAttributeValue(null, "minlon");
172        String minlat = parser.getAttributeValue(null, "minlat");
173        String maxlon = parser.getAttributeValue(null, "maxlon");
174        String maxlat = parser.getAttributeValue(null, "maxlat");
175        String origin = parser.getAttributeValue(null, "origin");
176        if (minlon != null && maxlon != null && minlat != null && maxlat != null) {
177            if (origin == null) {
178                origin = generator;
179            }
180            Bounds bounds = new Bounds(
181                    Double.parseDouble(minlat), Double.parseDouble(minlon),
182                    Double.parseDouble(maxlat), Double.parseDouble(maxlon));
183            if (bounds.isOutOfTheWorld()) {
184                Bounds copy = new Bounds(bounds);
185                bounds.normalize();
186                Main.info("Bbox " + copy + " is out of the world, normalized to " + bounds);
187            }
188            DataSource src = new DataSource(bounds, origin);
189            ds.dataSources.add(src);
190        } else {
191            throwException(tr("Missing mandatory attributes on element ''bounds''. " +
192                    "Got minlon=''{0}'',minlat=''{1}'',maxlon=''{3}'',maxlat=''{4}'', origin=''{5}''.",
193                    minlon, minlat, maxlon, maxlat, origin
194            ));
195        }
196        jumpToEnd();
197    }
198
199    protected Node parseNode() throws XMLStreamException {
200        NodeData nd = new NodeData();
201        String lat = parser.getAttributeValue(null, "lat");
202        String lon = parser.getAttributeValue(null, "lon");
203        if (lat != null && lon != null) {
204            nd.setCoor(new LatLon(Double.parseDouble(lat), Double.parseDouble(lon)));
205        }
206        readCommon(nd);
207        Node n = new Node(nd.getId(), nd.getVersion());
208        n.setVisible(nd.isVisible());
209        n.load(nd);
210        externalIdMap.put(nd.getPrimitiveId(), n);
211        while (true) {
212            int event = parser.next();
213            if (event == XMLStreamConstants.START_ELEMENT) {
214                if ("tag".equals(parser.getLocalName())) {
215                    parseTag(n);
216                } else {
217                    parseUnknown();
218                }
219            } else if (event == XMLStreamConstants.END_ELEMENT)
220                return n;
221        }
222    }
223
224    protected Way parseWay() throws XMLStreamException {
225        WayData wd = new WayData();
226        readCommon(wd);
227        Way w = new Way(wd.getId(), wd.getVersion());
228        w.setVisible(wd.isVisible());
229        w.load(wd);
230        externalIdMap.put(wd.getPrimitiveId(), w);
231
232        Collection<Long> nodeIds = new ArrayList<>();
233        while (true) {
234            int event = parser.next();
235            if (event == XMLStreamConstants.START_ELEMENT) {
236                switch (parser.getLocalName()) {
237                case "nd":
238                    nodeIds.add(parseWayNode(w));
239                    break;
240                case "tag":
241                    parseTag(w);
242                    break;
243                default:
244                    parseUnknown();
245                }
246            } else if (event == XMLStreamConstants.END_ELEMENT) {
247                break;
248            }
249        }
250        if (w.isDeleted() && !nodeIds.isEmpty()) {
251            Main.info(tr("Deleted way {0} contains nodes", w.getUniqueId()));
252            nodeIds = new ArrayList<>();
253        }
254        ways.put(wd.getUniqueId(), nodeIds);
255        return w;
256    }
257
258    private long parseWayNode(Way w) throws XMLStreamException {
259        if (parser.getAttributeValue(null, "ref") == null) {
260            throwException(
261                    tr("Missing mandatory attribute ''{0}'' on <nd> of way {1}.", "ref", w.getUniqueId())
262            );
263        }
264        long id = getLong("ref");
265        if (id == 0) {
266            throwException(
267                    tr("Illegal value of attribute ''ref'' of element <nd>. Got {0}.", id)
268            );
269        }
270        jumpToEnd();
271        return id;
272    }
273
274    protected Relation parseRelation() throws XMLStreamException {
275        RelationData rd = new RelationData();
276        readCommon(rd);
277        Relation r = new Relation(rd.getId(), rd.getVersion());
278        r.setVisible(rd.isVisible());
279        r.load(rd);
280        externalIdMap.put(rd.getPrimitiveId(), r);
281
282        Collection<RelationMemberData> members = new ArrayList<>();
283        while (true) {
284            int event = parser.next();
285            if (event == XMLStreamConstants.START_ELEMENT) {
286                switch (parser.getLocalName()) {
287                case "member":
288                    members.add(parseRelationMember(r));
289                    break;
290                case "tag":
291                    parseTag(r);
292                    break;
293                default:
294                    parseUnknown();
295                }
296            } else if (event == XMLStreamConstants.END_ELEMENT) {
297                break;
298            }
299        }
300        if (r.isDeleted() && !members.isEmpty()) {
301            Main.info(tr("Deleted relation {0} contains members", r.getUniqueId()));
302            members = new ArrayList<>();
303        }
304        relations.put(rd.getUniqueId(), members);
305        return r;
306    }
307
308    private RelationMemberData parseRelationMember(Relation r) throws XMLStreamException {
309        String role = null;
310        OsmPrimitiveType type = null;
311        long id = 0;
312        String value = parser.getAttributeValue(null, "ref");
313        if (value == null) {
314            throwException(tr("Missing attribute ''ref'' on member in relation {0}.", r.getUniqueId()));
315        }
316        try {
317            id = Long.parseLong(value);
318        } catch (NumberFormatException e) {
319            throwException(tr("Illegal value for attribute ''ref'' on member in relation {0}. Got {1}", Long.toString(r.getUniqueId()),
320                    value), e);
321        }
322        value = parser.getAttributeValue(null, "type");
323        if (value == null) {
324            throwException(tr("Missing attribute ''type'' on member {0} in relation {1}.", Long.toString(id), Long.toString(r.getUniqueId())));
325        }
326        try {
327            type = OsmPrimitiveType.fromApiTypeName(value);
328        } catch (IllegalArgumentException e) {
329            throwException(tr("Illegal value for attribute ''type'' on member {0} in relation {1}. Got {2}.",
330                    Long.toString(id), Long.toString(r.getUniqueId()), value), e);
331        }
332        value = parser.getAttributeValue(null, "role");
333        role = value;
334
335        if (id == 0) {
336            throwException(tr("Incomplete <member> specification with ref=0"));
337        }
338        jumpToEnd();
339        return new RelationMemberData(role, type, id);
340    }
341
342    private void parseChangeset(Long uploadChangesetId) throws XMLStreamException {
343
344        Long id = null;
345        if (parser.getAttributeValue(null, "id") != null) {
346            id = getLong("id");
347        }
348        // Read changeset info if neither upload-changeset nor id are set, or if they are both set to the same value
349        if (id == uploadChangesetId || (id != null && id.equals(uploadChangesetId))) {
350            uploadChangeset = new Changeset(id != null ? id.intValue() : 0);
351            while (true) {
352                int event = parser.next();
353                if (event == XMLStreamConstants.START_ELEMENT) {
354                    if ("tag".equals(parser.getLocalName())) {
355                        parseTag(uploadChangeset);
356                    } else {
357                        parseUnknown();
358                    }
359                } else if (event == XMLStreamConstants.END_ELEMENT)
360                    return;
361            }
362        } else {
363            jumpToEnd(false);
364        }
365    }
366
367    private void parseTag(Tagged t) throws XMLStreamException {
368        String key = parser.getAttributeValue(null, "k");
369        String value = parser.getAttributeValue(null, "v");
370        if (key == null || value == null) {
371            throwException(tr("Missing key or value attribute in tag."));
372        }
373        t.put(key.intern(), value.intern());
374        jumpToEnd();
375    }
376
377    protected void parseUnknown(boolean printWarning) throws XMLStreamException {
378        if (printWarning) {
379            Main.info(tr("Undefined element ''{0}'' found in input stream. Skipping.", parser.getLocalName()));
380        }
381        while (true) {
382            int event = parser.next();
383            if (event == XMLStreamConstants.START_ELEMENT) {
384                parseUnknown(false); /* no more warning for inner elements */
385            } else if (event == XMLStreamConstants.END_ELEMENT)
386                return;
387        }
388    }
389
390    protected void parseUnknown() throws XMLStreamException {
391        parseUnknown(true);
392    }
393
394    /**
395     * When cursor is at the start of an element, moves it to the end tag of that element.
396     * Nested content is skipped.
397     *
398     * This is basically the same code as parseUnknown(), except for the warnings, which
399     * are displayed for inner elements and not at top level.
400     * @throws XMLStreamException if there is an error processing the underlying XML source
401     */
402    private void jumpToEnd(boolean printWarning) throws XMLStreamException {
403        while (true) {
404            int event = parser.next();
405            if (event == XMLStreamConstants.START_ELEMENT) {
406                parseUnknown(printWarning);
407            } else if (event == XMLStreamConstants.END_ELEMENT)
408                return;
409        }
410    }
411
412    private void jumpToEnd() throws XMLStreamException {
413        jumpToEnd(true);
414    }
415
416    private User createUser(String uid, String name) throws XMLStreamException {
417        if (uid == null) {
418            if (name == null)
419                return null;
420            return User.createLocalUser(name);
421        }
422        try {
423            long id = Long.parseLong(uid);
424            return User.createOsmUser(id, name);
425        } catch (NumberFormatException e) {
426            throwException(MessageFormat.format("Illegal value for attribute ''uid''. Got ''{0}''.", uid), e);
427        }
428        return null;
429    }
430
431    /**
432     * Read out the common attributes and put them into current OsmPrimitive.
433     * @throws XMLStreamException if there is an error processing the underlying XML source
434     */
435    private void readCommon(PrimitiveData current) throws XMLStreamException {
436        current.setId(getLong("id"));
437        if (current.getUniqueId() == 0) {
438            throwException(tr("Illegal object with ID=0."));
439        }
440
441        String time = parser.getAttributeValue(null, "timestamp");
442        if (time != null && !time.isEmpty()) {
443            current.setRawTimestamp((int) (DateUtils.tsFromString(time)/1000));
444        }
445
446        String user = parser.getAttributeValue(null, "user");
447        String uid = parser.getAttributeValue(null, "uid");
448        current.setUser(createUser(uid, user));
449
450        String visible = parser.getAttributeValue(null, "visible");
451        if (visible != null) {
452            current.setVisible(Boolean.parseBoolean(visible));
453        }
454
455        String versionString = parser.getAttributeValue(null, "version");
456        int version = 0;
457        if (versionString != null) {
458            try {
459                version = Integer.parseInt(versionString);
460            } catch (NumberFormatException e) {
461                throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
462                        Long.toString(current.getUniqueId()), versionString), e);
463            }
464            switch (ds.getVersion()) {
465            case "0.6":
466                if (version <= 0 && !current.isNew()) {
467                    throwException(tr("Illegal value for attribute ''version'' on OSM primitive with ID {0}. Got {1}.",
468                            Long.toString(current.getUniqueId()), versionString));
469                } else if (version < 0 && current.isNew()) {
470                    Main.warn(tr("Normalizing value of attribute ''version'' of element {0} to {2}, API version is ''{3}''. Got {1}.",
471                            current.getUniqueId(), version, 0, "0.6"));
472                    version = 0;
473                }
474                break;
475            default:
476                // should not happen. API version has been checked before
477                throwException(tr("Unknown or unsupported API version. Got {0}.", ds.getVersion()));
478            }
479        } else {
480            // version expected for OSM primitives with an id assigned by the server (id > 0), since API 0.6
481            if (!current.isNew() && ds.getVersion() != null && "0.6".equals(ds.getVersion())) {
482                throwException(tr("Missing attribute ''version'' on OSM primitive with ID {0}.", Long.toString(current.getUniqueId())));
483            }
484        }
485        current.setVersion(version);
486
487        String action = parser.getAttributeValue(null, "action");
488        if (action == null) {
489            // do nothing
490        } else if ("delete".equals(action)) {
491            current.setDeleted(true);
492            current.setModified(current.isVisible());
493        } else if ("modify".equals(action)) {
494            current.setModified(true);
495        }
496
497        String v = parser.getAttributeValue(null, "changeset");
498        if (v == null) {
499            current.setChangesetId(0);
500        } else {
501            try {
502                current.setChangesetId(Integer.parseInt(v));
503            } catch (IllegalArgumentException e) {
504                Main.debug(e.getMessage());
505                if (current.isNew()) {
506                    // for a new primitive we just log a warning
507                    Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
508                            v, current.getUniqueId()));
509                    current.setChangesetId(0);
510                } else {
511                    // for an existing primitive this is a problem
512                    throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v), e);
513                }
514            } catch (IllegalStateException e) {
515                // thrown for positive changeset id on new primitives
516                Main.info(e.getMessage());
517                current.setChangesetId(0);
518            }
519            if (current.getChangesetId() <= 0) {
520                if (current.isNew()) {
521                    // for a new primitive we just log a warning
522                    Main.info(tr("Illegal value for attribute ''changeset'' on new object {1}. Got {0}. Resetting to 0.",
523                            v, current.getUniqueId()));
524                    current.setChangesetId(0);
525                } else {
526                    // for an existing primitive this is a problem
527                    throwException(tr("Illegal value for attribute ''changeset''. Got {0}.", v));
528                }
529            }
530        }
531    }
532
533    private long getLong(String name) throws XMLStreamException {
534        String value = parser.getAttributeValue(null, name);
535        if (value == null) {
536            throwException(tr("Missing required attribute ''{0}''.", name));
537        }
538        try {
539            return Long.parseLong(value);
540        } catch (NumberFormatException e) {
541            throwException(tr("Illegal long value for attribute ''{0}''. Got ''{1}''.", name, value), e);
542        }
543        return 0; // should not happen
544    }
545
546    private static class OsmParsingException extends XMLStreamException {
547
548        OsmParsingException(String msg, Location location) {
549            super(msg); /* cannot use super(msg, location) because it messes with the message preventing localization */
550            this.location = location;
551        }
552
553        OsmParsingException(String msg, Location location, Throwable th) {
554            super(msg, th);
555            this.location = location;
556        }
557
558        @Override
559        public String getMessage() {
560            String msg = super.getMessage();
561            if (msg == null) {
562                msg = getClass().getName();
563            }
564            if (getLocation() == null)
565                return msg;
566            msg += ' ' + tr("(at line {0}, column {1})", getLocation().getLineNumber(), getLocation().getColumnNumber());
567            int offset = getLocation().getCharacterOffset();
568            if (offset > -1) {
569                msg += ". "+ tr("{0} bytes have been read", offset);
570            }
571            return msg;
572        }
573    }
574
575    /**
576     * Exception thrown after user cancelation.
577     */
578    private static final class OsmParsingCanceledException extends OsmParsingException implements ImportCancelException {
579        /**
580         * Constructs a new {@code OsmParsingCanceledException}.
581         * @param msg The error message
582         * @param location The parser location
583         */
584        OsmParsingCanceledException(String msg, Location location) {
585            super(msg, location);
586        }
587    }
588
589    protected DataSet doParseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
590        if (progressMonitor == null) {
591            progressMonitor = NullProgressMonitor.INSTANCE;
592        }
593        ProgressMonitor.CancelListener cancelListener = new ProgressMonitor.CancelListener() {
594            @Override public void operationCanceled() {
595                cancel = true;
596            }
597        };
598        progressMonitor.addCancelListener(cancelListener);
599        CheckParameterUtil.ensureParameterNotNull(source, "source");
600        try {
601            progressMonitor.beginTask(tr("Prepare OSM data...", 2));
602            progressMonitor.indeterminateSubTask(tr("Parsing OSM data..."));
603
604            try (InputStreamReader ir = UTFInputStreamReader.create(source)) {
605                XMLStreamReader parser = XMLInputFactory.newInstance().createXMLStreamReader(ir);
606                setParser(parser);
607                parse();
608            }
609            progressMonitor.worked(1);
610
611            progressMonitor.indeterminateSubTask(tr("Preparing data set..."));
612            prepareDataSet();
613            progressMonitor.worked(1);
614
615            // iterate over registered postprocessors and give them each a chance
616            // to modify the dataset we have just loaded.
617            if (postprocessors != null) {
618                for (OsmServerReadPostprocessor pp : postprocessors) {
619                    pp.postprocessDataSet(getDataSet(), progressMonitor);
620                }
621            }
622            return getDataSet();
623        } catch (IllegalDataException e) {
624            throw e;
625        } catch (OsmParsingException e) {
626            throw new IllegalDataException(e.getMessage(), e);
627        } catch (XMLStreamException e) {
628            String msg = e.getMessage();
629            Pattern p = Pattern.compile("Message: (.+)");
630            Matcher m = p.matcher(msg);
631            if (m.find()) {
632                msg = m.group(1);
633            }
634            if (e.getLocation() != null)
635                throw new IllegalDataException(tr("Line {0} column {1}: ",
636                        e.getLocation().getLineNumber(), e.getLocation().getColumnNumber()) + msg, e);
637            else
638                throw new IllegalDataException(msg, e);
639        } catch (Exception e) {
640            throw new IllegalDataException(e);
641        } finally {
642            progressMonitor.finishTask();
643            progressMonitor.removeCancelListener(cancelListener);
644        }
645    }
646
647    /**
648     * Parse the given input source and return the dataset.
649     *
650     * @param source the source input stream. Must not be null.
651     * @param progressMonitor  the progress monitor. If null, {@link NullProgressMonitor#INSTANCE} is assumed
652     *
653     * @return the dataset with the parsed data
654     * @throws IllegalDataException if an error was found while parsing the data from the source
655     * @throws IllegalArgumentException if source is null
656     */
657    public static DataSet parseDataSet(InputStream source, ProgressMonitor progressMonitor) throws IllegalDataException {
658        return new OsmReader().doParseDataSet(source, progressMonitor);
659    }
660}