001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005import static org.openstreetmap.josm.tools.I18n.trn;
006
007import java.io.BufferedReader;
008import java.io.BufferedWriter;
009import java.io.IOException;
010import java.io.InputStream;
011import java.io.InputStreamReader;
012import java.io.OutputStream;
013import java.io.OutputStreamWriter;
014import java.io.PrintWriter;
015import java.io.StringReader;
016import java.io.StringWriter;
017import java.net.ConnectException;
018import java.net.HttpURLConnection;
019import java.net.MalformedURLException;
020import java.net.SocketTimeoutException;
021import java.net.URL;
022import java.nio.charset.StandardCharsets;
023import java.util.Collection;
024import java.util.Collections;
025import java.util.HashMap;
026import java.util.List;
027import java.util.Map;
028
029import javax.xml.parsers.ParserConfigurationException;
030
031import org.openstreetmap.josm.Main;
032import org.openstreetmap.josm.data.coor.LatLon;
033import org.openstreetmap.josm.data.notes.Note;
034import org.openstreetmap.josm.data.osm.Changeset;
035import org.openstreetmap.josm.data.osm.IPrimitive;
036import org.openstreetmap.josm.data.osm.OsmPrimitive;
037import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
038import org.openstreetmap.josm.gui.layer.ImageryLayer;
039import org.openstreetmap.josm.gui.layer.Layer;
040import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
041import org.openstreetmap.josm.gui.progress.ProgressMonitor;
042import org.openstreetmap.josm.io.Capabilities.CapabilitiesParser;
043import org.openstreetmap.josm.tools.CheckParameterUtil;
044import org.openstreetmap.josm.tools.Utils;
045import org.openstreetmap.josm.tools.XmlParsingException;
046import org.xml.sax.InputSource;
047import org.xml.sax.SAXException;
048import org.xml.sax.SAXParseException;
049
050/**
051 * Class that encapsulates the communications with the <a href="http://wiki.openstreetmap.org/wiki/API_v0.6">OSM API</a>.<br><br>
052 *
053 * All interaction with the server-side OSM API should go through this class.<br><br>
054 *
055 * It is conceivable to extract this into an interface later and create various
056 * classes implementing the interface, to be able to talk to various kinds of servers.
057 *
058 */
059public class OsmApi extends OsmConnection {
060
061    /**
062     * Maximum number of retries to send a request in case of HTTP 500 errors or timeouts
063     */
064    public static final int DEFAULT_MAX_NUM_RETRIES = 5;
065
066    /**
067     * Maximum number of concurrent download threads, imposed by
068     * <a href="http://wiki.openstreetmap.org/wiki/API_usage_policy#Technical_Usage_Requirements">
069     * OSM API usage policy.</a>
070     * @since 5386
071     */
072    public static final int MAX_DOWNLOAD_THREADS = 2;
073
074    /**
075     * Default URL of the standard OSM API.
076     * @since 5422
077     */
078    public static final String DEFAULT_API_URL = "https://api.openstreetmap.org/api";
079
080    // The collection of instantiated OSM APIs
081    private static Map<String, OsmApi> instances = new HashMap<>();
082
083    private URL url;
084
085    /**
086     * Replies the {@link OsmApi} for a given server URL
087     *
088     * @param serverUrl  the server URL
089     * @return the OsmApi
090     * @throws IllegalArgumentException if serverUrl is null
091     *
092     */
093    public static OsmApi getOsmApi(String serverUrl) {
094        OsmApi api = instances.get(serverUrl);
095        if (api == null) {
096            api = new OsmApi(serverUrl);
097            instances.put(serverUrl, api);
098        }
099        return api;
100    }
101
102    private static String getServerUrlFromPref() {
103        return Main.pref.get("osm-server.url", DEFAULT_API_URL);
104    }
105
106    /**
107     * Replies the {@link OsmApi} for the URL given by the preference <code>osm-server.url</code>
108     *
109     * @return the OsmApi
110     */
111    public static OsmApi getOsmApi() {
112        return getOsmApi(getServerUrlFromPref());
113    }
114
115    /** Server URL */
116    private String serverUrl;
117
118    /** Object describing current changeset */
119    private Changeset changeset;
120
121    /** API version used for server communications */
122    private String version;
123
124    /** API capabilities */
125    private Capabilities capabilities;
126
127    /** true if successfully initialized */
128    private boolean initialized;
129
130    /**
131     * Constructs a new {@code OsmApi} for a specific server URL.
132     *
133     * @param serverUrl the server URL. Must not be null
134     * @throws IllegalArgumentException if serverUrl is null
135     */
136    protected OsmApi(String serverUrl)  {
137        CheckParameterUtil.ensureParameterNotNull(serverUrl, "serverUrl");
138        this.serverUrl = serverUrl;
139    }
140
141    /**
142     * Replies the OSM protocol version we use to talk to the server.
143     * @return protocol version, or null if not yet negotiated.
144     */
145    public String getVersion() {
146        return version;
147    }
148
149    /**
150     * Replies the host name of the server URL.
151     * @return the host name of the server URL, or null if the server URL is malformed.
152     */
153    public String getHost() {
154        String host = null;
155        try {
156            host = (new URL(serverUrl)).getHost();
157        } catch (MalformedURLException e) {
158            Main.warn(e);
159        }
160        return host;
161    }
162
163    private class CapabilitiesCache extends CacheCustomContent<OsmTransferException> {
164
165        private static final String CAPABILITIES = "capabilities";
166
167        private ProgressMonitor monitor;
168        private boolean fastFail;
169
170        CapabilitiesCache(ProgressMonitor monitor, boolean fastFail) {
171            super(CAPABILITIES + getBaseUrl().hashCode(), CacheCustomContent.INTERVAL_WEEKLY);
172            this.monitor = monitor;
173            this.fastFail = fastFail;
174        }
175
176        @Override
177        protected void checkOfflineAccess() {
178            OnlineResource.OSM_API.checkOfflineAccess(getBaseUrl(getServerUrlFromPref(), "0.6")+CAPABILITIES, getServerUrlFromPref());
179        }
180
181        @Override
182        protected byte[] updateData() throws OsmTransferException {
183            return sendRequest("GET", CAPABILITIES, null, monitor, false, fastFail).getBytes(StandardCharsets.UTF_8);
184        }
185    }
186
187    /**
188     * Initializes this component by negotiating a protocol version with the server.
189     *
190     * @param monitor the progress monitor
191     * @throws OsmTransferCanceledException If the initialisation has been cancelled by user.
192     * @throws OsmApiInitializationException If any other exception occurs. Use getCause() to get the original exception.
193     */
194    public void initialize(ProgressMonitor monitor) throws OsmTransferCanceledException, OsmApiInitializationException {
195        initialize(monitor, false);
196    }
197
198    /**
199     * Initializes this component by negotiating a protocol version with the server, with the ability to control the timeout.
200     *
201     * @param monitor the progress monitor
202     * @param fastFail true to request quick initialisation with a small timeout (more likely to throw exception)
203     * @throws OsmTransferCanceledException If the initialisation has been cancelled by user.
204     * @throws OsmApiInitializationException If any other exception occurs. Use getCause() to get the original exception.
205     */
206    public void initialize(ProgressMonitor monitor, boolean fastFail) throws OsmTransferCanceledException, OsmApiInitializationException {
207        if (initialized)
208            return;
209        cancel = false;
210        try {
211            CapabilitiesCache cache = new CapabilitiesCache(monitor, fastFail);
212            try {
213                initializeCapabilities(cache.updateIfRequiredString());
214            } catch (SAXParseException parseException) {
215                // XML parsing may fail if JOSM previously stored a corrupted capabilities document (see #8278)
216                // In that case, force update and try again
217                initializeCapabilities(cache.updateForceString());
218            }
219            if (capabilities == null) {
220                if (Main.isOffline(OnlineResource.OSM_API)) {
221                    Main.warn(tr("{0} not available (offline mode)", tr("OSM API")));
222                } else {
223                    Main.error(tr("Unable to initialize OSM API."));
224                }
225                return;
226            } else if (!capabilities.supportsVersion("0.6")) {
227                Main.error(tr("This version of JOSM is incompatible with the configured server."));
228                Main.error(tr("It supports protocol version 0.6, while the server says it supports {0} to {1}.",
229                        capabilities.get("version", "minimum"), capabilities.get("version", "maximum")));
230                return;
231            } else {
232                version = "0.6";
233                initialized = true;
234            }
235
236            /* This is an interim solution for openstreetmap.org not currently
237             * transmitting their imagery blacklist in the capabilities call.
238             * remove this as soon as openstreetmap.org adds blacklists.
239             * If you want to update this list, please ask for update of
240             * http://trac.openstreetmap.org/ticket/5024
241             * This list should not be maintained by each OSM editor (see #9210) */
242            if (this.serverUrl.matches(".*openstreetmap.org/api.*") && capabilities.getImageryBlacklist().isEmpty()) {
243                capabilities.put("blacklist", "regex", ".*\\.google\\.com/.*");
244                capabilities.put("blacklist", "regex", ".*209\\.85\\.2\\d\\d.*");
245                capabilities.put("blacklist", "regex", ".*209\\.85\\.1[3-9]\\d.*");
246                capabilities.put("blacklist", "regex", ".*209\\.85\\.12[89].*");
247            }
248
249            /* This checks if there are any layers currently displayed that
250             * are now on the blacklist, and removes them. This is a rare
251             * situation - probably only occurs if the user changes the API URL
252             * in the preferences menu. Otherwise they would not have been able
253             * to load the layers in the first place because they would have
254             * been disabled! */
255            if (Main.isDisplayingMapView()) {
256                for (Layer l : Main.map.mapView.getLayersOfType(ImageryLayer.class)) {
257                    if (((ImageryLayer) l).getInfo().isBlacklisted()) {
258                        Main.info(tr("Removed layer {0} because it is not allowed by the configured API.", l.getName()));
259                        Main.main.removeLayer(l);
260                    }
261                }
262            }
263
264        } catch (OsmTransferCanceledException e) {
265            throw e;
266        } catch (OsmTransferException e) {
267            initialized = false;
268            Main.addNetworkError(url, Utils.getRootCause(e));
269            throw new OsmApiInitializationException(e);
270        } catch (Exception e) {
271            initialized = false;
272            throw new OsmApiInitializationException(e);
273        }
274    }
275
276    private synchronized void initializeCapabilities(String xml) throws SAXException, IOException, ParserConfigurationException {
277        if (xml != null) {
278            capabilities = CapabilitiesParser.parse(new InputSource(new StringReader(xml)));
279        }
280    }
281
282    /**
283     * Makes an XML string from an OSM primitive. Uses the OsmWriter class.
284     * @param o the OSM primitive
285     * @param addBody true to generate the full XML, false to only generate the encapsulating tag
286     * @return XML string
287     */
288    private String toXml(IPrimitive o, boolean addBody) {
289        StringWriter swriter = new StringWriter();
290        try (OsmWriter osmWriter = OsmWriterFactory.createOsmWriter(new PrintWriter(swriter), true, version)) {
291            swriter.getBuffer().setLength(0);
292            osmWriter.setWithBody(addBody);
293            osmWriter.setChangeset(changeset);
294            osmWriter.header();
295            o.accept(osmWriter);
296            osmWriter.footer();
297            osmWriter.flush();
298        } catch (IOException e) {
299            Main.warn(e);
300        }
301        return swriter.toString();
302    }
303
304    /**
305     * Makes an XML string from an OSM primitive. Uses the OsmWriter class.
306     * @param s the changeset
307     * @return XML string
308     */
309    private String toXml(Changeset s) {
310        StringWriter swriter = new StringWriter();
311        try (OsmWriter osmWriter = OsmWriterFactory.createOsmWriter(new PrintWriter(swriter), true, version)) {
312            swriter.getBuffer().setLength(0);
313            osmWriter.header();
314            osmWriter.visit(s);
315            osmWriter.footer();
316            osmWriter.flush();
317        } catch (IOException e) {
318            Main.warn(e);
319        }
320        return swriter.toString();
321    }
322
323    private static String getBaseUrl(String serverUrl, String version) {
324        StringBuilder rv = new StringBuilder(serverUrl);
325        if (version != null) {
326            rv.append('/').append(version);
327        }
328        rv.append('/');
329        // this works around a ruby (or lighttpd) bug where two consecutive slashes in
330        // an URL will cause a "404 not found" response.
331        int p;
332        while ((p = rv.indexOf("//", rv.indexOf("://")+2)) > -1) {
333            rv.delete(p, p + 1);
334        }
335        return rv.toString();
336    }
337
338    /**
339     * Returns the base URL for API requests, including the negotiated version number.
340     * @return base URL string
341     */
342    public String getBaseUrl() {
343        return getBaseUrl(serverUrl, version);
344    }
345
346    /**
347     * Creates an OSM primitive on the server. The OsmPrimitive object passed in
348     * is modified by giving it the server-assigned id.
349     *
350     * @param osm the primitive
351     * @param monitor the progress monitor
352     * @throws OsmTransferException if something goes wrong
353     */
354    public void createPrimitive(IPrimitive osm, ProgressMonitor monitor) throws OsmTransferException {
355        String ret = "";
356        try {
357            ensureValidChangeset();
358            initialize(monitor);
359            ret = sendRequest("PUT", OsmPrimitiveType.from(osm).getAPIName()+"/create", toXml(osm, true), monitor);
360            osm.setOsmId(Long.parseLong(ret.trim()), 1);
361            osm.setChangesetId(getChangeset().getId());
362        } catch (NumberFormatException e) {
363            throw new OsmTransferException(tr("Unexpected format of ID replied by the server. Got ''{0}''.", ret), e);
364        }
365    }
366
367    /**
368     * Modifies an OSM primitive on the server.
369     *
370     * @param osm the primitive. Must not be null.
371     * @param monitor the progress monitor
372     * @throws OsmTransferException if something goes wrong
373     */
374    public void modifyPrimitive(IPrimitive osm, ProgressMonitor monitor) throws OsmTransferException {
375        String ret = null;
376        try {
377            ensureValidChangeset();
378            initialize(monitor);
379            // normal mode (0.6 and up) returns new object version.
380            ret = sendRequest("PUT", OsmPrimitiveType.from(osm).getAPIName()+'/' + osm.getId(), toXml(osm, true), monitor);
381            osm.setOsmId(osm.getId(), Integer.parseInt(ret.trim()));
382            osm.setChangesetId(getChangeset().getId());
383            osm.setVisible(true);
384        } catch (NumberFormatException e) {
385            throw new OsmTransferException(tr("Unexpected format of new version of modified primitive ''{0}''. Got ''{1}''.",
386                    osm.getId(), ret), e);
387        }
388    }
389
390    /**
391     * Deletes an OSM primitive on the server.
392     * @param osm the primitive
393     * @param monitor the progress monitor
394     * @throws OsmTransferException if something goes wrong
395     */
396    public void deletePrimitive(OsmPrimitive osm, ProgressMonitor monitor) throws OsmTransferException {
397        ensureValidChangeset();
398        initialize(monitor);
399        // can't use a the individual DELETE method in the 0.6 API. Java doesn't allow
400        // submitting a DELETE request with content, the 0.6 API requires it, however. Falling back
401        // to diff upload.
402        //
403        uploadDiff(Collections.singleton(osm), monitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false));
404    }
405
406    /**
407     * Creates a new changeset based on the keys in <code>changeset</code>. If this
408     * method succeeds, changeset.getId() replies the id the server assigned to the new
409     * changeset
410     *
411     * The changeset must not be null, but its key/value-pairs may be empty.
412     *
413     * @param changeset the changeset toe be created. Must not be null.
414     * @param progressMonitor the progress monitor
415     * @throws OsmTransferException signifying a non-200 return code, or connection errors
416     * @throws IllegalArgumentException if changeset is null
417     */
418    public void openChangeset(Changeset changeset, ProgressMonitor progressMonitor) throws OsmTransferException {
419        CheckParameterUtil.ensureParameterNotNull(changeset, "changeset");
420        try {
421            progressMonitor.beginTask(tr("Creating changeset..."));
422            initialize(progressMonitor);
423            String ret = "";
424            try {
425                ret = sendRequest("PUT", "changeset/create", toXml(changeset), progressMonitor);
426                changeset.setId(Integer.parseInt(ret.trim()));
427                changeset.setOpen(true);
428            } catch (NumberFormatException e) {
429                throw new OsmTransferException(tr("Unexpected format of ID replied by the server. Got ''{0}''.", ret), e);
430            }
431            progressMonitor.setCustomText(tr("Successfully opened changeset {0}", changeset.getId()));
432        } finally {
433            progressMonitor.finishTask();
434        }
435    }
436
437    /**
438     * Updates a changeset with the keys in  <code>changesetUpdate</code>. The changeset must not
439     * be null and id &gt; 0 must be true.
440     *
441     * @param changeset the changeset to update. Must not be null.
442     * @param monitor the progress monitor. If null, uses the {@link NullProgressMonitor#INSTANCE}.
443     *
444     * @throws OsmTransferException if something goes wrong.
445     * @throws IllegalArgumentException if changeset is null
446     * @throws IllegalArgumentException if changeset.getId() &lt;= 0
447     *
448     */
449    public void updateChangeset(Changeset changeset, ProgressMonitor monitor) throws OsmTransferException {
450        CheckParameterUtil.ensureParameterNotNull(changeset, "changeset");
451        if (monitor == null) {
452            monitor = NullProgressMonitor.INSTANCE;
453        }
454        if (changeset.getId() <= 0)
455            throw new IllegalArgumentException(tr("Changeset ID > 0 expected. Got {0}.", changeset.getId()));
456        try {
457            monitor.beginTask(tr("Updating changeset..."));
458            initialize(monitor);
459            monitor.setCustomText(tr("Updating changeset {0}...", changeset.getId()));
460            sendRequest(
461                    "PUT",
462                    "changeset/" + changeset.getId(),
463                    toXml(changeset),
464                    monitor
465            );
466        } catch (ChangesetClosedException e) {
467            e.setSource(ChangesetClosedException.Source.UPDATE_CHANGESET);
468            throw e;
469        } catch (OsmApiException e) {
470            String errorHeader = e.getErrorHeader();
471            if (e.getResponseCode() == HttpURLConnection.HTTP_CONFLICT && ChangesetClosedException.errorHeaderMatchesPattern(errorHeader))
472                throw new ChangesetClosedException(errorHeader, ChangesetClosedException.Source.UPDATE_CHANGESET);
473            throw e;
474        } finally {
475            monitor.finishTask();
476        }
477    }
478
479    /**
480     * Closes a changeset on the server. Sets changeset.setOpen(false) if this operation succeeds.
481     *
482     * @param changeset the changeset to be closed. Must not be null. changeset.getId() &gt; 0 required.
483     * @param monitor the progress monitor. If null, uses {@link NullProgressMonitor#INSTANCE}
484     *
485     * @throws OsmTransferException if something goes wrong.
486     * @throws IllegalArgumentException if changeset is null
487     * @throws IllegalArgumentException if changeset.getId() &lt;= 0
488     */
489    public void closeChangeset(Changeset changeset, ProgressMonitor monitor) throws OsmTransferException {
490        CheckParameterUtil.ensureParameterNotNull(changeset, "changeset");
491        if (monitor == null) {
492            monitor = NullProgressMonitor.INSTANCE;
493        }
494        if (changeset.getId() <= 0)
495            throw new IllegalArgumentException(tr("Changeset ID > 0 expected. Got {0}.", changeset.getId()));
496        try {
497            monitor.beginTask(tr("Closing changeset..."));
498            initialize(monitor);
499            /* send "\r\n" instead of empty string, so we don't send zero payload - works around bugs
500               in proxy software */
501            sendRequest("PUT", "changeset" + "/" + changeset.getId() + "/close", "\r\n", monitor);
502            changeset.setOpen(false);
503        } finally {
504            monitor.finishTask();
505        }
506    }
507
508    /**
509     * Uploads a list of changes in "diff" form to the server.
510     *
511     * @param list the list of changed OSM Primitives
512     * @param  monitor the progress monitor
513     * @return list of processed primitives
514     * @throws OsmTransferException if something is wrong
515     */
516    public Collection<OsmPrimitive> uploadDiff(Collection<? extends OsmPrimitive> list, ProgressMonitor monitor)
517            throws OsmTransferException {
518        try {
519            monitor.beginTask("", list.size() * 2);
520            if (changeset == null)
521                throw new OsmTransferException(tr("No changeset present for diff upload."));
522
523            initialize(monitor);
524
525            // prepare upload request
526            //
527            OsmChangeBuilder changeBuilder = new OsmChangeBuilder(changeset);
528            monitor.subTask(tr("Preparing upload request..."));
529            changeBuilder.start();
530            changeBuilder.append(list);
531            changeBuilder.finish();
532            String diffUploadRequest = changeBuilder.getDocument();
533
534            // Upload to the server
535            //
536            monitor.indeterminateSubTask(
537                    trn("Uploading {0} object...", "Uploading {0} objects...", list.size(), list.size()));
538            String diffUploadResponse = sendRequest("POST", "changeset/" + changeset.getId() + "/upload", diffUploadRequest, monitor);
539
540            // Process the response from the server
541            //
542            DiffResultProcessor reader = new DiffResultProcessor(list);
543            reader.parse(diffUploadResponse, monitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false));
544            return reader.postProcess(
545                    getChangeset(),
546                    monitor.createSubTaskMonitor(ProgressMonitor.ALL_TICKS, false)
547            );
548        } catch (OsmTransferException e) {
549            throw e;
550        } catch (XmlParsingException e) {
551            throw new OsmTransferException(e);
552        } finally {
553            monitor.finishTask();
554        }
555    }
556
557    private void sleepAndListen(int retry, ProgressMonitor monitor) throws OsmTransferCanceledException {
558        Main.info(tr("Waiting 10 seconds ... "));
559        for (int i = 0; i < 10; i++) {
560            if (monitor != null) {
561                monitor.setCustomText(tr("Starting retry {0} of {1} in {2} seconds ...", getMaxRetries() - retry, getMaxRetries(), 10-i));
562            }
563            if (cancel)
564                throw new OsmTransferCanceledException("Operation canceled" + (i > 0 ? " in retry #"+i : ""));
565            try {
566                Thread.sleep(1000);
567            } catch (InterruptedException ex) {
568                Main.warn("InterruptedException in "+getClass().getSimpleName()+" during sleep");
569            }
570        }
571        Main.info(tr("OK - trying again."));
572    }
573
574    /**
575     * Replies the max. number of retries in case of 5XX errors on the server
576     *
577     * @return the max number of retries
578     */
579    protected int getMaxRetries() {
580        int ret = Main.pref.getInteger("osm-server.max-num-retries", DEFAULT_MAX_NUM_RETRIES);
581        return Math.max(ret, 0);
582    }
583
584    /**
585     * Determines if JOSM is configured to access OSM API via OAuth
586     * @return {@code true} if JOSM is configured to access OSM API via OAuth, {@code false} otherwise
587     * @since 6349
588     */
589    public static final boolean isUsingOAuth() {
590        return "oauth".equals(Main.pref.get("osm-server.auth-method", "basic"));
591    }
592
593    protected final String sendRequest(String requestMethod, String urlSuffix, String requestBody, ProgressMonitor monitor)
594            throws OsmTransferException {
595        return sendRequest(requestMethod, urlSuffix, requestBody, monitor, true, false);
596    }
597
598    /**
599     * Generic method for sending requests to the OSM API.
600     *
601     * This method will automatically re-try any requests that are answered with a 5xx
602     * error code, or that resulted in a timeout exception from the TCP layer.
603     *
604     * @param requestMethod The http method used when talking with the server.
605     * @param urlSuffix The suffix to add at the server url, not including the version number,
606     *    but including any object ids (e.g. "/way/1234/history").
607     * @param requestBody the body of the HTTP request, if any.
608     * @param monitor the progress monitor
609     * @param doAuthenticate  set to true, if the request sent to the server shall include authentication
610     * credentials;
611     * @param fastFail true to request a short timeout
612     *
613     * @return the body of the HTTP response, if and only if the response code was "200 OK".
614     * @throws OsmTransferException if the HTTP return code was not 200 (and retries have
615     *    been exhausted), or rewrapping a Java exception.
616     */
617    protected final String sendRequest(String requestMethod, String urlSuffix, String requestBody, ProgressMonitor monitor,
618            boolean doAuthenticate, boolean fastFail) throws OsmTransferException {
619        StringBuilder responseBody = new StringBuilder();
620        int retries = fastFail ? 0 : getMaxRetries();
621
622        while (true) { // the retry loop
623            try {
624                url = new URL(new URL(getBaseUrl()), urlSuffix);
625                Main.info(requestMethod + ' ' + url + "... ");
626                Main.debug(requestBody);
627                // fix #5369, see http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive
628                activeConnection = Utils.openHttpConnection(url, false);
629                activeConnection.setConnectTimeout(fastFail ? 1000 : Main.pref.getInteger("socket.timeout.connect", 15)*1000);
630                if (fastFail) {
631                    activeConnection.setReadTimeout(1000);
632                }
633                activeConnection.setRequestMethod(requestMethod);
634                if (doAuthenticate) {
635                    addAuth(activeConnection);
636                }
637
638                if ("PUT".equals(requestMethod) || "POST".equals(requestMethod) || "DELETE".equals(requestMethod)) {
639                    activeConnection.setDoOutput(true);
640                    activeConnection.setRequestProperty("Content-type", "text/xml");
641                    try (OutputStream out = activeConnection.getOutputStream()) {
642                        // It seems that certain bits of the Ruby API are very unhappy upon
643                        // receipt of a PUT/POST message without a Content-length header,
644                        // even if the request has no payload.
645                        // Since Java will not generate a Content-length header unless
646                        // we use the output stream, we create an output stream for PUT/POST
647                        // even if there is no payload.
648                        if (requestBody != null) {
649                            try (BufferedWriter bwr = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8))) {
650                                bwr.write(requestBody);
651                                bwr.flush();
652                            }
653                        }
654                    }
655                }
656
657                activeConnection.connect();
658                Main.info(activeConnection.getResponseMessage());
659                int retCode = activeConnection.getResponseCode();
660
661                if (retCode >= 500) {
662                    if (retries-- > 0) {
663                        sleepAndListen(retries, monitor);
664                        Main.info(tr("Starting retry {0} of {1}.", getMaxRetries() - retries, getMaxRetries()));
665                        continue;
666                    }
667                }
668
669                // populate return fields.
670                responseBody.setLength(0);
671
672                // If the API returned an error code like 403 forbidden, getInputStream will fail with an IOException.
673                InputStream i = getConnectionStream();
674                if (i != null) {
675                    // the input stream can be null if both the input and the error stream
676                    // are null. Seems to be the case if the OSM server replies a 401 Unauthorized, see #3887.
677                    String s;
678                    try (BufferedReader in = new BufferedReader(new InputStreamReader(i, StandardCharsets.UTF_8))) {
679                        while ((s = in.readLine()) != null) {
680                            responseBody.append(s);
681                            responseBody.append('\n');
682                        }
683                    }
684                }
685                String errorHeader = null;
686                // Look for a detailed error message from the server
687                if (activeConnection.getHeaderField("Error") != null) {
688                    errorHeader = activeConnection.getHeaderField("Error");
689                    Main.error("Error header: " + errorHeader);
690                } else if (retCode != HttpURLConnection.HTTP_OK && responseBody.length() > 0) {
691                    Main.error("Error body: " + responseBody);
692                }
693                activeConnection.disconnect();
694
695                if (Main.isDebugEnabled()) {
696                    Main.debug("RESPONSE: "+ activeConnection.getHeaderFields());
697                }
698
699                errorHeader = errorHeader == null ? null : errorHeader.trim();
700                String errorBody = responseBody.length() == 0 ? null : responseBody.toString().trim();
701                switch(retCode) {
702                case HttpURLConnection.HTTP_OK:
703                    return responseBody.toString();
704                case HttpURLConnection.HTTP_GONE:
705                    throw new OsmApiPrimitiveGoneException(errorHeader, errorBody);
706                case HttpURLConnection.HTTP_CONFLICT:
707                    if (ChangesetClosedException.errorHeaderMatchesPattern(errorHeader))
708                        throw new ChangesetClosedException(errorBody, ChangesetClosedException.Source.UPLOAD_DATA);
709                    else
710                        throw new OsmApiException(retCode, errorHeader, errorBody);
711                case HttpURLConnection.HTTP_FORBIDDEN:
712                    OsmApiException e = new OsmApiException(retCode, errorHeader, errorBody);
713                    e.setAccessedUrl(activeConnection.getURL().toString());
714                    throw e;
715                default:
716                    throw new OsmApiException(retCode, errorHeader, errorBody);
717                }
718            } catch (SocketTimeoutException | ConnectException e) {
719                if (retries-- > 0) {
720                    continue;
721                }
722                throw new OsmTransferException(e);
723            } catch (IOException e) {
724                throw new OsmTransferException(e);
725            } catch (OsmTransferException e) {
726                throw e;
727            }
728        }
729    }
730
731    private InputStream getConnectionStream() {
732        try {
733            return activeConnection.getInputStream();
734        } catch (IOException ioe) {
735            Main.warn(ioe);
736            return activeConnection.getErrorStream();
737        }
738    }
739
740    /**
741     * Replies the API capabilities.
742     *
743     * @return the API capabilities, or null, if the API is not initialized yet
744     */
745    public synchronized Capabilities getCapabilities() {
746        return capabilities;
747    }
748
749    /**
750     * Ensures that the current changeset can be used for uploading data
751     *
752     * @throws OsmTransferException if the current changeset can't be used for uploading data
753     */
754    protected void ensureValidChangeset() throws OsmTransferException {
755        if (changeset == null)
756            throw new OsmTransferException(tr("Current changeset is null. Cannot upload data."));
757        if (changeset.getId() <= 0)
758            throw new OsmTransferException(tr("ID of current changeset > 0 required. Current ID is {0}.", changeset.getId()));
759    }
760
761    /**
762     * Replies the changeset data uploads are currently directed to
763     *
764     * @return the changeset data uploads are currently directed to
765     */
766    public Changeset getChangeset() {
767        return changeset;
768    }
769
770    /**
771     * Sets the changesets to which further data uploads are directed. The changeset
772     * can be null. If it isn't null it must have been created, i.e. id &gt; 0 is required. Furthermore,
773     * it must be open.
774     *
775     * @param changeset the changeset
776     * @throws IllegalArgumentException if changeset.getId() &lt;= 0
777     * @throws IllegalArgumentException if !changeset.isOpen()
778     */
779    public void setChangeset(Changeset changeset) {
780        if (changeset == null) {
781            this.changeset = null;
782            return;
783        }
784        if (changeset.getId() <= 0)
785            throw new IllegalArgumentException(tr("Changeset ID > 0 expected. Got {0}.", changeset.getId()));
786        if (!changeset.isOpen())
787            throw new IllegalArgumentException(tr("Open changeset expected. Got closed changeset with id {0}.", changeset.getId()));
788        this.changeset = changeset;
789    }
790
791    private static StringBuilder noteStringBuilder(Note note) {
792        return new StringBuilder().append("notes/").append(note.getId());
793    }
794
795    /**
796     * Create a new note on the server.
797     * @param latlon Location of note
798     * @param text Comment entered by user to open the note
799     * @param monitor Progress monitor
800     * @return Note as it exists on the server after creation (ID assigned)
801     * @throws OsmTransferException if any error occurs during dialog with OSM API
802     */
803    public Note createNote(LatLon latlon, String text, ProgressMonitor monitor) throws OsmTransferException {
804        initialize(monitor);
805        String noteUrl = new StringBuilder()
806            .append("notes?lat=")
807            .append(latlon.lat())
808            .append("&lon=")
809            .append(latlon.lon())
810            .append("&text=")
811            .append(Utils.encodeUrl(text)).toString();
812
813        String response = sendRequest("POST", noteUrl, null, monitor, true, false);
814        return parseSingleNote(response);
815    }
816
817    /**
818     * Add a comment to an existing note.
819     * @param note The note to add a comment to
820     * @param comment Text of the comment
821     * @param monitor Progress monitor
822     * @return Note returned by the API after the comment was added
823     * @throws OsmTransferException if any error occurs during dialog with OSM API
824     */
825    public Note addCommentToNote(Note note, String comment, ProgressMonitor monitor) throws OsmTransferException {
826        initialize(monitor);
827        String noteUrl = noteStringBuilder(note)
828            .append("/comment?text=")
829            .append(Utils.encodeUrl(comment)).toString();
830
831        String response = sendRequest("POST", noteUrl, null, monitor, true, false);
832        return parseSingleNote(response);
833    }
834
835    /**
836     * Close a note.
837     * @param note Note to close. Must currently be open
838     * @param closeMessage Optional message supplied by the user when closing the note
839     * @param monitor Progress monitor
840     * @return Note returned by the API after the close operation
841     * @throws OsmTransferException if any error occurs during dialog with OSM API
842     */
843    public Note closeNote(Note note, String closeMessage, ProgressMonitor monitor) throws OsmTransferException {
844        initialize(monitor);
845        String encodedMessage = Utils.encodeUrl(closeMessage);
846        StringBuilder urlBuilder = noteStringBuilder(note)
847            .append("/close");
848        if (encodedMessage != null && !encodedMessage.trim().isEmpty()) {
849            urlBuilder.append("?text=");
850            urlBuilder.append(encodedMessage);
851        }
852
853        String response = sendRequest("POST", urlBuilder.toString(), null, monitor, true, false);
854        return parseSingleNote(response);
855    }
856
857    /**
858     * Reopen a closed note
859     * @param note Note to reopen. Must currently be closed
860     * @param reactivateMessage Optional message supplied by the user when reopening the note
861     * @param monitor Progress monitor
862     * @return Note returned by the API after the reopen operation
863     * @throws OsmTransferException if any error occurs during dialog with OSM API
864     */
865    public Note reopenNote(Note note, String reactivateMessage, ProgressMonitor monitor) throws OsmTransferException {
866        initialize(monitor);
867        String encodedMessage = Utils.encodeUrl(reactivateMessage);
868        StringBuilder urlBuilder = noteStringBuilder(note)
869            .append("/reopen");
870        if (encodedMessage != null && !encodedMessage.trim().isEmpty()) {
871            urlBuilder.append("?text=");
872            urlBuilder.append(encodedMessage);
873        }
874
875        String response = sendRequest("POST", urlBuilder.toString(), null, monitor, true, false);
876        return parseSingleNote(response);
877    }
878
879    /**
880     * Method for parsing API responses for operations on individual notes
881     * @param xml the API response as XML data
882     * @return the resulting Note
883     * @throws OsmTransferException if the API response cannot be parsed
884     */
885    private Note parseSingleNote(String xml) throws OsmTransferException {
886        try {
887            List<Note> newNotes = new NoteReader(xml).parse();
888            if (newNotes.size() == 1) {
889                return newNotes.get(0);
890            }
891            //Shouldn't ever execute. Server will either respond with an error (caught elsewhere) or one note
892            throw new OsmTransferException(tr("Note upload failed"));
893        } catch (SAXException | IOException e) {
894            Main.error(e, true);
895            throw new OsmTransferException(tr("Error parsing note response from server"), e);
896        }
897    }
898}