001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.io;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.BufferedReader;
007import java.io.IOException;
008import java.io.InputStream;
009import java.io.InputStreamReader;
010import java.net.HttpURLConnection;
011import java.net.MalformedURLException;
012import java.net.URL;
013import java.nio.charset.StandardCharsets;
014import java.util.List;
015import java.util.Map;
016import java.util.zip.GZIPInputStream;
017import java.util.zip.Inflater;
018import java.util.zip.InflaterInputStream;
019
020import org.openstreetmap.josm.Main;
021import org.openstreetmap.josm.data.gpx.GpxData;
022import org.openstreetmap.josm.data.notes.Note;
023import org.openstreetmap.josm.data.osm.DataSet;
024import org.openstreetmap.josm.gui.progress.ProgressMonitor;
025import org.openstreetmap.josm.tools.Utils;
026
027/**
028 * This DataReader reads directly from the REST API of the osm server.
029 *
030 * It supports plain text transfer as well as gzip or deflate encoded transfers;
031 * if compressed transfers are unwanted, set property osm-server.use-compression
032 * to false.
033 *
034 * @author imi
035 */
036public abstract class OsmServerReader extends OsmConnection {
037    private OsmApi api = OsmApi.getOsmApi();
038    private boolean doAuthenticate = false;
039    protected boolean gpxParsedProperly;
040
041    /**
042     * Open a connection to the given url and return a reader on the input stream
043     * from that connection. In case of user cancel, return <code>null</code>.
044     * Relative URL's are directed to API base URL.
045     * @param urlStr The url to connect to.
046     * @param progressMonitor progress monitoring and abort handler
047     * @return A reader reading the input stream (servers answer) or <code>null</code>.
048     * @throws OsmTransferException thrown if data transfer errors occur
049     */
050    protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException  {
051        return getInputStream(urlStr, progressMonitor, null);
052    }
053
054    /**
055     * Open a connection to the given url and return a reader on the input stream
056     * from that connection. In case of user cancel, return <code>null</code>.
057     * Relative URL's are directed to API base URL.
058     * @param urlStr The url to connect to.
059     * @param progressMonitor progress monitoring and abort handler
060     * @param reason The reason to show on console. Can be {@code null} if no reason is given
061     * @return A reader reading the input stream (servers answer) or <code>null</code>.
062     * @throws OsmTransferException thrown if data transfer errors occur
063     */
064    protected InputStream getInputStream(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException  {
065        try {
066            api.initialize(progressMonitor);
067            String url = urlStr.startsWith("http") ? urlStr : (getBaseUrl() + urlStr);
068            return getInputStreamRaw(url, progressMonitor, reason);
069        } finally {
070            progressMonitor.invalidate();
071        }
072    }
073
074    /**
075     * Return the base URL for relative URL requests
076     * @return base url of API
077     */
078    protected String getBaseUrl() {
079        return api.getBaseUrl();
080    }
081
082    /**
083     * Open a connection to the given url and return a reader on the input stream
084     * from that connection. In case of user cancel, return <code>null</code>.
085     * @param urlStr The exact url to connect to.
086     * @param progressMonitor progress monitoring and abort handler
087     * @return An reader reading the input stream (servers answer) or <code>null</code>.
088     * @throws OsmTransferException thrown if data transfer errors occur
089     */
090    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor) throws OsmTransferException {
091        return getInputStreamRaw(urlStr, progressMonitor, null);
092    }
093
094    /**
095     * Open a connection to the given url and return a reader on the input stream
096     * from that connection. In case of user cancel, return <code>null</code>.
097     * @param urlStr The exact url to connect to.
098     * @param progressMonitor progress monitoring and abort handler
099     * @param reason The reason to show on console. Can be {@code null} if no reason is given
100     * @return An reader reading the input stream (servers answer) or <code>null</code>.
101     * @throws OsmTransferException thrown if data transfer errors occur
102     */
103    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason) throws OsmTransferException {
104        return getInputStreamRaw(urlStr, progressMonitor, reason, false);
105    }
106
107    /**
108     * Open a connection to the given url and return a reader on the input stream
109     * from that connection. In case of user cancel, return <code>null</code>.
110     * @param urlStr The exact url to connect to.
111     * @param progressMonitor progress monitoring and abort handler
112     * @param reason The reason to show on console. Can be {@code null} if no reason is given
113     * @param uncompressAccordingToContentDisposition Whether to inspect the HTTP header {@code Content-Disposition}
114     *                                                for {@code filename} and uncompress a gzip/bzip2 stream.
115     * @return An reader reading the input stream (servers answer) or <code>null</code>.
116     * @throws OsmTransferException thrown if data transfer errors occur
117     */
118    @SuppressWarnings("resource")
119    protected InputStream getInputStreamRaw(String urlStr, ProgressMonitor progressMonitor, String reason, boolean uncompressAccordingToContentDisposition) throws OsmTransferException {
120        try {
121            OnlineResource.JOSM_WEBSITE.checkOfflineAccess(urlStr, Main.getJOSMWebsite());
122            OnlineResource.OSM_API.checkOfflineAccess(urlStr, Main.pref.get("osm-server.url", OsmApi.DEFAULT_API_URL));
123
124            URL url = null;
125            try {
126                url = new URL(urlStr.replace(" ", "%20"));
127            } catch(MalformedURLException e) {
128                throw new OsmTransferException(e);
129            }
130            try {
131                // fix #7640, see http://www.tikalk.com/java/forums/httpurlconnection-disable-keep-alive
132                activeConnection = Utils.openHttpConnection(url, false);
133            } catch(Exception e) {
134                throw new OsmTransferException(tr("Failed to open connection to API {0}.", url.toExternalForm()), e);
135            }
136            if (cancel) {
137                activeConnection.disconnect();
138                return null;
139            }
140
141            if (doAuthenticate) {
142                addAuth(activeConnection);
143            }
144            if (cancel)
145                throw new OsmTransferCanceledException();
146            if (Main.pref.getBoolean("osm-server.use-compression", true)) {
147                activeConnection.setRequestProperty("Accept-Encoding", "gzip, deflate");
148            }
149
150            activeConnection.setConnectTimeout(Main.pref.getInteger("socket.timeout.connect",15)*1000);
151
152            try {
153                if (reason != null && !reason.isEmpty()) {
154                    Main.info("GET " + url + " (" + reason + ")");
155                } else {
156                    Main.info("GET " + url);
157                }
158                activeConnection.connect();
159            } catch (Exception e) {
160                Main.error(e);
161                OsmTransferException ote = new OsmTransferException(tr("Could not connect to the OSM server. Please check your internet connection."), e);
162                ote.setUrl(url.toString());
163                throw ote;
164            }
165            try {
166                if (Main.isDebugEnabled()) {
167                    Main.debug("RESPONSE: "+activeConnection.getHeaderFields());
168                }
169                if (activeConnection.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED)
170                    throw new OsmApiException(HttpURLConnection.HTTP_UNAUTHORIZED,null,null);
171
172                if (activeConnection.getResponseCode() == HttpURLConnection.HTTP_PROXY_AUTH)
173                    throw new OsmTransferCanceledException();
174
175                String encoding = activeConnection.getContentEncoding();
176                if (activeConnection.getResponseCode() != HttpURLConnection.HTTP_OK) {
177                    String errorHeader = activeConnection.getHeaderField("Error");
178                    StringBuilder errorBody = new StringBuilder();
179                    try {
180                        InputStream i = fixEncoding(activeConnection.getErrorStream(), encoding);
181                        if (i != null) {
182                            BufferedReader in = new BufferedReader(new InputStreamReader(i, StandardCharsets.UTF_8));
183                            String s;
184                            while((s = in.readLine()) != null) {
185                                errorBody.append(s);
186                                errorBody.append("\n");
187                            }
188                        }
189                    }
190                    catch(Exception e) {
191                        errorBody.append(tr("Reading error text failed."));
192                    }
193
194                    throw new OsmApiException(activeConnection.getResponseCode(), errorHeader, errorBody.toString(), url.toString());
195                }
196
197                InputStream in = new ProgressInputStream(activeConnection, progressMonitor);
198                if (uncompressAccordingToContentDisposition) {
199                    in = uncompressAccordingToContentDisposition(in, activeConnection.getHeaderFields());
200                }
201                return fixEncoding(in, encoding);
202            } catch (OsmTransferException e) {
203                throw e;
204            } catch (Exception e) {
205                throw new OsmTransferException(e);
206            }
207        } finally {
208            progressMonitor.invalidate();
209        }
210    }
211
212    private InputStream fixEncoding(InputStream stream, String encoding) throws IOException {
213        if ("gzip".equalsIgnoreCase(encoding)) {
214            stream = new GZIPInputStream(stream);
215        } else if ("deflate".equalsIgnoreCase(encoding)) {
216            stream = new InflaterInputStream(stream, new Inflater(true));
217        }
218        return stream;
219    }
220
221    private InputStream uncompressAccordingToContentDisposition(InputStream stream, Map<String, List<String>> headerFields) throws IOException {
222        List<String> field = headerFields.get("Content-Disposition");
223        if (field != null && field.toString().contains(".gz\"")) {
224            return Compression.GZIP.getUncompressedInputStream(stream);
225        } else if (field != null && field.toString().contains(".bz2\"")) {
226            return Compression.BZIP2.getUncompressedInputStream(stream);
227        } else {
228            return stream;
229        }
230    }
231
232    /**
233     * Download OSM files from somewhere
234     * @param progressMonitor The progress monitor
235     * @return The corresponding dataset
236     * @throws OsmTransferException if any error occurs
237     */
238    public abstract DataSet parseOsm(final ProgressMonitor progressMonitor) throws OsmTransferException;
239
240    /**
241     * Download OSM Change files from somewhere
242     * @param progressMonitor The progress monitor
243     * @return The corresponding dataset
244     * @throws OsmTransferException if any error occurs
245     */
246    public DataSet parseOsmChange(final ProgressMonitor progressMonitor) throws OsmTransferException {
247        return null;
248    }
249
250    /**
251     * Download BZip2-compressed OSM Change files from somewhere
252     * @param progressMonitor The progress monitor
253     * @return The corresponding dataset
254     * @throws OsmTransferException if any error occurs
255     */
256    public DataSet parseOsmChangeBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException {
257        return null;
258    }
259
260    /**
261     * Download GZip-compressed OSM Change files from somewhere
262     * @param progressMonitor The progress monitor
263     * @return The corresponding dataset
264     * @throws OsmTransferException if any error occurs
265     */
266    public DataSet parseOsmChangeGzip(final ProgressMonitor progressMonitor) throws OsmTransferException {
267        return null;
268    }
269
270    /**
271     * Retrieve raw gps waypoints from the server API.
272     * @param progressMonitor The progress monitor
273     * @return The corresponding GPX tracks
274     * @throws OsmTransferException if any error occurs
275     */
276    public GpxData parseRawGps(final ProgressMonitor progressMonitor) throws OsmTransferException {
277        return null;
278    }
279
280    /**
281     * Retrieve BZip2-compressed GPX files from somewhere.
282     * @param progressMonitor The progress monitor
283     * @return The corresponding GPX tracks
284     * @throws OsmTransferException if any error occurs
285     * @since 6244
286     */
287    public GpxData parseRawGpsBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException {
288        return null;
289    }
290
291    /**
292     * Download BZip2-compressed OSM files from somewhere
293     * @param progressMonitor The progress monitor
294     * @return The corresponding dataset
295     * @throws OsmTransferException if any error occurs
296     */
297    public DataSet parseOsmBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException {
298        return null;
299    }
300
301    /**
302     * Download GZip-compressed OSM files from somewhere
303     * @param progressMonitor The progress monitor
304     * @return The corresponding dataset
305     * @throws OsmTransferException if any error occurs
306     */
307    public DataSet parseOsmGzip(final ProgressMonitor progressMonitor) throws OsmTransferException {
308        return null;
309    }
310
311    /**
312     * Download Zip-compressed OSM files from somewhere
313     * @param progressMonitor The progress monitor
314     * @return The corresponding dataset
315     * @throws OsmTransferException if any error occurs
316     * @since 6882
317     */
318    public DataSet parseOsmZip(final ProgressMonitor progressMonitor) throws OsmTransferException {
319        return null;
320    }
321
322    /**
323     * Returns true if this reader is adding authentication credentials to the read
324     * request sent to the server.
325     *
326     * @return true if this reader is adding authentication credentials to the read
327     * request sent to the server
328     */
329    public boolean isDoAuthenticate() {
330        return doAuthenticate;
331    }
332
333    /**
334     * Sets whether this reader adds authentication credentials to the read
335     * request sent to the server.
336     *
337     * @param doAuthenticate  true if  this reader adds authentication credentials to the read
338     * request sent to the server
339     */
340    public void setDoAuthenticate(boolean doAuthenticate) {
341        this.doAuthenticate = doAuthenticate;
342    }
343
344    /**
345     * Determines if the GPX data has been parsed properly.
346     * @return true if the GPX data has been parsed properly, false otherwise
347     * @see GpxReader#parse
348     */
349    public final boolean isGpxParsedProperly() {
350        return gpxParsedProperly;
351    }
352
353    /**
354     * Downloads notes from the API, given API limit parameters
355     *
356     * @param noteLimit How many notes to download. Defaults to 1000 if not specified. API has a hard limit of 10000
357     * @param daysClosed Return notes closed this many days in the past. -1 means all notes, ever. 0 means only unresolved notes.
358     * @param progressMonitor Progress monitor for user feedback
359     * @return List of notes returned by the API
360     * @throws OsmTransferException if any errors happen
361     */
362    public List<Note> parseNotes(Integer noteLimit, Integer daysClosed, ProgressMonitor progressMonitor) throws OsmTransferException {
363        return null;
364    }
365
366    /**
367     * Downloads notes from a given raw URL. The URL is assumed to be complete and no API limits are added
368     *
369     * @param progressMonitor
370     * @return A list of notes parsed from the URL
371     * @throws OsmTransferException
372     */
373    public List<Note> parseRawNotes(final ProgressMonitor progressMonitor) throws OsmTransferException {
374        return null;
375    }
376
377    /**
378     * Download notes from a URL that contains a bzip2 compressed notes dump file
379     * @param progressMonitor
380     * @return A list of notes parsed from the URL
381     * @throws OsmTransferException
382     */
383    public List<Note> parseRawNotesBzip2(final ProgressMonitor progressMonitor) throws OsmTransferException {
384        return null;
385    }
386}