001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.plugins;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.awt.Dimension;
007import java.awt.GridBagLayout;
008import java.io.BufferedReader;
009import java.io.ByteArrayInputStream;
010import java.io.File;
011import java.io.FileOutputStream;
012import java.io.FilenameFilter;
013import java.io.IOException;
014import java.io.InputStream;
015import java.io.InputStreamReader;
016import java.io.OutputStream;
017import java.io.OutputStreamWriter;
018import java.io.PrintWriter;
019import java.net.HttpURLConnection;
020import java.net.MalformedURLException;
021import java.net.URL;
022import java.nio.charset.StandardCharsets;
023import java.util.ArrayList;
024import java.util.Arrays;
025import java.util.Collection;
026import java.util.Collections;
027import java.util.HashSet;
028import java.util.LinkedList;
029import java.util.List;
030
031import javax.swing.JLabel;
032import javax.swing.JOptionPane;
033import javax.swing.JPanel;
034import javax.swing.JScrollPane;
035
036import org.openstreetmap.josm.Main;
037import org.openstreetmap.josm.gui.PleaseWaitRunnable;
038import org.openstreetmap.josm.gui.progress.NullProgressMonitor;
039import org.openstreetmap.josm.gui.progress.ProgressMonitor;
040import org.openstreetmap.josm.gui.util.GuiHelper;
041import org.openstreetmap.josm.gui.widgets.JosmTextArea;
042import org.openstreetmap.josm.io.OsmTransferException;
043import org.openstreetmap.josm.tools.GBC;
044import org.openstreetmap.josm.tools.ImageProvider;
045import org.openstreetmap.josm.tools.Utils;
046import org.xml.sax.SAXException;
047
048/**
049 * An asynchronous task for downloading plugin lists from the configured plugin download sites.
050 * @since 2817
051 */
052public class ReadRemotePluginInformationTask extends PleaseWaitRunnable {
053
054    private Collection<String> sites;
055    private boolean canceled;
056    private HttpURLConnection connection;
057    private List<PluginInformation> availablePlugins;
058    private boolean displayErrMsg;
059
060    protected enum CacheType {PLUGIN_LIST, ICON_LIST}
061
062    protected final void init(Collection<String> sites, boolean displayErrMsg) {
063        this.sites = sites;
064        if (sites == null) {
065            this.sites = Collections.emptySet();
066        }
067        this.availablePlugins = new LinkedList<>();
068        this.displayErrMsg = displayErrMsg;
069    }
070
071    /**
072     * Constructs a new {@code ReadRemotePluginInformationTask}.
073     *
074     * @param sites the collection of download sites. Defaults to the empty collection if null.
075     */
076    public ReadRemotePluginInformationTask(Collection<String> sites) {
077        super(tr("Download plugin list..."), false /* don't ignore exceptions */);
078        init(sites, true);
079    }
080
081    /**
082     * Constructs a new {@code ReadRemotePluginInformationTask}.
083     *
084     * @param monitor the progress monitor. Defaults to {@link NullProgressMonitor#INSTANCE} if null
085     * @param sites the collection of download sites. Defaults to the empty collection if null.
086     * @param displayErrMsg if {@code true}, a blocking error message is displayed in case of I/O exception.
087     */
088    public ReadRemotePluginInformationTask(ProgressMonitor monitor, Collection<String> sites, boolean displayErrMsg) {
089        super(tr("Download plugin list..."), monitor == null ? NullProgressMonitor.INSTANCE: monitor, false /* don't ignore exceptions */);
090        init(sites, displayErrMsg);
091    }
092
093    @Override
094    protected void cancel() {
095        canceled = true;
096        synchronized(this) {
097            if (connection != null) {
098                connection.disconnect();
099            }
100        }
101    }
102
103    @Override
104    protected void finish() {}
105
106    /**
107     * Creates the file name for the cached plugin list and the icon cache file.
108     *
109     * @param site the name of the site
110     * @param type icon cache or plugin list cache
111     * @return the file name for the cache file
112     */
113    protected File createSiteCacheFile(File pluginDir, String site, CacheType type) {
114        String name;
115        try {
116            site = site.replaceAll("%<(.*)>", "");
117            URL url = new URL(site);
118            StringBuilder sb = new StringBuilder();
119            sb.append("site-");
120            sb.append(url.getHost()).append("-");
121            if (url.getPort() != -1) {
122                sb.append(url.getPort()).append("-");
123            }
124            String path = url.getPath();
125            for (int i =0;i<path.length(); i++) {
126                char c = path.charAt(i);
127                if (Character.isLetterOrDigit(c)) {
128                    sb.append(c);
129                } else {
130                    sb.append("_");
131                }
132            }
133            switch (type) {
134            case PLUGIN_LIST:
135                sb.append(".txt");
136                break;
137            case ICON_LIST:
138                sb.append("-icons.zip");
139                break;
140            }
141            name = sb.toString();
142        } catch(MalformedURLException e) {
143            name = "site-unknown.txt";
144        }
145        return new File(pluginDir, name);
146    }
147
148    /**
149     * Downloads the list from a remote location
150     *
151     * @param site the site URL
152     * @param monitor a progress monitor
153     * @return the downloaded list
154     */
155    protected String downloadPluginList(String site, final ProgressMonitor monitor) {
156        /* replace %<x> with empty string or x=plugins (separated with comma) */
157        String pl = Utils.join(",", Main.pref.getCollection("plugins"));
158        String printsite = site.replaceAll("%<(.*)>", "");
159        if (pl != null && pl.length() != 0) {
160            site = site.replaceAll("%<(.*)>", "$1"+pl);
161        } else {
162            site = printsite;
163        }
164
165        try {
166            monitor.beginTask("");
167            monitor.indeterminateSubTask(tr("Downloading plugin list from ''{0}''", printsite));
168
169            URL url = new URL(site);
170            synchronized(this) {
171                connection = Utils.openHttpConnection(url);
172                connection.setRequestProperty("Cache-Control", "no-cache");
173                connection.setRequestProperty("Accept-Charset", "utf-8");
174            }
175            try (BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8))) {
176                StringBuilder sb = new StringBuilder();
177                String line;
178                while ((line = in.readLine()) != null) {
179                    sb.append(line).append("\n");
180                }
181                return sb.toString();
182            }
183        } catch (MalformedURLException e) {
184            if (canceled) return null;
185            Main.error(e);
186            return null;
187        } catch (IOException e) {
188            if (canceled) return null;
189            Main.addNetworkError(site, e);
190            handleIOException(monitor, e, tr("Plugin list download error"), tr("JOSM failed to download plugin list:"), displayErrMsg);
191            return null;
192        } finally {
193            synchronized(this) {
194                if (connection != null) {
195                    connection.disconnect();
196                }
197                connection = null;
198            }
199            monitor.finishTask();
200        }
201    }
202
203    private void handleIOException(final ProgressMonitor monitor, IOException e, final String title, final String firstMessage, boolean displayMsg) {
204        StringBuilder sb = new StringBuilder();
205        try (InputStream errStream = connection.getErrorStream()) {
206            if (errStream != null) {
207                try (BufferedReader err = new BufferedReader(new InputStreamReader(errStream, StandardCharsets.UTF_8))) {
208                    String line;
209                    while ((line = err.readLine()) != null) {
210                        sb.append(line).append("\n");
211                    }
212                } catch (Exception ex) {
213                    Main.error(e);
214                    Main.error(ex);
215                }
216            }
217        } catch (IOException ex) {
218            Main.warn(ex);
219        }
220        final String msg = e.getMessage();
221        final String details = sb.toString();
222        if (details.isEmpty()) {
223            Main.error(e.getClass().getSimpleName()+": " + msg);
224        } else {
225            Main.error(msg + " - Details:\n" + details);
226        }
227
228        if (displayMsg) {
229            displayErrorMessage(monitor, msg, details, title, firstMessage);
230        }
231    }
232
233    private void displayErrorMessage(final ProgressMonitor monitor, final String msg, final String details, final String title, final String firstMessage) {
234        GuiHelper.runInEDTAndWait(new Runnable() {
235            @Override public void run() {
236                JPanel panel = new JPanel(new GridBagLayout());
237                panel.add(new JLabel(firstMessage), GBC.eol().insets(0, 0, 0, 10));
238                StringBuilder b = new StringBuilder();
239                for (String part : msg.split("(?<=\\G.{200})")) {
240                    b.append(part).append("\n");
241                }
242                panel.add(new JLabel("<html><body width=\"500\"><b>"+b.toString().trim()+"</b></body></html>"), GBC.eol().insets(0, 0, 0, 10));
243                if (!details.isEmpty()) {
244                    panel.add(new JLabel(tr("Details:")), GBC.eol().insets(0, 0, 0, 10));
245                    JosmTextArea area = new JosmTextArea(details);
246                    area.setEditable(false);
247                    area.setLineWrap(true);
248                    area.setWrapStyleWord(true);
249                    JScrollPane scrollPane = new JScrollPane(area);
250                    scrollPane.setPreferredSize(new Dimension(500, 300));
251                    panel.add(scrollPane, GBC.eol().fill());
252                }
253                JOptionPane.showMessageDialog(monitor.getWindowParent(), panel, title, JOptionPane.ERROR_MESSAGE);
254            }
255        });
256    }
257
258    /**
259     * Downloads the icon archive from a remote location
260     *
261     * @param site the site URL
262     * @param monitor a progress monitor
263     */
264    protected void downloadPluginIcons(String site, File destFile, ProgressMonitor monitor) {
265        try {
266            site = site.replaceAll("%<(.*)>", "");
267
268            monitor.beginTask("");
269            monitor.indeterminateSubTask(tr("Downloading plugin list from ''{0}''", site));
270
271            URL url = new URL(site);
272            synchronized(this) {
273                connection = Utils.openHttpConnection(url);
274                connection.setRequestProperty("Cache-Control", "no-cache");
275            }
276            try (
277                InputStream in = connection.getInputStream();
278                OutputStream out = new FileOutputStream(destFile)
279            ) {
280                byte[] buffer = new byte[8192];
281                for (int read = in.read(buffer); read != -1; read = in.read(buffer)) {
282                    out.write(buffer, 0, read);
283                }
284            }
285        } catch (MalformedURLException e) {
286            if (canceled) return;
287            Main.error(e);
288            return;
289        } catch (IOException e) {
290            if (canceled) return;
291            handleIOException(monitor, e, tr("Plugin icons download error"), tr("JOSM failed to download plugin icons:"), displayErrMsg);
292            return;
293        } finally {
294            synchronized(this) {
295                if (connection != null) {
296                    connection.disconnect();
297                }
298                connection = null;
299            }
300            monitor.finishTask();
301        }
302        for (PluginInformation pi : availablePlugins) {
303            if (pi.icon == null && pi.iconPath != null) {
304                pi.icon = new ImageProvider(pi.name+".jar/"+pi.iconPath)
305                                .setArchive(destFile)
306                                .setMaxWidth(24)
307                                .setMaxHeight(24)
308                                .setOptional(true).get();
309            }
310        }
311    }
312
313    /**
314     * Writes the list of plugins to a cache file
315     *
316     * @param site the site from where the list was downloaded
317     * @param list the downloaded list
318     */
319    protected void cachePluginList(String site, String list) {
320        File pluginDir = Main.pref.getPluginsDirectory();
321        if (!pluginDir.exists() && !pluginDir.mkdirs()) {
322            Main.warn(tr("Failed to create plugin directory ''{0}''. Cannot cache plugin list from plugin site ''{1}''.", pluginDir.toString(), site));
323        }
324        File cacheFile = createSiteCacheFile(pluginDir, site, CacheType.PLUGIN_LIST);
325        getProgressMonitor().subTask(tr("Writing plugin list to local cache ''{0}''", cacheFile.toString()));
326        try (PrintWriter writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(cacheFile), StandardCharsets.UTF_8))) {
327            writer.write(list);
328            writer.flush();
329        } catch(IOException e) {
330            // just failed to write the cache file. No big deal, but log the exception anyway
331            Main.error(e);
332        }
333    }
334
335    /**
336     * Filter information about deprecated plugins from the list of downloaded
337     * plugins
338     *
339     * @param plugins the plugin informations
340     * @return the plugin informations, without deprecated plugins
341     */
342    protected List<PluginInformation> filterDeprecatedPlugins(List<PluginInformation> plugins) {
343        List<PluginInformation> ret = new ArrayList<>(plugins.size());
344        HashSet<String> deprecatedPluginNames = new HashSet<>();
345        for (PluginHandler.DeprecatedPlugin p : PluginHandler.DEPRECATED_PLUGINS) {
346            deprecatedPluginNames.add(p.name);
347        }
348        for (PluginInformation plugin: plugins) {
349            if (deprecatedPluginNames.contains(plugin.name)) {
350                continue;
351            }
352            ret.add(plugin);
353        }
354        return ret;
355    }
356
357    /**
358     * Parses the plugin list
359     *
360     * @param site the site from where the list was downloaded
361     * @param doc the document with the plugin list
362     */
363    protected void parsePluginListDocument(String site, String doc) {
364        try {
365            getProgressMonitor().subTask(tr("Parsing plugin list from site ''{0}''", site));
366            InputStream in = new ByteArrayInputStream(doc.getBytes(StandardCharsets.UTF_8));
367            List<PluginInformation> pis = new PluginListParser().parse(in);
368            availablePlugins.addAll(filterDeprecatedPlugins(pis));
369        } catch (PluginListParseException e) {
370            Main.error(tr("Failed to parse plugin list document from site ''{0}''. Skipping site. Exception was: {1}", site, e.toString()));
371            Main.error(e);
372        }
373    }
374
375    @Override
376    protected void realRun() throws SAXException, IOException, OsmTransferException {
377        if (sites == null) return;
378        getProgressMonitor().setTicksCount(sites.size() * 3);
379        File pluginDir = Main.pref.getPluginsDirectory();
380
381        // collect old cache files and remove if no longer in use
382        List<File> siteCacheFiles = new LinkedList<>();
383        for (String location : PluginInformation.getPluginLocations()) {
384            File [] f = new File(location).listFiles(
385                    new FilenameFilter() {
386                        @Override
387                        public boolean accept(File dir, String name) {
388                            return name.matches("^([0-9]+-)?site.*\\.txt$") ||
389                            name.matches("^([0-9]+-)?site.*-icons\\.zip$");
390                        }
391                    }
392            );
393            if(f != null && f.length > 0) {
394                siteCacheFiles.addAll(Arrays.asList(f));
395            }
396        }
397
398        for (String site: sites) {
399            String printsite = site.replaceAll("%<(.*)>", "");
400            getProgressMonitor().subTask(tr("Processing plugin list from site ''{0}''", printsite));
401            String list = downloadPluginList(site, getProgressMonitor().createSubTaskMonitor(0, false));
402            if (canceled) return;
403            siteCacheFiles.remove(createSiteCacheFile(pluginDir, site, CacheType.PLUGIN_LIST));
404            siteCacheFiles.remove(createSiteCacheFile(pluginDir, site, CacheType.ICON_LIST));
405            if (list != null) {
406                getProgressMonitor().worked(1);
407                cachePluginList(site, list);
408                if (canceled) return;
409                getProgressMonitor().worked(1);
410                parsePluginListDocument(site, list);
411                if (canceled) return;
412                getProgressMonitor().worked(1);
413                if (canceled) return;
414            }
415            downloadPluginIcons(site+"-icons.zip", createSiteCacheFile(pluginDir, site, CacheType.ICON_LIST), getProgressMonitor().createSubTaskMonitor(0, false));
416        }
417        // remove old stuff or whole update process is broken
418        for (File file: siteCacheFiles) {
419            file.delete();
420        }
421    }
422
423    /**
424     * Replies true if the task was canceled
425     * @return <code>true</code> if the task was stopped by the user
426     */
427    public boolean isCanceled() {
428        return canceled;
429    }
430
431    /**
432     * Replies the list of plugins described in the downloaded plugin lists
433     *
434     * @return  the list of plugins
435     * @since 5601
436     */
437    public List<PluginInformation> getAvailablePlugins() {
438        return availablePlugins;
439    }
440}