001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.gui.layer;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006import static org.openstreetmap.josm.tools.Utils.getSystemProperty;
007
008import java.io.BufferedReader;
009import java.io.File;
010import java.io.FileFilter;
011import java.io.IOException;
012import java.lang.management.ManagementFactory;
013import java.nio.charset.StandardCharsets;
014import java.nio.file.Files;
015import java.nio.file.Path;
016import java.util.ArrayList;
017import java.util.Collections;
018import java.util.Date;
019import java.util.Deque;
020import java.util.HashSet;
021import java.util.Iterator;
022import java.util.LinkedList;
023import java.util.List;
024import java.util.Locale;
025import java.util.Set;
026import java.util.Timer;
027import java.util.TimerTask;
028import java.util.concurrent.ExecutionException;
029import java.util.concurrent.Future;
030import java.util.concurrent.TimeUnit;
031import java.util.regex.Pattern;
032
033import org.openstreetmap.josm.actions.OpenFileAction.OpenFileTask;
034import org.openstreetmap.josm.data.osm.DataSet;
035import org.openstreetmap.josm.data.osm.NoteData;
036import org.openstreetmap.josm.data.osm.NoteData.NoteDataUpdateListener;
037import org.openstreetmap.josm.data.osm.event.AbstractDatasetChangedEvent;
038import org.openstreetmap.josm.data.osm.event.DataSetListenerAdapter;
039import org.openstreetmap.josm.data.osm.event.DataSetListenerAdapter.Listener;
040import org.openstreetmap.josm.data.preferences.BooleanProperty;
041import org.openstreetmap.josm.data.preferences.IntegerProperty;
042import org.openstreetmap.josm.gui.MainApplication;
043import org.openstreetmap.josm.gui.Notification;
044import org.openstreetmap.josm.gui.io.importexport.NoteExporter;
045import org.openstreetmap.josm.gui.io.importexport.NoteImporter;
046import org.openstreetmap.josm.gui.io.importexport.OsmExporter;
047import org.openstreetmap.josm.gui.io.importexport.OsmImporter;
048import org.openstreetmap.josm.gui.layer.LayerManager.LayerAddEvent;
049import org.openstreetmap.josm.gui.layer.LayerManager.LayerChangeListener;
050import org.openstreetmap.josm.gui.layer.LayerManager.LayerOrderChangeEvent;
051import org.openstreetmap.josm.gui.layer.LayerManager.LayerRemoveEvent;
052import org.openstreetmap.josm.gui.util.GuiHelper;
053import org.openstreetmap.josm.spi.preferences.Config;
054import org.openstreetmap.josm.tools.Logging;
055import org.openstreetmap.josm.tools.Utils;
056
057/**
058 * Saves data and note layers periodically so they can be recovered in case of a crash.
059 *
060 * There are 2 directories
061 *  - autosave dir: copies of the currently open data layers are saved here every
062 *      PROP_INTERVAL seconds. When a data layer is closed normally, the corresponding
063 *      files are removed. If this dir is non-empty on start, JOSM assumes
064 *      that it crashed last time.
065 *  - deleted layers dir: "secondary archive" - when autosaved layers are restored
066 *      they are copied to this directory. We cannot keep them in the autosave folder,
067 *      but just deleting it would be dangerous: Maybe a feature inside the file
068 *      caused JOSM to crash. If the data is valuable, the user can still try to
069 *      open with another versions of JOSM or fix the problem manually.
070 *
071 *      The deleted layers dir keeps at most PROP_DELETED_LAYERS files.
072 *
073 * @since  3378 (creation)
074 * @since 10386 (new LayerChangeListener interface)
075 */
076public class AutosaveTask extends TimerTask implements LayerChangeListener, Listener, NoteDataUpdateListener {
077
078    private static final char[] ILLEGAL_CHARACTERS = {'/', '\n', '\r', '\t', '\0', '\f', '`', '?', '*', '\\', '<', '>', '|', '\"', ':'};
079    private static final String AUTOSAVE_DIR = "autosave";
080    private static final String DELETED_LAYERS_DIR = "autosave/deleted_layers";
081
082    /**
083     * If autosave is enabled
084     */
085    public static final BooleanProperty PROP_AUTOSAVE_ENABLED = new BooleanProperty("autosave.enabled", true);
086    /**
087     * The number of files to store per layer
088     */
089    public static final IntegerProperty PROP_FILES_PER_LAYER = new IntegerProperty("autosave.filesPerLayer", 1);
090    /**
091     * How many deleted layers should be stored
092     */
093    public static final IntegerProperty PROP_DELETED_LAYERS = new IntegerProperty("autosave.deletedLayersBackupCount", 5);
094    /**
095     * The autosave interval, in seconds
096     */
097    public static final IntegerProperty PROP_INTERVAL = new IntegerProperty("autosave.interval", (int) TimeUnit.MINUTES.toSeconds(5));
098    /**
099     * The maximum number of autosave files to store
100     */
101    public static final IntegerProperty PROP_INDEX_LIMIT = new IntegerProperty("autosave.index-limit", 1000);
102    /**
103     * Defines if a notification should be displayed after each autosave
104     */
105    public static final BooleanProperty PROP_NOTIFICATION = new BooleanProperty("autosave.notification", false);
106
107    protected static final class AutosaveLayerInfo<T extends AbstractModifiableLayer> {
108        private final T layer;
109        private String layerName;
110        private String layerFileName;
111        private final Deque<File> backupFiles = new LinkedList<>();
112
113        AutosaveLayerInfo(T layer) {
114            this.layer = layer;
115        }
116    }
117
118    private final DataSetListenerAdapter datasetAdapter = new DataSetListenerAdapter(this);
119    private final Set<DataSet> changedDatasets = new HashSet<>();
120    private final Set<NoteData> changedNoteData = new HashSet<>();
121    private final List<AutosaveLayerInfo<?>> layersInfo = new ArrayList<>();
122    private final Object layersLock = new Object();
123    private final Deque<File> deletedLayers = new LinkedList<>();
124
125    private final File autosaveDir = new File(Config.getDirs().getUserDataDirectory(true), AUTOSAVE_DIR);
126    private final File deletedLayersDir = new File(Config.getDirs().getUserDataDirectory(true), DELETED_LAYERS_DIR);
127
128    /**
129     * Replies the autosave directory.
130     * @return the autosave directory
131     * @since 10299
132     */
133    public final Path getAutosaveDir() {
134        return autosaveDir.toPath();
135    }
136
137    /**
138     * Starts the autosave background task.
139     */
140    public void schedule() {
141        if (PROP_INTERVAL.get() > 0) {
142
143            if (!autosaveDir.exists() && !autosaveDir.mkdirs()) {
144                Logging.warn(tr("Unable to create directory {0}, autosave will be disabled", autosaveDir.getAbsolutePath()));
145                return;
146            }
147            if (!deletedLayersDir.exists() && !deletedLayersDir.mkdirs()) {
148                Logging.warn(tr("Unable to create directory {0}, autosave will be disabled", deletedLayersDir.getAbsolutePath()));
149                return;
150            }
151
152            File[] files = deletedLayersDir.listFiles();
153            if (files != null) {
154                for (File f: files) {
155                    deletedLayers.add(f); // FIXME: sort by mtime
156                }
157            }
158
159            new Timer(true).schedule(this, TimeUnit.SECONDS.toMillis(1), TimeUnit.SECONDS.toMillis(PROP_INTERVAL.get()));
160            MainApplication.getLayerManager().addAndFireLayerChangeListener(this);
161        }
162    }
163
164    private static String getFileName(String layerName, int index) {
165        String result = layerName;
166        for (char illegalCharacter : ILLEGAL_CHARACTERS) {
167            result = result.replaceAll(Pattern.quote(String.valueOf(illegalCharacter)),
168                    '&' + String.valueOf((int) illegalCharacter) + ';');
169        }
170        if (index != 0) {
171            result = result + '_' + index;
172        }
173        return result;
174    }
175
176    private void setLayerFileName(AutosaveLayerInfo<?> layer) {
177        int index = 0;
178        while (true) {
179            String filename = getFileName(layer.layer.getName(), index);
180            boolean foundTheSame = false;
181            for (AutosaveLayerInfo<?> info: layersInfo) {
182                if (info != layer && filename.equals(info.layerFileName)) {
183                    foundTheSame = true;
184                    break;
185                }
186            }
187
188            if (!foundTheSame) {
189                layer.layerFileName = filename;
190                return;
191            }
192
193            index++;
194        }
195    }
196
197    protected File getNewLayerFile(AutosaveLayerInfo<?> layer, Date now, int startIndex) {
198        int index = startIndex;
199        while (true) {
200            String filename = String.format(Locale.ENGLISH, "%1$s_%2$tY%2$tm%2$td_%2$tH%2$tM%2$tS%2$tL%3$s",
201                    layer.layerFileName, now, index == 0 ? "" : ('_' + Integer.toString(index)));
202            File result = new File(autosaveDir, filename + '.' +
203                    (layer.layer instanceof NoteLayer ?
204                            Config.getPref().get("autosave.notes.extension", "osn") :
205                            Config.getPref().get("autosave.extension", "osm")));
206            try {
207                if (index > PROP_INDEX_LIMIT.get())
208                    throw new IOException("index limit exceeded");
209                if (result.createNewFile()) {
210                    createNewPidFile(autosaveDir, filename);
211                    return result;
212                } else {
213                    Logging.warn(tr("Unable to create file {0}, other filename will be used", result.getAbsolutePath()));
214                }
215            } catch (IOException e) {
216                Logging.log(Logging.LEVEL_ERROR, tr("IOError while creating file, autosave will be skipped: {0}", e.getMessage()), e);
217                return null;
218            }
219            index++;
220        }
221    }
222
223    private static void createNewPidFile(File autosaveDir, String filename) {
224        File pidFile = new File(autosaveDir, filename+".pid");
225        try {
226            final String content = ManagementFactory.getRuntimeMXBean().getName();
227            Files.write(pidFile.toPath(), Collections.singleton(content), StandardCharsets.UTF_8);
228        } catch (IOException | SecurityException t) {
229            Logging.error(t);
230        }
231    }
232
233    private void savelayer(AutosaveLayerInfo<?> info) {
234        if (!info.layer.getName().equals(info.layerName)) {
235            setLayerFileName(info);
236            info.layerName = info.layer.getName();
237        }
238        try {
239            if (info.layer instanceof OsmDataLayer) {
240                OsmDataLayer dataLayer = (OsmDataLayer) info.layer;
241                if (changedDatasets.remove(dataLayer.data)) {
242                    File file = getNewLayerFile(info, new Date(), 0);
243                    if (file != null) {
244                        info.backupFiles.add(file);
245                        new OsmExporter().exportData(file, info.layer, true /* no backup with appended ~ */);
246                    }
247                }
248            } else if (info.layer instanceof NoteLayer) {
249                NoteLayer noteLayer = (NoteLayer) info.layer;
250                if (changedNoteData.remove(noteLayer.getNoteData())) {
251                    File file = getNewLayerFile(info, new Date(), 0);
252                    if (file != null) {
253                        info.backupFiles.add(file);
254                        new NoteExporter().exportData(file, info.layer);
255                    }
256                }
257            }
258        } catch (IOException e) {
259            Logging.error(e);
260        }
261        while (info.backupFiles.size() > PROP_FILES_PER_LAYER.get()) {
262            File oldFile = info.backupFiles.remove();
263            if (Utils.deleteFile(oldFile, marktr("Unable to delete old backup file {0}"))) {
264                Utils.deleteFile(getPidFile(oldFile), marktr("Unable to delete old backup file {0}"));
265            }
266        }
267    }
268
269    @Override
270    public void run() {
271        synchronized (layersLock) {
272            try {
273                for (AutosaveLayerInfo<?> info: layersInfo) {
274                    savelayer(info);
275                }
276                changedDatasets.clear();
277                changedNoteData.clear();
278                if (PROP_NOTIFICATION.get() && !layersInfo.isEmpty()) {
279                    GuiHelper.runInEDT(this::displayNotification);
280                }
281            } catch (RuntimeException t) { // NOPMD
282                // Don't let exception stop time thread
283                Logging.error("Autosave failed:");
284                Logging.error(t);
285            }
286        }
287    }
288
289    protected void displayNotification() {
290        new Notification(tr("Your work has been saved automatically."))
291        .setDuration(Notification.TIME_SHORT)
292        .show();
293    }
294
295    @Override
296    public void layerOrderChanged(LayerOrderChangeEvent e) {
297        // Do nothing
298    }
299
300    private void registerNewlayer(OsmDataLayer layer) {
301        synchronized (layersLock) {
302            layer.getDataSet().addDataSetListener(datasetAdapter);
303            layersInfo.add(new AutosaveLayerInfo<>(layer));
304        }
305    }
306
307    private void registerNewlayer(NoteLayer layer) {
308        synchronized (layersLock) {
309            layer.getNoteData().addNoteDataUpdateListener(this);
310            layersInfo.add(new AutosaveLayerInfo<>(layer));
311        }
312    }
313
314    @Override
315    public void layerAdded(LayerAddEvent e) {
316        if (e.getAddedLayer() instanceof OsmDataLayer) {
317            registerNewlayer((OsmDataLayer) e.getAddedLayer());
318        } else if (e.getAddedLayer() instanceof NoteLayer) {
319            registerNewlayer((NoteLayer) e.getAddedLayer());
320        }
321    }
322
323    @Override
324    public void layerRemoving(LayerRemoveEvent e) {
325        if (e.getRemovedLayer() instanceof OsmDataLayer) {
326            synchronized (layersLock) {
327                OsmDataLayer osmLayer = (OsmDataLayer) e.getRemovedLayer();
328                osmLayer.getDataSet().removeDataSetListener(datasetAdapter);
329                cleanupLayer(osmLayer);
330            }
331        } else if (e.getRemovedLayer() instanceof NoteLayer) {
332            synchronized (layersLock) {
333                NoteLayer noteLayer = (NoteLayer) e.getRemovedLayer();
334                noteLayer.getNoteData().removeNoteDataUpdateListener(this);
335                cleanupLayer(noteLayer);
336            }
337        }
338    }
339
340    private void cleanupLayer(AbstractModifiableLayer removedLayer) {
341        Iterator<AutosaveLayerInfo<?>> it = layersInfo.iterator();
342        while (it.hasNext()) {
343            AutosaveLayerInfo<?> info = it.next();
344            if (info.layer == removedLayer) {
345
346                savelayer(info);
347                File lastFile = info.backupFiles.pollLast();
348                if (lastFile != null) {
349                    moveToDeletedLayersFolder(lastFile);
350                }
351                for (File file: info.backupFiles) {
352                    if (Utils.deleteFile(file)) {
353                        Utils.deleteFile(getPidFile(file));
354                    }
355                }
356
357                it.remove();
358            }
359        }
360    }
361
362    @Override
363    public void processDatasetEvent(AbstractDatasetChangedEvent event) {
364        changedDatasets.add(event.getDataset());
365    }
366
367    @Override
368    public void noteDataUpdated(NoteData data) {
369        changedNoteData.add(data);
370    }
371
372    @Override
373    public void selectedNoteChanged(NoteData noteData) {
374        // Do nothing
375    }
376
377    protected File getPidFile(File osmFile) {
378        return new File(autosaveDir, osmFile.getName().replaceFirst("[.][^.]+$", ".pid"));
379    }
380
381    /**
382     * Replies the list of .osm files still present in autosave dir, that are not currently managed by another instance of JOSM.
383     * These files are hence unsaved layers from an old instance of JOSM that crashed and may be recovered by this instance.
384     * @return The list of .osm files still present in autosave dir, that are not currently managed by another instance of JOSM
385     */
386    public List<File> getUnsavedLayersFiles() {
387        List<File> result = new ArrayList<>();
388        try {
389            File[] files = autosaveDir.listFiles((FileFilter)
390                    pathname -> OsmImporter.FILE_FILTER.accept(pathname) || NoteImporter.FILE_FILTER.accept(pathname));
391            if (files == null)
392                return result;
393            for (File file: files) {
394                if (file.isFile()) {
395                    boolean skipFile = false;
396                    File pidFile = getPidFile(file);
397                    if (pidFile.exists()) {
398                        try (BufferedReader reader = Files.newBufferedReader(pidFile.toPath(), StandardCharsets.UTF_8)) {
399                            String jvmId = reader.readLine();
400                            if (jvmId != null) {
401                                String pid = jvmId.split("@")[0];
402                                skipFile = jvmPerfDataFileExists(pid);
403                            }
404                        } catch (IOException | SecurityException t) {
405                            Logging.error(t);
406                        }
407                    }
408                    if (!skipFile) {
409                        result.add(file);
410                    }
411                }
412            }
413        } catch (SecurityException e) {
414            Logging.log(Logging.LEVEL_ERROR, "Unable to list unsaved layers files", e);
415        }
416        return result;
417    }
418
419    private static boolean jvmPerfDataFileExists(final String jvmId) {
420        File jvmDir = new File(getSystemProperty("java.io.tmpdir") + File.separator + "hsperfdata_" + getSystemProperty("user.name"));
421        if (jvmDir.exists() && jvmDir.canRead()) {
422            File[] files = jvmDir.listFiles((FileFilter) file -> file.getName().equals(jvmId) && file.isFile());
423            return files != null && files.length == 1;
424        }
425        return false;
426    }
427
428    /**
429     * Recover the unsaved layers and open them asynchronously.
430     * @return A future that can be used to wait for the completion of this task.
431     */
432    public Future<?> recoverUnsavedLayers() {
433        List<File> files = getUnsavedLayersFiles();
434        final OpenFileTask openFileTsk = new OpenFileTask(files, null, tr("Restoring files"));
435        final Future<?> openFilesFuture = MainApplication.worker.submit(openFileTsk);
436        return MainApplication.worker.submit(() -> {
437            try {
438                // Wait for opened tasks to be generated.
439                openFilesFuture.get();
440                for (File f: openFileTsk.getSuccessfullyOpenedFiles()) {
441                    moveToDeletedLayersFolder(f);
442                }
443            } catch (InterruptedException | ExecutionException e) {
444                Logging.error(e);
445            }
446        });
447    }
448
449    /**
450     * Move file to the deleted layers directory.
451     * If moving does not work, it will try to delete the file directly.
452     * Afterwards, if the number of deleted layers gets larger than PROP_DELETED_LAYERS,
453     * some files in the deleted layers directory will be removed.
454     *
455     * @param f the file, usually from the autosave dir
456     */
457    private void moveToDeletedLayersFolder(File f) {
458        File backupFile = new File(deletedLayersDir, f.getName());
459        File pidFile = getPidFile(f);
460
461        if (backupFile.exists()) {
462            deletedLayers.remove(backupFile);
463            Utils.deleteFile(backupFile, marktr("Unable to delete old backup file {0}"));
464        }
465        if (f.renameTo(backupFile)) {
466            deletedLayers.add(backupFile);
467            Utils.deleteFile(pidFile);
468        } else {
469            Logging.warn(String.format("Could not move autosaved file %s to %s folder", f.getName(), deletedLayersDir.getName()));
470            // we cannot move to deleted folder, so just try to delete it directly
471            if (Utils.deleteFile(f, marktr("Unable to delete backup file {0}"))) {
472                Utils.deleteFile(pidFile, marktr("Unable to delete PID file {0}"));
473            }
474        }
475        while (deletedLayers.size() > PROP_DELETED_LAYERS.get()) {
476            File next = deletedLayers.remove();
477            if (next == null) {
478                break;
479            }
480            Utils.deleteFile(next, marktr("Unable to delete archived backup file {0}"));
481        }
482    }
483
484    /**
485     * Mark all unsaved layers as deleted. They are still preserved in the deleted layers folder.
486     */
487    public void discardUnsavedLayers() {
488        for (File f: getUnsavedLayersFiles()) {
489            moveToDeletedLayersFolder(f);
490        }
491    }
492}