001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.awt.GraphicsEnvironment;
007import java.io.File;
008import java.io.FileNotFoundException;
009import java.io.IOException;
010import java.nio.charset.StandardCharsets;
011import java.nio.file.Files;
012import java.nio.file.Path;
013import java.nio.file.Paths;
014import java.util.ArrayList;
015import java.util.Arrays;
016import java.util.Collection;
017import java.util.Collections;
018import java.util.EnumMap;
019import java.util.Enumeration;
020import java.util.HashMap;
021import java.util.Iterator;
022import java.util.List;
023import java.util.Map;
024import java.util.Map.Entry;
025import java.util.SortedMap;
026import java.util.TreeMap;
027import java.util.TreeSet;
028import java.util.function.Predicate;
029import java.util.regex.Pattern;
030import java.util.stream.Collectors;
031
032import javax.swing.JOptionPane;
033import javax.swing.JTree;
034import javax.swing.tree.DefaultMutableTreeNode;
035import javax.swing.tree.TreeModel;
036import javax.swing.tree.TreeNode;
037
038import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
039import org.openstreetmap.josm.data.projection.ProjectionRegistry;
040import org.openstreetmap.josm.data.validation.tests.Addresses;
041import org.openstreetmap.josm.data.validation.tests.ApiCapabilitiesTest;
042import org.openstreetmap.josm.data.validation.tests.BarriersEntrances;
043import org.openstreetmap.josm.data.validation.tests.Coastlines;
044import org.openstreetmap.josm.data.validation.tests.ConditionalKeys;
045import org.openstreetmap.josm.data.validation.tests.CrossingWays;
046import org.openstreetmap.josm.data.validation.tests.DuplicateNode;
047import org.openstreetmap.josm.data.validation.tests.DuplicateRelation;
048import org.openstreetmap.josm.data.validation.tests.DuplicateWay;
049import org.openstreetmap.josm.data.validation.tests.DuplicatedWayNodes;
050import org.openstreetmap.josm.data.validation.tests.Highways;
051import org.openstreetmap.josm.data.validation.tests.InternetTags;
052import org.openstreetmap.josm.data.validation.tests.Lanes;
053import org.openstreetmap.josm.data.validation.tests.LongSegment;
054import org.openstreetmap.josm.data.validation.tests.MapCSSTagChecker;
055import org.openstreetmap.josm.data.validation.tests.MultipolygonTest;
056import org.openstreetmap.josm.data.validation.tests.NameMismatch;
057import org.openstreetmap.josm.data.validation.tests.OpeningHourTest;
058import org.openstreetmap.josm.data.validation.tests.OverlappingWays;
059import org.openstreetmap.josm.data.validation.tests.PowerLines;
060import org.openstreetmap.josm.data.validation.tests.PublicTransportRouteTest;
061import org.openstreetmap.josm.data.validation.tests.RelationChecker;
062import org.openstreetmap.josm.data.validation.tests.RightAngleBuildingTest;
063import org.openstreetmap.josm.data.validation.tests.SelfIntersectingWay;
064import org.openstreetmap.josm.data.validation.tests.SimilarNamedWays;
065import org.openstreetmap.josm.data.validation.tests.TagChecker;
066import org.openstreetmap.josm.data.validation.tests.TurnrestrictionTest;
067import org.openstreetmap.josm.data.validation.tests.UnclosedWays;
068import org.openstreetmap.josm.data.validation.tests.UnconnectedWays;
069import org.openstreetmap.josm.data.validation.tests.UntaggedNode;
070import org.openstreetmap.josm.data.validation.tests.UntaggedWay;
071import org.openstreetmap.josm.data.validation.tests.WayConnectedToArea;
072import org.openstreetmap.josm.data.validation.tests.WronglyOrderedWays;
073import org.openstreetmap.josm.gui.MainApplication;
074import org.openstreetmap.josm.gui.layer.ValidatorLayer;
075import org.openstreetmap.josm.gui.preferences.projection.ProjectionPreference;
076import org.openstreetmap.josm.gui.util.GuiHelper;
077import org.openstreetmap.josm.spi.preferences.Config;
078import org.openstreetmap.josm.tools.AlphanumComparator;
079import org.openstreetmap.josm.tools.Logging;
080import org.openstreetmap.josm.tools.Utils;
081
082/**
083 * A OSM data validator.
084 *
085 * @author Francisco R. Santos <frsantos@gmail.com>
086 */
087public final class OsmValidator {
088
089    private OsmValidator() {
090        // Hide default constructor for utilities classes
091    }
092
093    private static volatile ValidatorLayer errorLayer;
094
095    /** Grid detail, multiplier of east,north values for valuable cell sizing */
096    private static double griddetail;
097
098    private static final SortedMap<String, String> ignoredErrors = new TreeMap<>();
099    /**
100     * All registered tests
101     */
102    private static final Collection<Class<? extends Test>> allTests = new ArrayList<>();
103    private static final Map<String, Test> allTestsMap = new HashMap<>();
104
105    /**
106     * All available tests in core
107     */
108    @SuppressWarnings("unchecked")
109    private static final Class<Test>[] CORE_TEST_CLASSES = new Class[] {
110        /* FIXME - unique error numbers for tests aren't properly unique - ignoring will not work as expected */
111        DuplicateNode.class, // ID    1 ..   99
112        OverlappingWays.class, // ID  101 ..  199
113        UntaggedNode.class, // ID  201 ..  299
114        UntaggedWay.class, // ID  301 ..  399
115        SelfIntersectingWay.class, // ID  401 ..  499
116        DuplicatedWayNodes.class, // ID  501 ..  599
117        CrossingWays.Ways.class, // ID  601 ..  699
118        CrossingWays.Boundaries.class, // ID  601 ..  699
119        CrossingWays.Barrier.class, // ID  601 ..  699
120        CrossingWays.SelfCrossing.class, // ID  601 ..  699
121        SimilarNamedWays.class, // ID  701 ..  799
122        Coastlines.class, // ID  901 ..  999
123        WronglyOrderedWays.class, // ID 1001 .. 1099
124        UnclosedWays.class, // ID 1101 .. 1199
125        TagChecker.class, // ID 1201 .. 1299
126        UnconnectedWays.UnconnectedHighways.class, // ID 1301 .. 1399
127        UnconnectedWays.UnconnectedRailways.class, // ID 1301 .. 1399
128        UnconnectedWays.UnconnectedWaterways.class, // ID 1301 .. 1399
129        UnconnectedWays.UnconnectedNaturalOrLanduse.class, // ID 1301 .. 1399
130        UnconnectedWays.UnconnectedPower.class, // ID 1301 .. 1399
131        DuplicateWay.class, // ID 1401 .. 1499
132        NameMismatch.class, // ID  1501 ..  1599
133        MultipolygonTest.class, // ID  1601 ..  1699
134        RelationChecker.class, // ID  1701 ..  1799
135        TurnrestrictionTest.class, // ID  1801 ..  1899
136        DuplicateRelation.class, // ID 1901 .. 1999
137        WayConnectedToArea.class, // ID 2301 .. 2399
138        PowerLines.class, // ID 2501 .. 2599
139        Addresses.class, // ID 2601 .. 2699
140        Highways.class, // ID 2701 .. 2799
141        BarriersEntrances.class, // ID 2801 .. 2899
142        OpeningHourTest.class, // 2901 .. 2999
143        MapCSSTagChecker.class, // 3000 .. 3099
144        Lanes.class, // 3100 .. 3199
145        ConditionalKeys.class, // 3200 .. 3299
146        InternetTags.class, // 3300 .. 3399
147        ApiCapabilitiesTest.class, // 3400 .. 3499
148        LongSegment.class, // 3500 .. 3599
149        PublicTransportRouteTest.class, // 3600 .. 3699
150        RightAngleBuildingTest.class, // 3700 .. 3799
151    };
152
153    /**
154     * Adds a test to the list of available tests
155     * @param testClass The test class
156     */
157    public static void addTest(Class<? extends Test> testClass) {
158        allTests.add(testClass);
159        try {
160            allTestsMap.put(testClass.getName(), testClass.getConstructor().newInstance());
161        } catch (ReflectiveOperationException e) {
162            Logging.error(e);
163        }
164    }
165
166    static {
167        for (Class<? extends Test> testClass : CORE_TEST_CLASSES) {
168            addTest(testClass);
169        }
170    }
171
172    /**
173     * Initializes {@code OsmValidator}.
174     */
175    public static void initialize() {
176        checkValidatorDir();
177        initializeGridDetail();
178        loadIgnoredErrors();
179    }
180
181    /**
182     * Returns the validator directory.
183     *
184     * @return The validator directory
185     */
186    public static String getValidatorDir() {
187        File dir = new File(Config.getDirs().getUserDataDirectory(true), "validator");
188        try {
189            return dir.getAbsolutePath();
190        } catch (SecurityException e) {
191            Logging.log(Logging.LEVEL_ERROR, null, e);
192            return dir.getPath();
193        }
194    }
195
196    /**
197     * Check if validator directory exists (store ignored errors file)
198     */
199    private static void checkValidatorDir() {
200        File pathDir = new File(getValidatorDir());
201        try {
202            if (!pathDir.exists()) {
203                Utils.mkDirs(pathDir);
204            }
205        } catch (SecurityException e) {
206            Logging.log(Logging.LEVEL_ERROR, "Unable to check validator directory", e);
207        }
208    }
209
210    private static void loadIgnoredErrors() {
211        ignoredErrors.clear();
212        if (ValidatorPrefHelper.PREF_USE_IGNORE.get()) {
213            Config.getPref().getListOfMaps(ValidatorPrefHelper.PREF_IGNORELIST).forEach(ignoredErrors::putAll);
214            Path path = Paths.get(getValidatorDir()).resolve("ignorederrors");
215            try {
216                if (path.toFile().exists()) {
217                    try {
218                        TreeSet<String> treeSet = new TreeSet<>();
219                        treeSet.addAll(Files.readAllLines(path, StandardCharsets.UTF_8));
220                        treeSet.forEach(ignore -> ignoredErrors.putIfAbsent(ignore, ""));
221
222                        saveIgnoredErrors();
223                        Files.deleteIfExists(path);
224
225                    } catch (FileNotFoundException e) {
226                        Logging.debug(Logging.getErrorMessage(e));
227                    } catch (IOException e) {
228                        Logging.error(e);
229                    }
230                }
231            } catch (SecurityException e) {
232                Logging.log(Logging.LEVEL_ERROR, "Unable to load ignored errors", e);
233            }
234        }
235    }
236
237    /**
238     * Adds an ignored error
239     * @param s The ignore group / sub group name
240     * @see TestError#getIgnoreGroup()
241     * @see TestError#getIgnoreSubGroup()
242     */
243    public static void addIgnoredError(String s) {
244        addIgnoredError(s, "");
245    }
246
247    /**
248     * Adds an ignored error
249     * @param s The ignore group / sub group name
250     * @param description What the error actually is
251     * @see TestError#getIgnoreGroup()
252     * @see TestError#getIgnoreSubGroup()
253     */
254    public static void addIgnoredError(String s, String description) {
255        if (description == null) description = "";
256        ignoredErrors.put(s, description);
257    }
258
259    /**
260     *  Make sure that we don't keep single entries for a "group ignore" or
261     *  multiple different entries for the single entries that are in the same group.
262     */
263    private static void cleanupIgnoredErrors() {
264        if (ignoredErrors.size() > 1) {
265            List<String> toRemove = new ArrayList<>();
266
267            Iterator<Entry<String, String>> iter = ignoredErrors.entrySet().iterator();
268            Entry<String, String> last = iter.next();
269            while (iter.hasNext()) {
270                Entry<String, String> entry = iter.next();
271                if (entry.getKey().startsWith(last.getKey())) {
272                    toRemove.add(entry.getKey());
273                } else {
274                    last = entry;
275                }
276            }
277            toRemove.forEach(ignoredErrors::remove);
278        }
279
280        Map<String, String> tmap = buildIgnore(buildJTreeList());
281        if (!tmap.isEmpty()) {
282            ignoredErrors.clear();
283            ignoredErrors.putAll(tmap);
284        }
285    }
286
287    /**
288     * Check if a error should be ignored
289     * @param s The ignore group / sub group name
290     * @return <code>true</code> to ignore that error
291     */
292    public static boolean hasIgnoredError(String s) {
293        return ignoredErrors.containsKey(s);
294    }
295
296    /**
297     * Get the list of all ignored errors
298     * @return The <code>Collection&lt;String&gt;</code> of errors that are ignored
299     */
300    public static SortedMap<String, String> getIgnoredErrors() {
301        return ignoredErrors;
302    }
303
304    /**
305     * Build a JTree with a list
306     * @return &lt;type&gt;list as a {@code JTree}
307     */
308    public static JTree buildJTreeList() {
309        DefaultMutableTreeNode root = new DefaultMutableTreeNode(tr("Ignore list"));
310        final Pattern elemId1Pattern = Pattern.compile(":(r|w|n)_");
311        final Pattern elemId2Pattern = Pattern.compile("^[0-9]+$");
312        for (Entry<String, String> e: ignoredErrors.entrySet()) {
313            String key = e.getKey();
314            String value = e.getValue();
315            ArrayList<String> ignoredWayList = new ArrayList<>();
316            String[] osmobjects = elemId1Pattern.split(key);
317            for (int i = 1; i < osmobjects.length; i++) {
318                String osmid = osmobjects[i];
319                if (elemId2Pattern.matcher(osmid).matches()) {
320                    osmid = '_' + osmid;
321                    int index = key.indexOf(osmid);
322                    if (index < key.lastIndexOf(']')) continue;
323                    char type = key.charAt(index - 1);
324                    ignoredWayList.add(type + osmid);
325                }
326            }
327            for (String osmignore : ignoredWayList) {
328                key = key.replace(':' + osmignore, "");
329            }
330
331            DefaultMutableTreeNode trunk;
332            DefaultMutableTreeNode branch;
333
334            if (value != null && !value.isEmpty()) {
335                trunk = inTree(root, value);
336                branch = inTree(trunk, key);
337                trunk.add(branch);
338            } else {
339                trunk = inTree(root, key);
340                branch = trunk;
341            }
342            ignoredWayList.forEach(osmignore -> branch.add(new DefaultMutableTreeNode(osmignore)));
343
344            root.add(trunk);
345        }
346        return new JTree(root);
347    }
348
349    private static DefaultMutableTreeNode inTree(DefaultMutableTreeNode root, String name) {
350        @SuppressWarnings("unchecked")
351        Enumeration<TreeNode> trunks = root.children();
352        while (trunks.hasMoreElements()) {
353            TreeNode ttrunk = trunks.nextElement();
354            if (ttrunk instanceof DefaultMutableTreeNode) {
355                DefaultMutableTreeNode trunk = (DefaultMutableTreeNode) ttrunk;
356                if (name.equals(trunk.getUserObject())) {
357                    return trunk;
358                }
359            }
360        }
361        return new DefaultMutableTreeNode(name);
362    }
363
364    /**
365     * Build a {@code HashMap} from a tree of ignored errors
366     * @param tree The JTree of ignored errors
367     * @return A {@code HashMap} of the ignored errors for comparison
368     */
369    public static Map<String, String> buildIgnore(JTree tree) {
370        TreeModel model = tree.getModel();
371        DefaultMutableTreeNode root = (DefaultMutableTreeNode) model.getRoot();
372        return buildIgnore(model, root);
373    }
374
375    private static Map<String, String> buildIgnore(TreeModel model, DefaultMutableTreeNode node) {
376        HashMap<String, String> rHashMap = new HashMap<>();
377
378        String osmids = node.getUserObject().toString();
379        String description = "";
380
381        if (!model.getRoot().equals(node)) {
382            description = ((DefaultMutableTreeNode) node.getParent()).getUserObject().toString();
383        } else {
384            description = node.getUserObject().toString();
385        }
386        if (tr("Ignore list").equals(description)) description = "";
387        if (!osmids.matches("^[0-9]+(_.*|$)")) {
388            description = osmids;
389            osmids = "";
390        }
391
392
393        StringBuilder sb = new StringBuilder();
394        for (int i = 0; i < model.getChildCount(node); i++) {
395            DefaultMutableTreeNode child = (DefaultMutableTreeNode) model.getChild(node, i);
396            if (model.getChildCount(child) == 0) {
397                String ignoreName = child.getUserObject().toString();
398                if (ignoreName.matches("^(r|w|n)_.*")) {
399                    sb.append(':').append(child.getUserObject().toString());
400                } else if (ignoreName.matches("^[0-9]+(_.*|)$")) {
401                    rHashMap.put(ignoreName, description);
402                }
403            } else {
404                rHashMap.putAll(buildIgnore(model, child));
405            }
406        }
407        osmids += sb.toString();
408        if (!osmids.isEmpty() && osmids.indexOf(':') != 0) rHashMap.put(osmids, description);
409        return rHashMap;
410    }
411
412    /**
413     * Reset the error list by deleting {@code validator.ignorelist}
414     */
415    public static void resetErrorList() {
416        saveIgnoredErrors();
417        Config.getPref().putListOfMaps(ValidatorPrefHelper.PREF_IGNORELIST, null);
418        OsmValidator.initialize();
419    }
420
421    /**
422     * Saves the names of the ignored errors to a preference
423     */
424    public static void saveIgnoredErrors() {
425        List<Map<String, String>> list = new ArrayList<>();
426        cleanupIgnoredErrors();
427        list.add(ignoredErrors);
428        int i = 0;
429        while (i < list.size()) {
430            if (list.get(i) == null || list.get(i).isEmpty()) {
431                list.remove(i);
432                continue;
433            }
434            i++;
435        }
436        if (list.isEmpty()) list = null;
437        Config.getPref().putListOfMaps(ValidatorPrefHelper.PREF_IGNORELIST, list);
438    }
439
440    /**
441     * Initializes error layer.
442     */
443    public static synchronized void initializeErrorLayer() {
444        if (!ValidatorPrefHelper.PREF_LAYER.get())
445            return;
446        if (errorLayer == null) {
447            errorLayer = new ValidatorLayer();
448            MainApplication.getLayerManager().addLayer(errorLayer);
449        }
450    }
451
452    /**
453     * Resets error layer.
454     * @since 11852
455     */
456    public static synchronized void resetErrorLayer() {
457        errorLayer = null;
458    }
459
460    /**
461     * Gets a map from simple names to all tests.
462     * @return A map of all tests, indexed and sorted by the name of their Java class
463     */
464    public static SortedMap<String, Test> getAllTestsMap() {
465        applyPrefs(allTestsMap, false);
466        applyPrefs(allTestsMap, true);
467        return new TreeMap<>(allTestsMap);
468    }
469
470    /**
471     * Returns the instance of the given test class.
472     * @param <T> testClass type
473     * @param testClass The class of test to retrieve
474     * @return the instance of the given test class, if any, or {@code null}
475     * @since 6670
476     */
477    @SuppressWarnings("unchecked")
478    public static <T extends Test> T getTest(Class<T> testClass) {
479        if (testClass == null) {
480            return null;
481        }
482        return (T) allTestsMap.get(testClass.getName());
483    }
484
485    private static void applyPrefs(Map<String, Test> tests, boolean beforeUpload) {
486        for (String testName : Config.getPref().getList(beforeUpload
487        ? ValidatorPrefHelper.PREF_SKIP_TESTS_BEFORE_UPLOAD : ValidatorPrefHelper.PREF_SKIP_TESTS)) {
488            Test test = tests.get(testName);
489            if (test != null) {
490                if (beforeUpload) {
491                    test.testBeforeUpload = false;
492                } else {
493                    test.enabled = false;
494                }
495            }
496        }
497    }
498
499    /**
500     * Gets all tests that are possible
501     * @return The tests
502     */
503    public static Collection<Test> getTests() {
504        return getAllTestsMap().values();
505    }
506
507    /**
508     * Gets all tests that are run
509     * @param beforeUpload To get the ones that are run before upload
510     * @return The tests
511     */
512    public static Collection<Test> getEnabledTests(boolean beforeUpload) {
513        Collection<Test> enabledTests = getTests();
514        for (Test t : new ArrayList<>(enabledTests)) {
515            if (beforeUpload ? t.testBeforeUpload : t.enabled) {
516                continue;
517            }
518            enabledTests.remove(t);
519        }
520        return enabledTests;
521    }
522
523    /**
524     * Gets the list of all available test classes
525     *
526     * @return A collection of the test classes
527     */
528    public static Collection<Class<? extends Test>> getAllAvailableTestClasses() {
529        return Collections.unmodifiableCollection(allTests);
530    }
531
532    /**
533     * Initialize grid details based on current projection system. Values based on
534     * the original value fixed for EPSG:4326 (10000) using heuristics (that is, test&amp;error
535     * until most bugs were discovered while keeping the processing time reasonable)
536     */
537    public static void initializeGridDetail() {
538        String code = ProjectionRegistry.getProjection().toCode();
539        if (Arrays.asList(ProjectionPreference.wgs84.allCodes()).contains(code)) {
540            OsmValidator.griddetail = 10_000;
541        } else if (Arrays.asList(ProjectionPreference.mercator.allCodes()).contains(code)) {
542            OsmValidator.griddetail = 0.01;
543        } else if (Arrays.asList(ProjectionPreference.lambert.allCodes()).contains(code)) {
544            OsmValidator.griddetail = 0.1;
545        } else {
546            OsmValidator.griddetail = 1.0;
547        }
548    }
549
550    /**
551     * Returns grid detail, multiplier of east,north values for valuable cell sizing
552     * @return grid detail
553     * @since 11852
554     */
555    public static double getGridDetail() {
556        return griddetail;
557    }
558
559    private static boolean testsInitialized;
560
561    /**
562     * Initializes all tests if this operations hasn't been performed already.
563     */
564    public static synchronized void initializeTests() {
565        if (!testsInitialized) {
566            Logging.debug("Initializing validator tests");
567            final long startTime = System.currentTimeMillis();
568            initializeTests(getTests());
569            testsInitialized = true;
570            if (Logging.isDebugEnabled()) {
571                final long elapsedTime = System.currentTimeMillis() - startTime;
572                Logging.debug("Initializing validator tests completed in {0}", Utils.getDurationString(elapsedTime));
573            }
574        }
575    }
576
577    /**
578     * Initializes all tests
579     * @param allTests The tests to initialize
580     */
581    public static void initializeTests(Collection<? extends Test> allTests) {
582        for (Test test : allTests) {
583            try {
584                if (test.enabled) {
585                    test.initialize();
586                }
587            } catch (Exception e) { // NOPMD
588                String message = tr("Error initializing test {0}:\n {1}", test.getClass().getSimpleName(), e);
589                Logging.error(message);
590                if (!GraphicsEnvironment.isHeadless()) {
591                    GuiHelper.runInEDT(() ->
592                        JOptionPane.showMessageDialog(MainApplication.getMainFrame(), message, tr("Error"), JOptionPane.ERROR_MESSAGE)
593                    );
594                }
595            }
596        }
597    }
598
599    /**
600     * Groups the given collection of errors by severity, then message, then description.
601     * @param errors list of errors to group
602     * @param filterToUse optional filter
603     * @return collection of errors grouped by severity, then message, then description
604     * @since 12667
605     */
606    public static Map<Severity, Map<String, Map<String, List<TestError>>>> getErrorsBySeverityMessageDescription(
607            Collection<TestError> errors, Predicate<? super TestError> filterToUse) {
608        return errors.stream().filter(filterToUse).collect(
609                Collectors.groupingBy(TestError::getSeverity, () -> new EnumMap<>(Severity.class),
610                        Collectors.groupingBy(TestError::getMessage, () -> new TreeMap<>(AlphanumComparator.getInstance()),
611                                Collectors.groupingBy(e -> e.getDescription() == null ? "" : e.getDescription(),
612                                        () -> new TreeMap<>(AlphanumComparator.getInstance()),
613                                        Collectors.toList()
614                                ))));
615    }
616}