001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.BufferedReader;
007import java.io.IOException;
008import java.io.InputStream;
009import java.io.Reader;
010import java.io.StringReader;
011import java.lang.reflect.Method;
012import java.text.MessageFormat;
013import java.util.ArrayList;
014import java.util.Collection;
015import java.util.Collections;
016import java.util.HashMap;
017import java.util.HashSet;
018import java.util.Iterator;
019import java.util.LinkedHashMap;
020import java.util.LinkedHashSet;
021import java.util.LinkedList;
022import java.util.List;
023import java.util.Locale;
024import java.util.Map;
025import java.util.Objects;
026import java.util.Optional;
027import java.util.Set;
028import java.util.function.Predicate;
029import java.util.regex.Matcher;
030import java.util.regex.Pattern;
031
032import org.openstreetmap.josm.command.ChangePropertyCommand;
033import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
034import org.openstreetmap.josm.command.Command;
035import org.openstreetmap.josm.command.DeleteCommand;
036import org.openstreetmap.josm.command.SequenceCommand;
037import org.openstreetmap.josm.data.coor.LatLon;
038import org.openstreetmap.josm.data.osm.DataSet;
039import org.openstreetmap.josm.data.osm.IPrimitive;
040import org.openstreetmap.josm.data.osm.OsmPrimitive;
041import org.openstreetmap.josm.data.osm.OsmUtils;
042import org.openstreetmap.josm.data.osm.Relation;
043import org.openstreetmap.josm.data.osm.Tag;
044import org.openstreetmap.josm.data.osm.Way;
045import org.openstreetmap.josm.data.preferences.sources.SourceEntry;
046import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
047import org.openstreetmap.josm.data.validation.OsmValidator;
048import org.openstreetmap.josm.data.validation.Severity;
049import org.openstreetmap.josm.data.validation.Test;
050import org.openstreetmap.josm.data.validation.TestError;
051import org.openstreetmap.josm.gui.mappaint.Environment;
052import org.openstreetmap.josm.gui.mappaint.Keyword;
053import org.openstreetmap.josm.gui.mappaint.MultiCascade;
054import org.openstreetmap.josm.gui.mappaint.mapcss.Condition;
055import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ClassCondition;
056import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ExpressionCondition;
057import org.openstreetmap.josm.gui.mappaint.mapcss.Expression;
058import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.ParameterFunction;
059import org.openstreetmap.josm.gui.mappaint.mapcss.Functions;
060import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction;
061import org.openstreetmap.josm.gui.mappaint.mapcss.LiteralExpression;
062import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
063import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration;
064import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
065import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource.MapCSSRuleIndex;
066import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
067import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.AbstractSelector;
068import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector;
069import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.OptimizedGeneralSelector;
070import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser;
071import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
072import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError;
073import org.openstreetmap.josm.gui.progress.ProgressMonitor;
074import org.openstreetmap.josm.io.CachedFile;
075import org.openstreetmap.josm.io.FileWatcher;
076import org.openstreetmap.josm.io.IllegalDataException;
077import org.openstreetmap.josm.io.UTFInputStreamReader;
078import org.openstreetmap.josm.spi.preferences.Config;
079import org.openstreetmap.josm.tools.CheckParameterUtil;
080import org.openstreetmap.josm.tools.DefaultGeoProperty;
081import org.openstreetmap.josm.tools.GeoProperty;
082import org.openstreetmap.josm.tools.GeoPropertyIndex;
083import org.openstreetmap.josm.tools.I18n;
084import org.openstreetmap.josm.tools.Logging;
085import org.openstreetmap.josm.tools.MultiMap;
086import org.openstreetmap.josm.tools.Territories;
087import org.openstreetmap.josm.tools.Utils;
088
089/**
090 * MapCSS-based tag checker/fixer.
091 * @since 6506
092 */
093public class MapCSSTagChecker extends Test.TagTest {
094    MapCSSTagCheckerIndex indexData;
095    final Set<OsmPrimitive> tested = new HashSet<>();
096
097    /**
098    * A grouped MapCSSRule with multiple selectors for a single declaration.
099    * @see MapCSSRule
100    */
101    public static class GroupedMapCSSRule {
102        /** MapCSS selectors **/
103        public final List<Selector> selectors;
104        /** MapCSS declaration **/
105        public final Declaration declaration;
106
107        /**
108         * Constructs a new {@code GroupedMapCSSRule}.
109         * @param selectors MapCSS selectors
110         * @param declaration MapCSS declaration
111         */
112        public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) {
113            this.selectors = selectors;
114            this.declaration = declaration;
115        }
116
117        @Override
118        public int hashCode() {
119            return Objects.hash(selectors, declaration);
120        }
121
122        @Override
123        public boolean equals(Object obj) {
124            if (this == obj) return true;
125            if (obj == null || getClass() != obj.getClass()) return false;
126            GroupedMapCSSRule that = (GroupedMapCSSRule) obj;
127            return Objects.equals(selectors, that.selectors) &&
128                    Objects.equals(declaration, that.declaration);
129        }
130
131        @Override
132        public String toString() {
133            return "GroupedMapCSSRule [selectors=" + selectors + ", declaration=" + declaration + ']';
134        }
135    }
136
137    /**
138     * The preference key for tag checker source entries.
139     * @since 6670
140     */
141    public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
142
143    /**
144     * Constructs a new {@code MapCSSTagChecker}.
145     */
146    public MapCSSTagChecker() {
147        super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
148    }
149
150    /**
151     * Represents a fix to a validation test. The fixing {@link Command} can be obtained by {@link #createCommand(OsmPrimitive, Selector)}.
152     */
153    @FunctionalInterface
154    interface FixCommand {
155        /**
156         * Creates the fixing {@link Command} for the given primitive. The {@code matchingSelector} is used to evaluate placeholders
157         * (cf. {@link MapCSSTagChecker.TagCheck#insertArguments(Selector, String, OsmPrimitive)}).
158         * @param p OSM primitive
159         * @param matchingSelector  matching selector
160         * @return fix command
161         */
162        Command createCommand(OsmPrimitive p, Selector matchingSelector);
163
164        /**
165         * Checks that object is either an {@link Expression} or a {@link String}.
166         * @param obj object to check
167         * @throws IllegalArgumentException if object is not an {@code Expression} or a {@code String}
168         */
169        static void checkObject(final Object obj) {
170            CheckParameterUtil.ensureThat(obj instanceof Expression || obj instanceof String,
171                    () -> "instance of Exception or String expected, but got " + obj);
172        }
173
174        /**
175         * Evaluates given object as {@link Expression} or {@link String} on the matched {@link OsmPrimitive} and {@code matchingSelector}.
176         * @param obj object to evaluate ({@link Expression} or {@link String})
177         * @param p OSM primitive
178         * @param matchingSelector matching selector
179         * @return result string
180         */
181        static String evaluateObject(final Object obj, final OsmPrimitive p, final Selector matchingSelector) {
182            final String s;
183            if (obj instanceof Expression) {
184                s = (String) ((Expression) obj).evaluate(new Environment(p));
185            } else if (obj instanceof String) {
186                s = (String) obj;
187            } else {
188                return null;
189            }
190            return TagCheck.insertArguments(matchingSelector, s, p);
191        }
192
193        /**
194         * Creates a fixing command which executes a {@link ChangePropertyCommand} on the specified tag.
195         * @param obj object to evaluate ({@link Expression} or {@link String})
196         * @return created fix command
197         */
198        static FixCommand fixAdd(final Object obj) {
199            checkObject(obj);
200            return new FixCommand() {
201                @Override
202                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
203                    final Tag tag = Tag.ofString(evaluateObject(obj, p, matchingSelector));
204                    return new ChangePropertyCommand(p, tag.getKey(), tag.getValue());
205                }
206
207                @Override
208                public String toString() {
209                    return "fixAdd: " + obj;
210                }
211            };
212        }
213
214        /**
215         * Creates a fixing command which executes a {@link ChangePropertyCommand} to delete the specified key.
216         * @param obj object to evaluate ({@link Expression} or {@link String})
217         * @return created fix command
218         */
219        static FixCommand fixRemove(final Object obj) {
220            checkObject(obj);
221            return new FixCommand() {
222                @Override
223                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
224                    final String key = evaluateObject(obj, p, matchingSelector);
225                    return new ChangePropertyCommand(p, key, "");
226                }
227
228                @Override
229                public String toString() {
230                    return "fixRemove: " + obj;
231                }
232            };
233        }
234
235        /**
236         * Creates a fixing command which executes a {@link ChangePropertyKeyCommand} on the specified keys.
237         * @param oldKey old key
238         * @param newKey new key
239         * @return created fix command
240         */
241        static FixCommand fixChangeKey(final String oldKey, final String newKey) {
242            return new FixCommand() {
243                @Override
244                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
245                    return new ChangePropertyKeyCommand(p,
246                            TagCheck.insertArguments(matchingSelector, oldKey, p),
247                            TagCheck.insertArguments(matchingSelector, newKey, p));
248                }
249
250                @Override
251                public String toString() {
252                    return "fixChangeKey: " + oldKey + " => " + newKey;
253                }
254            };
255        }
256    }
257
258    final MultiMap<String, TagCheck> checks = new MultiMap<>();
259
260    /**
261     * Result of {@link TagCheck#readMapCSS}
262     * @since 8936
263     */
264    public static class ParseResult {
265        /** Checks successfully parsed */
266        public final List<TagCheck> parseChecks;
267        /** Errors that occurred during parsing */
268        public final Collection<Throwable> parseErrors;
269
270        /**
271         * Constructs a new {@code ParseResult}.
272         * @param parseChecks Checks successfully parsed
273         * @param parseErrors Errors that occurred during parsing
274         */
275        public ParseResult(List<TagCheck> parseChecks, Collection<Throwable> parseErrors) {
276            this.parseChecks = parseChecks;
277            this.parseErrors = parseErrors;
278        }
279    }
280
281    /**
282     * Tag check.
283     */
284    public static class TagCheck implements Predicate<OsmPrimitive> {
285        /** The selector of this {@code TagCheck} */
286        protected final GroupedMapCSSRule rule;
287        /** Commands to apply in order to fix a matching primitive */
288        protected final List<FixCommand> fixCommands = new ArrayList<>();
289        /** Tags (or arbitraty strings) of alternatives to be presented to the user */
290        protected final List<String> alternatives = new ArrayList<>();
291        /** An {@link org.openstreetmap.josm.gui.mappaint.mapcss.Instruction.AssignmentInstruction}-{@link Severity} pair.
292         * Is evaluated on the matching primitive to give the error message. Map is checked to contain exactly one element. */
293        protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>();
294        /** Unit tests */
295        protected final Map<String, Boolean> assertions = new HashMap<>();
296        /** MapCSS Classes to set on matching primitives */
297        protected final Set<String> setClassExpressions = new HashSet<>();
298        /** Denotes whether the object should be deleted for fixing it */
299        protected boolean deletion;
300        /** A string used to group similar tests */
301        protected String group;
302
303        TagCheck(GroupedMapCSSRule rule) {
304            this.rule = rule;
305        }
306
307        private static final String POSSIBLE_THROWS = possibleThrows();
308
309        static final String possibleThrows() {
310            StringBuilder sb = new StringBuilder();
311            for (Severity s : Severity.values()) {
312                if (sb.length() > 0) {
313                    sb.append('/');
314                }
315                sb.append("throw")
316                .append(s.name().charAt(0))
317                .append(s.name().substring(1).toLowerCase(Locale.ENGLISH));
318            }
319            return sb.toString();
320        }
321
322        static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) throws IllegalDataException {
323            final TagCheck check = new TagCheck(rule);
324            for (Instruction i : rule.declaration.instructions) {
325                if (i instanceof Instruction.AssignmentInstruction) {
326                    final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i;
327                    if (ai.isSetInstruction) {
328                        check.setClassExpressions.add(ai.key);
329                        continue;
330                    }
331                    try {
332                        final String val = ai.val instanceof Expression
333                                ? Optional.ofNullable(((Expression) ai.val).evaluate(new Environment())).map(Object::toString).orElse(null)
334                                : ai.val instanceof String
335                                ? (String) ai.val
336                                : ai.val instanceof Keyword
337                                ? ((Keyword) ai.val).val
338                                : null;
339                        if (ai.key.startsWith("throw")) {
340                            try {
341                                check.errors.put(ai, Severity.valueOf(ai.key.substring("throw".length()).toUpperCase(Locale.ENGLISH)));
342                            } catch (IllegalArgumentException e) {
343                                Logging.log(Logging.LEVEL_WARN,
344                                        "Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS+'.', e);
345                            }
346                        } else if ("fixAdd".equals(ai.key)) {
347                            check.fixCommands.add(FixCommand.fixAdd(ai.val));
348                        } else if ("fixRemove".equals(ai.key)) {
349                            CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")),
350                                    "Unexpected '='. Please only specify the key to remove in: " + ai);
351                            check.fixCommands.add(FixCommand.fixRemove(ai.val));
352                        } else if (val != null && "fixChangeKey".equals(ai.key)) {
353                            CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!");
354                            final String[] x = val.split("=>", 2);
355                            check.fixCommands.add(FixCommand.fixChangeKey(Utils.removeWhiteSpaces(x[0]), Utils.removeWhiteSpaces(x[1])));
356                        } else if (val != null && "fixDeleteObject".equals(ai.key)) {
357                            CheckParameterUtil.ensureThat("this".equals(val), "fixDeleteObject must be followed by 'this'");
358                            check.deletion = true;
359                        } else if (val != null && "suggestAlternative".equals(ai.key)) {
360                            check.alternatives.add(val);
361                        } else if (val != null && "assertMatch".equals(ai.key)) {
362                            check.assertions.put(val, Boolean.TRUE);
363                        } else if (val != null && "assertNoMatch".equals(ai.key)) {
364                            check.assertions.put(val, Boolean.FALSE);
365                        } else if (val != null && "group".equals(ai.key)) {
366                            check.group = val;
367                        } else if (ai.key.startsWith("-")) {
368                            Logging.debug("Ignoring extension instruction: " + ai.key + ": " + ai.val);
369                        } else {
370                            throw new IllegalDataException("Cannot add instruction " + ai.key + ": " + ai.val + '!');
371                        }
372                    } catch (IllegalArgumentException e) {
373                        throw new IllegalDataException(e);
374                    }
375                }
376            }
377            if (check.errors.isEmpty() && check.setClassExpressions.isEmpty()) {
378                throw new IllegalDataException(
379                        "No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors);
380            } else if (check.errors.size() > 1) {
381                throw new IllegalDataException(
382                        "More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for "
383                                + rule.selectors);
384            }
385            return check;
386        }
387
388        static ParseResult readMapCSS(Reader css) throws ParseException {
389            CheckParameterUtil.ensureParameterNotNull(css, "css");
390
391            final MapCSSStyleSource source = new MapCSSStyleSource("");
392            final MapCSSParser preprocessor = new MapCSSParser(css, MapCSSParser.LexicalState.PREPROCESSOR);
393            try (StringReader mapcss = new StringReader(preprocessor.pp_root(source))) {
394                new MapCSSParser(mapcss, MapCSSParser.LexicalState.DEFAULT).sheet(source);
395            }
396            // Ignore "meta" rule(s) from external rules of JOSM wiki
397            source.removeMetaRules();
398            // group rules with common declaration block
399            Map<Declaration, List<Selector>> g = new LinkedHashMap<>();
400            for (MapCSSRule rule : source.rules) {
401                if (!g.containsKey(rule.declaration)) {
402                    List<Selector> sels = new ArrayList<>();
403                    sels.add(rule.selector);
404                    g.put(rule.declaration, sels);
405                } else {
406                    g.get(rule.declaration).add(rule.selector);
407                }
408            }
409            List<TagCheck> parseChecks = new ArrayList<>();
410            for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) {
411                try {
412                    parseChecks.add(TagCheck.ofMapCSSRule(
413                            new GroupedMapCSSRule(map.getValue(), map.getKey())));
414                } catch (IllegalDataException e) {
415                    Logging.error("Cannot add MapCss rule: "+e.getMessage());
416                    source.logError(e);
417                }
418            }
419            return new ParseResult(parseChecks, source.getErrors());
420        }
421
422        @Override
423        public boolean test(OsmPrimitive primitive) {
424            // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker.
425            return whichSelectorMatchesPrimitive(primitive) != null;
426        }
427
428        Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) {
429            return whichSelectorMatchesEnvironment(new Environment(primitive));
430        }
431
432        Selector whichSelectorMatchesEnvironment(Environment env) {
433            for (Selector i : rule.selectors) {
434                env.clearSelectorMatchingInformation();
435                if (i.matches(env)) {
436                    return i;
437                }
438            }
439            return null;
440        }
441
442        /**
443         * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the
444         * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}.
445         * @param matchingSelector matching selector
446         * @param index index
447         * @param type selector type ("key", "value" or "tag")
448         * @param p OSM primitive
449         * @return argument value, can be {@code null}
450         */
451        static String determineArgument(OptimizedGeneralSelector matchingSelector, int index, String type, OsmPrimitive p) {
452            try {
453                final Condition c = matchingSelector.getConditions().get(index);
454                final Tag tag = c instanceof Condition.ToTagConvertable
455                        ? ((Condition.ToTagConvertable) c).asTag(p)
456                        : null;
457                if (tag == null) {
458                    return null;
459                } else if ("key".equals(type)) {
460                    return tag.getKey();
461                } else if ("value".equals(type)) {
462                    return tag.getValue();
463                } else if ("tag".equals(type)) {
464                    return tag.toString();
465                }
466            } catch (IndexOutOfBoundsException ignore) {
467                Logging.debug(ignore);
468            }
469            return null;
470        }
471
472        /**
473         * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding
474         * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}.
475         * @param matchingSelector matching selector
476         * @param s any string
477         * @param p OSM primitive
478         * @return string with arguments inserted
479         */
480        static String insertArguments(Selector matchingSelector, String s, OsmPrimitive p) {
481            if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) {
482                return insertArguments(((Selector.ChildOrParentSelector) matchingSelector).right, s, p);
483            } else if (s == null || !(matchingSelector instanceof Selector.OptimizedGeneralSelector)) {
484                return s;
485            }
486            final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s);
487            final StringBuffer sb = new StringBuffer();
488            while (m.find()) {
489                final String argument = determineArgument((Selector.OptimizedGeneralSelector) matchingSelector,
490                        Integer.parseInt(m.group(1)), m.group(2), p);
491                try {
492                    // Perform replacement with null-safe + regex-safe handling
493                    m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", ""));
494                } catch (IndexOutOfBoundsException | IllegalArgumentException e) {
495                    Logging.log(Logging.LEVEL_ERROR, tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()), e);
496                }
497            }
498            m.appendTail(sb);
499            return sb.toString();
500        }
501
502        /**
503         * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive}
504         * if the error is fixable, or {@code null} otherwise.
505         *
506         * @param p the primitive to construct the fix for
507         * @return the fix or {@code null}
508         */
509        Command fixPrimitive(OsmPrimitive p) {
510            if (fixCommands.isEmpty() && !deletion) {
511                return null;
512            }
513            try {
514                final Selector matchingSelector = whichSelectorMatchesPrimitive(p);
515                Collection<Command> cmds = new LinkedList<>();
516                for (FixCommand fixCommand : fixCommands) {
517                    cmds.add(fixCommand.createCommand(p, matchingSelector));
518                }
519                if (deletion && !p.isDeleted()) {
520                    cmds.add(new DeleteCommand(p));
521                }
522                return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds);
523            } catch (IllegalArgumentException e) {
524                Logging.error(e);
525                return null;
526            }
527        }
528
529        /**
530         * Constructs a (localized) message for this deprecation check.
531         * @param p OSM primitive
532         *
533         * @return a message
534         */
535        String getMessage(OsmPrimitive p) {
536            if (errors.isEmpty()) {
537                // Return something to avoid NPEs
538                return rule.declaration.toString();
539            } else {
540                final Object val = errors.keySet().iterator().next().val;
541                return String.valueOf(
542                        val instanceof Expression
543                                ? ((Expression) val).evaluate(new Environment(p))
544                                : val
545                );
546            }
547        }
548
549        /**
550         * Constructs a (localized) description for this deprecation check.
551         * @param p OSM primitive
552         *
553         * @return a description (possibly with alternative suggestions)
554         * @see #getDescriptionForMatchingSelector
555         */
556        String getDescription(OsmPrimitive p) {
557            if (alternatives.isEmpty()) {
558                return getMessage(p);
559            } else {
560                /* I18N: {0} is the test error message and {1} is an alternative */
561                return tr("{0}, use {1} instead", getMessage(p), String.join(tr(" or "), alternatives));
562            }
563        }
564
565        /**
566         * Constructs a (localized) description for this deprecation check
567         * where any placeholders are replaced by values of the matched selector.
568         *
569         * @param matchingSelector matching selector
570         * @param p OSM primitive
571         * @return a description (possibly with alternative suggestions)
572         */
573        String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) {
574            return insertArguments(matchingSelector, getDescription(p), p);
575        }
576
577        Severity getSeverity() {
578            return errors.isEmpty() ? null : errors.values().iterator().next();
579        }
580
581        @Override
582        public String toString() {
583            return getDescription(null);
584        }
585
586        /**
587         * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error.
588         *
589         * @param p the primitive to construct the error for
590         * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error.
591         */
592        List<TestError> getErrorsForPrimitive(OsmPrimitive p) {
593            final Environment env = new Environment(p);
594            return getErrorsForPrimitive(p, whichSelectorMatchesEnvironment(env), env, null);
595        }
596
597        private List<TestError> getErrorsForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env, Test tester) {
598            List<TestError> res = new ArrayList<>();
599            if (matchingSelector != null && !errors.isEmpty()) {
600                final Command fix = fixPrimitive(p);
601                final String description = getDescriptionForMatchingSelector(p, matchingSelector);
602                final String description1 = group == null ? description : group;
603                final String description2 = group == null ? null : description;
604                TestError.Builder errorBuilder = TestError.builder(tester, getSeverity(), 3000)
605                        .messageWithManuallyTranslatedDescription(description1, description2, matchingSelector.toString());
606                if (fix != null) {
607                    errorBuilder = errorBuilder.fix(() -> fix);
608                }
609                if (env.child instanceof OsmPrimitive) {
610                    res.add(errorBuilder.primitives(p, (OsmPrimitive) env.child).build());
611                } else if (env.children != null) {
612                    for (IPrimitive c : env.children) {
613                        if (c instanceof OsmPrimitive) {
614                            errorBuilder = TestError.builder(tester, getSeverity(), 3000)
615                                    .messageWithManuallyTranslatedDescription(description1, description2,
616                                            matchingSelector.toString());
617                            if (fix != null) {
618                                errorBuilder = errorBuilder.fix(() -> fix);
619                            }
620                            res.add(errorBuilder.primitives(p, (OsmPrimitive) c).build());
621                        }
622                    }
623                } else {
624                    res.add(errorBuilder.primitives(p).build());
625                }
626            }
627            return res;
628        }
629
630        /**
631         * Returns the set of tagchecks on which this check depends on.
632         * @param schecks the collection of tagcheks to search in
633         * @return the set of tagchecks on which this check depends on
634         * @since 7881
635         */
636        public Set<TagCheck> getTagCheckDependencies(Collection<TagCheck> schecks) {
637            Set<TagCheck> result = new HashSet<>();
638            Set<String> classes = getClassesIds();
639            if (schecks != null && !classes.isEmpty()) {
640                for (TagCheck tc : schecks) {
641                    if (this.equals(tc)) {
642                        continue;
643                    }
644                    for (String id : tc.setClassExpressions) {
645                        if (classes.contains(id)) {
646                            result.add(tc);
647                            break;
648                        }
649                    }
650                }
651            }
652            return result;
653        }
654
655        /**
656         * Returns the list of ids of all MapCSS classes referenced in the rule selectors.
657         * @return the list of ids of all MapCSS classes referenced in the rule selectors
658         * @since 7881
659         */
660        public Set<String> getClassesIds() {
661            Set<String> result = new HashSet<>();
662            for (Selector s : rule.selectors) {
663                if (s instanceof AbstractSelector) {
664                    for (Condition c : ((AbstractSelector) s).getConditions()) {
665                        if (c instanceof ClassCondition) {
666                            result.add(((ClassCondition) c).id);
667                        }
668                    }
669                }
670            }
671            return result;
672        }
673    }
674
675    static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
676        public final GroupedMapCSSRule rule;
677
678        MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) {
679            this.rule = rule;
680        }
681
682        @Override
683        public String toString() {
684            return "MapCSSTagCheckerAndRule [rule=" + rule + ']';
685        }
686    }
687
688    /**
689     * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
690     * @param p The OSM primitive
691     * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned
692     * @return all errors for the given primitive, with or without those of "info" severity
693     */
694    public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
695        final List<TestError> res = new ArrayList<>();
696        if (indexData == null) {
697            indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverity, MapCSSTagCheckerIndex.ALL_TESTS);
698        }
699
700        MapCSSRuleIndex matchingRuleIndex = indexData.get(p);
701
702        Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
703        // the declaration indices are sorted, so it suffices to save the last used index
704        Declaration lastDeclUsed = null;
705
706        Iterator<MapCSSRule> candidates = matchingRuleIndex.getRuleCandidates(p);
707        while (candidates.hasNext()) {
708            MapCSSRule r = candidates.next();
709            env.clearSelectorMatchingInformation();
710            if (r.selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector)
711                TagCheck check = indexData.getCheck(r);
712                if (check != null) {
713                    if (r.declaration == lastDeclUsed)
714                        continue; // don't apply one declaration more than once
715                    lastDeclUsed = r.declaration;
716
717                    r.declaration.execute(env);
718                    if (!check.errors.isEmpty()) {
719                        for (TestError e: check.getErrorsForPrimitive(p, r.selector, env, new MapCSSTagCheckerAndRule(check.rule))) {
720                            addIfNotSimilar(e, res);
721                        }
722                    }
723                }
724            }
725        }
726        return res;
727    }
728
729    /**
730     * See #12627
731     * Add error to given list if list doesn't already contain a similar error.
732     * Similar means same code and description and same combination of primitives and same combination of highlighted objects,
733     * but maybe with different orders.
734     * @param toAdd the error to add
735     * @param errors the list of errors
736     */
737    private static void addIfNotSimilar(TestError toAdd, List<TestError> errors) {
738        boolean isDup = false;
739        if (toAdd.getPrimitives().size() >= 2) {
740            for (TestError e : errors) {
741                if (e.getCode() == toAdd.getCode() && e.getMessage().equals(toAdd.getMessage())
742                        && e.getPrimitives().size() == toAdd.getPrimitives().size()
743                        && e.getPrimitives().containsAll(toAdd.getPrimitives())
744                        && e.getHighlighted().size() == toAdd.getHighlighted().size()
745                        && e.getHighlighted().containsAll(toAdd.getHighlighted())) {
746                    isDup = true;
747                    break;
748                }
749            }
750        }
751        if (!isDup)
752            errors.add(toAdd);
753    }
754
755    private static Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity,
756            Collection<Set<TagCheck>> checksCol) {
757        final List<TestError> r = new ArrayList<>();
758        final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
759        for (Set<TagCheck> schecks : checksCol) {
760            for (TagCheck check : schecks) {
761                boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
762                // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
763                if (ignoreError && check.setClassExpressions.isEmpty()) {
764                    continue;
765                }
766                final Selector selector = check.whichSelectorMatchesEnvironment(env);
767                if (selector != null) {
768                    check.rule.declaration.execute(env);
769                    if (!ignoreError && !check.errors.isEmpty()) {
770                        r.addAll(check.getErrorsForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule)));
771                    }
772                }
773            }
774        }
775        return r;
776    }
777
778    /**
779     * Visiting call for primitives.
780     *
781     * @param p The primitive to inspect.
782     */
783    @Override
784    public void check(OsmPrimitive p) {
785        for (TestError e : getErrorsForPrimitive(p, ValidatorPrefHelper.PREF_OTHER.get())) {
786            addIfNotSimilar(e, errors);
787        }
788        if (partialSelection) {
789            tested.add(p);
790        }
791    }
792
793    /**
794     * Adds a new MapCSS config file from the given URL.
795     * @param url The unique URL of the MapCSS config file
796     * @return List of tag checks and parsing errors, or null
797     * @throws ParseException if the config file does not match MapCSS syntax
798     * @throws IOException if any I/O error occurs
799     * @since 7275
800     */
801    public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException {
802        CheckParameterUtil.ensureParameterNotNull(url, "url");
803        ParseResult result;
804        try (CachedFile cache = new CachedFile(url);
805             InputStream zip = cache.findZipEntryInputStream("validator.mapcss", "");
806             InputStream s = zip != null ? zip : cache.getInputStream();
807             Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) {
808            if (zip != null)
809                I18n.addTexts(cache.getFile());
810            result = TagCheck.readMapCSS(reader);
811            checks.remove(url);
812            checks.putAll(url, result.parseChecks);
813            indexData = null;
814            // Check assertions, useful for development of local files
815            if (Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url)) {
816                for (String msg : checkAsserts(result.parseChecks)) {
817                    Logging.warn(msg);
818                }
819            }
820        }
821        return result;
822    }
823
824    @Override
825    public synchronized void initialize() throws Exception {
826        checks.clear();
827        indexData = null;
828        for (SourceEntry source : new ValidatorPrefHelper().get()) {
829            if (!source.active) {
830                continue;
831            }
832            String i = source.url;
833            try {
834                if (!i.startsWith("resource:")) {
835                    Logging.info(tr("Adding {0} to tag checker", i));
836                } else if (Logging.isDebugEnabled()) {
837                    Logging.debug(tr("Adding {0} to tag checker", i));
838                }
839                addMapCSS(i);
840                if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) {
841                    FileWatcher.getDefaultInstance().registerSource(source);
842                }
843            } catch (IOException | IllegalStateException | IllegalArgumentException ex) {
844                Logging.warn(tr("Failed to add {0} to tag checker", i));
845                Logging.log(Logging.LEVEL_WARN, ex);
846            } catch (ParseException | TokenMgrError ex) {
847                Logging.warn(tr("Failed to add {0} to tag checker", i));
848                Logging.warn(ex);
849            }
850        }
851    }
852
853    private static Method getFunctionMethod(String method) {
854        try {
855            return Functions.class.getDeclaredMethod(method, Environment.class, String.class);
856        } catch (NoSuchMethodException | SecurityException e) {
857            Logging.error(e);
858            return null;
859        }
860    }
861
862    private static Optional<String> getFirstInsideCountry(TagCheck check, Method insideMethod) {
863        return check.rule.selectors.stream()
864                .filter(s -> s instanceof GeneralSelector)
865                .flatMap(s -> ((GeneralSelector) s).getConditions().stream())
866                .filter(c -> c instanceof ExpressionCondition)
867                .map(c -> ((ExpressionCondition) c).getExpression())
868                .filter(c -> c instanceof ParameterFunction)
869                .map(c -> (ParameterFunction) c)
870                .filter(c -> c.getMethod().equals(insideMethod))
871                .flatMap(c -> c.getArgs().stream())
872                .filter(e -> e instanceof LiteralExpression)
873                .map(e -> ((LiteralExpression) e).getLiteral())
874                .filter(l -> l instanceof String)
875                .map(l -> ((String) l).split(",")[0])
876                .findFirst();
877    }
878
879    private static LatLon getLocation(TagCheck check, Method insideMethod) {
880        Optional<String> inside = getFirstInsideCountry(check, insideMethod);
881        if (inside.isPresent()) {
882            GeoPropertyIndex<Boolean> index = Territories.getGeoPropertyIndex(inside.get());
883            if (index != null) {
884                GeoProperty<Boolean> prop = index.getGeoProperty();
885                if (prop instanceof DefaultGeoProperty) {
886                    return ((DefaultGeoProperty) prop).getRandomLatLon();
887                }
888            }
889        }
890        return LatLon.ZERO;
891    }
892
893    /**
894     * Checks that rule assertions are met for the given set of TagChecks.
895     * @param schecks The TagChecks for which assertions have to be checked
896     * @return A set of error messages, empty if all assertions are met
897     * @since 7356
898     */
899    public Set<String> checkAsserts(final Collection<TagCheck> schecks) {
900        Set<String> assertionErrors = new LinkedHashSet<>();
901        final Method insideMethod = getFunctionMethod("inside");
902        final DataSet ds = new DataSet();
903        for (final TagCheck check : schecks) {
904            Logging.debug("Check: {0}", check);
905            for (final Map.Entry<String, Boolean> i : check.assertions.entrySet()) {
906                Logging.debug("- Assertion: {0}", i);
907                final OsmPrimitive p = OsmUtils.createPrimitive(i.getKey(), getLocation(check, insideMethod), true);
908                // Build minimal ordered list of checks to run to test the assertion
909                List<Set<TagCheck>> checksToRun = new ArrayList<>();
910                Set<TagCheck> checkDependencies = check.getTagCheckDependencies(schecks);
911                if (!checkDependencies.isEmpty()) {
912                    checksToRun.add(checkDependencies);
913                }
914                checksToRun.add(Collections.singleton(check));
915                // Add primitive to dataset to avoid DataIntegrityProblemException when evaluating selectors
916                addPrimitive(ds, p);
917                final Collection<TestError> pErrors = getErrorsForPrimitive(p, true, checksToRun);
918                Logging.debug("- Errors: {0}", pErrors);
919                final boolean isError = pErrors.stream().anyMatch(e -> e.getTester() instanceof MapCSSTagCheckerAndRule
920                        && ((MapCSSTagCheckerAndRule) e.getTester()).rule.equals(check.rule));
921                if (isError != i.getValue()) {
922                    assertionErrors.add(MessageFormat.format("Expecting test ''{0}'' (i.e., {1}) to {2} {3} (i.e., {4})",
923                            check.getMessage(p), check.rule.selectors, i.getValue() ? "match" : "not match", i.getKey(), p.getKeys()));
924                }
925                if (isError) {
926                    // Check that autofix works as expected
927                    Command fix = check.fixPrimitive(p);
928                    if (fix != null && fix.executeCommand() && !getErrorsForPrimitive(p, true, checksToRun).isEmpty()) {
929                        assertionErrors.add(MessageFormat.format("Autofix does not work for test ''{0}'' (i.e., {1})",
930                                check.getMessage(p), check.rule.selectors));
931                    }
932                }
933                ds.removePrimitive(p);
934            }
935        }
936        return assertionErrors;
937    }
938
939    private static void addPrimitive(DataSet ds, OsmPrimitive p) {
940        if (p instanceof Way) {
941            ((Way) p).getNodes().forEach(n -> addPrimitive(ds, n));
942        } else if (p instanceof Relation) {
943            ((Relation) p).getMembers().forEach(m -> addPrimitive(ds, m.getMember()));
944        }
945        ds.addPrimitive(p);
946    }
947
948    /**
949     * Reload tagchecker rule.
950     * @param rule tagchecker rule to reload
951     * @since 12825
952     */
953    public static void reloadRule(SourceEntry rule) {
954        MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class);
955        if (tagChecker != null) {
956            try {
957                tagChecker.addMapCSS(rule.url);
958            } catch (IOException | ParseException | TokenMgrError e) {
959                Logging.warn(e);
960            }
961        }
962    }
963
964    @Override
965    public synchronized void startTest(ProgressMonitor progressMonitor) {
966        super.startTest(progressMonitor);
967        super.setShowElements(true);
968        if (indexData == null) {
969            indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverityChecks(), MapCSSTagCheckerIndex.ALL_TESTS);
970        }
971        tested.clear();
972    }
973
974    @Override
975    public synchronized void endTest() {
976        if (partialSelection && !tested.isEmpty()) {
977            // #14287: see https://josm.openstreetmap.de/ticket/14287#comment:15
978            // execute tests for objects which might contain or cross previously tested elements
979
980            // rebuild index with a reduced set of rules (those that use ChildOrParentSelector) and thus may have left selectors
981            // matching the previously tested elements
982            indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverityChecks(), MapCSSTagCheckerIndex.ONLY_SELECTED_TESTS);
983
984            Set<OsmPrimitive> surrounding = new HashSet<>();
985            for (OsmPrimitive p : tested) {
986                if (p.getDataSet() != null) {
987                    surrounding.addAll(p.getDataSet().searchWays(p.getBBox()));
988                    surrounding.addAll(p.getDataSet().searchRelations(p.getBBox()));
989                }
990            }
991            final boolean includeOtherSeverity = includeOtherSeverityChecks();
992            for (OsmPrimitive p : surrounding) {
993                if (tested.contains(p))
994                    continue;
995                Collection<TestError> additionalErrors = getErrorsForPrimitive(p, includeOtherSeverity);
996                for (TestError e : additionalErrors) {
997                    if (e.getPrimitives().stream().anyMatch(tested::contains))
998                        addIfNotSimilar(e, errors);
999                }
1000            }
1001            tested.clear();
1002        }
1003        super.endTest();
1004        // no need to keep the index, it is quickly build and doubles the memory needs
1005        indexData = null;
1006    }
1007}