001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.awt.Rectangle;
007import java.io.BufferedReader;
008import java.io.IOException;
009import java.io.InputStream;
010import java.io.Reader;
011import java.io.StringReader;
012import java.lang.reflect.Method;
013import java.text.MessageFormat;
014import java.util.ArrayList;
015import java.util.Collection;
016import java.util.Collections;
017import java.util.HashMap;
018import java.util.HashSet;
019import java.util.Iterator;
020import java.util.LinkedHashMap;
021import java.util.LinkedHashSet;
022import java.util.LinkedList;
023import java.util.List;
024import java.util.Locale;
025import java.util.Map;
026import java.util.Objects;
027import java.util.Optional;
028import java.util.Set;
029import java.util.function.Predicate;
030import java.util.regex.Matcher;
031import java.util.regex.Pattern;
032
033import org.openstreetmap.josm.command.ChangePropertyCommand;
034import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
035import org.openstreetmap.josm.command.Command;
036import org.openstreetmap.josm.command.DeleteCommand;
037import org.openstreetmap.josm.command.SequenceCommand;
038import org.openstreetmap.josm.data.coor.LatLon;
039import org.openstreetmap.josm.data.osm.DataSet;
040import org.openstreetmap.josm.data.osm.IPrimitive;
041import org.openstreetmap.josm.data.osm.OsmPrimitive;
042import org.openstreetmap.josm.data.osm.OsmUtils;
043import org.openstreetmap.josm.data.osm.Relation;
044import org.openstreetmap.josm.data.osm.Tag;
045import org.openstreetmap.josm.data.osm.Way;
046import org.openstreetmap.josm.data.preferences.sources.SourceEntry;
047import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
048import org.openstreetmap.josm.data.validation.OsmValidator;
049import org.openstreetmap.josm.data.validation.Severity;
050import org.openstreetmap.josm.data.validation.Test;
051import org.openstreetmap.josm.data.validation.TestError;
052import org.openstreetmap.josm.gui.mappaint.Environment;
053import org.openstreetmap.josm.gui.mappaint.Keyword;
054import org.openstreetmap.josm.gui.mappaint.MultiCascade;
055import org.openstreetmap.josm.gui.mappaint.mapcss.Condition;
056import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ClassCondition;
057import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ExpressionCondition;
058import org.openstreetmap.josm.gui.mappaint.mapcss.Expression;
059import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.ParameterFunction;
060import org.openstreetmap.josm.gui.mappaint.mapcss.Functions;
061import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction;
062import org.openstreetmap.josm.gui.mappaint.mapcss.LiteralExpression;
063import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
064import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration;
065import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
066import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource.MapCSSRuleIndex;
067import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
068import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.AbstractSelector;
069import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector;
070import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.OptimizedGeneralSelector;
071import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser;
072import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
073import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError;
074import org.openstreetmap.josm.gui.progress.ProgressMonitor;
075import org.openstreetmap.josm.io.CachedFile;
076import org.openstreetmap.josm.io.FileWatcher;
077import org.openstreetmap.josm.io.IllegalDataException;
078import org.openstreetmap.josm.io.UTFInputStreamReader;
079import org.openstreetmap.josm.spi.preferences.Config;
080import org.openstreetmap.josm.tools.CheckParameterUtil;
081import org.openstreetmap.josm.tools.DefaultGeoProperty;
082import org.openstreetmap.josm.tools.GeoProperty;
083import org.openstreetmap.josm.tools.GeoPropertyIndex;
084import org.openstreetmap.josm.tools.I18n;
085import org.openstreetmap.josm.tools.Logging;
086import org.openstreetmap.josm.tools.MultiMap;
087import org.openstreetmap.josm.tools.Territories;
088import org.openstreetmap.josm.tools.Utils;
089
090/**
091 * MapCSS-based tag checker/fixer.
092 * @since 6506
093 */
094public class MapCSSTagChecker extends Test.TagTest {
095    MapCSSTagCheckerIndex indexData;
096    final Set<OsmPrimitive> tested = new HashSet<>();
097
098
099    /**
100    * A grouped MapCSSRule with multiple selectors for a single declaration.
101    * @see MapCSSRule
102    */
103    public static class GroupedMapCSSRule {
104        /** MapCSS selectors **/
105        public final List<Selector> selectors;
106        /** MapCSS declaration **/
107        public final Declaration declaration;
108
109        /**
110         * Constructs a new {@code GroupedMapCSSRule}.
111         * @param selectors MapCSS selectors
112         * @param declaration MapCSS declaration
113         */
114        public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) {
115            this.selectors = selectors;
116            this.declaration = declaration;
117        }
118
119        @Override
120        public int hashCode() {
121            return Objects.hash(selectors, declaration);
122        }
123
124        @Override
125        public boolean equals(Object obj) {
126            if (this == obj) return true;
127            if (obj == null || getClass() != obj.getClass()) return false;
128            GroupedMapCSSRule that = (GroupedMapCSSRule) obj;
129            return Objects.equals(selectors, that.selectors) &&
130                    Objects.equals(declaration, that.declaration);
131        }
132
133        @Override
134        public String toString() {
135            return "GroupedMapCSSRule [selectors=" + selectors + ", declaration=" + declaration + ']';
136        }
137    }
138
139    /**
140     * The preference key for tag checker source entries.
141     * @since 6670
142     */
143    public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
144
145    /**
146     * Constructs a new {@code MapCSSTagChecker}.
147     */
148    public MapCSSTagChecker() {
149        super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
150    }
151
152    /**
153     * Represents a fix to a validation test. The fixing {@link Command} can be obtained by {@link #createCommand(OsmPrimitive, Selector)}.
154     */
155    @FunctionalInterface
156    interface FixCommand {
157        /**
158         * Creates the fixing {@link Command} for the given primitive. The {@code matchingSelector} is used to evaluate placeholders
159         * (cf. {@link MapCSSTagChecker.TagCheck#insertArguments(Selector, String, OsmPrimitive)}).
160         * @param p OSM primitive
161         * @param matchingSelector  matching selector
162         * @return fix command
163         */
164        Command createCommand(OsmPrimitive p, Selector matchingSelector);
165
166        /**
167         * Checks that object is either an {@link Expression} or a {@link String}.
168         * @param obj object to check
169         * @throws IllegalArgumentException if object is not an {@code Expression} or a {@code String}
170         */
171        static void checkObject(final Object obj) {
172            CheckParameterUtil.ensureThat(obj instanceof Expression || obj instanceof String,
173                    () -> "instance of Exception or String expected, but got " + obj);
174        }
175
176        /**
177         * Evaluates given object as {@link Expression} or {@link String} on the matched {@link OsmPrimitive} and {@code matchingSelector}.
178         * @param obj object to evaluate ({@link Expression} or {@link String})
179         * @param p OSM primitive
180         * @param matchingSelector matching selector
181         * @return result string
182         */
183        static String evaluateObject(final Object obj, final OsmPrimitive p, final Selector matchingSelector) {
184            final String s;
185            if (obj instanceof Expression) {
186                s = (String) ((Expression) obj).evaluate(new Environment(p));
187            } else if (obj instanceof String) {
188                s = (String) obj;
189            } else {
190                return null;
191            }
192            return TagCheck.insertArguments(matchingSelector, s, p);
193        }
194
195        /**
196         * Creates a fixing command which executes a {@link ChangePropertyCommand} on the specified tag.
197         * @param obj object to evaluate ({@link Expression} or {@link String})
198         * @return created fix command
199         */
200        static FixCommand fixAdd(final Object obj) {
201            checkObject(obj);
202            return new FixCommand() {
203                @Override
204                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
205                    final Tag tag = Tag.ofString(evaluateObject(obj, p, matchingSelector));
206                    return new ChangePropertyCommand(p, tag.getKey(), tag.getValue());
207                }
208
209                @Override
210                public String toString() {
211                    return "fixAdd: " + obj;
212                }
213            };
214        }
215
216        /**
217         * Creates a fixing command which executes a {@link ChangePropertyCommand} to delete the specified key.
218         * @param obj object to evaluate ({@link Expression} or {@link String})
219         * @return created fix command
220         */
221        static FixCommand fixRemove(final Object obj) {
222            checkObject(obj);
223            return new FixCommand() {
224                @Override
225                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
226                    final String key = evaluateObject(obj, p, matchingSelector);
227                    return new ChangePropertyCommand(p, key, "");
228                }
229
230                @Override
231                public String toString() {
232                    return "fixRemove: " + obj;
233                }
234            };
235        }
236
237        /**
238         * Creates a fixing command which executes a {@link ChangePropertyKeyCommand} on the specified keys.
239         * @param oldKey old key
240         * @param newKey new key
241         * @return created fix command
242         */
243        static FixCommand fixChangeKey(final String oldKey, final String newKey) {
244            return new FixCommand() {
245                @Override
246                public Command createCommand(OsmPrimitive p, Selector matchingSelector) {
247                    return new ChangePropertyKeyCommand(p,
248                            TagCheck.insertArguments(matchingSelector, oldKey, p),
249                            TagCheck.insertArguments(matchingSelector, newKey, p));
250                }
251
252                @Override
253                public String toString() {
254                    return "fixChangeKey: " + oldKey + " => " + newKey;
255                }
256            };
257        }
258    }
259
260    final MultiMap<String, TagCheck> checks = new MultiMap<>();
261
262    /**
263     * Result of {@link TagCheck#readMapCSS}
264     * @since 8936
265     */
266    public static class ParseResult {
267        /** Checks successfully parsed */
268        public final List<TagCheck> parseChecks;
269        /** Errors that occurred during parsing */
270        public final Collection<Throwable> parseErrors;
271
272        /**
273         * Constructs a new {@code ParseResult}.
274         * @param parseChecks Checks successfully parsed
275         * @param parseErrors Errors that occurred during parsing
276         */
277        public ParseResult(List<TagCheck> parseChecks, Collection<Throwable> parseErrors) {
278            this.parseChecks = parseChecks;
279            this.parseErrors = parseErrors;
280        }
281    }
282
283    /**
284     * Tag check.
285     */
286    public static class TagCheck implements Predicate<OsmPrimitive> {
287        /** The selector of this {@code TagCheck} */
288        protected final GroupedMapCSSRule rule;
289        /** Commands to apply in order to fix a matching primitive */
290        protected final List<FixCommand> fixCommands = new ArrayList<>();
291        /** Tags (or arbitraty strings) of alternatives to be presented to the user */
292        protected final List<String> alternatives = new ArrayList<>();
293        /** An {@link org.openstreetmap.josm.gui.mappaint.mapcss.Instruction.AssignmentInstruction}-{@link Severity} pair.
294         * Is evaluated on the matching primitive to give the error message. Map is checked to contain exactly one element. */
295        protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>();
296        /** Unit tests */
297        protected final Map<String, Boolean> assertions = new HashMap<>();
298        /** MapCSS Classes to set on matching primitives */
299        protected final Set<String> setClassExpressions = new HashSet<>();
300        /** Denotes whether the object should be deleted for fixing it */
301        protected boolean deletion;
302        /** A string used to group similar tests */
303        protected String group;
304
305        TagCheck(GroupedMapCSSRule rule) {
306            this.rule = rule;
307        }
308
309        private static final String POSSIBLE_THROWS = possibleThrows();
310
311        static final String possibleThrows() {
312            StringBuilder sb = new StringBuilder();
313            for (Severity s : Severity.values()) {
314                if (sb.length() > 0) {
315                    sb.append('/');
316                }
317                sb.append("throw")
318                .append(s.name().charAt(0))
319                .append(s.name().substring(1).toLowerCase(Locale.ENGLISH));
320            }
321            return sb.toString();
322        }
323
324        static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) throws IllegalDataException {
325            final TagCheck check = new TagCheck(rule);
326            for (Instruction i : rule.declaration.instructions) {
327                if (i instanceof Instruction.AssignmentInstruction) {
328                    final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i;
329                    if (ai.isSetInstruction) {
330                        check.setClassExpressions.add(ai.key);
331                        continue;
332                    }
333                    try {
334                        final String val = ai.val instanceof Expression
335                                ? Optional.ofNullable(((Expression) ai.val).evaluate(new Environment())).map(Object::toString).orElse(null)
336                                : ai.val instanceof String
337                                ? (String) ai.val
338                                : ai.val instanceof Keyword
339                                ? ((Keyword) ai.val).val
340                                : null;
341                        if (ai.key.startsWith("throw")) {
342                            try {
343                                check.errors.put(ai, Severity.valueOf(ai.key.substring("throw".length()).toUpperCase(Locale.ENGLISH)));
344                            } catch (IllegalArgumentException e) {
345                                Logging.log(Logging.LEVEL_WARN,
346                                        "Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS+'.', e);
347                            }
348                        } else if ("fixAdd".equals(ai.key)) {
349                            check.fixCommands.add(FixCommand.fixAdd(ai.val));
350                        } else if ("fixRemove".equals(ai.key)) {
351                            CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")),
352                                    "Unexpected '='. Please only specify the key to remove in: " + ai);
353                            check.fixCommands.add(FixCommand.fixRemove(ai.val));
354                        } else if (val != null && "fixChangeKey".equals(ai.key)) {
355                            CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!");
356                            final String[] x = val.split("=>", 2);
357                            check.fixCommands.add(FixCommand.fixChangeKey(Utils.removeWhiteSpaces(x[0]), Utils.removeWhiteSpaces(x[1])));
358                        } else if (val != null && "fixDeleteObject".equals(ai.key)) {
359                            CheckParameterUtil.ensureThat("this".equals(val), "fixDeleteObject must be followed by 'this'");
360                            check.deletion = true;
361                        } else if (val != null && "suggestAlternative".equals(ai.key)) {
362                            check.alternatives.add(val);
363                        } else if (val != null && "assertMatch".equals(ai.key)) {
364                            check.assertions.put(val, Boolean.TRUE);
365                        } else if (val != null && "assertNoMatch".equals(ai.key)) {
366                            check.assertions.put(val, Boolean.FALSE);
367                        } else if (val != null && "group".equals(ai.key)) {
368                            check.group = val;
369                        } else if (ai.key.startsWith("-")) {
370                            Logging.debug("Ignoring extension instruction: " + ai.key + ": " + ai.val);
371                        } else {
372                            throw new IllegalDataException("Cannot add instruction " + ai.key + ": " + ai.val + '!');
373                        }
374                    } catch (IllegalArgumentException e) {
375                        throw new IllegalDataException(e);
376                    }
377                }
378            }
379            if (check.errors.isEmpty() && check.setClassExpressions.isEmpty()) {
380                throw new IllegalDataException(
381                        "No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors);
382            } else if (check.errors.size() > 1) {
383                throw new IllegalDataException(
384                        "More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for "
385                                + rule.selectors);
386            }
387            return check;
388        }
389
390        static ParseResult readMapCSS(Reader css) throws ParseException {
391            CheckParameterUtil.ensureParameterNotNull(css, "css");
392
393            final MapCSSStyleSource source = new MapCSSStyleSource("");
394            final MapCSSParser preprocessor = new MapCSSParser(css, MapCSSParser.LexicalState.PREPROCESSOR);
395            final StringReader mapcss = new StringReader(preprocessor.pp_root(source));
396            final MapCSSParser parser = new MapCSSParser(mapcss, MapCSSParser.LexicalState.DEFAULT);
397            parser.sheet(source);
398            // Ignore "meta" rule(s) from external rules of JOSM wiki
399            source.removeMetaRules();
400            // group rules with common declaration block
401            Map<Declaration, List<Selector>> g = new LinkedHashMap<>();
402            for (MapCSSRule rule : source.rules) {
403                if (!g.containsKey(rule.declaration)) {
404                    List<Selector> sels = new ArrayList<>();
405                    sels.add(rule.selector);
406                    g.put(rule.declaration, sels);
407                } else {
408                    g.get(rule.declaration).add(rule.selector);
409                }
410            }
411            List<TagCheck> parseChecks = new ArrayList<>();
412            for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) {
413                try {
414                    parseChecks.add(TagCheck.ofMapCSSRule(
415                            new GroupedMapCSSRule(map.getValue(), map.getKey())));
416                } catch (IllegalDataException e) {
417                    Logging.error("Cannot add MapCss rule: "+e.getMessage());
418                    source.logError(e);
419                }
420            }
421            return new ParseResult(parseChecks, source.getErrors());
422        }
423
424        @Override
425        public boolean test(OsmPrimitive primitive) {
426            // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker.
427            return whichSelectorMatchesPrimitive(primitive) != null;
428        }
429
430        Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) {
431            return whichSelectorMatchesEnvironment(new Environment(primitive));
432        }
433
434        Selector whichSelectorMatchesEnvironment(Environment env) {
435            for (Selector i : rule.selectors) {
436                env.clearSelectorMatchingInformation();
437                if (i.matches(env)) {
438                    return i;
439                }
440            }
441            return null;
442        }
443
444        /**
445         * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the
446         * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}.
447         * @param matchingSelector matching selector
448         * @param index index
449         * @param type selector type ("key", "value" or "tag")
450         * @param p OSM primitive
451         * @return argument value, can be {@code null}
452         */
453        static String determineArgument(OptimizedGeneralSelector matchingSelector, int index, String type, OsmPrimitive p) {
454            try {
455                final Condition c = matchingSelector.getConditions().get(index);
456                final Tag tag = c instanceof Condition.ToTagConvertable
457                        ? ((Condition.ToTagConvertable) c).asTag(p)
458                        : null;
459                if (tag == null) {
460                    return null;
461                } else if ("key".equals(type)) {
462                    return tag.getKey();
463                } else if ("value".equals(type)) {
464                    return tag.getValue();
465                } else if ("tag".equals(type)) {
466                    return tag.toString();
467                }
468            } catch (IndexOutOfBoundsException ignore) {
469                Logging.debug(ignore);
470            }
471            return null;
472        }
473
474        /**
475         * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding
476         * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}.
477         * @param matchingSelector matching selector
478         * @param s any string
479         * @param p OSM primitive
480         * @return string with arguments inserted
481         */
482        static String insertArguments(Selector matchingSelector, String s, OsmPrimitive p) {
483            if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) {
484                return insertArguments(((Selector.ChildOrParentSelector) matchingSelector).right, s, p);
485            } else if (s == null || !(matchingSelector instanceof Selector.OptimizedGeneralSelector)) {
486                return s;
487            }
488            final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s);
489            final StringBuffer sb = new StringBuffer();
490            while (m.find()) {
491                final String argument = determineArgument((Selector.OptimizedGeneralSelector) matchingSelector,
492                        Integer.parseInt(m.group(1)), m.group(2), p);
493                try {
494                    // Perform replacement with null-safe + regex-safe handling
495                    m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", ""));
496                } catch (IndexOutOfBoundsException | IllegalArgumentException e) {
497                    Logging.log(Logging.LEVEL_ERROR, tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()), e);
498                }
499            }
500            m.appendTail(sb);
501            return sb.toString();
502        }
503
504        /**
505         * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive}
506         * if the error is fixable, or {@code null} otherwise.
507         *
508         * @param p the primitive to construct the fix for
509         * @return the fix or {@code null}
510         */
511        Command fixPrimitive(OsmPrimitive p) {
512            if (fixCommands.isEmpty() && !deletion) {
513                return null;
514            }
515            try {
516                final Selector matchingSelector = whichSelectorMatchesPrimitive(p);
517                Collection<Command> cmds = new LinkedList<>();
518                for (FixCommand fixCommand : fixCommands) {
519                    cmds.add(fixCommand.createCommand(p, matchingSelector));
520                }
521                if (deletion && !p.isDeleted()) {
522                    cmds.add(new DeleteCommand(p));
523                }
524                return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds);
525            } catch (IllegalArgumentException e) {
526                Logging.error(e);
527                return null;
528            }
529        }
530
531        /**
532         * Constructs a (localized) message for this deprecation check.
533         * @param p OSM primitive
534         *
535         * @return a message
536         */
537        String getMessage(OsmPrimitive p) {
538            if (errors.isEmpty()) {
539                // Return something to avoid NPEs
540                return rule.declaration.toString();
541            } else {
542                final Object val = errors.keySet().iterator().next().val;
543                return String.valueOf(
544                        val instanceof Expression
545                                ? ((Expression) val).evaluate(new Environment(p))
546                                : val
547                );
548            }
549        }
550
551        /**
552         * Constructs a (localized) description for this deprecation check.
553         * @param p OSM primitive
554         *
555         * @return a description (possibly with alternative suggestions)
556         * @see #getDescriptionForMatchingSelector
557         */
558        String getDescription(OsmPrimitive p) {
559            if (alternatives.isEmpty()) {
560                return getMessage(p);
561            } else {
562                /* I18N: {0} is the test error message and {1} is an alternative */
563                return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives));
564            }
565        }
566
567        /**
568         * Constructs a (localized) description for this deprecation check
569         * where any placeholders are replaced by values of the matched selector.
570         *
571         * @param matchingSelector matching selector
572         * @param p OSM primitive
573         * @return a description (possibly with alternative suggestions)
574         */
575        String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) {
576            return insertArguments(matchingSelector, getDescription(p), p);
577        }
578
579        Severity getSeverity() {
580            return errors.isEmpty() ? null : errors.values().iterator().next();
581        }
582
583        @Override
584        public String toString() {
585            return getDescription(null);
586        }
587
588        /**
589         * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error.
590         *
591         * @param p the primitive to construct the error for
592         * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error.
593         */
594        List<TestError> getErrorsForPrimitive(OsmPrimitive p) {
595            final Environment env = new Environment(p);
596            return getErrorsForPrimitive(p, whichSelectorMatchesEnvironment(env), env, null);
597        }
598
599        private List<TestError> getErrorsForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env, Test tester) {
600            List<TestError> res = new ArrayList<>();
601            if (matchingSelector != null && !errors.isEmpty()) {
602                final Command fix = fixPrimitive(p);
603                final String description = getDescriptionForMatchingSelector(p, matchingSelector);
604                final String description1 = group == null ? description : group;
605                final String description2 = group == null ? null : description;
606                TestError.Builder errorBuilder = TestError.builder(tester, getSeverity(), 3000)
607                        .messageWithManuallyTranslatedDescription(description1, description2, matchingSelector.toString());
608                if (fix != null) {
609                    errorBuilder = errorBuilder.fix(() -> fix);
610                }
611                if (env.child instanceof OsmPrimitive) {
612                    res.add(errorBuilder.primitives(p, (OsmPrimitive) env.child).build());
613                } else if (env.children != null) {
614                    for (IPrimitive c : env.children) {
615                        if (c instanceof OsmPrimitive) {
616                            errorBuilder = TestError.builder(tester, getSeverity(), 3000)
617                                    .messageWithManuallyTranslatedDescription(description1, description2,
618                                            matchingSelector.toString());
619                            if (fix != null) {
620                                errorBuilder = errorBuilder.fix(() -> fix);
621                            }
622                            res.add(errorBuilder.primitives(p, (OsmPrimitive) c).build());
623                        }
624                    }
625                } else {
626                    res.add(errorBuilder.primitives(p).build());
627                }
628            }
629            return res;
630        }
631
632        /**
633         * Returns the set of tagchecks on which this check depends on.
634         * @param schecks the collection of tagcheks to search in
635         * @return the set of tagchecks on which this check depends on
636         * @since 7881
637         */
638        public Set<TagCheck> getTagCheckDependencies(Collection<TagCheck> schecks) {
639            Set<TagCheck> result = new HashSet<>();
640            Set<String> classes = getClassesIds();
641            if (schecks != null && !classes.isEmpty()) {
642                for (TagCheck tc : schecks) {
643                    if (this.equals(tc)) {
644                        continue;
645                    }
646                    for (String id : tc.setClassExpressions) {
647                        if (classes.contains(id)) {
648                            result.add(tc);
649                            break;
650                        }
651                    }
652                }
653            }
654            return result;
655        }
656
657        /**
658         * Returns the list of ids of all MapCSS classes referenced in the rule selectors.
659         * @return the list of ids of all MapCSS classes referenced in the rule selectors
660         * @since 7881
661         */
662        public Set<String> getClassesIds() {
663            Set<String> result = new HashSet<>();
664            for (Selector s : rule.selectors) {
665                if (s instanceof AbstractSelector) {
666                    for (Condition c : ((AbstractSelector) s).getConditions()) {
667                        if (c instanceof ClassCondition) {
668                            result.add(((ClassCondition) c).id);
669                        }
670                    }
671                }
672            }
673            return result;
674        }
675    }
676
677    static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
678        public final GroupedMapCSSRule rule;
679
680        MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) {
681            this.rule = rule;
682        }
683
684        @Override
685        public synchronized boolean equals(Object obj) {
686            return super.equals(obj)
687                    || (obj instanceof TagCheck && rule.equals(((TagCheck) obj).rule))
688                    || (obj instanceof GroupedMapCSSRule && rule.equals(obj));
689        }
690
691        @Override
692        public synchronized int hashCode() {
693            return Objects.hash(super.hashCode(), rule);
694        }
695
696        @Override
697        public String toString() {
698            return "MapCSSTagCheckerAndRule [rule=" + rule + ']';
699        }
700    }
701
702    /**
703     * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
704     * @param p The OSM primitive
705     * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned
706     * @return all errors for the given primitive, with or without those of "info" severity
707     */
708    public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
709        final List<TestError> res = new ArrayList<>();
710        if (indexData == null) {
711            indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverity, MapCSSTagCheckerIndex.ALL_TESTS);
712        }
713
714        MapCSSRuleIndex matchingRuleIndex = indexData.get(p);
715
716        Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
717        // the declaration indices are sorted, so it suffices to save the last used index
718        Declaration lastDeclUsed = null;
719
720        Iterator<MapCSSRule> candidates = matchingRuleIndex.getRuleCandidates(p);
721        while (candidates.hasNext()) {
722            MapCSSRule r = candidates.next();
723            env.clearSelectorMatchingInformation();
724            if (r.selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector)
725                TagCheck check = indexData.getCheck(r);
726                if (check != null) {
727                    if (r.declaration == lastDeclUsed)
728                        continue; // don't apply one declaration more than once
729                    lastDeclUsed = r.declaration;
730
731                    r.declaration.execute(env);
732                    if (!check.errors.isEmpty()) {
733                        for (TestError e: check.getErrorsForPrimitive(p, r.selector, env, new MapCSSTagCheckerAndRule(check.rule))) {
734                            addIfNotSimilar(e, res);
735                        }
736                    }
737                }
738            }
739        }
740        return res;
741    }
742
743    /**
744     * See #12627
745     * Add error to given list if list doesn't already contain a similar error.
746     * Similar means same code and description and same combination of primitives and same combination of highlighted objects,
747     * but maybe with different orders.
748     * @param toAdd the error to add
749     * @param errors the list of errors
750     */
751    private static void addIfNotSimilar(TestError toAdd, List<TestError> errors) {
752        boolean isDup = false;
753        if (toAdd.getPrimitives().size() >= 2) {
754            for (TestError e : errors) {
755                if (e.getCode() == toAdd.getCode() && e.getMessage().equals(toAdd.getMessage())
756                        && e.getPrimitives().size() == toAdd.getPrimitives().size()
757                        && e.getPrimitives().containsAll(toAdd.getPrimitives())
758                        && e.getHighlighted().size() == toAdd.getHighlighted().size()
759                        && e.getHighlighted().containsAll(toAdd.getHighlighted())) {
760                    isDup = true;
761                    break;
762                }
763            }
764        }
765        if (!isDup)
766            errors.add(toAdd);
767    }
768
769    private static Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity,
770            Collection<Set<TagCheck>> checksCol) {
771        final List<TestError> r = new ArrayList<>();
772        final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
773        for (Set<TagCheck> schecks : checksCol) {
774            for (TagCheck check : schecks) {
775                boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity;
776                // Do not run "information" level checks if not wanted, unless they also set a MapCSS class
777                if (ignoreError && check.setClassExpressions.isEmpty()) {
778                    continue;
779                }
780                final Selector selector = check.whichSelectorMatchesEnvironment(env);
781                if (selector != null) {
782                    check.rule.declaration.execute(env);
783                    if (!ignoreError && !check.errors.isEmpty()) {
784                        r.addAll(check.getErrorsForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule)));
785                    }
786                }
787            }
788        }
789        return r;
790    }
791
792    /**
793     * Visiting call for primitives.
794     *
795     * @param p The primitive to inspect.
796     */
797    @Override
798    public void check(OsmPrimitive p) {
799        for (TestError e : getErrorsForPrimitive(p, ValidatorPrefHelper.PREF_OTHER.get())) {
800            addIfNotSimilar(e, errors);
801        }
802        if (partialSelection && p.isTagged()) {
803            tested.add(p);
804        }
805    }
806
807    /**
808     * Adds a new MapCSS config file from the given URL.
809     * @param url The unique URL of the MapCSS config file
810     * @return List of tag checks and parsing errors, or null
811     * @throws ParseException if the config file does not match MapCSS syntax
812     * @throws IOException if any I/O error occurs
813     * @since 7275
814     */
815    public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException {
816        CheckParameterUtil.ensureParameterNotNull(url, "url");
817        ParseResult result;
818        try (CachedFile cache = new CachedFile(url);
819             InputStream zip = cache.findZipEntryInputStream("validator.mapcss", "");
820             InputStream s = zip != null ? zip : cache.getInputStream();
821             Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) {
822            if (zip != null)
823                I18n.addTexts(cache.getFile());
824            result = TagCheck.readMapCSS(reader);
825            checks.remove(url);
826            checks.putAll(url, result.parseChecks);
827            indexData = null;
828            // Check assertions, useful for development of local files
829            if (Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url)) {
830                for (String msg : checkAsserts(result.parseChecks)) {
831                    Logging.warn(msg);
832                }
833            }
834        }
835        return result;
836    }
837
838    @Override
839    public synchronized void initialize() throws Exception {
840        checks.clear();
841        indexData = null;
842        for (SourceEntry source : new ValidatorPrefHelper().get()) {
843            if (!source.active) {
844                continue;
845            }
846            String i = source.url;
847            try {
848                if (!i.startsWith("resource:")) {
849                    Logging.info(tr("Adding {0} to tag checker", i));
850                } else if (Logging.isDebugEnabled()) {
851                    Logging.debug(tr("Adding {0} to tag checker", i));
852                }
853                addMapCSS(i);
854                if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) {
855                    FileWatcher.getDefaultInstance().registerSource(source);
856                }
857            } catch (IOException | IllegalStateException | IllegalArgumentException ex) {
858                Logging.warn(tr("Failed to add {0} to tag checker", i));
859                Logging.log(Logging.LEVEL_WARN, ex);
860            } catch (ParseException | TokenMgrError ex) {
861                Logging.warn(tr("Failed to add {0} to tag checker", i));
862                Logging.warn(ex);
863            }
864        }
865    }
866
867    private static Method getFunctionMethod(String method) {
868        try {
869            return Functions.class.getDeclaredMethod(method, Environment.class, String.class);
870        } catch (NoSuchMethodException | SecurityException e) {
871            Logging.error(e);
872            return null;
873        }
874    }
875
876    private static Optional<String> getFirstInsideCountry(TagCheck check, Method insideMethod) {
877        return check.rule.selectors.stream()
878                .filter(s -> s instanceof GeneralSelector)
879                .flatMap(s -> ((GeneralSelector) s).getConditions().stream())
880                .filter(c -> c instanceof ExpressionCondition)
881                .map(c -> ((ExpressionCondition) c).getExpression())
882                .filter(c -> c instanceof ParameterFunction)
883                .map(c -> (ParameterFunction) c)
884                .filter(c -> c.getMethod().equals(insideMethod))
885                .flatMap(c -> c.getArgs().stream())
886                .filter(e -> e instanceof LiteralExpression)
887                .map(e -> ((LiteralExpression) e).getLiteral())
888                .filter(l -> l instanceof String)
889                .map(l -> ((String) l).split(",")[0])
890                .findFirst();
891    }
892
893    private static LatLon getLocation(TagCheck check, Method insideMethod) {
894        Optional<String> inside = getFirstInsideCountry(check, insideMethod);
895        if (inside.isPresent()) {
896            GeoPropertyIndex<Boolean> index = Territories.getGeoPropertyIndex(inside.get());
897            if (index != null) {
898                GeoProperty<Boolean> prop = index.getGeoProperty();
899                if (prop instanceof DefaultGeoProperty) {
900                    Rectangle bounds = ((DefaultGeoProperty) prop).getArea().getBounds();
901                    return new LatLon(bounds.getCenterY(), bounds.getCenterX());
902                }
903            }
904        }
905        return LatLon.ZERO;
906    }
907
908    /**
909     * Checks that rule assertions are met for the given set of TagChecks.
910     * @param schecks The TagChecks for which assertions have to be checked
911     * @return A set of error messages, empty if all assertions are met
912     * @since 7356
913     */
914    public Set<String> checkAsserts(final Collection<TagCheck> schecks) {
915        Set<String> assertionErrors = new LinkedHashSet<>();
916        final Method insideMethod = getFunctionMethod("inside");
917        final DataSet ds = new DataSet();
918        for (final TagCheck check : schecks) {
919            Logging.debug("Check: {0}", check);
920            for (final Map.Entry<String, Boolean> i : check.assertions.entrySet()) {
921                Logging.debug("- Assertion: {0}", i);
922                final OsmPrimitive p = OsmUtils.createPrimitive(i.getKey(), getLocation(check, insideMethod), true);
923                // Build minimal ordered list of checks to run to test the assertion
924                List<Set<TagCheck>> checksToRun = new ArrayList<>();
925                Set<TagCheck> checkDependencies = check.getTagCheckDependencies(schecks);
926                if (!checkDependencies.isEmpty()) {
927                    checksToRun.add(checkDependencies);
928                }
929                checksToRun.add(Collections.singleton(check));
930                // Add primitive to dataset to avoid DataIntegrityProblemException when evaluating selectors
931                addPrimitive(ds, p);
932                final Collection<TestError> pErrors = getErrorsForPrimitive(p, true, checksToRun);
933                Logging.debug("- Errors: {0}", pErrors);
934                @SuppressWarnings({"EqualsBetweenInconvertibleTypes", "EqualsIncompatibleType"})
935                final boolean isError = pErrors.stream().anyMatch(e -> e.getTester().equals(check.rule));
936                if (isError != i.getValue()) {
937                    final String error = MessageFormat.format("Expecting test ''{0}'' (i.e., {1}) to {2} {3} (i.e., {4})",
938                            check.getMessage(p), check.rule.selectors, i.getValue() ? "match" : "not match", i.getKey(), p.getKeys());
939                    assertionErrors.add(error);
940                }
941                ds.removePrimitive(p);
942            }
943        }
944        return assertionErrors;
945    }
946
947    private static void addPrimitive(DataSet ds, OsmPrimitive p) {
948        if (p instanceof Way) {
949            ((Way) p).getNodes().forEach(n -> addPrimitive(ds, n));
950        } else if (p instanceof Relation) {
951            ((Relation) p).getMembers().forEach(m -> addPrimitive(ds, m.getMember()));
952        }
953        ds.addPrimitive(p);
954    }
955
956    @Override
957    public synchronized int hashCode() {
958        return Objects.hash(super.hashCode(), checks);
959    }
960
961    @Override
962    public synchronized boolean equals(Object obj) {
963        if (this == obj) return true;
964        if (obj == null || getClass() != obj.getClass()) return false;
965        if (!super.equals(obj)) return false;
966        MapCSSTagChecker that = (MapCSSTagChecker) obj;
967        return Objects.equals(checks, that.checks);
968    }
969
970    /**
971     * Reload tagchecker rule.
972     * @param rule tagchecker rule to reload
973     * @since 12825
974     */
975    public static void reloadRule(SourceEntry rule) {
976        MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class);
977        if (tagChecker != null) {
978            try {
979                tagChecker.addMapCSS(rule.url);
980            } catch (IOException | ParseException | TokenMgrError e) {
981                Logging.warn(e);
982            }
983        }
984    }
985
986    @Override
987    public synchronized void startTest(ProgressMonitor progressMonitor) {
988        super.startTest(progressMonitor);
989        super.setShowElements(true);
990        if (indexData == null) {
991            indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverityChecks(), MapCSSTagCheckerIndex.ALL_TESTS);
992        }
993        tested.clear();
994    }
995
996    @Override
997    public synchronized void endTest() {
998        if (partialSelection && !tested.isEmpty()) {
999            // #14287: see https://josm.openstreetmap.de/ticket/14287#comment:15
1000            // execute tests for objects which might contain or cross previously tested elements
1001
1002            // rebuild index with a reduced set of rules (those that use ChildOrParentSelector) and thus may have left selectors
1003            // matching the previously tested elements
1004            indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverityChecks(), MapCSSTagCheckerIndex.ONLY_SELECTED_TESTS);
1005
1006            Set<OsmPrimitive> surrounding = new HashSet<>();
1007            for (OsmPrimitive p : tested) {
1008                if (p.getDataSet() != null) {
1009                    surrounding.addAll(p.getDataSet().searchWays(p.getBBox()));
1010                    surrounding.addAll(p.getDataSet().searchRelations(p.getBBox()));
1011                }
1012            }
1013            final boolean includeOtherSeverity = includeOtherSeverityChecks();
1014            for (OsmPrimitive p : surrounding) {
1015                if (tested.contains(p))
1016                    continue;
1017                Collection<TestError> additionalErrors = getErrorsForPrimitive(p, includeOtherSeverity);
1018                for (TestError e : additionalErrors) {
1019                    if (e.getPrimitives().stream().anyMatch(tested::contains))
1020                        addIfNotSimilar(e, errors);
1021                }
1022            }
1023            tested.clear();
1024        }
1025        super.endTest();
1026        // no need to keep the index, it is quickly build and doubles the memory needs
1027        indexData = null;
1028    }
1029
1030    private boolean includeOtherSeverityChecks() {
1031        return isBeforeUpload ? ValidatorPrefHelper.PREF_OTHER_UPLOAD.get() : ValidatorPrefHelper.PREF_OTHER.get();
1032    }
1033
1034}