001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.tr;
005
006import java.io.BufferedReader;
007import java.io.IOException;
008import java.io.InputStream;
009import java.io.Reader;
010import java.util.ArrayList;
011import java.util.Arrays;
012import java.util.Collection;
013import java.util.Collections;
014import java.util.HashMap;
015import java.util.Iterator;
016import java.util.LinkedHashMap;
017import java.util.LinkedList;
018import java.util.List;
019import java.util.Map;
020import java.util.Set;
021import java.util.regex.Matcher;
022import java.util.regex.Pattern;
023
024import org.openstreetmap.josm.Main;
025import org.openstreetmap.josm.command.ChangePropertyCommand;
026import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
027import org.openstreetmap.josm.command.Command;
028import org.openstreetmap.josm.command.SequenceCommand;
029import org.openstreetmap.josm.data.osm.OsmPrimitive;
030import org.openstreetmap.josm.data.osm.Tag;
031import org.openstreetmap.josm.data.validation.FixableTestError;
032import org.openstreetmap.josm.data.validation.Severity;
033import org.openstreetmap.josm.data.validation.Test;
034import org.openstreetmap.josm.data.validation.TestError;
035import org.openstreetmap.josm.gui.mappaint.Environment;
036import org.openstreetmap.josm.gui.mappaint.MultiCascade;
037import org.openstreetmap.josm.gui.mappaint.mapcss.Condition;
038import org.openstreetmap.josm.gui.mappaint.mapcss.Expression;
039import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction;
040import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule;
041import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration;
042import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource;
043import org.openstreetmap.josm.gui.mappaint.mapcss.Selector;
044import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector;
045import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser;
046import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException;
047import org.openstreetmap.josm.gui.preferences.SourceEntry;
048import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference;
049import org.openstreetmap.josm.gui.preferences.validator.ValidatorTagCheckerRulesPreference;
050import org.openstreetmap.josm.io.CachedFile;
051import org.openstreetmap.josm.io.UTFInputStreamReader;
052import org.openstreetmap.josm.tools.CheckParameterUtil;
053import org.openstreetmap.josm.tools.MultiMap;
054import org.openstreetmap.josm.tools.Predicate;
055import org.openstreetmap.josm.tools.Utils;
056
057/**
058 * MapCSS-based tag checker/fixer.
059 * @since 6506
060 */
061public class MapCSSTagChecker extends Test.TagTest {
062
063    /**
064     * A grouped MapCSSRule with multiple selectors for a single declaration.
065     * @see MapCSSRule
066     */
067    public static class GroupedMapCSSRule {
068        /** MapCSS selectors **/
069        final public List<Selector> selectors;
070        /** MapCSS declaration **/
071        final public Declaration declaration;
072
073        /**
074         * Constructs a new {@code GroupedMapCSSRule}.
075         * @param selectors MapCSS selectors
076         * @param declaration MapCSS declaration
077         */
078        public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) {
079            this.selectors = selectors;
080            this.declaration = declaration;
081        }
082
083        @Override
084        public int hashCode() {
085            final int prime = 31;
086            int result = 1;
087            result = prime * result + ((declaration == null) ? 0 : declaration.hashCode());
088            result = prime * result + ((selectors == null) ? 0 : selectors.hashCode());
089            return result;
090        }
091
092        @Override
093        public boolean equals(Object obj) {
094            if (this == obj)
095                return true;
096            if (obj == null)
097                return false;
098            if (!(obj instanceof GroupedMapCSSRule))
099                return false;
100            GroupedMapCSSRule other = (GroupedMapCSSRule) obj;
101            if (declaration == null) {
102                if (other.declaration != null)
103                    return false;
104            } else if (!declaration.equals(other.declaration))
105                return false;
106            if (selectors == null) {
107                if (other.selectors != null)
108                    return false;
109            } else if (!selectors.equals(other.selectors))
110                return false;
111            return true;
112        }
113    }
114
115    /**
116     * The preference key for tag checker source entries.
117     * @since 6670
118     */
119    public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries";
120
121    /**
122     * Constructs a new {@code MapCSSTagChecker}.
123     */
124    public MapCSSTagChecker() {
125        super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values."));
126    }
127
128    final MultiMap<String, TagCheck> checks = new MultiMap<>();
129
130    static class TagCheck implements Predicate<OsmPrimitive> {
131        protected final GroupedMapCSSRule rule;
132        protected final List<PrimitiveToTag> change = new ArrayList<>();
133        protected final Map<String, String> keyChange = new LinkedHashMap<>();
134        protected final List<String> alternatives = new ArrayList<>();
135        protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>();
136        protected final Map<String, Boolean> assertions = new HashMap<>();
137
138        TagCheck(GroupedMapCSSRule rule) {
139            this.rule = rule;
140        }
141
142        /**
143         * A function mapping the matched {@link OsmPrimitive} to a {@link Tag}.
144         */
145        abstract static class PrimitiveToTag implements Utils.Function<OsmPrimitive, Tag> {
146
147            private PrimitiveToTag() {
148                // Hide implicit public constructor for utility class
149            }
150
151            /**
152             * Creates a new mapping from an {@code MapCSS} object.
153             * In case of an {@link Expression}, that is evaluated on the matched {@link OsmPrimitive}.
154             * In case of a {@link String}, that is "compiled" to a {@link Tag} instance.
155             */
156            static PrimitiveToTag ofMapCSSObject(final Object obj, final boolean keyOnly) {
157                if (obj instanceof Expression) {
158                    return new PrimitiveToTag() {
159                        @Override
160                        public Tag apply(OsmPrimitive p) {
161                            final String s = (String) ((Expression) obj).evaluate(new Environment().withPrimitive(p));
162                            return keyOnly? new Tag(s) : Tag.ofString(s);
163                        }
164                    };
165                } else if (obj instanceof String) {
166                    final Tag tag = keyOnly ? new Tag((String) obj) : Tag.ofString((String) obj);
167                    return new PrimitiveToTag() {
168                        @Override
169                        public Tag apply(OsmPrimitive ignore) {
170                            return tag;
171                        }
172                    };
173                } else {
174                    return null;
175                }
176            }
177        }
178
179        static final String POSSIBLE_THROWS = possibleThrows();
180
181        static final String possibleThrows() {
182            StringBuffer sb = new StringBuffer();
183            for (Severity s : Severity.values()) {
184                if (sb.length() > 0) {
185                    sb.append('/');
186                }
187                sb.append("throw")
188                .append(s.name().charAt(0))
189                .append(s.name().substring(1).toLowerCase());
190            }
191            return sb.toString();
192        }
193
194        static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) {
195            final TagCheck check = new TagCheck(rule);
196            boolean containsSetClassExpression = false;
197            for (Instruction i : rule.declaration.instructions) {
198                if (i instanceof Instruction.AssignmentInstruction) {
199                    final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i;
200                    if (ai.isSetInstruction) {
201                        containsSetClassExpression = true;
202                        continue;
203                    }
204                    final String val = ai.val instanceof Expression
205                            ? (String) ((Expression) ai.val).evaluate(new Environment())
206                            : ai.val instanceof String
207                            ? (String) ai.val
208                            : null;
209                    if (ai.key.startsWith("throw")) {
210                        try {
211                            final Severity severity = Severity.valueOf(ai.key.substring("throw".length()).toUpperCase());
212                            check.errors.put(ai, severity);
213                        } catch (IllegalArgumentException e) {
214                            Main.warn("Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS);
215                        }
216                    } else if ("fixAdd".equals(ai.key)) {
217                        final PrimitiveToTag toTag = PrimitiveToTag.ofMapCSSObject(ai.val, false);
218                        if (toTag != null) {
219                            check.change.add(toTag);
220                        } else {
221                            Main.warn("Invalid value for "+ai.key+": "+ai.val);
222                        }
223                    } else if ("fixRemove".equals(ai.key)) {
224                        CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")),
225                                "Unexpected '='. Please only specify the key to remove!");
226                        final PrimitiveToTag toTag = PrimitiveToTag.ofMapCSSObject(ai.val, true);
227                        if (toTag != null) {
228                            check.change.add(toTag);
229                        } else {
230                            Main.warn("Invalid value for "+ai.key+": "+ai.val);
231                        }
232                    } else if ("fixChangeKey".equals(ai.key) && val != null) {
233                        CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!");
234                        final String[] x = val.split("=>", 2);
235                        check.keyChange.put(Tag.removeWhiteSpaces(x[0]), Tag.removeWhiteSpaces(x[1]));
236                    } else if ("suggestAlternative".equals(ai.key) && val != null) {
237                        check.alternatives.add(val);
238                    } else if ("assertMatch".equals(ai.key) && val != null) {
239                        check.assertions.put(val, true);
240                    } else if ("assertNoMatch".equals(ai.key) && val != null) {
241                        check.assertions.put(val, false);
242                    } else {
243                        throw new RuntimeException("Cannot add instruction " + ai.key + ": " + ai.val + "!");
244                    }
245                }
246            }
247            if (check.errors.isEmpty() && !containsSetClassExpression) {
248                throw new RuntimeException("No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors);
249            } else if (check.errors.size() > 1) {
250                throw new RuntimeException("More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for " + rule.selectors);
251            }
252            return check;
253        }
254
255        static List<TagCheck> readMapCSS(Reader css) throws ParseException {
256            CheckParameterUtil.ensureParameterNotNull(css, "css");
257            return readMapCSS(new MapCSSParser(css));
258        }
259
260        static List<TagCheck> readMapCSS(MapCSSParser css) throws ParseException {
261            CheckParameterUtil.ensureParameterNotNull(css, "css");
262            final MapCSSStyleSource source = new MapCSSStyleSource("");
263            css.sheet(source);
264            assert source.getErrors().isEmpty();
265            // Ignore "meta" rule(s) from external rules of JOSM wiki
266            removeMetaRules(source);
267            // group rules with common declaration block
268            Map<Declaration, List<Selector>> g = new LinkedHashMap<>();
269            for (MapCSSRule rule : source.rules) {
270                if (!g.containsKey(rule.declaration)) {
271                    List<Selector> sels = new ArrayList<>();
272                    sels.add(rule.selector);
273                    g.put(rule.declaration, sels);
274                } else {
275                    g.get(rule.declaration).add(rule.selector);
276                }
277            }
278            List<TagCheck> result = new ArrayList<>();
279            for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) {
280                result.add(TagCheck.ofMapCSSRule(
281                        new GroupedMapCSSRule(map.getValue(), map.getKey())));
282            }
283            return result;
284        }
285
286        private static void removeMetaRules(MapCSSStyleSource source) {
287            for (Iterator<MapCSSRule> it = source.rules.iterator(); it.hasNext(); ) {
288                MapCSSRule x = it.next();
289                if (x.selector instanceof GeneralSelector) {
290                    GeneralSelector gs = (GeneralSelector) x.selector;
291                    if ("meta".equals(gs.base) && gs.getConditions().isEmpty()) {
292                        it.remove();
293                    }
294                }
295            }
296        }
297
298        @Override
299        public boolean evaluate(OsmPrimitive primitive) {
300            // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker.
301            return whichSelectorMatchesPrimitive(primitive) != null;
302        }
303
304        Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) {
305            return whichSelectorMatchesEnvironment(new Environment().withPrimitive(primitive));
306        }
307
308        Selector whichSelectorMatchesEnvironment(Environment env) {
309            for (Selector i : rule.selectors) {
310                env.clearSelectorMatchingInformation();
311                if (i.matches(env)) {
312                    return i;
313                }
314            }
315            return null;
316        }
317
318        /**
319         * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the
320         * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}.
321         */
322        static String determineArgument(Selector.GeneralSelector matchingSelector, int index, String type) {
323            try {
324                final Condition c = matchingSelector.getConditions().get(index);
325                final Tag tag = c instanceof Condition.KeyCondition
326                        ? ((Condition.KeyCondition) c).asTag()
327                        : c instanceof Condition.SimpleKeyValueCondition
328                        ? ((Condition.SimpleKeyValueCondition) c).asTag()
329                        : c instanceof Condition.KeyValueCondition
330                        ? ((Condition.KeyValueCondition) c).asTag()
331                        : null;
332                if (tag == null) {
333                    return null;
334                } else if ("key".equals(type)) {
335                    return tag.getKey();
336                } else if ("value".equals(type)) {
337                    return tag.getValue();
338                } else if ("tag".equals(type)) {
339                    return tag.toString();
340                }
341            } catch (IndexOutOfBoundsException ignore) {
342                Main.debug(ignore.getMessage());
343            }
344            return null;
345        }
346
347        /**
348         * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding
349         * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}.
350         */
351        static String insertArguments(Selector matchingSelector, String s) {
352            if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) {
353                return  insertArguments(((Selector.ChildOrParentSelector)matchingSelector).right, s);
354            } else if (s == null || !(matchingSelector instanceof GeneralSelector)) {
355                return s;
356            }
357            final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s);
358            final StringBuffer sb = new StringBuffer();
359            while (m.find()) {
360                final String argument = determineArgument((Selector.GeneralSelector) matchingSelector, Integer.parseInt(m.group(1)), m.group(2));
361                try {
362                    // Perform replacement with null-safe + regex-safe handling
363                    m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", ""));
364                } catch (IndexOutOfBoundsException | IllegalArgumentException e) {
365                    Main.error(tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()));
366                }
367            }
368            m.appendTail(sb);
369            return sb.toString();
370        }
371
372        /**
373         * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive}
374         * if the error is fixable, or {@code null} otherwise.
375         *
376         * @param p the primitive to construct the fix for
377         * @return the fix or {@code null}
378         */
379        Command fixPrimitive(OsmPrimitive p) {
380            if (change.isEmpty() && keyChange.isEmpty()) {
381                return null;
382            }
383            final Selector matchingSelector = whichSelectorMatchesPrimitive(p);
384            Collection<Command> cmds = new LinkedList<>();
385            for (PrimitiveToTag toTag : change) {
386                final Tag tag = toTag.apply(p);
387                final String key = insertArguments(matchingSelector, tag.getKey());
388                final String value = insertArguments(matchingSelector, tag.getValue());
389                cmds.add(new ChangePropertyCommand(p, key, value));
390            }
391            for (Map.Entry<String, String> i : keyChange.entrySet()) {
392                final String oldKey = insertArguments(matchingSelector, i.getKey());
393                final String newKey = insertArguments(matchingSelector, i.getValue());
394                cmds.add(new ChangePropertyKeyCommand(p, oldKey, newKey));
395            }
396            return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds);
397        }
398
399        /**
400         * Constructs a (localized) message for this deprecation check.
401         *
402         * @return a message
403         */
404        String getMessage(OsmPrimitive p) {
405            if (errors.isEmpty()) {
406                // Return something to avoid NPEs
407                return rule.declaration.toString();
408            } else {
409                final Object val = errors.keySet().iterator().next().val;
410                return String.valueOf(
411                        val instanceof Expression
412                                ? ((Expression) val).evaluate(new Environment().withPrimitive(p))
413                                : val
414                );
415            }
416        }
417
418        /**
419         * Constructs a (localized) description for this deprecation check.
420         *
421         * @return a description (possibly with alternative suggestions)
422         * @see #getDescriptionForMatchingSelector
423         */
424        String getDescription(OsmPrimitive p) {
425            if (alternatives.isEmpty()) {
426                return getMessage(p);
427            } else {
428                /* I18N: {0} is the test error message and {1} is an alternative */
429                return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives));
430            }
431        }
432
433        /**
434         * Constructs a (localized) description for this deprecation check
435         * where any placeholders are replaced by values of the matched selector.
436         *
437         * @return a description (possibly with alternative suggestions)
438         */
439        String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) {
440            return insertArguments(matchingSelector, getDescription(p));
441        }
442
443        Severity getSeverity() {
444            return errors.isEmpty() ? null : errors.values().iterator().next();
445        }
446
447        @Override
448        public String toString() {
449            return getDescription(null);
450        }
451
452        /**
453         * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error.
454         *
455         * @param p the primitive to construct the error for
456         * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error.
457         */
458        TestError getErrorForPrimitive(OsmPrimitive p) {
459            final Environment env = new Environment().withPrimitive(p);
460            return getErrorForPrimitive(p, whichSelectorMatchesEnvironment(env), env);
461        }
462
463        TestError getErrorForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env) {
464            if (matchingSelector != null && !errors.isEmpty()) {
465                final Command fix = fixPrimitive(p);
466                final String description = getDescriptionForMatchingSelector(p, matchingSelector);
467                final List<OsmPrimitive> primitives;
468                if (env.child != null) {
469                    primitives = Arrays.asList(p, env.child);
470                } else {
471                    primitives = Collections.singletonList(p);
472                }
473                if (fix != null) {
474                    return new FixableTestError(null, getSeverity(), description, null, matchingSelector.toString(), 3000, primitives, fix);
475                } else {
476                    return new TestError(null, getSeverity(), description, null, matchingSelector.toString(), 3000, primitives);
477                }
478            } else {
479                return null;
480            }
481        }
482    }
483
484    static class MapCSSTagCheckerAndRule extends MapCSSTagChecker {
485        public final GroupedMapCSSRule rule;
486
487        MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) {
488            this.rule = rule;
489        }
490
491        @Override
492        public boolean equals(Object obj) {
493            return super.equals(obj)
494                    || (obj instanceof TagCheck && rule.equals(((TagCheck) obj).rule))
495                    || (obj instanceof GroupedMapCSSRule && rule.equals(obj));
496        }
497
498        @Override
499        public int hashCode() {
500            final int prime = 31;
501            int result = super.hashCode();
502            result = prime * result + ((rule == null) ? 0 : rule.hashCode());
503            return result;
504        }
505    }
506
507    /**
508     * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}.
509     * @param p The OSM primitive
510     * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned
511     * @return all errors for the given primitive, with or without those of "info" severity
512     */
513    public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) {
514        final ArrayList<TestError> r = new ArrayList<>();
515        final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null);
516        for (Set<TagCheck> schecks : checks.values()) {
517            for (TagCheck check : schecks) {
518                if (Severity.OTHER.equals(check.getSeverity()) && !includeOtherSeverity) {
519                    continue;
520                }
521                final Selector selector = check.whichSelectorMatchesEnvironment(env);
522                if (selector != null) {
523                    check.rule.declaration.execute(env);
524                    final TestError error = check.getErrorForPrimitive(p, selector, env);
525                    if (error != null) {
526                        error.setTester(new MapCSSTagCheckerAndRule(check.rule));
527                        r.add(error);
528                    }
529                }
530            }
531        }
532        return r;
533    }
534
535    /**
536     * Visiting call for primitives.
537     *
538     * @param p The primitive to inspect.
539     */
540    @Override
541    public void check(OsmPrimitive p) {
542        errors.addAll(getErrorsForPrimitive(p, ValidatorPreference.PREF_OTHER.get()));
543    }
544
545    /**
546     * Adds a new MapCSS config file from the given URL.
547     * @param url The unique URL of the MapCSS config file
548     * @throws ParseException if the config file does not match MapCSS syntax
549     * @throws IOException if any I/O error occurs
550     * @since 7275
551     */
552    public synchronized void addMapCSS(String url) throws ParseException, IOException {
553        CheckParameterUtil.ensureParameterNotNull(url, "url");
554        CachedFile cache = new CachedFile(url);
555        try (InputStream s = cache.getInputStream()) {
556            List<TagCheck> tagchecks = TagCheck.readMapCSS(new BufferedReader(UTFInputStreamReader.create(s)));
557            checks.remove(url);
558            checks.putAll(url, tagchecks);
559        }
560    }
561
562    @Override
563    public synchronized void initialize() throws Exception {
564        checks.clear();
565        for (SourceEntry source : new ValidatorTagCheckerRulesPreference.RulePrefHelper().get()) {
566            if (!source.active) {
567                continue;
568            }
569            String i = source.url;
570            try {
571                if (i.startsWith("resource:")) {
572                    Main.debug(tr("Adding {0} to tag checker", i));
573                } else {
574                    Main.info(tr("Adding {0} to tag checker", i));
575                }
576                addMapCSS(i);
577                if (Main.pref.getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) {
578                    try {
579                        Main.fileWatcher.registerValidatorRule(source);
580                    } catch (IOException e) {
581                        Main.error(e);
582                    }
583                }
584            } catch (IOException ex) {
585                Main.warn(tr("Failed to add {0} to tag checker", i));
586                Main.warn(ex, false);
587            } catch (Exception ex) {
588                Main.warn(tr("Failed to add {0} to tag checker", i));
589                Main.warn(ex);
590            }
591        }
592    }
593
594    @Override
595    public synchronized int hashCode() {
596        final int prime = 31;
597        int result = super.hashCode();
598        result = prime * result + ((checks == null) ? 0 : checks.hashCode());
599        return result;
600    }
601
602    @Override
603    public synchronized boolean equals(Object obj) {
604        if (this == obj)
605            return true;
606        if (!super.equals(obj))
607            return false;
608        if (!(obj instanceof MapCSSTagChecker))
609            return false;
610        MapCSSTagChecker other = (MapCSSTagChecker) obj;
611        if (checks == null) {
612            if (other.checks != null)
613                return false;
614        } else if (!checks.equals(other.checks))
615            return false;
616        return true;
617    }
618}