001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedReader; 007import java.io.IOException; 008import java.io.InputStream; 009import java.io.Reader; 010import java.util.ArrayList; 011import java.util.Arrays; 012import java.util.Collection; 013import java.util.Collections; 014import java.util.HashMap; 015import java.util.Iterator; 016import java.util.LinkedHashMap; 017import java.util.LinkedList; 018import java.util.List; 019import java.util.Map; 020import java.util.Set; 021import java.util.regex.Matcher; 022import java.util.regex.Pattern; 023 024import org.openstreetmap.josm.Main; 025import org.openstreetmap.josm.command.ChangePropertyCommand; 026import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 027import org.openstreetmap.josm.command.Command; 028import org.openstreetmap.josm.command.SequenceCommand; 029import org.openstreetmap.josm.data.osm.OsmPrimitive; 030import org.openstreetmap.josm.data.osm.Tag; 031import org.openstreetmap.josm.data.validation.FixableTestError; 032import org.openstreetmap.josm.data.validation.Severity; 033import org.openstreetmap.josm.data.validation.Test; 034import org.openstreetmap.josm.data.validation.TestError; 035import org.openstreetmap.josm.gui.mappaint.Environment; 036import org.openstreetmap.josm.gui.mappaint.MultiCascade; 037import org.openstreetmap.josm.gui.mappaint.mapcss.Condition; 038import org.openstreetmap.josm.gui.mappaint.mapcss.Expression; 039import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction; 040import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule; 041import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration; 042import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource; 043import org.openstreetmap.josm.gui.mappaint.mapcss.Selector; 044import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector; 045import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser; 046import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException; 047import org.openstreetmap.josm.gui.preferences.SourceEntry; 048import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference; 049import org.openstreetmap.josm.gui.preferences.validator.ValidatorTagCheckerRulesPreference; 050import org.openstreetmap.josm.io.CachedFile; 051import org.openstreetmap.josm.io.UTFInputStreamReader; 052import org.openstreetmap.josm.tools.CheckParameterUtil; 053import org.openstreetmap.josm.tools.MultiMap; 054import org.openstreetmap.josm.tools.Predicate; 055import org.openstreetmap.josm.tools.Utils; 056 057/** 058 * MapCSS-based tag checker/fixer. 059 * @since 6506 060 */ 061public class MapCSSTagChecker extends Test.TagTest { 062 063 /** 064 * A grouped MapCSSRule with multiple selectors for a single declaration. 065 * @see MapCSSRule 066 */ 067 public static class GroupedMapCSSRule { 068 /** MapCSS selectors **/ 069 final public List<Selector> selectors; 070 /** MapCSS declaration **/ 071 final public Declaration declaration; 072 073 /** 074 * Constructs a new {@code GroupedMapCSSRule}. 075 * @param selectors MapCSS selectors 076 * @param declaration MapCSS declaration 077 */ 078 public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) { 079 this.selectors = selectors; 080 this.declaration = declaration; 081 } 082 083 @Override 084 public int hashCode() { 085 final int prime = 31; 086 int result = 1; 087 result = prime * result + ((declaration == null) ? 0 : declaration.hashCode()); 088 result = prime * result + ((selectors == null) ? 0 : selectors.hashCode()); 089 return result; 090 } 091 092 @Override 093 public boolean equals(Object obj) { 094 if (this == obj) 095 return true; 096 if (obj == null) 097 return false; 098 if (!(obj instanceof GroupedMapCSSRule)) 099 return false; 100 GroupedMapCSSRule other = (GroupedMapCSSRule) obj; 101 if (declaration == null) { 102 if (other.declaration != null) 103 return false; 104 } else if (!declaration.equals(other.declaration)) 105 return false; 106 if (selectors == null) { 107 if (other.selectors != null) 108 return false; 109 } else if (!selectors.equals(other.selectors)) 110 return false; 111 return true; 112 } 113 } 114 115 /** 116 * The preference key for tag checker source entries. 117 * @since 6670 118 */ 119 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries"; 120 121 /** 122 * Constructs a new {@code MapCSSTagChecker}. 123 */ 124 public MapCSSTagChecker() { 125 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values.")); 126 } 127 128 final MultiMap<String, TagCheck> checks = new MultiMap<>(); 129 130 static class TagCheck implements Predicate<OsmPrimitive> { 131 protected final GroupedMapCSSRule rule; 132 protected final List<PrimitiveToTag> change = new ArrayList<>(); 133 protected final Map<String, String> keyChange = new LinkedHashMap<>(); 134 protected final List<String> alternatives = new ArrayList<>(); 135 protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>(); 136 protected final Map<String, Boolean> assertions = new HashMap<>(); 137 138 TagCheck(GroupedMapCSSRule rule) { 139 this.rule = rule; 140 } 141 142 /** 143 * A function mapping the matched {@link OsmPrimitive} to a {@link Tag}. 144 */ 145 abstract static class PrimitiveToTag implements Utils.Function<OsmPrimitive, Tag> { 146 147 private PrimitiveToTag() { 148 // Hide implicit public constructor for utility class 149 } 150 151 /** 152 * Creates a new mapping from an {@code MapCSS} object. 153 * In case of an {@link Expression}, that is evaluated on the matched {@link OsmPrimitive}. 154 * In case of a {@link String}, that is "compiled" to a {@link Tag} instance. 155 */ 156 static PrimitiveToTag ofMapCSSObject(final Object obj, final boolean keyOnly) { 157 if (obj instanceof Expression) { 158 return new PrimitiveToTag() { 159 @Override 160 public Tag apply(OsmPrimitive p) { 161 final String s = (String) ((Expression) obj).evaluate(new Environment().withPrimitive(p)); 162 return keyOnly? new Tag(s) : Tag.ofString(s); 163 } 164 }; 165 } else if (obj instanceof String) { 166 final Tag tag = keyOnly ? new Tag((String) obj) : Tag.ofString((String) obj); 167 return new PrimitiveToTag() { 168 @Override 169 public Tag apply(OsmPrimitive ignore) { 170 return tag; 171 } 172 }; 173 } else { 174 return null; 175 } 176 } 177 } 178 179 static final String POSSIBLE_THROWS = possibleThrows(); 180 181 static final String possibleThrows() { 182 StringBuffer sb = new StringBuffer(); 183 for (Severity s : Severity.values()) { 184 if (sb.length() > 0) { 185 sb.append('/'); 186 } 187 sb.append("throw") 188 .append(s.name().charAt(0)) 189 .append(s.name().substring(1).toLowerCase()); 190 } 191 return sb.toString(); 192 } 193 194 static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) { 195 final TagCheck check = new TagCheck(rule); 196 boolean containsSetClassExpression = false; 197 for (Instruction i : rule.declaration.instructions) { 198 if (i instanceof Instruction.AssignmentInstruction) { 199 final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i; 200 if (ai.isSetInstruction) { 201 containsSetClassExpression = true; 202 continue; 203 } 204 final String val = ai.val instanceof Expression 205 ? (String) ((Expression) ai.val).evaluate(new Environment()) 206 : ai.val instanceof String 207 ? (String) ai.val 208 : null; 209 if (ai.key.startsWith("throw")) { 210 try { 211 final Severity severity = Severity.valueOf(ai.key.substring("throw".length()).toUpperCase()); 212 check.errors.put(ai, severity); 213 } catch (IllegalArgumentException e) { 214 Main.warn("Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS); 215 } 216 } else if ("fixAdd".equals(ai.key)) { 217 final PrimitiveToTag toTag = PrimitiveToTag.ofMapCSSObject(ai.val, false); 218 if (toTag != null) { 219 check.change.add(toTag); 220 } else { 221 Main.warn("Invalid value for "+ai.key+": "+ai.val); 222 } 223 } else if ("fixRemove".equals(ai.key)) { 224 CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")), 225 "Unexpected '='. Please only specify the key to remove!"); 226 final PrimitiveToTag toTag = PrimitiveToTag.ofMapCSSObject(ai.val, true); 227 if (toTag != null) { 228 check.change.add(toTag); 229 } else { 230 Main.warn("Invalid value for "+ai.key+": "+ai.val); 231 } 232 } else if ("fixChangeKey".equals(ai.key) && val != null) { 233 CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!"); 234 final String[] x = val.split("=>", 2); 235 check.keyChange.put(Tag.removeWhiteSpaces(x[0]), Tag.removeWhiteSpaces(x[1])); 236 } else if ("suggestAlternative".equals(ai.key) && val != null) { 237 check.alternatives.add(val); 238 } else if ("assertMatch".equals(ai.key) && val != null) { 239 check.assertions.put(val, true); 240 } else if ("assertNoMatch".equals(ai.key) && val != null) { 241 check.assertions.put(val, false); 242 } else { 243 throw new RuntimeException("Cannot add instruction " + ai.key + ": " + ai.val + "!"); 244 } 245 } 246 } 247 if (check.errors.isEmpty() && !containsSetClassExpression) { 248 throw new RuntimeException("No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors); 249 } else if (check.errors.size() > 1) { 250 throw new RuntimeException("More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for " + rule.selectors); 251 } 252 return check; 253 } 254 255 static List<TagCheck> readMapCSS(Reader css) throws ParseException { 256 CheckParameterUtil.ensureParameterNotNull(css, "css"); 257 return readMapCSS(new MapCSSParser(css)); 258 } 259 260 static List<TagCheck> readMapCSS(MapCSSParser css) throws ParseException { 261 CheckParameterUtil.ensureParameterNotNull(css, "css"); 262 final MapCSSStyleSource source = new MapCSSStyleSource(""); 263 css.sheet(source); 264 assert source.getErrors().isEmpty(); 265 // Ignore "meta" rule(s) from external rules of JOSM wiki 266 removeMetaRules(source); 267 // group rules with common declaration block 268 Map<Declaration, List<Selector>> g = new LinkedHashMap<>(); 269 for (MapCSSRule rule : source.rules) { 270 if (!g.containsKey(rule.declaration)) { 271 List<Selector> sels = new ArrayList<>(); 272 sels.add(rule.selector); 273 g.put(rule.declaration, sels); 274 } else { 275 g.get(rule.declaration).add(rule.selector); 276 } 277 } 278 List<TagCheck> result = new ArrayList<>(); 279 for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) { 280 result.add(TagCheck.ofMapCSSRule( 281 new GroupedMapCSSRule(map.getValue(), map.getKey()))); 282 } 283 return result; 284 } 285 286 private static void removeMetaRules(MapCSSStyleSource source) { 287 for (Iterator<MapCSSRule> it = source.rules.iterator(); it.hasNext(); ) { 288 MapCSSRule x = it.next(); 289 if (x.selector instanceof GeneralSelector) { 290 GeneralSelector gs = (GeneralSelector) x.selector; 291 if ("meta".equals(gs.base) && gs.getConditions().isEmpty()) { 292 it.remove(); 293 } 294 } 295 } 296 } 297 298 @Override 299 public boolean evaluate(OsmPrimitive primitive) { 300 // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker. 301 return whichSelectorMatchesPrimitive(primitive) != null; 302 } 303 304 Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) { 305 return whichSelectorMatchesEnvironment(new Environment().withPrimitive(primitive)); 306 } 307 308 Selector whichSelectorMatchesEnvironment(Environment env) { 309 for (Selector i : rule.selectors) { 310 env.clearSelectorMatchingInformation(); 311 if (i.matches(env)) { 312 return i; 313 } 314 } 315 return null; 316 } 317 318 /** 319 * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the 320 * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}. 321 */ 322 static String determineArgument(Selector.GeneralSelector matchingSelector, int index, String type) { 323 try { 324 final Condition c = matchingSelector.getConditions().get(index); 325 final Tag tag = c instanceof Condition.KeyCondition 326 ? ((Condition.KeyCondition) c).asTag() 327 : c instanceof Condition.SimpleKeyValueCondition 328 ? ((Condition.SimpleKeyValueCondition) c).asTag() 329 : c instanceof Condition.KeyValueCondition 330 ? ((Condition.KeyValueCondition) c).asTag() 331 : null; 332 if (tag == null) { 333 return null; 334 } else if ("key".equals(type)) { 335 return tag.getKey(); 336 } else if ("value".equals(type)) { 337 return tag.getValue(); 338 } else if ("tag".equals(type)) { 339 return tag.toString(); 340 } 341 } catch (IndexOutOfBoundsException ignore) { 342 Main.debug(ignore.getMessage()); 343 } 344 return null; 345 } 346 347 /** 348 * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding 349 * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}. 350 */ 351 static String insertArguments(Selector matchingSelector, String s) { 352 if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) { 353 return insertArguments(((Selector.ChildOrParentSelector)matchingSelector).right, s); 354 } else if (s == null || !(matchingSelector instanceof GeneralSelector)) { 355 return s; 356 } 357 final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s); 358 final StringBuffer sb = new StringBuffer(); 359 while (m.find()) { 360 final String argument = determineArgument((Selector.GeneralSelector) matchingSelector, Integer.parseInt(m.group(1)), m.group(2)); 361 try { 362 // Perform replacement with null-safe + regex-safe handling 363 m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", "")); 364 } catch (IndexOutOfBoundsException | IllegalArgumentException e) { 365 Main.error(tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage())); 366 } 367 } 368 m.appendTail(sb); 369 return sb.toString(); 370 } 371 372 /** 373 * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive} 374 * if the error is fixable, or {@code null} otherwise. 375 * 376 * @param p the primitive to construct the fix for 377 * @return the fix or {@code null} 378 */ 379 Command fixPrimitive(OsmPrimitive p) { 380 if (change.isEmpty() && keyChange.isEmpty()) { 381 return null; 382 } 383 final Selector matchingSelector = whichSelectorMatchesPrimitive(p); 384 Collection<Command> cmds = new LinkedList<>(); 385 for (PrimitiveToTag toTag : change) { 386 final Tag tag = toTag.apply(p); 387 final String key = insertArguments(matchingSelector, tag.getKey()); 388 final String value = insertArguments(matchingSelector, tag.getValue()); 389 cmds.add(new ChangePropertyCommand(p, key, value)); 390 } 391 for (Map.Entry<String, String> i : keyChange.entrySet()) { 392 final String oldKey = insertArguments(matchingSelector, i.getKey()); 393 final String newKey = insertArguments(matchingSelector, i.getValue()); 394 cmds.add(new ChangePropertyKeyCommand(p, oldKey, newKey)); 395 } 396 return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds); 397 } 398 399 /** 400 * Constructs a (localized) message for this deprecation check. 401 * 402 * @return a message 403 */ 404 String getMessage(OsmPrimitive p) { 405 if (errors.isEmpty()) { 406 // Return something to avoid NPEs 407 return rule.declaration.toString(); 408 } else { 409 final Object val = errors.keySet().iterator().next().val; 410 return String.valueOf( 411 val instanceof Expression 412 ? ((Expression) val).evaluate(new Environment().withPrimitive(p)) 413 : val 414 ); 415 } 416 } 417 418 /** 419 * Constructs a (localized) description for this deprecation check. 420 * 421 * @return a description (possibly with alternative suggestions) 422 * @see #getDescriptionForMatchingSelector 423 */ 424 String getDescription(OsmPrimitive p) { 425 if (alternatives.isEmpty()) { 426 return getMessage(p); 427 } else { 428 /* I18N: {0} is the test error message and {1} is an alternative */ 429 return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives)); 430 } 431 } 432 433 /** 434 * Constructs a (localized) description for this deprecation check 435 * where any placeholders are replaced by values of the matched selector. 436 * 437 * @return a description (possibly with alternative suggestions) 438 */ 439 String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) { 440 return insertArguments(matchingSelector, getDescription(p)); 441 } 442 443 Severity getSeverity() { 444 return errors.isEmpty() ? null : errors.values().iterator().next(); 445 } 446 447 @Override 448 public String toString() { 449 return getDescription(null); 450 } 451 452 /** 453 * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error. 454 * 455 * @param p the primitive to construct the error for 456 * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error. 457 */ 458 TestError getErrorForPrimitive(OsmPrimitive p) { 459 final Environment env = new Environment().withPrimitive(p); 460 return getErrorForPrimitive(p, whichSelectorMatchesEnvironment(env), env); 461 } 462 463 TestError getErrorForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env) { 464 if (matchingSelector != null && !errors.isEmpty()) { 465 final Command fix = fixPrimitive(p); 466 final String description = getDescriptionForMatchingSelector(p, matchingSelector); 467 final List<OsmPrimitive> primitives; 468 if (env.child != null) { 469 primitives = Arrays.asList(p, env.child); 470 } else { 471 primitives = Collections.singletonList(p); 472 } 473 if (fix != null) { 474 return new FixableTestError(null, getSeverity(), description, null, matchingSelector.toString(), 3000, primitives, fix); 475 } else { 476 return new TestError(null, getSeverity(), description, null, matchingSelector.toString(), 3000, primitives); 477 } 478 } else { 479 return null; 480 } 481 } 482 } 483 484 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker { 485 public final GroupedMapCSSRule rule; 486 487 MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) { 488 this.rule = rule; 489 } 490 491 @Override 492 public boolean equals(Object obj) { 493 return super.equals(obj) 494 || (obj instanceof TagCheck && rule.equals(((TagCheck) obj).rule)) 495 || (obj instanceof GroupedMapCSSRule && rule.equals(obj)); 496 } 497 498 @Override 499 public int hashCode() { 500 final int prime = 31; 501 int result = super.hashCode(); 502 result = prime * result + ((rule == null) ? 0 : rule.hashCode()); 503 return result; 504 } 505 } 506 507 /** 508 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}. 509 * @param p The OSM primitive 510 * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned 511 * @return all errors for the given primitive, with or without those of "info" severity 512 */ 513 public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) { 514 final ArrayList<TestError> r = new ArrayList<>(); 515 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null); 516 for (Set<TagCheck> schecks : checks.values()) { 517 for (TagCheck check : schecks) { 518 if (Severity.OTHER.equals(check.getSeverity()) && !includeOtherSeverity) { 519 continue; 520 } 521 final Selector selector = check.whichSelectorMatchesEnvironment(env); 522 if (selector != null) { 523 check.rule.declaration.execute(env); 524 final TestError error = check.getErrorForPrimitive(p, selector, env); 525 if (error != null) { 526 error.setTester(new MapCSSTagCheckerAndRule(check.rule)); 527 r.add(error); 528 } 529 } 530 } 531 } 532 return r; 533 } 534 535 /** 536 * Visiting call for primitives. 537 * 538 * @param p The primitive to inspect. 539 */ 540 @Override 541 public void check(OsmPrimitive p) { 542 errors.addAll(getErrorsForPrimitive(p, ValidatorPreference.PREF_OTHER.get())); 543 } 544 545 /** 546 * Adds a new MapCSS config file from the given URL. 547 * @param url The unique URL of the MapCSS config file 548 * @throws ParseException if the config file does not match MapCSS syntax 549 * @throws IOException if any I/O error occurs 550 * @since 7275 551 */ 552 public synchronized void addMapCSS(String url) throws ParseException, IOException { 553 CheckParameterUtil.ensureParameterNotNull(url, "url"); 554 CachedFile cache = new CachedFile(url); 555 try (InputStream s = cache.getInputStream()) { 556 List<TagCheck> tagchecks = TagCheck.readMapCSS(new BufferedReader(UTFInputStreamReader.create(s))); 557 checks.remove(url); 558 checks.putAll(url, tagchecks); 559 } 560 } 561 562 @Override 563 public synchronized void initialize() throws Exception { 564 checks.clear(); 565 for (SourceEntry source : new ValidatorTagCheckerRulesPreference.RulePrefHelper().get()) { 566 if (!source.active) { 567 continue; 568 } 569 String i = source.url; 570 try { 571 if (i.startsWith("resource:")) { 572 Main.debug(tr("Adding {0} to tag checker", i)); 573 } else { 574 Main.info(tr("Adding {0} to tag checker", i)); 575 } 576 addMapCSS(i); 577 if (Main.pref.getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) { 578 try { 579 Main.fileWatcher.registerValidatorRule(source); 580 } catch (IOException e) { 581 Main.error(e); 582 } 583 } 584 } catch (IOException ex) { 585 Main.warn(tr("Failed to add {0} to tag checker", i)); 586 Main.warn(ex, false); 587 } catch (Exception ex) { 588 Main.warn(tr("Failed to add {0} to tag checker", i)); 589 Main.warn(ex); 590 } 591 } 592 } 593 594 @Override 595 public synchronized int hashCode() { 596 final int prime = 31; 597 int result = super.hashCode(); 598 result = prime * result + ((checks == null) ? 0 : checks.hashCode()); 599 return result; 600 } 601 602 @Override 603 public synchronized boolean equals(Object obj) { 604 if (this == obj) 605 return true; 606 if (!super.equals(obj)) 607 return false; 608 if (!(obj instanceof MapCSSTagChecker)) 609 return false; 610 MapCSSTagChecker other = (MapCSSTagChecker) obj; 611 if (checks == null) { 612 if (other.checks != null) 613 return false; 614 } else if (!checks.equals(other.checks)) 615 return false; 616 return true; 617 } 618}