001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.tr; 005 006import java.io.BufferedReader; 007import java.io.IOException; 008import java.io.InputStream; 009import java.io.Reader; 010import java.io.StringReader; 011import java.lang.reflect.Method; 012import java.text.MessageFormat; 013import java.util.ArrayList; 014import java.util.Collection; 015import java.util.Collections; 016import java.util.HashMap; 017import java.util.HashSet; 018import java.util.Iterator; 019import java.util.LinkedHashMap; 020import java.util.LinkedHashSet; 021import java.util.LinkedList; 022import java.util.List; 023import java.util.Locale; 024import java.util.Map; 025import java.util.Objects; 026import java.util.Optional; 027import java.util.Set; 028import java.util.function.Predicate; 029import java.util.regex.Matcher; 030import java.util.regex.Pattern; 031 032import org.openstreetmap.josm.command.ChangePropertyCommand; 033import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 034import org.openstreetmap.josm.command.Command; 035import org.openstreetmap.josm.command.DeleteCommand; 036import org.openstreetmap.josm.command.SequenceCommand; 037import org.openstreetmap.josm.data.coor.LatLon; 038import org.openstreetmap.josm.data.osm.DataSet; 039import org.openstreetmap.josm.data.osm.IPrimitive; 040import org.openstreetmap.josm.data.osm.OsmPrimitive; 041import org.openstreetmap.josm.data.osm.OsmUtils; 042import org.openstreetmap.josm.data.osm.Relation; 043import org.openstreetmap.josm.data.osm.Tag; 044import org.openstreetmap.josm.data.osm.Way; 045import org.openstreetmap.josm.data.preferences.sources.SourceEntry; 046import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper; 047import org.openstreetmap.josm.data.validation.OsmValidator; 048import org.openstreetmap.josm.data.validation.Severity; 049import org.openstreetmap.josm.data.validation.Test; 050import org.openstreetmap.josm.data.validation.TestError; 051import org.openstreetmap.josm.gui.mappaint.Environment; 052import org.openstreetmap.josm.gui.mappaint.Keyword; 053import org.openstreetmap.josm.gui.mappaint.MultiCascade; 054import org.openstreetmap.josm.gui.mappaint.mapcss.Condition; 055import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ClassCondition; 056import org.openstreetmap.josm.gui.mappaint.mapcss.ConditionFactory.ExpressionCondition; 057import org.openstreetmap.josm.gui.mappaint.mapcss.Expression; 058import org.openstreetmap.josm.gui.mappaint.mapcss.ExpressionFactory.ParameterFunction; 059import org.openstreetmap.josm.gui.mappaint.mapcss.Functions; 060import org.openstreetmap.josm.gui.mappaint.mapcss.Instruction; 061import org.openstreetmap.josm.gui.mappaint.mapcss.LiteralExpression; 062import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule; 063import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSRule.Declaration; 064import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource; 065import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource.MapCSSRuleIndex; 066import org.openstreetmap.josm.gui.mappaint.mapcss.Selector; 067import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.AbstractSelector; 068import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector; 069import org.openstreetmap.josm.gui.mappaint.mapcss.Selector.OptimizedGeneralSelector; 070import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.MapCSSParser; 071import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException; 072import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.TokenMgrError; 073import org.openstreetmap.josm.gui.progress.ProgressMonitor; 074import org.openstreetmap.josm.io.CachedFile; 075import org.openstreetmap.josm.io.FileWatcher; 076import org.openstreetmap.josm.io.IllegalDataException; 077import org.openstreetmap.josm.io.UTFInputStreamReader; 078import org.openstreetmap.josm.spi.preferences.Config; 079import org.openstreetmap.josm.tools.CheckParameterUtil; 080import org.openstreetmap.josm.tools.DefaultGeoProperty; 081import org.openstreetmap.josm.tools.GeoProperty; 082import org.openstreetmap.josm.tools.GeoPropertyIndex; 083import org.openstreetmap.josm.tools.I18n; 084import org.openstreetmap.josm.tools.Logging; 085import org.openstreetmap.josm.tools.MultiMap; 086import org.openstreetmap.josm.tools.Territories; 087import org.openstreetmap.josm.tools.Utils; 088 089/** 090 * MapCSS-based tag checker/fixer. 091 * @since 6506 092 */ 093public class MapCSSTagChecker extends Test.TagTest { 094 MapCSSTagCheckerIndex indexData; 095 final Set<OsmPrimitive> tested = new HashSet<>(); 096 097 098 /** 099 * A grouped MapCSSRule with multiple selectors for a single declaration. 100 * @see MapCSSRule 101 */ 102 public static class GroupedMapCSSRule { 103 /** MapCSS selectors **/ 104 public final List<Selector> selectors; 105 /** MapCSS declaration **/ 106 public final Declaration declaration; 107 108 /** 109 * Constructs a new {@code GroupedMapCSSRule}. 110 * @param selectors MapCSS selectors 111 * @param declaration MapCSS declaration 112 */ 113 public GroupedMapCSSRule(List<Selector> selectors, Declaration declaration) { 114 this.selectors = selectors; 115 this.declaration = declaration; 116 } 117 118 @Override 119 public int hashCode() { 120 return Objects.hash(selectors, declaration); 121 } 122 123 @Override 124 public boolean equals(Object obj) { 125 if (this == obj) return true; 126 if (obj == null || getClass() != obj.getClass()) return false; 127 GroupedMapCSSRule that = (GroupedMapCSSRule) obj; 128 return Objects.equals(selectors, that.selectors) && 129 Objects.equals(declaration, that.declaration); 130 } 131 132 @Override 133 public String toString() { 134 return "GroupedMapCSSRule [selectors=" + selectors + ", declaration=" + declaration + ']'; 135 } 136 } 137 138 /** 139 * The preference key for tag checker source entries. 140 * @since 6670 141 */ 142 public static final String ENTRIES_PREF_KEY = "validator." + MapCSSTagChecker.class.getName() + ".entries"; 143 144 /** 145 * Constructs a new {@code MapCSSTagChecker}. 146 */ 147 public MapCSSTagChecker() { 148 super(tr("Tag checker (MapCSS based)"), tr("This test checks for errors in tag keys and values.")); 149 } 150 151 /** 152 * Represents a fix to a validation test. The fixing {@link Command} can be obtained by {@link #createCommand(OsmPrimitive, Selector)}. 153 */ 154 @FunctionalInterface 155 interface FixCommand { 156 /** 157 * Creates the fixing {@link Command} for the given primitive. The {@code matchingSelector} is used to evaluate placeholders 158 * (cf. {@link MapCSSTagChecker.TagCheck#insertArguments(Selector, String, OsmPrimitive)}). 159 * @param p OSM primitive 160 * @param matchingSelector matching selector 161 * @return fix command 162 */ 163 Command createCommand(OsmPrimitive p, Selector matchingSelector); 164 165 /** 166 * Checks that object is either an {@link Expression} or a {@link String}. 167 * @param obj object to check 168 * @throws IllegalArgumentException if object is not an {@code Expression} or a {@code String} 169 */ 170 static void checkObject(final Object obj) { 171 CheckParameterUtil.ensureThat(obj instanceof Expression || obj instanceof String, 172 () -> "instance of Exception or String expected, but got " + obj); 173 } 174 175 /** 176 * Evaluates given object as {@link Expression} or {@link String} on the matched {@link OsmPrimitive} and {@code matchingSelector}. 177 * @param obj object to evaluate ({@link Expression} or {@link String}) 178 * @param p OSM primitive 179 * @param matchingSelector matching selector 180 * @return result string 181 */ 182 static String evaluateObject(final Object obj, final OsmPrimitive p, final Selector matchingSelector) { 183 final String s; 184 if (obj instanceof Expression) { 185 s = (String) ((Expression) obj).evaluate(new Environment(p)); 186 } else if (obj instanceof String) { 187 s = (String) obj; 188 } else { 189 return null; 190 } 191 return TagCheck.insertArguments(matchingSelector, s, p); 192 } 193 194 /** 195 * Creates a fixing command which executes a {@link ChangePropertyCommand} on the specified tag. 196 * @param obj object to evaluate ({@link Expression} or {@link String}) 197 * @return created fix command 198 */ 199 static FixCommand fixAdd(final Object obj) { 200 checkObject(obj); 201 return new FixCommand() { 202 @Override 203 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 204 final Tag tag = Tag.ofString(evaluateObject(obj, p, matchingSelector)); 205 return new ChangePropertyCommand(p, tag.getKey(), tag.getValue()); 206 } 207 208 @Override 209 public String toString() { 210 return "fixAdd: " + obj; 211 } 212 }; 213 } 214 215 /** 216 * Creates a fixing command which executes a {@link ChangePropertyCommand} to delete the specified key. 217 * @param obj object to evaluate ({@link Expression} or {@link String}) 218 * @return created fix command 219 */ 220 static FixCommand fixRemove(final Object obj) { 221 checkObject(obj); 222 return new FixCommand() { 223 @Override 224 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 225 final String key = evaluateObject(obj, p, matchingSelector); 226 return new ChangePropertyCommand(p, key, ""); 227 } 228 229 @Override 230 public String toString() { 231 return "fixRemove: " + obj; 232 } 233 }; 234 } 235 236 /** 237 * Creates a fixing command which executes a {@link ChangePropertyKeyCommand} on the specified keys. 238 * @param oldKey old key 239 * @param newKey new key 240 * @return created fix command 241 */ 242 static FixCommand fixChangeKey(final String oldKey, final String newKey) { 243 return new FixCommand() { 244 @Override 245 public Command createCommand(OsmPrimitive p, Selector matchingSelector) { 246 return new ChangePropertyKeyCommand(p, 247 TagCheck.insertArguments(matchingSelector, oldKey, p), 248 TagCheck.insertArguments(matchingSelector, newKey, p)); 249 } 250 251 @Override 252 public String toString() { 253 return "fixChangeKey: " + oldKey + " => " + newKey; 254 } 255 }; 256 } 257 } 258 259 final MultiMap<String, TagCheck> checks = new MultiMap<>(); 260 261 /** 262 * Result of {@link TagCheck#readMapCSS} 263 * @since 8936 264 */ 265 public static class ParseResult { 266 /** Checks successfully parsed */ 267 public final List<TagCheck> parseChecks; 268 /** Errors that occurred during parsing */ 269 public final Collection<Throwable> parseErrors; 270 271 /** 272 * Constructs a new {@code ParseResult}. 273 * @param parseChecks Checks successfully parsed 274 * @param parseErrors Errors that occurred during parsing 275 */ 276 public ParseResult(List<TagCheck> parseChecks, Collection<Throwable> parseErrors) { 277 this.parseChecks = parseChecks; 278 this.parseErrors = parseErrors; 279 } 280 } 281 282 /** 283 * Tag check. 284 */ 285 public static class TagCheck implements Predicate<OsmPrimitive> { 286 /** The selector of this {@code TagCheck} */ 287 protected final GroupedMapCSSRule rule; 288 /** Commands to apply in order to fix a matching primitive */ 289 protected final List<FixCommand> fixCommands = new ArrayList<>(); 290 /** Tags (or arbitraty strings) of alternatives to be presented to the user */ 291 protected final List<String> alternatives = new ArrayList<>(); 292 /** An {@link org.openstreetmap.josm.gui.mappaint.mapcss.Instruction.AssignmentInstruction}-{@link Severity} pair. 293 * Is evaluated on the matching primitive to give the error message. Map is checked to contain exactly one element. */ 294 protected final Map<Instruction.AssignmentInstruction, Severity> errors = new HashMap<>(); 295 /** Unit tests */ 296 protected final Map<String, Boolean> assertions = new HashMap<>(); 297 /** MapCSS Classes to set on matching primitives */ 298 protected final Set<String> setClassExpressions = new HashSet<>(); 299 /** Denotes whether the object should be deleted for fixing it */ 300 protected boolean deletion; 301 /** A string used to group similar tests */ 302 protected String group; 303 304 TagCheck(GroupedMapCSSRule rule) { 305 this.rule = rule; 306 } 307 308 private static final String POSSIBLE_THROWS = possibleThrows(); 309 310 static final String possibleThrows() { 311 StringBuilder sb = new StringBuilder(); 312 for (Severity s : Severity.values()) { 313 if (sb.length() > 0) { 314 sb.append('/'); 315 } 316 sb.append("throw") 317 .append(s.name().charAt(0)) 318 .append(s.name().substring(1).toLowerCase(Locale.ENGLISH)); 319 } 320 return sb.toString(); 321 } 322 323 static TagCheck ofMapCSSRule(final GroupedMapCSSRule rule) throws IllegalDataException { 324 final TagCheck check = new TagCheck(rule); 325 for (Instruction i : rule.declaration.instructions) { 326 if (i instanceof Instruction.AssignmentInstruction) { 327 final Instruction.AssignmentInstruction ai = (Instruction.AssignmentInstruction) i; 328 if (ai.isSetInstruction) { 329 check.setClassExpressions.add(ai.key); 330 continue; 331 } 332 try { 333 final String val = ai.val instanceof Expression 334 ? Optional.ofNullable(((Expression) ai.val).evaluate(new Environment())).map(Object::toString).orElse(null) 335 : ai.val instanceof String 336 ? (String) ai.val 337 : ai.val instanceof Keyword 338 ? ((Keyword) ai.val).val 339 : null; 340 if (ai.key.startsWith("throw")) { 341 try { 342 check.errors.put(ai, Severity.valueOf(ai.key.substring("throw".length()).toUpperCase(Locale.ENGLISH))); 343 } catch (IllegalArgumentException e) { 344 Logging.log(Logging.LEVEL_WARN, 345 "Unsupported "+ai.key+" instruction. Allowed instructions are "+POSSIBLE_THROWS+'.', e); 346 } 347 } else if ("fixAdd".equals(ai.key)) { 348 check.fixCommands.add(FixCommand.fixAdd(ai.val)); 349 } else if ("fixRemove".equals(ai.key)) { 350 CheckParameterUtil.ensureThat(!(ai.val instanceof String) || !(val != null && val.contains("=")), 351 "Unexpected '='. Please only specify the key to remove in: " + ai); 352 check.fixCommands.add(FixCommand.fixRemove(ai.val)); 353 } else if (val != null && "fixChangeKey".equals(ai.key)) { 354 CheckParameterUtil.ensureThat(val.contains("=>"), "Separate old from new key by '=>'!"); 355 final String[] x = val.split("=>", 2); 356 check.fixCommands.add(FixCommand.fixChangeKey(Utils.removeWhiteSpaces(x[0]), Utils.removeWhiteSpaces(x[1]))); 357 } else if (val != null && "fixDeleteObject".equals(ai.key)) { 358 CheckParameterUtil.ensureThat("this".equals(val), "fixDeleteObject must be followed by 'this'"); 359 check.deletion = true; 360 } else if (val != null && "suggestAlternative".equals(ai.key)) { 361 check.alternatives.add(val); 362 } else if (val != null && "assertMatch".equals(ai.key)) { 363 check.assertions.put(val, Boolean.TRUE); 364 } else if (val != null && "assertNoMatch".equals(ai.key)) { 365 check.assertions.put(val, Boolean.FALSE); 366 } else if (val != null && "group".equals(ai.key)) { 367 check.group = val; 368 } else if (ai.key.startsWith("-")) { 369 Logging.debug("Ignoring extension instruction: " + ai.key + ": " + ai.val); 370 } else { 371 throw new IllegalDataException("Cannot add instruction " + ai.key + ": " + ai.val + '!'); 372 } 373 } catch (IllegalArgumentException e) { 374 throw new IllegalDataException(e); 375 } 376 } 377 } 378 if (check.errors.isEmpty() && check.setClassExpressions.isEmpty()) { 379 throw new IllegalDataException( 380 "No "+POSSIBLE_THROWS+" given! You should specify a validation error message for " + rule.selectors); 381 } else if (check.errors.size() > 1) { 382 throw new IllegalDataException( 383 "More than one "+POSSIBLE_THROWS+" given! You should specify a single validation error message for " 384 + rule.selectors); 385 } 386 return check; 387 } 388 389 static ParseResult readMapCSS(Reader css) throws ParseException { 390 CheckParameterUtil.ensureParameterNotNull(css, "css"); 391 392 final MapCSSStyleSource source = new MapCSSStyleSource(""); 393 final MapCSSParser preprocessor = new MapCSSParser(css, MapCSSParser.LexicalState.PREPROCESSOR); 394 final StringReader mapcss = new StringReader(preprocessor.pp_root(source)); 395 final MapCSSParser parser = new MapCSSParser(mapcss, MapCSSParser.LexicalState.DEFAULT); 396 parser.sheet(source); 397 // Ignore "meta" rule(s) from external rules of JOSM wiki 398 source.removeMetaRules(); 399 // group rules with common declaration block 400 Map<Declaration, List<Selector>> g = new LinkedHashMap<>(); 401 for (MapCSSRule rule : source.rules) { 402 if (!g.containsKey(rule.declaration)) { 403 List<Selector> sels = new ArrayList<>(); 404 sels.add(rule.selector); 405 g.put(rule.declaration, sels); 406 } else { 407 g.get(rule.declaration).add(rule.selector); 408 } 409 } 410 List<TagCheck> parseChecks = new ArrayList<>(); 411 for (Map.Entry<Declaration, List<Selector>> map : g.entrySet()) { 412 try { 413 parseChecks.add(TagCheck.ofMapCSSRule( 414 new GroupedMapCSSRule(map.getValue(), map.getKey()))); 415 } catch (IllegalDataException e) { 416 Logging.error("Cannot add MapCss rule: "+e.getMessage()); 417 source.logError(e); 418 } 419 } 420 return new ParseResult(parseChecks, source.getErrors()); 421 } 422 423 @Override 424 public boolean test(OsmPrimitive primitive) { 425 // Tests whether the primitive contains a deprecated tag which is represented by this MapCSSTagChecker. 426 return whichSelectorMatchesPrimitive(primitive) != null; 427 } 428 429 Selector whichSelectorMatchesPrimitive(OsmPrimitive primitive) { 430 return whichSelectorMatchesEnvironment(new Environment(primitive)); 431 } 432 433 Selector whichSelectorMatchesEnvironment(Environment env) { 434 for (Selector i : rule.selectors) { 435 env.clearSelectorMatchingInformation(); 436 if (i.matches(env)) { 437 return i; 438 } 439 } 440 return null; 441 } 442 443 /** 444 * Determines the {@code index}-th key/value/tag (depending on {@code type}) of the 445 * {@link org.openstreetmap.josm.gui.mappaint.mapcss.Selector.GeneralSelector}. 446 * @param matchingSelector matching selector 447 * @param index index 448 * @param type selector type ("key", "value" or "tag") 449 * @param p OSM primitive 450 * @return argument value, can be {@code null} 451 */ 452 static String determineArgument(OptimizedGeneralSelector matchingSelector, int index, String type, OsmPrimitive p) { 453 try { 454 final Condition c = matchingSelector.getConditions().get(index); 455 final Tag tag = c instanceof Condition.ToTagConvertable 456 ? ((Condition.ToTagConvertable) c).asTag(p) 457 : null; 458 if (tag == null) { 459 return null; 460 } else if ("key".equals(type)) { 461 return tag.getKey(); 462 } else if ("value".equals(type)) { 463 return tag.getValue(); 464 } else if ("tag".equals(type)) { 465 return tag.toString(); 466 } 467 } catch (IndexOutOfBoundsException ignore) { 468 Logging.debug(ignore); 469 } 470 return null; 471 } 472 473 /** 474 * Replaces occurrences of <code>{i.key}</code>, <code>{i.value}</code>, <code>{i.tag}</code> in {@code s} by the corresponding 475 * key/value/tag of the {@code index}-th {@link Condition} of {@code matchingSelector}. 476 * @param matchingSelector matching selector 477 * @param s any string 478 * @param p OSM primitive 479 * @return string with arguments inserted 480 */ 481 static String insertArguments(Selector matchingSelector, String s, OsmPrimitive p) { 482 if (s != null && matchingSelector instanceof Selector.ChildOrParentSelector) { 483 return insertArguments(((Selector.ChildOrParentSelector) matchingSelector).right, s, p); 484 } else if (s == null || !(matchingSelector instanceof Selector.OptimizedGeneralSelector)) { 485 return s; 486 } 487 final Matcher m = Pattern.compile("\\{(\\d+)\\.(key|value|tag)\\}").matcher(s); 488 final StringBuffer sb = new StringBuffer(); 489 while (m.find()) { 490 final String argument = determineArgument((Selector.OptimizedGeneralSelector) matchingSelector, 491 Integer.parseInt(m.group(1)), m.group(2), p); 492 try { 493 // Perform replacement with null-safe + regex-safe handling 494 m.appendReplacement(sb, String.valueOf(argument).replace("^(", "").replace(")$", "")); 495 } catch (IndexOutOfBoundsException | IllegalArgumentException e) { 496 Logging.log(Logging.LEVEL_ERROR, tr("Unable to replace argument {0} in {1}: {2}", argument, sb, e.getMessage()), e); 497 } 498 } 499 m.appendTail(sb); 500 return sb.toString(); 501 } 502 503 /** 504 * Constructs a fix in terms of a {@link org.openstreetmap.josm.command.Command} for the {@link OsmPrimitive} 505 * if the error is fixable, or {@code null} otherwise. 506 * 507 * @param p the primitive to construct the fix for 508 * @return the fix or {@code null} 509 */ 510 Command fixPrimitive(OsmPrimitive p) { 511 if (fixCommands.isEmpty() && !deletion) { 512 return null; 513 } 514 try { 515 final Selector matchingSelector = whichSelectorMatchesPrimitive(p); 516 Collection<Command> cmds = new LinkedList<>(); 517 for (FixCommand fixCommand : fixCommands) { 518 cmds.add(fixCommand.createCommand(p, matchingSelector)); 519 } 520 if (deletion && !p.isDeleted()) { 521 cmds.add(new DeleteCommand(p)); 522 } 523 return new SequenceCommand(tr("Fix of {0}", getDescriptionForMatchingSelector(p, matchingSelector)), cmds); 524 } catch (IllegalArgumentException e) { 525 Logging.error(e); 526 return null; 527 } 528 } 529 530 /** 531 * Constructs a (localized) message for this deprecation check. 532 * @param p OSM primitive 533 * 534 * @return a message 535 */ 536 String getMessage(OsmPrimitive p) { 537 if (errors.isEmpty()) { 538 // Return something to avoid NPEs 539 return rule.declaration.toString(); 540 } else { 541 final Object val = errors.keySet().iterator().next().val; 542 return String.valueOf( 543 val instanceof Expression 544 ? ((Expression) val).evaluate(new Environment(p)) 545 : val 546 ); 547 } 548 } 549 550 /** 551 * Constructs a (localized) description for this deprecation check. 552 * @param p OSM primitive 553 * 554 * @return a description (possibly with alternative suggestions) 555 * @see #getDescriptionForMatchingSelector 556 */ 557 String getDescription(OsmPrimitive p) { 558 if (alternatives.isEmpty()) { 559 return getMessage(p); 560 } else { 561 /* I18N: {0} is the test error message and {1} is an alternative */ 562 return tr("{0}, use {1} instead", getMessage(p), Utils.join(tr(" or "), alternatives)); 563 } 564 } 565 566 /** 567 * Constructs a (localized) description for this deprecation check 568 * where any placeholders are replaced by values of the matched selector. 569 * 570 * @param matchingSelector matching selector 571 * @param p OSM primitive 572 * @return a description (possibly with alternative suggestions) 573 */ 574 String getDescriptionForMatchingSelector(OsmPrimitive p, Selector matchingSelector) { 575 return insertArguments(matchingSelector, getDescription(p), p); 576 } 577 578 Severity getSeverity() { 579 return errors.isEmpty() ? null : errors.values().iterator().next(); 580 } 581 582 @Override 583 public String toString() { 584 return getDescription(null); 585 } 586 587 /** 588 * Constructs a {@link TestError} for the given primitive, or returns null if the primitive does not give rise to an error. 589 * 590 * @param p the primitive to construct the error for 591 * @return an instance of {@link TestError}, or returns null if the primitive does not give rise to an error. 592 */ 593 List<TestError> getErrorsForPrimitive(OsmPrimitive p) { 594 final Environment env = new Environment(p); 595 return getErrorsForPrimitive(p, whichSelectorMatchesEnvironment(env), env, null); 596 } 597 598 private List<TestError> getErrorsForPrimitive(OsmPrimitive p, Selector matchingSelector, Environment env, Test tester) { 599 List<TestError> res = new ArrayList<>(); 600 if (matchingSelector != null && !errors.isEmpty()) { 601 final Command fix = fixPrimitive(p); 602 final String description = getDescriptionForMatchingSelector(p, matchingSelector); 603 final String description1 = group == null ? description : group; 604 final String description2 = group == null ? null : description; 605 TestError.Builder errorBuilder = TestError.builder(tester, getSeverity(), 3000) 606 .messageWithManuallyTranslatedDescription(description1, description2, matchingSelector.toString()); 607 if (fix != null) { 608 errorBuilder = errorBuilder.fix(() -> fix); 609 } 610 if (env.child instanceof OsmPrimitive) { 611 res.add(errorBuilder.primitives(p, (OsmPrimitive) env.child).build()); 612 } else if (env.children != null) { 613 for (IPrimitive c : env.children) { 614 if (c instanceof OsmPrimitive) { 615 errorBuilder = TestError.builder(tester, getSeverity(), 3000) 616 .messageWithManuallyTranslatedDescription(description1, description2, 617 matchingSelector.toString()); 618 if (fix != null) { 619 errorBuilder = errorBuilder.fix(() -> fix); 620 } 621 res.add(errorBuilder.primitives(p, (OsmPrimitive) c).build()); 622 } 623 } 624 } else { 625 res.add(errorBuilder.primitives(p).build()); 626 } 627 } 628 return res; 629 } 630 631 /** 632 * Returns the set of tagchecks on which this check depends on. 633 * @param schecks the collection of tagcheks to search in 634 * @return the set of tagchecks on which this check depends on 635 * @since 7881 636 */ 637 public Set<TagCheck> getTagCheckDependencies(Collection<TagCheck> schecks) { 638 Set<TagCheck> result = new HashSet<>(); 639 Set<String> classes = getClassesIds(); 640 if (schecks != null && !classes.isEmpty()) { 641 for (TagCheck tc : schecks) { 642 if (this.equals(tc)) { 643 continue; 644 } 645 for (String id : tc.setClassExpressions) { 646 if (classes.contains(id)) { 647 result.add(tc); 648 break; 649 } 650 } 651 } 652 } 653 return result; 654 } 655 656 /** 657 * Returns the list of ids of all MapCSS classes referenced in the rule selectors. 658 * @return the list of ids of all MapCSS classes referenced in the rule selectors 659 * @since 7881 660 */ 661 public Set<String> getClassesIds() { 662 Set<String> result = new HashSet<>(); 663 for (Selector s : rule.selectors) { 664 if (s instanceof AbstractSelector) { 665 for (Condition c : ((AbstractSelector) s).getConditions()) { 666 if (c instanceof ClassCondition) { 667 result.add(((ClassCondition) c).id); 668 } 669 } 670 } 671 } 672 return result; 673 } 674 } 675 676 static class MapCSSTagCheckerAndRule extends MapCSSTagChecker { 677 public final GroupedMapCSSRule rule; 678 679 MapCSSTagCheckerAndRule(GroupedMapCSSRule rule) { 680 this.rule = rule; 681 } 682 683 @Override 684 public String toString() { 685 return "MapCSSTagCheckerAndRule [rule=" + rule + ']'; 686 } 687 } 688 689 /** 690 * Obtains all {@link TestError}s for the {@link OsmPrimitive} {@code p}. 691 * @param p The OSM primitive 692 * @param includeOtherSeverity if {@code true}, errors of severity {@link Severity#OTHER} (info) will also be returned 693 * @return all errors for the given primitive, with or without those of "info" severity 694 */ 695 public synchronized Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity) { 696 final List<TestError> res = new ArrayList<>(); 697 if (indexData == null) { 698 indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverity, MapCSSTagCheckerIndex.ALL_TESTS); 699 } 700 701 MapCSSRuleIndex matchingRuleIndex = indexData.get(p); 702 703 Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null); 704 // the declaration indices are sorted, so it suffices to save the last used index 705 Declaration lastDeclUsed = null; 706 707 Iterator<MapCSSRule> candidates = matchingRuleIndex.getRuleCandidates(p); 708 while (candidates.hasNext()) { 709 MapCSSRule r = candidates.next(); 710 env.clearSelectorMatchingInformation(); 711 if (r.selector.matches(env)) { // as side effect env.parent will be set (if s is a child selector) 712 TagCheck check = indexData.getCheck(r); 713 if (check != null) { 714 if (r.declaration == lastDeclUsed) 715 continue; // don't apply one declaration more than once 716 lastDeclUsed = r.declaration; 717 718 r.declaration.execute(env); 719 if (!check.errors.isEmpty()) { 720 for (TestError e: check.getErrorsForPrimitive(p, r.selector, env, new MapCSSTagCheckerAndRule(check.rule))) { 721 addIfNotSimilar(e, res); 722 } 723 } 724 } 725 } 726 } 727 return res; 728 } 729 730 /** 731 * See #12627 732 * Add error to given list if list doesn't already contain a similar error. 733 * Similar means same code and description and same combination of primitives and same combination of highlighted objects, 734 * but maybe with different orders. 735 * @param toAdd the error to add 736 * @param errors the list of errors 737 */ 738 private static void addIfNotSimilar(TestError toAdd, List<TestError> errors) { 739 boolean isDup = false; 740 if (toAdd.getPrimitives().size() >= 2) { 741 for (TestError e : errors) { 742 if (e.getCode() == toAdd.getCode() && e.getMessage().equals(toAdd.getMessage()) 743 && e.getPrimitives().size() == toAdd.getPrimitives().size() 744 && e.getPrimitives().containsAll(toAdd.getPrimitives()) 745 && e.getHighlighted().size() == toAdd.getHighlighted().size() 746 && e.getHighlighted().containsAll(toAdd.getHighlighted())) { 747 isDup = true; 748 break; 749 } 750 } 751 } 752 if (!isDup) 753 errors.add(toAdd); 754 } 755 756 private static Collection<TestError> getErrorsForPrimitive(OsmPrimitive p, boolean includeOtherSeverity, 757 Collection<Set<TagCheck>> checksCol) { 758 final List<TestError> r = new ArrayList<>(); 759 final Environment env = new Environment(p, new MultiCascade(), Environment.DEFAULT_LAYER, null); 760 for (Set<TagCheck> schecks : checksCol) { 761 for (TagCheck check : schecks) { 762 boolean ignoreError = Severity.OTHER == check.getSeverity() && !includeOtherSeverity; 763 // Do not run "information" level checks if not wanted, unless they also set a MapCSS class 764 if (ignoreError && check.setClassExpressions.isEmpty()) { 765 continue; 766 } 767 final Selector selector = check.whichSelectorMatchesEnvironment(env); 768 if (selector != null) { 769 check.rule.declaration.execute(env); 770 if (!ignoreError && !check.errors.isEmpty()) { 771 r.addAll(check.getErrorsForPrimitive(p, selector, env, new MapCSSTagCheckerAndRule(check.rule))); 772 } 773 } 774 } 775 } 776 return r; 777 } 778 779 /** 780 * Visiting call for primitives. 781 * 782 * @param p The primitive to inspect. 783 */ 784 @Override 785 public void check(OsmPrimitive p) { 786 for (TestError e : getErrorsForPrimitive(p, ValidatorPrefHelper.PREF_OTHER.get())) { 787 addIfNotSimilar(e, errors); 788 } 789 if (partialSelection && p.isTagged()) { 790 tested.add(p); 791 } 792 } 793 794 /** 795 * Adds a new MapCSS config file from the given URL. 796 * @param url The unique URL of the MapCSS config file 797 * @return List of tag checks and parsing errors, or null 798 * @throws ParseException if the config file does not match MapCSS syntax 799 * @throws IOException if any I/O error occurs 800 * @since 7275 801 */ 802 public synchronized ParseResult addMapCSS(String url) throws ParseException, IOException { 803 CheckParameterUtil.ensureParameterNotNull(url, "url"); 804 ParseResult result; 805 try (CachedFile cache = new CachedFile(url); 806 InputStream zip = cache.findZipEntryInputStream("validator.mapcss", ""); 807 InputStream s = zip != null ? zip : cache.getInputStream(); 808 Reader reader = new BufferedReader(UTFInputStreamReader.create(s))) { 809 if (zip != null) 810 I18n.addTexts(cache.getFile()); 811 result = TagCheck.readMapCSS(reader); 812 checks.remove(url); 813 checks.putAll(url, result.parseChecks); 814 indexData = null; 815 // Check assertions, useful for development of local files 816 if (Config.getPref().getBoolean("validator.check_assert_local_rules", false) && Utils.isLocalUrl(url)) { 817 for (String msg : checkAsserts(result.parseChecks)) { 818 Logging.warn(msg); 819 } 820 } 821 } 822 return result; 823 } 824 825 @Override 826 public synchronized void initialize() throws Exception { 827 checks.clear(); 828 indexData = null; 829 for (SourceEntry source : new ValidatorPrefHelper().get()) { 830 if (!source.active) { 831 continue; 832 } 833 String i = source.url; 834 try { 835 if (!i.startsWith("resource:")) { 836 Logging.info(tr("Adding {0} to tag checker", i)); 837 } else if (Logging.isDebugEnabled()) { 838 Logging.debug(tr("Adding {0} to tag checker", i)); 839 } 840 addMapCSS(i); 841 if (Config.getPref().getBoolean("validator.auto_reload_local_rules", true) && source.isLocal()) { 842 FileWatcher.getDefaultInstance().registerSource(source); 843 } 844 } catch (IOException | IllegalStateException | IllegalArgumentException ex) { 845 Logging.warn(tr("Failed to add {0} to tag checker", i)); 846 Logging.log(Logging.LEVEL_WARN, ex); 847 } catch (ParseException | TokenMgrError ex) { 848 Logging.warn(tr("Failed to add {0} to tag checker", i)); 849 Logging.warn(ex); 850 } 851 } 852 } 853 854 private static Method getFunctionMethod(String method) { 855 try { 856 return Functions.class.getDeclaredMethod(method, Environment.class, String.class); 857 } catch (NoSuchMethodException | SecurityException e) { 858 Logging.error(e); 859 return null; 860 } 861 } 862 863 private static Optional<String> getFirstInsideCountry(TagCheck check, Method insideMethod) { 864 return check.rule.selectors.stream() 865 .filter(s -> s instanceof GeneralSelector) 866 .flatMap(s -> ((GeneralSelector) s).getConditions().stream()) 867 .filter(c -> c instanceof ExpressionCondition) 868 .map(c -> ((ExpressionCondition) c).getExpression()) 869 .filter(c -> c instanceof ParameterFunction) 870 .map(c -> (ParameterFunction) c) 871 .filter(c -> c.getMethod().equals(insideMethod)) 872 .flatMap(c -> c.getArgs().stream()) 873 .filter(e -> e instanceof LiteralExpression) 874 .map(e -> ((LiteralExpression) e).getLiteral()) 875 .filter(l -> l instanceof String) 876 .map(l -> ((String) l).split(",")[0]) 877 .findFirst(); 878 } 879 880 private static LatLon getLocation(TagCheck check, Method insideMethod) { 881 Optional<String> inside = getFirstInsideCountry(check, insideMethod); 882 if (inside.isPresent()) { 883 GeoPropertyIndex<Boolean> index = Territories.getGeoPropertyIndex(inside.get()); 884 if (index != null) { 885 GeoProperty<Boolean> prop = index.getGeoProperty(); 886 if (prop instanceof DefaultGeoProperty) { 887 return ((DefaultGeoProperty) prop).getRandomLatLon(); 888 } 889 } 890 } 891 return LatLon.ZERO; 892 } 893 894 /** 895 * Checks that rule assertions are met for the given set of TagChecks. 896 * @param schecks The TagChecks for which assertions have to be checked 897 * @return A set of error messages, empty if all assertions are met 898 * @since 7356 899 */ 900 public Set<String> checkAsserts(final Collection<TagCheck> schecks) { 901 Set<String> assertionErrors = new LinkedHashSet<>(); 902 final Method insideMethod = getFunctionMethod("inside"); 903 final DataSet ds = new DataSet(); 904 for (final TagCheck check : schecks) { 905 Logging.debug("Check: {0}", check); 906 for (final Map.Entry<String, Boolean> i : check.assertions.entrySet()) { 907 Logging.debug("- Assertion: {0}", i); 908 final OsmPrimitive p = OsmUtils.createPrimitive(i.getKey(), getLocation(check, insideMethod), true); 909 // Build minimal ordered list of checks to run to test the assertion 910 List<Set<TagCheck>> checksToRun = new ArrayList<>(); 911 Set<TagCheck> checkDependencies = check.getTagCheckDependencies(schecks); 912 if (!checkDependencies.isEmpty()) { 913 checksToRun.add(checkDependencies); 914 } 915 checksToRun.add(Collections.singleton(check)); 916 // Add primitive to dataset to avoid DataIntegrityProblemException when evaluating selectors 917 addPrimitive(ds, p); 918 final Collection<TestError> pErrors = getErrorsForPrimitive(p, true, checksToRun); 919 Logging.debug("- Errors: {0}", pErrors); 920 final boolean isError = pErrors.stream().anyMatch(e -> e.getTester() instanceof MapCSSTagCheckerAndRule 921 && ((MapCSSTagCheckerAndRule) e.getTester()).rule.equals(check.rule)); 922 if (isError != i.getValue()) { 923 assertionErrors.add(MessageFormat.format("Expecting test ''{0}'' (i.e., {1}) to {2} {3} (i.e., {4})", 924 check.getMessage(p), check.rule.selectors, i.getValue() ? "match" : "not match", i.getKey(), p.getKeys())); 925 } 926 if (isError) { 927 // Check that autofix works as expected 928 Command fix = check.fixPrimitive(p); 929 if (fix != null && fix.executeCommand() && !getErrorsForPrimitive(p, true, checksToRun).isEmpty()) { 930 assertionErrors.add(MessageFormat.format("Autofix does not work for test ''{0}'' (i.e., {1})", 931 check.getMessage(p), check.rule.selectors)); 932 } 933 } 934 ds.removePrimitive(p); 935 } 936 } 937 return assertionErrors; 938 } 939 940 private static void addPrimitive(DataSet ds, OsmPrimitive p) { 941 if (p instanceof Way) { 942 ((Way) p).getNodes().forEach(n -> addPrimitive(ds, n)); 943 } else if (p instanceof Relation) { 944 ((Relation) p).getMembers().forEach(m -> addPrimitive(ds, m.getMember())); 945 } 946 ds.addPrimitive(p); 947 } 948 949 /** 950 * Reload tagchecker rule. 951 * @param rule tagchecker rule to reload 952 * @since 12825 953 */ 954 public static void reloadRule(SourceEntry rule) { 955 MapCSSTagChecker tagChecker = OsmValidator.getTest(MapCSSTagChecker.class); 956 if (tagChecker != null) { 957 try { 958 tagChecker.addMapCSS(rule.url); 959 } catch (IOException | ParseException | TokenMgrError e) { 960 Logging.warn(e); 961 } 962 } 963 } 964 965 @Override 966 public synchronized void startTest(ProgressMonitor progressMonitor) { 967 super.startTest(progressMonitor); 968 super.setShowElements(true); 969 if (indexData == null) { 970 indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverityChecks(), MapCSSTagCheckerIndex.ALL_TESTS); 971 } 972 tested.clear(); 973 } 974 975 @Override 976 public synchronized void endTest() { 977 if (partialSelection && !tested.isEmpty()) { 978 // #14287: see https://josm.openstreetmap.de/ticket/14287#comment:15 979 // execute tests for objects which might contain or cross previously tested elements 980 981 // rebuild index with a reduced set of rules (those that use ChildOrParentSelector) and thus may have left selectors 982 // matching the previously tested elements 983 indexData = new MapCSSTagCheckerIndex(checks, includeOtherSeverityChecks(), MapCSSTagCheckerIndex.ONLY_SELECTED_TESTS); 984 985 Set<OsmPrimitive> surrounding = new HashSet<>(); 986 for (OsmPrimitive p : tested) { 987 if (p.getDataSet() != null) { 988 surrounding.addAll(p.getDataSet().searchWays(p.getBBox())); 989 surrounding.addAll(p.getDataSet().searchRelations(p.getBBox())); 990 } 991 } 992 final boolean includeOtherSeverity = includeOtherSeverityChecks(); 993 for (OsmPrimitive p : surrounding) { 994 if (tested.contains(p)) 995 continue; 996 Collection<TestError> additionalErrors = getErrorsForPrimitive(p, includeOtherSeverity); 997 for (TestError e : additionalErrors) { 998 if (e.getPrimitives().stream().anyMatch(tested::contains)) 999 addIfNotSimilar(e, errors); 1000 } 1001 } 1002 tested.clear(); 1003 } 1004 super.endTest(); 1005 // no need to keep the index, it is quickly build and doubles the memory needs 1006 indexData = null; 1007 } 1008 1009 private boolean includeOtherSeverityChecks() { 1010 return isBeforeUpload ? ValidatorPrefHelper.PREF_OTHER_UPLOAD.get() : ValidatorPrefHelper.PREF_OTHER.get(); 1011 } 1012 1013}