001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.marktr; 005import static org.openstreetmap.josm.tools.I18n.tr; 006 007import java.util.ArrayList; 008import java.util.Arrays; 009import java.util.Collection; 010import java.util.Collections; 011import java.util.HashMap; 012import java.util.HashSet; 013import java.util.List; 014import java.util.Locale; 015import java.util.Map; 016import java.util.Map.Entry; 017import java.util.Objects; 018import java.util.Set; 019import java.util.stream.Collectors; 020import java.util.stream.Stream; 021 022import org.openstreetmap.josm.command.Command; 023import org.openstreetmap.josm.command.DeleteCommand; 024import org.openstreetmap.josm.data.coor.EastNorth; 025import org.openstreetmap.josm.data.coor.LatLon; 026import org.openstreetmap.josm.data.osm.Node; 027import org.openstreetmap.josm.data.osm.OsmPrimitive; 028import org.openstreetmap.josm.data.osm.Relation; 029import org.openstreetmap.josm.data.osm.RelationMember; 030import org.openstreetmap.josm.data.osm.TagMap; 031import org.openstreetmap.josm.data.osm.Way; 032import org.openstreetmap.josm.data.preferences.DoubleProperty; 033import org.openstreetmap.josm.data.validation.Severity; 034import org.openstreetmap.josm.data.validation.Test; 035import org.openstreetmap.josm.data.validation.TestError; 036import org.openstreetmap.josm.tools.Geometry; 037import org.openstreetmap.josm.tools.Logging; 038import org.openstreetmap.josm.tools.Pair; 039import org.openstreetmap.josm.tools.SubclassFilteredCollection; 040import org.openstreetmap.josm.tools.Territories; 041import org.openstreetmap.josm.tools.Utils; 042 043/** 044 * Performs validation tests on addresses (addr:housenumber) and associatedStreet relations. 045 * @since 5644 046 */ 047public class Addresses extends Test { 048 049 protected static final int HOUSE_NUMBER_WITHOUT_STREET = 2601; 050 protected static final int DUPLICATE_HOUSE_NUMBER = 2602; 051 protected static final int MULTIPLE_STREET_NAMES = 2603; 052 protected static final int MULTIPLE_STREET_RELATIONS = 2604; 053 protected static final int HOUSE_NUMBER_TOO_FAR = 2605; 054 protected static final int OBSOLETE_RELATION = 2606; 055 056 protected static final DoubleProperty MAX_DUPLICATE_DISTANCE = new DoubleProperty("validator.addresses.max_duplicate_distance", 200.0); 057 protected static final DoubleProperty MAX_STREET_DISTANCE = new DoubleProperty("validator.addresses.max_street_distance", 200.0); 058 059 // CHECKSTYLE.OFF: SingleSpaceSeparator 060 protected static final String ADDR_HOUSE_NUMBER = "addr:housenumber"; 061 protected static final String ADDR_INTERPOLATION = "addr:interpolation"; 062 protected static final String ADDR_NEIGHBOURHOOD = "addr:neighbourhood"; 063 protected static final String ADDR_PLACE = "addr:place"; 064 protected static final String ADDR_STREET = "addr:street"; 065 protected static final String ADDR_SUBURB = "addr:suburb"; 066 protected static final String ADDR_CITY = "addr:city"; 067 protected static final String ADDR_UNIT = "addr:unit"; 068 protected static final String ADDR_FLATS = "addr:flats"; 069 protected static final String ADDR_HOUSE_NAME = "addr:housename"; 070 protected static final String ADDR_POSTCODE = "addr:postcode"; 071 protected static final String ASSOCIATED_STREET = "associatedStreet"; 072 // CHECKSTYLE.ON: SingleSpaceSeparator 073 074 private Map<String, Collection<OsmPrimitive>> knownAddresses; 075 private Set<String> ignoredAddresses; 076 077 /** 078 * Constructor 079 */ 080 public Addresses() { 081 super(tr("Addresses"), tr("Checks for errors in addresses and associatedStreet relations.")); 082 } 083 084 protected List<Relation> getAndCheckAssociatedStreets(OsmPrimitive p) { 085 final List<Relation> list = p.referrers(Relation.class) 086 .filter(r -> r.hasTag("type", ASSOCIATED_STREET)) 087 .collect(Collectors.toList()); 088 if (list.size() > 1) { 089 Severity level; 090 // warning level only if several relations have different names, see #10945 091 final String name = list.get(0).get("name"); 092 if (name == null || SubclassFilteredCollection.filter(list, r -> r.hasTag("name", name)).size() < list.size()) { 093 level = Severity.WARNING; 094 } else { 095 level = Severity.OTHER; 096 } 097 List<OsmPrimitive> errorList = new ArrayList<>(list); 098 errorList.add(0, p); 099 errors.add(TestError.builder(this, level, MULTIPLE_STREET_RELATIONS) 100 .message(tr("Multiple associatedStreet relations")) 101 .primitives(errorList) 102 .build()); 103 } 104 return list; 105 } 106 107 /** 108 * Checks for house numbers for which the street is unknown. 109 * @param p primitive to test 110 * @return error found, or null 111 */ 112 protected TestError checkHouseNumbersWithoutStreet(OsmPrimitive p) { 113 // Find house number without proper location 114 // (neither addr:street, associatedStreet, addr:place, addr:neighbourhood or addr:interpolation) 115 if (p.hasKey(ADDR_HOUSE_NUMBER) && !p.hasKey(ADDR_STREET, ADDR_PLACE, ADDR_NEIGHBOURHOOD) 116 && getAndCheckAssociatedStreets(p).isEmpty() 117 && p.referrers(Way.class).noneMatch(w -> w.hasKey(ADDR_INTERPOLATION) && w.hasKey(ADDR_STREET))) { 118 // no street found 119 TestError e = TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_WITHOUT_STREET) 120 .message(tr("House number without street")) 121 .primitives(p) 122 .build(); 123 errors.add(e); 124 return e; 125 } 126 return null; 127 } 128 129 static boolean isPOI(OsmPrimitive p) { 130 return p.hasKey("shop", "amenity", "tourism", "leisure", "emergency", "craft", "office", "name"); 131 } 132 133 static boolean hasAddress(OsmPrimitive p) { 134 return p.hasKey(ADDR_HOUSE_NUMBER) && p.hasKey(ADDR_STREET, ADDR_PLACE); 135 } 136 137 /** 138 * adds the OsmPrimitive to the address map if it complies to the restrictions 139 * @param p OsmPrimitive that has an address 140 */ 141 private void collectAddress(OsmPrimitive p) { 142 if (!isPOI(p)) { 143 for (String simplifiedAddress : getSimplifiedAddresses(p)) { 144 if (!ignoredAddresses.contains(simplifiedAddress)) { 145 knownAddresses.computeIfAbsent(simplifiedAddress, x -> new ArrayList<>()).add(p); 146 } 147 } 148 } 149 } 150 151 protected void initAddressMap(OsmPrimitive primitive) { 152 knownAddresses = new HashMap<>(); 153 ignoredAddresses = new HashSet<>(); 154 for (OsmPrimitive p : primitive.getDataSet().allNonDeletedPrimitives()) { 155 if (p instanceof Node && p.hasKey(ADDR_UNIT, ADDR_FLATS)) { 156 for (OsmPrimitive r : p.getReferrers()) { 157 if (hasAddress(r)) { 158 // ignore addresses of buildings that are connected to addr:unit nodes 159 // it's quite reasonable that there are more buildings with this address 160 for (String simplifiedAddress : getSimplifiedAddresses(r)) { 161 if (!ignoredAddresses.contains(simplifiedAddress)) { 162 ignoredAddresses.add(simplifiedAddress); 163 } else if (knownAddresses.containsKey(simplifiedAddress)) { 164 knownAddresses.remove(simplifiedAddress); 165 } 166 } 167 } 168 } 169 } 170 if (hasAddress(p)) { 171 collectAddress(p); 172 } 173 } 174 } 175 176 @Override 177 public void endTest() { 178 knownAddresses = null; 179 ignoredAddresses = null; 180 super.endTest(); 181 } 182 183 protected List<TestError> checkForDuplicate(OsmPrimitive p) { 184 if (knownAddresses == null) { 185 initAddressMap(p); 186 } 187 if (!isPOI(p) && hasAddress(p)) { 188 List<TestError> result = new ArrayList<>(); 189 for (String simplifiedAddress : getSimplifiedAddresses(p)) { 190 if (!ignoredAddresses.contains(simplifiedAddress) && knownAddresses.containsKey(simplifiedAddress)) { 191 double maxDistance = MAX_DUPLICATE_DISTANCE.get(); 192 for (OsmPrimitive p2 : knownAddresses.get(simplifiedAddress)) { 193 if (p == p2) { 194 continue; 195 } 196 Severity severityLevel; 197 String city1 = p.get(ADDR_CITY); 198 String city2 = p2.get(ADDR_CITY); 199 double distance = getDistance(p, p2); 200 if (city1 != null && city2 != null) { 201 if (city1.equals(city2)) { 202 if ((!p.hasKey(ADDR_POSTCODE) || !p2.hasKey(ADDR_POSTCODE) 203 || p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) 204 && (!p.hasKey(ADDR_SUBURB) || !p2.hasKey(ADDR_SUBURB) 205 || p.get(ADDR_SUBURB).equals(p2.get(ADDR_SUBURB)))) { 206 severityLevel = Severity.WARNING; 207 } else { 208 // address including city identical but postcode or suburb differs 209 // most likely perfectly fine 210 severityLevel = Severity.OTHER; 211 } 212 } else { 213 // address differs only by city - notify if very close, otherwise ignore 214 if (distance < maxDistance) { 215 severityLevel = Severity.OTHER; 216 } else { 217 continue; 218 } 219 } 220 } else { 221 // at least one address has no city specified 222 if (p.hasKey(ADDR_POSTCODE) && p2.hasKey(ADDR_POSTCODE) 223 && p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) { 224 // address including postcode identical 225 severityLevel = Severity.WARNING; 226 } else { 227 // city/postcode unclear - warn if very close, otherwise only notify 228 // TODO: get city from surrounding boundaries? 229 if (distance < maxDistance) { 230 severityLevel = Severity.WARNING; 231 } else { 232 severityLevel = Severity.OTHER; 233 } 234 } 235 } 236 result.add(TestError.builder(this, severityLevel, DUPLICATE_HOUSE_NUMBER) 237 .message(tr("Duplicate house numbers"), marktr("''{0}'' ({1}m)"), simplifiedAddress, (int) distance) 238 .primitives(Arrays.asList(p, p2)).build()); 239 } 240 knownAddresses.get(simplifiedAddress).remove(p); // otherwise we would get every warning two times 241 } 242 } 243 errors.addAll(result); 244 return result; 245 } 246 return Collections.emptyList(); 247 } 248 249 static List<String> getSimplifiedAddresses(OsmPrimitive p) { 250 String simplifiedStreetName = p.hasKey(ADDR_STREET) ? p.get(ADDR_STREET) : p.get(ADDR_PLACE); 251 // ignore whitespaces and dashes in street name, so that "Mozart-Gasse", "Mozart Gasse" and "Mozartgasse" are all seen as equal 252 return expandHouseNumber(p.get(ADDR_HOUSE_NUMBER)).stream().map(addrHouseNumber -> Utils.strip(Stream.of( 253 simplifiedStreetName.replaceAll("[ -]", ""), 254 addrHouseNumber, 255 p.get(ADDR_HOUSE_NAME), 256 p.get(ADDR_UNIT), 257 p.get(ADDR_FLATS)) 258 .filter(Objects::nonNull) 259 .collect(Collectors.joining(" "))) 260 .toUpperCase(Locale.ENGLISH)).collect(Collectors.toList()); 261 } 262 263 /** 264 * Split addr:housenumber on , and ; (common separators) 265 * 266 * @param houseNumber The housenumber to be split 267 * @return A list of addr:housenumber equivalents 268 */ 269 static List<String> expandHouseNumber(String houseNumber) { 270 return Arrays.asList(houseNumber.split(",|;")); 271 } 272 273 @Override 274 public void visit(Node n) { 275 checkHouseNumbersWithoutStreet(n); 276 checkForDuplicate(n); 277 } 278 279 @Override 280 public void visit(Way w) { 281 checkHouseNumbersWithoutStreet(w); 282 checkForDuplicate(w); 283 } 284 285 @Override 286 public void visit(Relation r) { 287 checkHouseNumbersWithoutStreet(r); 288 checkForDuplicate(r); 289 if (r.hasTag("type", ASSOCIATED_STREET)) { 290 checkIfObsolete(r); 291 // Used to count occurrences of each house number in order to find duplicates 292 Map<String, List<OsmPrimitive>> map = new HashMap<>(); 293 // Used to detect different street names 294 String relationName = r.get("name"); 295 Set<OsmPrimitive> wrongStreetNames = new HashSet<>(); 296 // Used to check distance 297 Set<OsmPrimitive> houses = new HashSet<>(); 298 Set<Way> street = new HashSet<>(); 299 for (RelationMember m : r.getMembers()) { 300 String role = m.getRole(); 301 OsmPrimitive p = m.getMember(); 302 if ("house".equals(role)) { 303 houses.add(p); 304 String number = p.get(ADDR_HOUSE_NUMBER); 305 if (number != null) { 306 number = number.trim().toUpperCase(Locale.ENGLISH); 307 List<OsmPrimitive> list = map.get(number); 308 if (list == null) { 309 list = new ArrayList<>(); 310 map.put(number, list); 311 } 312 list.add(p); 313 } 314 if (relationName != null && p.hasKey(ADDR_STREET) && !relationName.equals(p.get(ADDR_STREET))) { 315 if (wrongStreetNames.isEmpty()) { 316 wrongStreetNames.add(r); 317 } 318 wrongStreetNames.add(p); 319 } 320 } else if ("street".equals(role)) { 321 if (p instanceof Way) { 322 street.add((Way) p); 323 } 324 if (relationName != null && p.hasTagDifferent("name", relationName)) { 325 if (wrongStreetNames.isEmpty()) { 326 wrongStreetNames.add(r); 327 } 328 wrongStreetNames.add(p); 329 } 330 } 331 } 332 // Report duplicate house numbers 333 for (Entry<String, List<OsmPrimitive>> entry : map.entrySet()) { 334 List<OsmPrimitive> list = entry.getValue(); 335 if (list.size() > 1) { 336 errors.add(TestError.builder(this, Severity.WARNING, DUPLICATE_HOUSE_NUMBER) 337 .message(tr("Duplicate house numbers"), marktr("House number ''{0}'' duplicated"), entry.getKey()) 338 .primitives(list) 339 .build()); 340 } 341 } 342 // Report wrong street names 343 if (!wrongStreetNames.isEmpty()) { 344 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_STREET_NAMES) 345 .message(tr("Multiple street names in relation")) 346 .primitives(wrongStreetNames) 347 .build()); 348 } 349 // Report addresses too far away 350 if (!street.isEmpty()) { 351 for (OsmPrimitive house : houses) { 352 if (house.isUsable()) { 353 checkDistance(house, street); 354 } 355 } 356 } 357 } 358 } 359 360 /** 361 * returns rough distance between two OsmPrimitives 362 * @param a primitive a 363 * @param b primitive b 364 * @return distance of center of bounding boxes in meters 365 */ 366 static double getDistance(OsmPrimitive a, OsmPrimitive b) { 367 LatLon centerA = a.getBBox().getCenter(); 368 LatLon centerB = b.getBBox().getCenter(); 369 return (centerA.greatCircleDistance(centerB)); 370 } 371 372 protected void checkDistance(OsmPrimitive house, Collection<Way> street) { 373 EastNorth centroid; 374 if (house instanceof Node) { 375 centroid = ((Node) house).getEastNorth(); 376 } else if (house instanceof Way) { 377 List<Node> nodes = ((Way) house).getNodes(); 378 if (house.hasKey(ADDR_INTERPOLATION)) { 379 for (Node n : nodes) { 380 if (n.hasKey(ADDR_HOUSE_NUMBER)) { 381 checkDistance(n, street); 382 } 383 } 384 return; 385 } 386 centroid = Geometry.getCentroid(nodes); 387 } else { 388 return; // TODO handle multipolygon houses ? 389 } 390 if (centroid == null) return; // fix #8305 391 double maxDistance = MAX_STREET_DISTANCE.get(); 392 boolean hasIncompleteWays = false; 393 for (Way streetPart : street) { 394 for (Pair<Node, Node> chunk : streetPart.getNodePairs(false)) { 395 EastNorth p1 = chunk.a.getEastNorth(); 396 EastNorth p2 = chunk.b.getEastNorth(); 397 if (p1 != null && p2 != null) { 398 EastNorth closest = Geometry.closestPointToSegment(p1, p2, centroid); 399 if (closest.distance(centroid) <= maxDistance) { 400 return; 401 } 402 } else { 403 Logging.warn("Addresses test skipped chunck "+chunk+" for street part "+streetPart+" because p1 or p2 is null"); 404 } 405 } 406 if (!hasIncompleteWays && streetPart.isIncomplete()) { 407 hasIncompleteWays = true; 408 } 409 } 410 // No street segment found near this house, report error on if the relation does not contain incomplete street ways (fix #8314) 411 if (hasIncompleteWays) return; 412 List<OsmPrimitive> errorList = new ArrayList<>(street); 413 errorList.add(0, house); 414 errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_TOO_FAR) 415 .message(tr("House number too far from street")) 416 .primitives(errorList) 417 .build()); 418 } 419 420 /** 421 * Check if an associatedStreet Relation is obsolete. This test marks only those relations which 422 * are complete and don't contain any information which isn't also tagged on the members. 423 * The strategy is to avoid any false positive. 424 * @param r the relation 425 */ 426 private void checkIfObsolete(Relation r) { 427 if (r.isIncomplete()) 428 return; 429 /** array of country codes for which the test should be performed. For now, only Germany */ 430 String[] countryCodes = {"DE"}; 431 TagMap neededtagsForHouse = new TagMap(); 432 for (Entry<String, String> tag : r.getKeys().entrySet()) { 433 String key = tag.getKey(); 434 if (key.startsWith("name:")) { 435 return; // maybe check if all members have corresponding tags? 436 } else if (key.startsWith("addr:")) { 437 neededtagsForHouse.put(key, tag.getValue()); 438 } else { 439 switch (key) { 440 case "name": 441 case "type": 442 case "source": 443 break; 444 default: 445 // unexpected tag in relation 446 return; 447 } 448 } 449 } 450 451 for (RelationMember m : r.getMembers()) { 452 if (m.getMember().isIncomplete() || !isInWarnCountry(m, countryCodes)) 453 return; 454 455 String role = m.getRole(); 456 if ("".equals(role)) { 457 if (m.isWay() && m.getMember().hasKey("highway")) { 458 role = "street"; 459 } else if (m.getMember().hasTag("building")) 460 role = "house"; 461 } 462 switch (role) { 463 case "house": 464 case "addr:houselink": 465 case "address": 466 if (!m.getMember().hasTag(ADDR_STREET) || !m.getMember().hasTag(ADDR_HOUSE_NUMBER)) 467 return; 468 for (Entry<String, String> tag : neededtagsForHouse.entrySet()) { 469 if (!m.getMember().hasTag(tag.getKey(), tag.getValue())) 470 return; 471 } 472 break; 473 case "street": 474 if (!m.getMember().hasTag("name") && r.hasTag("name")) 475 return; 476 break; 477 default: 478 // unknown role: don't create auto-fix 479 return; 480 } 481 } 482 errors.add(TestError.builder(this, Severity.WARNING, OBSOLETE_RELATION) 483 .message(tr("Relation is obsolete")) 484 .primitives(r) 485 .build()); 486 } 487 488 private static boolean isInWarnCountry(RelationMember m, String[] countryCodes) { 489 if (countryCodes.length == 0) 490 return true; 491 LatLon center = null; 492 493 if (m.isNode()) { 494 center = m.getNode().getCoor(); 495 } else if (m.isWay()) { 496 center = m.getWay().getBBox().getCenter(); 497 } else if (m.isRelation() && m.getRelation().isMultipolygon()) { 498 center = m.getRelation().getBBox().getCenter(); 499 } 500 if (center == null) 501 return false; 502 for (String country : countryCodes) { 503 if (Territories.isIso3166Code(country, center)) 504 return true; 505 } 506 return false; 507 } 508 509 /** 510 * remove obsolete relation. 511 */ 512 @Override 513 public Command fixError(TestError testError) { 514 return new DeleteCommand(testError.getPrimitives()); 515 } 516 517 @Override 518 public boolean isFixable(TestError testError) { 519 if (!(testError.getTester() instanceof Addresses)) 520 return false; 521 return testError.getCode() == OBSOLETE_RELATION; 522 } 523 524}