001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.marktr; 005import static org.openstreetmap.josm.tools.I18n.tr; 006 007import java.util.ArrayList; 008import java.util.Arrays; 009import java.util.Collection; 010import java.util.Collections; 011import java.util.HashMap; 012import java.util.HashSet; 013import java.util.List; 014import java.util.Locale; 015import java.util.Map; 016import java.util.Map.Entry; 017import java.util.Objects; 018import java.util.Set; 019import java.util.stream.Collectors; 020import java.util.stream.Stream; 021 022import org.openstreetmap.josm.command.Command; 023import org.openstreetmap.josm.command.DeleteCommand; 024import org.openstreetmap.josm.data.coor.EastNorth; 025import org.openstreetmap.josm.data.coor.LatLon; 026import org.openstreetmap.josm.data.osm.Node; 027import org.openstreetmap.josm.data.osm.OsmPrimitive; 028import org.openstreetmap.josm.data.osm.Relation; 029import org.openstreetmap.josm.data.osm.RelationMember; 030import org.openstreetmap.josm.data.osm.TagMap; 031import org.openstreetmap.josm.data.osm.Way; 032import org.openstreetmap.josm.data.preferences.DoubleProperty; 033import org.openstreetmap.josm.data.validation.Severity; 034import org.openstreetmap.josm.data.validation.Test; 035import org.openstreetmap.josm.data.validation.TestError; 036import org.openstreetmap.josm.tools.Geometry; 037import org.openstreetmap.josm.tools.Logging; 038import org.openstreetmap.josm.tools.Pair; 039import org.openstreetmap.josm.tools.SubclassFilteredCollection; 040import org.openstreetmap.josm.tools.Territories; 041import org.openstreetmap.josm.tools.Utils; 042 043/** 044 * Performs validation tests on addresses (addr:housenumber) and associatedStreet relations. 045 * @since 5644 046 */ 047public class Addresses extends Test { 048 049 protected static final int HOUSE_NUMBER_WITHOUT_STREET = 2601; 050 protected static final int DUPLICATE_HOUSE_NUMBER = 2602; 051 protected static final int MULTIPLE_STREET_NAMES = 2603; 052 protected static final int MULTIPLE_STREET_RELATIONS = 2604; 053 protected static final int HOUSE_NUMBER_TOO_FAR = 2605; 054 protected static final int OBSOLETE_RELATION = 2606; 055 056 protected static final DoubleProperty MAX_DUPLICATE_DISTANCE = new DoubleProperty("validator.addresses.max_duplicate_distance", 200.0); 057 protected static final DoubleProperty MAX_STREET_DISTANCE = new DoubleProperty("validator.addresses.max_street_distance", 200.0); 058 059 // CHECKSTYLE.OFF: SingleSpaceSeparator 060 protected static final String ADDR_HOUSE_NUMBER = "addr:housenumber"; 061 protected static final String ADDR_INTERPOLATION = "addr:interpolation"; 062 protected static final String ADDR_NEIGHBOURHOOD = "addr:neighbourhood"; 063 protected static final String ADDR_PLACE = "addr:place"; 064 protected static final String ADDR_STREET = "addr:street"; 065 protected static final String ADDR_SUBURB = "addr:suburb"; 066 protected static final String ADDR_CITY = "addr:city"; 067 protected static final String ADDR_UNIT = "addr:unit"; 068 protected static final String ADDR_FLATS = "addr:flats"; 069 protected static final String ADDR_HOUSE_NAME = "addr:housename"; 070 protected static final String ADDR_POSTCODE = "addr:postcode"; 071 protected static final String ASSOCIATED_STREET = "associatedStreet"; 072 // CHECKSTYLE.ON: SingleSpaceSeparator 073 074 private Map<String, Collection<OsmPrimitive>> knownAddresses; 075 private Set<String> ignoredAddresses; 076 077 /** 078 * Constructor 079 */ 080 public Addresses() { 081 super(tr("Addresses"), tr("Checks for errors in addresses and associatedStreet relations.")); 082 } 083 084 protected List<Relation> getAndCheckAssociatedStreets(OsmPrimitive p) { 085 final List<Relation> list = p.referrers(Relation.class) 086 .filter(r -> r.hasTag("type", ASSOCIATED_STREET)) 087 .collect(Collectors.toList()); 088 if (list.size() > 1) { 089 Severity level; 090 // warning level only if several relations have different names, see #10945 091 final String name = list.get(0).get("name"); 092 if (name == null || SubclassFilteredCollection.filter(list, r -> r.hasTag("name", name)).size() < list.size()) { 093 level = Severity.WARNING; 094 } else { 095 level = Severity.OTHER; 096 } 097 List<OsmPrimitive> errorList = new ArrayList<>(list); 098 errorList.add(0, p); 099 errors.add(TestError.builder(this, level, MULTIPLE_STREET_RELATIONS) 100 .message(tr("Multiple associatedStreet relations")) 101 .primitives(errorList) 102 .build()); 103 } 104 return list; 105 } 106 107 /** 108 * Checks for house numbers for which the street is unknown. 109 * @param p primitive to test 110 * @return error found, or null 111 */ 112 protected TestError checkHouseNumbersWithoutStreet(OsmPrimitive p) { 113 // Find house number without proper location 114 // (neither addr:street, associatedStreet, addr:place, addr:neighbourhood or addr:interpolation) 115 if (p.hasKey(ADDR_HOUSE_NUMBER) && !p.hasKey(ADDR_STREET, ADDR_PLACE, ADDR_NEIGHBOURHOOD) 116 && getAndCheckAssociatedStreets(p).isEmpty() 117 && p.referrers(Way.class).noneMatch(w -> w.hasKey(ADDR_INTERPOLATION) && w.hasKey(ADDR_STREET))) { 118 // no street found 119 TestError e = TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_WITHOUT_STREET) 120 .message(tr("House number without street")) 121 .primitives(p) 122 .build(); 123 errors.add(e); 124 return e; 125 } 126 return null; 127 } 128 129 static boolean isPOI(OsmPrimitive p) { 130 return p.hasKey("shop", "amenity", "tourism", "leisure", "emergency", "craft", "office", "name"); 131 } 132 133 static boolean hasAddress(OsmPrimitive p) { 134 return p.hasKey(ADDR_HOUSE_NUMBER) && p.hasKey(ADDR_STREET, ADDR_PLACE); 135 } 136 137 /** 138 * adds the OsmPrimitive to the address map if it complies to the restrictions 139 * @param p OsmPrimitive that has an address 140 */ 141 private void collectAddress(OsmPrimitive p) { 142 if (!isPOI(p)) { 143 String simplifiedAddress = getSimplifiedAddress(p); 144 if (!ignoredAddresses.contains(simplifiedAddress)) { 145 knownAddresses.computeIfAbsent(simplifiedAddress, x -> new ArrayList<>()).add(p); 146 } 147 } 148 } 149 150 protected void initAddressMap(OsmPrimitive primitive) { 151 knownAddresses = new HashMap<>(); 152 ignoredAddresses = new HashSet<>(); 153 for (OsmPrimitive p : primitive.getDataSet().allNonDeletedPrimitives()) { 154 if (p instanceof Node && p.hasKey(ADDR_UNIT, ADDR_FLATS)) { 155 for (OsmPrimitive r : p.getReferrers()) { 156 if (hasAddress(r)) { 157 // ignore addresses of buildings that are connected to addr:unit nodes 158 // it's quite reasonable that there are more buildings with this address 159 String simplifiedAddress = getSimplifiedAddress(r); 160 if (!ignoredAddresses.contains(simplifiedAddress)) { 161 ignoredAddresses.add(simplifiedAddress); 162 } else if (knownAddresses.containsKey(simplifiedAddress)) { 163 knownAddresses.remove(simplifiedAddress); 164 } 165 } 166 } 167 } 168 if (hasAddress(p)) { 169 collectAddress(p); 170 } 171 } 172 } 173 174 @Override 175 public void endTest() { 176 knownAddresses = null; 177 ignoredAddresses = null; 178 super.endTest(); 179 } 180 181 protected List<TestError> checkForDuplicate(OsmPrimitive p) { 182 if (knownAddresses == null) { 183 initAddressMap(p); 184 } 185 if (!isPOI(p) && hasAddress(p)) { 186 List<TestError> result = new ArrayList<>(); 187 String simplifiedAddress = getSimplifiedAddress(p); 188 if (!ignoredAddresses.contains(simplifiedAddress) && knownAddresses.containsKey(simplifiedAddress)) { 189 double maxDistance = MAX_DUPLICATE_DISTANCE.get(); 190 for (OsmPrimitive p2 : knownAddresses.get(simplifiedAddress)) { 191 if (p == p2) { 192 continue; 193 } 194 Severity severityLevel; 195 String city1 = p.get(ADDR_CITY); 196 String city2 = p2.get(ADDR_CITY); 197 double distance = getDistance(p, p2); 198 if (city1 != null && city2 != null) { 199 if (city1.equals(city2)) { 200 if ((!p.hasKey(ADDR_POSTCODE) || !p2.hasKey(ADDR_POSTCODE) || p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) 201 && (!p.hasKey(ADDR_SUBURB) || !p2.hasKey(ADDR_SUBURB) || p.get(ADDR_SUBURB).equals(p2.get(ADDR_SUBURB)))) { 202 severityLevel = Severity.WARNING; 203 } else { 204 // address including city identical but postcode or suburb differs 205 // most likely perfectly fine 206 severityLevel = Severity.OTHER; 207 } 208 } else { 209 // address differs only by city - notify if very close, otherwise ignore 210 if (distance < maxDistance) { 211 severityLevel = Severity.OTHER; 212 } else { 213 continue; 214 } 215 } 216 } else { 217 // at least one address has no city specified 218 if (p.hasKey(ADDR_POSTCODE) && p2.hasKey(ADDR_POSTCODE) && p.get(ADDR_POSTCODE).equals(p2.get(ADDR_POSTCODE))) { 219 // address including postcode identical 220 severityLevel = Severity.WARNING; 221 } else { 222 // city/postcode unclear - warn if very close, otherwise only notify 223 // TODO: get city from surrounding boundaries? 224 if (distance < maxDistance) { 225 severityLevel = Severity.WARNING; 226 } else { 227 severityLevel = Severity.OTHER; 228 } 229 } 230 } 231 result.add(TestError.builder(this, severityLevel, DUPLICATE_HOUSE_NUMBER) 232 .message(tr("Duplicate house numbers"), marktr("''{0}'' ({1}m)"), simplifiedAddress, (int) distance) 233 .primitives(Arrays.asList(p, p2)).build()); 234 } 235 knownAddresses.get(simplifiedAddress).remove(p); // otherwise we would get every warning two times 236 } 237 errors.addAll(result); 238 return result; 239 } 240 return Collections.emptyList(); 241 } 242 243 static String getSimplifiedAddress(OsmPrimitive p) { 244 String simplifiedStreetName = p.hasKey(ADDR_STREET) ? p.get(ADDR_STREET) : p.get(ADDR_PLACE); 245 // ignore whitespaces and dashes in street name, so that "Mozart-Gasse", "Mozart Gasse" and "Mozartgasse" are all seen as equal 246 return Utils.strip(Stream.of( 247 simplifiedStreetName.replaceAll("[ -]", ""), 248 p.get(ADDR_HOUSE_NUMBER), 249 p.get(ADDR_HOUSE_NAME), 250 p.get(ADDR_UNIT), 251 p.get(ADDR_FLATS)) 252 .filter(Objects::nonNull) 253 .collect(Collectors.joining(" "))) 254 .toUpperCase(Locale.ENGLISH); 255 } 256 257 @Override 258 public void visit(Node n) { 259 checkHouseNumbersWithoutStreet(n); 260 checkForDuplicate(n); 261 } 262 263 @Override 264 public void visit(Way w) { 265 checkHouseNumbersWithoutStreet(w); 266 checkForDuplicate(w); 267 } 268 269 @Override 270 public void visit(Relation r) { 271 checkHouseNumbersWithoutStreet(r); 272 checkForDuplicate(r); 273 if (r.hasTag("type", ASSOCIATED_STREET)) { 274 checkIfObsolete(r); 275 // Used to count occurrences of each house number in order to find duplicates 276 Map<String, List<OsmPrimitive>> map = new HashMap<>(); 277 // Used to detect different street names 278 String relationName = r.get("name"); 279 Set<OsmPrimitive> wrongStreetNames = new HashSet<>(); 280 // Used to check distance 281 Set<OsmPrimitive> houses = new HashSet<>(); 282 Set<Way> street = new HashSet<>(); 283 for (RelationMember m : r.getMembers()) { 284 String role = m.getRole(); 285 OsmPrimitive p = m.getMember(); 286 if ("house".equals(role)) { 287 houses.add(p); 288 String number = p.get(ADDR_HOUSE_NUMBER); 289 if (number != null) { 290 number = number.trim().toUpperCase(Locale.ENGLISH); 291 List<OsmPrimitive> list = map.get(number); 292 if (list == null) { 293 list = new ArrayList<>(); 294 map.put(number, list); 295 } 296 list.add(p); 297 } 298 if (relationName != null && p.hasKey(ADDR_STREET) && !relationName.equals(p.get(ADDR_STREET))) { 299 if (wrongStreetNames.isEmpty()) { 300 wrongStreetNames.add(r); 301 } 302 wrongStreetNames.add(p); 303 } 304 } else if ("street".equals(role)) { 305 if (p instanceof Way) { 306 street.add((Way) p); 307 } 308 if (relationName != null && p.hasTagDifferent("name", relationName)) { 309 if (wrongStreetNames.isEmpty()) { 310 wrongStreetNames.add(r); 311 } 312 wrongStreetNames.add(p); 313 } 314 } 315 } 316 // Report duplicate house numbers 317 for (Entry<String, List<OsmPrimitive>> entry : map.entrySet()) { 318 List<OsmPrimitive> list = entry.getValue(); 319 if (list.size() > 1) { 320 errors.add(TestError.builder(this, Severity.WARNING, DUPLICATE_HOUSE_NUMBER) 321 .message(tr("Duplicate house numbers"), marktr("House number ''{0}'' duplicated"), entry.getKey()) 322 .primitives(list) 323 .build()); 324 } 325 } 326 // Report wrong street names 327 if (!wrongStreetNames.isEmpty()) { 328 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_STREET_NAMES) 329 .message(tr("Multiple street names in relation")) 330 .primitives(wrongStreetNames) 331 .build()); 332 } 333 // Report addresses too far away 334 if (!street.isEmpty()) { 335 for (OsmPrimitive house : houses) { 336 if (house.isUsable()) { 337 checkDistance(house, street); 338 } 339 } 340 } 341 } 342 } 343 344 /** 345 * returns rough distance between two OsmPrimitives 346 * @param a primitive a 347 * @param b primitive b 348 * @return distance of center of bounding boxes in meters 349 */ 350 static double getDistance(OsmPrimitive a, OsmPrimitive b) { 351 LatLon centerA = a.getBBox().getCenter(); 352 LatLon centerB = b.getBBox().getCenter(); 353 return (centerA.greatCircleDistance(centerB)); 354 } 355 356 protected void checkDistance(OsmPrimitive house, Collection<Way> street) { 357 EastNorth centroid; 358 if (house instanceof Node) { 359 centroid = ((Node) house).getEastNorth(); 360 } else if (house instanceof Way) { 361 List<Node> nodes = ((Way) house).getNodes(); 362 if (house.hasKey(ADDR_INTERPOLATION)) { 363 for (Node n : nodes) { 364 if (n.hasKey(ADDR_HOUSE_NUMBER)) { 365 checkDistance(n, street); 366 } 367 } 368 return; 369 } 370 centroid = Geometry.getCentroid(nodes); 371 } else { 372 return; // TODO handle multipolygon houses ? 373 } 374 if (centroid == null) return; // fix #8305 375 double maxDistance = MAX_STREET_DISTANCE.get(); 376 boolean hasIncompleteWays = false; 377 for (Way streetPart : street) { 378 for (Pair<Node, Node> chunk : streetPart.getNodePairs(false)) { 379 EastNorth p1 = chunk.a.getEastNorth(); 380 EastNorth p2 = chunk.b.getEastNorth(); 381 if (p1 != null && p2 != null) { 382 EastNorth closest = Geometry.closestPointToSegment(p1, p2, centroid); 383 if (closest.distance(centroid) <= maxDistance) { 384 return; 385 } 386 } else { 387 Logging.warn("Addresses test skipped chunck "+chunk+" for street part "+streetPart+" because p1 or p2 is null"); 388 } 389 } 390 if (!hasIncompleteWays && streetPart.isIncomplete()) { 391 hasIncompleteWays = true; 392 } 393 } 394 // No street segment found near this house, report error on if the relation does not contain incomplete street ways (fix #8314) 395 if (hasIncompleteWays) return; 396 List<OsmPrimitive> errorList = new ArrayList<>(street); 397 errorList.add(0, house); 398 errors.add(TestError.builder(this, Severity.WARNING, HOUSE_NUMBER_TOO_FAR) 399 .message(tr("House number too far from street")) 400 .primitives(errorList) 401 .build()); 402 } 403 404 /** 405 * Check if an associatedStreet Relation is obsolete. This test marks only those relations which 406 * are complete and don't contain any information which isn't also tagged on the members. 407 * The strategy is to avoid any false positive. 408 * @param r the relation 409 */ 410 private void checkIfObsolete(Relation r) { 411 if (r.isIncomplete()) 412 return; 413 /** array of country codes for which the test should be performed. For now, only Germany */ 414 String[] countryCodes = {"DE"}; 415 TagMap neededtagsForHouse = new TagMap(); 416 for (Entry<String, String> tag : r.getKeys().entrySet()) { 417 String key = tag.getKey(); 418 if (key.startsWith("name:")) { 419 return; // maybe check if all members have corresponding tags? 420 } else if (key.startsWith("addr:")) { 421 neededtagsForHouse.put(key, tag.getValue()); 422 } else { 423 switch (key) { 424 case "name": 425 case "type": 426 case "source": 427 break; 428 default: 429 // unexpected tag in relation 430 return; 431 } 432 } 433 } 434 435 for (RelationMember m : r.getMembers()) { 436 if (m.getMember().isIncomplete() || !isInWarnCountry(m, countryCodes)) 437 return; 438 439 String role = m.getRole(); 440 if ("".equals(role)) { 441 if (m.isWay() && m.getMember().hasKey("highway")) { 442 role = "street"; 443 } else if (m.getMember().hasTag("building")) 444 role = "house"; 445 } 446 switch (role) { 447 case "house": 448 case "addr:houselink": 449 case "address": 450 if (!m.getMember().hasTag(ADDR_STREET) || !m.getMember().hasTag(ADDR_HOUSE_NUMBER)) 451 return; 452 for (Entry<String, String> tag : neededtagsForHouse.entrySet()) { 453 if (!m.getMember().hasTag(tag.getKey(), tag.getValue())) 454 return; 455 } 456 break; 457 case "street": 458 if (!m.getMember().hasTag("name") && r.hasTag("name")) 459 return; 460 break; 461 default: 462 // unknown role: don't create auto-fix 463 return; 464 } 465 } 466 errors.add(TestError.builder(this, Severity.WARNING, OBSOLETE_RELATION) 467 .message(tr("Relation is obsolete")) 468 .primitives(r) 469 .build()); 470 } 471 472 private static boolean isInWarnCountry(RelationMember m, String[] countryCodes) { 473 if (countryCodes.length == 0) 474 return true; 475 LatLon center = null; 476 477 if (m.isNode()) { 478 center = m.getNode().getCoor(); 479 } else if (m.isWay()) { 480 center = m.getWay().getBBox().getCenter(); 481 } else if (m.isRelation() && m.getRelation().isMultipolygon()) { 482 center = m.getRelation().getBBox().getCenter(); 483 } 484 if (center == null) 485 return false; 486 for (String country : countryCodes) { 487 if (Territories.isIso3166Code(country, center)) 488 return true; 489 } 490 return false; 491 } 492 493 /** 494 * remove obsolete relation. 495 */ 496 @Override 497 public Command fixError(TestError testError) { 498 return new DeleteCommand(testError.getPrimitives()); 499 } 500 501 @Override 502 public boolean isFixable(TestError testError) { 503 if (!(testError.getTester() instanceof Addresses)) 504 return false; 505 return testError.getCode() == OBSOLETE_RELATION; 506 } 507 508}