001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006
007import java.awt.GridBagConstraints;
008import java.awt.event.ActionListener;
009import java.io.BufferedReader;
010import java.io.IOException;
011import java.lang.Character.UnicodeBlock;
012import java.util.ArrayList;
013import java.util.Arrays;
014import java.util.Collection;
015import java.util.Collections;
016import java.util.EnumSet;
017import java.util.HashMap;
018import java.util.HashSet;
019import java.util.LinkedHashMap;
020import java.util.LinkedHashSet;
021import java.util.List;
022import java.util.Locale;
023import java.util.Map;
024import java.util.Map.Entry;
025import java.util.Set;
026import java.util.regex.Pattern;
027import java.util.stream.Collectors;
028
029import javax.swing.JCheckBox;
030import javax.swing.JLabel;
031import javax.swing.JPanel;
032
033import org.openstreetmap.josm.command.ChangePropertyCommand;
034import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
035import org.openstreetmap.josm.command.Command;
036import org.openstreetmap.josm.command.SequenceCommand;
037import org.openstreetmap.josm.data.osm.AbstractPrimitive;
038import org.openstreetmap.josm.data.osm.OsmPrimitive;
039import org.openstreetmap.josm.data.osm.Tag;
040import org.openstreetmap.josm.data.osm.TagMap;
041import org.openstreetmap.josm.data.osm.Tagged;
042import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper;
043import org.openstreetmap.josm.data.validation.Severity;
044import org.openstreetmap.josm.data.validation.Test.TagTest;
045import org.openstreetmap.josm.data.validation.TestError;
046import org.openstreetmap.josm.data.validation.util.Entities;
047import org.openstreetmap.josm.gui.progress.ProgressMonitor;
048import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
049import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
050import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetListener;
051import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetType;
052import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
053import org.openstreetmap.josm.gui.tagging.presets.items.Check;
054import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
055import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
056import org.openstreetmap.josm.gui.widgets.EditableList;
057import org.openstreetmap.josm.io.CachedFile;
058import org.openstreetmap.josm.spi.preferences.Config;
059import org.openstreetmap.josm.tools.GBC;
060import org.openstreetmap.josm.tools.Logging;
061import org.openstreetmap.josm.tools.MultiMap;
062import org.openstreetmap.josm.tools.Utils;
063
064/**
065 * Check for misspelled or wrong tags
066 *
067 * @author frsantos
068 * @since 3669
069 */
070public class TagChecker extends TagTest implements TaggingPresetListener {
071
072    /** The config file of ignored tags */
073    public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
074    /** The config file of dictionary words */
075    public static final String SPELL_FILE = "resource://data/validator/words.cfg";
076
077    /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
078    private static final Map<String, String> harmonizedKeys = new HashMap<>();
079    /** The spell check preset values which are not stored in TaggingPresets */
080    private static volatile HashSet<String> additionalPresetsValueData;
081    /** often used tags which are not in presets */
082    private static volatile MultiMap<String, String> oftenUsedTags = new MultiMap<>();
083    private static final Map<TaggingPreset, List<TaggingPresetItem>> presetIndex = new LinkedHashMap<>();
084
085    private static final Pattern UNWANTED_NON_PRINTING_CONTROL_CHARACTERS = Pattern.compile(
086            "[\\x00-\\x09\\x0B\\x0C\\x0E-\\x1F\\x7F\\u200e-\\u200f\\u202a-\\u202e]");
087
088    /** The TagChecker data */
089    private static final List<String> ignoreDataStartsWith = new ArrayList<>();
090    private static final Set<String> ignoreDataEquals = new HashSet<>();
091    private static final List<String> ignoreDataEndsWith = new ArrayList<>();
092    private static final List<Tag> ignoreDataTag = new ArrayList<>();
093    /** tag keys that have only numerical values in the presets */
094    private static final Set<String> ignoreForLevenshtein = new HashSet<>();
095
096    /** The preferences prefix */
097    protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName();
098
099    /**
100     * The preference key to check values
101     */
102    public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
103    /**
104     * The preference key to check keys
105     */
106    public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
107    /**
108     * The preference key to enable complex checks
109     */
110    public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
111    /**
112     * The preference key to search for fixme tags
113     */
114    public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
115    /**
116     * The preference key to check presets
117     */
118    public static final String PREF_CHECK_PRESETS_TYPES = PREFIX + ".checkPresetsTypes";
119
120    /**
121     * The preference key for source files
122     * @see #DEFAULT_SOURCES
123     */
124    public static final String PREF_SOURCES = PREFIX + ".source";
125
126    private static final String BEFORE_UPLOAD = "BeforeUpload";
127    /**
128     * The preference key to check keys - used before upload
129     */
130    public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + BEFORE_UPLOAD;
131    /**
132     * The preference key to check values - used before upload
133     */
134    public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + BEFORE_UPLOAD;
135    /**
136     * The preference key to run complex tests - used before upload
137     */
138    public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + BEFORE_UPLOAD;
139    /**
140     * The preference key to search for fixmes - used before upload
141     */
142    public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + BEFORE_UPLOAD;
143    /**
144     * The preference key to search for presets - used before upload
145     */
146    public static final String PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD = PREF_CHECK_PRESETS_TYPES + BEFORE_UPLOAD;
147
148    private static final int MAX_LEVENSHTEIN_DISTANCE = 2;
149
150    protected boolean includeOtherSeverity;
151
152    protected boolean checkKeys;
153    protected boolean checkValues;
154    /** Was used for special configuration file, might be used to disable value spell checker. */
155    protected boolean checkComplex;
156    protected boolean checkFixmes;
157    protected boolean checkPresetsTypes;
158
159    protected JCheckBox prefCheckKeys;
160    protected JCheckBox prefCheckValues;
161    protected JCheckBox prefCheckComplex;
162    protected JCheckBox prefCheckFixmes;
163    protected JCheckBox prefCheckPresetsTypes;
164
165    protected JCheckBox prefCheckKeysBeforeUpload;
166    protected JCheckBox prefCheckValuesBeforeUpload;
167    protected JCheckBox prefCheckComplexBeforeUpload;
168    protected JCheckBox prefCheckFixmesBeforeUpload;
169    protected JCheckBox prefCheckPresetsTypesBeforeUpload;
170
171    // CHECKSTYLE.OFF: SingleSpaceSeparator
172    protected static final int EMPTY_VALUES             = 1200;
173    protected static final int INVALID_KEY              = 1201;
174    protected static final int INVALID_VALUE            = 1202;
175    protected static final int FIXME                    = 1203;
176    protected static final int INVALID_SPACE            = 1204;
177    protected static final int INVALID_KEY_SPACE        = 1205;
178    protected static final int INVALID_HTML             = 1206; /* 1207 was PAINT */
179    protected static final int LONG_VALUE               = 1208;
180    protected static final int LONG_KEY                 = 1209;
181    protected static final int LOW_CHAR_VALUE           = 1210;
182    protected static final int LOW_CHAR_KEY             = 1211;
183    protected static final int MISSPELLED_VALUE         = 1212;
184    protected static final int MISSPELLED_KEY           = 1213;
185    protected static final int MULTIPLE_SPACES          = 1214;
186    protected static final int MISSPELLED_VALUE_NO_FIX  = 1215;
187    protected static final int UNUSUAL_UNICODE_CHAR_VALUE = 1216;
188    protected static final int INVALID_PRESETS_TYPE     = 1217;
189    // CHECKSTYLE.ON: SingleSpaceSeparator
190
191    protected EditableList sourcesList;
192
193    private static final List<String> DEFAULT_SOURCES = Arrays.asList(IGNORE_FILE, SPELL_FILE);
194
195    /**
196     * Constructor
197     */
198    public TagChecker() {
199        super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
200    }
201
202    @Override
203    public void initialize() throws IOException {
204        TaggingPresets.addListener(this);
205        initializeData();
206        initializePresets();
207        analysePresets();
208    }
209
210    /**
211     * Add presets that contain only numerical values to the ignore list
212     */
213    private static void analysePresets() {
214        for (String key : TaggingPresets.getPresetKeys()) {
215            if (isKeyIgnored(key))
216                continue;
217            boolean allNumerical = true;
218            Set<String> values = TaggingPresets.getPresetValues(key);
219            if (values.isEmpty())
220                allNumerical = false;
221            for (String val : values) {
222                if (!isNum(val)) {
223                    allNumerical = false;
224                    break;
225                }
226            }
227            if (allNumerical) {
228                ignoreForLevenshtein.add(key);
229            }
230        }
231    }
232
233    /**
234     * Reads the spell-check file into a HashMap.
235     * The data file is a list of words, beginning with +/-. If it starts with +,
236     * the word is valid, but if it starts with -, the word should be replaced
237     * by the nearest + word before this.
238     *
239     * @throws IOException if any I/O error occurs
240     */
241    private static void initializeData() throws IOException {
242        ignoreDataStartsWith.clear();
243        ignoreDataEquals.clear();
244        ignoreDataEndsWith.clear();
245        ignoreDataTag.clear();
246        harmonizedKeys.clear();
247        ignoreForLevenshtein.clear();
248        oftenUsedTags.clear();
249        presetIndex.clear();
250
251        StringBuilder errorSources = new StringBuilder();
252        for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) {
253            try (
254                CachedFile cf = new CachedFile(source);
255                BufferedReader reader = cf.getContentReader()
256            ) {
257                String okValue = null;
258                boolean tagcheckerfile = false;
259                boolean ignorefile = false;
260                boolean isFirstLine = true;
261                String line;
262                while ((line = reader.readLine()) != null) {
263                    if (line.isEmpty()) {
264                        // ignore
265                    } else if (line.startsWith("#")) {
266                        if (line.startsWith("# JOSM TagChecker")) {
267                            tagcheckerfile = true;
268                            Logging.error(tr("Ignoring {0}. Support was dropped", source));
269                        } else
270                        if (line.startsWith("# JOSM IgnoreTags")) {
271                            ignorefile = true;
272                            if (!DEFAULT_SOURCES.contains(source)) {
273                                Logging.info(tr("Adding {0} to ignore tags", source));
274                            }
275                        }
276                    } else if (ignorefile) {
277                        parseIgnoreFileLine(source, line);
278                    } else if (tagcheckerfile) {
279                        // ignore
280                    } else if (line.charAt(0) == '+') {
281                        okValue = line.substring(1);
282                    } else if (line.charAt(0) == '-' && okValue != null) {
283                        String hk = harmonizeKey(line.substring(1));
284                        if (!okValue.equals(hk) && harmonizedKeys.put(hk, okValue) != null) {
285                            Logging.debug(tr("Line was ignored: {0}", line));
286                        }
287                    } else {
288                        Logging.error(tr("Invalid spellcheck line: {0}", line));
289                    }
290                    if (isFirstLine) {
291                        isFirstLine = false;
292                        if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
293                            Logging.info(tr("Adding {0} to spellchecker", source));
294                        }
295                    }
296                }
297            } catch (IOException e) {
298                Logging.error(e);
299                errorSources.append(source).append('\n');
300            }
301        }
302
303        if (errorSources.length() > 0)
304            throw new IOException(tr("Could not access data file(s):\n{0}", errorSources));
305    }
306
307    /**
308     * Parse a line found in a configuration file
309     * @param source name of configuration file
310     * @param line the line to parse
311     */
312    private static void parseIgnoreFileLine(String source, String line) {
313        line = line.trim();
314        if (line.length() < 4) {
315            return;
316        }
317        try {
318            String key = line.substring(0, 2);
319            line = line.substring(2);
320
321            switch (key) {
322            case "S:":
323                ignoreDataStartsWith.add(line);
324                break;
325            case "E:":
326                ignoreDataEquals.add(line);
327                addToKeyDictionary(line);
328                break;
329            case "F:":
330                ignoreDataEndsWith.add(line);
331                break;
332            case "K:":
333                Tag tag = Tag.ofString(line);
334                ignoreDataTag.add(tag);
335                oftenUsedTags.put(tag.getKey(), tag.getValue());
336                addToKeyDictionary(tag.getKey());
337                break;
338            default:
339                if (!key.startsWith(";")) {
340                    Logging.warn("Unsupported TagChecker key: " + key);
341                }
342            }
343        } catch (IllegalArgumentException e) {
344            Logging.error("Invalid line in {0} : {1}", source, e.getMessage());
345            Logging.trace(e);
346        }
347    }
348
349    private static void addToKeyDictionary(String key) {
350        if (key != null) {
351            String hk = harmonizeKey(key);
352            if (!key.equals(hk)) {
353                harmonizedKeys.put(hk, key);
354            }
355        }
356    }
357
358    /**
359     * Reads the presets data.
360     *
361     */
362    public static void initializePresets() {
363
364        if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true))
365            return;
366
367        Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
368        if (!presets.isEmpty()) {
369            initAdditionalPresetsValueData();
370            for (TaggingPreset p : presets) {
371                List<TaggingPresetItem> minData = new ArrayList<>();
372                for (TaggingPresetItem i : p.data) {
373                    if (i instanceof KeyedItem) {
374                        if (!"none".equals(((KeyedItem) i).match))
375                            minData.add(i);
376                        addPresetValue((KeyedItem) i);
377                    } else if (i instanceof CheckGroup) {
378                        for (Check c : ((CheckGroup) i).checks) {
379                            addPresetValue(c);
380                        }
381                    }
382                }
383                if (!minData.isEmpty()) {
384                    presetIndex .put(p, minData);
385                }
386            }
387        }
388    }
389
390    private static void initAdditionalPresetsValueData() {
391        additionalPresetsValueData = new HashSet<>();
392        for (String a : AbstractPrimitive.getUninterestingKeys()) {
393            additionalPresetsValueData.add(a);
394        }
395        for (String a : Config.getPref().getList(ValidatorPrefHelper.PREFIX + ".knownkeys",
396                Arrays.asList("is_in", "int_ref", "fixme", "population"))) {
397            additionalPresetsValueData.add(a);
398        }
399    }
400
401    private static void addPresetValue(KeyedItem ky) {
402        if (ky.key != null && ky.getValues() != null) {
403            addToKeyDictionary(ky.key);
404        }
405    }
406
407    /**
408     * Checks given string (key or value) if it contains unwanted non-printing control characters (either ASCII or Unicode bidi characters)
409     * @param s string to check
410     * @return {@code true} if {@code s} contains non-printing control characters
411     */
412    static boolean containsUnwantedNonPrintingControlCharacter(String s) {
413        return s != null && !s.isEmpty() && (
414                isJoiningChar(s.charAt(0)) ||
415                isJoiningChar(s.charAt(s.length() - 1)) ||
416                s.chars().anyMatch(c -> (isAsciiControlChar(c) && !isNewLineChar(c)) || isBidiControlChar(c))
417                );
418    }
419
420    private static boolean isAsciiControlChar(int c) {
421        return c < 0x20 || c == 0x7F;
422    }
423
424    private static boolean isNewLineChar(int c) {
425        return c == 0x0a || c == 0x0d;
426    }
427
428    private static boolean isJoiningChar(int c) {
429        return c == 0x200c || c == 0x200d; // ZWNJ, ZWJ
430    }
431
432    private static boolean isBidiControlChar(int c) {
433        /* check for range 0x200e to 0x200f (LRM, RLM) or
434                           0x202a to 0x202e (LRE, RLE, PDF, LRO, RLO) */
435        return (c >= 0x200e && c <= 0x200f) || (c >= 0x202a && c <= 0x202e);
436    }
437
438    static String removeUnwantedNonPrintingControlCharacters(String s) {
439        // Remove all unwanted characters
440        String result = UNWANTED_NON_PRINTING_CONTROL_CHARACTERS.matcher(s).replaceAll("");
441        // Remove joining characters located at the beginning of the string
442        while (!result.isEmpty() && isJoiningChar(result.charAt(0))) {
443            result = result.substring(1);
444        }
445        // Remove joining characters located at the end of the string
446        while (!result.isEmpty() && isJoiningChar(result.charAt(result.length() - 1))) {
447            result = result.substring(0, result.length() - 1);
448        }
449        return result;
450    }
451
452    static boolean containsUnusualUnicodeCharacter(String key, String value) {
453        return value != null && value.chars().anyMatch(c -> isUnusualUnicodeBlock(key, c));
454    }
455
456    /**
457     * Detects highly suspicious Unicode characters that have been seen in OSM database.
458     * @param key tag key
459     * @param c current character code point
460     * @return {@code true} if the current unicode block is very unusual for the given key
461     */
462    private static boolean isUnusualUnicodeBlock(String key, int c) {
463        UnicodeBlock b = UnicodeBlock.of(c);
464        return isUnusualPhoneticUse(key, b, c) || isUnusualBmpUse(b) || isUnusualSmpUse(b);
465    }
466
467    private static boolean isAllowedPhoneticCharacter(String key, int c) {
468        return c == 0x0259                                          // U+0259 is used as a standard character in azerbaidjani
469            || (key.endsWith("ref") && 0x1D2C <= c && c <= 0x1D42); // allow uppercase superscript latin characters in *ref tags
470    }
471
472    private static boolean isUnusualPhoneticUse(String key, UnicodeBlock b, int c) {
473        return !isAllowedPhoneticCharacter(key, c)
474            && (b == UnicodeBlock.IPA_EXTENSIONS                        // U+0250..U+02AF
475             || b == UnicodeBlock.PHONETIC_EXTENSIONS                   // U+1D00..U+1D7F
476             || b == UnicodeBlock.PHONETIC_EXTENSIONS_SUPPLEMENT)       // U+1D80..U+1DBF
477                && !key.endsWith(":pronunciation");
478    }
479
480    private static boolean isUnusualBmpUse(UnicodeBlock b) {
481        // CHECKSTYLE.OFF: BooleanExpressionComplexity
482        return b == UnicodeBlock.COMBINING_MARKS_FOR_SYMBOLS            // U+20D0..U+20FF
483            || b == UnicodeBlock.MATHEMATICAL_OPERATORS                 // U+2200..U+22FF
484            || b == UnicodeBlock.ENCLOSED_ALPHANUMERICS                 // U+2460..U+24FF
485            || b == UnicodeBlock.BOX_DRAWING                            // U+2500..U+257F
486            || b == UnicodeBlock.GEOMETRIC_SHAPES                       // U+25A0..U+25FF
487            || b == UnicodeBlock.DINGBATS                               // U+2700..U+27BF
488            || b == UnicodeBlock.MISCELLANEOUS_SYMBOLS_AND_ARROWS       // U+2B00..U+2BFF
489            || b == UnicodeBlock.GLAGOLITIC                             // U+2C00..U+2C5F
490            || b == UnicodeBlock.HANGUL_COMPATIBILITY_JAMO              // U+3130..U+318F
491            || b == UnicodeBlock.ENCLOSED_CJK_LETTERS_AND_MONTHS        // U+3200..U+32FF
492            || b == UnicodeBlock.LATIN_EXTENDED_D                       // U+A720..U+A7FF
493            || b == UnicodeBlock.CJK_COMPATIBILITY_IDEOGRAPHS           // U+F900..U+FAFF
494            || b == UnicodeBlock.ALPHABETIC_PRESENTATION_FORMS          // U+FB00..U+FB4F
495            || b == UnicodeBlock.VARIATION_SELECTORS                    // U+FE00..U+FE0F
496            || b == UnicodeBlock.SPECIALS;                              // U+FFF0..U+FFFF
497            // CHECKSTYLE.ON: BooleanExpressionComplexity
498    }
499
500    private static boolean isUnusualSmpUse(UnicodeBlock b) {
501        // UnicodeBlock.SUPPLEMENTAL_SYMBOLS_AND_PICTOGRAPHS is only defined in Java 9+
502        return b == UnicodeBlock.MUSICAL_SYMBOLS                        // U+1D100..U+1D1FF
503            || b == UnicodeBlock.ENCLOSED_ALPHANUMERIC_SUPPLEMENT       // U+1F100..U+1F1FF
504            || b == UnicodeBlock.EMOTICONS                              // U+1F600..U+1F64F
505            || b == UnicodeBlock.TRANSPORT_AND_MAP_SYMBOLS;             // U+1F680..U+1F6FF
506    }
507
508    /**
509     * Get set of preset values for the given key.
510     * @param key the key
511     * @return null if key is not in presets or in additionalPresetsValueData,
512     *  else a set which might be empty.
513     */
514    private static Set<String> getPresetValues(String key) {
515        Set<String> res = TaggingPresets.getPresetValues(key);
516        if (res != null)
517            return res;
518        if (additionalPresetsValueData.contains(key))
519            return Collections.emptySet();
520        // null means key is not known
521        return null;
522    }
523
524    /**
525     * Determines if the given key is in internal presets.
526     * @param key key
527     * @return {@code true} if the given key is in internal presets
528     * @since 9023
529     */
530    public static boolean isKeyInPresets(String key) {
531        return TaggingPresets.getPresetValues(key) != null;
532    }
533
534    /**
535     * Determines if the given tag is in internal presets.
536     * @param key key
537     * @param value value
538     * @return {@code true} if the given tag is in internal presets
539     * @since 9023
540     */
541    public static boolean isTagInPresets(String key, String value) {
542        final Set<String> values = getPresetValues(key);
543        return values != null && values.contains(value);
544    }
545
546    /**
547     * Returns the list of ignored tags.
548     * @return the list of ignored tags
549     * @since 9023
550     */
551    public static List<Tag> getIgnoredTags() {
552        return new ArrayList<>(ignoreDataTag);
553    }
554
555    /**
556     * Determines if the given tag key is ignored for checks "key/tag not in presets".
557     * @param key key
558     * @return true if the given key is ignored
559     */
560    private static boolean isKeyIgnored(String key) {
561        if (ignoreDataEquals.contains(key)) {
562            return true;
563        }
564        for (String a : ignoreDataStartsWith) {
565            if (key.startsWith(a)) {
566                return true;
567            }
568        }
569        for (String a : ignoreDataEndsWith) {
570            if (key.endsWith(a)) {
571                return true;
572            }
573        }
574        return false;
575    }
576
577    /**
578     * Determines if the given tag is ignored for checks "key/tag not in presets".
579     * @param key key
580     * @param value value
581     * @return {@code true} if the given tag is ignored
582     * @since 9023
583     */
584    public static boolean isTagIgnored(String key, String value) {
585        if (isKeyIgnored(key))
586            return true;
587        final Set<String> values = getPresetValues(key);
588        if (values != null && values.isEmpty())
589            return true;
590        if (!isTagInPresets(key, value)) {
591            for (Tag a : ignoreDataTag) {
592                if (key.equals(a.getKey()) && value.equals(a.getValue())) {
593                    return true;
594                }
595            }
596        }
597        return false;
598    }
599
600    /**
601     * Checks the primitive tags
602     * @param p The primitive to check
603     */
604    @Override
605    public void check(OsmPrimitive p) {
606        // Just a collection to know if a primitive has been already marked with error
607        MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
608
609        for (Entry<String, String> prop : p.getKeys().entrySet()) {
610            String s = marktr("Tag ''{0}'' invalid.");
611            String key = prop.getKey();
612            String value = prop.getValue();
613
614            if (checkKeys) {
615                checkSingleTagKeySimple(withErrors, p, s, key);
616            }
617            if (checkValues) {
618                checkSingleTagValueSimple(withErrors, p, s, key, value);
619                checkSingleTagComplex(withErrors, p, key, value);
620            }
621            if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) {
622                errors.add(TestError.builder(this, Severity.OTHER, FIXME)
623                        .message(tr("FIXMES"))
624                        .primitives(p)
625                        .build());
626                withErrors.put(p, "FIXME");
627            }
628        }
629
630        if (checkPresetsTypes) {
631            TagMap tags = p.getKeys();
632            TaggingPresetType presetType = TaggingPresetType.forPrimitive(p);
633            EnumSet<TaggingPresetType> presetTypes = EnumSet.of(presetType);
634
635            Collection<TaggingPreset> matchingPresets = new LinkedHashSet<>();
636            for (Entry<TaggingPreset, List<TaggingPresetItem>> e : presetIndex.entrySet()) {
637                if (TaggingPresetItem.matches(e.getValue(), tags)) {
638                    matchingPresets.add(e.getKey());
639                }
640            }
641            Collection<TaggingPreset> matchingPresetsOK = matchingPresets.stream().filter(
642                    tp -> tp.typeMatches(presetTypes)).collect(Collectors.toList());
643            Collection<TaggingPreset> matchingPresetsKO = matchingPresets.stream().filter(
644                    tp -> !tp.typeMatches(presetTypes)).collect(Collectors.toList());
645
646            for (TaggingPreset tp : matchingPresetsKO) {
647                // Potential error, unless matching tags are all known by a supported preset
648                Map<String, String> matchingTags = tp.data.stream()
649                    .filter(i -> Boolean.TRUE.equals(i.matches(tags)))
650                    .filter(i -> i instanceof KeyedItem).map(i -> ((KeyedItem) i).key)
651                    .collect(Collectors.toMap(k -> k, tags::get));
652                if (matchingPresetsOK.stream().noneMatch(
653                        tp2 -> matchingTags.entrySet().stream().allMatch(
654                                e -> tp2.data.stream().anyMatch(
655                                        i -> i instanceof KeyedItem && ((KeyedItem) i).key.equals(e.getKey()))))) {
656                    errors.add(TestError.builder(this, Severity.OTHER, INVALID_PRESETS_TYPE)
657                            .message(tr("Object type not in preset"),
658                                    marktr("Object type {0} is not supported by tagging preset: {1}"),
659                                    tr(presetType.getName()), tp.getLocaleName())
660                            .primitives(p)
661                            .build());
662                }
663            }
664        }
665    }
666
667    private void checkSingleTagValueSimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key, String value) {
668        if (!checkValues || value == null)
669            return;
670        if ((containsUnwantedNonPrintingControlCharacter(value)) && !withErrors.contains(p, "ICV")) {
671            errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE)
672                    .message(tr("Tag value contains non-printing (usually invisible) character"), s, key)
673                    .primitives(p)
674                    .fix(() -> new ChangePropertyCommand(p, key, removeUnwantedNonPrintingControlCharacters(value)))
675                    .build());
676            withErrors.put(p, "ICV");
677        }
678        if ((containsUnusualUnicodeCharacter(key, value)) && !withErrors.contains(p, "UUCV")) {
679            errors.add(TestError.builder(this, Severity.WARNING, UNUSUAL_UNICODE_CHAR_VALUE)
680                    .message(tr("Tag value contains unusual Unicode character"), s, key)
681                    .primitives(p)
682                    .build());
683            withErrors.put(p, "UUCV");
684        }
685        if ((value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) {
686            errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE)
687                    .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key)
688                    .primitives(p)
689                    .build());
690            withErrors.put(p, "LV");
691        }
692        if ((value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
693            errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES)
694                    .message(tr("Tags with empty values"), s, key)
695                    .primitives(p)
696                    .build());
697            withErrors.put(p, "EV");
698        }
699        final String errTypeSpace = "SPACE";
700        if ((value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, errTypeSpace)) {
701            errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE)
702                    .message(tr("Property values start or end with white space"), s, key)
703                    .primitives(p)
704                    .build());
705            withErrors.put(p, errTypeSpace);
706        }
707        if (value.contains("  ") && !withErrors.contains(p, errTypeSpace)) {
708            errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES)
709                    .message(tr("Property values contain multiple white spaces"), s, key)
710                    .primitives(p)
711                    .build());
712            withErrors.put(p, errTypeSpace);
713        }
714        if (includeOtherSeverity && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
715            errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML)
716                    .message(tr("Property values contain HTML entity"), s, key)
717                    .primitives(p)
718                    .build());
719            withErrors.put(p, "HTML");
720        }
721    }
722
723    private void checkSingleTagKeySimple(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String s, String key) {
724        if (!checkKeys || key == null)
725            return;
726        if ((containsUnwantedNonPrintingControlCharacter(key)) && !withErrors.contains(p, "ICK")) {
727            errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY)
728                    .message(tr("Tag key contains non-printing character"), s, key)
729                    .primitives(p)
730                    .fix(() -> new ChangePropertyCommand(p, key, removeUnwantedNonPrintingControlCharacters(key)))
731                    .build());
732            withErrors.put(p, "ICK");
733        }
734        if (key.length() > Tagged.MAX_TAG_LENGTH && !withErrors.contains(p, "LK")) {
735            errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY)
736                    .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key)
737                    .primitives(p)
738                    .build());
739            withErrors.put(p, "LK");
740        }
741        if (key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
742            errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE)
743                    .message(tr("Invalid white space in property key"), s, key)
744                    .primitives(p)
745                    .build());
746            withErrors.put(p, "IPK");
747        }
748    }
749
750    private void checkSingleTagComplex(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key, String value) {
751        if (!checkValues || key == null || value == null || value.isEmpty())
752            return;
753        if (additionalPresetsValueData != null && !isTagIgnored(key, value)) {
754            if (!isKeyInPresets(key)) {
755                spellCheckKey(withErrors, p, key);
756            } else if (!isTagInPresets(key, value)) {
757                if (oftenUsedTags.contains(key, value)) {
758                    // tag is quite often used but not in presets
759                    errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
760                            .message(tr("Presets do not contain property value"),
761                                    marktr("Value ''{0}'' for key ''{1}'' not in presets, but is known."), value, key)
762                            .primitives(p)
763                            .build());
764                    withErrors.put(p, "UPV");
765                } else {
766                    tryGuess(p, key, value, withErrors);
767                }
768            }
769        }
770    }
771
772    private void spellCheckKey(MultiMap<OsmPrimitive, String> withErrors, OsmPrimitive p, String key) {
773        String prettifiedKey = harmonizeKey(key);
774        String fixedKey;
775        if (ignoreDataEquals.contains(prettifiedKey)) {
776            fixedKey = prettifiedKey;
777        } else {
778            fixedKey = isKeyInPresets(prettifiedKey) ? prettifiedKey : harmonizedKeys.get(prettifiedKey);
779        }
780        if (fixedKey == null) {
781            for (Tag a : ignoreDataTag) {
782                if (a.getKey().equals(prettifiedKey)) {
783                    fixedKey = prettifiedKey;
784                    break;
785                }
786            }
787        }
788
789        if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
790            final String proposedKey = fixedKey;
791            // misspelled preset key
792            final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY)
793                    .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, proposedKey)
794                    .primitives(p);
795            if (p.hasKey(fixedKey)) {
796                errors.add(error.build());
797            } else {
798                errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, proposedKey)).build());
799            }
800            withErrors.put(p, "WPK");
801        } else if (includeOtherSeverity) {
802            errors.add(TestError.builder(this, Severity.OTHER, INVALID_KEY)
803                    .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key)
804                    .primitives(p)
805                    .build());
806            withErrors.put(p, "UPK");
807        }
808    }
809
810    private void tryGuess(OsmPrimitive p, String key, String value, MultiMap<OsmPrimitive, String> withErrors) {
811        // try to fix common typos and check again if value is still unknown
812        final String harmonizedValue = harmonizeValue(value);
813        if (harmonizedValue == null || harmonizedValue.isEmpty())
814            return;
815        String fixedValue = null;
816        List<Set<String>> sets = new ArrayList<>();
817        Set<String> presetValues = getPresetValues(key);
818        if (presetValues != null)
819            sets.add(presetValues);
820        Set<String> usedValues = oftenUsedTags.get(key);
821        if (usedValues != null)
822            sets.add(usedValues);
823        for (Set<String> possibleValues: sets) {
824            if (possibleValues.contains(harmonizedValue)) {
825                fixedValue = harmonizedValue;
826                break;
827            }
828        }
829        if (fixedValue == null && !ignoreForLevenshtein.contains(key)) {
830            int maxPresetValueLen = 0;
831            List<String> fixVals = new ArrayList<>();
832            // use Levenshtein distance to find typical typos
833            int minDist = MAX_LEVENSHTEIN_DISTANCE + 1;
834            String closest = null;
835            for (Set<String> possibleValues: sets) {
836                for (String possibleVal : possibleValues) {
837                    if (possibleVal.isEmpty())
838                        continue;
839                    maxPresetValueLen = Math.max(maxPresetValueLen, possibleVal.length());
840                    if (harmonizedValue.length() < 3 && possibleVal.length() >= harmonizedValue.length() + MAX_LEVENSHTEIN_DISTANCE) {
841                        // don't suggest fix value when given value is short and lengths are too different
842                        // for example surface=u would result in surface=mud
843                        continue;
844                    }
845                    int dist = Utils.getLevenshteinDistance(possibleVal, harmonizedValue);
846                    if (dist >= harmonizedValue.length()) {
847                        // short value, all characters are different. Don't warn, might say Value '10' for key 'fee' looks like 'no'.
848                        continue;
849                    }
850                    if (dist < minDist) {
851                        closest = possibleVal;
852                        minDist = dist;
853                        fixVals.clear();
854                        fixVals.add(possibleVal);
855                    } else if (dist == minDist) {
856                        fixVals.add(possibleVal);
857                    }
858                }
859            }
860
861            if (minDist <= MAX_LEVENSHTEIN_DISTANCE && maxPresetValueLen > MAX_LEVENSHTEIN_DISTANCE
862                    && (harmonizedValue.length() > 3 || minDist < MAX_LEVENSHTEIN_DISTANCE)) {
863                if (fixVals.size() < 2) {
864                    fixedValue = closest;
865                } else {
866                    Collections.sort(fixVals);
867                    // misspelled preset value with multiple good alternatives
868                    errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE_NO_FIX)
869                            .message(tr("Unknown property value"),
870                                    marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe one of {2} is meant?"),
871                                    value, key, fixVals)
872                            .primitives(p).build());
873                    withErrors.put(p, "WPV");
874                    return;
875                }
876            }
877        }
878        if (fixedValue != null && !fixedValue.equals(value)) {
879            final String newValue = fixedValue;
880            // misspelled preset value
881            errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE)
882                    .message(tr("Unknown property value"),
883                            marktr("Value ''{0}'' for key ''{1}'' is unknown, maybe ''{2}'' is meant?"), value, key, newValue)
884                    .primitives(p)
885                    .build());
886            withErrors.put(p, "WPV");
887        } else if (includeOtherSeverity) {
888            // unknown preset value
889            errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE)
890                    .message(tr("Presets do not contain property value"),
891                            marktr("Value ''{0}'' for key ''{1}'' not in presets."), value, key)
892                    .primitives(p)
893                    .build());
894            withErrors.put(p, "UPV");
895        }
896    }
897
898    private static boolean isNum(String harmonizedValue) {
899        try {
900            Double.parseDouble(harmonizedValue);
901            return true;
902        } catch (NumberFormatException e) {
903            return false;
904        }
905    }
906
907    private static boolean isFixme(String key, String value) {
908        return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo")
909          || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete");
910    }
911
912    private static String harmonizeKey(String key) {
913        return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,");
914    }
915
916    private static String harmonizeValue(String value) {
917        return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,");
918    }
919
920    @Override
921    public void startTest(ProgressMonitor monitor) {
922        super.startTest(monitor);
923        includeOtherSeverity = includeOtherSeverityChecks();
924        checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true);
925        if (isBeforeUpload) {
926            checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
927        }
928
929        checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true);
930        if (isBeforeUpload) {
931            checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
932        }
933
934        checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true);
935        if (isBeforeUpload) {
936            checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
937        }
938
939        checkFixmes = includeOtherSeverity && Config.getPref().getBoolean(PREF_CHECK_FIXMES, true);
940        if (isBeforeUpload) {
941            checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
942        }
943
944        checkPresetsTypes = includeOtherSeverity && Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES, true);
945        if (isBeforeUpload) {
946            checkPresetsTypes = checkPresetsTypes && Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, true);
947        }
948    }
949
950    @Override
951    public void visit(Collection<OsmPrimitive> selection) {
952        if (checkKeys || checkValues || checkComplex || checkFixmes || checkPresetsTypes) {
953            super.visit(selection);
954        }
955    }
956
957    @Override
958    public void addGui(JPanel testPanel) {
959        GBC a = GBC.eol();
960        a.anchor = GridBagConstraints.EAST;
961
962        testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
963
964        prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true));
965        prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
966        testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
967
968        prefCheckKeysBeforeUpload = new JCheckBox();
969        prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
970        testPanel.add(prefCheckKeysBeforeUpload, a);
971
972        prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true));
973        prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
974        testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
975
976        prefCheckComplexBeforeUpload = new JCheckBox();
977        prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
978        testPanel.add(prefCheckComplexBeforeUpload, a);
979
980        final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES);
981        sourcesList = new EditableList(tr("TagChecker source"));
982        sourcesList.setItems(sources);
983        testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
984        testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
985
986        ActionListener disableCheckActionListener = e -> handlePrefEnable();
987        prefCheckKeys.addActionListener(disableCheckActionListener);
988        prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
989        prefCheckComplex.addActionListener(disableCheckActionListener);
990        prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
991
992        handlePrefEnable();
993
994        prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true));
995        prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
996        testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
997
998        prefCheckValuesBeforeUpload = new JCheckBox();
999        prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
1000        testPanel.add(prefCheckValuesBeforeUpload, a);
1001
1002        prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true));
1003        prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
1004        testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
1005
1006        prefCheckFixmesBeforeUpload = new JCheckBox();
1007        prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
1008        testPanel.add(prefCheckFixmesBeforeUpload, a);
1009
1010        prefCheckPresetsTypes = new JCheckBox(tr("Check for presets types."), Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES, true));
1011        prefCheckPresetsTypes.setToolTipText(tr("Validate that objects types are valid checking against presets."));
1012        testPanel.add(prefCheckPresetsTypes, GBC.std().insets(20, 0, 0, 0));
1013
1014        prefCheckPresetsTypesBeforeUpload = new JCheckBox();
1015        prefCheckPresetsTypesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, true));
1016        testPanel.add(prefCheckPresetsTypesBeforeUpload, a);
1017    }
1018
1019    /**
1020     * Enables/disables the source list field
1021     */
1022    public void handlePrefEnable() {
1023        boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
1024                || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
1025        sourcesList.setEnabled(selected);
1026    }
1027
1028    @Override
1029    public boolean ok() {
1030        enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
1031        testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
1032                || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
1033
1034        Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected());
1035        Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
1036        Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
1037        Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
1038        Config.getPref().putBoolean(PREF_CHECK_PRESETS_TYPES, prefCheckPresetsTypes.isSelected());
1039        Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
1040        Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
1041        Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
1042        Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
1043        Config.getPref().putBoolean(PREF_CHECK_PRESETS_TYPES_BEFORE_UPLOAD, prefCheckPresetsTypesBeforeUpload.isSelected());
1044        return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems());
1045    }
1046
1047    @Override
1048    public Command fixError(TestError testError) {
1049        List<Command> commands = new ArrayList<>(50);
1050
1051        Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
1052        for (OsmPrimitive p : primitives) {
1053            Map<String, String> tags = p.getKeys();
1054            if (tags.isEmpty()) {
1055                continue;
1056            }
1057
1058            for (Entry<String, String> prop: tags.entrySet()) {
1059                String key = prop.getKey();
1060                String value = prop.getValue();
1061                if (value == null || value.trim().isEmpty()) {
1062                    commands.add(new ChangePropertyCommand(p, key, null));
1063                } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains("  ")) {
1064                    commands.add(new ChangePropertyCommand(p, key, Utils.removeWhiteSpaces(value)));
1065                } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains("  ")) {
1066                    commands.add(new ChangePropertyKeyCommand(p, key, Utils.removeWhiteSpaces(key)));
1067                } else {
1068                    String evalue = Entities.unescape(value);
1069                    if (!evalue.equals(value)) {
1070                        commands.add(new ChangePropertyCommand(p, key, evalue));
1071                    }
1072                }
1073            }
1074        }
1075
1076        if (commands.isEmpty())
1077            return null;
1078        if (commands.size() == 1)
1079            return commands.get(0);
1080
1081        return new SequenceCommand(tr("Fix tags"), commands);
1082    }
1083
1084    @Override
1085    public boolean isFixable(TestError testError) {
1086        if (testError.getTester() instanceof TagChecker) {
1087            int code = testError.getCode();
1088            return code == EMPTY_VALUES || code == INVALID_SPACE ||
1089                   code == INVALID_KEY_SPACE || code == INVALID_HTML ||
1090                   code == MULTIPLE_SPACES;
1091        }
1092
1093        return false;
1094    }
1095
1096    @Override
1097    public void taggingPresetsModified() {
1098        try {
1099            initializeData();
1100            initializePresets();
1101            analysePresets();
1102        } catch (IOException e) {
1103            Logging.error(e);
1104        }
1105    }
1106}