001// License: GPL. For details, see LICENSE file.
002package org.openstreetmap.josm.data.validation.tests;
003
004import static org.openstreetmap.josm.tools.I18n.marktr;
005import static org.openstreetmap.josm.tools.I18n.tr;
006
007import java.awt.GridBagConstraints;
008import java.awt.event.ActionEvent;
009import java.awt.event.ActionListener;
010import java.io.BufferedReader;
011import java.io.IOException;
012import java.io.InputStream;
013import java.text.MessageFormat;
014import java.util.ArrayList;
015import java.util.Arrays;
016import java.util.Collection;
017import java.util.HashMap;
018import java.util.List;
019import java.util.Locale;
020import java.util.Map;
021import java.util.Map.Entry;
022import java.util.Set;
023import java.util.regex.Matcher;
024import java.util.regex.Pattern;
025import java.util.regex.PatternSyntaxException;
026
027import javax.swing.JCheckBox;
028import javax.swing.JLabel;
029import javax.swing.JPanel;
030
031import org.openstreetmap.josm.Main;
032import org.openstreetmap.josm.command.ChangePropertyCommand;
033import org.openstreetmap.josm.command.ChangePropertyKeyCommand;
034import org.openstreetmap.josm.command.Command;
035import org.openstreetmap.josm.command.SequenceCommand;
036import org.openstreetmap.josm.data.osm.OsmPrimitive;
037import org.openstreetmap.josm.data.osm.OsmPrimitiveType;
038import org.openstreetmap.josm.data.osm.OsmUtils;
039import org.openstreetmap.josm.data.osm.Tag;
040import org.openstreetmap.josm.data.validation.FixableTestError;
041import org.openstreetmap.josm.data.validation.Severity;
042import org.openstreetmap.josm.data.validation.Test.TagTest;
043import org.openstreetmap.josm.data.validation.TestError;
044import org.openstreetmap.josm.data.validation.util.Entities;
045import org.openstreetmap.josm.gui.preferences.validator.ValidatorPreference;
046import org.openstreetmap.josm.gui.progress.ProgressMonitor;
047import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset;
048import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem;
049import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets;
050import org.openstreetmap.josm.gui.tagging.presets.items.Check;
051import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup;
052import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem;
053import org.openstreetmap.josm.gui.widgets.EditableList;
054import org.openstreetmap.josm.io.CachedFile;
055import org.openstreetmap.josm.io.UTFInputStreamReader;
056import org.openstreetmap.josm.tools.GBC;
057import org.openstreetmap.josm.tools.MultiMap;
058import org.openstreetmap.josm.tools.Utils;
059
060/**
061 * Check for misspelled or wrong tags
062 *
063 * @author frsantos
064 * @since 3669
065 */
066public class TagChecker extends TagTest {
067
068    /** The config file of ignored tags */
069    public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg";
070    /** The config file of dictionary words */
071    public static final String SPELL_FILE = "resource://data/validator/words.cfg";
072
073    /** Normalized keys: the key should be substituted by the value if the key was not found in presets */
074    private static final Map<String, String> harmonizedKeys = new HashMap<>();
075    /** The spell check preset values */
076    private static volatile MultiMap<String, String> presetsValueData;
077    /** The TagChecker data */
078    private static final List<CheckerData> checkerData = new ArrayList<>();
079    private static final List<String> ignoreDataStartsWith = new ArrayList<>();
080    private static final List<String> ignoreDataEquals = new ArrayList<>();
081    private static final List<String> ignoreDataEndsWith = new ArrayList<>();
082    private static final List<Tag> ignoreDataTag = new ArrayList<>();
083
084    /** The preferences prefix */
085    protected static final String PREFIX = ValidatorPreference.PREFIX + "." + TagChecker.class.getSimpleName();
086
087    public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues";
088    public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys";
089    public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex";
090    public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes";
091
092    public static final String PREF_SOURCES = PREFIX + ".source";
093
094    public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload";
095    public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload";
096    public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload";
097    public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload";
098
099    protected boolean checkKeys;
100    protected boolean checkValues;
101    protected boolean checkComplex;
102    protected boolean checkFixmes;
103
104    protected JCheckBox prefCheckKeys;
105    protected JCheckBox prefCheckValues;
106    protected JCheckBox prefCheckComplex;
107    protected JCheckBox prefCheckFixmes;
108    protected JCheckBox prefCheckPaint;
109
110    protected JCheckBox prefCheckKeysBeforeUpload;
111    protected JCheckBox prefCheckValuesBeforeUpload;
112    protected JCheckBox prefCheckComplexBeforeUpload;
113    protected JCheckBox prefCheckFixmesBeforeUpload;
114    protected JCheckBox prefCheckPaintBeforeUpload;
115
116    protected static final int EMPTY_VALUES      = 1200;
117    protected static final int INVALID_KEY       = 1201;
118    protected static final int INVALID_VALUE     = 1202;
119    protected static final int FIXME             = 1203;
120    protected static final int INVALID_SPACE     = 1204;
121    protected static final int INVALID_KEY_SPACE = 1205;
122    protected static final int INVALID_HTML      = 1206; /* 1207 was PAINT */
123    protected static final int LONG_VALUE        = 1208;
124    protected static final int LONG_KEY          = 1209;
125    protected static final int LOW_CHAR_VALUE    = 1210;
126    protected static final int LOW_CHAR_KEY      = 1211;
127    protected static final int MISSPELLED_VALUE  = 1212;
128    protected static final int MISSPELLED_KEY    = 1213;
129    /** 1250 and up is used by tagcheck */
130
131    protected EditableList sourcesList;
132
133    private static final List<String> DEFAULT_SOURCES = Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE);
134
135    /**
136     * Constructor
137     */
138    public TagChecker() {
139        super(tr("Tag checker"), tr("This test checks for errors in tag keys and values."));
140    }
141
142    @Override
143    public void initialize() throws IOException {
144        initializeData();
145        initializePresets();
146    }
147
148    /**
149     * Reads the spellcheck file into a HashMap.
150     * The data file is a list of words, beginning with +/-. If it starts with +,
151     * the word is valid, but if it starts with -, the word should be replaced
152     * by the nearest + word before this.
153     *
154     * @throws IOException if any I/O error occurs
155     */
156    private static void initializeData() throws IOException {
157        checkerData.clear();
158        ignoreDataStartsWith.clear();
159        ignoreDataEquals.clear();
160        ignoreDataEndsWith.clear();
161        ignoreDataTag.clear();
162        harmonizedKeys.clear();
163
164        StringBuilder errorSources = new StringBuilder();
165        for (String source : Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES)) {
166            try (
167                InputStream s = new CachedFile(source).getInputStream();
168                BufferedReader reader = new BufferedReader(UTFInputStreamReader.create(s));
169            ) {
170                String okValue = null;
171                boolean tagcheckerfile = false;
172                boolean ignorefile = false;
173                boolean isFirstLine = true;
174                String line;
175                while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) {
176                    if (line.startsWith("#")) {
177                        if (line.startsWith("# JOSM TagChecker")) {
178                            tagcheckerfile = true;
179                            if (!DEFAULT_SOURCES.contains(source)) {
180                                Main.info(tr("Adding {0} to tag checker", source));
181                            }
182                        } else
183                        if (line.startsWith("# JOSM IgnoreTags")) {
184                            ignorefile = true;
185                            if (!DEFAULT_SOURCES.contains(source)) {
186                                Main.info(tr("Adding {0} to ignore tags", source));
187                            }
188                        }
189                    } else if (ignorefile) {
190                        line = line.trim();
191                        if (line.length() < 4) {
192                            continue;
193                        }
194
195                        String key = line.substring(0, 2);
196                        line = line.substring(2);
197
198                        switch (key) {
199                        case "S:":
200                            ignoreDataStartsWith.add(line);
201                            break;
202                        case "E:":
203                            ignoreDataEquals.add(line);
204                            break;
205                        case "F:":
206                            ignoreDataEndsWith.add(line);
207                            break;
208                        case "K:":
209                            ignoreDataTag.add(Tag.ofString(line));
210                        }
211                    } else if (tagcheckerfile) {
212                        if (!line.isEmpty()) {
213                            CheckerData d = new CheckerData();
214                            String err = d.getData(line);
215
216                            if (err == null) {
217                                checkerData.add(d);
218                            } else {
219                                Main.error(tr("Invalid tagchecker line - {0}: {1}", err, line));
220                            }
221                        }
222                    } else if (line.charAt(0) == '+') {
223                        okValue = line.substring(1);
224                    } else if (line.charAt(0) == '-' && okValue != null) {
225                        harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue);
226                    } else {
227                        Main.error(tr("Invalid spellcheck line: {0}", line));
228                    }
229                    if (isFirstLine) {
230                        isFirstLine = false;
231                        if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) {
232                            Main.info(tr("Adding {0} to spellchecker", source));
233                        }
234                    }
235                }
236            } catch (IOException e) {
237                errorSources.append(source).append('\n');
238            }
239        }
240
241        if (errorSources.length() > 0)
242            throw new IOException(tr("Could not access data file(s):\n{0}", errorSources));
243    }
244
245    /**
246     * Reads the presets data.
247     *
248     */
249    public static void initializePresets() {
250
251        if (!Main.pref.getBoolean(PREF_CHECK_VALUES, true))
252            return;
253
254        Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets();
255        if (!presets.isEmpty()) {
256            presetsValueData = new MultiMap<>();
257            for (String a : OsmPrimitive.getUninterestingKeys()) {
258                presetsValueData.putVoid(a);
259            }
260            // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead)
261            for (String a : Main.pref.getCollection(ValidatorPreference.PREFIX + ".knownkeys",
262                    Arrays.asList(new String[]{"is_in", "int_ref", "fixme", "population"}))) {
263                presetsValueData.putVoid(a);
264            }
265            for (TaggingPreset p : presets) {
266                for (TaggingPresetItem i : p.data) {
267                    if (i instanceof KeyedItem) {
268                        addPresetValue(p, (KeyedItem) i);
269                    } else if (i instanceof CheckGroup) {
270                        for (Check c : ((CheckGroup) i).checks) {
271                            addPresetValue(p, c);
272                        }
273                    }
274                }
275            }
276        }
277    }
278
279    private static void addPresetValue(TaggingPreset p, KeyedItem ky) {
280        Collection<String> values = ky.getValues();
281        if (ky.key != null && values != null) {
282            try {
283                presetsValueData.putAll(ky.key, values);
284                harmonizedKeys.put(harmonizeKey(ky.key), ky.key);
285            } catch (NullPointerException e) {
286                Main.error(p+": Unable to initialize "+ky);
287            }
288        }
289    }
290
291    /**
292     * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters)
293     * @param s string to check
294     * @return {@code true} if {@code s} contains characters with code below 0x20
295     */
296    private static boolean containsLow(String s) {
297        if (s == null)
298            return false;
299        for (int i = 0; i < s.length(); i++) {
300            if (s.charAt(i) < 0x20)
301                return true;
302        }
303        return false;
304    }
305
306    /**
307     * Determines if the given key is in internal presets.
308     * @param key key
309     * @return {@code true} if the given key is in internal presets
310     * @since 9023
311     */
312    public static boolean isKeyInPresets(String key) {
313        return presetsValueData.get(key) != null;
314    }
315
316    /**
317     * Determines if the given tag is in internal presets.
318     * @param key key
319     * @param value value
320     * @return {@code true} if the given tag is in internal presets
321     * @since 9023
322     */
323    public static boolean isTagInPresets(String key, String value) {
324        final Set<String> values = presetsValueData.get(key);
325        return values != null && (values.isEmpty() || values.contains(value));
326    }
327
328    /**
329     * Returns the list of ignored tags.
330     * @return the list of ignored tags
331     * @since 9023
332     */
333    public static List<Tag> getIgnoredTags() {
334        return new ArrayList<>(ignoreDataTag);
335    }
336
337    /**
338     * Determines if the given tag is ignored for checks "key/tag not in presets".
339     * @param key key
340     * @param value value
341     * @return {@code true} if the given tag is ignored
342     * @since 9023
343     */
344    public static boolean isTagIgnored(String key, String value) {
345        boolean tagInPresets = isTagInPresets(key, value);
346        boolean ignore = false;
347
348        for (String a : ignoreDataStartsWith) {
349            if (key.startsWith(a)) {
350                ignore = true;
351            }
352        }
353        for (String a : ignoreDataEquals) {
354            if (key.equals(a)) {
355                ignore = true;
356            }
357        }
358        for (String a : ignoreDataEndsWith) {
359            if (key.endsWith(a)) {
360                ignore = true;
361            }
362        }
363
364        if (!tagInPresets) {
365            for (Tag a : ignoreDataTag) {
366                if (key.equals(a.getKey()) && value.equals(a.getValue())) {
367                    ignore = true;
368                }
369            }
370        }
371        return ignore;
372    }
373
374    /**
375     * Checks the primitive tags
376     * @param p The primitive to check
377     */
378    @Override
379    public void check(OsmPrimitive p) {
380        // Just a collection to know if a primitive has been already marked with error
381        MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>();
382
383        if (checkComplex) {
384            Map<String, String> keys = p.getKeys();
385            for (CheckerData d : checkerData) {
386                if (d.match(p, keys)) {
387                    errors.add(new TestError(this, d.getSeverity(), tr("Suspicious tag/value combinations"),
388                            d.getDescription(), d.getDescriptionOrig(), d.getCode(), p));
389                    withErrors.put(p, "TC");
390                }
391            }
392        }
393
394        for (Entry<String, String> prop : p.getKeys().entrySet()) {
395            String s = marktr("Key ''{0}'' invalid.");
396            String key = prop.getKey();
397            String value = prop.getValue();
398            if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) {
399                errors.add(new TestError(this, Severity.WARNING, tr("Tag value contains character with code less than 0x20"),
400                        tr(s, key), MessageFormat.format(s, key), LOW_CHAR_VALUE, p));
401                withErrors.put(p, "ICV");
402            }
403            if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) {
404                errors.add(new TestError(this, Severity.WARNING, tr("Tag key contains character with code less than 0x20"),
405                        tr(s, key), MessageFormat.format(s, key), LOW_CHAR_KEY, p));
406                withErrors.put(p, "ICK");
407            }
408            if (checkValues && (value != null && value.length() > 255) && !withErrors.contains(p, "LV")) {
409                errors.add(new TestError(this, Severity.ERROR, tr("Tag value longer than allowed"),
410                        tr(s, key), MessageFormat.format(s, key), LONG_VALUE, p));
411                withErrors.put(p, "LV");
412            }
413            if (checkKeys && (key != null && key.length() > 255) && !withErrors.contains(p, "LK")) {
414                errors.add(new TestError(this, Severity.ERROR, tr("Tag key longer than allowed"),
415                        tr(s, key), MessageFormat.format(s, key), LONG_KEY, p));
416                withErrors.put(p, "LK");
417            }
418            if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) {
419                errors.add(new TestError(this, Severity.WARNING, tr("Tags with empty values"),
420                        tr(s, key), MessageFormat.format(s, key), EMPTY_VALUES, p));
421                withErrors.put(p, "EV");
422            }
423            if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) {
424                errors.add(new TestError(this, Severity.WARNING, tr("Invalid white space in property key"),
425                        tr(s, key), MessageFormat.format(s, key), INVALID_KEY_SPACE, p));
426                withErrors.put(p, "IPK");
427            }
428            if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) {
429                errors.add(new TestError(this, Severity.WARNING, tr("Property values start or end with white space"),
430                        tr(s, key), MessageFormat.format(s, key), INVALID_SPACE, p));
431                withErrors.put(p, "SPACE");
432            }
433            if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) {
434                errors.add(new TestError(this, Severity.OTHER, tr("Property values contain HTML entity"),
435                        tr(s, key), MessageFormat.format(s, key), INVALID_HTML, p));
436                withErrors.put(p, "HTML");
437            }
438            if (checkValues && key != null && value != null && !value.isEmpty() && presetsValueData != null) {
439                if (!isTagIgnored(key, value)) {
440                    if (!isKeyInPresets(key)) {
441                        String prettifiedKey = harmonizeKey(key);
442                        String fixedKey = harmonizedKeys.get(prettifiedKey);
443                        if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) {
444                            // misspelled preset key
445                            String i = marktr("Key ''{0}'' looks like ''{1}''.");
446                            errors.add(new FixableTestError(this, Severity.WARNING, tr("Misspelled property key"),
447                                    tr(i, key, fixedKey),
448                                    MessageFormat.format(i, key, fixedKey), MISSPELLED_KEY, p,
449                                    new ChangePropertyKeyCommand(p, key, fixedKey)));
450                            withErrors.put(p, "WPK");
451                        } else {
452                            String i = marktr("Key ''{0}'' not in presets.");
453                            errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property key"),
454                                    tr(i, key), MessageFormat.format(i, key), INVALID_VALUE, p));
455                            withErrors.put(p, "UPK");
456                        }
457                    } else if (!isTagInPresets(key, value)) {
458                        // try to fix common typos and check again if value is still unknown
459                        String fixedValue = harmonizeValue(prop.getValue());
460                        Map<String, String> possibleValues = getPossibleValues(presetsValueData.get(key));
461                        if (possibleValues.containsKey(fixedValue)) {
462                            fixedValue = possibleValues.get(fixedValue);
463                            // misspelled preset value
464                            String i = marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''.");
465                            errors.add(new FixableTestError(this, Severity.WARNING, tr("Misspelled property value"),
466                                    tr(i, prop.getValue(), key, fixedValue), MessageFormat.format(i, prop.getValue(), fixedValue),
467                                    MISSPELLED_VALUE, p, new ChangePropertyCommand(p, key, fixedValue)));
468                            withErrors.put(p, "WPV");
469                        } else {
470                            // unknown preset value
471                            String i = marktr("Value ''{0}'' for key ''{1}'' not in presets.");
472                            errors.add(new TestError(this, Severity.OTHER, tr("Presets do not contain property value"),
473                                    tr(i, prop.getValue(), key), MessageFormat.format(i, prop.getValue(), key), INVALID_VALUE, p));
474                            withErrors.put(p, "UPV");
475                        }
476                    }
477                }
478            }
479            if (checkFixmes && key != null && value != null && !value.isEmpty()) {
480                if ((value.toLowerCase(Locale.ENGLISH).contains("fixme")
481                        || value.contains("check and delete")
482                        || key.contains("todo") || key.toLowerCase(Locale.ENGLISH).contains("fixme"))
483                        && !withErrors.contains(p, "FIXME")) {
484                    errors.add(new TestError(this, Severity.OTHER,
485                            tr("FIXMES"), FIXME, p));
486                    withErrors.put(p, "FIXME");
487                }
488            }
489        }
490    }
491
492    private static Map<String, String> getPossibleValues(Set<String> values) {
493        // generate a map with common typos
494        Map<String, String> map = new HashMap<>();
495        if (values != null) {
496            for (String value : values) {
497                map.put(value, value);
498                if (value.contains("_")) {
499                    map.put(value.replace("_", ""), value);
500                }
501            }
502        }
503        return map;
504    }
505
506    private static String harmonizeKey(String key) {
507        key = key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_');
508        return Utils.strip(key, "-_;:,");
509    }
510
511    private static String harmonizeValue(String value) {
512        value = value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_');
513        return Utils.strip(value, "-_;:,");
514    }
515
516    @Override
517    public void startTest(ProgressMonitor monitor) {
518        super.startTest(monitor);
519        checkKeys = Main.pref.getBoolean(PREF_CHECK_KEYS, true);
520        if (isBeforeUpload) {
521            checkKeys = checkKeys && Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true);
522        }
523
524        checkValues = Main.pref.getBoolean(PREF_CHECK_VALUES, true);
525        if (isBeforeUpload) {
526            checkValues = checkValues && Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true);
527        }
528
529        checkComplex = Main.pref.getBoolean(PREF_CHECK_COMPLEX, true);
530        if (isBeforeUpload) {
531            checkComplex = checkValues && Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true);
532        }
533
534        checkFixmes = Main.pref.getBoolean(PREF_CHECK_FIXMES, true);
535        if (isBeforeUpload) {
536            checkFixmes = checkFixmes && Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true);
537        }
538    }
539
540    @Override
541    public void visit(Collection<OsmPrimitive> selection) {
542        if (checkKeys || checkValues || checkComplex || checkFixmes) {
543            super.visit(selection);
544        }
545    }
546
547    @Override
548    public void addGui(JPanel testPanel) {
549        GBC a = GBC.eol();
550        a.anchor = GridBagConstraints.EAST;
551
552        testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0));
553
554        prefCheckKeys = new JCheckBox(tr("Check property keys."), Main.pref.getBoolean(PREF_CHECK_KEYS, true));
555        prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words."));
556        testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0));
557
558        prefCheckKeysBeforeUpload = new JCheckBox();
559        prefCheckKeysBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true));
560        testPanel.add(prefCheckKeysBeforeUpload, a);
561
562        prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Main.pref.getBoolean(PREF_CHECK_COMPLEX, true));
563        prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules."));
564        testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0));
565
566        prefCheckComplexBeforeUpload = new JCheckBox();
567        prefCheckComplexBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true));
568        testPanel.add(prefCheckComplexBeforeUpload, a);
569
570        final Collection<String> sources = Main.pref.getCollection(PREF_SOURCES, DEFAULT_SOURCES);
571        sourcesList = new EditableList(tr("TagChecker source"));
572        sourcesList.setItems(sources);
573        testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0));
574        testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0));
575
576        ActionListener disableCheckActionListener = new ActionListener() {
577            @Override
578            public void actionPerformed(ActionEvent e) {
579                handlePrefEnable();
580            }
581        };
582        prefCheckKeys.addActionListener(disableCheckActionListener);
583        prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener);
584        prefCheckComplex.addActionListener(disableCheckActionListener);
585        prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener);
586
587        handlePrefEnable();
588
589        prefCheckValues = new JCheckBox(tr("Check property values."), Main.pref.getBoolean(PREF_CHECK_VALUES, true));
590        prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets."));
591        testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0));
592
593        prefCheckValuesBeforeUpload = new JCheckBox();
594        prefCheckValuesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true));
595        testPanel.add(prefCheckValuesBeforeUpload, a);
596
597        prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Main.pref.getBoolean(PREF_CHECK_FIXMES, true));
598        prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value."));
599        testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0));
600
601        prefCheckFixmesBeforeUpload = new JCheckBox();
602        prefCheckFixmesBeforeUpload.setSelected(Main.pref.getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true));
603        testPanel.add(prefCheckFixmesBeforeUpload, a);
604    }
605
606    public void handlePrefEnable() {
607        boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected()
608                || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected();
609        sourcesList.setEnabled(selected);
610    }
611
612    @Override
613    public boolean ok() {
614        enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected();
615        testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected()
616                || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected();
617
618        Main.pref.put(PREF_CHECK_VALUES, prefCheckValues.isSelected());
619        Main.pref.put(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected());
620        Main.pref.put(PREF_CHECK_KEYS, prefCheckKeys.isSelected());
621        Main.pref.put(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected());
622        Main.pref.put(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected());
623        Main.pref.put(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected());
624        Main.pref.put(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected());
625        Main.pref.put(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected());
626        return Main.pref.putCollection(PREF_SOURCES, sourcesList.getItems());
627    }
628
629    @Override
630    public Command fixError(TestError testError) {
631        List<Command> commands = new ArrayList<>(50);
632
633        if (testError instanceof FixableTestError) {
634            commands.add(testError.getFix());
635        } else {
636            Collection<? extends OsmPrimitive> primitives = testError.getPrimitives();
637            for (OsmPrimitive p : primitives) {
638                Map<String, String> tags = p.getKeys();
639                if (tags == null || tags.isEmpty()) {
640                    continue;
641                }
642
643                for (Entry<String, String> prop: tags.entrySet()) {
644                    String key = prop.getKey();
645                    String value = prop.getValue();
646                    if (value == null || value.trim().isEmpty()) {
647                        commands.add(new ChangePropertyCommand(p, key, null));
648                    } else if (value.startsWith(" ") || value.endsWith(" ")) {
649                        commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value)));
650                    } else if (key.startsWith(" ") || key.endsWith(" ")) {
651                        commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key)));
652                    } else {
653                        String evalue = Entities.unescape(value);
654                        if (!evalue.equals(value)) {
655                            commands.add(new ChangePropertyCommand(p, key, evalue));
656                        }
657                    }
658                }
659            }
660        }
661
662        if (commands.isEmpty())
663            return null;
664        if (commands.size() == 1)
665            return commands.get(0);
666
667        return new SequenceCommand(tr("Fix tags"), commands);
668    }
669
670    @Override
671    public boolean isFixable(TestError testError) {
672        if (testError.getTester() instanceof TagChecker) {
673            int code = testError.getCode();
674            return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE ||
675                   code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE;
676        }
677
678        return false;
679    }
680
681    protected static class CheckerData {
682        private String description;
683        protected List<CheckerElement> data = new ArrayList<>();
684        private OsmPrimitiveType type;
685        private int code;
686        protected Severity severity;
687        protected static final int TAG_CHECK_ERROR  = 1250;
688        protected static final int TAG_CHECK_WARN   = 1260;
689        protected static final int TAG_CHECK_INFO   = 1270;
690
691        protected static class CheckerElement {
692            public Object tag;
693            public Object value;
694            public boolean noMatch;
695            public boolean tagAll;
696            public boolean valueAll;
697            public boolean valueBool;
698
699            private static Pattern getPattern(String str) throws PatternSyntaxException {
700                if (str.endsWith("/i"))
701                    return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE);
702                if (str.endsWith("/"))
703                    return Pattern.compile(str.substring(1, str.length()-1));
704
705                throw new IllegalStateException();
706            }
707
708            public CheckerElement(String exp) throws PatternSyntaxException {
709                Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp);
710                m.matches();
711
712                String n = m.group(1).trim();
713
714                if ("*".equals(n)) {
715                    tagAll = true;
716                } else {
717                    tag = n.startsWith("/") ? getPattern(n) : n;
718                    noMatch = "!=".equals(m.group(2));
719                    n = m.group(3).trim();
720                    if ("*".equals(n)) {
721                        valueAll = true;
722                    } else if ("BOOLEAN_TRUE".equals(n)) {
723                        valueBool = true;
724                        value = OsmUtils.trueval;
725                    } else if ("BOOLEAN_FALSE".equals(n)) {
726                        valueBool = true;
727                        value = OsmUtils.falseval;
728                    } else {
729                        value = n.startsWith("/") ? getPattern(n) : n;
730                    }
731                }
732            }
733
734            public boolean match(Map<String, String> keys) {
735                for (Entry<String, String> prop: keys.entrySet()) {
736                    String key = prop.getKey();
737                    String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue();
738                    if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag)))
739                            && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value))))
740                        return !noMatch;
741                }
742                return noMatch;
743            }
744        }
745
746        private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$");
747        private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *");
748        private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *");
749
750        public String getData(final String str) {
751            Matcher m = CLEAN_STR_PATTERN.matcher(str);
752            String trimmed = m.replaceFirst("").trim();
753            try {
754                description = m.group(1);
755                if (description != null && description.isEmpty()) {
756                    description = null;
757                }
758            } catch (IllegalStateException e) {
759                description = null;
760            }
761            String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3);
762            switch (n[0]) {
763            case "way":
764                type = OsmPrimitiveType.WAY;
765                break;
766            case "node":
767                type = OsmPrimitiveType.NODE;
768                break;
769            case "relation":
770                type = OsmPrimitiveType.RELATION;
771                break;
772            case "*":
773                type = null;
774                break;
775            default:
776                return tr("Could not find element type");
777            }
778            if (n.length != 3)
779                return tr("Incorrect number of parameters");
780
781            switch (n[1]) {
782            case "W":
783                severity = Severity.WARNING;
784                code = TAG_CHECK_WARN;
785                break;
786            case "E":
787                severity = Severity.ERROR;
788                code = TAG_CHECK_ERROR;
789                break;
790            case "I":
791                severity = Severity.OTHER;
792                code = TAG_CHECK_INFO;
793                break;
794            default:
795                return tr("Could not find warning level");
796            }
797            for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) {
798                try {
799                    data.add(new CheckerElement(exp));
800                } catch (IllegalStateException e) {
801                    return tr("Illegal expression ''{0}''", exp);
802                } catch (PatternSyntaxException e) {
803                    return tr("Illegal regular expression ''{0}''", exp);
804                }
805            }
806            return null;
807        }
808
809        public boolean match(OsmPrimitive osm, Map<String, String> keys) {
810            if (type != null && OsmPrimitiveType.from(osm) != type)
811                return false;
812
813            for (CheckerElement ce : data) {
814                if (!ce.match(keys))
815                    return false;
816            }
817            return true;
818        }
819
820        public String getDescription() {
821            return tr(description);
822        }
823
824        public String getDescriptionOrig() {
825            return description;
826        }
827
828        public Severity getSeverity() {
829            return severity;
830        }
831
832        public int getCode() {
833            if (type == null)
834                return code;
835
836            return code + type.ordinal() + 1;
837        }
838    }
839}