001// License: GPL. For details, see LICENSE file. 002package org.openstreetmap.josm.data.validation.tests; 003 004import static org.openstreetmap.josm.tools.I18n.marktr; 005import static org.openstreetmap.josm.tools.I18n.tr; 006 007import java.awt.GridBagConstraints; 008import java.awt.event.ActionListener; 009import java.io.BufferedReader; 010import java.io.IOException; 011import java.util.ArrayList; 012import java.util.Arrays; 013import java.util.Collection; 014import java.util.HashMap; 015import java.util.List; 016import java.util.Locale; 017import java.util.Map; 018import java.util.Map.Entry; 019import java.util.Set; 020import java.util.regex.Matcher; 021import java.util.regex.Pattern; 022import java.util.regex.PatternSyntaxException; 023 024import javax.swing.JCheckBox; 025import javax.swing.JLabel; 026import javax.swing.JPanel; 027 028import org.openstreetmap.josm.command.ChangePropertyCommand; 029import org.openstreetmap.josm.command.ChangePropertyKeyCommand; 030import org.openstreetmap.josm.command.Command; 031import org.openstreetmap.josm.command.SequenceCommand; 032import org.openstreetmap.josm.data.osm.OsmPrimitive; 033import org.openstreetmap.josm.data.osm.OsmPrimitiveType; 034import org.openstreetmap.josm.data.osm.OsmUtils; 035import org.openstreetmap.josm.data.osm.Tag; 036import org.openstreetmap.josm.data.osm.Tagged; 037import org.openstreetmap.josm.data.preferences.sources.ValidatorPrefHelper; 038import org.openstreetmap.josm.data.validation.Severity; 039import org.openstreetmap.josm.data.validation.Test.TagTest; 040import org.openstreetmap.josm.data.validation.TestError; 041import org.openstreetmap.josm.data.validation.util.Entities; 042import org.openstreetmap.josm.gui.progress.ProgressMonitor; 043import org.openstreetmap.josm.gui.tagging.presets.TaggingPreset; 044import org.openstreetmap.josm.gui.tagging.presets.TaggingPresetItem; 045import org.openstreetmap.josm.gui.tagging.presets.TaggingPresets; 046import org.openstreetmap.josm.gui.tagging.presets.items.Check; 047import org.openstreetmap.josm.gui.tagging.presets.items.CheckGroup; 048import org.openstreetmap.josm.gui.tagging.presets.items.KeyedItem; 049import org.openstreetmap.josm.gui.widgets.EditableList; 050import org.openstreetmap.josm.io.CachedFile; 051import org.openstreetmap.josm.spi.preferences.Config; 052import org.openstreetmap.josm.tools.GBC; 053import org.openstreetmap.josm.tools.Logging; 054import org.openstreetmap.josm.tools.MultiMap; 055import org.openstreetmap.josm.tools.Utils; 056 057/** 058 * Check for misspelled or wrong tags 059 * 060 * @author frsantos 061 * @since 3669 062 */ 063public class TagChecker extends TagTest { 064 065 /** The config file of ignored tags */ 066 public static final String IGNORE_FILE = "resource://data/validator/ignoretags.cfg"; 067 /** The config file of dictionary words */ 068 public static final String SPELL_FILE = "resource://data/validator/words.cfg"; 069 070 /** Normalized keys: the key should be substituted by the value if the key was not found in presets */ 071 private static final Map<String, String> harmonizedKeys = new HashMap<>(); 072 /** The spell check preset values which are not stored in TaggingPresets */ 073 private static volatile MultiMap<String, String> additionalPresetsValueData; 074 /** The TagChecker data */ 075 private static final List<CheckerData> checkerData = new ArrayList<>(); 076 private static final List<String> ignoreDataStartsWith = new ArrayList<>(); 077 private static final List<String> ignoreDataEquals = new ArrayList<>(); 078 private static final List<String> ignoreDataEndsWith = new ArrayList<>(); 079 private static final List<Tag> ignoreDataTag = new ArrayList<>(); 080 081 /** The preferences prefix */ 082 protected static final String PREFIX = ValidatorPrefHelper.PREFIX + "." + TagChecker.class.getSimpleName(); 083 084 /** 085 * The preference key to check values 086 */ 087 public static final String PREF_CHECK_VALUES = PREFIX + ".checkValues"; 088 /** 089 * The preference key to check keys 090 */ 091 public static final String PREF_CHECK_KEYS = PREFIX + ".checkKeys"; 092 /** 093 * The preference key to enable complex checks 094 */ 095 public static final String PREF_CHECK_COMPLEX = PREFIX + ".checkComplex"; 096 /** 097 * The preference key to search for fixme tags 098 */ 099 public static final String PREF_CHECK_FIXMES = PREFIX + ".checkFixmes"; 100 101 /** 102 * The preference key for source files 103 * @see #DEFAULT_SOURCES 104 */ 105 public static final String PREF_SOURCES = PREFIX + ".source"; 106 107 /** 108 * The preference key to check keys - used before upload 109 */ 110 public static final String PREF_CHECK_KEYS_BEFORE_UPLOAD = PREF_CHECK_KEYS + "BeforeUpload"; 111 /** 112 * The preference key to check values - used before upload 113 */ 114 public static final String PREF_CHECK_VALUES_BEFORE_UPLOAD = PREF_CHECK_VALUES + "BeforeUpload"; 115 /** 116 * The preference key to run complex tests - used before upload 117 */ 118 public static final String PREF_CHECK_COMPLEX_BEFORE_UPLOAD = PREF_CHECK_COMPLEX + "BeforeUpload"; 119 /** 120 * The preference key to search for fixmes - used before upload 121 */ 122 public static final String PREF_CHECK_FIXMES_BEFORE_UPLOAD = PREF_CHECK_FIXMES + "BeforeUpload"; 123 124 protected boolean checkKeys; 125 protected boolean checkValues; 126 protected boolean checkComplex; 127 protected boolean checkFixmes; 128 129 protected JCheckBox prefCheckKeys; 130 protected JCheckBox prefCheckValues; 131 protected JCheckBox prefCheckComplex; 132 protected JCheckBox prefCheckFixmes; 133 protected JCheckBox prefCheckPaint; 134 135 protected JCheckBox prefCheckKeysBeforeUpload; 136 protected JCheckBox prefCheckValuesBeforeUpload; 137 protected JCheckBox prefCheckComplexBeforeUpload; 138 protected JCheckBox prefCheckFixmesBeforeUpload; 139 protected JCheckBox prefCheckPaintBeforeUpload; 140 141 // CHECKSTYLE.OFF: SingleSpaceSeparator 142 protected static final int EMPTY_VALUES = 1200; 143 protected static final int INVALID_KEY = 1201; 144 protected static final int INVALID_VALUE = 1202; 145 protected static final int FIXME = 1203; 146 protected static final int INVALID_SPACE = 1204; 147 protected static final int INVALID_KEY_SPACE = 1205; 148 protected static final int INVALID_HTML = 1206; /* 1207 was PAINT */ 149 protected static final int LONG_VALUE = 1208; 150 protected static final int LONG_KEY = 1209; 151 protected static final int LOW_CHAR_VALUE = 1210; 152 protected static final int LOW_CHAR_KEY = 1211; 153 protected static final int MISSPELLED_VALUE = 1212; 154 protected static final int MISSPELLED_KEY = 1213; 155 protected static final int MULTIPLE_SPACES = 1214; 156 // CHECKSTYLE.ON: SingleSpaceSeparator 157 // 1250 and up is used by tagcheck 158 159 protected EditableList sourcesList; 160 161 private static final List<String> DEFAULT_SOURCES = Arrays.asList(/*DATA_FILE, */IGNORE_FILE, SPELL_FILE); 162 163 /** 164 * Constructor 165 */ 166 public TagChecker() { 167 super(tr("Tag checker"), tr("This test checks for errors in tag keys and values.")); 168 } 169 170 @Override 171 public void initialize() throws IOException { 172 initializeData(); 173 initializePresets(); 174 } 175 176 /** 177 * Reads the spellcheck file into a HashMap. 178 * The data file is a list of words, beginning with +/-. If it starts with +, 179 * the word is valid, but if it starts with -, the word should be replaced 180 * by the nearest + word before this. 181 * 182 * @throws IOException if any I/O error occurs 183 */ 184 private static void initializeData() throws IOException { 185 checkerData.clear(); 186 ignoreDataStartsWith.clear(); 187 ignoreDataEquals.clear(); 188 ignoreDataEndsWith.clear(); 189 ignoreDataTag.clear(); 190 harmonizedKeys.clear(); 191 192 StringBuilder errorSources = new StringBuilder(); 193 for (String source : Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES)) { 194 try ( 195 CachedFile cf = new CachedFile(source); 196 BufferedReader reader = cf.getContentReader() 197 ) { 198 String okValue = null; 199 boolean tagcheckerfile = false; 200 boolean ignorefile = false; 201 boolean isFirstLine = true; 202 String line; 203 while ((line = reader.readLine()) != null && (tagcheckerfile || !line.isEmpty())) { 204 if (line.startsWith("#")) { 205 if (line.startsWith("# JOSM TagChecker")) { 206 tagcheckerfile = true; 207 if (!DEFAULT_SOURCES.contains(source)) { 208 Logging.info(tr("Adding {0} to tag checker", source)); 209 } 210 } else 211 if (line.startsWith("# JOSM IgnoreTags")) { 212 ignorefile = true; 213 if (!DEFAULT_SOURCES.contains(source)) { 214 Logging.info(tr("Adding {0} to ignore tags", source)); 215 } 216 } 217 } else if (ignorefile) { 218 line = line.trim(); 219 if (line.length() < 4) { 220 continue; 221 } 222 223 String key = line.substring(0, 2); 224 line = line.substring(2); 225 226 switch (key) { 227 case "S:": 228 ignoreDataStartsWith.add(line); 229 break; 230 case "E:": 231 ignoreDataEquals.add(line); 232 break; 233 case "F:": 234 ignoreDataEndsWith.add(line); 235 break; 236 case "K:": 237 ignoreDataTag.add(Tag.ofString(line)); 238 break; 239 default: 240 if (!key.startsWith(";")) { 241 Logging.warn("Unsupported TagChecker key: " + key); 242 } 243 } 244 } else if (tagcheckerfile) { 245 if (!line.isEmpty()) { 246 CheckerData d = new CheckerData(); 247 String err = d.getData(line); 248 249 if (err == null) { 250 checkerData.add(d); 251 } else { 252 Logging.error(tr("Invalid tagchecker line - {0}: {1}", err, line)); 253 } 254 } 255 } else if (line.charAt(0) == '+') { 256 okValue = line.substring(1); 257 } else if (line.charAt(0) == '-' && okValue != null) { 258 harmonizedKeys.put(harmonizeKey(line.substring(1)), okValue); 259 } else { 260 Logging.error(tr("Invalid spellcheck line: {0}", line)); 261 } 262 if (isFirstLine) { 263 isFirstLine = false; 264 if (!(tagcheckerfile || ignorefile) && !DEFAULT_SOURCES.contains(source)) { 265 Logging.info(tr("Adding {0} to spellchecker", source)); 266 } 267 } 268 } 269 } catch (IOException e) { 270 Logging.error(e); 271 errorSources.append(source).append('\n'); 272 } 273 } 274 275 if (errorSources.length() > 0) 276 throw new IOException(tr("Could not access data file(s):\n{0}", errorSources)); 277 } 278 279 /** 280 * Reads the presets data. 281 * 282 */ 283 public static void initializePresets() { 284 285 if (!Config.getPref().getBoolean(PREF_CHECK_VALUES, true)) 286 return; 287 288 Collection<TaggingPreset> presets = TaggingPresets.getTaggingPresets(); 289 if (!presets.isEmpty()) { 290 additionalPresetsValueData = new MultiMap<>(); 291 for (String a : OsmPrimitive.getUninterestingKeys()) { 292 additionalPresetsValueData.putVoid(a); 293 } 294 // TODO directionKeys are no longer in OsmPrimitive (search pattern is used instead) 295 for (String a : Config.getPref().getList(ValidatorPrefHelper.PREFIX + ".knownkeys", 296 Arrays.asList("is_in", "int_ref", "fixme", "population"))) { 297 additionalPresetsValueData.putVoid(a); 298 } 299 for (TaggingPreset p : presets) { 300 for (TaggingPresetItem i : p.data) { 301 if (i instanceof KeyedItem) { 302 addPresetValue((KeyedItem) i); 303 } else if (i instanceof CheckGroup) { 304 for (Check c : ((CheckGroup) i).checks) { 305 addPresetValue(c); 306 } 307 } 308 } 309 } 310 } 311 } 312 313 private static void addPresetValue(KeyedItem ky) { 314 Collection<String> values = ky.getValues(); 315 if (ky.key != null && values != null) { 316 harmonizedKeys.put(harmonizeKey(ky.key), ky.key); 317 } 318 } 319 320 /** 321 * Checks given string (key or value) if it contains characters with code below 0x20 (either newline or some other special characters) 322 * @param s string to check 323 * @return {@code true} if {@code s} contains characters with code below 0x20 324 */ 325 private static boolean containsLow(String s) { 326 if (s == null) 327 return false; 328 for (int i = 0; i < s.length(); i++) { 329 if (s.charAt(i) < 0x20) 330 return true; 331 } 332 return false; 333 } 334 335 private static Set<String> getPresetValues(String key) { 336 Set<String> res = TaggingPresets.getPresetValues(key); 337 if (res != null) 338 return res; 339 return additionalPresetsValueData.get(key); 340 } 341 342 /** 343 * Determines if the given key is in internal presets. 344 * @param key key 345 * @return {@code true} if the given key is in internal presets 346 * @since 9023 347 */ 348 public static boolean isKeyInPresets(String key) { 349 return getPresetValues(key) != null; 350 } 351 352 /** 353 * Determines if the given tag is in internal presets. 354 * @param key key 355 * @param value value 356 * @return {@code true} if the given tag is in internal presets 357 * @since 9023 358 */ 359 public static boolean isTagInPresets(String key, String value) { 360 final Set<String> values = getPresetValues(key); 361 return values != null && (values.isEmpty() || values.contains(value)); 362 } 363 364 /** 365 * Returns the list of ignored tags. 366 * @return the list of ignored tags 367 * @since 9023 368 */ 369 public static List<Tag> getIgnoredTags() { 370 return new ArrayList<>(ignoreDataTag); 371 } 372 373 /** 374 * Determines if the given tag is ignored for checks "key/tag not in presets". 375 * @param key key 376 * @param value value 377 * @return {@code true} if the given tag is ignored 378 * @since 9023 379 */ 380 public static boolean isTagIgnored(String key, String value) { 381 boolean tagInPresets = isTagInPresets(key, value); 382 boolean ignore = false; 383 384 for (String a : ignoreDataStartsWith) { 385 if (key.startsWith(a)) { 386 ignore = true; 387 } 388 } 389 for (String a : ignoreDataEquals) { 390 if (key.equals(a)) { 391 ignore = true; 392 } 393 } 394 for (String a : ignoreDataEndsWith) { 395 if (key.endsWith(a)) { 396 ignore = true; 397 } 398 } 399 400 if (!tagInPresets) { 401 for (Tag a : ignoreDataTag) { 402 if (key.equals(a.getKey()) && value.equals(a.getValue())) { 403 ignore = true; 404 } 405 } 406 } 407 return ignore; 408 } 409 410 /** 411 * Checks the primitive tags 412 * @param p The primitive to check 413 */ 414 @Override 415 public void check(OsmPrimitive p) { 416 // Just a collection to know if a primitive has been already marked with error 417 MultiMap<OsmPrimitive, String> withErrors = new MultiMap<>(); 418 419 if (checkComplex) { 420 Map<String, String> keys = p.getKeys(); 421 for (CheckerData d : checkerData) { 422 if (d.match(p, keys)) { 423 errors.add(TestError.builder(this, d.getSeverity(), d.getCode()) 424 .message(tr("Suspicious tag/value combinations"), d.getDescription()) 425 .primitives(p) 426 .build()); 427 withErrors.put(p, "TC"); 428 } 429 } 430 } 431 432 for (Entry<String, String> prop : p.getKeys().entrySet()) { 433 String s = marktr("Key ''{0}'' invalid."); 434 String key = prop.getKey(); 435 String value = prop.getValue(); 436 if (checkValues && (containsLow(value)) && !withErrors.contains(p, "ICV")) { 437 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_VALUE) 438 .message(tr("Tag value contains character with code less than 0x20"), s, key) 439 .primitives(p) 440 .build()); 441 withErrors.put(p, "ICV"); 442 } 443 if (checkKeys && (containsLow(key)) && !withErrors.contains(p, "ICK")) { 444 errors.add(TestError.builder(this, Severity.WARNING, LOW_CHAR_KEY) 445 .message(tr("Tag key contains character with code less than 0x20"), s, key) 446 .primitives(p) 447 .build()); 448 withErrors.put(p, "ICK"); 449 } 450 if (checkValues && (value != null && value.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LV")) { 451 errors.add(TestError.builder(this, Severity.ERROR, LONG_VALUE) 452 .message(tr("Tag value longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, value.length()), s, key) 453 .primitives(p) 454 .build()); 455 withErrors.put(p, "LV"); 456 } 457 if (checkKeys && (key != null && key.length() > Tagged.MAX_TAG_LENGTH) && !withErrors.contains(p, "LK")) { 458 errors.add(TestError.builder(this, Severity.ERROR, LONG_KEY) 459 .message(tr("Tag key longer than {0} characters ({1} characters)", Tagged.MAX_TAG_LENGTH, key.length()), s, key) 460 .primitives(p) 461 .build()); 462 withErrors.put(p, "LK"); 463 } 464 if (checkValues && (value == null || value.trim().isEmpty()) && !withErrors.contains(p, "EV")) { 465 errors.add(TestError.builder(this, Severity.WARNING, EMPTY_VALUES) 466 .message(tr("Tags with empty values"), s, key) 467 .primitives(p) 468 .build()); 469 withErrors.put(p, "EV"); 470 } 471 if (checkKeys && key != null && key.indexOf(' ') >= 0 && !withErrors.contains(p, "IPK")) { 472 errors.add(TestError.builder(this, Severity.WARNING, INVALID_KEY_SPACE) 473 .message(tr("Invalid white space in property key"), s, key) 474 .primitives(p) 475 .build()); 476 withErrors.put(p, "IPK"); 477 } 478 if (checkValues && value != null && (value.startsWith(" ") || value.endsWith(" ")) && !withErrors.contains(p, "SPACE")) { 479 errors.add(TestError.builder(this, Severity.WARNING, INVALID_SPACE) 480 .message(tr("Property values start or end with white space"), s, key) 481 .primitives(p) 482 .build()); 483 withErrors.put(p, "SPACE"); 484 } 485 if (checkValues && value != null && value.contains(" ") && !withErrors.contains(p, "SPACE")) { 486 errors.add(TestError.builder(this, Severity.WARNING, MULTIPLE_SPACES) 487 .message(tr("Property values contain multiple white spaces"), s, key) 488 .primitives(p) 489 .build()); 490 withErrors.put(p, "SPACE"); 491 } 492 if (checkValues && value != null && !value.equals(Entities.unescape(value)) && !withErrors.contains(p, "HTML")) { 493 errors.add(TestError.builder(this, Severity.OTHER, INVALID_HTML) 494 .message(tr("Property values contain HTML entity"), s, key) 495 .primitives(p) 496 .build()); 497 withErrors.put(p, "HTML"); 498 } 499 if (checkValues && key != null && value != null && !value.isEmpty() && additionalPresetsValueData != null 500 && !isTagIgnored(key, value)) { 501 if (!isKeyInPresets(key)) { 502 String prettifiedKey = harmonizeKey(key); 503 String fixedKey = harmonizedKeys.get(prettifiedKey); 504 if (fixedKey != null && !"".equals(fixedKey) && !fixedKey.equals(key)) { 505 // misspelled preset key 506 final TestError.Builder error = TestError.builder(this, Severity.WARNING, MISSPELLED_KEY) 507 .message(tr("Misspelled property key"), marktr("Key ''{0}'' looks like ''{1}''."), key, fixedKey) 508 .primitives(p); 509 if (p.hasKey(fixedKey)) { 510 errors.add(error.build()); 511 } else { 512 errors.add(error.fix(() -> new ChangePropertyKeyCommand(p, key, fixedKey)).build()); 513 } 514 withErrors.put(p, "WPK"); 515 } else { 516 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE) 517 .message(tr("Presets do not contain property key"), marktr("Key ''{0}'' not in presets."), key) 518 .primitives(p) 519 .build()); 520 withErrors.put(p, "UPK"); 521 } 522 } else if (!isTagInPresets(key, value)) { 523 // try to fix common typos and check again if value is still unknown 524 String fixedValue = harmonizeValue(prop.getValue()); 525 Map<String, String> possibleValues = getPossibleValues(getPresetValues(key)); 526 if (possibleValues.containsKey(fixedValue)) { 527 final String newKey = possibleValues.get(fixedValue); 528 // misspelled preset value 529 errors.add(TestError.builder(this, Severity.WARNING, MISSPELLED_VALUE) 530 .message(tr("Misspelled property value"), 531 marktr("Value ''{0}'' for key ''{1}'' looks like ''{2}''."), prop.getValue(), key, fixedValue) 532 .primitives(p) 533 .fix(() -> new ChangePropertyCommand(p, key, newKey)) 534 .build()); 535 withErrors.put(p, "WPV"); 536 } else { 537 // unknown preset value 538 errors.add(TestError.builder(this, Severity.OTHER, INVALID_VALUE) 539 .message(tr("Presets do not contain property value"), 540 marktr("Value ''{0}'' for key ''{1}'' not in presets."), prop.getValue(), key) 541 .primitives(p) 542 .build()); 543 withErrors.put(p, "UPV"); 544 } 545 } 546 } 547 if (checkFixmes && key != null && value != null && !value.isEmpty() && isFixme(key, value) && !withErrors.contains(p, "FIXME")) { 548 errors.add(TestError.builder(this, Severity.OTHER, FIXME) 549 .message(tr("FIXMES")) 550 .primitives(p) 551 .build()); 552 withErrors.put(p, "FIXME"); 553 } 554 } 555 } 556 557 private static boolean isFixme(String key, String value) { 558 return key.toLowerCase(Locale.ENGLISH).contains("fixme") || key.contains("todo") 559 || value.toLowerCase(Locale.ENGLISH).contains("fixme") || value.contains("check and delete"); 560 } 561 562 private static Map<String, String> getPossibleValues(Set<String> values) { 563 // generate a map with common typos 564 Map<String, String> map = new HashMap<>(); 565 if (values != null) { 566 for (String value : values) { 567 map.put(value, value); 568 if (value.contains("_")) { 569 map.put(value.replace("_", ""), value); 570 } 571 } 572 } 573 return map; 574 } 575 576 private static String harmonizeKey(String key) { 577 return Utils.strip(key.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(':', '_').replace(' ', '_'), "-_;:,"); 578 } 579 580 private static String harmonizeValue(String value) { 581 return Utils.strip(value.toLowerCase(Locale.ENGLISH).replace('-', '_').replace(' ', '_'), "-_;:,"); 582 } 583 584 @Override 585 public void startTest(ProgressMonitor monitor) { 586 super.startTest(monitor); 587 checkKeys = Config.getPref().getBoolean(PREF_CHECK_KEYS, true); 588 if (isBeforeUpload) { 589 checkKeys = checkKeys && Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true); 590 } 591 592 checkValues = Config.getPref().getBoolean(PREF_CHECK_VALUES, true); 593 if (isBeforeUpload) { 594 checkValues = checkValues && Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true); 595 } 596 597 checkComplex = Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true); 598 if (isBeforeUpload) { 599 checkComplex = checkComplex && Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true); 600 } 601 602 checkFixmes = Config.getPref().getBoolean(PREF_CHECK_FIXMES, true); 603 if (isBeforeUpload) { 604 checkFixmes = checkFixmes && Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true); 605 } 606 } 607 608 @Override 609 public void visit(Collection<OsmPrimitive> selection) { 610 if (checkKeys || checkValues || checkComplex || checkFixmes) { 611 super.visit(selection); 612 } 613 } 614 615 @Override 616 public void addGui(JPanel testPanel) { 617 GBC a = GBC.eol(); 618 a.anchor = GridBagConstraints.EAST; 619 620 testPanel.add(new JLabel(name+" :"), GBC.eol().insets(3, 0, 0, 0)); 621 622 prefCheckKeys = new JCheckBox(tr("Check property keys."), Config.getPref().getBoolean(PREF_CHECK_KEYS, true)); 623 prefCheckKeys.setToolTipText(tr("Validate that property keys are valid checking against list of words.")); 624 testPanel.add(prefCheckKeys, GBC.std().insets(20, 0, 0, 0)); 625 626 prefCheckKeysBeforeUpload = new JCheckBox(); 627 prefCheckKeysBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, true)); 628 testPanel.add(prefCheckKeysBeforeUpload, a); 629 630 prefCheckComplex = new JCheckBox(tr("Use complex property checker."), Config.getPref().getBoolean(PREF_CHECK_COMPLEX, true)); 631 prefCheckComplex.setToolTipText(tr("Validate property values and tags using complex rules.")); 632 testPanel.add(prefCheckComplex, GBC.std().insets(20, 0, 0, 0)); 633 634 prefCheckComplexBeforeUpload = new JCheckBox(); 635 prefCheckComplexBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, true)); 636 testPanel.add(prefCheckComplexBeforeUpload, a); 637 638 final Collection<String> sources = Config.getPref().getList(PREF_SOURCES, DEFAULT_SOURCES); 639 sourcesList = new EditableList(tr("TagChecker source")); 640 sourcesList.setItems(sources); 641 testPanel.add(new JLabel(tr("Data sources ({0})", "*.cfg")), GBC.eol().insets(23, 0, 0, 0)); 642 testPanel.add(sourcesList, GBC.eol().fill(GridBagConstraints.HORIZONTAL).insets(23, 0, 0, 0)); 643 644 ActionListener disableCheckActionListener = e -> handlePrefEnable(); 645 prefCheckKeys.addActionListener(disableCheckActionListener); 646 prefCheckKeysBeforeUpload.addActionListener(disableCheckActionListener); 647 prefCheckComplex.addActionListener(disableCheckActionListener); 648 prefCheckComplexBeforeUpload.addActionListener(disableCheckActionListener); 649 650 handlePrefEnable(); 651 652 prefCheckValues = new JCheckBox(tr("Check property values."), Config.getPref().getBoolean(PREF_CHECK_VALUES, true)); 653 prefCheckValues.setToolTipText(tr("Validate that property values are valid checking against presets.")); 654 testPanel.add(prefCheckValues, GBC.std().insets(20, 0, 0, 0)); 655 656 prefCheckValuesBeforeUpload = new JCheckBox(); 657 prefCheckValuesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, true)); 658 testPanel.add(prefCheckValuesBeforeUpload, a); 659 660 prefCheckFixmes = new JCheckBox(tr("Check for FIXMES."), Config.getPref().getBoolean(PREF_CHECK_FIXMES, true)); 661 prefCheckFixmes.setToolTipText(tr("Looks for nodes or ways with FIXME in any property value.")); 662 testPanel.add(prefCheckFixmes, GBC.std().insets(20, 0, 0, 0)); 663 664 prefCheckFixmesBeforeUpload = new JCheckBox(); 665 prefCheckFixmesBeforeUpload.setSelected(Config.getPref().getBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, true)); 666 testPanel.add(prefCheckFixmesBeforeUpload, a); 667 } 668 669 /** 670 * Enables/disables the source list field 671 */ 672 public void handlePrefEnable() { 673 boolean selected = prefCheckKeys.isSelected() || prefCheckKeysBeforeUpload.isSelected() 674 || prefCheckComplex.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 675 sourcesList.setEnabled(selected); 676 } 677 678 @Override 679 public boolean ok() { 680 enabled = prefCheckKeys.isSelected() || prefCheckValues.isSelected() || prefCheckComplex.isSelected() || prefCheckFixmes.isSelected(); 681 testBeforeUpload = prefCheckKeysBeforeUpload.isSelected() || prefCheckValuesBeforeUpload.isSelected() 682 || prefCheckFixmesBeforeUpload.isSelected() || prefCheckComplexBeforeUpload.isSelected(); 683 684 Config.getPref().putBoolean(PREF_CHECK_VALUES, prefCheckValues.isSelected()); 685 Config.getPref().putBoolean(PREF_CHECK_COMPLEX, prefCheckComplex.isSelected()); 686 Config.getPref().putBoolean(PREF_CHECK_KEYS, prefCheckKeys.isSelected()); 687 Config.getPref().putBoolean(PREF_CHECK_FIXMES, prefCheckFixmes.isSelected()); 688 Config.getPref().putBoolean(PREF_CHECK_VALUES_BEFORE_UPLOAD, prefCheckValuesBeforeUpload.isSelected()); 689 Config.getPref().putBoolean(PREF_CHECK_COMPLEX_BEFORE_UPLOAD, prefCheckComplexBeforeUpload.isSelected()); 690 Config.getPref().putBoolean(PREF_CHECK_KEYS_BEFORE_UPLOAD, prefCheckKeysBeforeUpload.isSelected()); 691 Config.getPref().putBoolean(PREF_CHECK_FIXMES_BEFORE_UPLOAD, prefCheckFixmesBeforeUpload.isSelected()); 692 return Config.getPref().putList(PREF_SOURCES, sourcesList.getItems()); 693 } 694 695 @Override 696 public Command fixError(TestError testError) { 697 List<Command> commands = new ArrayList<>(50); 698 699 Collection<? extends OsmPrimitive> primitives = testError.getPrimitives(); 700 for (OsmPrimitive p : primitives) { 701 Map<String, String> tags = p.getKeys(); 702 if (tags.isEmpty()) { 703 continue; 704 } 705 706 for (Entry<String, String> prop: tags.entrySet()) { 707 String key = prop.getKey(); 708 String value = prop.getValue(); 709 if (value == null || value.trim().isEmpty()) { 710 commands.add(new ChangePropertyCommand(p, key, null)); 711 } else if (value.startsWith(" ") || value.endsWith(" ") || value.contains(" ")) { 712 commands.add(new ChangePropertyCommand(p, key, Tag.removeWhiteSpaces(value))); 713 } else if (key.startsWith(" ") || key.endsWith(" ") || key.contains(" ")) { 714 commands.add(new ChangePropertyKeyCommand(p, key, Tag.removeWhiteSpaces(key))); 715 } else { 716 String evalue = Entities.unescape(value); 717 if (!evalue.equals(value)) { 718 commands.add(new ChangePropertyCommand(p, key, evalue)); 719 } 720 } 721 } 722 } 723 724 if (commands.isEmpty()) 725 return null; 726 if (commands.size() == 1) 727 return commands.get(0); 728 729 return new SequenceCommand(tr("Fix tags"), commands); 730 } 731 732 @Override 733 public boolean isFixable(TestError testError) { 734 if (testError.getTester() instanceof TagChecker) { 735 int code = testError.getCode(); 736 return code == INVALID_KEY || code == EMPTY_VALUES || code == INVALID_SPACE || 737 code == INVALID_KEY_SPACE || code == INVALID_HTML || code == MISSPELLED_VALUE || 738 code == MULTIPLE_SPACES; 739 } 740 741 return false; 742 } 743 744 protected static class CheckerData { 745 private String description; 746 protected List<CheckerElement> data = new ArrayList<>(); 747 private OsmPrimitiveType type; 748 private TagCheckLevel level; 749 protected Severity severity; 750 751 private enum TagCheckLevel { 752 TAG_CHECK_ERROR(1250), 753 TAG_CHECK_WARN(1260), 754 TAG_CHECK_INFO(1270); 755 756 final int code; 757 758 TagCheckLevel(int code) { 759 this.code = code; 760 } 761 } 762 763 protected static class CheckerElement { 764 public Object tag; 765 public Object value; 766 public boolean noMatch; 767 public boolean tagAll; 768 public boolean valueAll; 769 public boolean valueBool; 770 771 private static Pattern getPattern(String str) { 772 if (str.endsWith("/i")) 773 return Pattern.compile(str.substring(1, str.length()-2), Pattern.CASE_INSENSITIVE); 774 if (str.endsWith("/")) 775 return Pattern.compile(str.substring(1, str.length()-1)); 776 777 throw new IllegalStateException(); 778 } 779 780 public CheckerElement(String exp) { 781 Matcher m = Pattern.compile("(.+)([!=]=)(.+)").matcher(exp); 782 m.matches(); 783 784 String n = m.group(1).trim(); 785 786 if ("*".equals(n)) { 787 tagAll = true; 788 } else { 789 tag = n.startsWith("/") ? getPattern(n) : n; 790 noMatch = "!=".equals(m.group(2)); 791 n = m.group(3).trim(); 792 if ("*".equals(n)) { 793 valueAll = true; 794 } else if ("BOOLEAN_TRUE".equals(n)) { 795 valueBool = true; 796 value = OsmUtils.TRUE_VALUE; 797 } else if ("BOOLEAN_FALSE".equals(n)) { 798 valueBool = true; 799 value = OsmUtils.FALSE_VALUE; 800 } else { 801 value = n.startsWith("/") ? getPattern(n) : n; 802 } 803 } 804 } 805 806 public boolean match(Map<String, String> keys) { 807 for (Entry<String, String> prop: keys.entrySet()) { 808 String key = prop.getKey(); 809 String val = valueBool ? OsmUtils.getNamedOsmBoolean(prop.getValue()) : prop.getValue(); 810 if ((tagAll || (tag instanceof Pattern ? ((Pattern) tag).matcher(key).matches() : key.equals(tag))) 811 && (valueAll || (value instanceof Pattern ? ((Pattern) value).matcher(val).matches() : val.equals(value)))) 812 return !noMatch; 813 } 814 return noMatch; 815 } 816 } 817 818 private static final Pattern CLEAN_STR_PATTERN = Pattern.compile(" *# *([^#]+) *$"); 819 private static final Pattern SPLIT_TRIMMED_PATTERN = Pattern.compile(" *: *"); 820 private static final Pattern SPLIT_ELEMENTS_PATTERN = Pattern.compile(" *&& *"); 821 822 public String getData(final String str) { 823 Matcher m = CLEAN_STR_PATTERN.matcher(str); 824 String trimmed = m.replaceFirst("").trim(); 825 try { 826 description = m.group(1); 827 if (description != null && description.isEmpty()) { 828 description = null; 829 } 830 } catch (IllegalStateException e) { 831 Logging.error(e); 832 description = null; 833 } 834 String[] n = SPLIT_TRIMMED_PATTERN.split(trimmed, 3); 835 switch (n[0]) { 836 case "way": 837 type = OsmPrimitiveType.WAY; 838 break; 839 case "node": 840 type = OsmPrimitiveType.NODE; 841 break; 842 case "relation": 843 type = OsmPrimitiveType.RELATION; 844 break; 845 case "*": 846 type = null; 847 break; 848 default: 849 return tr("Could not find element type"); 850 } 851 if (n.length != 3) 852 return tr("Incorrect number of parameters"); 853 854 switch (n[1]) { 855 case "W": 856 severity = Severity.WARNING; 857 level = TagCheckLevel.TAG_CHECK_WARN; 858 break; 859 case "E": 860 severity = Severity.ERROR; 861 level = TagCheckLevel.TAG_CHECK_ERROR; 862 break; 863 case "I": 864 severity = Severity.OTHER; 865 level = TagCheckLevel.TAG_CHECK_INFO; 866 break; 867 default: 868 return tr("Could not find warning level"); 869 } 870 for (String exp: SPLIT_ELEMENTS_PATTERN.split(n[2])) { 871 try { 872 data.add(new CheckerElement(exp)); 873 } catch (IllegalStateException e) { 874 Logging.trace(e); 875 return tr("Illegal expression ''{0}''", exp); 876 } catch (PatternSyntaxException e) { 877 Logging.trace(e); 878 return tr("Illegal regular expression ''{0}''", exp); 879 } 880 } 881 return null; 882 } 883 884 public boolean match(OsmPrimitive osm, Map<String, String> keys) { 885 if (type != null && OsmPrimitiveType.from(osm) != type) 886 return false; 887 888 for (CheckerElement ce : data) { 889 if (!ce.match(keys)) 890 return false; 891 } 892 return true; 893 } 894 895 /** 896 * Returns the error description. 897 * @return the error description 898 */ 899 public String getDescription() { 900 return description; 901 } 902 903 /** 904 * Returns the error severity. 905 * @return the error severity 906 */ 907 public Severity getSeverity() { 908 return severity; 909 } 910 911 /** 912 * Returns the error code. 913 * @return the error code 914 */ 915 public int getCode() { 916 if (type == null) 917 return level.code; 918 919 return level.code + type.ordinal() + 1; 920 } 921 } 922}