diff --git a/src/org/openstreetmap/josm/data/validation/ValidatorCLI.java b/src/org/openstreetmap/josm/data/validation/ValidatorCLI.java new file mode 100644 index 0000000000..2403bb38c4 --- /dev/null +++ b/src/org/openstreetmap/josm/data/validation/ValidatorCLI.java @@ -0,0 +1,468 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.validation; + +import static org.openstreetmap.josm.tools.I18n.tr; +import static org.openstreetmap.josm.tools.I18n.trn; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; +import java.util.logging.Level; +import java.util.stream.Collectors; + +import org.apache.commons.compress.utils.FileNameUtils; +import org.openstreetmap.josm.actions.ExtensionFileFilter; +import org.openstreetmap.josm.cli.CLIModule; +import org.openstreetmap.josm.data.Preferences; +import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.data.preferences.JosmBaseDirectories; +import org.openstreetmap.josm.data.preferences.JosmUrls; +import org.openstreetmap.josm.data.projection.ProjectionRegistry; +import org.openstreetmap.josm.data.projection.Projections; +import org.openstreetmap.josm.data.validation.tests.MapCSSTagChecker; +import org.openstreetmap.josm.gui.MainApplication; +import org.openstreetmap.josm.gui.io.CustomConfigurator; +import org.openstreetmap.josm.gui.io.importexport.FileImporter; +import org.openstreetmap.josm.gui.layer.OsmDataLayer; +import org.openstreetmap.josm.gui.mappaint.MapPaintStyles; +import org.openstreetmap.josm.gui.mappaint.mapcss.MapCSSStyleSource; +import org.openstreetmap.josm.gui.mappaint.mapcss.parsergen.ParseException; +import org.openstreetmap.josm.gui.progress.CLIProgressMonitor; +import org.openstreetmap.josm.gui.progress.ProgressMonitor; +import org.openstreetmap.josm.io.Compression; +import org.openstreetmap.josm.io.GeoJSONMapRouletteWriter; +import org.openstreetmap.josm.io.IllegalDataException; +import org.openstreetmap.josm.io.OsmChangeReader; +import org.openstreetmap.josm.spi.lifecycle.Lifecycle; +import org.openstreetmap.josm.spi.preferences.Config; +import org.openstreetmap.josm.spi.preferences.IPreferences; +import org.openstreetmap.josm.spi.preferences.MemoryPreferences; +import org.openstreetmap.josm.tools.Http1Client; +import org.openstreetmap.josm.tools.HttpClient; +import org.openstreetmap.josm.tools.I18n; +import org.openstreetmap.josm.tools.JosmRuntimeException; +import org.openstreetmap.josm.tools.Logging; +import org.openstreetmap.josm.tools.OptionParser; +import org.openstreetmap.josm.tools.Stopwatch; +import org.openstreetmap.josm.tools.Territories; +import org.openstreetmap.josm.tools.Utils; + +/** + * Add a validate command to the JOSM command line interface. + * @author Taylor Smock + * @since xxx + */ +public class ValidatorCLI implements CLIModule { + public static final ValidatorCLI INSTANCE = new ValidatorCLI(); + + /** The input file(s) */ + private final List input = new ArrayList<>(); + /** The change files. input file -> list of change files */ + private final Map> changeFiles = new HashMap<>(); + /** The output file(s). If {@code null}, use input filename as base (replace extension with geojson). input -> output */ + private final Map output = new HashMap<>(); + + private static final Supplier progressMonitorFactory = CLIProgressMonitor::new; + + /** The log level */ + private Level logLevel; + + private enum Option { + /** --help Show the help for validate */ + HELP(false, 'h'), + /** --input=<input-file> Set the current input file */ + INPUT(true, 'i', OptionParser.OptionCount.MULTIPLE), + /** --output=<output-file> Set the output file for the current input file */ + OUTPUT(true, 'o', OptionParser.OptionCount.MULTIPLE), + /** --change-file=<change-file> Add a change file */ + CHANGE_FILE(true, 'c', OptionParser.OptionCount.MULTIPLE), + /** --debug Set logging level to debug */ + DEBUG(false, '*'), + /** --trace Set logging level to trace */ + TRACE(false, '*'), + /** --language=<language> Set the language */ + LANGUAGE(true, 'l'), + /** --load-preferences=<url-to-xml> Changes preferences according to the XML file */ + LOAD_PREFERENCES(true, 'p'), + /** --set=<key>=<value> Set preference key to value */ + SET(true, 's'); + + private final String name; + private final boolean requiresArgument; + private final char shortOption; + private final OptionParser.OptionCount optionCount; + Option(final boolean requiresArgument, final char shortOption) { + this(requiresArgument, shortOption, OptionParser.OptionCount.OPTIONAL); + } + + Option(final boolean requiresArgument, final char shortOption, final OptionParser.OptionCount optionCount) { + this.name = name().toLowerCase(Locale.ROOT).replace('_', '-'); + this.requiresArgument = requiresArgument; + this.shortOption = shortOption; + this.optionCount = optionCount; + } + + /** + * Replies the option name + * @return The option name, in lowercase + */ + public String getName() { + return this.name; + } + + /** + * Get the number of times this option should be seen + * @return The option count + */ + public OptionParser.OptionCount getOptionCount() { + return this.optionCount; + } + + /** + * Replies the short option (single letter) associated with this option. + * @return the short option or '*' if there is no short option + */ + public char getShortOption() { + return this.shortOption; + } + + /** + * Determines if this option requires an argument. + * @return {@code true} if this option requires an argument, {@code false} otherwise + */ + public boolean requiresArgument() { + return this.requiresArgument; + } + + } + + @Override + public String getActionKeyword() { + return "validate"; + } + + @Override + public void processArguments(final String[] argArray) { + try { + // Ensure that preferences are only in memory + Config.setPreferencesInstance(new MemoryPreferences()); + Logging.setLogLevel(Level.INFO); + this.parseArguments(argArray); + if (this.input.isEmpty()) { + throw new IllegalArgumentException(tr("Missing argument - input data file ({0})", "--input|-i")); + } + this.initialize(); + final ProgressMonitor fileMonitor = progressMonitorFactory.get(); + fileMonitor.beginTask(tr("Processing files..."), this.input.size()); + for (String inputFile : this.input) { + if (inputFile.endsWith(".validator.mapcss")) { + this.processValidatorFile(inputFile); + } else if (inputFile.endsWith(".mapcss")) { + this.processMapcssFile(inputFile); + } else { + this.processFile(inputFile); + } + fileMonitor.worked(1); + } + fileMonitor.finishTask(); + } catch (Exception e) { + Logging.info(e); + Lifecycle.exitJosm(true, 1); + } + Lifecycle.exitJosm(true, 0); + } + + /** + * Process a standard mapcss file + * @param inputFile The mapcss file to validate + * @throws ParseException if the file does not match the mapcss syntax + */ + private void processMapcssFile(final String inputFile) throws ParseException { + final MapCSSStyleSource styleSource = new MapCSSStyleSource(new File(inputFile).toURI().getPath(), inputFile, inputFile); + styleSource.loadStyleSource(); + if (!styleSource.getErrors().isEmpty()) { + throw new ParseException(trn("{0} had {1} error", "{0} had {1} errors", styleSource.getErrors().size(), + inputFile, styleSource.getErrors().size())); + } else { + Logging.info(tr("{0} had no errors", inputFile)); + } + } + + /** + * Process a validator file + * @param inputFile The file to check + * @throws IOException if there is a problem reading the file + * @throws ParseException if the file does not match the validator mapcss syntax + */ + private void processValidatorFile(final String inputFile) throws ParseException, IOException { + // Check asserts + Config.getPref().putBoolean("validator.check_assert_local_rules", true); + final MapCSSTagChecker mapCSSTagChecker = new MapCSSTagChecker(); + final Collection assertionErrors = new ArrayList<>(); + final MapCSSTagChecker.ParseResult result = mapCSSTagChecker.addMapCSS(new File(inputFile).toURI().getPath(), + assertionErrors::add); + if (!result.parseErrors.isEmpty() || !assertionErrors.isEmpty()) { + for (Throwable throwable : result.parseErrors) { + Logging.error(throwable); + } + for (String error : assertionErrors) { + Logging.error(error); + } + throw new ParseException(trn("{0} had {1} error", "{0} had {1} errors", result.parseErrors.size() + assertionErrors.size(), + inputFile, result.parseErrors.size() + assertionErrors.size())); + } else { + Logging.info(tr("{0} had no errors"), inputFile); + } + } + + /** + * Process an OSM file + * @param inputFile The input filename + * @throws IllegalArgumentException If an argument is not valid + * @throws IllegalDataException If there is bad data + * @throws IOException If a file could not be read or written + */ + private void processFile(final String inputFile) throws IllegalDataException, IOException { + final File inputFileFile = new File(inputFile); + final List inputFileImporters = ExtensionFileFilter.getImporters().stream() + .filter(importer -> importer.acceptFile(inputFileFile)).collect(Collectors.toList()); + final Stopwatch stopwatch = Stopwatch.createStarted(); + if (inputFileImporters.stream().noneMatch(fileImporter -> + fileImporter.importDataHandleExceptions(inputFileFile, progressMonitorFactory.get()))) { + throw new IOException(tr("Could not load input file: {0}", inputFile)); + } + final String outputFile = Optional.ofNullable(this.output.get(inputFile)).orElseGet(() -> getDefaultOutputName(inputFile)); + final String task = tr("Validating {0}, saving output to {1}", inputFile, outputFile); + OsmDataLayer dataLayer = null; + try { + Logging.info(task); + OsmValidator.initializeTests(); + dataLayer = MainApplication.getLayerManager().getLayersOfType(OsmDataLayer.class) + .stream().filter(layer -> inputFileFile.equals(layer.getAssociatedFile())) + .findFirst().orElseThrow(() -> new JosmRuntimeException(tr("Could not find a layer for {0}", inputFile))); + final DataSet dataSet = dataLayer.getDataSet(); + if (this.changeFiles.containsKey(inputFile)) { + ProgressMonitor changeFilesMonitor = progressMonitorFactory.get(); + for (String changeFile : this.changeFiles.getOrDefault(inputFile, Collections.emptyList())) { + try (InputStream changeStream = Compression.getUncompressedFileInputStream(Paths.get(changeFile))) { + dataSet.mergeFrom(OsmChangeReader.parseDataSet(changeStream, changeFilesMonitor)); + } + } + } + Collection tests = OsmValidator.getEnabledTests(false); + if (Files.isRegularFile(Paths.get(outputFile)) && !Files.deleteIfExists(Paths.get(outputFile))) { + Logging.error("Could not delete {0}, attempting to append", outputFile); + } + GeoJSONMapRouletteWriter geoJSONMapRouletteWriter = new GeoJSONMapRouletteWriter(dataSet); + try (OutputStream fileOutputStream = Files.newOutputStream(Paths.get(outputFile))) { + tests.parallelStream().forEach(test -> runTest(test, geoJSONMapRouletteWriter, fileOutputStream, dataSet)); + } + } finally { + if (dataLayer != null) { + MainApplication.getLayerManager().removeLayer(dataLayer); + } + Logging.info(stopwatch.toString(task)); + } + } + + /** + * Get the default output name + * @param inputString The input file + * @return The default output name for the input file (extension stripped, ".geojson" added) + */ + private static String getDefaultOutputName(final String inputString) { + final String extension = FileNameUtils.getExtension(inputString); + if (!Arrays.asList("zip", "bz", "xz", "geojson").contains(extension)) { + return FileNameUtils.getBaseName(inputString) + ".geojson"; + } else if ("geojson".equals(extension)) { + // Account for geojson input files + return FileNameUtils.getBaseName(inputString) + ".validated.geojson"; + } + return FileNameUtils.getBaseName(FileNameUtils.getBaseName(inputString)) + ".geojson"; + } + + /** + * Run a test + * @param test The test to run + * @param geoJSONMapRouletteWriter The object to use to create challenges + * @param fileOutputStream The location to write data to + * @param dataSet The dataset to check + */ + private void runTest(final Test test, final GeoJSONMapRouletteWriter geoJSONMapRouletteWriter, + final OutputStream fileOutputStream, DataSet dataSet) { + test.startTest(progressMonitorFactory.get()); + test.visit(dataSet.allPrimitives()); + test.endTest(); + test.getErrors().stream().map(geoJSONMapRouletteWriter::write) + .filter(Optional::isPresent).map(Optional::get) + .map(jsonObject -> jsonObject.toString().getBytes(StandardCharsets.UTF_8)).forEach(bytes -> { + try { + writeToFile(fileOutputStream, bytes); + } catch (IOException e) { + throw new JosmRuntimeException(e); + } + }); + test.clear(); + } + + /** + * Write to a file. Synchronized to avoid writing to the same file in different threads. + * + * @param fileOutputStream The file output stream to read + * @param bytes The bytes to write (surrounded by RS and LF) + * @throws IOException If we couldn't write to file + */ + private synchronized void writeToFile(final OutputStream fileOutputStream, final byte[] bytes) + throws IOException { + // Write the ASCII Record Separator character + fileOutputStream.write(0x1e); + fileOutputStream.write(bytes); + // Write the ASCII Line Feed character + fileOutputStream.write(0x0a); + } + + /** + * Initialize everything that might be needed + * + * Arguments may need to be parsed first. + */ + void initialize() { + Logging.setLogLevel(this.logLevel); + HttpClient.setFactory(Http1Client::new); + Config.setBaseDirectoriesProvider(JosmBaseDirectories.getInstance()); // for right-left-hand traffic cache file + Config.setUrlsProvider(JosmUrls.getInstance()); + ProjectionRegistry.setProjection(Projections.getProjectionByCode("epsg:3857".toUpperCase(Locale.ROOT))); + + Territories.initializeInternalData(); + OsmValidator.initialize(); + MapPaintStyles.readFromPreferences(); + } + + /** + * Parse command line arguments and do some low-level error checking. + * @param argArray the arguments array + */ + void parseArguments(String[] argArray) { + Logging.setLogLevel(Level.INFO); + + OptionParser parser = new OptionParser("JOSM validate"); + final AtomicReference currentInput = new AtomicReference<>(null); + for (Option o : Option.values()) { + if (o.requiresArgument()) { + parser.addArgumentParameter(o.getName(), + o.getOptionCount(), + arg -> handleOption(currentInput.get(), o, arg).ifPresent(currentInput::set)); + } else { + parser.addFlagParameter(o.getName(), () -> handleOption(o)); + } + if (o.getShortOption() != '*') { + parser.addShortAlias(o.getName(), Character.toString(o.getShortOption())); + } + } + parser.parseOptionsOrExit(Arrays.asList(argArray)); + } + + private void handleOption(final Option option) { + switch (option) { + case HELP: + showHelp(); + System.exit(0); + break; + case DEBUG: + this.logLevel = Logging.LEVEL_DEBUG; + break; + case TRACE: + this.logLevel = Logging.LEVEL_TRACE; + break; + default: + throw new AssertionError("Unexpected option: " + option); + } + } + + /** + * Handle an option + * @param currentInput The current input file, if any. May be {@code null}. + * @param option The option to parse + * @param argument The argument for the option + * @return The new input file, if any. + */ + private Optional handleOption(final String currentInput, final Option option, final String argument) { + switch (option) { + case INPUT: + this.input.add(argument); + return Optional.of(argument); + case OUTPUT: + this.output.put(currentInput, argument); + break; + case CHANGE_FILE: + this.changeFiles.computeIfAbsent(currentInput, key -> new ArrayList<>()).add(argument); + break; + case LANGUAGE: + I18n.set(argument); + break; + case LOAD_PREFERENCES: + final Preferences tempPreferences = new Preferences(); + tempPreferences.enableSaveOnPut(false); + CustomConfigurator.XMLCommandProcessor config = new CustomConfigurator.XMLCommandProcessor(tempPreferences); + try (InputStream is = Utils.openStream(new File(argument).toURI().toURL())) { + config.openAndReadXML(is); + } catch (IOException e) { + throw new JosmRuntimeException(e); + } + final IPreferences pref = Config.getPref(); + if (pref instanceof MemoryPreferences) { + final MemoryPreferences memoryPreferences = (MemoryPreferences) pref; + tempPreferences.getAllSettings().entrySet().stream().filter(entry -> entry.getValue().isNew()) + .forEach(entry -> memoryPreferences.putSetting(entry.getKey(), entry.getValue())); + } else { + throw new JosmRuntimeException(tr("Preferences are not the expected type")); + } + break; + case SET: + + default: + throw new AssertionError("Unexpected option: " + option); + } + return Optional.empty(); + } + + private static void showHelp() { + System.out.println(getHelp()); + } + + private static String getHelp() { + final String helpPadding = "\t "; + // CHECKSTYLE.OFF: SingleSpaceSeparator + return tr("JOSM Validation command line interface") + "\n\n" + + tr("Usage") + ":\n" + + "\tjava -jar josm.jar validate \n\n" + + tr("Description") + ":\n" + + tr("Validates data and saves the result to a file.") + "\n\n"+ + tr("Options") + ":\n" + + "\t--help|-h " + tr("Show this help") + "\n" + + "\t--input|-i " + tr("Input data file name (.osm, .validator.mapcss, .mapcss).") + '\n' + + helpPadding + tr("OSM files can be specified multiple times. Required.") + '\n' + + helpPadding + tr(".validator.mapcss and .mapcss files will stop processing on first error.") + '\n' + + helpPadding + tr("Non-osm files do not use --output or --change-file") + '\n' + + "\t--output|-o " + tr("Output data file name (.geojson, line-by-line delimited for MapRoulette). Optional.") + + '\n' + + "\t--change-file|-c " + tr("Change file name (.osc). Can be specified multiple times per input.") + '\n' + + helpPadding + tr("Changes will be applied in the specified order. Optional."); + // CHECKSTYLE.ON: SingleSpaceSeparator + } +} diff --git a/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagChecker.java b/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagChecker.java index 7db32bd554..664dac89ed 100644 --- a/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagChecker.java +++ b/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagChecker.java @@ -253,13 +253,6 @@ public class MapCSSTagChecker extends Test.TagTest { } } - /** - * A handler for assertion error messages (for not fulfilled "assertMatch", "assertNoMatch"). - */ - @FunctionalInterface - interface AssertionConsumer extends Consumer { - } - /** * Adds a new MapCSS config file from the given URL. * @param url The unique URL of the MapCSS config file @@ -274,7 +267,18 @@ public class MapCSSTagChecker extends Test.TagTest { return addMapCSS(url, checkAssertions ? Logging::warn : null); } - synchronized ParseResult addMapCSS(String url, AssertionConsumer assertionConsumer) throws ParseException, IOException { + /** + * Adds a new MapCSS config file from the given URL.
+ * NOTE: You should prefer {@link #addMapCSS(String)} unless you need to know what the assertions return. + * + * @param url The unique URL of the MapCSS config file + * @param assertionConsumer A string consumer for error messages. + * @return List of tag checks and parsing errors, or null + * @throws ParseException if the config file does not match MapCSS syntax + * @throws IOException if any I/O error occurs + * @since xxx (public, primarily for ValidatorCLI) + */ + public synchronized ParseResult addMapCSS(String url, Consumer assertionConsumer) throws ParseException, IOException { CheckParameterUtil.ensureParameterNotNull(url, "url"); ParseResult result; try (CachedFile cache = new CachedFile(url); diff --git a/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagCheckerAsserts.java b/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagCheckerAsserts.java index d004882872..838868f9d5 100644 --- a/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagCheckerAsserts.java +++ b/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagCheckerAsserts.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; import java.util.stream.Collectors; import org.openstreetmap.josm.command.Command; @@ -46,7 +47,7 @@ final class MapCSSTagCheckerAsserts { * @param assertionConsumer The handler for assertion error messages */ static void checkAsserts(final MapCSSTagCheckerRule check, final Map assertions, - final MapCSSTagChecker.AssertionConsumer assertionConsumer) { + final Consumer assertionConsumer) { final DataSet ds = new DataSet(); Logging.debug("Check: {0}", check); for (final Map.Entry i : assertions.entrySet()) { diff --git a/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagCheckerRule.java b/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagCheckerRule.java index 4e012467e8..478eb57650 100644 --- a/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagCheckerRule.java +++ b/src/org/openstreetmap/josm/data/validation/tests/MapCSSTagCheckerRule.java @@ -15,6 +15,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Consumer; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -31,7 +32,6 @@ import org.openstreetmap.josm.data.osm.WaySegment; import org.openstreetmap.josm.data.validation.Severity; import org.openstreetmap.josm.data.validation.Test; import org.openstreetmap.josm.data.validation.TestError; -import org.openstreetmap.josm.data.validation.tests.MapCSSTagChecker.AssertionConsumer; import org.openstreetmap.josm.gui.mappaint.Environment; import org.openstreetmap.josm.gui.mappaint.Keyword; import org.openstreetmap.josm.gui.mappaint.mapcss.Condition; @@ -106,7 +106,7 @@ final class MapCSSTagCheckerRule implements Predicate { private static final String POSSIBLE_THROWS = "throwError/throwWarning/throwOther"; - static MapCSSTagCheckerRule ofMapCSSRule(final MapCSSRule rule, AssertionConsumer assertionConsumer) throws IllegalDataException { + static MapCSSTagCheckerRule ofMapCSSRule(final MapCSSRule rule, Consumer assertionConsumer) throws IllegalDataException { final MapCSSTagCheckerRule check = new MapCSSTagCheckerRule(rule); final Map assertions = new HashMap<>(); for (Instruction i : rule.declaration.instructions) { @@ -185,7 +185,7 @@ final class MapCSSTagCheckerRule implements Predicate { return readMapCSS(css, null); } - static MapCSSTagChecker.ParseResult readMapCSS(Reader css, AssertionConsumer assertionConsumer) throws ParseException { + static MapCSSTagChecker.ParseResult readMapCSS(Reader css, Consumer assertionConsumer) throws ParseException { CheckParameterUtil.ensureParameterNotNull(css, "css"); final MapCSSStyleSource source = new MapCSSStyleSource(""); diff --git a/src/org/openstreetmap/josm/gui/MainApplication.java b/src/org/openstreetmap/josm/gui/MainApplication.java index 05da963669..8aa31512fa 100644 --- a/src/org/openstreetmap/josm/gui/MainApplication.java +++ b/src/org/openstreetmap/josm/gui/MainApplication.java @@ -98,6 +98,7 @@ import org.openstreetmap.josm.data.projection.ProjectionRegistry; import org.openstreetmap.josm.data.projection.datum.NTV2GridShiftFileSource; import org.openstreetmap.josm.data.projection.datum.NTV2GridShiftFileWrapper; import org.openstreetmap.josm.data.projection.datum.NTV2Proj4DirGridShiftFileSource; +import org.openstreetmap.josm.data.validation.ValidatorCLI; import org.openstreetmap.josm.data.validation.tests.MapCSSTagChecker; import org.openstreetmap.josm.gui.ProgramArguments.Option; import org.openstreetmap.josm.gui.SplashScreen.SplashProgressMonitor; @@ -311,6 +312,7 @@ public class MainApplication { registerCLIModule(JOSM_CLI_MODULE); registerCLIModule(ProjectionCLI.INSTANCE); registerCLIModule(RenderingCLI.INSTANCE); + registerCLIModule(ValidatorCLI.INSTANCE); } /** @@ -660,7 +662,8 @@ public class MainApplication { tr("commands")+":\n"+ "\trunjosm "+tr("launch JOSM (default, performed when no command is specified)")+'\n'+ "\trender "+tr("render data and save the result to an image file")+'\n'+ - "\tproject "+tr("convert coordinates from one coordinate reference system to another")+"\n\n"+ + "\tproject " + tr("convert coordinates from one coordinate reference system to another")+ '\n' + + "\tvalidate " + tr("validate data") + "\n\n" + tr("For details on the {0} and {1} commands, run them with the {2} option.", "render", "project", "--help")+'\n'+ tr("The remainder of this help page documents the {0} command.", "runjosm")+"\n\n"+ tr("options")+":\n"+ diff --git a/src/org/openstreetmap/josm/gui/io/importexport/OsmImporter.java b/src/org/openstreetmap/josm/gui/io/importexport/OsmImporter.java index d839be5963..4216843f60 100644 --- a/src/org/openstreetmap/josm/gui/io/importexport/OsmImporter.java +++ b/src/org/openstreetmap/josm/gui/io/importexport/OsmImporter.java @@ -97,10 +97,11 @@ public class OsmImporter extends FileImporter { final OsmImporterData data = loadLayer(in, associatedFile, associatedFile == null ? OsmDataLayer.createNewName() : associatedFile.getName(), pm); + final OsmDataLayer layer = data.getLayer(); + // Note: addLayer calls GuiHelper.runInEDTAndWaitWithException + MainApplication.getLayerManager().addLayer(layer); // FIXME: remove UI stuff from IO subsystem GuiHelper.runInEDT(() -> { - OsmDataLayer layer = data.getLayer(); - MainApplication.getLayerManager().addLayer(layer); data.getPostLayerTask().run(); data.getLayer().onPostLoadFromFile(); }); diff --git a/src/org/openstreetmap/josm/gui/preferences/projection/CustomProjectionChoice.java b/src/org/openstreetmap/josm/gui/preferences/projection/CustomProjectionChoice.java index e4db57d1d0..6dfc53eb17 100644 --- a/src/org/openstreetmap/josm/gui/preferences/projection/CustomProjectionChoice.java +++ b/src/org/openstreetmap/josm/gui/preferences/projection/CustomProjectionChoice.java @@ -23,6 +23,7 @@ import org.openstreetmap.josm.data.projection.Projection; import org.openstreetmap.josm.data.projection.ProjectionConfigurationException; import org.openstreetmap.josm.data.projection.Projections; import org.openstreetmap.josm.gui.ExtendedDialog; +import org.openstreetmap.josm.gui.tagging.ac.AutoCompTextField; import org.openstreetmap.josm.gui.widgets.AbstractTextComponentValidator; import org.openstreetmap.josm.gui.widgets.HistoryComboBox; import org.openstreetmap.josm.gui.widgets.HtmlPanel; @@ -52,7 +53,7 @@ public class CustomProjectionChoice extends AbstractProjectionChoice implements private static class PreferencePanel extends JPanel { - public JosmTextField input; + public AutoCompTextField input; private HistoryComboBox cbInput; PreferencePanel(String initialText, ActionListener listener) { @@ -60,7 +61,7 @@ public class CustomProjectionChoice extends AbstractProjectionChoice implements } private void build(String initialText, final ActionListener listener) { - input = new JosmTextField(30); + input = new AutoCompTextField<>(30); cbInput = new HistoryComboBox(); cbInput.setEditor(new BasicComboBoxEditor() { @Override diff --git a/src/org/openstreetmap/josm/gui/progress/AbstractProgressMonitor.java b/src/org/openstreetmap/josm/gui/progress/AbstractProgressMonitor.java index 31e2c0b3c9..f8d2381f66 100644 --- a/src/org/openstreetmap/josm/gui/progress/AbstractProgressMonitor.java +++ b/src/org/openstreetmap/josm/gui/progress/AbstractProgressMonitor.java @@ -233,6 +233,10 @@ public abstract class AbstractProgressMonitor implements ProgressMonitor { * Ticks handling ==================*/ + /** + * Update progress message + * @param value The percentage of completion (this and child progress) + */ protected abstract void updateProgress(double value); @Override diff --git a/src/org/openstreetmap/josm/gui/progress/CLIProgressMonitor.java b/src/org/openstreetmap/josm/gui/progress/CLIProgressMonitor.java new file mode 100644 index 0000000000..b41ab40ace --- /dev/null +++ b/src/org/openstreetmap/josm/gui/progress/CLIProgressMonitor.java @@ -0,0 +1,93 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.gui.progress; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.awt.Component; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import org.openstreetmap.josm.tools.Logging; +import org.openstreetmap.josm.tools.Stopwatch; +import org.openstreetmap.josm.tools.Utils; + +/** + * CLI implementation of a {@link ProgressMonitor} + * @author Taylor Smock + * @since xxx + */ +public class CLIProgressMonitor extends AbstractProgressMonitor { + /** The current task id */ + private ProgressTaskId taskId; + /** The current task title */ + private String title = ""; + /** The custom text (prepended with '/') */ + private String customText = ""; + /** The last time we updated the progress information */ + private Stopwatch lastUpdateTime; + /** The start time of the monitor */ + private Stopwatch startTime; + + /** + * Create a new {@link CLIProgressMonitor} + */ + public CLIProgressMonitor() { + super(new CancelHandler()); + } + + @Override + protected void doBeginTask() { + if (!Utils.isBlank(this.title)) { + Logging.info(tr("Beginning task{2}: {0}{1}", this.title, this.customText, + Optional.ofNullable(this.taskId).map(ProgressTaskId::getId).map(id -> ' ' + id).orElse(""))); + } + this.startTime = Stopwatch.createStarted(); + this.lastUpdateTime = this.startTime; + } + + @Override + protected void doFinishTask() { + Logging.info(tr("Finishing task{2}: {0}{1} ({3})", this.title, this.customText, + Optional.ofNullable(this.taskId).map(ProgressTaskId::getId).map(id -> ' ' + id).orElse(""), this.startTime)); + this.lastUpdateTime = null; + } + + @Override + protected void doSetIntermediate(boolean value) { + // Do nothing for now + } + + @Override + protected void doSetTitle(String title) { + this.title = Optional.ofNullable(title).orElse(""); + } + + @Override + protected void doSetCustomText(String customText) { + this.customText = Optional.ofNullable(customText).map(str -> '/' + str).orElse(""); + } + + @Override + protected void updateProgress(double value) { + if (this.lastUpdateTime == null || this.lastUpdateTime.elapsed() > TimeUnit.SECONDS.toMillis(10)) { + this.lastUpdateTime = Stopwatch.createStarted(); + Logging.info(tr("Progress of task{2}: {0}{1} is {3}% ({4})", this.title, this.customText, + Optional.ofNullable(this.taskId).map(ProgressTaskId::getId).map(id -> ' ' + id).orElse(""), value * 100, this.startTime)); + } + } + + @Override + public void setProgressTaskId(ProgressTaskId taskId) { + this.taskId = taskId; + } + + @Override + public ProgressTaskId getProgressTaskId() { + return this.taskId; + } + + @Override + public Component getWindowParent() { + return null; + } +} diff --git a/src/org/openstreetmap/josm/gui/util/GuiHelper.java b/src/org/openstreetmap/josm/gui/util/GuiHelper.java index 89ab9f84b0..4c17334af2 100644 --- a/src/org/openstreetmap/josm/gui/util/GuiHelper.java +++ b/src/org/openstreetmap/josm/gui/util/GuiHelper.java @@ -288,7 +288,7 @@ public final class GuiHelper { * @since 10271 */ public static void assertCallFromEdt() { - if (!SwingUtilities.isEventDispatchThread()) { + if (!SwingUtilities.isEventDispatchThread() && !GraphicsEnvironment.isHeadless()) { throw new IllegalStateException( "Needs to be called from the EDT thread, not from " + Thread.currentThread().getName()); } diff --git a/src/org/openstreetmap/josm/io/GeoJSONMapRouletteWriter.java b/src/org/openstreetmap/josm/io/GeoJSONMapRouletteWriter.java new file mode 100644 index 0000000000..fca07f34ac --- /dev/null +++ b/src/org/openstreetmap/josm/io/GeoJSONMapRouletteWriter.java @@ -0,0 +1,90 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.io; + +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Stream; + +import javax.json.Json; +import javax.json.JsonArray; +import javax.json.JsonArrayBuilder; +import javax.json.JsonObject; +import javax.json.JsonObjectBuilder; +import javax.json.JsonValue; + +import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.data.osm.OsmPrimitive; +import org.openstreetmap.josm.data.osm.WaySegment; +import org.openstreetmap.josm.data.validation.TestError; +import org.openstreetmap.josm.tools.Logging; + +/** + * Convert {@link TestError} to MapRoulette Tasks + * @author Taylor Smock + * @since xxx + */ +public class GeoJSONMapRouletteWriter extends GeoJSONWriter { + + /** + * Constructs a new {@code GeoJSONWriter}. + * @param ds The originating OSM dataset + */ + public GeoJSONMapRouletteWriter(DataSet ds) { + super(ds); + super.setOptions(Options.RIGHT_HAND_RULE, Options.WRITE_OSM_INFORMATION); + } + + /** + * Convert a test error to a string + * @param testError The test error to convert + * @return The MapRoulette challenge object + */ + public Optional write(final TestError testError) { + final JsonObjectBuilder jsonObjectBuilder = Json.createObjectBuilder(); + final JsonArrayBuilder featuresBuilder = Json.createArrayBuilder(); + final JsonObjectBuilder propertiesBuilder = Json.createObjectBuilder(); + propertiesBuilder.add("message", testError.getMessage()); + Optional.ofNullable(testError.getDescription()).ifPresent(description -> propertiesBuilder.add("description", description)); + propertiesBuilder.add("code", testError.getCode()); + propertiesBuilder.add("fixable", testError.isFixable()); + propertiesBuilder.add("severity", testError.getSeverity().toString()); + propertiesBuilder.add("severityInteger", testError.getSeverity().getLevel()); + propertiesBuilder.add("test", testError.getTester().getName()); + Stream.concat(testError.getPrimitives().stream(), testError.getHighlighted().stream()).distinct().map(p -> { + if (p instanceof OsmPrimitive) { + return p; + } else if (p instanceof WaySegment) { + return ((WaySegment) p).toWay(); + } + Logging.trace("Could not convert {0} to an OsmPrimitive", p); + return null; + }).filter(Objects::nonNull).distinct().map(OsmPrimitive.class::cast) + .forEach(primitive -> super.appendPrimitive(primitive, featuresBuilder)); + final JsonArray featureArray = featuresBuilder.build(); + final JsonArrayBuilder featuresMessageBuilder = Json.createArrayBuilder(); + if (featureArray.isEmpty()) { + Logging.trace("Could not generate task for {0}", testError.getMessage()); + return Optional.empty(); + } + JsonObject primitive = featureArray.getJsonObject(0); + JsonObjectBuilder replacementPrimitive = Json.createObjectBuilder(primitive); + final JsonObjectBuilder properties; + if (primitive.containsKey("properties") && primitive.get("properties").getValueType() == JsonValue.ValueType.OBJECT) { + properties = Json.createObjectBuilder(primitive.getJsonObject("properties")); + } else { + properties = Json.createObjectBuilder(); + } + properties.addAll(propertiesBuilder); + replacementPrimitive.add("properties", properties); + featuresMessageBuilder.add(replacementPrimitive); + for (int i = 1; i < featureArray.size(); i++) { + featuresMessageBuilder.add(featureArray.get(i)); + } + // For now, don't add any cooperativeWork objects, as JOSM should be able to find the fixes. + // This should change if the ValidatorCLI can use plugins (especially those introducing external data, like + // the ElevationProfile plugin (which provides elevation data)). + jsonObjectBuilder.add("type", "FeatureCollection"); + jsonObjectBuilder.add("features", featuresMessageBuilder); + return Optional.of(jsonObjectBuilder.build()); + } +} diff --git a/src/org/openstreetmap/josm/io/GeoJSONWriter.java b/src/org/openstreetmap/josm/io/GeoJSONWriter.java index d2eb644846..23ac2ba7a4 100644 --- a/src/org/openstreetmap/josm/io/GeoJSONWriter.java +++ b/src/org/openstreetmap/josm/io/GeoJSONWriter.java @@ -6,14 +6,18 @@ import java.io.StringWriter; import java.io.Writer; import java.math.BigDecimal; import java.math.RoundingMode; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.EnumSet; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.json.Json; @@ -30,19 +34,23 @@ import org.openstreetmap.josm.data.Bounds; import org.openstreetmap.josm.data.coor.EastNorth; import org.openstreetmap.josm.data.coor.LatLon; import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.data.osm.INode; +import org.openstreetmap.josm.data.osm.IWay; import org.openstreetmap.josm.data.osm.MultipolygonBuilder; -import org.openstreetmap.josm.data.osm.MultipolygonBuilder.JoinedPolygon; import org.openstreetmap.josm.data.osm.Node; import org.openstreetmap.josm.data.osm.OsmPrimitive; import org.openstreetmap.josm.data.osm.Relation; +import org.openstreetmap.josm.data.osm.RelationMember; import org.openstreetmap.josm.data.osm.Way; import org.openstreetmap.josm.data.osm.visitor.OsmPrimitiveVisitor; import org.openstreetmap.josm.data.preferences.BooleanProperty; import org.openstreetmap.josm.data.projection.Projection; import org.openstreetmap.josm.data.projection.Projections; import org.openstreetmap.josm.gui.mappaint.ElemStyles; +import org.openstreetmap.josm.tools.Geometry; import org.openstreetmap.josm.tools.Logging; import org.openstreetmap.josm.tools.Pair; +import org.openstreetmap.josm.tools.Utils; /** * Writes OSM data as a GeoJSON string, using JSR 353: Java API for JSON Processing (JSON-P). @@ -51,11 +59,21 @@ import org.openstreetmap.josm.tools.Pair; */ public class GeoJSONWriter { + enum Options { + /** If using the right hand rule, we have to ensure that the "right" side is the interior of the object. */ + RIGHT_HAND_RULE, + /** Write OSM information to the feature properties field. This tries to follow the Overpass turbo format. */ + WRITE_OSM_INFORMATION, + /** Skip empty nodes */ + SKIP_EMPTY_NODES + } + private final DataSet data; - private final Projection projection; + private static final Projection projection = Projections.getProjectionByCode("EPSG:4326"); // WGS 84 private static final BooleanProperty SKIP_EMPTY_NODES = new BooleanProperty("geojson.export.skip-empty-nodes", true); private static final BooleanProperty UNTAGGED_CLOSED_IS_POLYGON = new BooleanProperty("geojson.export.untagged-closed-is-polygon", false); private static final Set processedMultipolygonWays = new HashSet<>(); + private EnumSet options = EnumSet.noneOf(Options.class); /** * This is used to determine that a tag should be interpreted as a json @@ -77,7 +95,18 @@ public class GeoJSONWriter { */ public GeoJSONWriter(DataSet ds) { this.data = ds; - this.projection = Projections.getProjectionByCode("EPSG:4326"); // WGS 84 + if (Boolean.TRUE.equals(SKIP_EMPTY_NODES.get())) { + this.options.add(Options.SKIP_EMPTY_NODES); + } + } + + /** + * Set the options for this writer. See {@link Options}. + * @param options The options to set. + */ + void setOptions(final Options... options) { + this.options.clear(); + this.options.addAll(Arrays.asList(options)); } /** @@ -117,6 +146,9 @@ public class GeoJSONWriter { } } + /** + * Convert a primitive to a json object + */ private class GeometryPrimitiveVisitor implements OsmPrimitiveVisitor { private final JsonObjectBuilder geomObj; @@ -141,9 +173,13 @@ public class GeoJSONWriter { // no need to write this object again return; } - final JsonArrayBuilder array = getCoorsArray(w.getNodes()); boolean writeAsPolygon = w.isClosed() && ((!w.isTagged() && UNTAGGED_CLOSED_IS_POLYGON.get()) || ElemStyles.hasAreaElemStyle(w, false)); + final List nodes = w.getNodes(); + if (writeAsPolygon && options.contains(Options.RIGHT_HAND_RULE) && Geometry.isClockwise(nodes)) { + Collections.reverse(nodes); + } + final JsonArrayBuilder array = getCoorsArray(nodes); if (writeAsPolygon) { geomObj.add("type", "Polygon"); geomObj.add("coordinates", Json.createArrayBuilder().add(array)); @@ -159,25 +195,116 @@ public class GeoJSONWriter { if (r == null || !r.isMultipolygon() || r.hasIncompleteMembers()) { return; } - try { - final Pair, List> mp = MultipolygonBuilder.joinWays(r); + if (r.isMultipolygon()) { + try { + this.visitMultipolygon(r); + return; + } catch (MultipolygonBuilder.JoinedPolygonCreationException ex) { + Logging.warn("GeoJSON: Failed to export multipolygon {0}, falling back to other multi geometry types", r.getUniqueId()); + Logging.warn(ex); + } + } + // These are run if (a) r is not a multipolygon or (b) r is not a well-formed multipolygon. + if (r.getMemberPrimitives().stream().allMatch(IWay.class::isInstance)) { + this.visitMultiLineString(r); + } else if (r.getMemberPrimitives().stream().allMatch(INode.class::isInstance)) { + this.visitMultiPoints(r); + } else { + this.visitMultiGeometry(r); + } + } + + /** + * Visit a multi-part geometry. + * Note: Does not currently recurse down relations. RFC 7946 indicates that we + * should avoid nested geometry collections. This behavior may change any time in the future! + * @param r The relation to visit. + */ + private void visitMultiGeometry(final Relation r) { + final JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); + r.getMemberPrimitives().stream().filter(p -> !(p instanceof Relation)) + .map(p -> { + final JsonObjectBuilder tempGeomObj = Json.createObjectBuilder(); + p.accept(new GeometryPrimitiveVisitor(tempGeomObj)); + return tempGeomObj.build(); + }).forEach(jsonArrayBuilder::add); + geomObj.add("type", "GeometryCollection"); + geomObj.add("geometries", jsonArrayBuilder); + } + + /** + * Visit a relation that only contains points + * @param r The relation to visit + */ + private void visitMultiPoints(final Relation r) { + final JsonArrayBuilder multiPoint = Json.createArrayBuilder(); + r.getMembers().stream().map(RelationMember::getMember).filter(Node.class::isInstance).map(Node.class::cast) + .map(Node::getCoor).map(latLon -> getCoorArray(null, latLon)) + .forEach(multiPoint::add); + geomObj.add("type", "MultiPoint"); + geomObj.add("coordinates", multiPoint); + } + + /** + * Visit a relation that is a multi line string + * @param r The relation to convert + */ + private void visitMultiLineString(final Relation r) { + final JsonArrayBuilder multiLine = Json.createArrayBuilder(); + r.getMembers().stream().map(RelationMember::getMember).filter(Way.class::isInstance).map(Way.class::cast) + .map(Way::getNodes).map(p -> { + JsonArrayBuilder array = getCoorsArray(p); + LatLon ll = p.get(0).getCoor(); + // since first node is not duplicated as last node + return ll != null ? array.add(getCoorArray(null, ll)) : array; + }).forEach(multiLine::add); + geomObj.add("type", "MultiLineString"); + geomObj.add("coordinates", multiLine); + processedMultipolygonWays.addAll(r.getMemberPrimitives(Way.class)); + } + + /** + * Convert a multipolygon to geojson + * @param r The relation to convert + * @throws MultipolygonBuilder.JoinedPolygonCreationException See {@link MultipolygonBuilder#joinWays(Relation)}. + * Note that if the exception is thrown, {@link #geomObj} will not have been modified. + */ + private void visitMultipolygon(final Relation r) throws MultipolygonBuilder.JoinedPolygonCreationException { + final Pair, List> mp = + MultipolygonBuilder.joinWays(r); final JsonArrayBuilder polygon = Json.createArrayBuilder(); - Stream.concat(mp.a.stream(), mp.b.stream()) + // Peek would theoretically be better for these two streams, but SonarLint doesn't like it. + // java:S3864: "Stream.peek" should be used with caution + final Stream> outer = mp.a.stream().map(MultipolygonBuilder.JoinedPolygon::getNodes).map(nodes -> { + final ArrayList tempNodes = new ArrayList<>(nodes); + tempNodes.add(tempNodes.get(0)); + if (options.contains(Options.RIGHT_HAND_RULE) && Geometry.isClockwise(tempNodes)) { + Collections.reverse(nodes); + } + return nodes; + }); + final Stream> inner = mp.b.stream().map(MultipolygonBuilder.JoinedPolygon::getNodes).map(nodes -> { + final ArrayList tempNodes = new ArrayList<>(nodes); + tempNodes.add(tempNodes.get(0)); + // Note that we are checking !Geometry.isClockwise, which is different from the outer + // ring check. + if (options.contains(Options.RIGHT_HAND_RULE) && !Geometry.isClockwise(tempNodes)) { + Collections.reverse(nodes); + } + return nodes; + }); + Stream.concat(outer, inner) .map(p -> { - JsonArrayBuilder array = getCoorsArray(p.getNodes()); - LatLon ll = p.getNodes().get(0).getCoor(); + JsonArrayBuilder array = getCoorsArray(p); + LatLon ll = p.get(0).getCoor(); // since first node is not duplicated as last node return ll != null ? array.add(getCoorArray(null, ll)) : array; - }) + }) .forEach(polygon::add); - geomObj.add("type", "MultiPolygon"); final JsonArrayBuilder multiPolygon = Json.createArrayBuilder().add(polygon); + geomObj.add("type", "MultiPolygon"); geomObj.add("coordinates", multiPolygon); processedMultipolygonWays.addAll(r.getMemberPrimitives(Way.class)); - } catch (MultipolygonBuilder.JoinedPolygonCreationException ex) { - Logging.warn("GeoJSON: Failed to export multipolygon {0}", r.getUniqueId()); - Logging.warn(ex); - } } private JsonArrayBuilder getCoorsArray(Iterable nodes) { @@ -204,14 +331,49 @@ public class GeoJSONWriter { protected void appendPrimitive(OsmPrimitive p, JsonArrayBuilder array) { if (p.isIncomplete() || - (SKIP_EMPTY_NODES.get() && p instanceof Node && p.getKeys().isEmpty())) { + (this.options.contains(Options.SKIP_EMPTY_NODES) && p instanceof Node && p.getKeys().isEmpty())) { return; } // Properties final JsonObjectBuilder propObj = Json.createObjectBuilder(); - for (Entry t : p.getKeys().entrySet()) { - propObj.add(t.getKey(), convertValueToJson(t.getValue())); + for (Map.Entry t : p.getKeys().entrySet()) { + // If writing OSM information, follow Overpass syntax (escape `@` with another `@`) + final String key = options.contains(Options.WRITE_OSM_INFORMATION) && t.getKey().startsWith("@") + ? '@' + t.getKey() : t.getKey(); + propObj.add(key, convertValueToJson(t.getValue())); + } + if (options.contains(Options.WRITE_OSM_INFORMATION)) { + // Use the same format as Overpass + propObj.add("@id", p.getPrimitiveId().getType().getAPIName() + '/' + p.getUniqueId()); // type/id + if (!p.isNew()) { + propObj.add("@timestamp", Instant.ofEpochSecond(p.getRawTimestamp()).toString()); + propObj.add("@version", Integer.toString(p.getVersion())); + propObj.add("@changeset", Long.toString(p.getChangesetId())); + } + if (p.getUser() != null) { + propObj.add("@user", p.getUser().getName()); + propObj.add("@uid", p.getUser().getId()); + } + if (options.contains(Options.WRITE_OSM_INFORMATION) && p.getReferrers(true).stream().anyMatch(Relation.class::isInstance)) { + final JsonArrayBuilder jsonArrayBuilder = Json.createArrayBuilder(); + for (Relation relation : Utils.filteredCollection(p.getReferrers(), Relation.class)) { + final JsonObjectBuilder relationObject = Json.createObjectBuilder(); + relationObject.add("rel", relation.getId()); + Collection members = relation.getMembersFor(Collections.singleton(p)); + // Each role is a separate object in overpass-turbo geojson export. For now, just concat them. + relationObject.add("role", + members.stream().map(RelationMember::getRole).collect(Collectors.joining(";"))); + final JsonObjectBuilder relationKeys = Json.createObjectBuilder(); + // Uncertain if the @relation reltags need to be @ escaped. I don't think so, as example output + // didn't have any metadata in it. + for (Map.Entry tag : relation.getKeys().entrySet()) { + relationKeys.add(tag.getKey(), convertValueToJson(tag.getValue())); + } + relationObject.add("reltags", relationKeys); + } + propObj.add("@relations", jsonArrayBuilder); + } } final JsonObject prop = propObj.build(); diff --git a/test/unit/org/openstreetmap/josm/data/validation/ValidatorCLITest.java b/test/unit/org/openstreetmap/josm/data/validation/ValidatorCLITest.java new file mode 100644 index 0000000000..92466c9175 --- /dev/null +++ b/test/unit/org/openstreetmap/josm/data/validation/ValidatorCLITest.java @@ -0,0 +1,234 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.validation; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.PrintWriter; +import java.lang.reflect.InvocationTargetException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.logging.Handler; +import java.util.logging.LogRecord; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.json.Json; +import javax.json.JsonObject; +import javax.json.JsonReader; +import javax.swing.SwingUtilities; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; +import org.openstreetmap.josm.TestUtils; +import org.openstreetmap.josm.data.coor.LatLon; +import org.openstreetmap.josm.data.osm.DataSet; +import org.openstreetmap.josm.data.osm.Node; +import org.openstreetmap.josm.gui.MainApplication; +import org.openstreetmap.josm.io.OsmWriter; +import org.openstreetmap.josm.io.OsmWriterFactory; +import org.openstreetmap.josm.spi.lifecycle.Lifecycle; +import org.openstreetmap.josm.testutils.annotations.BasicPreferences; +import org.openstreetmap.josm.tools.Logging; +import org.openstreetmap.josm.tools.Utils; + +import mockit.Mock; +import mockit.MockUp; + +/** + * Test class for {@link ValidatorCLI} + * @author Taylor Smock + */ +@BasicPreferences +class ValidatorCLITest { + @TempDir + static File temporaryDirectory; + + TestHandler handler; + + private static void synchronizeThreads() { + MainApplication.worker.execute(() -> { /* Sync worker thread */ }); + try { + SwingUtilities.invokeAndWait(() -> { /* Sync EDT thread */ }); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail(e); + } catch (InvocationTargetException e) { + fail(e); + } + } + + @BeforeEach + void setup() { + TestUtils.assumeWorkingJMockit(); + new LifecycleMock(); + this.handler = new TestHandler(); + Logging.getLogger().addHandler(this.handler); + } + + @AfterEach + void tearDown() throws InterruptedException, InvocationTargetException { + synchronizeThreads(); + Logging.getLogger().removeHandler(this.handler); + this.handler.close(); + this.handler = null; + } + + @ParameterizedTest + @ValueSource(strings = {"resources/styles/standard/elemstyles.mapcss", "resources/styles/standard/potlatch2.mapcss"}) + void testInternalMapcss(final String resourceLocation) { + new ValidatorCLI().processArguments(new String[]{"--input", resourceLocation}); + assertEquals(2, this.handler.logRecordList.size()); + assertEquals(resourceLocation + " had no errors", this.handler.logRecordList.get(0).getMessage()); + assertTrue(this.handler.logRecordList.get(1).getMessage().contains("Finishing task")); + } + + static Stream testInternalValidatorMapcss() { + return Stream.of(Objects.requireNonNull(new File("resources/data/validator").listFiles())) + .filter(file -> file.getPath().endsWith(".mapcss")) + .map(file -> { + // External validator mapcss files must have validator.mapcss as the extension. + final String renamedValidator = file.getName().endsWith(".validator.mapcss") ? + file.getName() : file.getName().replace(".mapcss", ".validator.mapcss"); + try { + return Files.copy(file.toPath(), Paths.get(temporaryDirectory.getPath(), renamedValidator)).getFileName().toString(); + } catch (IOException e) { + fail(e); + } + return null; + }).map(Arguments::of); + } + + @ParameterizedTest + @MethodSource + void testInternalValidatorMapcss(final String resourceLocation) { + final String path = Paths.get(temporaryDirectory.getPath(), resourceLocation).toString(); + new ValidatorCLI().processArguments(new String[]{"--input", path}); + assertEquals(2, this.handler.logRecordList.size(), this.handler.logRecordList.stream().map(LogRecord::getMessage).collect( + Collectors.joining(",\n"))); + assertEquals(path + " had no errors", this.handler.logRecordList.get(0).getMessage()); + assertTrue(this.handler.logRecordList.get(1).getMessage().contains("Finishing task")); + } + + @Test + void testBadDataTicket13165() { + // Ticket #13165 was a validator non-regression test. + final String dataPath = TestUtils.getRegressionDataFile(13165, "13165.osm"); + final String outputPath = Paths.get(temporaryDirectory.getPath(), "testBadDataTicket13165.geojson").toString(); + new ValidatorCLI().processArguments(new String[]{"--input", dataPath, "--output", outputPath}); + final File outputFile = new File(outputPath); + assertTrue(outputFile.exists()); + synchronizeThreads(); + final List errors = readJsonObjects(outputFile.toPath()); + assertEquals(3, errors.stream().map(ValidatorCLITest::getMessage).filter("Overlapping Identical Landuses"::equals).count()); + assertEquals(3, errors.size(), errors.stream().map(ValidatorCLITest::getMessage).collect(Collectors.joining("\n"))); + } + + @Test + void testBadDataPlusChangeFile() throws IOException { + final ValidatorCLI validatorCLI = new ValidatorCLI(); + // Write test data out + final String osmPath = Paths.get(temporaryDirectory.getPath(), "testBadDataPlusChangeFile.osm").toString(); + final String changePath = Paths.get(temporaryDirectory.getPath(), "testBadDataPlusChangeFile.osc").toString(); + final String errorPath = Paths.get(temporaryDirectory.getPath(), "testBadDataPlusChangeFile.geojson").toString(); + final DataSet dataSet = new DataSet(); + final Node node = new Node(LatLon.ZERO); + node.setOsmId(1, 1); + dataSet.addPrimitive(node); + final PrintWriter printWriter = new PrintWriter(Files.newOutputStream(Paths.get(osmPath)), true); + final OsmWriter writer = OsmWriterFactory.createOsmWriter(printWriter, true, "0.6"); + writer.write(dataSet); + printWriter.flush(); + final PrintWriter changeWriter = new PrintWriter(Files.newOutputStream(Paths.get(changePath)), true); + changeWriter.write(""); + changeWriter.write(""); + changeWriter.write(""); + changeWriter.flush(); + + validatorCLI.processArguments(new String[] {"--input", osmPath, "--output", errorPath}); + final List errors = readJsonObjects(Paths.get(errorPath)); + // There is already a mapped weather buoy at 0,0 (3000), and the node has no tags (201). + assertEquals(2, errors.size()); + Files.deleteIfExists(Paths.get(errorPath)); + + validatorCLI.processArguments(new String[] {"--input", osmPath, "--change-file", changePath, "--output", errorPath}); + errors.clear(); + errors.addAll(readJsonObjects(Paths.get(errorPath))); + assertEquals(0, errors.size()); + Files.deleteIfExists(Paths.get(errorPath)); + } + + /** + * Read json objects from a file + * @param path The file to read + * @return The json objects + */ + private static List readJsonObjects(final Path path) { + if (Files.exists(path)) { + final List lines = assertDoesNotThrow(() -> Files.readAllLines(path)); + lines.replaceAll(line -> Utils.strip(line.replace((char) 0x1e, ' '))); + return lines.stream().map(str -> Json.createReader(new ByteArrayInputStream(str.getBytes(StandardCharsets.UTF_8)))) + .map(JsonReader::readObject).collect(Collectors.toList()); + } + return Collections.emptyList(); + } + + /** + * Get the validation message from a json object + * @param jsonObject The json object to parse + * @return The validator message + */ + private static String getMessage(JsonObject jsonObject) { + return jsonObject.getJsonArray("features").getValuesAs(JsonObject.class) + .stream().filter(feature -> feature.containsKey("properties")).map(feature -> feature.getJsonObject("properties")) + .filter(properties -> properties.containsKey("message")).map(properties -> properties.getJsonString("message").getString()) + .collect(Collectors.joining(",")); + } + + /** + * This exists to avoid exiting the tests. + */ + private static final class LifecycleMock extends MockUp { + @Mock + public static boolean exitJosm(boolean exit, int exitCode) { + // No-op for now + return true; + } + } + + private static final class TestHandler extends Handler { + final List logRecordList = new ArrayList<>(); + + @Override + public void publish(LogRecord record) { + this.logRecordList.add(record); + } + + @Override + public void flush() { + this.logRecordList.clear(); + } + + @Override + public void close() throws SecurityException { + this.flush(); + } + } +}