Index: src/org/openstreetmap/josm/actions/downloadtasks/DownloadReferrersTask.java =================================================================== --- src/org/openstreetmap/josm/actions/downloadtasks/DownloadReferrersTask.java (revision 15845) +++ src/org/openstreetmap/josm/actions/downloadtasks/DownloadReferrersTask.java (working copy) @@ -14,6 +14,7 @@ import javax.swing.JOptionPane; import javax.swing.SwingUtilities; +import org.openstreetmap.josm.data.Bounds; import org.openstreetmap.josm.data.osm.DataSet; import org.openstreetmap.josm.data.osm.DataSetMerger; import org.openstreetmap.josm.data.osm.Node; @@ -26,10 +27,12 @@ import org.openstreetmap.josm.gui.PleaseWaitRunnable; import org.openstreetmap.josm.gui.layer.OsmDataLayer; import org.openstreetmap.josm.gui.progress.ProgressMonitor; +import org.openstreetmap.josm.io.MultiFetchOverpassObjectReader; import org.openstreetmap.josm.io.MultiFetchServerObjectReader; import org.openstreetmap.josm.io.OsmServerBackreferenceReader; import org.openstreetmap.josm.io.OsmServerReader; import org.openstreetmap.josm.io.OsmTransferException; +import org.openstreetmap.josm.io.OverpassDownloadReader; import org.openstreetmap.josm.tools.CheckParameterUtil; import org.openstreetmap.josm.tools.ExceptionUtil; import org.xml.sax.SAXException; @@ -150,37 +153,45 @@ @Override protected void realRun() throws SAXException, IOException, OsmTransferException { try { - progressMonitor.setTicksCount(children.size()); - int i = 1; - for (PrimitiveId p : children) { - if (canceled) - return; - String msg; - String id = Long.toString(p.getUniqueId()); - switch(p.getType()) { - case NODE: msg = tr("({0}/{1}) Loading parents of node {2}", i, children.size(), id); break; - case WAY: msg = tr("({0}/{1}) Loading parents of way {2}", i, children.size(), id); break; - case RELATION: msg = tr("({0}/{1}) Loading parents of relation {2}", i, children.size(), id); break; - default: throw new AssertionError(); + if (Boolean.TRUE.equals(OverpassDownloadReader.FOR_MULTI_FETCH.get())) { + String request = MultiFetchOverpassObjectReader.genOverpassQuery(children, false, true, false); + reader = new OverpassDownloadReader(new Bounds(0, 0, 0, 0), + OverpassDownloadReader.OVERPASS_SERVER.get(), request); + DataSet ds = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false)); + new DataSetMerger(parents, ds).merge(); + } else { + progressMonitor.setTicksCount(children.size()); + int i = 1; + for (PrimitiveId p : children) { + if (canceled) + return; + String msg; + String id = Long.toString(p.getUniqueId()); + switch(p.getType()) { + case NODE: msg = tr("({0}/{1}) Loading parents of node {2}", i, children.size(), id); break; + case WAY: msg = tr("({0}/{1}) Loading parents of way {2}", i, children.size(), id); break; + case RELATION: msg = tr("({0}/{1}) Loading parents of relation {2}", i, children.size(), id); break; + default: throw new AssertionError(); + } + progressMonitor.subTask(msg); + downloadParents(p.getUniqueId(), p.getType(), progressMonitor); + i++; } - progressMonitor.subTask(msg); - downloadParents(p.getUniqueId(), p.getType(), progressMonitor); - i++; - } - Collection ways = parents.getWays(); + Collection ways = parents.getWays(); - if (!ways.isEmpty()) { - // Collect incomplete nodes of parent ways - Set nodes = ways.stream().flatMap(w -> w.getNodes().stream().filter(OsmPrimitive::isIncomplete)) - .collect(Collectors.toSet()); - if (!nodes.isEmpty()) { - reader = MultiFetchServerObjectReader.create(); - ((MultiFetchServerObjectReader) reader).append(nodes); - DataSet wayNodes = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false)); - synchronized (this) { // avoid race condition in cancel() - reader = null; + if (!ways.isEmpty()) { + // Collect incomplete nodes of parent ways + Set nodes = ways.stream().flatMap(w -> w.getNodes().stream().filter(OsmPrimitive::isIncomplete)) + .collect(Collectors.toSet()); + if (!nodes.isEmpty()) { + reader = MultiFetchServerObjectReader.create(); + ((MultiFetchServerObjectReader) reader).append(nodes); + DataSet wayNodes = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false)); + synchronized (this) { // avoid race condition in cancel() + reader = null; + } + new DataSetMerger(parents, wayNodes).merge(); } - new DataSetMerger(parents, wayNodes).merge(); } } } catch (OsmTransferException e) { @@ -189,4 +200,5 @@ lastException = e; } } + } Index: src/org/openstreetmap/josm/gui/io/DownloadPrimitivesWithReferrersTask.java =================================================================== --- src/org/openstreetmap/josm/gui/io/DownloadPrimitivesWithReferrersTask.java (revision 15845) +++ src/org/openstreetmap/josm/gui/io/DownloadPrimitivesWithReferrersTask.java (working copy) @@ -31,6 +31,7 @@ import org.openstreetmap.josm.gui.util.GuiHelper; import org.openstreetmap.josm.gui.widgets.HtmlPanel; import org.openstreetmap.josm.gui.widgets.JosmTextArea; +import org.openstreetmap.josm.io.MultiFetchOverpassObjectReader; import org.openstreetmap.josm.io.OsmTransferException; import org.openstreetmap.josm.tools.GBC; import org.xml.sax.SAXException; @@ -101,6 +102,7 @@ @Override protected void realRun() throws SAXException, IOException, OsmTransferException { + MultiFetchOverpassObjectReader.genOverpassQuery(ids, true, downloadReferrers, full); // TODO: remove getProgressMonitor().setTicksCount(ids.size()+1); // First, download primitives mainTask = new DownloadPrimitivesTask(tmpLayer, ids, full, getProgressMonitor().createSubTaskMonitor(1, false)); Index: src/org/openstreetmap/josm/io/MultiFetchOverpassObjectReader.java =================================================================== --- src/org/openstreetmap/josm/io/MultiFetchOverpassObjectReader.java (revision 15845) +++ src/org/openstreetmap/josm/io/MultiFetchOverpassObjectReader.java (working copy) @@ -1,11 +1,16 @@ // License: GPL. For details, see LICENSE file. package org.openstreetmap.josm.io; +import java.util.Collection; +import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; import java.util.stream.Collectors; import org.openstreetmap.josm.data.osm.OsmPrimitiveType; +import org.openstreetmap.josm.data.osm.PrimitiveId; import org.openstreetmap.josm.tools.Logging; /** @@ -17,7 +22,7 @@ private static String getPackageString(final OsmPrimitiveType type, Set idPackage) { return idPackage.stream().map(String::valueOf) - .collect(Collectors.joining(",", type.getAPIName() + (idPackage.size() == 1 ? "(" : "(id:"), ");")); + .collect(Collectors.joining(",", type.getAPIName() + (idPackage.size() == 1 ? "(" : "(id:"), ")")); } /** @@ -30,7 +35,7 @@ for (Entry> e : primitivesMap.entrySet()) { if (!e.getValue().isEmpty()) { countTypes++; - String list = getPackageString(e.getKey(), e.getValue()); + String list = getPackageString(e.getKey(), e.getValue()) + ";"; switch (e.getKey()) { case MULTIPOLYGON: case RELATION: @@ -56,6 +61,96 @@ return query; } + /** + * Generate single overpass query to retrieve multiple primitives. Can be used to download parents, + * children, the objects, or any combination of them. + * @param ids the collection of ids + * @param includeObjects if false, don't retrieve the primitives (e.g. only the referrers) + * @param recurseUp if true, referrers (parents) of the objects are downloaded and all nodes of these parents + * @param recurseDownRelations true: yes, recurse down to retrieve complete relations + * @return the overpass query + */ + public static String genOverpassQuery(Collection ids, boolean includeObjects, boolean recurseUp, + boolean recurseDownRelations) { + Map> primitivesMap = new TreeMap<>(); + for (PrimitiveId p : ids) { + primitivesMap.computeIfAbsent(p.getType(), k -> new TreeSet<>()).add(p.getUniqueId()); + } + return genOverpassQuery(primitivesMap, includeObjects, recurseUp, recurseDownRelations); + } + + /** + * Generate single overpass query to retrieve multiple primitives. Can be used to download parents, + * children, the objects, or any combination of them. + * @param primitivesMap map containing the primitives + * @param includeObjects if false, don't retrieve the primitives (e.g. only the referrers) + * @param recurseUp if true, referrers (parents) of the primitives are downloaded and all nodes of these parents + * @param recurseDownRelations true: yes, recurse down to retrieve complete relations + * @return the overpass query + */ + public static String genOverpassQuery(Map> primitivesMap, boolean includeObjects, + boolean recurseUp, boolean recurseDownRelations) { + boolean getWayNodes = false; + StringBuilder sb = new StringBuilder(); + StringBuilder setsToInclude = new StringBuilder("("); + for (Entry> e : primitivesMap.entrySet()) { + if (!e.getValue().isEmpty()) { + String list = getPackageString(e.getKey(), e.getValue()); + switch (e.getKey()) { + case NODE: + sb.append(list).append("->.n;"); + if (includeObjects) { + setsToInclude.append(".n;"); + } + if (recurseUp) { + sb.append(".n;way(bn)->.wn;.n;rel(bn)->.rn;"); + setsToInclude.append(".wn;.rn;"); + getWayNodes = true; + } + break; + case CLOSEDWAY: + case WAY: + sb.append(list).append("->.w;"); + if (includeObjects) { + setsToInclude.append(".w;"); + getWayNodes = true; + } + if (recurseUp) { + sb.append(".w;rel(bw)->.pw;"); + setsToInclude.append(".pw;"); + } + break; + case MULTIPOLYGON: + case RELATION: + sb.append(list).append("->.r;"); + if (includeObjects) { + setsToInclude.append(".r;"); + } + if (recurseUp) { + sb.append(".r;rel(br)->.pr;"); + setsToInclude.append(".pr;"); + } + if (recurseDownRelations) { + sb.append(".r;>>->.rm;"); + setsToInclude.append(".rm;"); + } + break; + } + } + } + setsToInclude.append(");"); + // retrieve all incomplete nodes for union of parents ? + if (getWayNodes) { + sb.append("(").append(setsToInclude).append("node(w););"); + } else { + sb.append(setsToInclude); + } + sb.append("out meta;"); + String query = sb.toString(); + Logging.debug("{0} {1}", "Possible Overpass query:", query); + return query; + } + @Override protected String getBaseUrl() { return OverpassDownloadReader.OVERPASS_SERVER.get(); Index: src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java =================================================================== --- src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java (revision 15846) +++ src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java (working copy) @@ -12,12 +12,12 @@ import java.util.Collections; import java.util.HashSet; import java.util.Iterator; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; +import java.util.TreeMap; import java.util.concurrent.Callable; import java.util.concurrent.CompletionService; import java.util.concurrent.ExecutionException; @@ -89,7 +89,7 @@ relations = new LinkedHashSet<>(); this.outputDataSet = new DataSet(); this.missingPrimitives = new LinkedHashSet<>(); - primitivesMap = new LinkedHashMap<>(); + primitivesMap = new TreeMap<>(); primitivesMap.put(OsmPrimitiveType.RELATION, relations); primitivesMap.put(OsmPrimitiveType.WAY, ways); primitivesMap.put(OsmPrimitiveType.NODE, nodes); @@ -382,11 +382,11 @@ try { if (this instanceof MultiFetchOverpassObjectReader) { // calculate a single request for all the objects - String request = ((MultiFetchOverpassObjectReader) this).buildComplexRequestString(); + String request = MultiFetchOverpassObjectReader.genOverpassQuery(primitivesMap, true, false, recurseDownRelations); if (isCanceled()) return null; OverpassDownloadReader reader = new OverpassDownloadReader(new Bounds(0, 0, 0, 0), getBaseUrl(), request); - DataSet ds = reader.parseOsm(progressMonitor.createSubTaskMonitor(n, false)); + DataSet ds = reader.parseOsm(progressMonitor.createSubTaskMonitor(1, false)); new DataSetMerger(outputDataSet, ds).merge(); checkMissing(outputDataSet, progressMonitor); } else {