Subject: [PATCH] Fix #22726: Child objects do not need to get full history all the time --- Index: src/org/openstreetmap/josm/actions/downloadtasks/DownloadOsmChangeTask.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== diff --git a/src/org/openstreetmap/josm/actions/downloadtasks/DownloadOsmChangeTask.java b/src/org/openstreetmap/josm/actions/downloadtasks/DownloadOsmChangeTask.java --- a/src/org/openstreetmap/josm/actions/downloadtasks/DownloadOsmChangeTask.java (revision 18655) +++ b/src/org/openstreetmap/josm/actions/downloadtasks/DownloadOsmChangeTask.java (date 1675958434230) @@ -3,6 +3,7 @@ import static org.openstreetmap.josm.tools.I18n.tr; +import java.io.IOException; import java.time.Instant; import java.util.Arrays; import java.util.HashMap; @@ -12,7 +13,9 @@ import java.util.Optional; import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; +import java.util.function.Function; import java.util.regex.Matcher; +import java.util.stream.Collectors; import org.openstreetmap.josm.data.Bounds; import org.openstreetmap.josm.data.osm.AbstractPrimitive; @@ -23,7 +26,9 @@ import org.openstreetmap.josm.data.osm.OsmPrimitiveType; import org.openstreetmap.josm.data.osm.PrimitiveData; import org.openstreetmap.josm.data.osm.PrimitiveId; +import org.openstreetmap.josm.data.osm.Relation; import org.openstreetmap.josm.data.osm.RelationData; +import org.openstreetmap.josm.data.osm.Way; import org.openstreetmap.josm.data.osm.WayData; import org.openstreetmap.josm.data.osm.history.History; import org.openstreetmap.josm.data.osm.history.HistoryDataSet; @@ -36,12 +41,14 @@ import org.openstreetmap.josm.gui.history.HistoryLoadTask; import org.openstreetmap.josm.gui.progress.ProgressMonitor; import org.openstreetmap.josm.io.Compression; +import org.openstreetmap.josm.io.MultiFetchServerObjectReader; import org.openstreetmap.josm.io.OsmApi; import org.openstreetmap.josm.io.OsmServerLocationReader; import org.openstreetmap.josm.io.OsmServerReader; import org.openstreetmap.josm.io.OsmTransferException; import org.openstreetmap.josm.io.UrlPatterns.OsmChangeUrlPattern; import org.openstreetmap.josm.tools.Logging; +import org.xml.sax.SAXException; /** * Task allowing to download OsmChange data (http://wiki.openstreetmap.org/wiki/OsmChange). @@ -103,6 +110,77 @@ compression); } + @Override + public void realRun() throws IOException, SAXException, OsmTransferException { + super.realRun(); + final Map toLoadNext = new HashMap<>(); + final Map toLoad = getToLoad(dataSet); + while (!toLoad.isEmpty()) { + loadLastVersions(toLoad, toLoadNext); + toLoad.putAll(toLoadNext); + toLoadNext.clear(); + } + } + + /** + * This gets the last versions of references primitives. This may enough for many of the primitives. + * @param toLoad The primitives to load + * @param toLoadNext The primitives to load next (filled by this method) + */ + private void loadLastVersions(Map toLoad, Map toLoadNext) throws OsmTransferException { + final Map> typeMap = toLoad.entrySet().stream() + .collect(Collectors.groupingBy(entry -> entry.getKey().getType(), Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); + final Map idMap = toLoad.keySet().stream() + .collect(Collectors.toMap(OsmPrimitive::getPrimitiveId, Function.identity())); + for (OsmPrimitiveType type : Arrays.asList(OsmPrimitiveType.NODE, OsmPrimitiveType.WAY, OsmPrimitiveType.RELATION)) { + if (!typeMap.containsKey(type)) { + continue; + } + final MultiFetchServerObjectReader reader = MultiFetchServerObjectReader.create(); + typeMap.get(type).forEach((primitive, instant) -> reader.append(primitive)); + final DataSet ds = reader.parseOsm(this.progressMonitor.createSubTaskMonitor(1, false)); + switch (type) { + case NODE: + for (Node node : ds.getNodes()) { + Node original = (Node) idMap.get(node.getPrimitiveId()); + if (original != null && toLoad.get(original).isAfter(node.getInstant())) { + original.load(node.save()); + } + toLoad.remove(original); + } + break; + case WAY: + for (Way way : ds.getWays()) { + Way original = (Way) idMap.get(way.getPrimitiveId()); + if (original != null && toLoad.get(original).isAfter(way.getInstant())) { + Instant date = toLoad.get(original); + original.load(way.save()); + for (Long nodeId : way.getNodeIds()) { + if (way.getDataSet().getPrimitiveById(nodeId, OsmPrimitiveType.NODE) == null) { + Node n = new Node(nodeId); + way.getDataSet().addPrimitive(n); + toLoadNext.put(n, date); + } + } + } + toLoad.remove(original); + } + break; + case RELATION: + for (Relation relation : ds.getRelations()) { + Relation original = (Relation) idMap.get(relation.getPrimitiveId()); + if (original != null && toLoad.get(original).isAfter(relation.getInstant())) { + original.load(relation.save()); + } + toLoad.remove(relation); + } + break; + default: + throw new IllegalStateException("Only Node, Ways, and Relations should be returned by the API"); + } + } + } + @Override protected void finish() { super.finish(); @@ -111,25 +189,35 @@ try { // A changeset does not contain all referred primitives, this is the map of incomplete ones // For each incomplete primitive, we'll have to get its state at date it was referred - Map toLoad = new HashMap<>(); - for (OsmPrimitive p : downloadedData.allNonDeletedPrimitives()) { - if (p.isIncomplete()) { - Instant timestamp = p.getReferrers().stream() - .filter(ref -> !ref.isTimestampEmpty()) - .findFirst() - .map(AbstractPrimitive::getInstant) - .orElse(null); - toLoad.put(p, timestamp); - } - } - if (isCanceled()) return; - // Let's load all required history - MainApplication.worker.submit(new HistoryLoaderAndListener(toLoad)); - } catch (RejectedExecutionException e) { - rememberException(e); - setFailed(true); + Map toLoad = getToLoad(downloadedData); + if (isCanceled()) return; + // Let's load all required history + MainApplication.worker.submit(new HistoryLoaderAndListener(toLoad)); + } catch (RejectedExecutionException e) { + rememberException(e); + setFailed(true); + } + } + } + + /** + * Get the primitives to load more information + * @param ds The dataset to look for incomplete primitives from + * @return The objects that still need to be loaded + */ + private static Map getToLoad(DataSet ds) { + Map toLoad = new HashMap<>(); + for (OsmPrimitive p : ds.allNonDeletedPrimitives()) { + if (p.isIncomplete()) { + Instant timestamp = p.getReferrers().stream() + .filter(ref -> !ref.isTimestampEmpty()) + .findFirst() + .map(AbstractPrimitive::getInstant) + .orElse(null); + toLoad.put(p, timestamp); } } + return toLoad; } /**