Index: src/org/openstreetmap/josm/data/osm/DataSetMerger.java
===================================================================
--- src/org/openstreetmap/josm/data/osm/DataSetMerger.java	(revision 17135)
+++ src/org/openstreetmap/josm/data/osm/DataSetMerger.java	(working copy)
@@ -424,30 +424,40 @@
             progressMonitor.beginTask(tr("Merging data..."), sourceDataSet.allPrimitives().size());
         }
         targetDataSet.update(() -> {
-            List<? extends OsmPrimitive> candidates = new ArrayList<>(targetDataSet.getNodes());
+            List<? extends OsmPrimitive> candidates = null;
             for (Node node: sourceDataSet.getNodes()) {
+                // lazy initialisation to improve performance, see #19898
+                if (node.isNew() && candidates == null) {
+                    candidates = new ArrayList<>(targetDataSet.getNodes());
+                }
                 mergePrimitive(node, candidates);
                 if (progressMonitor != null) {
                     progressMonitor.worked(1);
                 }
             }
-            candidates.clear();
-            candidates = new ArrayList<>(targetDataSet.getWays());
+            candidates = null;
             for (Way way: sourceDataSet.getWays()) {
+                // lazy initialisation to improve performance
+                if (way.isNew() && candidates == null) {
+                    candidates = new ArrayList<>(targetDataSet.getWays());
+                }
                 mergePrimitive(way, candidates);
                 if (progressMonitor != null) {
                     progressMonitor.worked(1);
                 }
             }
-            candidates.clear();
-            candidates = new ArrayList<>(targetDataSet.getRelations());
+            candidates = null;
             for (Relation relation: sourceDataSet.getRelations()) {
+                // lazy initialisation to improve performance
+                if (relation.isNew() && candidates == null) {
+                    candidates = new ArrayList<>(targetDataSet.getRelations());
+                }
                 mergePrimitive(relation, candidates);
                 if (progressMonitor != null) {
                     progressMonitor.worked(1);
                 }
             }
-            candidates.clear();
+            candidates = null;
             fixReferences();
 
             Area a = targetDataSet.getDataSourceArea();
Index: src/org/openstreetmap/josm/io/OsmServerBackreferenceReader.java
===================================================================
--- src/org/openstreetmap/josm/io/OsmServerBackreferenceReader.java	(revision 17135)
+++ src/org/openstreetmap/josm/io/OsmServerBackreferenceReader.java	(working copy)
@@ -214,8 +214,8 @@
     protected DataSet readIncompletePrimitives(DataSet ds, ProgressMonitor progressMonitor) throws OsmTransferException {
         progressMonitor.beginTask(null, 2);
         try {
-            Collection<Way> waysToCheck = new ArrayList<>(ds.getWays());
             if (isReadFull() || (primitiveType == OsmPrimitiveType.NODE && !isAllowIncompleteParentWays())) {
+                Collection<Way> waysToCheck = new ArrayList<>(ds.getWays());
                 for (Way way: waysToCheck) {
                     if (!way.isNew() && way.hasIncompleteNodes()) {
                         OsmServerObjectReader reader = new OsmServerObjectReader(way.getId(), OsmPrimitiveType.from(way), true /* read full */);