Index: trunk/src/org/openstreetmap/josm/data/osm/DataSetMerger.java
===================================================================
--- trunk/src/org/openstreetmap/josm/data/osm/DataSetMerger.java	(revision 19227)
+++ trunk/src/org/openstreetmap/josm/data/osm/DataSetMerger.java	(revision 19228)
@@ -141,4 +141,10 @@
     protected void addConflict(OsmPrimitive my, OsmPrimitive their) {
         addConflict(new Conflict<>(my, their));
+    }
+
+    private void replaceConflict(Conflict<?> oldConflict, Conflict<?> newConflict) {
+        newConflict.setMergedMap(mergedMap);
+        conflicts.remove(oldConflict);
+        conflicts.add(newConflict);
     }
 
@@ -326,6 +332,8 @@
             // otherwise too many conflicts when refreshing from the server
             // but, if source is modified, there is a conflict
+            Conflict<?> currentConflict = null;
             if (source.isModified()) {
-                addConflict(new Conflict<>(target, source, true));
+                currentConflict = new Conflict<>(target, source, true);
+                addConflict(currentConflict);
             }
             // or, if source has a referrer that is not in the target dataset there is a conflict
@@ -333,5 +341,10 @@
             for (OsmPrimitive referrer: source.getReferrers()) {
                 if (targetDataSet.getPrimitiveById(referrer.getPrimitiveId()) == null) {
-                    addConflict(new Conflict<>(target, source, true));
+                    final Conflict<?> newConflict = new Conflict<>(target, source, true);
+                    if (currentConflict != null) { // See #23930
+                        replaceConflict(currentConflict, newConflict);
+                    } else {
+                        addConflict(newConflict);
+                    }
                     target.setDeleted(false);
                     break;