Index: src/org/openstreetmap/josm/gui/layer/OsmDataLayer.java
===================================================================
--- src/org/openstreetmap/josm/gui/layer/OsmDataLayer.java	(revision 5114)
+++ src/org/openstreetmap/josm/gui/layer/OsmDataLayer.java	(working copy)
@@ -707,6 +707,7 @@
     @Override
     public void destroy() {
         DataSet.removeSelectionListener(this);
+        data.clear();
     }
 
     public void processDatasetEvent(AbstractDatasetChangedEvent event) {