Index: src/org/openstreetmap/josm/gui/layer/markerlayer/MarkerLayer.java
===================================================================
--- src/org/openstreetmap/josm/gui/layer/markerlayer/MarkerLayer.java	(revision 18947)
+++ src/org/openstreetmap/josm/gui/layer/markerlayer/MarkerLayer.java	(working copy)
@@ -100,6 +100,7 @@
     final BasicStroke markerStroke = new StrokeProperty("draw.rawgps.markers.stroke", "1").get();
 
     private final ListenerList<IGeoImageLayer.ImageChangeListener> imageChangeListenerListenerList = ListenerList.create();
+    private MarkerMouseAdapter mouseAdapter;
 
     /**
      * The default color that is used for drawing markers.
@@ -192,12 +193,15 @@
         fromLayer = null;
         data.forEach(Marker::destroy);
         data.clear();
+        if (mouseAdapter != null)
+            mouseAdapter.getMapView().removeMouseListener(mouseAdapter);
         super.destroy();
     }
 
     @Override
     public LayerPainter attachToMapView(MapViewEvent event) {
-        event.getMapView().addMouseListener(new MarkerMouseAdapter());
+        mouseAdapter = new MarkerMouseAdapter(event.getMapView());
+        event.getMapView().addMouseListener(mouseAdapter);
 
         if (event.getMapView().playHeadMarker == null) {
             event.getMapView().playHeadMarker = PlayHeadMarker.create();
@@ -581,6 +585,12 @@
     }
 
     private final class MarkerMouseAdapter extends MouseAdapter {
+        private final MapView mv;
+
+        MarkerMouseAdapter(MapView mapView) {
+            mv = mapView;
+        }
+
         @Override
         public void mousePressed(MouseEvent e) {
             if (e.getButton() != MouseEvent.BUTTON1)
@@ -608,6 +618,13 @@
             }
             invalidate();
         }
+
+        /**
+         * @return the mapView
+         */
+        public MapView getMapView() {
+            return mv;
+        }
     }
 
     public static final class ShowHideMarkerText extends AbstractAction implements LayerAction {