Index: src/org/openstreetmap/josm/gui/layer/AutosaveTask.java
===================================================================
--- src/org/openstreetmap/josm/gui/layer/AutosaveTask.java	(revision 17023)
+++ src/org/openstreetmap/josm/gui/layer/AutosaveTask.java	(working copy)
@@ -316,6 +316,10 @@
                 noteLayer.getNoteData().removeNoteDataUpdateListener(this);
                 cleanupLayer(noteLayer);
             }
+        } else if (e.getRemovedLayer() instanceof AbstractModifiableLayer) {
+            synchronized (layersLock) {
+                cleanupLayer((AbstractModifiableLayer) e.getRemovedLayer());
+            }
         }
     }
 
Index: src/org/openstreetmap/josm/gui/layer/markerlayer/Marker.java
===================================================================
--- src/org/openstreetmap/josm/gui/layer/markerlayer/Marker.java	(revision 17023)
+++ src/org/openstreetmap/josm/gui/layer/markerlayer/Marker.java	(working copy)
@@ -31,6 +31,8 @@
 import org.openstreetmap.josm.gui.MapView;
 import org.openstreetmap.josm.gui.layer.GpxLayer;
 import org.openstreetmap.josm.gui.preferences.display.GPXSettingsPanel;
+import org.openstreetmap.josm.spi.preferences.PreferenceChangedListener;
+import org.openstreetmap.josm.tools.Destroyable;
 import org.openstreetmap.josm.tools.ImageProvider;
 import org.openstreetmap.josm.tools.Logging;
 import org.openstreetmap.josm.tools.template_engine.ParseError;
@@ -73,7 +75,7 @@
  *
  * @author Frederik Ramm
  */
-public class Marker implements TemplateEngineDataProvider, ILatLon {
+public class Marker implements TemplateEngineDataProvider, ILatLon, Destroyable {
 
     /**
      * Plugins can add their Marker creation stuff at the bottom or top of this list
@@ -147,6 +149,7 @@
     private String cachedDefaultTemplate;
 
     private CachedLatLon coor;
+    private PreferenceChangedListener listener = l -> updateText();
 
     private boolean erroneous;
 
@@ -174,7 +177,7 @@
         this.dataProvider = dataProvider;
         this.text = text;
 
-        Preferences.main().addKeyPreferenceChangeListener("draw.rawgps." + getTextTemplateKey(), l -> updateText());
+        Preferences.main().addKeyPreferenceChangeListener(getPreferenceKey(), listener);
     }
 
     /**
@@ -369,7 +372,7 @@
     public void updateText() {
         cachedText = null;
         cachedDefaultTemplate = null;
-        cachedTemplates = new HashMap<>();
+        cachedTemplates.clear();
     }
 
     @Override
@@ -432,4 +435,14 @@
             redSymbol = null;
         }
     }
+
+    @Override
+    public void destroy() {
+        cachedTemplates.clear();
+        Preferences.main().removeKeyPreferenceChangeListener(getPreferenceKey(), listener);
+    }
+
+    private String getPreferenceKey() {
+        return "draw.rawgps." + getTextTemplateKey();
+    }
 }
Index: src/org/openstreetmap/josm/gui/layer/markerlayer/MarkerLayer.java
===================================================================
--- src/org/openstreetmap/josm/gui/layer/markerlayer/MarkerLayer.java	(revision 17023)
+++ src/org/openstreetmap/josm/gui/layer/markerlayer/MarkerLayer.java	(working copy)
@@ -171,6 +171,7 @@
         syncAudioMarker = null;
         currentMarker = null;
         fromLayer = null;
+        data.forEach(Marker::destroy);
         data.clear();
         super.destroy();
     }