Index: /trunk/src/org/openstreetmap/josm/data/validation/tests/RelationChecker.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/data/validation/tests/RelationChecker.java	(revision 12001)
+++ /trunk/src/org/openstreetmap/josm/data/validation/tests/RelationChecker.java	(revision 12002)
@@ -8,4 +8,5 @@
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.LinkedList;
 import java.util.List;
@@ -101,5 +102,5 @@
     @Override
     public void visit(Relation n) {
-        List<Role> allroles = buildAllRoles(n);
+        Map<Role, String> allroles = buildAllRoles(n);
         if (allroles.isEmpty() && n.hasTag("type", "route")
                 && n.hasTag("route", "train", "subway", "monorail", "tram", "bus", "trolleybus", "aerialway", "ferry")) {
@@ -141,6 +142,6 @@
 
     // return Roles grouped by key
-    private static List<Role> buildAllRoles(Relation n) {
-        List<Role> allroles = new LinkedList<>();
+    private static Map<Role, String> buildAllRoles(Relation n) {
+        Map<Role, String> allroles = new LinkedHashMap<>();
 
         for (TaggingPreset p : relationpresets) {
@@ -148,5 +149,7 @@
             final Roles r = Utils.find(p.data, Roles.class);
             if (matches && r != null) {
-                allroles.addAll(r.roles);
+                for (Role role: r.roles) {
+                    allroles.put(role, p.name);
+                }
             }
         }
@@ -185,5 +188,5 @@
      *
      */
-    private boolean checkMemberExpressionAndType(List<Role> allroles, RelationMember member, Relation n) {
+    private boolean checkMemberExpressionAndType(Map<Role, String> allroles, RelationMember member, Relation n) {
         String role = member.getRole();
         String name = null;
@@ -193,9 +196,10 @@
         // iterate through all of the role definition within preset
         // and look for any matching definition
-        for (Role r: allroles) {
+        for (Map.Entry<Role, String> e : allroles.entrySet()) {
+            Role r = e.getKey();
             if (!r.isRole(role)) {
                 continue;
             }
-            name = r.key;
+            name = e.getValue();
             types.addAll(r.types);
             if (checkMemberType(r, member)) {
@@ -266,5 +270,5 @@
      * @param map contains statistics of occurances of specified role types in relation
      */
-    private void checkRoles(Relation n, List<Role> allroles, Map<String, RoleInfo> map) {
+    private void checkRoles(Relation n, Map<Role, String> allroles, Map<String, RoleInfo> map) {
         // go through all members of relation
         for (RelationMember member: n.getMembers()) {
@@ -274,5 +278,5 @@
 
         // verify role counts based on whole role sets
-        for (Role r: allroles) {
+        for (Role r: allroles.keySet()) {
             String keyname = r.key;
             if (keyname.isEmpty()) {
@@ -287,5 +291,5 @@
         for (String key : map.keySet()) {
             boolean found = false;
-            for (Role r: allroles) {
+            for (Role r: allroles.keySet()) {
                 if (r.isRole(key)) {
                     found = true;
@@ -295,5 +299,5 @@
 
             if (!found) {
-                String templates = allroles.stream().map(r -> r.key).collect(Collectors.joining("/"));
+                String templates = allroles.keySet().stream().map(r -> r.key).collect(Collectors.joining("/"));
 
                 if (!key.isEmpty()) {
Index: /trunk/test/unit/org/openstreetmap/josm/data/validation/tests/RelationCheckerTest.java
===================================================================
--- /trunk/test/unit/org/openstreetmap/josm/data/validation/tests/RelationCheckerTest.java	(revision 12001)
+++ /trunk/test/unit/org/openstreetmap/josm/data/validation/tests/RelationCheckerTest.java	(revision 12002)
@@ -139,4 +139,19 @@
 
     @Test
+    public void testBuildingMemberExpression() {
+        Relation r = createRelation("type=building");
+        r.addMember(new RelationMember("outline", new Way()));
+        r.addMember(new RelationMember("part", new Way()));
+        r.addMember(new RelationMember("level_-12", new Relation()));
+        r.addMember(new RelationMember("level_0", new Relation()));
+        r.addMember(new RelationMember("level_12", new Relation()));
+        r.addMember(new RelationMember("level_x", new Relation())); // fails
+
+        List<TestError> errors = testRelation(r);
+        assertEquals(1, errors.size());
+        assertEquals("Role 'level_x' unknown in templates 'outline/part/ridge/edge/entrance/level_-?\\d+'", errors.get(0).getDescription());
+    }
+
+    @Test
     public void testRouteMemberExpression() {
         Relation r = createRelation("type=route route=tram public_transport:version=2");