source: josm/trunk/scripts/SyncEditorLayerIndex.java@ 15035

Last change on this file since 15035 was 15034, checked in by Don-vip, 7 years ago

checkstyle/PMD
  • Property svn:eol-style set to native
File size: 60.8 KB
RevLine 
[7726]1// License: GPL. For details, see LICENSE file.
[15034]2import static java.nio.charset.StandardCharsets.UTF_8;
[15033]3import static org.apache.commons.lang3.StringUtils.isBlank;
4import static org.apache.commons.lang3.StringUtils.isNotBlank;
5
6import java.io.BufferedReader;
7import java.io.IOException;
8import java.io.InputStreamReader;
[15034]9import java.io.OutputStream;
[15033]10import java.io.OutputStreamWriter;
11import java.lang.reflect.Field;
[15034]12import java.nio.file.Files;
13import java.nio.file.Paths;
[15033]14import java.text.DecimalFormat;
15import java.text.ParseException;
16import java.text.SimpleDateFormat;
17import java.util.ArrayList;
18import java.util.Arrays;
19import java.util.Calendar;
20import java.util.Collection;
21import java.util.Collections;
22import java.util.Date;
23import java.util.HashMap;
24import java.util.LinkedList;
25import java.util.List;
26import java.util.Locale;
27import java.util.Map;
28import java.util.Map.Entry;
29import java.util.Objects;
30import java.util.regex.Matcher;
31import java.util.regex.Pattern;
32import java.util.stream.Collectors;
33
34import javax.json.Json;
35import javax.json.JsonArray;
36import javax.json.JsonNumber;
37import javax.json.JsonObject;
38import javax.json.JsonReader;
39import javax.json.JsonString;
40import javax.json.JsonValue;
41
42import org.openstreetmap.gui.jmapviewer.Coordinate;
43import org.openstreetmap.josm.data.Preferences;
44import org.openstreetmap.josm.data.imagery.ImageryInfo;
45import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryBounds;
46import org.openstreetmap.josm.data.imagery.Shape;
47import org.openstreetmap.josm.data.preferences.JosmBaseDirectories;
48import org.openstreetmap.josm.data.projection.Projections;
49import org.openstreetmap.josm.data.validation.routines.DomainValidator;
50import org.openstreetmap.josm.io.imagery.ImageryReader;
51import org.openstreetmap.josm.spi.preferences.Config;
[15034]52import org.openstreetmap.josm.tools.Logging;
[15033]53import org.openstreetmap.josm.tools.OptionParser;
54import org.openstreetmap.josm.tools.OptionParser.OptionCount;
55import org.openstreetmap.josm.tools.ReflectionUtils;
56import org.xml.sax.SAXException;
57
[7726]58/**
[11854]59 * Compare and analyse the differences of the editor layer index and the JOSM imagery list.
[7726]60 * The goal is to keep both lists in sync.
61 *
[11854]62 * The editor layer index project (https://github.com/osmlab/editor-layer-index)
[11694]63 * provides also a version in the JOSM format, but the GEOJSON is the original source
[7726]64 * format, so we read that.
65 *
66 * How to run:
67 * -----------
68 *
69 * Main JOSM binary needs to be in classpath, e.g.
70 *
[15033]71 * $ java -cp ../dist/josm-custom.jar SyncEditorLayerIndex
[9667]72 *
[7726]73 * Add option "-h" to show the available command line flags.
74 */
[15033]75public class SyncEditorLayerIndex {
[14019]76
[15034]77 private static final int MAXLEN = 140;
78
[15033]79 private List<ImageryInfo> josmEntries;
80 private JsonArray eliEntries;
[7726]81
[15034]82 private final Map<String, JsonObject> eliUrls = new HashMap<>();
83 private final Map<String, ImageryInfo> josmUrls = new HashMap<>();
84 private final Map<String, ImageryInfo> josmMirrors = new HashMap<>();
85 private static final Map<String, String> oldproj = new HashMap<>();
86 private static final List<String> ignoreproj = new LinkedList<>();
[7726]87
[15033]88 private static String eliInputFile = "imagery_eli.geojson";
89 private static String josmInputFile = "imagery_josm.imagery.xml";
90 private static String ignoreInputFile = "imagery_josm.ignores.txt";
[15034]91 private static OutputStream outputFile;
92 private static OutputStreamWriter outputStream;
[15033]93 private static String optionOutput;
94 private static boolean optionShorten;
95 private static boolean optionNoSkip;
96 private static boolean optionXhtmlBody;
97 private static boolean optionXhtml;
98 private static String optionEliXml;
99 private static String optionJosmXml;
100 private static String optionEncoding;
101 private static boolean optionNoEli;
102 private Map<String, String> skip = new HashMap<>();
[7726]103
104 /**
105 * Main method.
[15033]106 * @param args program arguments
107 * @throws IOException if any I/O error occurs
108 * @throws ReflectiveOperationException if any reflective operation error occurs
109 * @throws SAXException if any SAX error occurs
[7726]110 */
[15033]111 public static void main(String[] args) throws IOException, SAXException, ReflectiveOperationException {
112 Locale.setDefault(Locale.ROOT);
[15034]113 parseCommandLineArguments(args);
[15033]114 Preferences pref = new Preferences(JosmBaseDirectories.getInstance());
115 Config.setPreferencesInstance(pref);
116 pref.init(false);
117 SyncEditorLayerIndex script = new SyncEditorLayerIndex();
118 script.setupProj();
119 script.loadSkip();
120 script.start();
121 script.loadJosmEntries();
122 if (optionJosmXml != null) {
[15034]123 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionJosmXml)), UTF_8)) {
124 script.printentries(script.josmEntries, stream);
125 }
[11964]126 }
[15033]127 script.loadELIEntries();
128 if (optionEliXml != null) {
[15034]129 try (OutputStreamWriter stream = new OutputStreamWriter(Files.newOutputStream(Paths.get(optionEliXml)), UTF_8)) {
130 script.printentries(script.eliEntries, stream);
131 }
[11964]132 }
[15033]133 script.checkInOneButNotTheOther();
134 script.checkCommonEntries();
135 script.end();
136 if (outputStream != null) {
137 outputStream.close();
[9505]138 }
[15033]139 if (outputFile != null) {
140 outputFile.close();
[9505]141 }
[7726]142 }
[9653]143
[7726]144 /**
[15033]145 * Displays help on the console
146 */
147 private static void showHelp() {
148 System.out.println(getHelp());
149 System.exit(0);
150 }
151
152 static String getHelp() {
153 return "usage: java -cp build SyncEditorLayerIndex\n" +
154 "-c,--encoding <encoding> output encoding (defaults to UTF-8 or cp850 on Windows)\n" +
[15034]155 "-e,--eli_input <eli_input> Input file for the editor layer index (geojson). " +
156 "Default is imagery_eli.geojson (current directory).\n" +
[15033]157 "-h,--help show this help\n" +
158 "-i,--ignore_input <ignore_input> Input file for the ignore list. Default is imagery_josm.ignores.txt (current directory).\n" +
[15034]159 "-j,--josm_input <josm_input> Input file for the JOSM imagery list (xml). " +
160 "Default is imagery_josm.imagery.xml (current directory).\n" +
[15033]161 "-m,--noeli don't show output for ELI problems\n" +
162 "-n,--noskip don't skip known entries\n" +
163 "-o,--output <output> Output file, - prints to stdout (default: -)\n" +
164 "-p,--elixml <elixml> ELI entries for use in JOSM as XML file (incomplete)\n" +
165 "-q,--josmxml <josmxml> JOSM entries reoutput as XML file (incomplete)\n" +
166 "-s,--shorten shorten the output, so it is easier to read in a console window\n" +
167 "-x,--xhtmlbody create XHTML body for display in a web page\n" +
168 "-X,--xhtml create XHTML for display in a web page\n";
169 }
170
171 /**
[7726]172 * Parse command line arguments.
[15033]173 * @param args program arguments
[15034]174 * @throws IOException in case of I/O error
[7726]175 */
[15034]176 static void parseCommandLineArguments(String[] args) throws IOException {
[15033]177 new OptionParser("JOSM/ELI synchronization script")
178 .addFlagParameter("help", SyncEditorLayerIndex::showHelp)
179 .addShortAlias("help", "h")
180 .addArgumentParameter("output", OptionCount.OPTIONAL, x -> optionOutput = x)
181 .addShortAlias("output", "o")
182 .addArgumentParameter("eli_input", OptionCount.OPTIONAL, x -> eliInputFile = x)
183 .addShortAlias("eli_input", "e")
184 .addArgumentParameter("josm_input", OptionCount.OPTIONAL, x -> josmInputFile = x)
185 .addShortAlias("josm_input", "j")
186 .addArgumentParameter("ignore_input", OptionCount.OPTIONAL, x -> ignoreInputFile = x)
187 .addShortAlias("ignore_input", "i")
188 .addFlagParameter("shorten", () -> optionShorten = true)
189 .addShortAlias("shorten", "s")
190 .addFlagParameter("noskip", () -> optionNoSkip = true)
191 .addShortAlias("noskip", "n")
192 .addFlagParameter("xhtmlbody", () -> optionXhtmlBody = true)
193 .addShortAlias("xhtmlbody", "x")
194 .addFlagParameter("xhtml", () -> optionXhtml = true)
195 .addShortAlias("xhtml", "X")
196 .addArgumentParameter("elixml", OptionCount.OPTIONAL, x -> optionEliXml = x)
197 .addShortAlias("elixml", "p")
198 .addArgumentParameter("josmxml", OptionCount.OPTIONAL, x -> optionJosmXml = x)
199 .addShortAlias("josmxml", "q")
200 .addFlagParameter("noeli", () -> optionNoEli = true)
201 .addShortAlias("noeli", "m")
202 .addArgumentParameter("encoding", OptionCount.OPTIONAL, x -> optionEncoding = x)
203 .addShortAlias("encoding", "c")
204 .parseOptionsOrExit(Arrays.asList(args));
[7726]205
[15034]206 if (optionOutput != null && !"-".equals(optionOutput)) {
207 outputFile = Files.newOutputStream(Paths.get(optionOutput));
[15033]208 outputStream = new OutputStreamWriter(outputFile, optionEncoding != null ? optionEncoding : "UTF-8");
209 } else if (optionEncoding != null) {
210 outputStream = new OutputStreamWriter(System.out, optionEncoding);
[7726]211 }
212 }
213
[13530]214 void setupProj() {
[15033]215 oldproj.put("EPSG:3359", "EPSG:3404");
216 oldproj.put("EPSG:3785", "EPSG:3857");
217 oldproj.put("EPSG:31297", "EPGS:31287");
218 oldproj.put("EPSG:31464", "EPSG:31468");
219 oldproj.put("EPSG:54004", "EPSG:3857");
220 oldproj.put("EPSG:102100", "EPSG:3857");
221 oldproj.put("EPSG:102113", "EPSG:3857");
222 oldproj.put("EPSG:900913", "EPGS:3857");
223 ignoreproj.add("EPSG:4267");
224 ignoreproj.add("EPSG:5221");
225 ignoreproj.add("EPSG:5514");
226 ignoreproj.add("EPSG:32019");
227 ignoreproj.add("EPSG:102066");
228 ignoreproj.add("EPSG:102067");
229 ignoreproj.add("EPSG:102685");
230 ignoreproj.add("EPSG:102711");
[13530]231 }
232
[15033]233 void loadSkip() throws IOException {
234 final Pattern pattern = Pattern.compile("^\\|\\| *(ELI|Ignore) *\\|\\| *\\{\\{\\{(.+)\\}\\}\\} *\\|\\|");
[15034]235 try (BufferedReader fr = new BufferedReader(new InputStreamReader(Files.newInputStream(Paths.get(ignoreInputFile)), UTF_8))) {
[15033]236 String line;
[11238]237
[15033]238 while ((line = fr.readLine()) != null) {
239 Matcher res = pattern.matcher(line);
240 if (res.matches()) {
241 if ("Ignore".equals(res.group(1))) {
242 skip.put(res.group(2), "green");
243 } else {
244 skip.put(res.group(2), "darkgoldenrod");
245 }
[11238]246 }
247 }
[11234]248 }
[11238]249 }
[9653]250
[15033]251 void myprintlnfinal(String s) throws IOException {
252 if (outputStream != null) {
253 outputStream.write(s + System.getProperty("line.separator"));
[9658]254 } else {
[15033]255 System.out.println(s);
[9658]256 }
257 }
258
[15033]259 void myprintln(String s) throws IOException {
260 if (skip.containsKey(s)) {
261 String color = skip.get(s);
262 skip.remove(s);
263 if (optionXhtmlBody || optionXhtml) {
[15034]264 s = "<pre style=\"margin:3px;color:"+color+"\">"
265 + s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
[9658]266 }
[15033]267 if (!optionNoSkip) {
268 return;
[9662]269 }
[15034]270 } else if (optionXhtmlBody || optionXhtml) {
[15033]271 String color =
272 s.startsWith("***") ? "black" :
273 ((s.startsWith("+ ") || s.startsWith("+++ ELI")) ? "blue" :
274 (s.startsWith("#") ? "indigo" :
275 (s.startsWith("!") ? "orange" : "red")));
[15034]276 s = "<pre style=\"margin:3px;color:"+color+"\">"+s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;")+"</pre>";
[9505]277 }
[15033]278 if ((s.startsWith("+ ") || s.startsWith("+++ ELI") || s.startsWith("#")) && optionNoEli) {
279 return;
[11965]280 }
[15033]281 myprintlnfinal(s);
[9658]282 }
283
[15033]284 void start() throws IOException {
285 if (optionXhtml) {
[15034]286 myprintlnfinal(
287 "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.0 Strict//EN\" \"http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd\">\n");
288 myprintlnfinal(
289 "<html><head><meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\"/>"+
290 "<title>JOSM - ELI differences</title></head><body>\n");
[9505]291 }
292 }
[9653]293
[15033]294 void end() throws IOException {
295 for (String s : skip.keySet()) {
296 myprintln("+++ Obsolete skip entry: " + s);
[9658]297 }
[15033]298 if (optionXhtml) {
299 myprintlnfinal("</body></html>\n");
[9658]300 }
301 }
302
[15033]303 void loadELIEntries() throws IOException {
[15034]304 try (JsonReader jr = Json.createReader(new InputStreamReader(Files.newInputStream(Paths.get(eliInputFile)), UTF_8))) {
[15033]305 eliEntries = jr.readObject().getJsonArray("features");
306 }
[9653]307
[15033]308 for (JsonValue e : eliEntries) {
309 String url = getUrlStripped(e);
[9653]310 if (url.contains("{z}")) {
[15033]311 myprintln("+++ ELI-URL uses {z} instead of {zoom}: "+url);
[15034]312 url = url.replace("{z}", "{zoom}");
[9653]313 }
[11582]314 if (eliUrls.containsKey(url)) {
[15033]315 myprintln("+++ ELI-URL is not unique: "+url);
[9505]316 } else {
[15033]317 eliUrls.put(url, e.asJsonObject());
[9505]318 }
[15033]319 JsonArray s = e.asJsonObject().get("properties").asJsonObject().getJsonArray("available_projections");
320 if (s != null) {
321 String urlLc = url.toLowerCase(Locale.ENGLISH);
322 List<String> old = new LinkedList<>();
323 for (JsonValue p : s) {
324 String proj = ((JsonString) p).getString();
325 if (oldproj.containsKey(proj) || ("CRS:84".equals(proj) && !urlLc.contains("version=1.3"))) {
326 old.add(proj);
[13530]327 }
328 }
[15033]329 if (!old.isEmpty()) {
330 myprintln("+ ELI Projections "+String.join(", ", old)+" not useful: "+getDescription(e));
[13530]331 }
332 }
[7726]333 }
[15033]334 myprintln("*** Loaded "+eliEntries.size()+" entries (ELI). ***");
[7726]335 }
[15033]336
337 String cdata(String s) {
338 return cdata(s, false);
[11968]339 }
[7726]340
[15033]341 String cdata(String s, boolean escape) {
342 if (escape) {
343 return s.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
344 } else if (s.matches("[<>&]"))
345 return "<![CDATA["+s+"]]>";
346 return s;
347 }
348
349 String maininfo(Object entry, String offset) {
350 String t = getType(entry);
351 String res = offset + "<type>"+t+"</type>\n";
352 res += offset + "<url>"+cdata(getUrl(entry))+"</url>\n";
353 if (getMinZoom(entry) != null)
354 res += offset + "<min-zoom>"+getMinZoom(entry)+"</min-zoom>\n";
355 if (getMaxZoom(entry) != null)
356 res += offset + "<max-zoom>"+getMaxZoom(entry)+"</max-zoom>\n";
357 if ("wms".equals(t)) {
358 List<String> p = getProjections(entry);
359 if (p != null) {
360 res += offset + "<projections>\n";
[15034]361 for (String c : p) {
[15033]362 res += offset + " <code>"+c+"</code>\n";
[15034]363 }
[15033]364 res += offset + "</projections>\n";
[11975]365 }
[11967]366 }
[15033]367 return res;
[11967]368 }
[12061]369
[15033]370 void printentries(List<?> entries, OutputStreamWriter stream) throws IOException {
371 DecimalFormat df = new DecimalFormat("#.#######");
372 df.setRoundingMode(java.math.RoundingMode.CEILING);
373 stream.write("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n");
374 stream.write("<imagery xmlns=\"http://josm.openstreetmap.de/maps-1.0\">\n");
375 for (Object e : entries) {
[13536]376 stream.write(" <entry"
[15034]377 + ("eli-best".equals(getQuality(e)) ? " eli-best=\"true\"" : "")
378 + (getOverlay(e) ? " overlay=\"true\"" : "")
[15033]379 + ">\n");
380 String t;
381 if (isNotBlank(t = getName(e)))
382 stream.write(" <name>"+cdata(t, true)+"</name>\n");
383 if (isNotBlank(t = getId(e)))
384 stream.write(" <id>"+t+"</id>\n");
385 if (isNotBlank(t = getCategory(e)))
386 stream.write(" <category>"+t+"</category>\n");
387 if (isNotBlank(t = getDate(e)))
388 stream.write(" <date>"+t+"</date>\n");
389 if (isNotBlank(t = getCountryCode(e)))
390 stream.write(" <country-code>"+t+"</country-code>\n");
391 if ((getDefault(e)))
392 stream.write(" <default>true</default>\n");
393 stream.write(maininfo(e, " "));
394 if (isNotBlank(t = getAttributionText(e)))
395 stream.write(" <attribution-text mandatory=\"true\">"+cdata(t, true)+"</attribution-text>\n");
396 if (isNotBlank(t = getAttributionUrl(e)))
397 stream.write(" <attribution-url>"+cdata(t)+"</attribution-url>\n");
398 if (isNotBlank(t = getLogoImage(e)))
399 stream.write(" <logo-image>"+cdata(t, true)+"</logo-image>\n");
400 if (isNotBlank(t = getLogoUrl(e)))
401 stream.write(" <logo-url>"+cdata(t)+"</logo-url>\n");
402 if (isNotBlank(t = getTermsOfUseText(e)))
403 stream.write(" <terms-of-use-text>"+cdata(t, true)+"</terms-of-use-text>\n");
404 if (isNotBlank(t = getTermsOfUseUrl(e)))
405 stream.write(" <terms-of-use-url>"+cdata(t)+"</terms-of-use-url>\n");
406 if (isNotBlank(t = getPermissionReferenceUrl(e)))
407 stream.write(" <permission-ref>"+cdata(t)+"</permission-ref>\n");
408 if ((getValidGeoreference(e)))
409 stream.write(" <valid-georeference>true</valid-georeference>\n");
410 if (isNotBlank(t = getIcon(e)))
411 stream.write(" <icon>"+cdata(t)+"</icon>\n");
412 for (Entry<String, String> d : getDescriptions(e).entrySet()) {
413 stream.write(" <description lang=\""+d.getKey()+"\">"+d.getValue()+"</description>\n");
[11975]414 }
[15033]415 for (ImageryInfo m : getMirrors(e)) {
416 stream.write(" <mirror>\n"+maininfo(m, " ")+" </mirror>\n");
[11967]417 }
[15033]418 double minlat = 1000;
419 double minlon = 1000;
420 double maxlat = -1000;
421 double maxlon = -1000;
422 String shapes = "";
423 String sep = "\n ";
[13771]424 try {
[15033]425 for (Shape s: getShapes(e)) {
426 shapes += " <shape>";
427 int i = 0;
428 for (Coordinate p: s.getPoints()) {
429 double lat = p.getLat();
430 double lon = p.getLon();
431 if (lat > maxlat) maxlat = lat;
432 if (lon > maxlon) maxlon = lon;
433 if (lat < minlat) minlat = lat;
434 if (lon < minlon) minlon = lon;
[15034]435 if ((i++ % 3) == 0) {
[15033]436 shapes += sep + " ";
[13771]437 }
[15033]438 shapes += "<point lat='"+df.format(lat)+"' lon='"+df.format(lon)+"'/>";
[11964]439 }
[15033]440 shapes += sep + "</shape>\n";
[11964]441 }
[15034]442 } catch (IllegalArgumentException ignored) {
443 Logging.trace(ignored);
[11964]444 }
[15033]445 if (!shapes.isEmpty()) {
[15034]446 stream.write(" <bounds min-lat='"+df.format(minlat)
447 +"' min-lon='"+df.format(minlon)
448 +"' max-lat='"+df.format(maxlat)
449 +"' max-lon='"+df.format(maxlon)+"'>\n");
[15033]450 stream.write(shapes + " </bounds>\n");
[11964]451 }
[15033]452 stream.write(" </entry>\n");
[11964]453 }
[15033]454 stream.write("</imagery>\n");
455 stream.close();
[11964]456 }
457
[15033]458 void loadJosmEntries() throws IOException, SAXException, ReflectiveOperationException {
459 try (ImageryReader reader = new ImageryReader(josmInputFile)) {
460 josmEntries = reader.parse();
461 }
[9667]462
[15033]463 for (ImageryInfo e : josmEntries) {
464 if (isBlank(getUrl(e))) {
465 myprintln("+++ JOSM-Entry without URL: " + getDescription(e));
466 continue;
[14554]467 }
[15033]468 if (isBlank(getName(e))) {
469 myprintln("+++ JOSM-Entry without Name: " + getDescription(e));
470 continue;
[14554]471 }
[15033]472 String url = getUrlStripped(e);
[9658]473 if (url.contains("{z}")) {
[15033]474 myprintln("+++ JOSM-URL uses {z} instead of {zoom}: "+url);
[15034]475 url = url.replace("{z}", "{zoom}");
[9658]476 }
[9505]477 if (josmUrls.containsKey(url)) {
[15033]478 myprintln("+++ JOSM-URL is not unique: "+url);
[9505]479 } else {
[15033]480 josmUrls.put(url, e);
[7726]481 }
[15033]482 for (ImageryInfo m : e.getMirrors()) {
483 url = getUrlStripped(m);
484 Field origNameField = ImageryInfo.class.getDeclaredField("origName");
485 ReflectionUtils.setObjectsAccessible(origNameField);
[15034]486 origNameField.set(m, m.getOriginalName().replaceAll(" mirror server( \\d+)?", ""));
[9658]487 if (josmUrls.containsKey(url)) {
[15033]488 myprintln("+++ JOSM-Mirror-URL is not unique: "+url);
[9658]489 } else {
[15033]490 josmUrls.put(url, m);
491 josmMirrors.put(url, m);
[9658]492 }
493 }
[7726]494 }
[15033]495 myprintln("*** Loaded "+josmEntries.size()+" entries (JOSM). ***");
[7726]496 }
497
[15033]498 void checkInOneButNotTheOther() throws IOException {
499 List<String> le = new LinkedList<>(eliUrls.keySet());
500 List<String> lj = new LinkedList<>(josmUrls.keySet());
[13593]501
[15033]502 List<String> ke = new LinkedList<>(le);
503 for (String url : ke) {
504 if (lj.contains(url)) {
505 le.remove(url);
506 lj.remove(url);
[13593]507 }
508 }
509
[15033]510 if (!le.isEmpty() && !lj.isEmpty()) {
511 ke = new LinkedList<>(le);
512 for (String urle : ke) {
513 JsonObject e = eliUrls.get(urle);
514 String ide = getId(e);
[15034]515 String urlhttps = urle.replace("http:", "https:");
[15033]516 if (lj.contains(urlhttps)) {
517 myprintln("+ Missing https: "+getDescription(e));
518 eliUrls.put(urlhttps, eliUrls.get(urle));
519 eliUrls.remove(urle);
520 le.remove(urle);
521 lj.remove(urlhttps);
522 } else if (isNotBlank(ide)) {
523 List<String> kj = new LinkedList<>(lj);
524 for (String urlj : kj) {
525 ImageryInfo j = josmUrls.get(urlj);
526 String idj = getId(j);
[13714]527
[15033]528 if (ide.equals(idj) && Objects.equals(getType(j), getType(e))) {
529 myprintln("* URL for id "+idj+" differs ("+urle+"): "+getDescription(j));
530 le.remove(urle);
531 lj.remove(urlj);
532 // replace key for this entry with JOSM URL
533 eliUrls.remove(urle);
[15034]534 eliUrls.put(urlj, e);
[15033]535 break;
[13593]536 }
537 }
[13517]538 }
[7726]539 }
540 }
[13714]541
[15033]542 myprintln("*** URLs found in ELI but not in JOSM ("+le.size()+"): ***");
543 Collections.sort(le);
[13593]544 if (!le.isEmpty()) {
[15033]545 for (String l : le) {
546 myprintln("- " + getDescription(eliUrls.get(l)));
[11412]547 }
[7726]548 }
[15033]549 myprintln("*** URLs found in JOSM but not in ELI ("+lj.size()+"): ***");
550 Collections.sort(lj);
[13593]551 if (!lj.isEmpty()) {
[15033]552 for (String l : lj) {
553 myprintln("+ " + getDescription(josmUrls.get(l)));
[11412]554 }
[7726]555 }
556 }
[9667]557
[15033]558 void checkCommonEntries() throws IOException {
[15034]559 doSameUrlButDifferentName();
560 doSameUrlButDifferentId();
561 doSameUrlButDifferentType();
562 doSameUrlButDifferentZoomBounds();
563 doSameUrlButDifferentCountryCode();
564 doSameUrlButDifferentQuality();
565 doSameUrlButDifferentDates();
566 doSameUrlButDifferentInformation();
567 doMismatchingShapes();
568 doMismatchingIcons();
569 doMiscellaneousChecks();
570 }
571
572 void doSameUrlButDifferentName() throws IOException {
[15033]573 myprintln("*** Same URL, but different name: ***");
574 for (String url : eliUrls.keySet()) {
575 JsonObject e = eliUrls.get(url);
576 if (!josmUrls.containsKey(url)) continue;
577 ImageryInfo j = josmUrls.get(url);
[15034]578 String ename = getName(e).replace("'", "\u2019");
579 String jname = getName(j).replace("'", "\u2019");
[11951]580 if (!ename.equals(jname)) {
[15033]581 myprintln("* Name differs ('"+getName(e)+"' != '"+getName(j)+"'): "+getUrl(j));
[7726]582 }
583 }
[15034]584 }
[9667]585
[15034]586 void doSameUrlButDifferentId() throws IOException {
[15033]587 myprintln("*** Same URL, but different Id: ***");
588 for (String url : eliUrls.keySet()) {
589 JsonObject e = eliUrls.get(url);
590 if (!josmUrls.containsKey(url)) continue;
591 ImageryInfo j = josmUrls.get(url);
592 String ename = getId(e);
593 String jname = getId(j);
594 if (!Objects.equals(ename, jname)) {
595 myprintln("# Id differs ('"+getId(e)+"' != '"+getId(j)+"'): "+getUrl(j));
[12126]596 }
[12226]597 }
[15034]598 }
[12126]599
[15034]600 void doSameUrlButDifferentType() throws IOException {
[15033]601 myprintln("*** Same URL, but different type: ***");
602 for (String url : eliUrls.keySet()) {
603 JsonObject e = eliUrls.get(url);
604 if (!josmUrls.containsKey(url)) continue;
605 ImageryInfo j = josmUrls.get(url);
606 if (!Objects.equals(getType(e), getType(j))) {
607 myprintln("* Type differs ("+getType(e)+" != "+getType(j)+"): "+getName(j)+" - "+getUrl(j));
[7726]608 }
609 }
[15034]610 }
[9667]611
[15034]612 void doSameUrlButDifferentZoomBounds() throws IOException {
[15033]613 myprintln("*** Same URL, but different zoom bounds: ***");
614 for (String url : eliUrls.keySet()) {
615 JsonObject e = eliUrls.get(url);
616 if (!josmUrls.containsKey(url)) continue;
617 ImageryInfo j = josmUrls.get(url);
[7726]618
[15033]619 Integer eMinZoom = getMinZoom(e);
620 Integer jMinZoom = getMinZoom(j);
[13997]621 /* dont warn for entries copied from the base of the mirror */
[15033]622 if (eMinZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
[13997]623 jMinZoom = null;
[15033]624 if (!Objects.equals(eMinZoom, jMinZoom) && !(Objects.equals(eMinZoom, 0) && jMinZoom == null)) {
625 myprintln("* Minzoom differs ("+eMinZoom+" != "+jMinZoom+"): "+getDescription(j));
[7726]626 }
[15033]627 Integer eMaxZoom = getMaxZoom(e);
628 Integer jMaxZoom = getMaxZoom(j);
[13997]629 /* dont warn for entries copied from the base of the mirror */
[15033]630 if (eMaxZoom == null && "wms".equals(getType(j)) && j.getName().contains(" mirror"))
[13997]631 jMaxZoom = null;
[15033]632 if (!Objects.equals(eMaxZoom, jMaxZoom)) {
633 myprintln("* Maxzoom differs ("+eMaxZoom+" != "+jMaxZoom+"): "+getDescription(j));
[7726]634 }
635 }
[15034]636 }
[9667]637
[15034]638 void doSameUrlButDifferentCountryCode() throws IOException {
[15033]639 myprintln("*** Same URL, but different country code: ***");
640 for (String url : eliUrls.keySet()) {
641 JsonObject e = eliUrls.get(url);
642 if (!josmUrls.containsKey(url)) continue;
643 ImageryInfo j = josmUrls.get(url);
644 String cce = getCountryCode(e);
[13931]645 if ("ZZ".equals(cce)) { /* special ELI country code */
[15033]646 cce = null;
[13931]647 }
[15033]648 if (cce != null && !cce.equals(getCountryCode(j))) {
649 myprintln("* Country code differs ("+getCountryCode(e)+" != "+getCountryCode(j)+"): "+getDescription(j));
[7726]650 }
651 }
[15034]652 }
653
654 void doSameUrlButDifferentQuality() throws IOException {
[15033]655 myprintln("*** Same URL, but different quality: ***");
656 for (String url : eliUrls.keySet()) {
657 JsonObject e = eliUrls.get(url);
[9515]658 if (!josmUrls.containsKey(url)) {
[15033]659 String q = getQuality(e);
660 if ("eli-best".equals(q)) {
661 myprintln("- Quality best entry not in JOSM for "+getDescription(e));
[9515]662 }
[15033]663 continue;
[9515]664 }
[15033]665 ImageryInfo j = josmUrls.get(url);
666 if (!Objects.equals(getQuality(e), getQuality(j))) {
667 myprintln("* Quality differs ("+getQuality(e)+" != "+getQuality(j)+"): "+getDescription(j));
[9505]668 }
[11599]669 }
[15034]670 }
671
672 void doSameUrlButDifferentDates() throws IOException {
[15033]673 myprintln("*** Same URL, but different dates: ***");
674 Pattern pattern = Pattern.compile("^(.*;)(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?$");
675 for (String url : eliUrls.keySet()) {
676 String ed = getDate(eliUrls.get(url));
677 if (!josmUrls.containsKey(url)) continue;
678 ImageryInfo j = josmUrls.get(url);
679 String jd = getDate(j);
[11612]680 // The forms 2015;- or -;2015 or 2015;2015 are handled equal to 2015
[15034]681 String ef = ed.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
[11639]682 // ELI has a strange and inconsistent used end_date definition, so we try again with subtraction by one
[15033]683 String ed2 = ed;
684 Matcher m = pattern.matcher(ed);
685 if (m.matches()) {
686 Calendar cal = Calendar.getInstance();
687 cal.set(Integer.valueOf(m.group(2)),
688 m.group(4) == null ? 0 : Integer.valueOf(m.group(4))-1,
689 m.group(6) == null ? 1 : Integer.valueOf(m.group(6)));
690 cal.add(Calendar.DAY_OF_MONTH, -1);
691 ed2 = m.group(1) + cal.get(Calendar.YEAR);
692 if (m.group(4) != null)
693 ed2 += "-" + String.format("%02d", cal.get(Calendar.MONTH)+1);
694 if (m.group(6) != null)
695 ed2 += "-" + String.format("%02d", cal.get(Calendar.DAY_OF_MONTH));
[11639]696 }
[15034]697 String ef2 = ed2.replaceAll("\\A-;", "").replaceAll(";-\\z", "").replaceAll("\\A([0-9-]+);\\1\\z", "$1");
[11639]698 if (!ed.equals(jd) && !ef.equals(jd) && !ed2.equals(jd) && !ef2.equals(jd)) {
[15033]699 String t = "'"+ed+"'";
[11612]700 if (!ed.equals(ef)) {
[15033]701 t += " or '"+ef+"'";
[11612]702 }
[11666]703 if (jd.isEmpty()) {
[15033]704 myprintln("- Missing JOSM date ("+t+"): "+getDescription(j));
[11668]705 } else if (!ed.isEmpty()) {
[15033]706 myprintln("* Date differs ('"+t+"' != '"+jd+"'): "+getDescription(j));
707 } else if (!optionNoEli) {
708 myprintln("+ Missing ELI date ('"+jd+"'): "+getDescription(j));
[11666]709 }
[11573]710 }
[11665]711 }
[15034]712 }
713
714 void doSameUrlButDifferentInformation() throws IOException {
[15033]715 myprintln("*** Same URL, but different information: ***");
716 for (String url : eliUrls.keySet()) {
717 if (!josmUrls.containsKey(url)) continue;
718 JsonObject e = eliUrls.get(url);
719 ImageryInfo j = josmUrls.get(url);
[11981]720
[15033]721 String et = getDescriptions(e).getOrDefault("en", "");
722 String jt = getDescriptions(j).getOrDefault("en", "");
[12086]723 if (!et.equals(jt)) {
[15033]724 if (jt.isEmpty()) {
725 myprintln("- Missing JOSM description ("+et+"): "+getDescription(j));
726 } else if (!et.isEmpty()) {
727 myprintln("* Description differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
728 } else if (!optionNoEli) {
729 myprintln("+ Missing ELI description ('"+jt+"'): "+getDescription(j));
[11981]730 }
731 }
732
[15033]733 et = getPermissionReferenceUrl(e);
734 jt = getPermissionReferenceUrl(j);
735 String jt2 = getTermsOfUseUrl(j);
736 if (isBlank(jt)) jt = jt2;
737 if (!Objects.equals(et, jt)) {
738 if (isBlank(jt)) {
739 myprintln("- Missing JOSM license URL ("+et+"): "+getDescription(j));
740 } else if (isNotBlank(et)) {
741 String ethttps = et.replace("http:", "https:");
742 if (isBlank(jt2) || !(jt2.equals(ethttps) || jt2.equals(et+"/") || jt2.equals(ethttps+"/"))) {
743 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
744 myprintln("+ License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13578]745 } else {
[15033]746 String ja = getAttributionUrl(j);
747 if (ja != null && (ja.equals(et) || ja.equals(ethttps) || ja.equals(et+"/") || ja.equals(ethttps+"/"))) {
748 myprintln("+ ELI License URL in JOSM Attribution: "+getDescription(j));
[13579]749 } else {
[15033]750 myprintln("* License URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13579]751 }
[13578]752 }
[13551]753 }
[15033]754 } else if (!optionNoEli) {
755 myprintln("+ Missing ELI license URL ('"+jt+"'): "+getDescription(j));
[11981]756 }
757 }
758
[15033]759 et = getAttributionUrl(e);
760 jt = getAttributionUrl(j);
761 if (!Objects.equals(et, jt)) {
762 if (isBlank(jt)) {
763 myprintln("- Missing JOSM attribution URL ("+et+"): "+getDescription(j));
764 } else if (isNotBlank(et)) {
[15034]765 String ethttps = et.replace("http:", "https:");
[15033]766 if (jt.equals(ethttps) || jt.equals(et+"/") || jt.equals(ethttps+"/")) {
767 myprintln("+ Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13518]768 } else {
[15033]769 myprintln("* Attribution URL differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13518]770 }
[15033]771 } else if (!optionNoEli) {
772 myprintln("+ Missing ELI attribution URL ('"+jt+"'): "+getDescription(j));
[11981]773 }
774 }
775
[15033]776 et = getAttributionText(e);
777 jt = getAttributionText(j);
778 if (!Objects.equals(et, jt)) {
779 if (isBlank(jt)) {
780 myprintln("- Missing JOSM attribution text ("+et+"): "+getDescription(j));
781 } else if (isNotBlank(et)) {
782 myprintln("* Attribution text differs ('"+et+"' != '"+jt+"'): "+getDescription(j));
783 } else if (!optionNoEli) {
784 myprintln("+ Missing ELI attribution text ('"+jt+"'): "+getDescription(j));
[11981]785 }
786 }
787
[15033]788 et = getProjections(e).stream().sorted().collect(Collectors.joining(" "));
789 jt = getProjections(j).stream().sorted().collect(Collectors.joining(" "));
790 if (!Objects.equals(et, jt)) {
791 if (isBlank(jt)) {
792 String t = getType(e);
793 if ("wms_endpoint".equals(t) || "tms".equals(t)) {
794 myprintln("+ ELI projections for type "+t+": "+getDescription(j));
795 } else {
796 myprintln("- Missing JOSM projections ("+et+"): "+getDescription(j));
[13592]797 }
[15033]798 } else if (isNotBlank(et)) {
799 if ("EPSG:3857 EPSG:4326".equals(et) || "EPSG:3857".equals(et) || "EPSG:4326".equals(et)) {
800 myprintln("+ ELI has minimal projections ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13538]801 } else {
[15033]802 myprintln("* Projections differ ('"+et+"' != '"+jt+"'): "+getDescription(j));
[13538]803 }
[15033]804 } else if (!optionNoEli && !"tms".equals(getType(e))) {
805 myprintln("+ Missing ELI projections ('"+jt+"'): "+getDescription(j));
[11981]806 }
807 }
[12226]808
[15033]809 boolean ed = getDefault(e);
810 boolean jd = getDefault(j);
811 if (ed != jd) {
812 if (!jd) {
813 myprintln("- Missing JOSM default: "+getDescription(j));
814 } else if (!optionNoEli) {
815 myprintln("+ Missing ELI default: "+getDescription(j));
[12226]816 }
817 }
[15033]818 boolean eo = getOverlay(e);
819 boolean jo = getOverlay(j);
820 if (eo != jo) {
821 if (!jo) {
822 myprintln("- Missing JOSM overlay flag: "+getDescription(j));
823 } else if (!optionNoEli) {
824 myprintln("+ Missing ELI overlay flag: "+getDescription(j));
[13536]825 }
826 }
[11981]827 }
[15034]828 }
829
830 void doMismatchingShapes() throws IOException {
[15033]831 myprintln("*** Mismatching shapes: ***");
832 for (String url : josmUrls.keySet()) {
833 ImageryInfo j = josmUrls.get(url);
834 int num = 1;
835 for (Shape shape : getShapes(j)) {
836 List<Coordinate> p = shape.getPoints();
[15034]837 if (!p.get(0).equals(p.get(p.size()-1))) {
[15033]838 myprintln("+++ JOSM shape "+num+" unclosed: "+getDescription(j));
[11410]839 }
[15033]840 for (int nump = 1; nump < p.size(); ++nump) {
841 if (Objects.equals(p.get(nump-1), p.get(nump))) {
842 myprintln("+++ JOSM shape "+num+" double point at "+(nump-1)+": "+getDescription(j));
[11964]843 }
844 }
[15033]845 ++num;
[11410]846 }
847 }
[15033]848 for (String url : eliUrls.keySet()) {
849 JsonObject e = eliUrls.get(url);
850 int num = 1;
851 List<Shape> s = null;
[13771]852 try {
[15033]853 s = getShapes(e);
854 for (Shape shape : s) {
855 List<Coordinate> p = shape.getPoints();
[15034]856 if (!p.get(0).equals(p.get(p.size()-1)) && !optionNoEli) {
[15033]857 myprintln("+++ ELI shape "+num+" unclosed: "+getDescription(e));
[11964]858 }
[15033]859 for (int nump = 1; nump < p.size(); ++nump) {
860 if (Objects.equals(p.get(nump-1), p.get(nump))) {
861 myprintln("+++ ELI shape "+num+" double point at "+(nump-1)+": "+getDescription(e));
[13771]862 }
863 }
[15033]864 ++num;
[11964]865 }
[15033]866 } catch (IllegalArgumentException err) {
867 String desc = getDescription(e);
868 myprintln("* Invalid data in ELI geometry for "+desc+": "+err.getMessage());
[11410]869 }
[13780]870 if (s == null || !josmUrls.containsKey(url)) {
[15033]871 continue;
[11410]872 }
[15033]873 ImageryInfo j = josmUrls.get(url);
874 List<Shape> js = getShapes(j);
875 if (s.isEmpty() && !js.isEmpty()) {
876 if (!optionNoEli) {
877 myprintln("+ No ELI shape: "+getDescription(j));
[11414]878 }
[15034]879 } else if (js.isEmpty() && !s.isEmpty()) {
[11415]880 // don't report boundary like 5 point shapes as difference
[15033]881 if (s.size() != 1 || s.get(0).getPoints().size() != 5) {
882 myprintln("- No JOSM shape: "+getDescription(j));
[11415]883 }
[15034]884 } else if (s.size() != js.size()) {
[15033]885 myprintln("* Different number of shapes ("+s.size()+" != "+js.size()+"): "+getDescription(j));
[11413]886 } else {
[15033]887 boolean[] edone = new boolean[s.size()];
888 boolean[] jdone = new boolean[js.size()];
889 for (int enums = 0; enums < s.size(); ++enums) {
890 List<Coordinate> ep = s.get(enums).getPoints();
891 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
892 List<Coordinate> jp = js.get(jnums).getPoints();
893 if (ep.size() == jp.size() && !jdone[jnums]) {
[14576]894 boolean err = false;
[15034]895 for (int nump = 0; nump < ep.size() && !err; ++nump) {
[15033]896 Coordinate ept = ep.get(nump);
897 Coordinate jpt = jp.get(nump);
[15034]898 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001)
[15033]899 err = true;
[11413]900 }
[15034]901 if (!err) {
[15033]902 edone[enums] = true;
903 jdone[jnums] = true;
904 break;
[14576]905 }
[11413]906 }
907 }
[11411]908 }
[15033]909 for (int enums = 0; enums < s.size(); ++enums) {
910 List<Coordinate> ep = s.get(enums).getPoints();
911 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
912 List<Coordinate> jp = js.get(jnums).getPoints();
913 if (ep.size() == jp.size() && !jdone[jnums]) {
[14576]914 boolean err = false;
[15033]915 for (int nump = 0; nump < ep.size() && !err; ++nump) {
916 Coordinate ept = ep.get(nump);
917 Coordinate jpt = jp.get(nump);
918 if (Math.abs(ept.getLat()-jpt.getLat()) > 0.00001 || Math.abs(ept.getLon()-jpt.getLon()) > 0.00001) {
919 String numtxt = Integer.toString(enums+1);
920 if (enums != jnums) {
921 numtxt += '/' + Integer.toString(jnums+1);
922 }
923 myprintln("* Different coordinate for point "+(nump+1)+" of shape "+numtxt+": "+getDescription(j));
924 break;
[14576]925 }
926 }
[15033]927 edone[enums] = true;
928 jdone[jnums] = true;
929 break;
[14576]930 }
931 }
932 }
[15033]933 for (int enums = 0; enums < s.size(); ++enums) {
934 List<Coordinate> ep = s.get(enums).getPoints();
935 for (int jnums = 0; jnums < js.size() && !edone[enums]; ++jnums) {
936 List<Coordinate> jp = js.get(jnums).getPoints();
937 if (!jdone[jnums]) {
938 String numtxt = Integer.toString(enums+1);
939 if (enums != jnums) {
940 numtxt += '/' + Integer.toString(jnums+1);
941 }
[15034]942 myprintln("* Different number of points for shape "+numtxt+" ("+ep.size()+" ! = "+jp.size()+")): "
943 + getDescription(j));
[15033]944 edone[enums] = true;
945 jdone[jnums] = true;
946 break;
[14576]947 }
948 }
949 }
[11410]950 }
951 }
[15034]952 }
953
954 void doMismatchingIcons() throws IOException {
[15033]955 myprintln("*** Mismatching icons: ***");
956 for (String url : eliUrls.keySet()) {
957 JsonObject e = eliUrls.get(url);
[11420]958 if (!josmUrls.containsKey(url)) {
[15033]959 continue;
[11420]960 }
[15033]961 ImageryInfo j = josmUrls.get(url);
962 String ij = getIcon(j);
963 String ie = getIcon(e);
964 boolean ijok = isNotBlank(ij);
965 boolean ieok = isNotBlank(ie);
966 if (ijok && !ieok) {
967 if (!optionNoEli) {
968 myprintln("+ No ELI icon: "+getDescription(j));
[11420]969 }
[15033]970 } else if (!ijok && ieok) {
971 myprintln("- No JOSM icon: "+getDescription(j));
972 } else if (ijok && ieok && !Objects.equals(ij, ie) && !(
[14351]973 (ie.startsWith("https://osmlab.github.io/editor-layer-index/")
974 || ie.startsWith("https://raw.githubusercontent.com/osmlab/editor-layer-index/")) &&
[14293]975 ij.startsWith("data:"))) {
[15034]976 String iehttps = ie.replace("http:", "https:");
[15033]977 if (ij.equals(iehttps)) {
978 myprintln("+ Different icons: "+getDescription(j));
[14294]979 } else {
[15033]980 myprintln("* Different icons: "+getDescription(j));
[14294]981 }
[11420]982 }
983 }
[15034]984 }
985
986 void doMiscellaneousChecks() throws IOException {
[15033]987 myprintln("*** Miscellaneous checks: ***");
988 Map<String, ImageryInfo> josmIds = new HashMap<>();
989 Collection<String> all = Projections.getAllProjectionCodes();
[13551]990 DomainValidator dv = DomainValidator.getInstance();
[15033]991 for (String url : josmUrls.keySet()) {
992 ImageryInfo j = josmUrls.get(url);
993 String id = getId(j);
994 if ("wms".equals(getType(j))) {
995 String urlLc = url.toLowerCase(Locale.ENGLISH);
996 if (getProjections(j).isEmpty()) {
997 myprintln("* WMS without projections: "+getDescription(j));
[13526]998 } else {
[15033]999 List<String> unsupported = new LinkedList<>();
1000 List<String> old = new LinkedList<>();
1001 for (String p : getProjectionsUnstripped(j)) {
1002 if ("CRS:84".equals(p)) {
1003 if (!urlLc.contains("version=1.3")) {
1004 myprintln("* CRS:84 without WMS 1.3: "+getDescription(j));
[13526]1005 }
[15033]1006 } else if (oldproj.containsKey(p)) {
1007 old.add(p);
1008 } else if (!all.contains(p) && !ignoreproj.contains(p)) {
1009 unsupported.add(p);
[13526]1010 }
1011 }
[15033]1012 if (!unsupported.isEmpty()) {
1013 myprintln("* Projections "+String.join(", ", unsupported)+" not supported by JOSM: "+getDescription(j));
[13526]1014 }
[15033]1015 for (String o : old) {
[15034]1016 myprintln("* Projection "+o+" is an old unsupported code and has been replaced by "+oldproj.get(o)+": "
1017 + getDescription(j));
[13527]1018 }
[13526]1019 }
[15033]1020 if (urlLc.contains("version=1.3") && !urlLc.contains("crs={proj}")) {
1021 myprintln("* WMS 1.3 with strange CRS specification: "+getDescription(j));
1022 } else if (urlLc.contains("version=1.1") && !urlLc.contains("srs={proj}")) {
1023 myprintln("* WMS 1.1 with strange SRS specification: "+getDescription(j));
[13526]1024 }
[13511]1025 }
[15033]1026 List<String> urls = new LinkedList<>();
1027 if (!"scanex".equals(getType(j))) {
1028 urls.add(url);
[13532]1029 }
[15033]1030 String jt = getPermissionReferenceUrl(j);
1031 if (isNotBlank(jt) && !"Public Domain".equalsIgnoreCase(jt))
1032 urls.add(jt);
1033 jt = getTermsOfUseUrl(j);
1034 if (isNotBlank(jt))
1035 urls.add(jt);
1036 jt = getAttributionUrl(j);
1037 if (isNotBlank(jt))
1038 urls.add(jt);
1039 jt = getIcon(j);
1040 if (isNotBlank(jt) && !jt.startsWith("data:image/png;base64,"))
1041 urls.add(jt);
1042 Pattern patternU = Pattern.compile("^https?://([^/]+?)(:\\d+)?/.*");
1043 for (String u : urls) {
1044 Matcher m = patternU.matcher(u);
1045 if (!m.matches() || u.matches(".*[ \t]+$")) {
1046 myprintln("* Strange URL '"+u+"': "+getDescription(j));
1047 } else {
[15034]1048 String domain = m.group(1).replaceAll("\\{switch:.*\\}", "x");
[15033]1049 String port = m.group(2);
1050 if (!(domain.matches("^\\d+\\.\\d+\\.\\d+\\.\\d+$")) && !dv.isValid(domain))
1051 myprintln("* Strange Domain '"+domain+"': "+getDescription(j));
1052 else if (":80".equals(port) || ":443".equals(port)) {
1053 myprintln("* Useless port '"+port+"': "+getDescription(j));
[13554]1054 }
[13551]1055 }
1056 }
1057
[15033]1058 if (josmMirrors.containsKey(url)) {
1059 continue;
[11420]1060 }
[15033]1061 if (isBlank(id)) {
1062 myprintln("* No JOSM-ID: "+getDescription(j));
1063 } else if (josmIds.containsKey(id)) {
1064 myprintln("* JOSM-ID "+id+" not unique: "+getDescription(j));
[11420]1065 } else {
[15033]1066 josmIds.put(id, j);
[11420]1067 }
[15033]1068 String d = getDate(j);
1069 if (isNotBlank(d)) {
1070 Pattern patternD = Pattern.compile("^(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?)(;(-|(\\d\\d\\d\\d)(-(\\d\\d)(-(\\d\\d))?)?))?$");
1071 Matcher m = patternD.matcher(d);
1072 if (!m.matches()) {
1073 myprintln("* JOSM-Date '"+d+"' is strange: "+getDescription(j));
[11572]1074 } else {
1075 try {
[15033]1076 Date first = verifyDate(m.group(2), m.group(4), m.group(6));
1077 Date second = verifyDate(m.group(9), m.group(11), m.group(13));
1078 if (second.compareTo(first) < 0) {
1079 myprintln("* JOSM-Date '"+d+"' is strange (second earlier than first): "+getDescription(j));
[11572]1080 }
[15034]1081 } catch (Exception e) {
[15033]1082 myprintln("* JOSM-Date '"+d+"' is strange ("+e.getMessage()+"): "+getDescription(j));
[11572]1083 }
1084 }
[11603]1085 }
[15033]1086 if (isNotBlank(getAttributionUrl(j)) && isBlank(getAttributionText(j))) {
1087 myprintln("* Attribution link without text: "+getDescription(j));
[12261]1088 }
[15033]1089 if (isNotBlank(getLogoUrl(j)) && isBlank(getLogoImage(j))) {
1090 myprintln("* Logo link without image: "+getDescription(j));
[12261]1091 }
[15033]1092 if (isNotBlank(getTermsOfUseText(j)) && isBlank(getTermsOfUseUrl(j))) {
1093 myprintln("* Terms of Use text without link: "+getDescription(j));
[12261]1094 }
[15033]1095 List<Shape> js = getShapes(j);
1096 if (!js.isEmpty()) {
1097 double minlat = 1000;
1098 double minlon = 1000;
1099 double maxlat = -1000;
1100 double maxlon = -1000;
1101 for (Shape s: js) {
1102 for (Coordinate p: s.getPoints()) {
1103 double lat = p.getLat();
1104 double lon = p.getLon();
[15034]1105 if (lat > maxlat) maxlat = lat;
1106 if (lon > maxlon) maxlon = lon;
1107 if (lat < minlat) minlat = lat;
1108 if (lon < minlon) minlon = lon;
[11422]1109 }
1110 }
[15033]1111 ImageryBounds b = j.getBounds();
1112 if (b.getMinLat() != minlat || b.getMinLon() != minlon || b.getMaxLat() != maxlat || b.getMaxLon() != maxlon) {
1113 myprintln("* Bounds do not match shape (is "+b.getMinLat()+","+b.getMinLon()+","+b.getMaxLat()+","+b.getMaxLon()
[15034]1114 + ", calculated <bounds min-lat='"+minlat+"' min-lon='"+minlon+"' max-lat='"+maxlat+"' max-lon='"+maxlon+"'>): "
1115 + getDescription(j));
[11422]1116 }
1117 }
[15033]1118 List<String> knownCategories = Arrays.asList("photo", "map", "historicmap", "osmbasedmap", "historicphoto", "other");
1119 String cat = getCategory(j);
1120 if (isBlank(cat)) {
1121 myprintln("* No category: "+getDescription(j));
1122 } else if (!knownCategories.contains(cat)) {
1123 myprintln("* Strange category "+cat+": "+getDescription(j));
[13792]1124 }
[11420]1125 }
[7726]1126 }
[9667]1127
[15033]1128 /*
[7726]1129 * Utility functions that allow uniform access for both ImageryInfo and JsonObject.
1130 */
[15033]1131
[7726]1132 static String getUrl(Object e) {
[15033]1133 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getUrl();
1134 return ((Map<String, JsonObject>) e).get("properties").getString("url");
[7726]1135 }
[15033]1136
[12242]1137 static String getUrlStripped(Object e) {
[15034]1138 return getUrl(e).replaceAll("\\?(apikey|access_token)=.*", "");
[12242]1139 }
[15033]1140
[11572]1141 static String getDate(Object e) {
[15033]1142 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getDate() != null ? ((ImageryInfo) e).getDate() : "";
1143 JsonObject p = ((Map<String, JsonObject>) e).get("properties");
1144 String start = p.containsKey("start_date") ? p.getString("start_date") : "";
1145 String end = p.containsKey("end_date") ? p.getString("end_date") : "";
[15034]1146 if (!start.isEmpty() && !end.isEmpty())
[15033]1147 return start+";"+end;
[15034]1148 else if (!start.isEmpty())
[15033]1149 return start+";-";
[15034]1150 else if (!end.isEmpty())
[15033]1151 return "-;"+end;
1152 return "";
[11572]1153 }
[15033]1154
1155 static Date verifyDate(String year, String month, String day) throws ParseException {
1156 String date;
[15034]1157 if (year == null) {
[15033]1158 date = "3000-01-01";
[11854]1159 } else {
[15033]1160 date = year + "-" + (month == null ? "01" : month) + "-" + (day == null ? "01" : day);
[11854]1161 }
[15033]1162 SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd");
1163 df.setLenient(false);
1164 return df.parse(date);
[11572]1165 }
[15033]1166
[11420]1167 static String getId(Object e) {
[15033]1168 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getId();
1169 return ((Map<String, JsonObject>) e).get("properties").getString("id");
[11420]1170 }
[15033]1171
[7726]1172 static String getName(Object e) {
[15033]1173 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getOriginalName();
1174 return ((Map<String, JsonObject>) e).get("properties").getString("name");
[7726]1175 }
[15033]1176
1177 static List<ImageryInfo> getMirrors(Object e) {
1178 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getMirrors();
1179 return Collections.emptyList();
[11967]1180 }
[15033]1181
1182 static List<String> getProjections(Object e) {
1183 List<String> r = new ArrayList<>();
1184 List<String> u = getProjectionsUnstripped(e);
1185 if (u != null) {
1186 for (String p : u) {
1187 if (!oldproj.containsKey(p) && !("CRS:84".equals(p) && !(getUrlStripped(e).matches("(?i)version=1\\.3")))) {
1188 r.add(p);
[13530]1189 }
1190 }
1191 }
[15033]1192 return r;
[13530]1193 }
[15033]1194
1195 static List<String> getProjectionsUnstripped(Object e) {
1196 List<String> r = null;
[11975]1197 if (e instanceof ImageryInfo) {
[15033]1198 r = ((ImageryInfo) e).getServerProjections();
[11975]1199 } else {
[15033]1200 JsonValue s = ((Map<String, JsonObject>) e).get("properties").get("available_projections");
1201 if (s != null) {
1202 r = new ArrayList<>();
1203 for (JsonValue p : s.asJsonArray()) {
1204 r.add(((JsonString) p).getString());
[13530]1205 }
[11981]1206 }
[11975]1207 }
[15033]1208 return r != null ? r : Collections.emptyList();
[11975]1209 }
[15033]1210
[11410]1211 static List<Shape> getShapes(Object e) {
1212 if (e instanceof ImageryInfo) {
[15033]1213 ImageryBounds bounds = ((ImageryInfo) e).getBounds();
[15034]1214 if (bounds != null) {
[15033]1215 return bounds.getShapes();
[11411]1216 }
[15033]1217 return Collections.emptyList();
[11410]1218 }
[15033]1219 JsonValue ex = ((Map<String, JsonValue>) e).get("geometry");
1220 if (ex != null && !JsonValue.NULL.equals(ex) && !ex.asJsonObject().isNull("coordinates")) {
1221 JsonArray poly = ex.asJsonObject().getJsonArray("coordinates");
1222 List<Shape> l = new ArrayList<>();
1223 for (JsonValue shapes: poly) {
1224 Shape s = new Shape();
1225 for (JsonValue point: shapes.asJsonArray()) {
1226 String lon = point.asJsonArray().getJsonNumber(0).toString();
1227 String lat = point.asJsonArray().getJsonNumber(1).toString();
1228 s.addPoint(lat, lon);
[11410]1229 }
[15033]1230 l.add(s);
[11410]1231 }
[15033]1232 return l;
[11410]1233 }
[15033]1234 return Collections.emptyList();
[11410]1235 }
[15033]1236
[7726]1237 static String getType(Object e) {
[15033]1238 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getImageryType().getTypeString();
1239 return ((Map<String, JsonObject>) e).get("properties").getString("type");
[7726]1240 }
[15033]1241
[7726]1242 static Integer getMinZoom(Object e) {
1243 if (e instanceof ImageryInfo) {
[15033]1244 int mz = ((ImageryInfo) e).getMinZoom();
1245 return mz == 0 ? null : mz;
[7726]1246 } else {
[15033]1247 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("min_zoom");
1248 if (num == null) return null;
1249 return num.intValue();
[7726]1250 }
1251 }
[15033]1252
[7726]1253 static Integer getMaxZoom(Object e) {
1254 if (e instanceof ImageryInfo) {
[15033]1255 int mz = ((ImageryInfo) e).getMaxZoom();
1256 return mz == 0 ? null : mz;
[7726]1257 } else {
[15033]1258 JsonNumber num = ((Map<String, JsonObject>) e).get("properties").getJsonNumber("max_zoom");
1259 if (num == null) return null;
1260 return num.intValue();
[7726]1261 }
1262 }
[15033]1263
[7726]1264 static String getCountryCode(Object e) {
[15033]1265 if (e instanceof ImageryInfo) return "".equals(((ImageryInfo) e).getCountryCode()) ? null : ((ImageryInfo) e).getCountryCode();
1266 return ((Map<String, JsonObject>) e).get("properties").getString("country_code", null);
[7726]1267 }
[15033]1268
[9505]1269 static String getQuality(Object e) {
[15033]1270 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isBestMarked() ? "eli-best" : null;
1271 return (((Map<String, JsonObject>) e).get("properties").containsKey("best")
1272 && ((Map<String, JsonObject>) e).get("properties").getBoolean("best")) ? "eli-best" : null;
[9505]1273 }
[15033]1274
1275 static boolean getOverlay(Object e) {
1276 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isOverlay();
1277 return (((Map<String, JsonObject>) e).get("properties").containsKey("overlay")
1278 && ((Map<String, JsonObject>) e).get("properties").getBoolean("overlay"));
[13536]1279 }
[15033]1280
[11420]1281 static String getIcon(Object e) {
[15033]1282 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getIcon();
1283 return ((Map<String, JsonObject>) e).get("properties").getString("icon", null);
[11420]1284 }
[15033]1285
[11975]1286 static String getAttributionText(Object e) {
[15033]1287 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionText(0, null, null);
1288 try {
1289 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("text", null);
1290 } catch (NullPointerException ex) {
1291 return null;
1292 }
[11975]1293 }
[15033]1294
[11975]1295 static String getAttributionUrl(Object e) {
[15033]1296 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionLinkURL();
1297 try {
1298 return ((Map<String, JsonObject>) e).get("properties").getJsonObject("attribution").getString("url", null);
1299 } catch (NullPointerException ex) {
1300 return null;
1301 }
[11975]1302 }
[15033]1303
[11975]1304 static String getTermsOfUseText(Object e) {
[15033]1305 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseText();
1306 return null;
[11975]1307 }
[15033]1308
[11975]1309 static String getTermsOfUseUrl(Object e) {
[15033]1310 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getTermsOfUseURL();
1311 return null;
[11975]1312 }
[15033]1313
[13792]1314 static String getCategory(Object e) {
1315 if (e instanceof ImageryInfo) {
[15033]1316 return ((ImageryInfo) e).getImageryCategoryOriginalString();
[13792]1317 }
[15033]1318 return null;
[13792]1319 }
[15033]1320
[12261]1321 static String getLogoImage(Object e) {
[15033]1322 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageRaw();
1323 return null;
[12261]1324 }
[15033]1325
[12261]1326 static String getLogoUrl(Object e) {
[15033]1327 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getAttributionImageURL();
1328 return null;
[12261]1329 }
[15033]1330
[11975]1331 static String getPermissionReferenceUrl(Object e) {
[15033]1332 if (e instanceof ImageryInfo) return ((ImageryInfo) e).getPermissionReferenceURL();
1333 return ((Map<String, JsonObject>) e).get("properties").getString("license_url", null);
[11975]1334 }
[15033]1335
[15034]1336 static Map<String, String> getDescriptions(Object e) {
1337 Map<String, String> res = new HashMap<String, String>();
[11975]1338 if (e instanceof ImageryInfo) {
[15033]1339 String a = ((ImageryInfo) e).getDescription();
1340 if (a != null) res.put("en", a);
[11975]1341 } else {
[15033]1342 String a = ((Map<String, JsonObject>) e).get("properties").getString("description", null);
[15034]1343 if (a != null) res.put("en", a.replaceAll("''", "'"));
[11975]1344 }
[15033]1345 return res;
[11975]1346 }
[15033]1347
1348 static boolean getValidGeoreference(Object e) {
1349 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isGeoreferenceValid();
1350 return false;
[11975]1351 }
[15033]1352
1353 static boolean getDefault(Object e) {
1354 if (e instanceof ImageryInfo) return ((ImageryInfo) e).isDefaultEntry();
1355 return ((Map<String, JsonObject>) e).get("properties").getBoolean("default", false);
[12226]1356 }
[15033]1357
[7726]1358 String getDescription(Object o) {
[15033]1359 String url = getUrl(o);
1360 String cc = getCountryCode(o);
[7726]1361 if (cc == null) {
[15033]1362 ImageryInfo j = josmUrls.get(url);
1363 if (j != null) cc = getCountryCode(j);
[7726]1364 if (cc == null) {
[15033]1365 JsonObject e = eliUrls.get(url);
1366 if (e != null) cc = getCountryCode(e);
[7726]1367 }
1368 }
1369 if (cc == null) {
[15033]1370 cc = "";
[7726]1371 } else {
[15033]1372 cc = "["+cc+"] ";
[7726]1373 }
[15033]1374 String d = cc + getName(o) + " - " + getUrl(o);
1375 if (optionShorten) {
1376 if (d.length() > MAXLEN) d = d.substring(0, MAXLEN-1) + "...";
[7726]1377 }
[15033]1378 return d;
[7726]1379 }
1380}
Note: See TracBrowser for help on using the repository browser.