Ticket #7914: 7914.patch

core/src/org/openstreetmap/josm/io/MultiFetchServerObjectReader.java

@@ -12 +12 @@
 import java.util.LinkedHashSet;
 import java.util.NoSuchElementException;
 import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Executors;
 
 import org.openstreetmap.josm.data.osm.DataSet;
 import org.openstreetmap.josm.data.osm.DataSetMerger;
@@ -253 +259 @@
      * @param idPackage  the package of ids
      * @return the request string
      */
-    protected String buildRequestString(OsmPrimitiveType type, Set<Long> idPackage) {
+    protected static String buildRequestString(OsmPrimitiveType type, Set<Long> idPackage) {
         StringBuilder sb = new StringBuilder();
         sb.append(type.getAPIName()).append("s?")
         .append(type.getAPIName()).append("s=");
@@ -276 +282 @@
      * @param id the id
      * @return the request string
      */
-    protected String buildRequestString(OsmPrimitiveType type, long id) {
+    protected static String buildRequestString(OsmPrimitiveType type, long id) {
         StringBuilder sb = new StringBuilder();
         sb.append(type.getAPIName()).append("s?")
         .append(type.getAPIName()).append("s=")
@@ -284 +290 @@
         return sb.toString();
     }
 
-    /**
-     * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}.
-     * The retrieved primitives are merged to {@link #outputDataSet}.
-     *
-     * @param type the type
-     * @param pkg the package of ids
-     * @exception OsmTransferException thrown if an error occurs while communicating with the API server
-     *
-     */
-    protected void multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException {
-        String request = buildRequestString(type, pkg);
-        final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE);
-        if (in == null) return;
-        progressMonitor.subTask(tr("Downloading OSM data..."));
-        try {
-            DataSet loaded = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false));
-            rememberNodesOfIncompleteWaysToLoad(loaded);
-            merge(loaded);
-        } catch(Exception e) {
-            throw new OsmTransferException(e);
-        }
-    }
-
-    /**
-     * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}.
-     * The retrieved primitive is merged to {@link #outputDataSet}.
-     *
-     * @param type the type
-     * @param id the id
-     * @exception OsmTransferException thrown if an error occurs while communicating with the API server
-     *
-     */
-    protected void singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException {
-        String request = buildRequestString(type, id);
-        final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE);
-        if (in == null)
-            return;
-        progressMonitor.subTask(tr("Downloading OSM data..."));
-        try {
-            DataSet loaded = OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false));
-            rememberNodesOfIncompleteWaysToLoad(loaded);
-            merge(loaded);
-        } catch(Exception e) {
-            throw new OsmTransferException(e);
-        }
-    }
-
-    /**
-     * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}.
-     * The retrieved primitives are merged to {@link #outputDataSet}.
-     *
-     * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404).
-     * If the set is fetched with this method it is possible to find out which of the ids doesn't exist.
-     * Unfortunatelly, the server does not provide an error header or an error body for a 404 reply.
-     *
-     * @param type the type
-     * @param pkg the set of ids
-     * @exception OsmTransferException thrown if an error occurs while communicating with the API server
-     *
-     */
-    protected void singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException {
-        for (long id : pkg) {
-            try {
-                String msg = "";
-                switch(type) {
-                case NODE: msg = tr("Fetching node with id {0} from ''{1}''", id, OsmApi.getOsmApi().getBaseUrl()); break;
-                case WAY: msg = tr("Fetching way with id {0} from ''{1}''", id, OsmApi.getOsmApi().getBaseUrl()); break;
-                case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, OsmApi.getOsmApi().getBaseUrl()); break;
-                }
-                progressMonitor.setCustomText(msg);
-                singleGetId(type, id, progressMonitor);
-            } catch(OsmApiException e) {
-                if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
-                    System.out.println(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id)));
-                    missingPrimitives.add(new SimplePrimitiveId(id, type));
-                    continue;
-                }
-                throw e;
-            }
-        }
-    }
-
     protected void rememberNodesOfIncompleteWaysToLoad(DataSet from) {
         for (Way w: from.getWays()) {
             if (w.hasIncompleteNodes()) {
@@ -396 +320 @@
      * @param type the  type
      * @exception OsmTransferException thrown if an error occurs while communicating with the API server
      */
-    protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException{
+    protected void fetchPrimitives(Set<Long> ids, OsmPrimitiveType type, ProgressMonitor progressMonitor) throws OsmTransferException {
         String msg = "";
         switch(type) {
         case NODE: msg = tr("Fetching a package of nodes from ''{0}''", OsmApi.getOsmApi().getBaseUrl()); break;
@@ -405 +329 @@
         }
         progressMonitor.setTicksCount(ids.size());
         progressMonitor.setTicks(0);
+        // The complete set containg all primitives to fetch
         Set<Long> toFetch = new HashSet<Long>(ids);
-        while(! toFetch.isEmpty() && !isCanceled()) {
-            Set<Long> pkg = extractIdPackage(toFetch);
+        // Build a list of fetchers that will  download smaller sets containing only MAX_IDS_PER_REQUEST (200) primitives each
+        // On a multicore architecture, we will run up to MAX_DOWNLOAD_THREADS concurrent fetchers.
+        Executor exec = Executors.newFixedThreadPool(
+                Math.min(OsmApi.MAX_DOWNLOAD_THREADS, Runtime.getRuntime().availableProcessors()));
+        CompletionService<FetchResult> ecs = new ExecutorCompletionService<FetchResult>(exec);
+        int n = 0;
+        while (!toFetch.isEmpty()) {
+            ecs.submit(new Fetcher(type, extractIdPackage(toFetch), progressMonitor));
+            n++;
+        }
+        // Run the fetchers
+        for (int i = 0; i < n && !isCanceled(); i++) {
             progressMonitor.subTask(msg + "... " + progressMonitor.getTicks() + "/" + progressMonitor.getTicksCount());
             try {
-                multiGetIdPackage(type, pkg, progressMonitor);
-            } catch(OsmApiException e) {
-                if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
-                    System.out.println(tr("Server replied with response code 404, retrying with an individual request for each object."));
-                    singleGetIdPackage(type, pkg, progressMonitor);
-                } else
-                    throw e;
+                FetchResult result = ecs.take().get();
+                if (result.missingPrimitives != null) {
+                    missingPrimitives.addAll(result.missingPrimitives);
+                }
+                if (result.dataSet != null) {
+                    rememberNodesOfIncompleteWaysToLoad(result.dataSet);
+                    merge(result.dataSet);
+                }
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            } catch (ExecutionException e) {
+                e.printStackTrace();
             }
         }
     }
@@ -467 +407 @@
     public Set<PrimitiveId> getMissingPrimitives() {
         return missingPrimitives;
     }
+    
+    protected static class FetchResult {
+        public final DataSet dataSet;
+        public final Set<PrimitiveId> missingPrimitives;
+        public FetchResult(DataSet dataSet, Set<PrimitiveId> missingPrimitives) {
+            this.dataSet = dataSet;
+            this.missingPrimitives = missingPrimitives;
+        }
+    }
+    
+    protected static class Fetcher extends OsmServerReader implements Callable<FetchResult> {
+
+        private final Set<Long> pkg;
+        private final OsmPrimitiveType type;
+        private final ProgressMonitor progressMonitor;
+
+        public Fetcher(OsmPrimitiveType type, Set<Long> extractIdPackage, ProgressMonitor progressMonitor) {
+            this.pkg = extractIdPackage;
+            this.type = type;
+            this.progressMonitor = progressMonitor;
+        }
+
+        @Override
+        public DataSet parseOsm(ProgressMonitor progressMonitor) throws OsmTransferException {
+            return fetch(progressMonitor).dataSet;
+        }
+        
+        @Override
+        public FetchResult call() throws Exception {
+            return fetch(progressMonitor);
+        }
+        
+        private FetchResult fetch(ProgressMonitor progressMonitor) throws OsmTransferException {
+            try {
+                return multiGetIdPackage(type, pkg, progressMonitor);
+            } catch (OsmApiException e) {
+                if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
+                    System.out.println(tr("Server replied with response code 404, retrying with an individual request for each object."));
+                    return singleGetIdPackage(type, pkg, progressMonitor);
+                } else {
+                    throw e;
+                }
+            }
+        }
+        
+        /**
+         * invokes a Multi Get for a set of ids and a given {@link OsmPrimitiveType}.
+         * The retrieved primitives are merged to {@link #outputDataSet}.
+         *
+         * @param type the type
+         * @param pkg the package of ids
+         * @exception OsmTransferException thrown if an error occurs while communicating with the API server
+         *
+         */
+        protected FetchResult multiGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException {
+            String request = buildRequestString(type, pkg);
+            final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE);
+            if (in == null) return null;
+            progressMonitor.subTask(tr("Downloading OSM data..."));
+            try {
+                return new FetchResult(OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(pkg.size(), false)), null);
+            } catch (Exception e) {
+                throw new OsmTransferException(e);
+            }
+        }
+
+        /**
+         * invokes a Multi Get for a single id and a given {@link OsmPrimitiveType}.
+         * The retrieved primitive is merged to {@link #outputDataSet}.
+         *
+         * @param type the type
+         * @param id the id
+         * @exception OsmTransferException thrown if an error occurs while communicating with the API server
+         *
+         */
+        protected DataSet singleGetId(OsmPrimitiveType type, long id, ProgressMonitor progressMonitor) throws OsmTransferException {
+            String request = buildRequestString(type, id);
+            final InputStream in = getInputStream(request, NullProgressMonitor.INSTANCE);
+            if (in == null) return null;
+            progressMonitor.subTask(tr("Downloading OSM data..."));
+            try {
+                return OsmReader.parseDataSet(in, progressMonitor.createSubTaskMonitor(1, false));
+            } catch (Exception e) {
+                throw new OsmTransferException(e);
+            }
+        }
+
+        /**
+         * invokes a sequence of Multi Gets for individual ids in a set of ids and a given {@link OsmPrimitiveType}.
+         * The retrieved primitives are merged to {@link #outputDataSet}.
+         *
+         * This method is used if one of the ids in pkg doesn't exist (the server replies with return code 404).
+         * If the set is fetched with this method it is possible to find out which of the ids doesn't exist.
+         * Unfortunatelly, the server does not provide an error header or an error body for a 404 reply.
+         *
+         * @param type the type
+         * @param pkg the set of ids
+         * @return 
+         * @exception OsmTransferException thrown if an error occurs while communicating with the API server
+         *
+         */
+        protected FetchResult singleGetIdPackage(OsmPrimitiveType type, Set<Long> pkg, ProgressMonitor progressMonitor) throws OsmTransferException {
+            FetchResult result = new FetchResult(new DataSet(), new HashSet<PrimitiveId>());
+            String baseUrl = OsmApi.getOsmApi().getBaseUrl();
+            for (long id : pkg) {
+                try {
+                    String msg = "";
+                    switch (type) {
+                        case NODE:     msg = tr("Fetching node with id {0} from ''{1}''",     id, baseUrl); break;
+                        case WAY:      msg = tr("Fetching way with id {0} from ''{1}''",      id, baseUrl); break;
+                        case RELATION: msg = tr("Fetching relation with id {0} from ''{1}''", id, baseUrl); break;
+                    }
+                    progressMonitor.setCustomText(msg);
+                    result.dataSet.mergeFrom(singleGetId(type, id, progressMonitor));
+                } catch (OsmApiException e) {
+                    if (e.getResponseCode() == HttpURLConnection.HTTP_NOT_FOUND) {
+                        System.out.println(tr("Server replied with response code 404 for id {0}. Skipping.", Long.toString(id)));
+                        result.missingPrimitives.add(new SimplePrimitiveId(id, type));
+                    } else {
+                        throw e;
+                    }
+                }
+            }
+            return result;
+        }
+    }
 }

core/src/org/openstreetmap/josm/io/OsmApi.java

@@ -54 +54 @@
     /** max number of retries to send a request in case of HTTP 500 errors or timeouts */
     static public final int DEFAULT_MAX_NUM_RETRIES = 5;
 
+    /**
+     * max number of concurrent download threads, imposed by 
+     * <a href="http://wiki.openstreetmap.org/wiki/API_usage_policy#Technical_Usage_Requirements">
+     * OSM API usage policy.</a>
+     */
+    static public final int MAX_DOWNLOAD_THREADS = 2;
+
     /** the collection of instantiated OSM APIs */
     private static HashMap<String, OsmApi> instances = new HashMap<String, OsmApi>();