Index: /trunk/build.xml
===================================================================
--- /trunk/build.xml (revision 8167)
+++ /trunk/build.xml (revision 8168)
@@ -221,4 +221,12 @@
+
+
+
+
+
+
+
+
Index: /trunk/src/org/openstreetmap/josm/Main.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/Main.java (revision 8167)
+++ /trunk/src/org/openstreetmap/josm/Main.java (revision 8168)
@@ -68,4 +68,5 @@
import org.openstreetmap.josm.data.UndoRedoHandler;
import org.openstreetmap.josm.data.ViewportData;
+import org.openstreetmap.josm.data.cache.JCSCacheManager;
import org.openstreetmap.josm.data.coor.CoordinateFormat;
import org.openstreetmap.josm.data.coor.LatLon;
@@ -1089,4 +1090,5 @@
*/
public static boolean exitJosm(boolean exit, int exitCode) {
+ JCSCacheManager.shutdown();
if (Main.saveUnsavedModifications()) {
geometry.remember("gui.geometry");
Index: /trunk/src/org/openstreetmap/josm/data/cache/BufferedImageCacheEntry.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/data/cache/BufferedImageCacheEntry.java (revision 8168)
+++ /trunk/src/org/openstreetmap/josm/data/cache/BufferedImageCacheEntry.java (revision 8168)
@@ -0,0 +1,91 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.cache;
+
+import java.awt.image.BufferedImage;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+
+import javax.imageio.ImageIO;
+
+
+/**
+ * Cache Entry that has methods to get the BufferedImage, that will be cached along in memory
+ * but will be not serialized when saved to the disk (to avoid duplication of data)
+ * @author Wiktor Niesiobędzki
+ *
+ */
+public class BufferedImageCacheEntry extends CacheEntry {
+ private static final long serialVersionUID = 1L; //version
+ // transient to avoid serialization, volatile to avoid synchronization of whole getImage() method
+ private transient volatile BufferedImage img = null;
+ private transient volatile boolean writtenToDisk = false;
+
+ /**
+ *
+ * @param content byte array containing image
+ */
+ public BufferedImageCacheEntry(byte[] content) {
+ super(content);
+ }
+
+ /**
+ * Returns BufferedImage from for the content. Subsequent calls will return the same instance,
+ * to reduce overhead of ImageIO
+ *
+ * @return BufferedImage of cache entry content
+ * @throws IOException
+ */
+ public BufferedImage getImage() throws IOException {
+ if (img != null)
+ return img;
+ synchronized(this) {
+ if (img != null)
+ return img;
+ byte[] content = getContent();
+ if (content != null) {
+ img = ImageIO.read(new ByteArrayInputStream(content));
+
+ if (writtenToDisk)
+ content = null;
+ }
+
+ }
+ return img;
+ }
+
+
+ private void writeObject(java.io.ObjectOutputStream out) throws IOException {
+ /*
+ * This method below will be needed, if Apache Commons JCS (or any other caching system), will update
+ * disk representation of object from memory, once it is put into the cache (for example - at closing the cache)
+ *
+ * For now it is not the case, as we use DiskUsagePattern.UPDATE, which on JCS shutdown doesn't write again memory
+ * contents to file, so the fact, that we've cleared never gets saved to the disk
+ *
+ * This method is commented out, as it will convert all cache entries to PNG files regardless of what was returned.
+ * It might cause recompression/change of format which may result in decreased quality of imagery
+ */
+ /* synchronized (this) {
+ if (content == null && img != null) {
+ ByteArrayOutputStream restoredData = new ByteArrayOutputStream();
+ ImageIO.write(img, "png", restoredData);
+ content = restoredData.toByteArray();
+ }
+ out.writeObject(this);
+ }
+ */
+ synchronized (this) {
+ if (content == null && img != null) {
+ throw new AssertionError("Trying to serialize (save to disk?) an BufferedImageCacheEntry that was converted to BufferedImage and no raw data is present anymore");
+ }
+ out.writeObject(this);
+ // ugly hack to wait till element will get to disk to clean the memory
+ writtenToDisk = true;
+
+ if (img != null) {
+ content = null;
+ }
+
+ }
+ }
+}
Index: /trunk/src/org/openstreetmap/josm/data/cache/CacheEntry.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/data/cache/CacheEntry.java (revision 8168)
+++ /trunk/src/org/openstreetmap/josm/data/cache/CacheEntry.java (revision 8168)
@@ -0,0 +1,29 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.cache;
+
+import java.io.Serializable;
+
+/**
+ * @author Wiktor Niesiobędzki
+ *
+ * Class that will hold JCS cache entries
+ *
+ */
+public class CacheEntry implements Serializable {
+ private static final long serialVersionUID = 1L; //version
+ protected byte[] content;
+
+ /**
+ * @param content of the cache entry
+ */
+ public CacheEntry(byte[] content) {
+ this.content = content;
+ }
+
+ /**
+ * @return cache entry content
+ */
+ public byte[] getContent() {
+ return content;
+ }
+}
Index: /trunk/src/org/openstreetmap/josm/data/cache/CacheEntryAttributes.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/data/cache/CacheEntryAttributes.java (revision 8168)
+++ /trunk/src/org/openstreetmap/josm/data/cache/CacheEntryAttributes.java (revision 8168)
@@ -0,0 +1,66 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.cache;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.jcs.engine.ElementAttributes;
+
+/**
+ * Class that contains attirubtes for JCS cache entries. Parameters are used to properly handle HTTP caching
+ *
+ * @author Wiktor Niesiobędzki
+ *
+ */
+public class CacheEntryAttributes extends ElementAttributes {
+ private static final long serialVersionUID = 1L; //version
+ private Map attrs = new HashMap();
+ private final static String NO_TILE_AT_ZOOM = "noTileAtZoom";
+ private final static String ETAG = "Etag";
+ private final static String LAST_MODIFICATION = "lastModification";
+ private final static String EXPIRATION_TIME = "expirationTime";
+
+ public CacheEntryAttributes() {
+ super();
+ attrs.put(NO_TILE_AT_ZOOM, "false");
+ attrs.put(ETAG, null);
+ attrs.put(LAST_MODIFICATION, "0");
+ attrs.put(EXPIRATION_TIME, "0");
+ }
+
+ public boolean isNoTileAtZoom() {
+ return Boolean.toString(true).equals(attrs.get(NO_TILE_AT_ZOOM));
+ }
+ public void setNoTileAtZoom(boolean noTileAtZoom) {
+ attrs.put(NO_TILE_AT_ZOOM, Boolean.toString(noTileAtZoom));
+ }
+ public String getEtag() {
+ return attrs.get(ETAG);
+ }
+ public void setEtag(String etag) {
+ attrs.put(ETAG, etag);
+ }
+
+ private long getLongAttr(String key) {
+ try {
+ return Long.parseLong(attrs.get(key));
+ } catch (NumberFormatException e) {
+ attrs.put(key, "0");
+ return 0;
+ }
+ }
+
+ public long getLastModification() {
+ return getLongAttr(LAST_MODIFICATION);
+ }
+ public void setLastModification(long lastModification) {
+ attrs.put(LAST_MODIFICATION, Long.toString(lastModification));
+ }
+ public long getExpirationTime() {
+ return getLongAttr(EXPIRATION_TIME);
+ }
+ public void setExpirationTime(long expirationTime) {
+ attrs.put(EXPIRATION_TIME, Long.toString(expirationTime));
+ }
+
+}
Index: /trunk/src/org/openstreetmap/josm/data/cache/ICachedLoaderJob.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/data/cache/ICachedLoaderJob.java (revision 8168)
+++ /trunk/src/org/openstreetmap/josm/data/cache/ICachedLoaderJob.java (revision 8168)
@@ -0,0 +1,47 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.cache;
+
+import java.net.URL;
+
+
+/**
+ *
+ * @author Wiktor Niesiobędzki
+ *
+ * @param cache key type
+ */
+public interface ICachedLoaderJob {
+ /**
+ * returns cache entry key
+ *
+ * @param tile
+ * @return cache key for tile
+ */
+ public K getCacheKey();
+
+ /**
+ * method to get download URL for Job
+ * @return URL that should be fetched
+ *
+ */
+ public URL getUrl();
+ /**
+ * implements the main algorithm for fetching
+ */
+ public void run();
+
+ /**
+ * fetches object from cache, or returns null when object is not found
+ *
+ * @return filled tile with data or null when no cache entry found
+ */
+ public CacheEntry get();
+
+ /**
+ * Submit job for background fetch, and listener will be
+ * fed with value object
+ *
+ * @param listener
+ */
+ public void submit(ICachedLoaderListener listener);
+}
Index: /trunk/src/org/openstreetmap/josm/data/cache/ICachedLoaderListener.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/data/cache/ICachedLoaderListener.java (revision 8168)
+++ /trunk/src/org/openstreetmap/josm/data/cache/ICachedLoaderListener.java (revision 8168)
@@ -0,0 +1,13 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.cache;
+
+public interface ICachedLoaderListener {
+ /**
+ * Will be called when K object was successfully downloaded
+ *
+ * @param data
+ * @param success
+ */
+ public void loadingFinished(CacheEntry data, boolean success);
+
+}
Index: /trunk/src/org/openstreetmap/josm/data/cache/JCSCacheManager.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/data/cache/JCSCacheManager.java (revision 8168)
+++ /trunk/src/org/openstreetmap/josm/data/cache/JCSCacheManager.java (revision 8168)
@@ -0,0 +1,180 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.cache;
+
+import java.io.File;
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.Properties;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.LogRecord;
+import java.util.logging.Logger;
+
+import org.apache.commons.jcs.access.CacheAccess;
+import org.apache.commons.jcs.auxiliary.AuxiliaryCache;
+import org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCache;
+import org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes;
+import org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheManager;
+import org.apache.commons.jcs.engine.CompositeCacheAttributes;
+import org.apache.commons.jcs.engine.behavior.ICompositeCacheAttributes.DiskUsagePattern;
+import org.apache.commons.jcs.engine.control.CompositeCache;
+import org.apache.commons.jcs.engine.control.CompositeCacheManager;
+import org.apache.commons.jcs.utils.serialization.StandardSerializer;
+import org.openstreetmap.gui.jmapviewer.FeatureAdapter;
+import org.openstreetmap.josm.Main;
+import org.openstreetmap.josm.data.preferences.IntegerProperty;
+
+
+/**
+ * @author Wiktor Niesiobędzki
+ *
+ * Wrapper class for JCS Cache. Sets some sane environment and returns instances of cache objects.
+ * Static configuration for now assumes some small LRU cache in memory and larger LRU cache on disk
+ *
+ */
+public class JCSCacheManager {
+ private static final Logger log = FeatureAdapter.getLogger(JCSCacheManager.class.getCanonicalName());
+
+ private static volatile CompositeCacheManager cacheManager = null;
+ private static long maxObjectTTL = Long.MAX_VALUE;
+ private final static String PREFERENCE_PREFIX = "jcs.cache";
+
+ /**
+ * default objects to be held in memory by JCS caches (per region)
+ */
+ public static final IntegerProperty DEFAULT_MAX_OBJECTS_IN_MEMORY = new IntegerProperty(PREFERENCE_PREFIX + ".max_objects_in_memory", 1000);
+
+ private static void initialize() throws IOException {
+ File cacheDir = new File(Main.pref.getCacheDirectory(), "jcs");
+
+ if ((!cacheDir.exists() && !cacheDir.mkdirs()))
+ throw new IOException("Cannot access cache directory");
+
+ // raising logging level gives ~500x performance gain
+ // http://westsworld.dk/blog/2008/01/jcs-and-performance/
+ Logger jcsLog = Logger.getLogger("org.apache.commons.jcs");
+ jcsLog.setLevel(Level.INFO);
+ jcsLog.setUseParentHandlers(false);
+ //Logger.getLogger("org.apache.common").setUseParentHandlers(false);
+ // we need a separate handler from Main's, as we downgrade LEVEL.INFO to DEBUG level
+ jcsLog.addHandler(new Handler() {
+ @Override
+ public void publish(LogRecord record) {
+ String msg = MessageFormat.format(record.getMessage(), record.getParameters());
+ if (record.getLevel().intValue() >= Level.SEVERE.intValue()) {
+ Main.error(msg);
+ } else if (record.getLevel().intValue() >= Level.WARNING.intValue()) {
+ Main.warn(msg);
+ // downgrade INFO level to debug, as JCS is too verbose at INFO level
+ } else if (record.getLevel().intValue() >= Level.INFO.intValue()) {
+ Main.debug(msg);
+ } else {
+ Main.trace(msg);
+ }
+ }
+
+ @Override
+ public void flush() {
+ }
+
+ @Override
+ public void close() throws SecurityException {
+ }
+ });
+
+
+ CompositeCacheManager cm = CompositeCacheManager.getUnconfiguredInstance();
+ // this could be moved to external file
+ Properties props = new Properties();
+ // these are default common to all cache regions
+ // use of auxiliary cache and sizing of the caches is done with giving proper geCache(...) params
+ props.setProperty("jcs.default.cacheattributes", org.apache.commons.jcs.engine.CompositeCacheAttributes.class.getCanonicalName());
+ props.setProperty("jcs.default.cacheattributes.MaxObjects", DEFAULT_MAX_OBJECTS_IN_MEMORY.get().toString());
+ props.setProperty("jcs.default.cacheattributes.UseMemoryShrinker", "true");
+ props.setProperty("jcs.default.cacheattributes.DiskUsagePatternName", "UPDATE"); // store elements on disk on put
+ props.setProperty("jcs.default.elementattributes", CacheEntryAttributes.class.getCanonicalName());
+ props.setProperty("jcs.default.elementattributes.IsEternal", "false");
+ props.setProperty("jcs.default.elementattributes.MaxLife", Long.toString(maxObjectTTL));
+ props.setProperty("jcs.default.elementattributes.IdleTime", Long.toString(maxObjectTTL));
+ props.setProperty("jcs.default.elementattributes.IsSpool", "true");
+ cm.configure(props);
+ cacheManager = cm;
+
+ }
+
+ /**
+ * Returns configured cache object for named cache region
+ * @param cacheName region name
+ * @return cache access object
+ * @throws IOException if directory is not found
+ */
+ public static CacheAccess getCache(String cacheName) throws IOException {
+ return getCache(cacheName, DEFAULT_MAX_OBJECTS_IN_MEMORY.get().intValue(), 0, null);
+ }
+
+ /**
+ * Returns configured cache object with defined limits of memory cache and disk cache
+ * @param cacheName region name
+ * @param maxMemoryObjects number of objects to keep in memory
+ * @param maxDiskObjects number of objects to keep on disk (if cachePath provided)
+ * @param cachePath path to disk cache. if null, no disk cache will be created
+ * @return cache access object
+ * @throws IOException if directory is not found
+ */
+ public static CacheAccess getCache(String cacheName, int maxMemoryObjects, int maxDiskObjects, String cachePath) throws IOException {
+ if (cacheManager != null)
+ return getCacheInner(cacheName, maxMemoryObjects, maxDiskObjects, cachePath);
+
+ synchronized (JCSCacheManager.class) {
+ if (cacheManager == null)
+ initialize();
+ return getCacheInner(cacheName, maxMemoryObjects, maxDiskObjects, cachePath);
+ }
+ }
+
+
+ @SuppressWarnings("unchecked")
+ private static CacheAccess getCacheInner(String cacheName, int maxMemoryObjects, int maxDiskObjects, String cachePath) {
+ CompositeCache cc = cacheManager.getCache(cacheName, getCacheAttributes(maxMemoryObjects));
+
+ if (cachePath != null) {
+ IndexedDiskCacheAttributes diskAttributes = getDiskCacheAttributes(maxDiskObjects, cachePath);
+ diskAttributes.setCacheName(cacheName);
+ IndexedDiskCache diskCache = IndexedDiskCacheManager.getInstance(null, null, new StandardSerializer()).getCache(diskAttributes);
+ cc.setAuxCaches(new AuxiliaryCache[]{diskCache});
+ }
+ return new CacheAccess(cc);
+ }
+
+ /**
+ * Close all files to ensure, that all indexes and data are properly written
+ */
+ public static void shutdown() {
+ // use volatile semantics to get consistent object
+ CompositeCacheManager localCacheManager = cacheManager;
+ if (localCacheManager != null) {
+ localCacheManager.shutDown();
+ }
+ }
+
+ private static IndexedDiskCacheAttributes getDiskCacheAttributes(int maxDiskObjects, String cachePath) {
+ IndexedDiskCacheAttributes ret = new IndexedDiskCacheAttributes();
+ ret.setMaxKeySize(maxDiskObjects);
+ if (cachePath != null) {
+ File path = new File(cachePath);
+ if (!path.exists() && !path.mkdirs()) {
+ log.log(Level.WARNING, "Failed to create cache path: {0}", cachePath);
+ } else {
+ ret.setDiskPath(path);
+ }
+ }
+ return ret;
+ }
+
+ private static CompositeCacheAttributes getCacheAttributes(int maxMemoryElements) {
+ CompositeCacheAttributes ret = new CompositeCacheAttributes();
+ ret.setMaxObjects(maxMemoryElements);
+ ret.setDiskUsagePattern(DiskUsagePattern.UPDATE);
+ return ret;
+ }
+}
Index: /trunk/src/org/openstreetmap/josm/data/cache/JCSCachedTileLoaderJob.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/data/cache/JCSCachedTileLoaderJob.java (revision 8168)
+++ /trunk/src/org/openstreetmap/josm/data/cache/JCSCachedTileLoaderJob.java (revision 8168)
@@ -0,0 +1,409 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.cache;
+
+import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URLConnection;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Executor;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
+import org.apache.commons.jcs.engine.behavior.ICacheElement;
+import org.openstreetmap.gui.jmapviewer.FeatureAdapter;
+import org.openstreetmap.josm.data.preferences.IntegerProperty;
+
+/**
+ * @author Wiktor Niesiobędzki
+ *
+ * @param cache entry key type
+ *
+ * Generic loader for HTTP based tiles. Uses custom attribute, to check, if entry has expired
+ * according to HTTP headers sent with tile. If so, it tries to verify using Etags
+ * or If-Modified-Since / Last-Modified.
+ *
+ * If the tile is not valid, it will try to download it from remote service and put it
+ * to cache. If remote server will fail it will try to use stale entry.
+ *
+ * This class will keep only one Job running for specified tile. All others will just finish, but
+ * listeners will be gathered and notified, once download job will be finished
+ */
+public abstract class JCSCachedTileLoaderJob implements ICachedLoaderJob, Runnable {
+ private static final Logger log = FeatureAdapter.getLogger(JCSCachedTileLoaderJob.class.getCanonicalName());
+ protected static final long DEFAULT_EXPIRE_TIME = 1000L * 60 * 60 * 24 * 7; // 7 days
+ // Limit for the max-age value send by the server.
+ protected static final long EXPIRE_TIME_SERVER_LIMIT = 1000L * 60 * 60 * 24 * 28; // 4 weeks
+ // Absolute expire time limit. Cached tiles that are older will not be used,
+ // even if the refresh from the server fails.
+ protected static final long ABSOLUTE_EXPIRE_TIME_LIMIT = Long.MAX_VALUE; // unlimited
+
+ /**
+ * maximum download threads that will be started
+ */
+ public static IntegerProperty THREAD_LIMIT = new IntegerProperty("cache.jcs.max_threads", 10);
+ private static Executor DOWNLOAD_JOB_DISPATCHER = new ThreadPoolExecutor(
+ 2, // we have a small queue, so threads will be quickly started (threads are started only, when queue is full)
+ THREAD_LIMIT.get().intValue(), // do not this number of threads
+ 30, // keepalive for thread
+ TimeUnit.SECONDS,
+ // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see)
+ new LinkedBlockingDeque(5) {
+ /* keep the queue size fairly small, we do not want to
+ download a lot of tiles, that user is not seeing anyway */
+ @Override
+ public boolean offer(Runnable t) {
+ return super.offerFirst(t);
+ }
+
+ @Override
+ public Runnable remove() {
+ return super.removeFirst();
+ }
+ }
+ );
+ private static ConcurrentMap> inProgress = new ConcurrentHashMap<>();
+ private static ConcurrentMap useHead = new ConcurrentHashMap<>();
+
+ private long now; // when the job started
+
+ private ICacheAccess cache;
+ private ICacheElement cacheElement;
+ protected V cacheData = null;
+ protected CacheEntryAttributes attributes = null;
+
+ // HTTP connection parameters
+ private int connectTimeout;
+ private int readTimeout;
+ private Map headers;
+
+ /**
+ * @param cache cache instance that we will work on
+ * @param headers
+ * @param readTimeout
+ * @param connectTimeout
+ */
+ public JCSCachedTileLoaderJob(ICacheAccess cache,
+ int connectTimeout, int readTimeout,
+ Map headers) {
+
+ this.cache = cache;
+ this.now = System.currentTimeMillis();
+ this.connectTimeout = connectTimeout;
+ this.readTimeout = readTimeout;
+ this.headers = headers;
+ }
+
+ private void ensureCacheElement() {
+ if (cacheElement == null && getCacheKey() != null) {
+ cacheElement = cache.getCacheElement(getCacheKey());
+ if (cacheElement != null) {
+ attributes = (CacheEntryAttributes) cacheElement.getElementAttributes();
+ cacheData = cacheElement.getVal();
+ }
+ }
+ }
+
+ public V get() {
+ ensureCacheElement();
+ return cacheData;
+ }
+
+ @Override
+ public void submit(ICachedLoaderListener listener) {
+ boolean first = false;
+ String url = getUrl().toString();
+ if (url == null) {
+ log.log(Level.WARNING, "No url returned for: {0}, skipping", getCacheKey());
+ return;
+ }
+ synchronized (inProgress) {
+ Set newListeners = inProgress.get(url);
+ if (newListeners == null) {
+ newListeners = new HashSet<>();
+ inProgress.put(url, newListeners);
+ first = true;
+ }
+ newListeners.add(listener);
+ }
+
+ if (first) {
+ ensureCacheElement();
+ if (cacheElement != null && isCacheElementValid() && (isObjectLoadable())) {
+ // we got something in cache, and it's valid, so lets return it
+ log.log(Level.FINE, "JCS - Returning object from cache: {0}", getCacheKey());
+ finishLoading(true);
+ return;
+ }
+ // object not in cache, so submit work to separate thread
+ try {
+ // use getter method, so subclasses may override executors, to get separate thread pool
+ getDownloadExecutor().execute(JCSCachedTileLoaderJob.this);
+ } catch (RejectedExecutionException e) {
+ // queue was full, try again later
+ log.log(Level.FINE, "JCS - rejected job for: {0}", getCacheKey());
+ finishLoading(false);
+ }
+ }
+ }
+
+ /**
+ *
+ * @return checks if object from cache has sufficient data to be returned
+ */
+ protected boolean isObjectLoadable() {
+ byte[] content = cacheData.getContent();
+ return content != null && content.length > 0;
+ }
+
+ /**
+ *
+ * @return cache object as empty, regardless of what remote resource has returned (ex. based on headers)
+ */
+ protected boolean cacheAsEmpty() {
+ return false;
+ }
+
+ /**
+ * @return key under which discovered server settings will be kept
+ */
+ protected String getServerKey() {
+ return getUrl().getHost();
+ }
+
+ /**
+ * this needs to be non-static, so it can be overridden by subclasses
+ */
+ protected Executor getDownloadExecutor() {
+ return DOWNLOAD_JOB_DISPATCHER;
+ }
+
+
+ public void run() {
+ final Thread currentThread = Thread.currentThread();
+ final String oldName = currentThread.getName();
+ currentThread.setName("JCS Downloading: " + getUrl());
+ try {
+ // try to load object from remote resource
+ if (loadObject()) {
+ finishLoading(true);
+ } else {
+ // if loading failed - check if we can return stale entry
+ if (isObjectLoadable()) {
+ // try to get stale entry in cache
+ finishLoading(true);
+ log.log(Level.FINE, "JCS - found stale object in cache: {0}", getUrl());
+ } else {
+ // failed completely
+ finishLoading(false);
+ }
+ }
+ } finally {
+ currentThread.setName(oldName);
+ }
+ }
+
+
+ private void finishLoading(boolean success) {
+ Set listeners = null;
+ synchronized (inProgress) {
+ listeners = inProgress.remove(getUrl().toString());
+ }
+ if (listeners == null) {
+ log.log(Level.WARNING, "Listener not found for URL: {0}. Listener not notified!", getUrl());
+ return;
+ }
+ try {
+ for (ICachedLoaderListener l: listeners) {
+ l.loadingFinished(cacheData, success);
+ }
+ } catch (Exception e) {
+ log.log(Level.WARNING, "JCS - Error while loading object from cache: {0}; {1}", new Object[]{e.getMessage(), getUrl()});
+ log.log(Level.FINE, "Stacktrace", e);
+ for (ICachedLoaderListener l: listeners) {
+ l.loadingFinished(cacheData, false);
+ }
+
+ }
+
+ }
+
+ private boolean isCacheElementValid() {
+ long expires = attributes.getExpirationTime();
+
+ // check by expire date set by server
+ if (expires != 0L) {
+ // put a limit to the expire time (some servers send a value
+ // that is too large)
+ expires = Math.min(expires, attributes.getCreateTime() + EXPIRE_TIME_SERVER_LIMIT);
+ if (now > expires) {
+ log.log(Level.FINE, "JCS - Object {0} has expired -> valid to {1}, now is: {2}", new Object[]{getUrl(), Long.toString(expires), Long.toString(now)});
+ return false;
+ }
+ } else {
+ // check by file modification date
+ if (now - attributes.getLastModification() > DEFAULT_EXPIRE_TIME) {
+ log.log(Level.FINE, "JCS - Object has expired, maximum file age reached {0}", getUrl());
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private boolean loadObject() {
+ try {
+ // if we have object in cache, and host doesn't support If-Modified-Since nor If-None-Match
+ // then just use HEAD request and check returned values
+ if (isObjectLoadable() &&
+ Boolean.TRUE.equals(useHead.get(getServerKey())) &&
+ isCacheValidUsingHead()) {
+ log.log(Level.FINE, "JCS - cache entry verified using HEAD request: {0}", getUrl());
+ return true;
+ }
+ URLConnection urlConn = getURLConnection();
+
+ if (isObjectLoadable() &&
+ (now - attributes.getLastModification()) <= ABSOLUTE_EXPIRE_TIME_LIMIT) {
+ urlConn.setIfModifiedSince(attributes.getLastModification());
+ }
+ if (isObjectLoadable() && attributes.getEtag() != null) {
+ urlConn.addRequestProperty("If-None-Match", attributes.getEtag());
+ }
+ if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) {
+ // If isModifiedSince or If-None-Match has been set
+ // and the server answers with a HTTP 304 = "Not Modified"
+ log.log(Level.FINE, "JCS - IfModifiedSince/Etag test: local version is up to date: {0}", getUrl());
+ return true;
+ } else if (isObjectLoadable()) {
+ // we have an object in cache, but we haven't received 304 resposne code
+ // check if we should use HEAD request to verify
+ if((attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) ||
+ attributes.getLastModification() == urlConn.getLastModified()) {
+ // we sent ETag or If-Modified-Since, but didn't get 304 response code
+ // for further requests - use HEAD
+ String serverKey = getServerKey();
+ log.log(Level.INFO, "JCS - Host: {0} found not to return 304 codes for If-Modifed-Since or If-None-Match headers", serverKey);
+ useHead.put(serverKey, Boolean.TRUE);
+ }
+ }
+
+ attributes = parseHeaders(urlConn);
+
+ for (int i = 0; i < 5; ++i) {
+ if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) {
+ Thread.sleep(5000+(new Random()).nextInt(5000));
+ continue;
+ }
+ byte[] raw = read(urlConn);
+
+ if (!cacheAsEmpty() && raw != null && raw.length > 0) {
+ cacheData = createCacheEntry(raw);
+ cache.put(getCacheKey(), cacheData, attributes);
+ log.log(Level.FINE, "JCS - downloaded key: {0}, length: {1}, url: {2}",
+ new Object[] {getCacheKey(), raw.length, getUrl()});
+ return true;
+ } else {
+ cacheData = createCacheEntry(new byte[]{});
+ cache.put(getCacheKey(), cacheData, attributes);
+ log.log(Level.FINE, "JCS - Caching empty object {0}", getUrl());
+ return true;
+ }
+ }
+ } catch (FileNotFoundException e) {
+ log.log(Level.FINE, "JCS - Caching empty object as server returned 404 for: {0}", getUrl());
+ cache.put(getCacheKey(), createCacheEntry(new byte[]{}), attributes);
+ handleNotFound();
+ return true;
+ } catch (Exception e) {
+ log.log(Level.WARNING, "JCS - Exception during download " + getUrl(), e);
+ }
+ log.log(Level.WARNING, "JCS - Silent failure during download: {0}", getUrl());
+ return false;
+
+ }
+
+ protected abstract void handleNotFound();
+
+ protected abstract V createCacheEntry(byte[] content);
+
+ private CacheEntryAttributes parseHeaders(URLConnection urlConn) {
+ CacheEntryAttributes ret = new CacheEntryAttributes();
+ ret.setNoTileAtZoom("no-tile".equals(urlConn.getHeaderField("X-VE-Tile-Info")));
+
+ Long lng = urlConn.getExpiration();
+ if (lng.equals(0L)) {
+ try {
+ String str = urlConn.getHeaderField("Cache-Control");
+ if (str != null) {
+ for (String token: str.split(",")) {
+ if (token.startsWith("max-age=")) {
+ lng = Long.parseLong(token.substring(8)) * 1000 +
+ System.currentTimeMillis();
+ }
+ }
+ }
+ } catch (NumberFormatException e) {} //ignore malformed Cache-Control headers
+ }
+
+ ret.setExpirationTime(lng);
+ ret.setLastModification(now);
+ ret.setEtag(urlConn.getHeaderField("ETag"));
+ return ret;
+ }
+
+ private HttpURLConnection getURLConnection() throws IOException, MalformedURLException {
+ HttpURLConnection urlConn = (HttpURLConnection) getUrl().openConnection();
+ urlConn.setRequestProperty("Accept", "text/html, image/png, image/jpeg, image/gif, */*");
+ urlConn.setReadTimeout(readTimeout); // 30 seconds read timeout
+ urlConn.setConnectTimeout(connectTimeout);
+ for(Map.Entry e: headers.entrySet()) {
+ urlConn.setRequestProperty(e.getKey(), e.getValue());
+ }
+ return urlConn;
+ }
+
+ private boolean isCacheValidUsingHead() throws IOException {
+ HttpURLConnection urlConn = (HttpURLConnection) getUrl().openConnection();
+ urlConn.setRequestMethod("HEAD");
+ long lastModified = urlConn.getLastModified();
+ return (
+ (attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) ||
+ (lastModified != 0 && lastModified <= attributes.getLastModification())
+ );
+ }
+
+ private static byte[] read(URLConnection urlConn) throws IOException {
+ InputStream input = urlConn.getInputStream();
+ try {
+ ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available());
+ byte[] buffer = new byte[2048];
+ boolean finished = false;
+ do {
+ int read = input.read(buffer);
+ if (read >= 0) {
+ bout.write(buffer, 0, read);
+ } else {
+ finished = true;
+ }
+ } while (!finished);
+ if (bout.size() == 0)
+ return null;
+ return bout.toByteArray();
+ } finally {
+ input.close();
+ }
+ }
+}
Index: /trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoader.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoader.java (revision 8168)
+++ /trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoader.java (revision 8168)
@@ -0,0 +1,91 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.imagery;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
+import org.openstreetmap.gui.jmapviewer.Tile;
+import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader;
+import org.openstreetmap.gui.jmapviewer.interfaces.TileCache;
+import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
+import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader;
+import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
+import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
+import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
+import org.openstreetmap.josm.data.cache.JCSCacheManager;
+import org.openstreetmap.josm.data.preferences.IntegerProperty;
+
+/**
+ * @author Wiktor Niesiobędzki
+ *
+ * Wrapper class that bridges between JCS cache and Tile Loaders
+ *
+ */
+public class TMSCachedTileLoader implements TileLoader, CachedTileLoader, TileCache {
+
+ private ICacheAccess cache;
+ private int connectTimeout;
+ private int readTimeout;
+ private Map headers;
+ private TileLoaderListener listener;
+ public static final String PREFERENCE_PREFIX = "imagery.tms.cache.";
+ // average tile size is about 20kb
+ public static IntegerProperty MAX_OBJECTS_ON_DISK = new IntegerProperty(PREFERENCE_PREFIX + "max_objects_disk", 25000); // 25000 is around 500MB under this assumptions
+
+
+ /**
+ * Constructor
+ * @param listener called when tile loading has finished
+ * @param name of the cache
+ * @param connectTimeout to remote resource
+ * @param readTimeout to remote resource
+ * @param headers to be sent along with request
+ * @param cacheDir where cache file shall reside
+ * @throws IOException when cache initialization fails
+ */
+ public TMSCachedTileLoader(TileLoaderListener listener, String name, int connectTimeout, int readTimeout, Map headers, String cacheDir) throws IOException {
+ this.cache = JCSCacheManager.getCache(name,
+ 1000, // use JCS memory cache instead of MemoryTileCache
+ MAX_OBJECTS_ON_DISK.get(),
+ cacheDir);
+ this.connectTimeout = connectTimeout;
+ this.readTimeout = readTimeout;
+ this.headers = headers;
+ this.listener = listener;
+ }
+
+ @Override
+ public TileJob createTileLoaderJob(Tile tile) {
+ return new TMSCachedTileLoaderJob(listener, tile, cache, connectTimeout, readTimeout, headers);
+ }
+
+ @Override
+ public void clearCache() {
+ this.cache.clear();
+ }
+
+ @Override
+ public Tile getTile(TileSource source, int x, int y, int z) {
+ return createTileLoaderJob(new Tile(source,x, y, z)).getTile();
+ }
+
+ @Override
+ public void addTile(Tile tile) {
+ createTileLoaderJob(tile).submit();
+ }
+
+ @Override
+ public int getTileCount() {
+ return 0;
+ }
+
+ @Override
+ public void clear() {
+ cache.clear();
+ }
+
+ public String getStats() {
+ return cache.getStats();
+ }
+}
Index: /trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java (revision 8168)
+++ /trunk/src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java (revision 8168)
@@ -0,0 +1,233 @@
+// License: GPL. For details, see LICENSE file.
+package org.openstreetmap.josm.data.imagery;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.net.URL;
+import java.util.Map;
+import java.util.concurrent.Executor;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
+import org.openstreetmap.gui.jmapviewer.FeatureAdapter;
+import org.openstreetmap.gui.jmapviewer.Tile;
+import org.openstreetmap.gui.jmapviewer.interfaces.TileJob;
+import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
+import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
+import org.openstreetmap.gui.jmapviewer.tilesources.AbstractTMSTileSource;
+import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry;
+import org.openstreetmap.josm.data.cache.CacheEntry;
+import org.openstreetmap.josm.data.cache.ICachedLoaderListener;
+import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob;
+import org.openstreetmap.josm.data.preferences.IntegerProperty;
+
+/**
+ * @author Wiktor Niesiobędzki
+ *
+ * Class bridging TMS requests to JCS cache requests
+ *
+ */
+public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob implements TileJob, ICachedLoaderListener {
+ private static final Logger log = FeatureAdapter.getLogger(TMSCachedTileLoaderJob.class.getCanonicalName());
+ private Tile tile;
+ private TileLoaderListener listener;
+ private volatile URL url;
+
+ /**
+ * overrides the THREAD_LIMIT in superclass, as we want to have separate limit and pool for TMS
+ */
+ public static IntegerProperty THREAD_LIMIT = new IntegerProperty("imagery.tms.tmsloader.maxjobs", 25);
+ /**
+ * separate from JCS thread pool for TMS loader, so we can have different thread pools for default JCS
+ * and for TMS imagery
+ */
+ private static ThreadPoolExecutor DOWNLOAD_JOB_DISPATCHER = new ThreadPoolExecutor(
+ THREAD_LIMIT.get().intValue(), // keep the thread number constant
+ THREAD_LIMIT.get().intValue(), // do not this number of threads
+ 30, // keepalive for thread
+ TimeUnit.SECONDS,
+ // make queue of LIFO type - so recently requested tiles will be loaded first (assuming that these are which user is waiting to see)
+ new LinkedBlockingDeque(5) {
+ /* keep the queue size fairly small, we do not want to
+ download a lot of tiles, that user is not seeing anyway */
+ @Override
+ public boolean offer(Runnable t) {
+ return super.offerFirst(t);
+ }
+
+ @Override
+ public Runnable remove() {
+ return super.removeFirst();
+ }
+ }
+ );
+
+ /**
+ * Constructor for creating a job, to get a specific tile from cache
+ * @param listener
+ * @param tile to be fetched from cache
+ * @param cache object
+ * @param connectTimeout when connecting to remote resource
+ * @param readTimeout when connecting to remote resource
+ * @param headers to be sent together with request
+ */
+ public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile, ICacheAccess cache, int connectTimeout, int readTimeout,
+ Map headers) {
+ super(cache, connectTimeout, readTimeout, headers);
+ this.tile = tile;
+ this.listener = listener;
+ }
+
+ @Override
+ public Tile getTile() {
+ return getCachedTile();
+ }
+
+ @Override
+ public String getCacheKey() {
+ if (tile != null)
+ return tile.getKey();
+ return null;
+ }
+
+ /*
+ * this doesn't needs to be synchronized, as it's not that costly to keep only one execution
+ * in parallel, but URL creation and Tile.getUrl() are costly and are not needed when fetching
+ * data from cache
+ *
+ * We need to have static url value for TileLoaderJob, as for some TileSources we might get different
+ * URL's each call we made (servers switching), and URL's are used below as a key for duplicate detection
+ *
+ */
+ @Override
+ public URL getUrl() {
+ if (url == null) {
+ try {
+ synchronized (this) {
+ if (url == null)
+ url = new URL(tile.getUrl());
+ }
+ } catch (IOException e) {
+ log.log(Level.WARNING, "JCS TMS Cache - error creating URL for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
+ log.log(Level.INFO, "Exception: ", e);
+ }
+ }
+ return url;
+ }
+
+ @Override
+ public boolean isObjectLoadable() {
+ if (cacheData != null) {
+ byte[] content = cacheData.getContent();
+ try {
+ return (content != null && content.length > 0) || cacheData.getImage() != null || cacheAsEmpty();
+ } catch (IOException e) {
+ log.log(Level.WARNING, "JCS TMS - error loading from cache for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
+ }
+ }
+ return false;
+ }
+
+ @Override
+ protected boolean cacheAsEmpty() {
+ if (attributes != null && attributes.isNoTileAtZoom()) {
+ // do not remove file - keep the information, that there is no tile, for further requests
+ // the code above will check, if this information is still valid
+ log.log(Level.FINE, "JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile);
+ tile.setError("No tile at this zoom level");
+ tile.putValue("tile-info", "no-tile");
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ protected Executor getDownloadExecutor() {
+ return DOWNLOAD_JOB_DISPATCHER;
+ }
+
+ public void submit() {
+ tile.initLoading();
+ super.submit(this);
+ }
+
+ @Override
+ public void loadingFinished(CacheEntry object, boolean success) {
+ try {
+ loadTile(object);
+ if (listener != null) {
+ listener.tileLoadingFinished(tile, success);
+ }
+ } catch (IOException e) {
+ log.log(Level.WARNING, "JCS TMS - error loading object for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
+ tile.setError(e.getMessage());
+ tile.setLoaded(false);
+ if (listener != null) {
+ listener.tileLoadingFinished(tile, false);
+ }
+ }
+ }
+
+ /**
+ * Method for getting the tile from cache only, without trying to reach remote resource
+ * @return tile or null, if nothing (useful) was found in cache
+ */
+ public Tile getCachedTile() {
+ BufferedImageCacheEntry data = super.get();
+ if (isObjectLoadable()) {
+ try {
+ loadTile(data);
+ return tile;
+ } catch (IOException e) {
+ log.log(Level.WARNING, "JCS TMS - error loading object for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()});
+ return null;
+ }
+
+ } else {
+ return null;
+ }
+ }
+
+ private void loadTile(CacheEntry object) throws IOException {
+ tile.finishLoading();
+ if (object != null) {
+ byte[] content = object.getContent();
+ if (content != null && content.length > 0) {
+ tile.loadImage(new ByteArrayInputStream(content));
+ }
+ }
+ }
+
+ private void loadTile(BufferedImageCacheEntry object) throws IOException {
+ tile.finishLoading();
+ if (object != null) {
+ if (object.getImage() != null) {
+ tile.setImage(object.getImage());
+ }
+ }
+ }
+
+ @Override
+ protected void handleNotFound() {
+ tile.setError("No tile at this zoom level");
+ tile.putValue("tile-info", "no-tile");
+ }
+
+ @Override
+ protected String getServerKey() {
+ TileSource ts = tile.getSource();
+ if (ts instanceof AbstractTMSTileSource) {
+ return ((AbstractTMSTileSource) ts).getBaseUrl();
+ }
+ return super.getServerKey();
+ }
+
+ @Override
+ protected BufferedImageCacheEntry createCacheEntry(byte[] content) {
+ return new BufferedImageCacheEntry(content);
+ }
+}
Index: /trunk/src/org/openstreetmap/josm/gui/bbox/SlippyMapBBoxChooser.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/gui/bbox/SlippyMapBBoxChooser.java (revision 8167)
+++ /trunk/src/org/openstreetmap/josm/gui/bbox/SlippyMapBBoxChooser.java (revision 8168)
@@ -12,6 +12,8 @@
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.CopyOnWriteArrayList;
@@ -26,4 +28,5 @@
import org.openstreetmap.gui.jmapviewer.OsmTileLoader;
import org.openstreetmap.gui.jmapviewer.interfaces.MapMarker;
+import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader;
import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
import org.openstreetmap.gui.jmapviewer.tilesources.MapQuestOpenAerialTileSource;
@@ -114,5 +117,5 @@
public static final String RESIZE_PROP = SlippyMapBBoxChooser.class.getName() + ".resize";
- private OsmTileLoader cachedLoader;
+ private TileLoader cachedLoader;
private OsmTileLoader uncachedLoader;
@@ -132,9 +135,12 @@
SpringLayout springLayout = new SpringLayout();
setLayout(springLayout);
- TMSLayer.setMaxWorkers();
- cachedLoader = TMSLayer.loaderFactory.makeTileLoader(this);
+
+ Map headers = new HashMap<>();
+ headers.put("User-Agent", Version.getInstance().getFullAgentString());
+
+ cachedLoader = TMSLayer.loaderFactory.makeTileLoader(this, headers);
uncachedLoader = new OsmTileLoader(this);
- uncachedLoader.headers.put("User-Agent", Version.getInstance().getFullAgentString());
+ uncachedLoader.headers.putAll(headers);
setZoomContolsVisible(Main.pref.getBoolean("slippy_map_chooser.zoomcontrols",false));
setMapMarkerVisible(false);
Index: /trunk/src/org/openstreetmap/josm/gui/layer/TMSLayer.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/gui/layer/TMSLayer.java (revision 8167)
+++ /trunk/src/org/openstreetmap/josm/gui/layer/TMSLayer.java (revision 8168)
@@ -22,11 +22,9 @@
import java.util.ArrayList;
import java.util.Collections;
-import java.util.HashSet;
+import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Scanner;
-import java.util.Set;
import java.util.concurrent.Callable;
import java.util.regex.Matcher;
@@ -42,12 +40,10 @@
import org.openstreetmap.gui.jmapviewer.AttributionSupport;
import org.openstreetmap.gui.jmapviewer.Coordinate;
-import org.openstreetmap.gui.jmapviewer.JobDispatcher;
import org.openstreetmap.gui.jmapviewer.MemoryTileCache;
-import org.openstreetmap.gui.jmapviewer.OsmFileCacheTileLoader;
import org.openstreetmap.gui.jmapviewer.OsmTileLoader;
-import org.openstreetmap.gui.jmapviewer.TMSFileCacheTileLoader;
import org.openstreetmap.gui.jmapviewer.Tile;
import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader;
-import org.openstreetmap.gui.jmapviewer.interfaces.TileClearController;
+import org.openstreetmap.gui.jmapviewer.interfaces.TileCache;
+import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader;
import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener;
import org.openstreetmap.gui.jmapviewer.interfaces.TileSource;
@@ -64,4 +60,5 @@
import org.openstreetmap.josm.data.imagery.ImageryInfo;
import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryType;
+import org.openstreetmap.josm.data.imagery.TMSCachedTileLoader;
import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor;
import org.openstreetmap.josm.data.preferences.BooleanProperty;
@@ -76,5 +73,4 @@
import org.openstreetmap.josm.gui.dialogs.LayerListPopup;
import org.openstreetmap.josm.gui.progress.ProgressMonitor;
-import org.openstreetmap.josm.gui.progress.ProgressMonitor.CancelListener;
import org.openstreetmap.josm.io.CacheCustomContent;
import org.openstreetmap.josm.io.OsmTransferException;
@@ -109,5 +105,4 @@
//public static final BooleanProperty PROP_DRAW_DEBUG = new BooleanProperty(PREFERENCE_PREFIX + ".draw_debug", false);
public static final BooleanProperty PROP_ADD_TO_SLIPPYMAP_CHOOSER = new BooleanProperty(PREFERENCE_PREFIX + ".add_to_slippymap_chooser", true);
- public static final IntegerProperty PROP_TMS_JOBS = new IntegerProperty("tmsloader.maxjobs", 25);
public static final StringProperty PROP_TILECACHE_DIR;
@@ -123,26 +118,37 @@
public interface TileLoaderFactory {
- OsmTileLoader makeTileLoader(TileLoaderListener listener);
- }
-
- protected MemoryTileCache tileCache;
+ TileLoader makeTileLoader(TileLoaderListener listener);
+ TileLoader makeTileLoader(TileLoaderListener listener, Map headers);
+ }
+
+ protected TileCache tileCache;
protected TileSource tileSource;
- protected OsmTileLoader tileLoader;
+ protected TileLoader tileLoader;
+
public static TileLoaderFactory loaderFactory = new TileLoaderFactory() {
@Override
- public OsmTileLoader makeTileLoader(TileLoaderListener listener) {
- String cachePath = TMSLayer.PROP_TILECACHE_DIR.get();
- if (cachePath != null && !cachePath.isEmpty()) {
- try {
- OsmFileCacheTileLoader loader;
- loader = new TMSFileCacheTileLoader(listener, new File(cachePath));
- loader.headers.put("User-Agent", Version.getInstance().getFullAgentString());
- return loader;
- } catch (IOException e) {
- Main.warn(e);
- }
+ public TileLoader makeTileLoader(TileLoaderListener listener, Map inputHeaders) {
+ Map headers = new HashMap<>();
+ headers.put("User-Agent", Version.getInstance().getFullAgentString());
+ headers.put("Accept", "text/html, image/png, image/jpeg, image/gif, */*");
+ if (inputHeaders != null)
+ headers.putAll(inputHeaders);
+
+ try {
+ return new TMSCachedTileLoader(listener, "TMS",
+ Main.pref.getInteger("socket.timeout.connect",15) * 1000,
+ Main.pref.getInteger("socket.timeout.read", 30) * 1000,
+ headers,
+ PROP_TILECACHE_DIR.get());
+ } catch (IOException e) {
+ Main.warn(e);
}
return null;
+ }
+
+ @Override
+ public TileLoader makeTileLoader(TileLoaderListener listener) {
+ return makeTileLoader(listener, null);
}
};
@@ -151,9 +157,8 @@
* Plugins that wish to set custom tile loader should call this method
*/
+
public static void setCustomTileLoaderFactory(TileLoaderFactory loaderFactory) {
TMSLayer.loaderFactory = loaderFactory;
}
-
- private Set tileRequestsOutstanding = new HashSet<>();
@Override
@@ -166,53 +171,11 @@
tile.setImage(sharpenImage(tile.getImage()));
}
- tile.setLoaded(true);
+ tile.setLoaded(success);
needRedraw = true;
if (Main.map != null) {
Main.map.repaint(100);
}
- tileRequestsOutstanding.remove(tile);
if (Main.isDebugEnabled()) {
Main.debug("tileLoadingFinished() tile: " + tile + " success: " + success);
- }
- }
-
- private static class TmsTileClearController implements TileClearController, CancelListener {
-
- private final ProgressMonitor monitor;
- private boolean cancel = false;
-
- public TmsTileClearController(ProgressMonitor monitor) {
- this.monitor = monitor;
- this.monitor.addCancelListener(this);
- }
-
- @Override
- public void initClearDir(File dir) {
- }
-
- @Override
- public void initClearFiles(File[] files) {
- monitor.setTicksCount(files.length);
- monitor.setTicks(0);
- }
-
- @Override
- public boolean cancel() {
- return cancel;
- }
-
- @Override
- public void fileDeleted(File file) {
- monitor.setTicks(monitor.getTicks()+1);
- }
-
- @Override
- public void clearFinished() {
- monitor.finishTask();
- }
-
- @Override
- public void operationCanceled() {
- cancel = true;
}
}
@@ -226,5 +189,4 @@
*
* @param monitor
- * @see MemoryTileCache#clear()
* @see OsmFileCacheTileLoader#clearCache(org.openstreetmap.gui.jmapviewer.interfaces.TileSource, org.openstreetmap.gui.jmapviewer.interfaces.TileClearController)
*/
@@ -232,5 +194,5 @@
tileCache.clear();
if (tileLoader instanceof CachedTileLoader) {
- ((CachedTileLoader)tileLoader).clearCache(tileSource, new TmsTileClearController(monitor));
+ ((CachedTileLoader)tileLoader).clearCache();
}
}
@@ -416,24 +378,18 @@
currentZoomLevel = getBestZoom();
- tileCache = new MemoryTileCache();
-
- tileLoader = loaderFactory.makeTileLoader(this);
- if (tileLoader == null) {
+ Map headers = null;
+ if (tileSource instanceof TemplatedTMSTileSource) {
+ headers = (((TemplatedTMSTileSource)tileSource).getHeaders());
+ }
+
+ // FIXME: tileCache = new MemoryTileCache();
+ tileLoader = loaderFactory.makeTileLoader(this, headers);
+ if (tileLoader instanceof TMSCachedTileLoader) {
+ tileCache = (TileCache) tileLoader;
+ } else {
+ tileCache = new MemoryTileCache();
+ }
+ if (tileLoader == null)
tileLoader = new OsmTileLoader(this);
- }
- tileLoader.timeoutConnect = Main.pref.getInteger("socket.timeout.connect",15) * 1000;
- tileLoader.timeoutRead = Main.pref.getInteger("socket.timeout.read", 30) * 1000;
- if (tileSource instanceof TemplatedTMSTileSource) {
- for(Entry e : ((TemplatedTMSTileSource)tileSource).getHeaders().entrySet()) {
- tileLoader.headers.put(e.getKey(), e.getValue());
- }
- }
- tileLoader.headers.put("User-Agent", Version.getInstance().getFullAgentString());
- }
-
- @Override
- public void setOffset(double dx, double dy) {
- super.setOffset(dx, dy);
- needRedraw = true;
}
@@ -472,25 +428,15 @@
}
- /**
- * Function to set the maximum number of workers for tile loading to the value defined
- * in preferences.
- */
- public static void setMaxWorkers() {
- JobDispatcher.setMaxWorkers(PROP_TMS_JOBS.get());
- JobDispatcher.getInstance().setLIFO(true);
- }
-
@SuppressWarnings("serial")
public TMSLayer(ImageryInfo info) {
super(info);
- setMaxWorkers();
if(!isProjectionSupported(Main.getProjection())) {
JOptionPane.showMessageDialog(Main.parent,
- tr("TMS layers do not support the projection {0}.\n{1}\n"
- + "Change the projection or remove the layer.",
- Main.getProjection().toCode(), nameSupportedProjections()),
- tr("Warning"),
- JOptionPane.WARNING_MESSAGE);
+ tr("TMS layers do not support the projection {0}.\n{1}\n"
+ + "Change the projection or remove the layer.",
+ Main.getProjection().toCode(), nameSupportedProjections()),
+ tr("Warning"),
+ JOptionPane.WARNING_MESSAGE);
}
@@ -685,6 +631,4 @@
}
needRedraw = true;
- JobDispatcher.getInstance().cancelOutstandingJobs();
- tileRequestsOutstanding.clear();
}
@@ -771,5 +715,5 @@
* into the tileCache.
*/
- synchronized Tile tempCornerTile(Tile t) {
+ Tile tempCornerTile(Tile t) {
int x = t.getXtile() + 1;
int y = t.getYtile() + 1;
@@ -781,5 +725,5 @@
}
- synchronized Tile getOrCreateTile(int x, int y, int zoom) {
+ Tile getOrCreateTile(int x, int y, int zoom) {
Tile tile = getTile(x, y, zoom);
if (tile == null) {
@@ -795,5 +739,5 @@
* already in the cache.
*/
- synchronized Tile getTile(int x, int y, int zoom) {
+ Tile getTile(int x, int y, int zoom) {
int max = (1 << zoom);
if (x < 0 || x >= max || y < 0 || y >= max)
@@ -802,15 +746,12 @@
}
- synchronized boolean loadTile(Tile tile, boolean force) {
+ boolean loadTile(Tile tile, boolean force) {
if (tile == null)
return false;
- if (!force && (tile.hasError() || tile.isLoaded()))
+ if (!force && (tile.isLoaded() || tile.hasError()))
return false;
if (tile.isLoading())
return false;
- if (tileRequestsOutstanding.contains(tile))
- return false;
- tileRequestsOutstanding.add(tile);
- JobDispatcher.getInstance().addJob(tileLoader.createTileLoaderJob(tile));
+ tileLoader.createTileLoaderJob(tile).submit();
return true;
}
@@ -1269,20 +1210,25 @@
if (zoom < minZoom)
return nullTileSet;
- TileSet ts = tileSets[zoom-minZoom];
- if (ts == null) {
- ts = new TileSet(topLeft, botRight, zoom);
- tileSets[zoom-minZoom] = ts;
- }
- return ts;
- }
+ synchronized (tileSets) {
+ TileSet ts = tileSets[zoom-minZoom];
+ if (ts == null) {
+ ts = new TileSet(topLeft, botRight, zoom);
+ tileSets[zoom-minZoom] = ts;
+ }
+ return ts;
+ }
+ }
+
public TileSetInfo getTileSetInfo(int zoom) {
if (zoom < minZoom)
return new TileSetInfo();
- TileSetInfo tsi = tileSetInfos[zoom-minZoom];
- if (tsi == null) {
- tsi = TMSLayer.getTileSetInfo(getTileSet(zoom));
- tileSetInfos[zoom-minZoom] = tsi;
- }
- return tsi;
+ synchronized (tileSetInfos) {
+ TileSetInfo tsi = tileSetInfos[zoom-minZoom];
+ if (tsi == null) {
+ tsi = TMSLayer.getTileSetInfo(getTileSet(zoom));
+ tileSetInfos[zoom-minZoom] = tsi;
+ }
+ return tsi;
+ }
}
}
@@ -1290,5 +1236,4 @@
@Override
public void paint(Graphics2D g, MapView mv, Bounds bounds) {
- //long start = System.currentTimeMillis();
EastNorth topLeft = mv.getEastNorth(0, 0);
EastNorth botRight = mv.getEastNorth(mv.getWidth(), mv.getHeight());
@@ -1436,4 +1381,12 @@
myDrawString(g, tr("Pixel scale: {0}", getScaleFactor(currentZoomLevel)), 50, 170);
myDrawString(g, tr("Best zoom: {0}", Math.log(getScaleFactor(1))/Math.log(2)/2+1), 50, 185);
+ if(tileLoader instanceof TMSCachedTileLoader) {
+ TMSCachedTileLoader cachedTileLoader = (TMSCachedTileLoader)tileLoader;
+ int offset = 185;
+ for(String part: cachedTileLoader.getStats().split("\n")) {
+ myDrawString(g, tr("Cache stats: {0}", part), 50, offset+=15);
+ }
+
+ }
}
}
Index: /trunk/src/org/openstreetmap/josm/gui/preferences/imagery/TMSSettingsPanel.java
===================================================================
--- /trunk/src/org/openstreetmap/josm/gui/preferences/imagery/TMSSettingsPanel.java (revision 8167)
+++ /trunk/src/org/openstreetmap/josm/gui/preferences/imagery/TMSSettingsPanel.java (revision 8168)
@@ -12,7 +12,9 @@
import javax.swing.SpinnerNumberModel;
+import org.openstreetmap.josm.data.imagery.TMSCachedTileLoader;
+import org.openstreetmap.josm.data.imagery.TMSCachedTileLoaderJob;
import org.openstreetmap.josm.gui.layer.TMSLayer;
+import org.openstreetmap.josm.gui.widgets.JosmTextField;
import org.openstreetmap.josm.tools.GBC;
-import org.openstreetmap.josm.gui.widgets.JosmTextField;
/**
@@ -29,4 +31,7 @@
private final JCheckBox addToSlippyMapChosser = new JCheckBox();
private final JosmTextField tilecacheDir = new JosmTextField();
+ private final JSpinner maxElementsOnDisk;
+ private final JSpinner maxConcurrentDownloads;
+
/**
@@ -37,4 +42,6 @@
minZoomLvl = new JSpinner(new SpinnerNumberModel(TMSLayer.DEFAULT_MIN_ZOOM, TMSLayer.MIN_ZOOM, TMSLayer.MAX_ZOOM, 1));
maxZoomLvl = new JSpinner(new SpinnerNumberModel(TMSLayer.DEFAULT_MAX_ZOOM, TMSLayer.MIN_ZOOM, TMSLayer.MAX_ZOOM, 1));
+ maxElementsOnDisk = new JSpinner(new SpinnerNumberModel(TMSCachedTileLoader.MAX_OBJECTS_ON_DISK.get().intValue(), 0, Integer.MAX_VALUE, 1));
+ maxConcurrentDownloads = new JSpinner(new SpinnerNumberModel(TMSCachedTileLoaderJob.THREAD_LIMIT.get().intValue(), 0, Integer.MAX_VALUE, 1));
add(new JLabel(tr("Auto zoom by default: ")), GBC.std());
@@ -61,6 +68,15 @@
add(GBC.glue(5, 0), GBC.std());
add(tilecacheDir, GBC.eol().fill(GBC.HORIZONTAL));
+
+ add(new JLabel(tr("Maximum concurrent downloads: ")), GBC.std());
+ add(GBC.glue(5, 0), GBC.std());
+ add(maxConcurrentDownloads, GBC.eol());
+
+ add(new JLabel(tr("Maximum elements in disk cache: ")), GBC.std());
+ add(GBC.glue(5, 0), GBC.std());
+ add(this.maxElementsOnDisk, GBC.eol());
+
}
-
+
/**
* Loads the TMS settings.
@@ -73,6 +89,8 @@
this.minZoomLvl.setValue(TMSLayer.getMinZoomLvl(null));
this.tilecacheDir.setText(TMSLayer.PROP_TILECACHE_DIR.get());
+ this.maxElementsOnDisk.setValue(TMSCachedTileLoader.MAX_OBJECTS_ON_DISK.get());
+ this.maxConcurrentDownloads.setValue(TMSCachedTileLoaderJob.THREAD_LIMIT.get());
}
-
+
/**
* Saves the TMS settings.
@@ -81,5 +99,5 @@
public boolean saveSettings() {
boolean restartRequired = false;
-
+
if (TMSLayer.PROP_ADD_TO_SLIPPYMAP_CHOOSER.get() != this.addToSlippyMapChosser.isSelected()) {
restartRequired = true;
@@ -90,6 +108,17 @@
TMSLayer.setMaxZoomLvl((Integer)this.maxZoomLvl.getValue());
TMSLayer.setMinZoomLvl((Integer)this.minZoomLvl.getValue());
- TMSLayer.PROP_TILECACHE_DIR.put(this.tilecacheDir.getText());
-
+
+ TMSCachedTileLoader.MAX_OBJECTS_ON_DISK.put((Integer) this.maxElementsOnDisk.getValue());
+
+ if (!TMSCachedTileLoaderJob.THREAD_LIMIT.get().equals(this.maxConcurrentDownloads.getValue())) {
+ restartRequired = true;
+ TMSCachedTileLoaderJob.THREAD_LIMIT.put((Integer) this.maxConcurrentDownloads.getValue());
+ }
+
+ if (!TMSLayer.PROP_TILECACHE_DIR.get().equals(this.tilecacheDir.getText())) {
+ restartRequired = true;
+ TMSLayer.PROP_TILECACHE_DIR.put(this.tilecacheDir.getText());
+ }
+
return restartRequired;
}