Index: src/org/apache/commons =================================================================== --- src/org/apache/commons (revision 8109) +++ src/org/apache/commons (working copy) Property changes on: src/org/apache/commons ___________________________________________________________________ Modified: svn:externals ## -1 +1,3 ## -codec http://svn.apache.org/repos/asf/commons/proper/codec/trunk/src/main/java/org/apache/commons/codec +http://svn.apache.org/repos/asf/commons/proper/codec/trunk/src/main/java/org/apache/commons/codec codec +http://svn.apache.org/repos/asf/commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs jcs +http://svn.apache.org/repos/asf/commons/proper/logging/trunk/src/main/java/org/apache/commons/logging logging Index: build.xml =================================================================== --- build.xml (revision 8109) +++ build.xml (working copy) @@ -220,6 +220,14 @@ destdir="build" target="1.7" source="1.7" debug="on" includeantruntime="false" createMissingPackageInfoClass="false" encoding="iso-8859-1"> + + + + + + + + + Index: src/org/openstreetmap/josm/Main.java =================================================================== --- src/org/openstreetmap/josm/Main.java (revision 8117) +++ src/org/openstreetmap/josm/Main.java (working copy) @@ -67,6 +67,7 @@ import org.openstreetmap.josm.data.ProjectionBounds; import org.openstreetmap.josm.data.UndoRedoHandler; import org.openstreetmap.josm.data.ViewportData; +import org.openstreetmap.josm.data.cache.JCSCacheManager; import org.openstreetmap.josm.data.coor.CoordinateFormat; import org.openstreetmap.josm.data.coor.LatLon; import org.openstreetmap.josm.data.osm.DataSet; @@ -1088,6 +1089,7 @@ * @since 3378 */ public static boolean exitJosm(boolean exit, int exitCode) { + JCSCacheManager.shutdown(); if (Main.saveUnsavedModifications()) { geometry.remember("gui.geometry"); if (map != null) { Index: src/org/openstreetmap/josm/data/cache/JCSCacheManager.java =================================================================== --- src/org/openstreetmap/josm/data/cache/JCSCacheManager.java (revision 0) +++ src/org/openstreetmap/josm/data/cache/JCSCacheManager.java (working copy) @@ -0,0 +1,119 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.cache; + +import java.io.File; +import java.io.IOException; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.jcs.access.CacheAccess; +import org.apache.commons.jcs.auxiliary.AuxiliaryCache; +import org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes; +import org.apache.commons.jcs.engine.control.CompositeCache; +import org.apache.commons.jcs.engine.control.CompositeCacheManager; +import org.openstreetmap.josm.Main; + + +/** + * @author Wiktor Niesiobędzki + * + * Wrapper class for JCS Cache. Sets some sane environment and returns instances of cache objects. + * Static configuration for now assumes some small LRU cache in memory and larger LRU cache on disk + * + */ +public class JCSCacheManager { + + private static volatile CompositeCacheManager cacheManager = null; + private static long maxObjectTTL = Long.MAX_VALUE; + + private static int maxObjectsInMemory = 1000; + private static int maxObjectsOnDisk = 25000; + + private static void initialize() throws IOException { + File cacheDir = new File(Main.pref.getCacheDirectory(), "jcs"); + + if ((!cacheDir.exists() && !cacheDir.mkdirs())) + throw new IOException("Cannot access cache directory"); + + // raising logging level gives ~500x performance gain + // http://westsworld.dk/blog/2008/01/jcs-and-performance/ + Logger.getLogger("org.apache.commons.jcs").setLevel(Level.INFO); + + CompositeCacheManager cm = CompositeCacheManager.getUnconfiguredInstance(); + // this could be moved to external file + Properties props = new Properties(); + props.setProperty("jcs.default", "DC"); + props.setProperty("jcs.default.cacheattributes", org.apache.commons.jcs.engine.CompositeCacheAttributes.class.getCanonicalName()); + props.setProperty("jcs.default.cacheattributes.MaxObjects", Long.toString(maxObjectsInMemory)); + props.setProperty("jcs.default.cacheattributes.UseMemoryShrinker", "true"); + props.setProperty("jcs.default.cacheattributes.DiskUsagePatternName", "UPDATE"); // store elements on disk on put + props.setProperty("jcs.default.elementattributes", CacheEntryAttributes.class.getCanonicalName()); + props.setProperty("jcs.default.elementattributes.IsEternal", "false"); + props.setProperty("jcs.default.elementattributes.MaxLife", Long.toString(maxObjectTTL)); + props.setProperty("jcs.default.elementattributes.IdleTime", Long.toString(maxObjectTTL)); + props.setProperty("jcs.default.elementattributes.IsSpool", "true"); + props.setProperty("jcs.auxiliary.DC", org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheFactory.class.getCanonicalName()); + props.setProperty("jcs.auxiliary.DC.attributes", org.apache.commons.jcs.auxiliary.disk.indexed.IndexedDiskCacheAttributes.class.getCanonicalName()); + props.setProperty("jcs.auxiliary.DC.attributes.DiskPath", cacheDir.getAbsolutePath()); + props.setProperty("jcs.auxiliary.DC.attributes.maxKeySize", Long.toString(maxObjectsOnDisk)); + //props.setProperty("jcs.auxiliary.DC.attributes.blockSizeBytes", "1024"); - for BlockDiskCacheAttributes + cm.configure(props); + cacheManager = cm; + + } + + /** + * Returns configured cache object for named cache region + * @param cacheName region name + * @return cache access object + * @throws IOException if directory is not found + */ + public static CacheAccess getCache(String cacheName) throws IOException { + return getCache(cacheName, maxObjectsInMemory, maxObjectsOnDisk); + } + + /** + * Returns configured cache object with defined limits of memory cache and disk cache + * @param cacheName region name + * @param maxMemoryObjects number of objects to keep in memory + * @param maxDiskObjects number of objects to keep on disk + * @return cache access object + * @throws IOException if directory is not found + */ + public static CacheAccess getCache(String cacheName, int maxMemoryObjects, int maxDiskObjects) throws IOException { + if (cacheManager != null) + return getCacheInner(cacheName, maxMemoryObjects, maxDiskObjects); + + synchronized (JCSCacheManager.class) { + if (cacheManager == null) + initialize(); + return getCacheInner(cacheName, maxMemoryObjects, maxDiskObjects); + } + } + + private static CacheAccess getCacheInner(String cacheName, int maxMemoryObjects, int maxDiskObjects) { + CompositeCache cc = cacheManager.getCache(cacheName); + cc.getCacheAttributes().setMaxObjects(maxMemoryObjects); + AuxiliaryCache ac[] = cc.getAuxCaches(); + if (ac!=null && ac.length > 0) { + if (ac[0].getAuxiliaryCacheAttributes() instanceof IndexedDiskCacheAttributes) { + ((IndexedDiskCacheAttributes) ac[0].getAuxiliaryCacheAttributes()).setMaxKeySize(maxDiskObjects); + } + } + return new CacheAccess(cc); + } + + /** + * Close all files to ensure, that all indexes and data are properly written + */ + public static void shutdown() { + // use volatile semantics to get consistent object + CompositeCacheManager localCacheManager = cacheManager; + if (localCacheManager != null) { + localCacheManager.shutDown(); + } + + } + +} Index: src/org/openstreetmap/josm/data/cache/JCSJobDispatcher.java =================================================================== --- src/org/openstreetmap/josm/data/cache/JCSJobDispatcher.java (revision 0) +++ src/org/openstreetmap/josm/data/cache/JCSJobDispatcher.java (working copy) @@ -0,0 +1,176 @@ +// License: GPL. For details, see Readme.txt file. +package org.openstreetmap.josm.data.cache; + +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; + +import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; + +/** + * A generic class that processes a list of {@link Runnable} one-by-one using + * one or more {@link Thread}-instances. The number of instances varies between + * 1 and {@link #workerThreadMaxCount} (default: 8). If an instance is idle + * more than {@link #workerThreadTimeout} seconds (default: 30), the instance + * ends itself. + * + * @author Jan Peter Stotz + */ +public class JCSJobDispatcher { + + private static final JCSJobDispatcher instance = new JCSJobDispatcher(); + + /** + * @return the singelton instance of the {@link JCSJobDispatcher} + */ + public static JCSJobDispatcher getInstance() { + return instance; + } + + private JCSJobDispatcher() { + addWorkerThread().firstThread = true; + } + + protected BlockingDeque jobQueue = new LinkedBlockingDeque<>(); + + protected static int workerThreadMaxCount = 8; + + /** + * Specifies the time span in seconds that a worker thread waits for new + * jobs to perform. If the time span has elapsed the worker thread + * terminates itself. Only the first worker thread works differently, it + * ignores the timeout and will never terminate itself. + */ + protected static int workerThreadTimeout = 30; + + /** + * Type of queue, FIFO if false, LIFO if true + */ + protected boolean modeLIFO = false; + + /** + * Total number of worker threads currently idle or active + */ + protected int workerThreadCount = 0; + + /** + * Number of worker threads currently idle + */ + protected int workerThreadIdleCount = 0; + + /** + * Just an id for identifying an worker thread instance + */ + protected int workerThreadId = 0; + + /** + * Removes all jobs from the queue that are currently not being processed. + */ + public void cancelOutstandingJobs() { + jobQueue.clear(); + } + + /** + * Function to set the maximum number of workers for tile loading. + */ + static public void setMaxWorkers(int workers) { + workerThreadMaxCount = workers; + } + + /** + * Function to set the LIFO/FIFO mode for tile loading job. + * + * @param lifo true for LIFO mode, false for FIFO mode + */ + public void setLIFO(boolean lifo) { + modeLIFO = lifo; + } + + /** + * Adds a job to the queue. + * Jobs for tiles already contained in the are ignored (using a null tile + * prevents skipping). + * + * @param job the the job to be added + */ + public void addJob(ICachedLoaderJob job) { + try { + if(job.getCacheKey() != null) { + for(ICachedLoaderJob oldJob : jobQueue) { + if(oldJob.getCacheKey() == job.getCacheKey()) { + return; + } + } + } + jobQueue.put(job); + if (workerThreadIdleCount == 0 && workerThreadCount < workerThreadMaxCount) + addWorkerThread(); + } catch (InterruptedException e) { + } + } + + protected JobThread addWorkerThread() { + JobThread jobThread = new JobThread(++workerThreadId); + synchronized (this) { + workerThreadCount++; + } + jobThread.start(); + return jobThread; + } + + public class JobThread extends Thread { + + ICachedLoaderJob job; + boolean firstThread = false; + + public JobThread(int threadId) { + super("OSMJobThread " + threadId); + setDaemon(true); + job = null; + } + + @Override + public void run() { + executeJobs(); + synchronized (instance) { + workerThreadCount--; + } + } + + protected void executeJobs() { + while (!isInterrupted()) { + try { + synchronized (instance) { + workerThreadIdleCount++; + } + if(modeLIFO) { + if (firstThread) + job = jobQueue.takeLast(); + else + job = jobQueue.pollLast(workerThreadTimeout, TimeUnit.SECONDS); + } else { + if (firstThread) + job = jobQueue.take(); + else + job = jobQueue.poll(workerThreadTimeout, TimeUnit.SECONDS); + } + } catch (InterruptedException e1) { + return; + } finally { + synchronized (instance) { + workerThreadIdleCount--; + } + } + if (job == null) + return; + try { + job.run(); + job = null; + } catch (Exception e) { + e.printStackTrace(); + } + } + } + } + +} Index: src/org/openstreetmap/josm/data/cache/CacheEntry.java =================================================================== --- src/org/openstreetmap/josm/data/cache/CacheEntry.java (revision 0) +++ src/org/openstreetmap/josm/data/cache/CacheEntry.java (working copy) @@ -0,0 +1,29 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.cache; + +import java.io.Serializable; + +/** + * @author Wiktor Niesiobędzki + * + * Class that will hold JCS cache entries + * + */ +public class CacheEntry implements Serializable { + private static final long serialVersionUID = 1L; //version + private byte[] content; + + /** + * @param content of the cache entry + */ + public CacheEntry(byte[] content) { + this.content = content; + } + + /** + * @return cache entry content + */ + public byte[] getContent() { + return content; + } +} Index: src/org/openstreetmap/josm/data/cache/ICachedLoaderListener.java =================================================================== --- src/org/openstreetmap/josm/data/cache/ICachedLoaderListener.java (revision 0) +++ src/org/openstreetmap/josm/data/cache/ICachedLoaderListener.java (working copy) @@ -0,0 +1,13 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.cache; + +public interface ICachedLoaderListener { + /** + * Will be called when K object was successfully downloaded + * + * @param object + * @param success + */ + public void loadingFinished(byte[] object, boolean success); + +} Index: src/org/openstreetmap/josm/data/cache/JCSCachedTileLoaderJob.java =================================================================== --- src/org/openstreetmap/josm/data/cache/JCSCachedTileLoaderJob.java (revision 0) +++ src/org/openstreetmap/josm/data/cache/JCSCachedTileLoaderJob.java (working copy) @@ -0,0 +1,340 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.cache; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.URL; +import java.net.URLConnection; +import java.util.HashSet; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.jcs.access.behavior.ICacheAccess; +import org.apache.commons.jcs.engine.behavior.ICacheElement; +import org.openstreetmap.gui.jmapviewer.FeatureAdapter; + +/** + * @author Wiktor Niesiobędzki + * + * @param cache entry key type + * + * Generic loader for HTTP based tiles. Uses custom attribute, to check, if entry has expired + * according to HTTP headers sent with tile. If so, it tries to verify using Etags + * or If-Modified-Since / Last-Modified. + * + * If the tile is not valid, it will try to download it from remote service and put it + * to cache. If remote server will fail it will try to use stale entry. + * + * This class will keep only one Job running for specified tile. All others will just finish, but + * listeners will be gathered and notified, once download job will be finished + */ +public abstract class JCSCachedTileLoaderJob implements ICachedLoaderJob { + private static final Logger log = FeatureAdapter.getLogger(JCSCachedTileLoaderJob.class.getCanonicalName()); + protected static final long DEFAULT_EXPIRE_TIME = 1000L * 60 * 60 * 24 * 7; // 7 days + // Limit for the max-age value send by the server. + protected static final long EXPIRE_TIME_SERVER_LIMIT = 1000L * 60 * 60 * 24 * 28; // 4 weeks + // Absolute expire time limit. Cached tiles that are older will not be used, + // even if the refresh from the server fails. + protected static final long ABSOLUTE_EXPIRE_TIME_LIMIT = Long.MAX_VALUE; // unlimited + + private ICacheAccess cache; + private long now; + private ICacheElement cacheElement; + private int connectTimeout; + private int readTimeout; + private Map headers; + private static ConcurrentMap> inProgress = new ConcurrentHashMap<>(); + + protected CacheEntryAttributes attributes = null; + protected byte[] data = null; + private static ConcurrentMap useHead = new ConcurrentHashMap<>(); + + + + /** + * @param cache cache instance that we will work on + * @param headers + * @param readTimeout + * @param connectTimeout + */ + public JCSCachedTileLoaderJob(ICacheAccess cache, + int connectTimeout, int readTimeout, + Map headers) { + + this.cache = cache; + this.now = System.currentTimeMillis(); + this.connectTimeout = connectTimeout; + this.readTimeout = readTimeout; + this.headers = headers; + } + + private void ensureCacheElement() { + if (cacheElement == null && getCacheKey() != null) { + cacheElement = cache.getCacheElement(getCacheKey()); + if (cacheElement != null) { + attributes = (CacheEntryAttributes) cacheElement.getElementAttributes(); + data = cacheElement.getVal().getContent(); + } + } + } + + public byte[] get() { + ensureCacheElement(); + if (cacheElement != null) { + return cacheElement.getVal().getContent(); + } + return null; + } + + @Override + public void submit(ICachedLoaderListener listener) { + boolean first = false; + URL url = getUrl(); + if (url == null) { + log.log(Level.WARNING, "No url returned for: {0}, skipping", getCacheKey()); + return; + } + synchronized (inProgress) { + Set newListeners = inProgress.get(url); + if (newListeners == null) { + newListeners = new HashSet<>(); + inProgress.put(url, newListeners); + first = true; + } + newListeners.add(listener); + } + + if (first) { + ensureCacheElement(); + if (cacheElement != null && isCacheElementValid() && (isObjectLoadable())) { + // we got something in cache, verify it + log.log(Level.FINE, "JCS - Returning object from cache: {0}", getCacheKey()); + finishLoading(true, true); + return; + } + // object not in cache, so submit work to separate thread + JCSJobDispatcher.getInstance().addJob(this); + } + + } + + /** + * + * @return checks if object from cache has sufficient data to be returned + */ + public boolean isObjectLoadable() { + return data != null && data.length > 0; + } + + /** + * + * @return cache object as empty, regardless of what remote resource has returned (ex. based on headers) + */ + protected boolean cacheAsEmpty() { + return false; + } + + public void run() { + // try to load object from remote resource + if (loadObject()) { + finishLoading(true, true); + } else { + // if loading failed - check if we can return stale entry + if (isObjectLoadable()) { + // try to get stale entry in cache + finishLoading(true, true); + log.log(Level.FINE, "JCS - found stale object in cache: {0}", getUrl()); + } else { + // failed completely + finishLoading(false, true); + } + } + } + + + private void finishLoading(boolean success, boolean loaded) { + Set listeners = null; + synchronized (inProgress) { + listeners = inProgress.remove(getUrl()); + } + if (listeners == null) { + log.log(Level.WARNING, "Listener not found for URL: {0}. Listener not notified!", getUrl()); + return; + } + try { + for (ICachedLoaderListener l: listeners) { + l.loadingFinished(data, success); + } + } catch (Exception e) { + log.log(Level.WARNING, "JCS - Error while loading object from cache: {0}; {1}", new Object[]{e.getMessage(), getUrl()}); + log.log(Level.FINE, "Stacktrace", e); + for (ICachedLoaderListener l: listeners) { + l.loadingFinished(data, false); + } + + } + + } + + private boolean isCacheElementValid() { + long expires = attributes.getExpirationTime(); + + // check by expire date set by server + if (expires != 0L) { + // put a limit to the expire time (some servers send a value + // that is too large) + expires = Math.min(expires, attributes.getCreateTime() + EXPIRE_TIME_SERVER_LIMIT); + if (now > expires) { + log.log(Level.FINE, "JCS - Object {0} has expired -> valid to {1}, now is: {2}", new Object[]{getUrl(), Long.toString(expires), Long.toString(now)}); + return false; + } + } else { + // check by file modification date + if (now - attributes.getLastModification() > DEFAULT_EXPIRE_TIME) { + log.log(Level.FINE, "JCS - Object has expired, maximum file age reached {0}", getUrl()); + return false; + } + } + return true; + } + + private boolean loadObject() { + try { + // if we have object in cache, and host doesn't support If-Modified-Since nor If-None-Match + // then just use HEAD request and check returned values + if (isObjectLoadable() && + Boolean.TRUE.equals(useHead.get(getUrl().getHost())) && + isCacheValidUsingHead()) { + log.log(Level.FINE, "JCS - cache entry verified using HEAD request: {0}", getUrl()); + return true; + } + URLConnection urlConn = getURLConnection(); + + if (isObjectLoadable() && + (now - attributes.getLastModification()) <= ABSOLUTE_EXPIRE_TIME_LIMIT) { + urlConn.setIfModifiedSince(attributes.getLastModification()); + } + if (isObjectLoadable() && attributes.getEtag() != null) { + urlConn.addRequestProperty("If-None-Match", attributes.getEtag()); + } + if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) { + // If isModifiedSince or If-None-Match has been set + // and the server answers with a HTTP 304 = "Not Modified" + log.log(Level.FINE, "JCS - IfModifiedSince/Etag test: local version is up to date: {0}", getUrl()); + return true; + } else if (isObjectLoadable()) { + // we have an object in cache, but we haven't received 304 resposne code + // check if we should use HEAD request to verify + if((attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) || + attributes.getLastModification() == urlConn.getLastModified()) { + // we sent ETag or If-Modified-Since, but didn't get 304 response code + // for further requests - use HEAD + String host = getUrl().getHost(); + log.log(Level.INFO, "JCS - Host: {0} found not to return 304 codes for If-Modifed-Since or If-None-Match headers", host); + useHead.put(host, Boolean.TRUE); + } + } + + attributes = parseHeaders(urlConn); + + for (int i = 0; i < 5; ++i) { + if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) { + Thread.sleep(5000+(new Random()).nextInt(5000)); + continue; + } + data = read(urlConn); + if (!cacheAsEmpty() && data != null && data.length > 0) { + cache.put(getCacheKey(), new CacheEntry(data), attributes); + log.log(Level.FINE, "JCS - downloaded key: {0}, length: {1}, url: {2}", + new Object[] {getCacheKey(), data.length, getUrl()}); + return true; + } else { + log.log(Level.FINE, "JCS - Caching empty object {0}", getUrl()); + cache.put(getCacheKey(), new CacheEntry(new byte[]{}), attributes); + return true; + } + } + } catch (Exception e) { + log.log(Level.WARNING, "JCS - Exception during download " + getUrl(), e); + } + log.log(Level.WARNING, "JCS - Silent failure during download: {0}", getUrl()); + return false; + + } + + private CacheEntryAttributes parseHeaders(URLConnection urlConn) { + CacheEntryAttributes ret = new CacheEntryAttributes(); + ret.setNoTileAtZoom("no-tile".equals(urlConn.getHeaderField("X-VE-Tile-Info"))); + + Long lng = urlConn.getExpiration(); + if (lng.equals(0L)) { + try { + String str = urlConn.getHeaderField("Cache-Control"); + if (str != null) { + for (String token: str.split(",")) { + if (token.startsWith("max-age=")) { + lng = Long.parseLong(token.substring(8)) * 1000 + + System.currentTimeMillis(); + } + } + } + } catch (NumberFormatException e) {} //ignore malformed Cache-Control headers + } + + ret.setExpirationTime(lng); + ret.setLastModification(now); + ret.setEtag(urlConn.getHeaderField("ETag")); + return ret; + } + + private HttpURLConnection getURLConnection() throws IOException, MalformedURLException { + HttpURLConnection urlConn = (HttpURLConnection) getUrl().openConnection(); + urlConn.setRequestProperty("Accept", "text/html, image/png, image/jpeg, image/gif, */*"); + urlConn.setReadTimeout(readTimeout); // 30 seconds read timeout + urlConn.setConnectTimeout(connectTimeout); + for(Map.Entry e: headers.entrySet()) { + urlConn.setRequestProperty(e.getKey(), e.getValue()); + } + return urlConn; + } + + private boolean isCacheValidUsingHead() throws IOException { + HttpURLConnection urlConn = (HttpURLConnection) getUrl().openConnection(); + urlConn.setRequestMethod("HEAD"); + long lastModified = urlConn.getLastModified(); + return ( + (attributes.getEtag() != null && attributes.getEtag().equals(urlConn.getRequestProperty("ETag"))) || + (lastModified != 0 && lastModified <= attributes.getLastModification()) + ); + } + + private static byte[] read(URLConnection urlConn) throws IOException { + InputStream input = urlConn.getInputStream(); + try { + ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available()); + byte[] buffer = new byte[2048]; + boolean finished = false; + do { + int read = input.read(buffer); + if (read >= 0) { + bout.write(buffer, 0, read); + } else { + finished = true; + } + } while (!finished); + if (bout.size() == 0) + return null; + return bout.toByteArray(); + } finally { + input.close(); + } + } +} Index: src/org/openstreetmap/josm/data/cache/CacheEntryAttributes.java =================================================================== --- src/org/openstreetmap/josm/data/cache/CacheEntryAttributes.java (revision 0) +++ src/org/openstreetmap/josm/data/cache/CacheEntryAttributes.java (working copy) @@ -0,0 +1,64 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.cache; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.jcs.engine.ElementAttributes; + +public class CacheEntryAttributes extends ElementAttributes { + private static final long serialVersionUID = 1L; //version + private Map attrs = new HashMap(); + private final static String NO_TILE_AT_ZOOM = "noTileAtZoom"; + private final static String ETAG = "Etag"; + private final static String LAST_MODIFICATION = "lastModification"; + private final static String EXPIRATION_TIME = "expirationTime"; + //private boolean noTileAtZoom = false; + private String Etag = null; + private long lastModification = 0; + private long expirationTime = 0; + + + public CacheEntryAttributes() { + super(); + attrs.put(NO_TILE_AT_ZOOM, "false"); + attrs.put(ETAG, null); + attrs.put(LAST_MODIFICATION, "0"); + attrs.put(EXPIRATION_TIME, "0"); + } + public boolean isNoTileAtZoom() { + return Boolean.toString(true).equals(attrs.get(NO_TILE_AT_ZOOM)); + } + public void setNoTileAtZoom(boolean noTileAtZoom) { + attrs.put(NO_TILE_AT_ZOOM, Boolean.toString(noTileAtZoom)); + } + public String getEtag() { + return attrs.get(ETAG); + } + public void setEtag(String etag) { + attrs.put(ETAG, etag); + } + + private long getLongAttr(String key) { + try { + return Long.parseLong(attrs.get(key)); + } catch (NumberFormatException e) { + attrs.put(key, "0"); + return 0; + } + } + + public long getLastModification() { + return getLongAttr(LAST_MODIFICATION); + } + public void setLastModification(long lastModification) { + attrs.put(LAST_MODIFICATION, Long.toString(lastModification)); + } + public long getExpirationTime() { + return getLongAttr(EXPIRATION_TIME); + } + public void setExpirationTime(long expirationTime) { + attrs.put(EXPIRATION_TIME, Long.toString(expirationTime)); + } + +} Index: src/org/openstreetmap/josm/data/cache/ICachedLoaderJob.java =================================================================== --- src/org/openstreetmap/josm/data/cache/ICachedLoaderJob.java (revision 0) +++ src/org/openstreetmap/josm/data/cache/ICachedLoaderJob.java (working copy) @@ -0,0 +1,48 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.cache; + +import java.net.URL; + + +/** + * + * @author Wiktor Niesiobędzki + * + * @param cache key type + * @param value that is returned from cache + */ +public interface ICachedLoaderJob { + /** + * returns cache entry key + * + * @param tile + * @return cache key for tile + */ + public K getCacheKey(); + + /** + * method to get download URL for Job + * @return URL that should be fetched + * + */ + public URL getUrl(); + /** + * implements the main algorithm for fetching + */ + public void run(); + + /** + * fetches object from cache, or returns null when object is not found + * + * @return filled tile with data or null when no cache entry found + */ + public byte[] get(); + + /** + * Submit job for background fetch, and listener will be + * fed with value object + * + * @param listener + */ + public void submit(ICachedLoaderListener listener); +} Index: src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoader.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoader.java (revision 0) +++ src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoader.java (working copy) @@ -0,0 +1,60 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery; + +import java.io.IOException; +import java.util.Map; + +import org.apache.commons.jcs.access.behavior.ICacheAccess; +import org.openstreetmap.gui.jmapviewer.Tile; +import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader; +import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; +import org.openstreetmap.josm.data.cache.CacheEntry; +import org.openstreetmap.josm.data.cache.JCSCacheManager; +import org.openstreetmap.josm.data.preferences.IntegerProperty; + +/** + * @author Wiktor Niesiobędzki + * + * Wrapper class that bridges between JCS cache and Tile Loaders + * + */ +public class TMSCachedTileLoader implements TileLoader, CachedTileLoader { + + private ICacheAccess cache; + private int connectTimeout; + private int readTimeout; + private Map headers; + private TileLoaderListener listener; + public static final String PREFERENCE_PREFIX = "imagery.tms.cache."; + // average tile size is about 20kb + public static IntegerProperty MAX_OBJECTS_IN_MEMORY = new IntegerProperty(PREFERENCE_PREFIX + "max_objects_memory", 1000); // 1000 is around 20MB under this assumptions + public static IntegerProperty MAX_OBJECTS_ON_DISK = new IntegerProperty(PREFERENCE_PREFIX + "max_objects_disk", 25000); // 25000 is around 500MB under this assumptions + + /** + * Constructor + * @param name of the cache + * @param connectTimeout to remote resource + * @param readTimeout to remote resource + * @param headers to be sent along with request + * @throws IOException when cache initialization fails + */ + public TMSCachedTileLoader(TileLoaderListener listener, String name, int connectTimeout, int readTimeout, Map headers) throws IOException { + this.cache = JCSCacheManager.getCache(name, MAX_OBJECTS_IN_MEMORY.get(), MAX_OBJECTS_ON_DISK.get()); + this.connectTimeout = connectTimeout; + this.readTimeout = readTimeout; + this.headers = headers; + this.listener = listener; + } + + @Override + public TileJob createTileLoaderJob(Tile tile) { + return new TMSCachedTileLoaderJob(listener, tile, cache, connectTimeout, readTimeout, headers); + } + + @Override + public void clearCache() { + this.cache.clear(); + } +} Index: src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java (revision 0) +++ src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java (working copy) @@ -0,0 +1,128 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.net.URL; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.jcs.access.behavior.ICacheAccess; +import org.openstreetmap.gui.jmapviewer.FeatureAdapter; +import org.openstreetmap.gui.jmapviewer.Tile; +import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; +import org.openstreetmap.josm.data.cache.CacheEntry; +import org.openstreetmap.josm.data.cache.ICachedLoaderListener; +import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob; + +/** + * @author Wiktor Niesiobędzki + * + * Class bridging TMS requests to JCS cache requests + * + */ +public class TMSCachedTileLoaderJob extends JCSCachedTileLoaderJob implements TileJob, ICachedLoaderListener { + private static final Logger log = FeatureAdapter.getLogger(TMSCachedTileLoaderJob.class.getCanonicalName()); + private Tile tile; + private TileLoaderListener listener; + private URL url; + + /** + * Constructor for creating a job, to get a specific tile from cache + * @param listener + * @param tile to be fetched from cache + * @param cache object + * @param connectTimeout when connecting to remote resource + * @param readTimeout when connecting to remote resource + * @param headers to be sent together with request + */ + public TMSCachedTileLoaderJob(TileLoaderListener listener, Tile tile, ICacheAccess cache, int connectTimeout, int readTimeout, + Map headers) { + super(cache, connectTimeout, readTimeout, headers); + this.tile = tile; + this.listener = listener; + // URLs tend to change for some tile providers. Make a static reference here, so the tile URL might be used as a key + // for request deduplication + try { + this.url = new URL(tile.getUrl()); + } catch (IOException e) { + log.log(Level.WARNING, "JCS TMS Cache - error creating URL for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()}); + } + + } + + @Override + public Tile getTile() { + return tile; + } + + @Override + public String getCacheKey() { + if (tile != null) + return tile.getKey(); + return null; + } + + @Override + public URL getUrl() { + return url; + } + + @Override + public boolean isObjectLoadable() { + return (data != null && data.length > 0) || cacheAsEmpty(); + } + + @Override + protected boolean cacheAsEmpty() { + if (attributes != null && attributes.isNoTileAtZoom()) { + // do not remove file - keep the information, that there is no tile, for further requests + // the code above will check, if this information is still valid + log.log(Level.FINE, "JCS TMS - Tile valid, but no file, as no tiles at this level {0}", tile); + tile.setError("No tile at this zoom level"); + tile.putValue("tile-info", "no-tile"); + return true; + } + return false; + } + + public void submit() { + super.submit(this); + } + + @Override + public void loadingFinished(byte[] object, boolean success) { + try { + if (object != null && object.length > 0) { + tile.loadImage(new ByteArrayInputStream(object)); + } + tile.setLoaded(true); + if (listener != null) { + listener.tileLoadingFinished(tile, success); + } + } catch (IOException e) { + log.log(Level.WARNING, "JCS TMS - error loading object for tile {0}: {1}", new Object[] {tile.getKey(), e.getMessage()}); + tile.setError(e.getMessage()); + tile.setLoaded(false); + if (listener != null) { + listener.tileLoadingFinished(tile, false); + } + } + } + + /** + * Method for getting the tile from cache only, without trying to reach remote resource + * @return tile or null, if nothing (useful) was found in cache + */ + public Tile getCachedTile() { + byte[] data = super.get(); + if (isObjectLoadable()) { + loadingFinished(data, true); + return tile; + } else { + return null; + } + } +} Index: src/org/openstreetmap/josm/gui/layer/TMSLayer.java =================================================================== --- src/org/openstreetmap/josm/gui/layer/TMSLayer.java (revision 8117) +++ src/org/openstreetmap/josm/gui/layer/TMSLayer.java (working copy) @@ -21,13 +21,11 @@ import java.net.URL; import java.util.ArrayList; import java.util.Collections; -import java.util.HashSet; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Scanner; -import java.util.Set; import java.util.concurrent.Callable; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -43,12 +41,10 @@ import org.openstreetmap.gui.jmapviewer.Coordinate; import org.openstreetmap.gui.jmapviewer.JobDispatcher; import org.openstreetmap.gui.jmapviewer.MemoryTileCache; -import org.openstreetmap.gui.jmapviewer.OsmFileCacheTileLoader; import org.openstreetmap.gui.jmapviewer.OsmTileLoader; -import org.openstreetmap.gui.jmapviewer.TMSFileCacheTileLoader; import org.openstreetmap.gui.jmapviewer.Tile; import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader; -import org.openstreetmap.gui.jmapviewer.interfaces.TileClearController; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; import org.openstreetmap.gui.jmapviewer.tilesources.BingAerialTileSource; @@ -63,6 +59,7 @@ import org.openstreetmap.josm.data.coor.LatLon; import org.openstreetmap.josm.data.imagery.ImageryInfo; import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryType; +import org.openstreetmap.josm.data.imagery.TMSCachedTileLoader; import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor; import org.openstreetmap.josm.data.preferences.BooleanProperty; import org.openstreetmap.josm.data.preferences.IntegerProperty; @@ -75,7 +72,6 @@ import org.openstreetmap.josm.gui.dialogs.LayerListDialog; import org.openstreetmap.josm.gui.dialogs.LayerListPopup; import org.openstreetmap.josm.gui.progress.ProgressMonitor; -import org.openstreetmap.josm.gui.progress.ProgressMonitor.CancelListener; import org.openstreetmap.josm.io.CacheCustomContent; import org.openstreetmap.josm.io.OsmTransferException; import org.openstreetmap.josm.io.UTFInputStreamReader; @@ -122,40 +118,51 @@ } public interface TileLoaderFactory { - OsmTileLoader makeTileLoader(TileLoaderListener listener); + TileLoader makeTileLoader(TileLoaderListener listener); + TileLoader makeTileLoader(TileLoaderListener listener, Map headers); } + // MemoryTileCache caches rendered tiles, to reduce latency during map panning + // ImageIO.read() takes a lot of time, so we can't use JCS cache protected MemoryTileCache tileCache; protected TileSource tileSource; - protected OsmTileLoader tileLoader; + protected TileLoader tileLoader; + public static TileLoaderFactory loaderFactory = new TileLoaderFactory() { @Override - public OsmTileLoader makeTileLoader(TileLoaderListener listener) { - String cachePath = TMSLayer.PROP_TILECACHE_DIR.get(); - if (cachePath != null && !cachePath.isEmpty()) { - try { - OsmFileCacheTileLoader loader; - loader = new TMSFileCacheTileLoader(listener, new File(cachePath)); - loader.headers.put("User-Agent", Version.getInstance().getFullAgentString()); - return loader; - } catch (IOException e) { - Main.warn(e); - } + public TileLoader makeTileLoader(TileLoaderListener listener, Map inputHeaders) { + Map headers = new HashMap<>(); + headers.put("User-Agent", Version.getInstance().getFullAgentString()); + headers.put("Accept", "text/html, image/png, image/jpeg, image/gif, */*"); + if (inputHeaders != null) + headers.putAll(inputHeaders); + + try { + return new TMSCachedTileLoader(listener, "TMS", + Main.pref.getInteger("socket.timeout.connect",15) * 1000, + Main.pref.getInteger("socket.timeout.read", 30) * 1000, + headers); + } catch (IOException e) { + Main.warn(e); } return null; } + + @Override + public TileLoader makeTileLoader(TileLoaderListener listener) { + return makeTileLoader(listener, null); + } }; /** * Plugins that wish to set custom tile loader should call this method */ + public static void setCustomTileLoaderFactory(TileLoaderFactory loaderFactory) { TMSLayer.loaderFactory = loaderFactory; } - private Set tileRequestsOutstanding = new HashSet<>(); - @Override public synchronized void tileLoadingFinished(Tile tile, boolean success) { if (tile.hasError()) { @@ -170,53 +177,11 @@ if (Main.map != null) { Main.map.repaint(100); } - tileRequestsOutstanding.remove(tile); if (Main.isDebugEnabled()) { Main.debug("tileLoadingFinished() tile: " + tile + " success: " + success); } } - private static class TmsTileClearController implements TileClearController, CancelListener { - - private final ProgressMonitor monitor; - private boolean cancel = false; - - public TmsTileClearController(ProgressMonitor monitor) { - this.monitor = monitor; - this.monitor.addCancelListener(this); - } - - @Override - public void initClearDir(File dir) { - } - - @Override - public void initClearFiles(File[] files) { - monitor.setTicksCount(files.length); - monitor.setTicks(0); - } - - @Override - public boolean cancel() { - return cancel; - } - - @Override - public void fileDeleted(File file) { - monitor.setTicks(monitor.getTicks()+1); - } - - @Override - public void clearFinished() { - monitor.finishTask(); - } - - @Override - public void operationCanceled() { - cancel = true; - } - } - /** * Clears the tile cache. * @@ -231,7 +196,7 @@ void clearTileCache(ProgressMonitor monitor) { tileCache.clear(); if (tileLoader instanceof CachedTileLoader) { - ((CachedTileLoader)tileLoader).clearCache(tileSource, new TmsTileClearController(monitor)); + ((CachedTileLoader)tileLoader).clearCache(); } } @@ -412,23 +377,17 @@ private void initTileSource(TileSource tileSource) { this.tileSource = tileSource; attribution.initialize(tileSource); - currentZoomLevel = getBestZoom(); - tileCache = new MemoryTileCache(); - - tileLoader = loaderFactory.makeTileLoader(this); - if (tileLoader == null) { - tileLoader = new OsmTileLoader(this); - } - tileLoader.timeoutConnect = Main.pref.getInteger("socket.timeout.connect",15) * 1000; - tileLoader.timeoutRead = Main.pref.getInteger("socket.timeout.read", 30) * 1000; + Map headers = null; if (tileSource instanceof TemplatedTMSTileSource) { - for(Entry e : ((TemplatedTMSTileSource)tileSource).getHeaders().entrySet()) { - tileLoader.headers.put(e.getKey(), e.getValue()); - } + headers = (((TemplatedTMSTileSource)tileSource).getHeaders()); } - tileLoader.headers.put("User-Agent", Version.getInstance().getFullAgentString()); + + tileCache = new MemoryTileCache(); + tileLoader = loaderFactory.makeTileLoader(this, headers); + if (tileLoader == null) + tileLoader = new OsmTileLoader(this); } @Override @@ -487,11 +446,11 @@ setMaxWorkers(); if(!isProjectionSupported(Main.getProjection())) { JOptionPane.showMessageDialog(Main.parent, - tr("TMS layers do not support the projection {0}.\n{1}\n" - + "Change the projection or remove the layer.", - Main.getProjection().toCode(), nameSupportedProjections()), - tr("Warning"), - JOptionPane.WARNING_MESSAGE); + tr("TMS layers do not support the projection {0}.\n{1}\n" + + "Change the projection or remove the layer.", + Main.getProjection().toCode(), nameSupportedProjections()), + tr("Warning"), + JOptionPane.WARNING_MESSAGE); } setBackgroundLayer(true); @@ -685,7 +644,6 @@ } needRedraw = true; JobDispatcher.getInstance().cancelOutstandingJobs(); - tileRequestsOutstanding.clear(); } int getMaxZoomLvl() { @@ -770,7 +728,7 @@ * are temporary only and intentionally not inserted * into the tileCache. */ - synchronized Tile tempCornerTile(Tile t) { + Tile tempCornerTile(Tile t) { int x = t.getXtile() + 1; int y = t.getYtile() + 1; int zoom = t.getZoom(); @@ -780,7 +738,7 @@ return new Tile(tileSource, x, y, zoom); } - synchronized Tile getOrCreateTile(int x, int y, int zoom) { + Tile getOrCreateTile(int x, int y, int zoom) { Tile tile = getTile(x, y, zoom); if (tile == null) { tile = new Tile(tileSource, x, y, zoom); @@ -794,24 +752,21 @@ * This can and will return null for tiles that are not * already in the cache. */ - synchronized Tile getTile(int x, int y, int zoom) { + Tile getTile(int x, int y, int zoom) { int max = (1 << zoom); if (x < 0 || x >= max || y < 0 || y >= max) return null; return tileCache.getTile(tileSource, x, y, zoom); } - synchronized boolean loadTile(Tile tile, boolean force) { + boolean loadTile(Tile tile, boolean force) { if (tile == null) return false; - if (!force && (tile.hasError() || tile.isLoaded())) + if (!force && (tile.isLoaded() || tile.hasError())) return false; if (tile.isLoading()) return false; - if (tileRequestsOutstanding.contains(tile)) - return false; - tileRequestsOutstanding.add(tile); - JobDispatcher.getInstance().addJob(tileLoader.createTileLoaderJob(tile)); + tileLoader.createTileLoaderJob(tile).submit(); return true; } @@ -1268,28 +1223,32 @@ public TileSet getTileSet(int zoom) { if (zoom < minZoom) return nullTileSet; - TileSet ts = tileSets[zoom-minZoom]; - if (ts == null) { - ts = new TileSet(topLeft, botRight, zoom); - tileSets[zoom-minZoom] = ts; + synchronized (tileSets) { + TileSet ts = tileSets[zoom-minZoom]; + if (ts == null) { + ts = new TileSet(topLeft, botRight, zoom); + tileSets[zoom-minZoom] = ts; + } + return ts; } - return ts; } + public TileSetInfo getTileSetInfo(int zoom) { if (zoom < minZoom) return new TileSetInfo(); - TileSetInfo tsi = tileSetInfos[zoom-minZoom]; - if (tsi == null) { - tsi = TMSLayer.getTileSetInfo(getTileSet(zoom)); - tileSetInfos[zoom-minZoom] = tsi; + synchronized (tileSetInfos) { + TileSetInfo tsi = tileSetInfos[zoom-minZoom]; + if (tsi == null) { + tsi = TMSLayer.getTileSetInfo(getTileSet(zoom)); + tileSetInfos[zoom-minZoom] = tsi; + } + return tsi; } - return tsi; } } @Override public void paint(Graphics2D g, MapView mv, Bounds bounds) { - //long start = System.currentTimeMillis(); EastNorth topLeft = mv.getEastNorth(0, 0); EastNorth botRight = mv.getEastNorth(mv.getWidth(), mv.getHeight()); Index: src/org/openstreetmap/josm/gui/bbox/SlippyMapBBoxChooser.java =================================================================== --- src/org/openstreetmap/josm/gui/bbox/SlippyMapBBoxChooser.java (revision 8117) +++ src/org/openstreetmap/josm/gui/bbox/SlippyMapBBoxChooser.java (working copy) @@ -25,6 +25,7 @@ import org.openstreetmap.gui.jmapviewer.MemoryTileCache; import org.openstreetmap.gui.jmapviewer.OsmTileLoader; import org.openstreetmap.gui.jmapviewer.interfaces.MapMarker; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; import org.openstreetmap.gui.jmapviewer.tilesources.MapQuestOpenAerialTileSource; import org.openstreetmap.gui.jmapviewer.tilesources.MapQuestOsmTileSource; @@ -113,7 +114,7 @@ private static final StringProperty PROP_MAPSTYLE = new StringProperty("slippy_map_chooser.mapstyle", "Mapnik"); public static final String RESIZE_PROP = SlippyMapBBoxChooser.class.getName() + ".resize"; - private OsmTileLoader cachedLoader; + private TileLoader cachedLoader; private OsmTileLoader uncachedLoader; private final SizeButton iSizeButton; Index: src/org/openstreetmap/gui/jmapviewer/TMSFileCacheTileLoader.java =================================================================== --- src/org/openstreetmap/gui/jmapviewer/TMSFileCacheTileLoader.java (revision 31054) +++ src/org/openstreetmap/gui/jmapviewer/TMSFileCacheTileLoader.java (working copy) @@ -1,76 +0,0 @@ -// License: GPL. For details, see Readme.txt file. -package org.openstreetmap.gui.jmapviewer; - -import java.io.File; -import java.io.IOException; -import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; -import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; -import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; - -/** - * Reworked version of the OsmFileCacheTileLoader. - * - * When class OsmFileCacheTileLoader is no longer needed, it can be integrated - * here and removed. - */ -public class TMSFileCacheTileLoader extends OsmFileCacheTileLoader { - - public TMSFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException { - super(map, cacheDir); - } - - @Override - public TileJob createTileLoaderJob(final Tile tile) { - return new TMSFileLoadJob(tile); - } - - protected class TMSFileLoadJob extends FileLoadJob { - - public TMSFileLoadJob(Tile tile) { - super(tile); - } - - @Override - protected File getTileFile() { - return getDataFile(tile.getSource().getTileType()); - } - - @Override - protected File getTagsFile() { - return getDataFile(TAGS_FILE_EXT); - } - - protected File getDataFile(String ext) { - int nDigits = (int) Math.ceil(Math.log10(1 << tile.getZoom())); - String x = String.format("%0" + nDigits + "d", tile.getXtile()); - String y = String.format("%0" + nDigits + "d", tile.getYtile()); - File path = new File(tileCacheDir, "z" + tile.getZoom()); - for (int i=0; i|]", "_")); - } - if (!dir.exists()) { - dir.mkdirs(); - } - } - return dir; - } - -} Index: src/org/openstreetmap/gui/jmapviewer/OsmFileCacheTileLoader.java =================================================================== --- src/org/openstreetmap/gui/jmapviewer/OsmFileCacheTileLoader.java (revision 31055) +++ src/org/openstreetmap/gui/jmapviewer/OsmFileCacheTileLoader.java (working copy) @@ -1,521 +0,0 @@ -// License: GPL. For details, see Readme.txt file. -package org.openstreetmap.gui.jmapviewer; - -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLConnection; -import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Random; -import java.util.logging.Level; -import java.util.logging.Logger; - -import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader; -import org.openstreetmap.gui.jmapviewer.interfaces.TileClearController; -import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; -import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; -import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; -import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; -import org.openstreetmap.gui.jmapviewer.interfaces.TileSource.TileUpdate; - -/** - * A {@link TileLoader} implementation that loads tiles from OSM via HTTP and - * saves all loaded files in a directory located in the temporary directory. - * If a tile is present in this file cache it will not be loaded from OSM again. - * - * @author Jan Peter Stotz - * @author Stefan Zeller - */ -public class OsmFileCacheTileLoader extends OsmTileLoader implements CachedTileLoader { - - private static final Logger log = FeatureAdapter.getLogger(OsmFileCacheTileLoader.class.getName()); - - protected static final String TAGS_FILE_EXT = "tags"; - - private static final Charset TAGS_CHARSET = Charset.forName("UTF-8"); - - // Default expire time (i.e. maximum age of cached tile before refresh). - // Used when the server does not send an expires or max-age value in the http header. - protected static final long DEFAULT_EXPIRE_TIME = 1000L * 60 * 60 * 24 * 7; // 7 days - // Limit for the max-age value send by the server. - protected static final long EXPIRE_TIME_SERVER_LIMIT = 1000L * 60 * 60 * 24 * 28; // 4 weeks - // Absolute expire time limit. Cached tiles that are older will not be used, - // even if the refresh from the server fails. - protected static final long ABSOLUTE_EXPIRE_TIME_LIMIT = Long.MAX_VALUE; // unlimited - - protected String cacheDirBase; - - protected final Map sourceCacheDirMap; - - - public static File getDefaultCacheDir() throws SecurityException { - String tempDir = null; - String userName = System.getProperty("user.name"); - try { - tempDir = System.getProperty("java.io.tmpdir"); - } catch (SecurityException e) { - log.log(Level.WARNING, - "Failed to access system property ''java.io.tmpdir'' for security reasons. Exception was: " - + e.toString()); - throw e; // rethrow - } - try { - if (tempDir == null) - throw new IOException("No temp directory set"); - String subDirName = "JMapViewerTiles"; - // On Linux/Unix systems we do not have a per user tmp directory. - // Therefore we add the user name for getting a unique dir name. - if (userName != null && userName.length() > 0) { - subDirName += "_" + userName; - } - File cacheDir = new File(tempDir, subDirName); - return cacheDir; - } catch (Exception e) { - } - return null; - } - - /** - * Create a OSMFileCacheTileLoader with given cache directory. - * If cacheDir is not set or invalid, IOException will be thrown. - * @param map the listener checking for tile load events (usually the map for display) - * @param cacheDir directory to store cached tiles - */ - public OsmFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException { - super(map); - if (cacheDir == null || (!cacheDir.exists() && !cacheDir.mkdirs())) - throw new IOException("Cannot access cache directory"); - - log.finest("Tile cache directory: " + cacheDir); - cacheDirBase = cacheDir.getAbsolutePath(); - sourceCacheDirMap = new HashMap<>(); - } - - /** - * Create a OSMFileCacheTileLoader with system property temp dir. - * If not set an IOException will be thrown. - * @param map the listener checking for tile load events (usually the map for display) - */ - public OsmFileCacheTileLoader(TileLoaderListener map) throws SecurityException, IOException { - this(map, getDefaultCacheDir()); - } - - @Override - public TileJob createTileLoaderJob(final Tile tile) { - return new FileLoadJob(tile); - } - - protected File getSourceCacheDir(TileSource source) { - File dir = sourceCacheDirMap.get(source); - if (dir == null) { - dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_")); - if (!dir.exists()) { - dir.mkdirs(); - } - } - return dir; - } - - protected class FileLoadJob implements TileJob { - InputStream input = null; - - Tile tile; - File tileCacheDir; - File tileFile = null; - File tagsFile = null; - Long fileMtime = null; - Long now = null; // current time in milliseconds (keep consistent value for the whole run) - - public FileLoadJob(Tile tile) { - this.tile = tile; - } - - @Override - public Tile getTile() { - return tile; - } - - @Override - public void run() { - synchronized (tile) { - if ((tile.isLoaded() && !tile.hasError()) || tile.isLoading()) - return; - tile.loaded = false; - tile.error = false; - tile.loading = true; - } - now = System.currentTimeMillis(); - tileCacheDir = getSourceCacheDir(tile.getSource()); - tileFile = getTileFile(); - tagsFile = getTagsFile(); - - loadTagsFromFile(); - - if (isCacheValid() && (isNoTileAtZoom() || loadTileFromFile())) { - log.log(Level.FINE, "TMS - found in tile cache: {0}", tile); - tile.setLoaded(true); - listener.tileLoadingFinished(tile, true); - return; - } - - TileJob job = new TileJob() { - - @Override - public void run() { - if (loadOrUpdateTile()) { - tile.setLoaded(true); - listener.tileLoadingFinished(tile, true); - } else { - // failed to download - use old cache file if available - if (isNoTileAtZoom() || loadTileFromFile()) { - tile.setLoaded(true); - tile.error = false; - listener.tileLoadingFinished(tile, true); - log.log(Level.FINE, "TMS - found stale tile in cache: {0}", tile); - } else { - // failed completely - tile.setLoaded(true); - listener.tileLoadingFinished(tile, false); - } - } - } - @Override - public Tile getTile() { - return tile; - } - }; - JobDispatcher.getInstance().addJob(job); - } - - protected boolean loadOrUpdateTile() { - try { - URLConnection urlConn = loadTileFromOsm(tile); - if (fileMtime != null && now - fileMtime <= ABSOLUTE_EXPIRE_TIME_LIMIT) { - switch (tile.getSource().getTileUpdate()) { - case IfModifiedSince: - urlConn.setIfModifiedSince(fileMtime); - break; - case LastModified: - if (!isOsmTileNewer(fileMtime)) { - log.log(Level.FINE, "TMS - LastModified test: local version is up to date: {0}", tile); - tileFile.setLastModified(now); - return true; - } - break; - default: - break; - } - } - if (tile.getSource().getTileUpdate() == TileUpdate.ETag || tile.getSource().getTileUpdate() == TileUpdate.IfNoneMatch) { - String fileETag = tile.getValue("etag"); - if (fileETag != null) { - switch (tile.getSource().getTileUpdate()) { - case IfNoneMatch: - urlConn.addRequestProperty("If-None-Match", fileETag); - break; - case ETag: - if (hasOsmTileETag(fileETag)) { - log.log(Level.FINE, "TMS - ETag test: local version is up to date: {0}", tile); - tileFile.setLastModified(now); - return true; - } - default: - break; - } - } - tile.putValue("etag", urlConn.getHeaderField("ETag")); - } - if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) { - // If isModifiedSince or If-None-Match has been set - // and the server answers with a HTTP 304 = "Not Modified" - switch (tile.getSource().getTileUpdate()) { - case IfModifiedSince: - log.log(Level.FINE, "TMS - IfModifiedSince test: local version is up to date: {0}", tile); - break; - case IfNoneMatch: - log.log(Level.FINE, "TMS - IfNoneMatch test: local version is up to date: {0}", tile); - break; - default: - break; - } - loadTileFromFile(); - tileFile.setLastModified(now); - return true; - } - - loadTileMetadata(tile, urlConn); - saveTagsToFile(); - - if ("no-tile".equals(tile.getValue("tile-info"))) - { - log.log(Level.FINE, "TMS - No tile: tile-info=no-tile: {0}", tile); - tile.setError("No tile at this zoom level"); - return true; - } else { - for (int i = 0; i < 5; ++i) { - if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 503) { - Thread.sleep(5000+(new Random()).nextInt(5000)); - continue; - } - byte[] buffer = loadTileInBuffer(urlConn); - if (buffer != null) { - tile.loadImage(new ByteArrayInputStream(buffer)); - saveTileToFile(buffer); - log.log(Level.FINE, "TMS - downloaded tile from server: {0}", tile.getUrl()); - return true; - } - } - } - } catch (Exception e) { - tile.setError(e.getMessage()); - if (input == null) { - try { - log.log(Level.WARNING, "TMS - Failed downloading {0}: {1}", new Object[]{tile.getUrl(), e.getMessage()}); - return false; - } catch(IOException i) { - } - } - } - log.log(Level.WARNING, "TMS - Failed downloading tile: {0}", tile); - return false; - } - - protected boolean isCacheValid() { - Long expires = null; - if (tileFile.exists()) { - fileMtime = tileFile.lastModified(); - } else if (tagsFile.exists()) { - fileMtime = tagsFile.lastModified(); - } else - return false; - - try { - expires = Long.parseLong(tile.getValue("expires")); - } catch (NumberFormatException e) {} - - // check by expire date set by server - if (expires != null && !expires.equals(0L)) { - // put a limit to the expire time (some servers send a value - // that is too large) - expires = Math.min(expires, fileMtime + EXPIRE_TIME_SERVER_LIMIT); - if (now > expires) { - log.log(Level.FINE, "TMS - Tile has expired -> not valid {0}", tile); - return false; - } - } else { - // check by file modification date - if (now - fileMtime > DEFAULT_EXPIRE_TIME) { - log.log(Level.FINE, "TMS - Tile has expired, maximum file age reached {0}", tile); - return false; - } - } - return true; - } - - protected boolean isNoTileAtZoom() { - if ("no-tile".equals(tile.getValue("tile-info"))) { - // do not remove file - keep the information, that there is no tile, for further requests - // the code above will check, if this information is still valid - log.log(Level.FINE, "TMS - Tile valid, but no file, as no tiles at this level {0}", tile); - tile.setError("No tile at this zoom level"); - return true; - } - return false; - } - - protected boolean loadTileFromFile() { - if (!tileFile.exists()) - return false; - - try (FileInputStream fin = new FileInputStream(tileFile)) { - if (fin.available() == 0) - throw new IOException("File empty"); - tile.loadImage(fin); - return true; - } catch (Exception e) { - log.log(Level.WARNING, "TMS - Error while loading image from tile cache: {0}; {1}", new Object[]{e.getMessage(), tile}); - tileFile.delete(); - if (tagsFile.exists()) { - tagsFile.delete(); - } - tileFile = null; - fileMtime = null; - } - return false; - } - - protected byte[] loadTileInBuffer(URLConnection urlConn) throws IOException { - input = urlConn.getInputStream(); - try { - ByteArrayOutputStream bout = new ByteArrayOutputStream(input.available()); - byte[] buffer = new byte[2048]; - boolean finished = false; - do { - int read = input.read(buffer); - if (read >= 0) { - bout.write(buffer, 0, read); - } else { - finished = true; - } - } while (!finished); - if (bout.size() == 0) - return null; - return bout.toByteArray(); - } finally { - input.close(); - input = null; - } - } - - /** - * Performs a HEAD request for retrieving the - * LastModified header value. - * - * Note: This does only work with servers providing the - * LastModified header: - *
    - *
  • {@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.CycleMap} - supported
    • - *
  • {@link org.openstreetmap.gui.jmapviewer.tilesources.OsmTileSource.Mapnik} - not supported
    • - *
- * - * @param fileAge time of the - * @return true if the tile on the server is newer than the - * file - * @throws IOException - */ - protected boolean isOsmTileNewer(long fileAge) throws IOException { - URL url; - url = new URL(tile.getUrl()); - HttpURLConnection urlConn = (HttpURLConnection) url.openConnection(); - prepareHttpUrlConnection(urlConn); - urlConn.setRequestMethod("HEAD"); - urlConn.setReadTimeout(30000); // 30 seconds read timeout - // System.out.println("Tile age: " + new - // Date(urlConn.getLastModified()) + " / " - // + new Date(fileMtime)); - long lastModified = urlConn.getLastModified(); - if (lastModified == 0) - return true; // no LastModified time returned - return (lastModified > fileAge); - } - - protected boolean hasOsmTileETag(String eTag) throws IOException { - URL url; - url = new URL(tile.getUrl()); - HttpURLConnection urlConn = (HttpURLConnection) url.openConnection(); - prepareHttpUrlConnection(urlConn); - urlConn.setRequestMethod("HEAD"); - urlConn.setReadTimeout(30000); // 30 seconds read timeout - // System.out.println("Tile age: " + new - // Date(urlConn.getLastModified()) + " / " - // + new Date(fileMtime)); - String osmETag = urlConn.getHeaderField("ETag"); - if (osmETag == null) - return true; - return (osmETag.equals(eTag)); - } - - protected File getTileFile() { - return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "." - + tile.getSource().getTileType()); - } - - protected File getTagsFile() { - return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "." - + TAGS_FILE_EXT); - } - - protected void saveTileToFile(byte[] rawData) { - File file = getTileFile(); - file.getParentFile().mkdirs(); - try (FileOutputStream f = new FileOutputStream(file)) { - f.write(rawData); - } catch (Exception e) { - log.log(Level.SEVERE, "Failed to save tile content: {0}", e.getLocalizedMessage()); - } - } - - protected void saveTagsToFile() { - File tagsFile = getTagsFile(); - tagsFile.getParentFile().mkdirs(); - if (tile.getMetadata() == null) { - tagsFile.delete(); - return; - } - try (PrintWriter f = new PrintWriter(new OutputStreamWriter(new FileOutputStream(tagsFile), TAGS_CHARSET))) { - for (Entry entry : tile.getMetadata().entrySet()) { - f.println(entry.getKey() + "=" + entry.getValue()); - } - } catch (Exception e) { - System.err.println("Failed to save tile tags: " + e.getLocalizedMessage()); - } - } - - protected boolean loadTagsFromFile() { - File tagsFile = getTagsFile(); - try (BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(tagsFile), TAGS_CHARSET))) { - for (String line = f.readLine(); line != null; line = f.readLine()) { - final int i = line.indexOf('='); - if (i == -1 || i == 0) { - System.err.println("Malformed tile tag in file '" + tagsFile.getName() + "':" + line); - continue; - } - tile.putValue(line.substring(0,i),line.substring(i+1)); - } - } catch (FileNotFoundException e) { - } catch (Exception e) { - System.err.println("Failed to load tile tags: " + e.getLocalizedMessage()); - } - - return true; - } - } - - public String getCacheDirBase() { - return cacheDirBase; - } - - public void setTileCacheDir(String tileCacheDir) { - File dir = new File(tileCacheDir); - dir.mkdirs(); - this.cacheDirBase = dir.getAbsolutePath(); - } - - @Override - public void clearCache(TileSource source) { - clearCache(source, null); - } - - @Override - public void clearCache(TileSource source, TileClearController controller) { - File dir = getSourceCacheDir(source); - if (dir != null) { - if (controller != null) controller.initClearDir(dir); - if (dir.isDirectory()) { - File[] files = dir.listFiles(); - if (controller != null) controller.initClearFiles(files); - for (File file : files) { - if (controller != null && controller.cancel()) return; - file.delete(); - if (controller != null) controller.fileDeleted(file); - } - } - dir.delete(); - } - if (controller != null) controller.clearFinished(); - } -} Index: src/org/openstreetmap/gui/jmapviewer/interfaces/TileJob.java =================================================================== --- src/org/openstreetmap/gui/jmapviewer/interfaces/TileJob.java (revision 31054) +++ src/org/openstreetmap/gui/jmapviewer/interfaces/TileJob.java (working copy) @@ -17,4 +17,9 @@ * @return {@link Tile} to be handled */ public Tile getTile(); + + /** + * submits download job to backend. + */ + void submit(); } Index: src/org/openstreetmap/gui/jmapviewer/interfaces/CachedTileLoader.java =================================================================== --- src/org/openstreetmap/gui/jmapviewer/interfaces/CachedTileLoader.java (revision 31054) +++ src/org/openstreetmap/gui/jmapviewer/interfaces/CachedTileLoader.java (working copy) @@ -5,6 +5,5 @@ * Interface that allow cleaning the tile cache without specifying exact type of loader */ public interface CachedTileLoader { - public void clearCache(TileSource source); - public void clearCache(TileSource source, TileClearController controller); + public void clearCache(); } Index: src/org/openstreetmap/gui/jmapviewer/interfaces/TileLoader.java =================================================================== --- src/org/openstreetmap/gui/jmapviewer/interfaces/TileLoader.java (revision 31054) +++ src/org/openstreetmap/gui/jmapviewer/interfaces/TileLoader.java (working copy) @@ -20,4 +20,5 @@ * action. */ public TileJob createTileLoaderJob(Tile tile); + } Index: src/org/openstreetmap/gui/jmapviewer/Demo.java =================================================================== --- src/org/openstreetmap/gui/jmapviewer/Demo.java (revision 31054) +++ src/org/openstreetmap/gui/jmapviewer/Demo.java (working copy) @@ -10,7 +10,6 @@ import java.awt.event.ItemListener; import java.awt.event.MouseAdapter; import java.awt.event.MouseEvent; -import java.io.IOException; import javax.swing.JButton; import javax.swing.JCheckBox; @@ -101,11 +100,7 @@ } }); JComboBox tileLoaderSelector; - try { - tileLoaderSelector = new JComboBox<>(new TileLoader[] { new OsmFileCacheTileLoader(map()), new OsmTileLoader(map()) }); - } catch (IOException e) { - tileLoaderSelector = new JComboBox<>(new TileLoader[] { new OsmTileLoader(map()) }); - } + tileLoaderSelector = new JComboBox<>(new TileLoader[] { new OsmTileLoader(map()) }); tileLoaderSelector.addItemListener(new ItemListener() { public void itemStateChanged(ItemEvent e) { map().setTileLoader((TileLoader) e.getItem()); Index: src/org/openstreetmap/gui/jmapviewer/TileController.java =================================================================== --- src/org/openstreetmap/gui/jmapviewer/TileController.java (revision 31054) +++ src/org/openstreetmap/gui/jmapviewer/TileController.java (working copy) @@ -45,7 +45,7 @@ tile.loadPlaceholderFromCache(tileCache); } if (!tile.isLoaded()) { - jobDispatcher.addJob(tileLoader.createTileLoaderJob(tile)); + tileLoader.createTileLoaderJob(tile).submit(); } return tile; } Index: src/org/openstreetmap/gui/jmapviewer/OsmTileLoader.java =================================================================== --- src/org/openstreetmap/gui/jmapviewer/OsmTileLoader.java (revision 31054) +++ src/org/openstreetmap/gui/jmapviewer/OsmTileLoader.java (working copy) @@ -85,6 +85,12 @@ public Tile getTile() { return tile; } + + @Override + public void submit() { + run(); + + } }; } @@ -142,5 +148,4 @@ public String toString() { return getClass().getSimpleName(); } - }