Index: build.xml =================================================================== --- build.xml (revision 8109) +++ build.xml (working copy) @@ -220,6 +220,14 @@ destdir="build" target="1.7" source="1.7" debug="on" includeantruntime="false" createMissingPackageInfoClass="false" encoding="iso-8859-1"> + + + + + + + + + Index: src/org/apache/commons =================================================================== --- src/org/apache/commons (revision 8117) +++ src/org/apache/commons (working copy) Property changes on: src/org/apache/commons ___________________________________________________________________ Modified: svn:externals ## -1 +1,3 ## -codec http://svn.apache.org/repos/asf/commons/proper/codec/trunk/src/main/java/org/apache/commons/codec +http://svn.apache.org/repos/asf/commons/proper/codec/trunk/src/main/java/org/apache/commons/codec codec +http://svn.apache.org/repos/asf/commons/proper/jcs/trunk/commons-jcs-core/src/main/java/org/apache/commons/jcs jcs +http://svn.apache.org/repos/asf/commons/proper/logging/trunk/src/main/java/org/apache/commons/logging logging Index: src/org/openstreetmap/gui/jmapviewer/OsmFileCacheTileLoader.java =================================================================== --- src/org/openstreetmap/gui/jmapviewer/OsmFileCacheTileLoader.java (revision 31055) +++ src/org/openstreetmap/gui/jmapviewer/OsmFileCacheTileLoader.java (working copy) @@ -1,29 +1,22 @@ // License: GPL. For details, see Readme.txt file. package org.openstreetmap.gui.jmapviewer; -import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStreamWriter; -import java.io.PrintWriter; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; import java.nio.charset.Charset; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; import java.util.Random; import java.util.logging.Level; import java.util.logging.Logger; +import org.apache.commons.jcs.access.behavior.ICacheAccess; +import org.apache.commons.jcs.engine.behavior.ICacheElement; +import org.apache.commons.jcs.engine.behavior.IElementAttributes; import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader; import org.openstreetmap.gui.jmapviewer.interfaces.TileClearController; import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; @@ -31,6 +24,9 @@ import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; import org.openstreetmap.gui.jmapviewer.interfaces.TileSource.TileUpdate; +import org.openstreetmap.josm.data.imagery.cache.CacheEntry; +import org.openstreetmap.josm.data.imagery.cache.CacheEntryAttributes; +import org.openstreetmap.josm.data.imagery.cache.TileCacheManager; /** * A {@link TileLoader} implementation that loads tiles from OSM via HTTP and @@ -57,9 +53,7 @@ // even if the refresh from the server fails. protected static final long ABSOLUTE_EXPIRE_TIME_LIMIT = Long.MAX_VALUE; // unlimited - protected String cacheDirBase; - - protected final Map sourceCacheDirMap; + protected ICacheAccess cache; public static File getDefaultCacheDir() throws SecurityException { @@ -70,7 +64,7 @@ } catch (SecurityException e) { log.log(Level.WARNING, "Failed to access system property ''java.io.tmpdir'' for security reasons. Exception was: " - + e.toString()); + + e.toString()); throw e; // rethrow } try { @@ -97,12 +91,7 @@ */ public OsmFileCacheTileLoader(TileLoaderListener map, File cacheDir) throws IOException { super(map); - if (cacheDir == null || (!cacheDir.exists() && !cacheDir.mkdirs())) - throw new IOException("Cannot access cache directory"); - - log.finest("Tile cache directory: " + cacheDir); - cacheDirBase = cacheDir.getAbsolutePath(); - sourceCacheDirMap = new HashMap<>(); + cache = TileCacheManager.getCache("TMS"); } /** @@ -119,25 +108,12 @@ return new FileLoadJob(tile); } - protected File getSourceCacheDir(TileSource source) { - File dir = sourceCacheDirMap.get(source); - if (dir == null) { - dir = new File(cacheDirBase, source.getName().replaceAll("[\\\\/:*?\"<>|]", "_")); - if (!dir.exists()) { - dir.mkdirs(); - } - } - return dir; - } - protected class FileLoadJob implements TileJob { InputStream input = null; Tile tile; - File tileCacheDir; - File tileFile = null; - File tagsFile = null; - Long fileMtime = null; + ICacheElement element; + CacheEntryAttributes elementAttributes; Long now = null; // current time in milliseconds (keep consistent value for the whole run) public FileLoadJob(Tile tile) { @@ -159,11 +135,9 @@ tile.loading = true; } now = System.currentTimeMillis(); - tileCacheDir = getSourceCacheDir(tile.getSource()); - tileFile = getTileFile(); - tagsFile = getTagsFile(); - loadTagsFromFile(); + element = cache.getCacheElement(tile.getKey()); + elementAttributes = getCacheEntryAttributes(element); if (isCacheValid() && (isNoTileAtZoom() || loadTileFromFile())) { log.log(Level.FINE, "TMS - found in tile cache: {0}", tile); @@ -204,15 +178,16 @@ protected boolean loadOrUpdateTile() { try { URLConnection urlConn = loadTileFromOsm(tile); - if (fileMtime != null && now - fileMtime <= ABSOLUTE_EXPIRE_TIME_LIMIT) { + + if (now - elementAttributes.getLastModification() <= ABSOLUTE_EXPIRE_TIME_LIMIT) { switch (tile.getSource().getTileUpdate()) { case IfModifiedSince: - urlConn.setIfModifiedSince(fileMtime); + urlConn.setIfModifiedSince(elementAttributes.getLastModification()); break; case LastModified: - if (!isOsmTileNewer(fileMtime)) { + if (!isOsmTileNewer(elementAttributes.getLastModification())) { log.log(Level.FINE, "TMS - LastModified test: local version is up to date: {0}", tile); - tileFile.setLastModified(now); + elementAttributes.setLastModification(now); return true; } break; @@ -221,7 +196,7 @@ } } if (tile.getSource().getTileUpdate() == TileUpdate.ETag || tile.getSource().getTileUpdate() == TileUpdate.IfNoneMatch) { - String fileETag = tile.getValue("etag"); + String fileETag = elementAttributes.getEtag(); if (fileETag != null) { switch (tile.getSource().getTileUpdate()) { case IfNoneMatch: @@ -230,14 +205,14 @@ case ETag: if (hasOsmTileETag(fileETag)) { log.log(Level.FINE, "TMS - ETag test: local version is up to date: {0}", tile); - tileFile.setLastModified(now); + elementAttributes.setLastModification(now); return true; } default: break; } } - tile.putValue("etag", urlConn.getHeaderField("ETag")); + elementAttributes.setEtag(urlConn.getHeaderField("ETag")); } if (urlConn instanceof HttpURLConnection && ((HttpURLConnection)urlConn).getResponseCode() == 304) { // If isModifiedSince or If-None-Match has been set @@ -253,17 +228,17 @@ break; } loadTileFromFile(); - tileFile.setLastModified(now); + elementAttributes.setLastModification(now); return true; } loadTileMetadata(tile, urlConn); - saveTagsToFile(); - if ("no-tile".equals(tile.getValue("tile-info"))) + if (elementAttributes.isNoTileAtZoom()) { log.log(Level.FINE, "TMS - No tile: tile-info=no-tile: {0}", tile); tile.setError("No tile at this zoom level"); + cache.put(tile.getKey(), new CacheEntry(new byte[]{}), elementAttributes); return true; } else { for (int i = 0; i < 5; ++i) { @@ -274,7 +249,7 @@ byte[] buffer = loadTileInBuffer(urlConn); if (buffer != null) { tile.loadImage(new ByteArrayInputStream(buffer)); - saveTileToFile(buffer); + cache.put(tile.getKey(), new CacheEntry(buffer), elementAttributes); log.log(Level.FINE, "TMS - downloaded tile from server: {0}", tile.getUrl()); return true; } @@ -294,31 +269,58 @@ return false; } + + private void loadTileMetadata(Tile tile, URLConnection urlConn) { + elementAttributes.setNoTileAtZoom("no-tile".equals(urlConn.getHeaderField("X-VE-Tile-Info"))); + + Long lng = urlConn.getExpiration(); + if (lng.equals(0L)) { + try { + String str = urlConn.getHeaderField("Cache-Control"); + if (str != null) { + for (String token: str.split(",")) { + if (token.startsWith("max-age=")) { + lng = Long.parseLong(token.substring(8)) * 1000 + + System.currentTimeMillis(); + } + } + } + } catch (NumberFormatException e) {} //ignore malformed Cache-Control headers + } + + elementAttributes.setExpirationTime(lng); + } + + private final CacheEntryAttributes getCacheEntryAttributes(ICacheElement e) { + if (e != null) { + IElementAttributes ea = e.getElementAttributes(); + return (CacheEntryAttributes) ea; + } else { + CacheEntryAttributes ret = new CacheEntryAttributes(); + ret.setLastModification(now); + ret.setExpirationTime(now + DEFAULT_EXPIRE_TIME); + return ret; + } + } + protected boolean isCacheValid() { - Long expires = null; - if (tileFile.exists()) { - fileMtime = tileFile.lastModified(); - } else if (tagsFile.exists()) { - fileMtime = tagsFile.lastModified(); - } else + if (element == null) return false; - try { - expires = Long.parseLong(tile.getValue("expires")); - } catch (NumberFormatException e) {} + long expires = elementAttributes.getExpirationTime(); // check by expire date set by server - if (expires != null && !expires.equals(0L)) { + if (expires != 0L) { // put a limit to the expire time (some servers send a value // that is too large) - expires = Math.min(expires, fileMtime + EXPIRE_TIME_SERVER_LIMIT); + expires = Math.min(expires, elementAttributes.getCreateTime() + EXPIRE_TIME_SERVER_LIMIT); if (now > expires) { - log.log(Level.FINE, "TMS - Tile has expired -> not valid {0}", tile); + log.log(Level.FINE, "TMS - Tile has expired ({0})-> not valid {1}", new Object[]{Long.toString(expires), tile}); return false; } } else { // check by file modification date - if (now - fileMtime > DEFAULT_EXPIRE_TIME) { + if (now - elementAttributes.getLastModification() > DEFAULT_EXPIRE_TIME) { log.log(Level.FINE, "TMS - Tile has expired, maximum file age reached {0}", tile); return false; } @@ -327,7 +329,7 @@ } protected boolean isNoTileAtZoom() { - if ("no-tile".equals(tile.getValue("tile-info"))) { + if (elementAttributes.isNoTileAtZoom()) { // do not remove file - keep the information, that there is no tile, for further requests // the code above will check, if this information is still valid log.log(Level.FINE, "TMS - Tile valid, but no file, as no tiles at this level {0}", tile); @@ -338,22 +340,13 @@ } protected boolean loadTileFromFile() { - if (!tileFile.exists()) - return false; - - try (FileInputStream fin = new FileInputStream(tileFile)) { + try (InputStream fin = new ByteArrayInputStream(element.getVal().getContent())) { if (fin.available() == 0) throw new IOException("File empty"); tile.loadImage(fin); return true; } catch (Exception e) { log.log(Level.WARNING, "TMS - Error while loading image from tile cache: {0}; {1}", new Object[]{e.getMessage(), tile}); - tileFile.delete(); - if (tagsFile.exists()) { - tagsFile.delete(); - } - tileFile = null; - fileMtime = null; } return false; } @@ -428,72 +421,9 @@ return true; return (osmETag.equals(eTag)); } - - protected File getTileFile() { - return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "." - + tile.getSource().getTileType()); - } - - protected File getTagsFile() { - return new File(tileCacheDir + "/" + tile.getZoom() + "_" + tile.getXtile() + "_" + tile.getYtile() + "." - + TAGS_FILE_EXT); - } - - protected void saveTileToFile(byte[] rawData) { - File file = getTileFile(); - file.getParentFile().mkdirs(); - try (FileOutputStream f = new FileOutputStream(file)) { - f.write(rawData); - } catch (Exception e) { - log.log(Level.SEVERE, "Failed to save tile content: {0}", e.getLocalizedMessage()); - } - } - - protected void saveTagsToFile() { - File tagsFile = getTagsFile(); - tagsFile.getParentFile().mkdirs(); - if (tile.getMetadata() == null) { - tagsFile.delete(); - return; - } - try (PrintWriter f = new PrintWriter(new OutputStreamWriter(new FileOutputStream(tagsFile), TAGS_CHARSET))) { - for (Entry entry : tile.getMetadata().entrySet()) { - f.println(entry.getKey() + "=" + entry.getValue()); - } - } catch (Exception e) { - System.err.println("Failed to save tile tags: " + e.getLocalizedMessage()); - } - } - - protected boolean loadTagsFromFile() { - File tagsFile = getTagsFile(); - try (BufferedReader f = new BufferedReader(new InputStreamReader(new FileInputStream(tagsFile), TAGS_CHARSET))) { - for (String line = f.readLine(); line != null; line = f.readLine()) { - final int i = line.indexOf('='); - if (i == -1 || i == 0) { - System.err.println("Malformed tile tag in file '" + tagsFile.getName() + "':" + line); - continue; - } - tile.putValue(line.substring(0,i),line.substring(i+1)); - } - } catch (FileNotFoundException e) { - } catch (Exception e) { - System.err.println("Failed to load tile tags: " + e.getLocalizedMessage()); - } - - return true; - } } - public String getCacheDirBase() { - return cacheDirBase; - } - public void setTileCacheDir(String tileCacheDir) { - File dir = new File(tileCacheDir); - dir.mkdirs(); - this.cacheDirBase = dir.getAbsolutePath(); - } @Override public void clearCache(TileSource source) { @@ -502,20 +432,7 @@ @Override public void clearCache(TileSource source, TileClearController controller) { - File dir = getSourceCacheDir(source); - if (dir != null) { - if (controller != null) controller.initClearDir(dir); - if (dir.isDirectory()) { - File[] files = dir.listFiles(); - if (controller != null) controller.initClearFiles(files); - for (File file : files) { - if (controller != null && controller.cancel()) return; - file.delete(); - if (controller != null) controller.fileDeleted(file); - } - } - dir.delete(); - } + cache.clear(); if (controller != null) controller.clearFinished(); } } Index: src/org/openstreetmap/gui/jmapviewer/TMSFileCacheTileLoader.java =================================================================== --- src/org/openstreetmap/gui/jmapviewer/TMSFileCacheTileLoader.java (revision 31054) +++ src/org/openstreetmap/gui/jmapviewer/TMSFileCacheTileLoader.java (working copy) @@ -3,9 +3,9 @@ import java.io.File; import java.io.IOException; + import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; -import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; /** * Reworked version of the OsmFileCacheTileLoader. @@ -30,47 +30,6 @@ super(tile); } - @Override - protected File getTileFile() { - return getDataFile(tile.getSource().getTileType()); - } - - @Override - protected File getTagsFile() { - return getDataFile(TAGS_FILE_EXT); - } - - protected File getDataFile(String ext) { - int nDigits = (int) Math.ceil(Math.log10(1 << tile.getZoom())); - String x = String.format("%0" + nDigits + "d", tile.getXtile()); - String y = String.format("%0" + nDigits + "d", tile.getYtile()); - File path = new File(tileCacheDir, "z" + tile.getZoom()); - for (int i=0; i|]", "_")); - } - if (!dir.exists()) { - dir.mkdirs(); - } - } - return dir; } - } Index: src/org/openstreetmap/josm/data/imagery/cache/CacheEntry.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/cache/CacheEntry.java (revision 0) +++ src/org/openstreetmap/josm/data/imagery/cache/CacheEntry.java (revision 0) @@ -0,0 +1,17 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.cache; + +import java.io.Serializable; + +public class CacheEntry implements Serializable { + private static final long serialVersionUID = 1L; //version + private byte[] content; + + public CacheEntry(byte[] content) { + this.content = content; + } + + public byte[] getContent() { + return content; + } +} Index: src/org/openstreetmap/josm/data/imagery/cache/CacheEntryAttributes.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/cache/CacheEntryAttributes.java (revision 0) +++ src/org/openstreetmap/josm/data/imagery/cache/CacheEntryAttributes.java (revision 0) @@ -0,0 +1,38 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.cache; + +import org.apache.commons.jcs.engine.ElementAttributes; + +public class CacheEntryAttributes extends ElementAttributes { + private boolean noTileAtZoom = false; + private String Etag = null; + private long lastModification = 0; + private long expirationTime = 0; + + + public boolean isNoTileAtZoom() { + return noTileAtZoom; + } + public void setNoTileAtZoom(boolean noTileAtZoom) { + this.noTileAtZoom = noTileAtZoom; + } + public String getEtag() { + return Etag; + } + public void setEtag(String etag) { + Etag = etag; + } + public long getLastModification() { + return lastModification; + } + public void setLastModification(long lastModification) { + this.lastModification = lastModification; + } + public long getExpirationTime() { + return expirationTime; + } + public void setExpirationTime(long expirationTime) { + this.expirationTime = expirationTime; + } + +} Index: src/org/openstreetmap/josm/data/imagery/cache/TileCacheManager.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/cache/TileCacheManager.java (revision 0) +++ src/org/openstreetmap/josm/data/imagery/cache/TileCacheManager.java (revision 0) @@ -0,0 +1,70 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.cache; + +import java.io.File; +import java.io.IOException; +import java.util.Properties; + +import org.apache.commons.jcs.access.CacheAccess; +import org.apache.commons.jcs.access.behavior.ICacheAccess; +import org.apache.commons.jcs.engine.control.CompositeCache; +import org.apache.commons.jcs.engine.control.CompositeCacheManager; +import org.openstreetmap.josm.Main; + +public abstract class TileCacheManager { + + private static volatile CompositeCacheManager cacheManager = null; + private static long maxObjectTTL = 1000L * 60 * 60 * 24 * 31; // 31 days + + // average tile size is about 20kb + // 1000 is around 20MB under this assumptions + private static long maxObjectsInMemory = 1000; + private static long maxObjectsOnDisk = 100000; + + private static void initialize() throws IOException { + File cacheDir = new File(Main.pref.getCacheDirectory(), "jcs"); + + if ((!cacheDir.exists() && !cacheDir.mkdirs())) + throw new IOException("Cannot access cache directory"); + + CompositeCacheManager cm = CompositeCacheManager.getUnconfiguredInstance(); + Properties props = new Properties(); + props.setProperty("jcs.default", "DC"); + props.setProperty("jcs.default.cacheattributes", org.apache.commons.jcs.engine.CompositeCacheAttributes.class.getCanonicalName()); + props.setProperty("jcs.default.cacheattributes.MaxObjects", Long.toString(maxObjectsInMemory)); + props.setProperty("jcs.default.cacheattributes.UseMemoryShrinker", "true"); + props.setProperty("jcs.default.cacheattributes.DiskUsagePatternName", "UPDATE"); // store elements on disk on put + props.setProperty("jcs.default.elementattributes", CacheEntryAttributes.class.getCanonicalName()); + props.setProperty("jcs.default.elementattributes.IsEternal", "false"); + props.setProperty("jcs.default.elementattributes.MaxLife", Long.toString(maxObjectTTL)); + props.setProperty("jcs.default.elementattributes.IdleTime", Long.toString(maxObjectTTL)); + props.setProperty("jcs.default.elementattributes.IsSpool", "true"); + + props.setProperty("jcs.auxiliary.DC", org.apache.commons.jcs.auxiliary.disk.block.BlockDiskCacheFactory.class.getCanonicalName()); + props.setProperty("jcs.auxiliary.DC.attributes", org.apache.commons.jcs.auxiliary.disk.block.BlockDiskCacheAttributes.class.getCanonicalName()); + props.setProperty("jcs.auxiliary.DC.attributes.DiskPath", cacheDir.getAbsolutePath()); + props.setProperty("jcs.auxiliary.DC.attributes.maxKeySize", Long.toString(maxObjectsOnDisk)); + props.setProperty("jcs.auxiliary.DC.attributes.blockSizeBytes", "1024"); + + + cm.configure(props); + cacheManager = cm; + } + + public static ICacheAccess getCache(String cacheName) throws IOException { + if (cacheManager != null) + return getCacheInner(cacheName); + + synchronized (TileCacheManager.class) { + if (cacheManager == null) + initialize(); + return getCacheInner(cacheName); + } + } + + private static ICacheAccess getCacheInner(String cacheName) { + CompositeCache cc = cacheManager.getCache(cacheName); + return new CacheAccess(cc); + } + +}