Index: resources/images/dialogs/add_mvt.svg =================================================================== --- resources/images/dialogs/add_mvt.svg (nonexistent) +++ resources/images/dialogs/add_mvt.svg (working copy) @@ -0,0 +1,148 @@ + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + V + + Index: src/org/openstreetmap/josm/data/imagery/ImageryInfo.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/ImageryInfo.java (revision 17488) +++ src/org/openstreetmap/josm/data/imagery/ImageryInfo.java (working copy) @@ -61,7 +61,9 @@ /** A WMS endpoint entry only stores the WMS server info, without layer, which are chosen later by the user. **/ WMS_ENDPOINT("wms_endpoint"), /** WMTS stores GetCapabilities URL. Does not store any information about the layer **/ - WMTS("wmts"); + WMTS("wmts"), + /** MapBox Vector Tiles entry*/ + MVT("mvt"); private final String typeString; Index: src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java (revision 17488) +++ src/org/openstreetmap/josm/data/imagery/TMSCachedTileLoaderJob.java (working copy) @@ -32,6 +32,7 @@ import org.openstreetmap.josm.data.cache.CacheEntryAttributes; import org.openstreetmap.josm.data.cache.ICachedLoaderListener; import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob; +import org.openstreetmap.josm.data.imagery.vectortile.VectorTile; import org.openstreetmap.josm.data.preferences.LongProperty; import org.openstreetmap.josm.tools.HttpClient; import org.openstreetmap.josm.tools.Logging; @@ -295,7 +296,8 @@ if (content.length > 0) { try (ByteArrayInputStream in = new ByteArrayInputStream(content)) { tile.loadImage(in); - if (tile.getImage() == null) { + if ((!(tile instanceof VectorTile) && tile.getImage() == null) + || ((tile instanceof VectorTile) && !tile.isLoaded())) { String s = new String(content, StandardCharsets.UTF_8); Matcher m = SERVICE_EXCEPTION_PATTERN.matcher(s); if (m.matches()) { Index: src/org/openstreetmap/josm/data/imagery/vectortile/VectorTile.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/VectorTile.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/VectorTile.java (working copy) @@ -0,0 +1,41 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile; + +import java.awt.Graphics; +import java.awt.image.ImageObserver; + +/** + * An interface that is used to draw vector tiles, instead of using images + * @author Taylor Smock + * @since xxx + */ +public interface VectorTile { + /** + * Paints the vector tile on the {@link Graphics} g at the + * position x/y. + * + * @param g the Graphics object + * @param x x-coordinate in g + * @param y y-coordinate in g + */ + void paint(Graphics g, int x, int y); + + /** + * Paints the vector tile on the {@link Graphics} g at the + * position x/y. + * @param g the Graphics object + * @param x x-coordinate in g + * @param y y-coordinate in g + * @param width width that tile should have + * @param height height that tile should have + * @param observer The paint observer. May be {@code null}. + * @param zoom The current zoom level + */ + void paint(Graphics g, int x, int y, int width, int height, int zoom, ImageObserver observer); + + /** + * Get the extent of the tile (in pixels) + * @return The tile extent (pixels) + */ + int getExtent(); +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Command.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Command.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Command.java (working copy) @@ -0,0 +1,48 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +/** + * Command integers for Mapbox Vector Tiles + * @author Taylor Smock + * @since xxx + */ +public enum Command { + /** + * For {@link GeometryTypes#POINT}, each {@link #MoveTo} is a new point. + * For {@link GeometryTypes#LINESTRING} and {@link GeometryTypes#POLYGON}, each {@link #MoveTo} is a new geometry of the same type. + */ + MoveTo((byte) 1, (byte) 2), + /** + * While not explicitly prohibited for {@link GeometryTypes#POINT}, it should be ignored. + * For {@link GeometryTypes#LINESTRING} and {@link GeometryTypes#POLYGON}, each {@link #LineTo} extends that geometry. + */ + LineTo((byte) 2, (byte) 2), + /** + * This is only explicitly valid for {@link GeometryTypes#POLYGON}. It closes the {@link GeometryTypes#POLYGON}. + */ + ClosePath((byte) 7, (byte) 0); + + private final byte id; + private final byte parameters; + + Command(byte id, byte parameters) { + this.id = id; + this.parameters = parameters; + } + + /** + * Get the command id + * @return The id + */ + public byte getId() { + return this.id; + } + + /** + * Get the number of parameters + * @return The number of parameters + */ + public byte getParameterNumber() { + return this.parameters; + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/CommandInteger.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/CommandInteger.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/CommandInteger.java (working copy) @@ -0,0 +1,62 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +import java.util.stream.Stream; + +/** + * An indicator for a command to be executed + * @author Taylor Smock + * @since xxx + */ +public class CommandInteger { + private final Command type; + private final short[] parameters; + private int added; + + /** + * Create a new command + * @param command the command (treated as an unsigned int) + */ + public CommandInteger(final int command) { + // Technically, the int is unsigned, but it is easier to work with the long + final long unsigned = Integer.toUnsignedLong(command); + this.type = Stream.of(Command.values()).filter(e -> e.getId() == (unsigned & 0x7)).findAny() + .orElseThrow(InvalidMapboxVectorTileException::new); + // This is safe, since we are shifting right 3 when we converted an int to a long (for unsigned). + // So we cannot lose anything. + final int operationsInt = (int) (unsigned >> 3); + this.parameters = new short[operationsInt * this.type.getParameterNumber()]; + } + + /** + * Add a parameter + * @param parameterInteger The parameter to add (converted to {@link short}). + */ + public void addParameter(Number parameterInteger) { + this.parameters[added++] = parameterInteger.shortValue(); + } + + /** + * Get the operations for the command + * @return The operations + */ + public short[] getOperations() { + return this.parameters; + } + + /** + * Get the command type + * @return the command type + */ + public Command getType() { + return this.type; + } + + /** + * Get the expected parameter length + * @return The expected parameter size + */ + public boolean hasAllExpectedParameters() { + return this.added >= this.parameters.length; + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Feature.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Feature.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Feature.java (working copy) @@ -0,0 +1,150 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +import java.io.IOException; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.List; + +import org.openstreetmap.josm.data.osm.TagMap; +import org.openstreetmap.josm.data.protobuf.ProtoBufPacked; +import org.openstreetmap.josm.data.protobuf.ProtoBufParser; +import org.openstreetmap.josm.data.protobuf.ProtoBufRecord; +import org.openstreetmap.josm.tools.Utils; + +/** + * A Feature for a {@link Layer} + * @author Taylor Smock + * @since xxx + */ +public class Feature { + private static final byte ID_FIELD = 1; + private static final byte TAG_FIELD = 2; + private static final byte GEOMETRY_TYPE_FIELD = 3; + private static final byte GEOMETRY_FIELD = 4; + /** The geometry of the feature. Required. */ + private final List geometry = new ArrayList<>(); + + /** The geometry type of the feature. Required. */ + private final GeometryTypes geometryType; + + /** The tags of the feature. Optional. */ + private TagMap tags; + + /** The id of the feature. Optional. */ + // Technically, uint64 + private final long id; + + /** + * Create a new Feature + * @param layer The layer the feature is part of (required for tags) + * @param record The record to create the feature from + * @throws IOException - if an IO error occurs + */ + public Feature(Layer layer, ProtoBufRecord record) throws IOException { + long tId = 0; + GeometryTypes geometryTypeTemp = GeometryTypes.UNKNOWN; + String key = null; + try (ProtoBufParser parser = new ProtoBufParser(record.getBytes())) { + while (parser.hasNext()) { + try (ProtoBufRecord next = new ProtoBufRecord(parser)) { + if (next.getField() == TAG_FIELD) { + if (tags == null) { + tags = new TagMap(); + } + // This is packed in v1 and v2 + ProtoBufPacked packed = new ProtoBufPacked(next.getBytes()); + for (Number number : packed.getArray()) { + key = parseTagValue(key, layer, number); + } + } else if (next.getField() == GEOMETRY_FIELD) { + // This is packed in v1 and v2 + ProtoBufPacked packed = new ProtoBufPacked(next.getBytes()); + CommandInteger currentCommand = null; + for (Number number : packed.getArray()) { + if (currentCommand != null && currentCommand.hasAllExpectedParameters()) { + currentCommand = null; + } + if (currentCommand == null) { + currentCommand = new CommandInteger(number.intValue()); + this.geometry.add(currentCommand); + } else { + currentCommand.addParameter(ParameterInteger.decode(number.intValue())); + } + } + // TODO fallback to non-packed + } else if (next.getField() == GEOMETRY_TYPE_FIELD) { + geometryTypeTemp = GeometryTypes.values()[next.asUnsignedVarInt().intValue()]; + } else if (next.getField() == ID_FIELD) { + tId = next.asUnsignedVarInt().longValue(); + } + } + } + } + this.id = tId; + this.geometryType = geometryTypeTemp; + record.close(); + } + + /** + * Parse a tag value + * @param key The current key (or {@code null}, if {@code null}, the returned value will be the new key) + * @param layer The layer with key/value information + * @param number The number to get the value from + * @return The new key (if {@code null}, then a value was parsed and added to tags) + */ + private String parseTagValue(String key, Layer layer, Number number) { + if (key == null) { + key = layer.getKey(number.intValue()); + } else { + Object value = layer.getValue(number.intValue()); + if (value instanceof Double || value instanceof Float) { + // reset grouping if the instance is a singleton + final NumberFormat numberFormat = NumberFormat.getNumberInstance(); + final boolean grouping = numberFormat.isGroupingUsed(); + try { + numberFormat.setGroupingUsed(false); + this.tags.put(key, numberFormat.format(value)); + } finally { + numberFormat.setGroupingUsed(grouping); + } + } else { + this.tags.put(key, Utils.intern(value.toString())); + } + key = null; + } + return key; + } + + /** + * Get the geometry instructions + * @return The geometry + */ + public List getGeometry() { + return this.geometry; + } + + /** + * Get the geometry type + * @return The {@link GeometryTypes} + */ + public GeometryTypes getGeometryType() { + return this.geometryType; + } + + /** + * Get the id of the object + * @return The unique id in the layer, or 0. + */ + public long getId() { + return this.id; + } + + /** + * Get the tags + * @return A tag map + */ + public TagMap getTags() { + return this.tags; + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Geometry.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Geometry.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Geometry.java (working copy) @@ -0,0 +1,99 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.awt.Shape; +import java.awt.geom.Area; +import java.awt.geom.Ellipse2D; +import java.awt.geom.Path2D; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; + +/** + * A class to generate geometry for a vector tile + * @author Taylor Smock + * @since xxx + */ +public class Geometry { + private static final byte CIRCLE_SIZE = 1; + final Collection shapes = new HashSet<>(); + private final Feature feature; + + /** + * Create a {@link Geometry} for a {@link Feature} + * @param feature the {@link Feature} for the geometry + */ + public Geometry(final Feature feature) { + this.feature = feature; + final GeometryTypes geometryType = this.feature.getGeometryType(); + final List commands = this.feature.getGeometry(); + final byte circleSize = CIRCLE_SIZE; + if (geometryType == GeometryTypes.POINT) { + for (CommandInteger command : commands) { + final short[] operations = command.getOperations(); + // Each MoveTo command is a new point + if (command.getType() == Command.MoveTo && operations.length % 2 == 0) { + for (int i = 0; i < operations.length / 2; i++) { + // move left/up by 1/2 circleSize, so that the circle is centered + shapes.add(new Ellipse2D.Float(operations[2 * i] - circleSize / 2f, + operations[2 * i + 1] - circleSize / 2f, circleSize, circleSize)); + } + } else { + throw new IllegalArgumentException(tr("{0} with {1} arguments is not understood", geometryType, operations.length)); + } + } + } else if (geometryType == GeometryTypes.LINESTRING || geometryType == GeometryTypes.POLYGON) { + Path2D.Float line = null; + for (CommandInteger command : commands) { + final short[] operations = command.getOperations(); + // Technically, there is no reason why there can be multiple MoveTo operations in one command, but that is undefined behavior + if (command.getType() == Command.MoveTo && operations.length == 2) { + final double x; + final double y; + if (line != null) { + x = line.getCurrentPoint().getX() + operations[0]; + y = line.getCurrentPoint().getY() + operations[1]; + } else { + x = operations[0]; + y = operations[1]; + } + line = new Path2D.Float(); + line.moveTo(x, y); + shapes.add(line); + } else if (command.getType() == Command.LineTo && operations.length % 2 == 0 && line != null) { + for (int i = 0; i < operations.length / 2; i++) { + final double x = line.getCurrentPoint().getX() + operations[2 * i]; + final double y = line.getCurrentPoint().getY() + operations[2 * i + 1]; + line.lineTo(x, y); + } + // ClosePath should only be used with Polygon geometry + } else if (geometryType == GeometryTypes.POLYGON && command.getType() == Command.ClosePath && line != null) { + line.closePath(); + shapes.remove(line); + shapes.add(new Area(line)); + } else { + throw new IllegalArgumentException(tr("{0} with {1} arguments is not understood", geometryType, operations.length)); + } + } + } + } + + /** + * Get the feature for this geometry + * @return The feature + */ + public Feature getFeature() { + return this.feature; + } + + /** + * Get the shapes to draw this geometry with + * @return A collection of shapes + */ + public Collection getShapes() { + return Collections.unmodifiableCollection(this.shapes); + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/GeometryTypes.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/GeometryTypes.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/GeometryTypes.java (working copy) @@ -0,0 +1,44 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +/** + * Geometry types used by Mapbox Vector Tiles + * @author Taylor Smock + * @since xxx + */ +public enum GeometryTypes { + /** May be ignored */ + UNKNOWN((byte) 0), + /** May be a point or a multipoint geometry. Uses only {@link Command#MoveTo}. Multiple {@link Command#MoveTo} + * indicates that it is a multi-point object. */ + POINT((byte) 1), + /** May be a line or a multiline geometry. Each line {@link Command#MoveTo} and one or more {@link Command#LineTo}. */ + LINESTRING((byte) 2), + /** May be a polygon or a multipolygon. Each ring uses a {@link Command#MoveTo}, one or more {@link Command#LineTo}, + * and one {@link Command#ClosePath} command. See {@link Ring}s. */ + POLYGON((byte) 3); + + private final byte id; + GeometryTypes(byte id) { + this.id = id; + } + + /** + * Get the id for the geometry type + * @return The id + */ + public byte getId() { + return this.id; + } + + /** + * Rings used by {@link GeometryTypes#POLYGON} + * @author Taylor Smock + */ + public enum Ring { + /** A ring that goes in the clockwise direction */ + ExteriorRing, + /** A ring that goes in the anti-clockwise direction */ + InteriorRing + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/InvalidMapboxVectorTileException.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/InvalidMapboxVectorTileException.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/InvalidMapboxVectorTileException.java (working copy) @@ -0,0 +1,25 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +/** + * Thrown when a mapbox vector tile does not match specifications. + * + * @author Taylor Smock + * @since xxx + */ +public class InvalidMapboxVectorTileException extends RuntimeException { + /** + * Create a default {@link InvalidMapboxVectorTileException}. + */ + public InvalidMapboxVectorTileException() { + super(); + } + + /** + * Create a new {@link InvalidMapboxVectorTile} exception with a message + * @param message The message + */ + public InvalidMapboxVectorTileException(final String message) { + super(message); + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Layer.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Layer.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Layer.java (working copy) @@ -0,0 +1,216 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.openstreetmap.josm.data.protobuf.ProtoBufParser; +import org.openstreetmap.josm.data.protobuf.ProtoBufRecord; +import org.openstreetmap.josm.tools.Logging; + +/** + * A Mapbox Vector Tile Layer + * @author Taylor Smock + * @since xxx + */ +public class Layer { + private static final class ValueFields { + static final ValueFields STRING = new ValueFields<>(1, ProtoBufRecord::asString); + static final ValueFields FLOAT = new ValueFields<>(2, ProtoBufRecord::asFloat); + static final ValueFields DOUBLE = new ValueFields<>(3, ProtoBufRecord::asDouble); + static final ValueFields INT64 = new ValueFields<>(4, ProtoBufRecord::asUnsignedVarInt); + // This may have issues if there are actual uint_values (i.e., more than {@link Long#MAX_VALUE}) + static final ValueFields UINT64 = new ValueFields<>(5, ProtoBufRecord::asUnsignedVarInt); + static final ValueFields SINT64 = new ValueFields<>(6, ProtoBufRecord::asSignedVarInt); + static final ValueFields BOOL = new ValueFields<>(7, r -> r.asUnsignedVarInt().longValue() != 0); + + public static final Collection> MAPPERS = Arrays.asList(STRING, FLOAT, DOUBLE, INT64, UINT64, SINT64, BOOL); + + private final byte field; + private final Function conversion; + private ValueFields(int field, Function conversion) { + this.field = (byte) field; + this.conversion = conversion; + } + + /** + * Get the field identifier for the value + * @return The identifier + */ + public byte getField() { + return this.field; + } + + /** + * Convert a protobuf record to a value + * @param protobufRecord The record to convert + * @return the converted value + */ + public T convertValue(ProtoBufRecord protobufRecord) { + return this.conversion.apply(protobufRecord); + } + } + + /** The field value for a layer (in {@link ProtoBufRecord#getField}) */ + public static final byte LAYER_FIELD = 3; + private static final byte VERSION_FIELD = 15; + private static final byte NAME_FIELD = 1; + private static final byte FEATURE_FIELD = 2; + private static final byte KEY_FIELD = 3; + private static final byte VALUE_FIELD = 4; + private static final byte EXTENT_FIELD = 5; + /** The default extent for a vector tile */ + static final int DEFAULT_EXTENT = 4096; + private static final byte DEFAULT_VERSION = 1; + /** This is technically an integer, but there are currently only two major versions (1, 2). Required. */ + private final byte version; + /** A unique name for the layer. This must be unique on a per-tile basis. Required. */ + private final String name; + + /** The extent of the tile, typically 4096. Required. */ + private final int extent; + + /** A list of unique keys. Order is important. Optional. */ + private final List keyList = new ArrayList<>(); + /** A list of unique values. Order is important. Optional. */ + private final List valueList = new ArrayList<>(); + /** The actual features of this layer in this tile */ + private final List featureCollection; + /** The shapes to use to draw this layer */ + private final List geometryCollection; + + /** + * Create a layer from a collection of records + * @param records The records to convert to a layer + * @throws IOException - if an IO error occurs + */ + public Layer(Collection records) throws IOException { + // Do the unique required fields first + Map> sorted = records.stream().collect(Collectors.groupingBy(ProtoBufRecord::getField)); + this.version = sorted.get((int) VERSION_FIELD).parallelStream().map(ProtoBufRecord::asUnsignedVarInt).map(Number::byteValue) + .findFirst().orElse(DEFAULT_VERSION); + // Per spec, we cannot continue past this until we have checked the version number + if (this.version != 1 && this.version != 2) { + throw new IllegalArgumentException(tr("We do not understand version {0} of the vector tile specification", this.version)); + } + this.name = sorted.getOrDefault((int) NAME_FIELD, Collections.emptyList()).parallelStream().map(ProtoBufRecord::asString).findFirst() + .orElseThrow(() -> new IllegalArgumentException(tr("Vector tile layers must have a layer name"))); + this.extent = sorted.getOrDefault((int) EXTENT_FIELD, Collections.emptyList()).parallelStream().map(ProtoBufRecord::asSignedVarInt) + .map(Number::intValue).findAny().orElse(DEFAULT_EXTENT); + + sorted.getOrDefault((int) KEY_FIELD, Collections.emptyList()).parallelStream().map(ProtoBufRecord::asString) + .forEachOrdered(this.keyList::add); + sorted.getOrDefault((int) VALUE_FIELD, Collections.emptyList()).parallelStream().map(ProtoBufRecord::getBytes) + .map(ProtoBufParser::new).map(parser1 -> { + try { + return new ProtoBufRecord(parser1); + } catch (IOException e) { + return null; + } + }) + .filter(Objects::nonNull) + .map(value -> ValueFields.MAPPERS.parallelStream() + .filter(v -> v.getField() == value.getField()) + .map(v -> v.convertValue(value)).findFirst() + .orElseThrow(() -> new IllegalArgumentException(tr("Unknown field in vector tile layer value ({0})", value.getField())))) + .forEachOrdered(this.valueList::add); + Collection exceptions = new HashSet<>(0); + this.featureCollection = sorted.getOrDefault((int) FEATURE_FIELD, Collections.emptyList()).parallelStream().map(feature -> { + try { + return new Feature(this, feature); + } catch (IOException e) { + exceptions.add(e); + } + return null; + }).collect(Collectors.toList()); + this.geometryCollection = this.featureCollection.stream().map(Geometry::new).collect(Collectors.toList()); + if (!exceptions.isEmpty()) { + throw exceptions.iterator().next(); + } + // Cleanup bytes (for memory) + for (ProtoBufRecord record : records) { + try { + record.close(); + } catch (Exception e) { + Logging.error(e); + } + } + } + + /** + * Create a new layer + * @param bytes The bytes that the layer comes from + * @throws IOException - if an IO error occurs + */ + public Layer(byte[] bytes) throws IOException { + this(new ProtoBufParser(bytes).allRecords()); + } + + /** + * Get the extent of the tile + * @return The layer extent + */ + public int getExtent() { + return this.extent; + } + + /** + * Get the feature on this layer + * @return the features + */ + public Collection getFeatures() { + return Collections.unmodifiableCollection(this.featureCollection); + } + + /** + * Get the geometry for this layer + * @return The geometry + */ + public Collection getGeometry() { + return Collections.unmodifiableCollection(this.geometryCollection); + } + + /** + * Get a specified key + * @param index The index in the key list + * @return The actual key + */ + public String getKey(int index) { + return this.keyList.get(index); + } + + /** + * Get the name of the layer + * @return The layer name + */ + public String getName() { + return this.name; + } + + /** + * Get a specified value + * @param index The index in the value list + * @return The actual value. This can be a {@link String}, {@link Boolean}, {@link Integer}, or {@link Float} value. + */ + public Object getValue(int index) { + return this.valueList.get(index); + } + + /** + * Get the MapBox Vector Tile version specification for this layer + * @return The version of the MapBox Vector Tile specification + */ + public byte getVersion() { + return this.version; + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MVTFile.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MVTFile.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MVTFile.java (working copy) @@ -0,0 +1,33 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +import java.util.Arrays; +import java.util.List; + +/** + * Items that MAY be used to figure out if a file or server response MAY BE a Mapbox Vector Tile + * @author Taylor Smock + * @since xxx + */ +public final class MVTFile { + /** + * Extensions for Mapbox Vector Tiles. + * This is a SHOULD, not a MUST. + */ + public static final List EXTENSION = Arrays.asList("mvt"); + + /** + * mimetypes for Mapbox Vector Tiles + * This is a SHOULD, not a MUST. + */ + public static final List MIMETYPE = Arrays.asList("application/vnd.mapbox-vector-tile"); + + /** + * The default projection. This is Web Mercator, per specification. + */ + public static final String DEFAULT_PROJECTION = "EPSG:3857"; + + private MVTFile() { + // Hide the constructor + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MVTTile.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MVTTile.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MVTTile.java (working copy) @@ -0,0 +1,133 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +import java.awt.Color; +import java.awt.Graphics; +import java.awt.Graphics2D; +import java.awt.Shape; +import java.awt.Stroke; +import java.awt.geom.AffineTransform; +import java.awt.geom.Area; +import java.awt.geom.Ellipse2D; +import java.awt.geom.Path2D; +import java.awt.image.BufferedImage; +import java.awt.image.ImageObserver; +import java.io.IOException; +import java.io.InputStream; +import java.util.Collection; +import java.util.HashSet; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.openstreetmap.gui.jmapviewer.Tile; +import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; +import org.openstreetmap.josm.data.imagery.vectortile.VectorTile; +import org.openstreetmap.josm.data.protobuf.ProtoBufParser; +import org.openstreetmap.josm.data.protobuf.ProtoBufRecord; + +/** + * A class for MapBox Vector Tiles + * @author Taylor Smock + * @since xxx + */ +public class MVTTile extends Tile implements VectorTile { + private Collection geometry; + private int extent = Layer.DEFAULT_EXTENT; + + public MVTTile(TileSource source, int xtile, int ytile, int zoom) { + super(source, xtile, ytile, zoom); + } + + @Override + public void paint(final Graphics g, final int x, final int y) { + this.paint(g, x, y, 256, 256); + } + + @Override + public void paint(Graphics g, int x, int y, int width, int height, int zoom, ImageObserver observer) { + if (!(g instanceof Graphics2D) || this.geometry == null) { + if (getImage() != null) { + g.drawImage(image, x, y, width, height, observer); + } + return; + } + final Graphics2D graphics = (Graphics2D) g; + graphics.setColor(Color.GREEN); + final AffineTransform originalTransform = graphics.getTransform(); + final Stroke originalStroke = graphics.getStroke(); + try { + graphics.translate(x, y); + // TODO figure out HiDPI (maybe GuiSizesHelper?) + // 131072 seems to be the magic number for my screens. This needs to be investigated more. + final double scale = width / (double) 131072; + // The scaleTransform is separate to avoid wide lines at high zoom (e.g., when vector tiles go to z14, but + // we are currently at z20, the graphics.scale function makes everything big. Unfortunately, this creates + // a new shape object. + final Function scaleShape; + if (scale > 1) { + final AffineTransform scaleTransform = AffineTransform.getScaleInstance(scale, scale); + scaleShape = scaleTransform::createTransformedShape; + } else { + scaleShape = Function.identity(); + graphics.scale(scale, scale); + } + final Color transparentYellow = new Color(Color.YELLOW.getRed(), Color.YELLOW.getGreen(), Color.YELLOW.getBlue(), 120); + this.geometry.forEach(shapes -> { + for (Shape shape : shapes.getShapes()) { + final Shape scaledShape = scaleShape.apply(shape); + if (shape instanceof Ellipse2D) { + graphics.setColor(Color.GREEN); + } else if (shape instanceof Path2D) { + graphics.setColor(Color.RED); + } else if (shape instanceof Area) { + graphics.setColor(transparentYellow); + graphics.fill(scaledShape); + graphics.setColor(Color.YELLOW); + } + graphics.draw(scaledShape); + } + }); + } finally { + graphics.setTransform(originalTransform); + graphics.setStroke(originalStroke); + } + graphics.setColor(Color.RED); + graphics.drawString("0, 0", 1024, 1024); + } + + @Override + public void loadImage(final InputStream inputStream) throws IOException { + if (this.image == null || this.image == Tile.LOADING_IMAGE || this.image == Tile.ERROR_IMAGE) { + this.initLoading(); + ProtoBufParser parser = new ProtoBufParser(inputStream); + Collection protoBufRecords = parser.allRecords(); + Collection layers = new HashSet<>(); + for (ProtoBufRecord record : protoBufRecords) { + if (record.getField() == Layer.LAYER_FIELD) { + Layer mvtLayer = new Layer(new ProtoBufParser(record.getBytes()).allRecords()); + layers.add(mvtLayer); + // Cleanup bytes + record.close(); + } + } + // TODO Store layers separately + this.geometry = layers.stream().flatMap(layer -> layer.getGeometry().stream()).collect(Collectors.toList()); + this.extent = layers.stream().map(Layer::getExtent).max(Integer::compare).orElse(Layer.DEFAULT_EXTENT); + BufferedImage bufferedImage = new BufferedImage(this.extent, this.extent, BufferedImage.TYPE_4BYTE_ABGR); + Graphics2D graphics = bufferedImage.createGraphics(); + this.paint(graphics, 0, 0); + + // TODO figure out a better way to free memory + final int maxSize = 256; + final BufferedImage resized = new BufferedImage(maxSize, maxSize, bufferedImage.getType()); + resized.getGraphics().drawImage(bufferedImage, 0, 0, resized.getWidth(), resized.getHeight(), null); + this.image = resized; + this.finishLoading(); + } + } + + @Override + public int getExtent() { + return this.extent; + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MapBoxVectorCachedTileLoader.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MapBoxVectorCachedTileLoader.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MapBoxVectorCachedTileLoader.java (working copy) @@ -0,0 +1,78 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.commons.jcs3.access.behavior.ICacheAccess; +import org.openstreetmap.gui.jmapviewer.Tile; +import org.openstreetmap.gui.jmapviewer.interfaces.CachedTileLoader; +import org.openstreetmap.gui.jmapviewer.interfaces.TileJob; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; +import org.openstreetmap.gui.jmapviewer.interfaces.TileSource; +import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry; +import org.openstreetmap.josm.data.cache.JCSCachedTileLoaderJob; +import org.openstreetmap.josm.data.imagery.TMSCachedTileLoader; +import org.openstreetmap.josm.data.imagery.TileJobOptions; +import org.openstreetmap.josm.data.preferences.IntegerProperty; +import org.openstreetmap.josm.tools.CheckParameterUtil; + +/** + * A TileLoader class for MVT tiles + * @author Taylor Smock + * @since xxx + */ +public class MapBoxVectorCachedTileLoader implements TileLoader, CachedTileLoader { + protected final ICacheAccess cache; + protected final TileLoaderListener listener; + protected final TileJobOptions options; + private static final IntegerProperty THREAD_LIMIT = + new IntegerProperty("imagery.vector.mvtloader.maxjobs", TMSCachedTileLoader.THREAD_LIMIT.getDefaultValue()); + private static final ThreadPoolExecutor DEFAULT_DOWNLOAD_JOB_DISPATCHER = + TMSCachedTileLoader.getNewThreadPoolExecutor("MVT-downloader-%d", THREAD_LIMIT.get()); + + /** + * Constructor + * @param listener called when tile loading has finished + * @param cache of the cache + * @param options tile job options + */ + public MapBoxVectorCachedTileLoader(TileLoaderListener listener, ICacheAccess cache, + TileJobOptions options) { + CheckParameterUtil.ensureParameterNotNull(cache, "cache"); + this.cache = cache; + this.options = options; + this.listener = listener; + } + + @Override + public void clearCache(TileSource source) { + this.cache.remove(source.getName() + ':'); + } + + @Override + public TileJob createTileLoaderJob(Tile tile) { + return new MapBoxVectorCachedTileLoaderJob( + listener, + tile, + cache, + options, + getDownloadExecutor()); + } + + @Override + public void cancelOutstandingTasks() { + final ThreadPoolExecutor executor = getDownloadExecutor(); + executor.getQueue().stream().filter(executor::remove).filter(MapBoxVectorCachedTileLoaderJob.class::isInstance) + .map(MapBoxVectorCachedTileLoaderJob.class::cast).forEach(JCSCachedTileLoaderJob::handleJobCancellation); + } + + @Override + public boolean hasOutstandingTasks() { + return getDownloadExecutor().getTaskCount() > getDownloadExecutor().getCompletedTaskCount(); + } + + private ThreadPoolExecutor getDownloadExecutor() { + return DEFAULT_DOWNLOAD_JOB_DISPATCHER; + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MapBoxVectorCachedTileLoaderJob.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MapBoxVectorCachedTileLoaderJob.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MapBoxVectorCachedTileLoaderJob.java (working copy) @@ -0,0 +1,25 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.commons.jcs3.access.behavior.ICacheAccess; +import org.openstreetmap.gui.jmapviewer.Tile; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoaderListener; +import org.openstreetmap.josm.data.cache.BufferedImageCacheEntry; +import org.openstreetmap.josm.data.imagery.TMSCachedTileLoaderJob; +import org.openstreetmap.josm.data.imagery.TileJobOptions; + +/** + * Bridge to JCS cache for MVT tiles + * @author Taylor Smock + * @since xxx + */ +public class MapBoxVectorCachedTileLoaderJob extends TMSCachedTileLoaderJob { + + public MapBoxVectorCachedTileLoaderJob(TileLoaderListener listener, Tile tile, + ICacheAccess cache, TileJobOptions options, + ThreadPoolExecutor downloadExecutor) { + super(listener, tile, cache, options, downloadExecutor); + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MapboxVectorTileSource.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MapboxVectorTileSource.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MapboxVectorTileSource.java (working copy) @@ -0,0 +1,17 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +import org.openstreetmap.josm.data.imagery.ImageryInfo; +import org.openstreetmap.josm.data.imagery.JosmTemplatedTMSTileSource; +import org.openstreetmap.josm.data.projection.Projection; + +/** + * Tile Source handling for Mapbox Vector Tile sources + * @author Taylor Smock + * @since xxx + */ +public class MapboxVectorTileSource extends JosmTemplatedTMSTileSource { + public MapboxVectorTileSource(ImageryInfo info) { + super(info); + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/ParameterInteger.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/ParameterInteger.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/ParameterInteger.java (working copy) @@ -0,0 +1,22 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +/** + * The parameters that follow the {@link CommandInteger}. + * @author Taylor Smock + * @since xxx + */ +public final class ParameterInteger { + private ParameterInteger() { + // Hide constructor + } + + /** + * Get the value for this ParameterInteger + * @param value The zig-zag and delta encoded value to decode + * @return The decoded integer value + */ + public static int decode(int value) { + return ((value >> 1) ^ -(value & 1)); + } +} Index: src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java (working copy) @@ -0,0 +1,65 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import java.util.ArrayList; +import java.util.List; + +/** + * Parse packed values (only numerical values) + * @author Taylor Smock + * @since xxx + */ +public class ProtoBufPacked { + private final byte[] bytes; + private int location; + private final Number[] numbers; + /** + * Create a new ProtoBufPacked object + * @param bytes The packed bytes + */ + public ProtoBufPacked(byte[] bytes) { + this.location = 0; + this.bytes = bytes; + List numbersT = new ArrayList<>(); + while (this.location < bytes.length) { + numbersT.add(ProtoBufParser.convertByteArray(this.nextVarInt(), ProtoBufParser.VAR_INT_BYTE_SIZE)); + } + + this.numbers = new Number[numbersT.size()]; + for (int i = 0; i < numbersT.size(); i++) { + this.numbers[i] = numbersT.get(i); + } + } + + /** + * The number of expected values + * @return The expected values + */ + public int size() { + return this.numbers.length; + } + + /** + * Get the parsed number array + * @return The number array + */ + public Number[] getArray() { + return this.numbers; + } + + private byte[] nextVarInt() { + List byteList = new ArrayList<>(); + while ((this.bytes[this.location] & ProtoBufParser.MOST_SIGNIFICANT_BYTE) == ProtoBufParser.MOST_SIGNIFICANT_BYTE) { + // Get rid of the leading bit (shift left 1, then shift right 1 unsigned) + byteList.add((byte) (this.bytes[this.location++] ^ ProtoBufParser.MOST_SIGNIFICANT_BYTE)); + } + // The last byte doesn't drop the most significant bit + byteList.add(this.bytes[this.location++]); + byte[] byteArray = new byte[byteList.size()]; + for (int i = 0; i < byteList.size(); i++) { + byteArray[i] = byteList.get(i); + } + + return byteArray; + } +} Index: src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java (working copy) @@ -0,0 +1,202 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.openstreetmap.josm.tools.Logging; + +/** + * A basic Protobuf parser + * @author Taylor Smock + * @since xxx + */ +public class ProtoBufParser implements AutoCloseable { + /** + * Used to get the most significant byte + */ + static final byte MOST_SIGNIFICANT_BYTE = (byte) (1 << 7); + /** The default byte size (see {@link #VAR_INT_BYTE_SIZE} for var ints) */ + public static final byte BYTE_SIZE = 8; + /** The byte size for var ints (since the first byte is just an indicator for if the var int is done) */ + public static final byte VAR_INT_BYTE_SIZE = BYTE_SIZE - 1; + // TODO switch to a better parser + private final InputStream inputStream; + /** + * Create a new parser + * @param bytes The bytes to parse + */ + public ProtoBufParser(byte[] bytes) { + this(new ByteArrayInputStream(bytes)); + } + + /** + * Create a new parser + * @param inputStream The InputStream (will be fully read at this time) + */ + public ProtoBufParser(InputStream inputStream) { + if (inputStream.markSupported()) { + this.inputStream = inputStream; + } else { + this.inputStream = new BufferedInputStream(inputStream); + } + } + + @Override + public void close() { + try { + this.inputStream.close(); + } catch (IOException e) { + Logging.error(e); + } + } + + /** + * Get the "next" WireType + * @return {@link WireType} expected + * @throws IOException - if an IO error occurs + */ + public WireType next() throws IOException { + this.inputStream.mark(16); + try { + return WireType.values()[this.inputStream.read() << 3]; + } finally { + this.inputStream.reset(); + } + } + + /** + * Get the next byte + * @return The next byte + * @throws IOException - if an IO error occurs + */ + public int nextByte() throws IOException { + return this.inputStream.read(); + } + + /** + * Check if there is more data to read + * @return {@code true} if there is more data to read + * @throws IOException - if an IO error occurs + */ + public boolean hasNext() throws IOException { + return this.inputStream.available() > 0; + } + + /** + * Get the next var int ({@code WireType#VARINT}) + * @return The next var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum}) + * @throws IOException - if an IO error occurs + */ + public byte[] nextVarInt() throws IOException { + List byteList = new ArrayList<>(); + int currentByte = this.nextByte(); + while ((byte) (currentByte & MOST_SIGNIFICANT_BYTE) == MOST_SIGNIFICANT_BYTE) { + // Get rid of the leading bit (shift left 1, then shift right 1 unsigned) + byteList.add((byte) (currentByte ^ MOST_SIGNIFICANT_BYTE)); + currentByte = this.nextByte(); + } + // The last byte doesn't drop the most significant bit + byteList.add((byte) currentByte); + byte[] byteArray = new byte[byteList.size()]; + for (int i = 0; i < byteList.size(); i++) { + byteArray[i] = byteList.get(i); + } + + return byteArray; + } + + /** + * Get the next 32 bits ({@link WireType#THIRTY_TWO_BIT}) + * @return a byte array of the next 32 bits (4 bytes) + * @throws IOException - if an IO error occurs + */ + public byte[] nextFixed32() throws IOException { + // 4 bytes == 32 bits + return readNextBytes(4); + } + + /** + * Get the next 64 bits ({@link WireType#SIXTY_FOUR_BIT}) + * @return a byte array of the next 64 bits (8 bytes) + * @throws IOException - if an IO error occurs + */ + public byte[] nextFixed64() throws IOException { + // 8 bytes == 64 bits + return readNextBytes(8); + } + + /** + * Read an arbitrary number of bytes + * @param size The number of bytes to read + * @return a byte array of the specified size, filled with bytes read (unsigned) + * @throws IOException - if an IO error occurs + */ + private byte[] readNextBytes(int size) throws IOException { + byte[] bytesRead = new byte[size]; + for (int i = 0; i < bytesRead.length; i++) { + bytesRead[i] = (byte) this.nextByte(); + } + return bytesRead; + } + + /** + * Get the next delimited message ({@link WireType#LENGTH_DELIMITED}) + * @return The next length delimited message + * @throws IOException - if an IO error occurs + */ + public byte[] nextLengthDelimited() throws IOException { + int length = convertByteArray(this.nextVarInt(), VAR_INT_BYTE_SIZE).intValue(); + return readNextBytes(length); + } + + /** + * Convert a byte array to a number (little endian) + * @param bytes The bytes to convert + * @param byteSize The size of the byte. For var ints, this is 7, for other ints, this is 8. + * @return An appropriate {@link Number} class. + */ + public static Number convertByteArray(byte[] bytes, byte byteSize) { + long number = 0; + for (int i = 0; i < bytes.length; i++) { + // Need to convert to uint64 in order to avoid bit operation from filling in 1's and overflow issues + number += Byte.toUnsignedLong(bytes[i]) << (byteSize * i); + } + return convertLong(number); + } + + /** + * Convert a long to an appropriate {@link Number} class + * @param number The long to convert + * @return A {@link Number} + */ + public static Number convertLong(long number) { + // TODO deal with booleans + if (number <= Byte.MAX_VALUE && number >= Byte.MIN_VALUE) { + return (byte) number; + } else if (number <= Short.MAX_VALUE && number >= Short.MIN_VALUE) { + return (short) number; + } else if (number <= Integer.MAX_VALUE && number >= Integer.MIN_VALUE) { + return (int) number; + } + return number; + } + + /** + * Read all records + * @return A collection of all records + * @throws IOException - if an IO error occurs + */ + public Collection allRecords() throws IOException { + Collection records = new ArrayList<>(); + while (this.hasNext()) { + records.add(new ProtoBufRecord(this)); + } + return records; + } +} Index: src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java (working copy) @@ -0,0 +1,142 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; + +import org.openstreetmap.josm.tools.Utils; + +/** + * A protobuf record, storing the {@link WireType}, the parsed field number, and the bytes for it. + * @author Taylor Smock + * @since xxx + */ +public class ProtoBufRecord implements AutoCloseable { + private static final byte[] EMPTY_BYTES = {}; + private final WireType type; + private final int field; + private byte[] bytes; + + /** + * Create a new Protobuf record + * @param parser The parser to use to create the record + * @throws IOException - if an IO error occurs + */ + public ProtoBufRecord(ProtoBufParser parser) throws IOException { + Number number = ProtoBufParser.convertByteArray(parser.nextVarInt(), ProtoBufParser.VAR_INT_BYTE_SIZE); + // I don't foresee having field numbers > {@code Integer#MAX_VALUE >> 3} + this.field = (int) number.longValue() >> 3; + // 7 is 111 (so last three bits) + byte wireType = (byte) (number.longValue() & 7); + this.type = Stream.of(WireType.values()).filter(wType -> wType.getTypeRepresentation() == wireType).findFirst().orElse(WireType.UNKNOWN); + + if (this.type == WireType.VARINT) { + this.bytes = parser.nextVarInt(); + } else if (this.type == WireType.SIXTY_FOUR_BIT) { + this.bytes = parser.nextFixed64(); + } else if (this.type == WireType.THIRTY_TWO_BIT) { + this.bytes = parser.nextFixed32(); + } else if (this.type == WireType.LENGTH_DELIMITED) { + this.bytes = parser.nextLengthDelimited(); + } else { + this.bytes = EMPTY_BYTES; + } + } + + /** + * Get the field value + * @return The field value + */ + public int getField() { + return this.field; + } + + /** + * Get the WireType of the data + * @return The {@link WireType} of the data + */ + public WireType getType() { + return this.type; + } + + /** + * Get the raw bytes for this record + * @return The bytes + */ + public byte[] getBytes() { + return this.bytes; + } + + /** + * Get the var int ({@code WireType#VARINT}) + * @return The var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum}) + */ + public Number asUnsignedVarInt() { + return ProtoBufParser.convertByteArray(this.bytes, ProtoBufParser.VAR_INT_BYTE_SIZE); + } + + /** + * Get the signed var int ({@code WireType#VARINT}). + * These are specially encoded so that they take up less space. + * + * @return The signed var int ({@code sint32} or {@code sint64}) + */ + public Number asSignedVarInt() { + final Number signed = this.asUnsignedVarInt(); + final long value = signed.longValue(); + final long number = (value << 1) ^ (value >> 31); + return ProtoBufParser.convertLong(number); + } + + /** + * Get as a double ({@link WireType#SIXTY_FOUR_BIT}) + * @return the double + */ + public double asDouble() { + long doubleNumber = ProtoBufParser.convertByteArray(asFixed64(), ProtoBufParser.BYTE_SIZE).longValue(); + return Double.longBitsToDouble(doubleNumber); + } + + /** + * Get as a float ({@link WireType#THIRTY_TWO_BIT}) + * @return the float + */ + public float asFloat() { + int floatNumber = ProtoBufParser.convertByteArray(asFixed32(), ProtoBufParser.BYTE_SIZE).intValue(); + return Float.intBitsToFloat(floatNumber); + } + + /** + * Get as a string ({@link WireType#LENGTH_DELIMITED}) + * @return The string (encoded as {@link StandardCharsets#UTF_8}) + */ + public String asString() { + return Utils.intern(new String(this.bytes, StandardCharsets.UTF_8)); + } + + /** + * Get as 32 bits ({@link WireType#THIRTY_TWO_BIT}) + * @return a byte array of the 32 bits (4 bytes) + */ + public byte[] asFixed32() { + // TODO verify, or just assume? + // 4 bytes == 32 bits + return this.bytes; + } + + /** + * Get as 64 bits ({@link WireType#SIXTY_FOUR_BIT}) + * @return a byte array of the 64 bits (8 bytes) + */ + public byte[] asFixed64() { + // TODO verify, or just assume? + // 8 bytes == 64 bits + return this.bytes; + } + + @Override + public void close() { + this.bytes = null; + } +} Index: src/org/openstreetmap/josm/data/protobuf/WireType.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/WireType.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/WireType.java (working copy) @@ -0,0 +1,46 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +/** + * The WireTypes + * @author Taylor Smock + * @since xxx + */ +public enum WireType { + /** int32, int64, uint32, uint64, sing32, sint64, bool, enum */ + VARINT(0), + /** fixed64, sfixed64, double */ + SIXTY_FOUR_BIT(1), + /** string, bytes, embedded messages, packed repeated fields */ + LENGTH_DELIMITED(2), + /** + * start groups + * @deprecated Unknown reason. Deprecated since at least 2012. + */ + @Deprecated + START_GROUP(3), + /** + * end groups + * @deprecated Unknown reason. Deprecated since at least 2012. + */ + @Deprecated + END_GROUP(4), + /** fixed32, sfixed32, float */ + THIRTY_TWO_BIT(5), + + /** For unknown WireTypes */ + UNKNOWN(Byte.MAX_VALUE); + + private final byte type; + WireType(int value) { + this.type = (byte) value; + } + + /** + * Get the type representation (byte form) + * @return The wire type byte representation + */ + public byte getTypeRepresentation() { + return this.type; + } +} Index: src/org/openstreetmap/josm/gui/layer/AbstractTileSourceLayer.java =================================================================== --- src/org/openstreetmap/josm/gui/layer/AbstractTileSourceLayer.java (revision 17488) +++ src/org/openstreetmap/josm/gui/layer/AbstractTileSourceLayer.java (working copy) @@ -87,6 +87,7 @@ import org.openstreetmap.josm.data.imagery.OffsetBookmark; import org.openstreetmap.josm.data.imagery.TMSCachedTileLoader; import org.openstreetmap.josm.data.imagery.TileLoaderFactory; +import org.openstreetmap.josm.data.imagery.vectortile.VectorTile; import org.openstreetmap.josm.data.osm.visitor.BoundingXYVisitor; import org.openstreetmap.josm.data.preferences.BooleanProperty; import org.openstreetmap.josm.data.preferences.IntegerProperty; @@ -868,7 +869,7 @@ if (coordinateConverter.requiresReprojection()) { tile = new ReprojectionTile(tileSource, x, y, zoom); } else { - tile = new Tile(tileSource, x, y, zoom); + tile = createTile(tileSource, x, y, zoom); } tileCache.addTile(tile); } @@ -1007,6 +1008,33 @@ } } + /** + * Draw a vector tile on screen. + * @param g the Graphics2D + * @param tile the vector tile + * @param anchorImage tile anchor in image coordinates + * @param anchorScreen tile anchor in screen coordinates + * @param clip clipping region in screen coordinates (can be null) + */ + private void drawVectorTileInside(Graphics2D g, VectorTile tile, TileAnchor anchorImage, TileAnchor anchorScreen, Shape clip) { + AffineTransform imageToScreen = anchorImage.convert(anchorScreen); + Point2D screen0 = imageToScreen.transform(new Point2D.Double(0, 0), null); + Point2D screen1 = imageToScreen.transform(new Point2D.Double( + tile.getExtent(), tile.getExtent()), null); + + Shape oldClip = null; + if (clip != null) { + oldClip = g.getClip(); + g.clip(clip); + } + tile.paint(g, (int) Math.round(screen0.getX()), (int) Math.round(screen0.getY()), + (int) Math.round(screen1.getX()) - (int) Math.round(screen0.getX()), + (int) Math.round(screen1.getY()) - (int) Math.round(screen0.getY()), this.currentZoomLevel, this); + if (clip != null) { + g.setClip(oldClip); + } + } + private List paintTileImages(Graphics2D g, TileSet ts) { Object paintMutex = new Object(); List missed = Collections.synchronizedList(new ArrayList<>()); @@ -1021,7 +1049,7 @@ img = getLoadedTileImage(tile); anchorImage = getAnchor(tile, img); } - if (img == null || anchorImage == null) { + if (img == null || anchorImage == null || (tile instanceof VectorTile && !tile.isLoaded())) { miss = true; } } @@ -1030,12 +1058,18 @@ return; } - img = applyImageProcessors(img); + if (img != null) { + img = applyImageProcessors(img); + } TileAnchor anchorScreen = coordinateConverter.getScreenAnchorForTile(tile); synchronized (paintMutex) { //cannot paint in parallel - drawImageInside(g, img, anchorImage, anchorScreen, null); + if (tile instanceof VectorTile) { + drawVectorTileInside(g, (VectorTile) tile, anchorImage, anchorScreen, null); + } else { + drawImageInside(g, img, anchorImage, anchorScreen, null); + } } MapView mapView = MainApplication.getMap().mapView; if (tile instanceof ReprojectionTile && ((ReprojectionTile) tile).needsUpdate(mapView.getScale())) { @@ -1830,7 +1864,7 @@ for (int x = minX; x <= maxX; x++) { for (int y = minY; y <= maxY; y++) { - requestedTiles.add(new Tile(tileSource, x, y, currentZoomLevel)); + requestedTiles.add(createTile(tileSource, x, y, currentZoomLevel)); } } } @@ -1929,6 +1963,20 @@ return SaveActionBase.createAndOpenSaveFileChooser(tr("Save WMS file"), WMSLayerImporter.FILE_FILTER); } + /** + * Create a new tile. Added to allow use of custom {@link Tile} objects. + * + * @param source Tile source + * @param x X coordinate + * @param y Y coordinate + * @param zoom Zoom level + * @return The new {@link Tile} + * @since xxx + */ + public Tile createTile(T source, int x, int y, int zoom) { + return new Tile(source, x, y, zoom); + } + @Override public synchronized void destroy() { super.destroy(); Index: src/org/openstreetmap/josm/gui/layer/ImageryLayer.java =================================================================== --- src/org/openstreetmap/josm/gui/layer/ImageryLayer.java (revision 17488) +++ src/org/openstreetmap/josm/gui/layer/ImageryLayer.java (working copy) @@ -37,6 +37,7 @@ import org.openstreetmap.josm.gui.MapView; import org.openstreetmap.josm.gui.MenuScroller; import org.openstreetmap.josm.gui.layer.imagery.ImageryFilterSettings; +import org.openstreetmap.josm.gui.layer.imagery.MVTLayer; import org.openstreetmap.josm.gui.widgets.UrlLabel; import org.openstreetmap.josm.tools.GBC; import org.openstreetmap.josm.tools.ImageProcessor; @@ -168,6 +169,8 @@ case BING: case SCANEX: return new TMSLayer(info); + case MVT: + return new MVTLayer(info); default: throw new AssertionError(tr("Unsupported imagery type: {0}", info.getImageryType())); } Index: src/org/openstreetmap/josm/gui/layer/imagery/MVTLayer.java =================================================================== --- src/org/openstreetmap/josm/gui/layer/imagery/MVTLayer.java (nonexistent) +++ src/org/openstreetmap/josm/gui/layer/imagery/MVTLayer.java (working copy) @@ -0,0 +1,58 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.gui.layer.imagery; + +import java.util.Collection; +import java.util.Collections; + +import org.openstreetmap.gui.jmapviewer.Tile; +import org.openstreetmap.gui.jmapviewer.interfaces.TileLoader; +import org.openstreetmap.josm.data.imagery.ImageryInfo; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MVTFile; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MVTTile; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MapBoxVectorCachedTileLoader; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MapboxVectorTileSource; +import org.openstreetmap.josm.data.projection.Projections; +import org.openstreetmap.josm.gui.layer.AbstractCachedTileSourceLayer; + +/** + * A layer for MapBox Vector Tiles + * @author Taylor Smock + * @since xxx + */ +public class MVTLayer extends AbstractCachedTileSourceLayer { + private static final String CACHE_REGION_NAME = "MVT"; + + public MVTLayer(ImageryInfo info) { + super(info); + } + + @Override + protected Class getTileLoaderClass() { + return MapBoxVectorCachedTileLoader.class; + } + + @Override + protected String getCacheName() { + return CACHE_REGION_NAME; + } + + @Override + public Collection getNativeProjections() { + // MapBox Vector Tiles specifically only support EPSG:3857 + // ("it is exclusively geared towards square pixel tiles in {link to EPSG:3857}"). + return Collections.singleton(MVTFile.DEFAULT_PROJECTION); + } + + @Override + protected MapboxVectorTileSource getTileSource() { + MapboxVectorTileSource source = new MapboxVectorTileSource(this.info); + this.info.setAttribution(source); + return source; + } + + @Override + public Tile createTile(MapboxVectorTileSource source, int x, int y, int zoom) { + return new MVTTile(source, x, y, zoom); + } + +} Index: src/org/openstreetmap/josm/gui/preferences/imagery/AddMVTLayerPanel.java =================================================================== --- src/org/openstreetmap/josm/gui/preferences/imagery/AddMVTLayerPanel.java (nonexistent) +++ src/org/openstreetmap/josm/gui/preferences/imagery/AddMVTLayerPanel.java (working copy) @@ -0,0 +1,92 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.gui.preferences.imagery; +import org.openstreetmap.josm.data.imagery.ImageryInfo; +import org.openstreetmap.josm.data.imagery.ImageryInfo.ImageryType; +import org.openstreetmap.josm.gui.widgets.JosmTextArea; +import org.openstreetmap.josm.gui.widgets.JosmTextField; +import org.openstreetmap.josm.tools.GBC; +import org.openstreetmap.josm.tools.Utils; + +import javax.swing.JLabel; +import java.awt.event.KeyAdapter; +import java.awt.event.KeyEvent; +import java.util.Arrays; + +import static org.openstreetmap.josm.tools.I18n.tr; + +/** + * A panel for adding MapBox Vector Tile layers + * @author Taylor Smock + * @since xxx + */ +public class AddMVTLayerPanel extends AddImageryPanel { + private final JosmTextField mvtZoom = new JosmTextField(); + private final JosmTextArea mvtUrl = new JosmTextArea(3, 40).transferFocusOnTab(); + + /** + * Constructs a new {@code AddMVTLayerPanel}. + */ + public AddMVTLayerPanel() { + + add(new JLabel(tr("{0} Make sure OSM has the permission to use this service", "1.")), GBC.eol()); + add(new JLabel(tr("{0} Enter URL", "2.")), GBC.eol()); + add(new JLabel("" + Utils.joinAsHtmlUnorderedList(Arrays.asList( + tr("{0} is replaced by tile zoom level, also supported:
" + + "offsets to the zoom level: {1} or {2}
" + + "reversed zoom level: {3}", "{zoom}", "{zoom+1}", "{zoom-1}", "{19-zoom}"), + tr("{0} is replaced by X-coordinate of the tile", "{x}"), + tr("{0} is replaced by Y-coordinate of the tile", "{y}"), + tr("{0} is replaced by a random selection from the given comma separated list, e.g. {1}", "{switch:...}", "{switch:a,b,c}") + )) + ""), GBC.eol().fill()); + + final KeyAdapter keyAdapter = new KeyAdapter() { + @Override + public void keyReleased(KeyEvent e) { + mvtUrl.setText(buildMvtUrl()); + } + }; + + add(rawUrl, GBC.eop().fill()); + rawUrl.setLineWrap(true); + rawUrl.addKeyListener(keyAdapter); + + add(new JLabel(tr("{0} Enter maximum zoom (optional)", "3.")), GBC.eol()); + mvtZoom.addKeyListener(keyAdapter); + add(mvtZoom, GBC.eop().fill()); + + add(new JLabel(tr("{0} Edit generated {1} URL (optional)", "4.", "MVT")), GBC.eol()); + add(mvtUrl, GBC.eop().fill()); + mvtUrl.setLineWrap(true); + + add(new JLabel(tr("{0} Enter name for this layer", "5.")), GBC.eol()); + add(name, GBC.eop().fill()); + + registerValidableComponent(mvtUrl); + } + + private String buildMvtUrl() { + StringBuilder a = new StringBuilder("mvt"); + String z = sanitize(mvtZoom.getText()); + if (!z.isEmpty()) { + a.append('[').append(z).append(']'); + } + a.append(':').append(sanitize(getImageryRawUrl(), ImageryType.MVT)); + return a.toString(); + } + + @Override + public ImageryInfo getImageryInfo() { + ImageryInfo generated = new ImageryInfo(getImageryName(), getMvtUrl()); + generated.setImageryType(ImageryType.MVT); + return generated; + } + + protected final String getMvtUrl() { + return sanitize(mvtUrl.getText()); + } + + @Override + protected boolean isImageryValid() { + return !getImageryName().isEmpty() && !getMvtUrl().isEmpty(); + } +} Index: src/org/openstreetmap/josm/gui/preferences/imagery/ImageryProvidersPanel.java =================================================================== --- src/org/openstreetmap/josm/gui/preferences/imagery/ImageryProvidersPanel.java (revision 17488) +++ src/org/openstreetmap/josm/gui/preferences/imagery/ImageryProvidersPanel.java (working copy) @@ -312,6 +312,7 @@ activeToolbar.add(new NewEntryAction(ImageryInfo.ImageryType.WMS)); activeToolbar.add(new NewEntryAction(ImageryInfo.ImageryType.TMS)); activeToolbar.add(new NewEntryAction(ImageryInfo.ImageryType.WMTS)); + activeToolbar.add(new NewEntryAction(ImageryInfo.ImageryType.MVT)); activeToolbar.add(remove); activePanel.add(activeToolbar, BorderLayout.EAST); add(activePanel, GBC.eol().fill(GridBagConstraints.BOTH).weight(2.0, 0.4).insets(5, 0, 0, 5)); @@ -440,6 +441,9 @@ case WMTS: icon = /* ICON(dialogs/) */ "add_wmts"; break; + case MVT: + icon = /* ICON(dialogs/) */ "add_mvt"; + break; default: break; } @@ -460,6 +464,9 @@ case WMTS: p = new AddWMTSLayerPanel(); break; + case MVT: + p = new AddMVTLayerPanel(); + break; default: throw new IllegalStateException("Type " + type + " not supported"); } Index: test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java =================================================================== --- test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java (nonexistent) +++ test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java (working copy) @@ -0,0 +1,74 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.openstreetmap.josm.TestUtils; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.Feature; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.Layer; +import org.openstreetmap.josm.io.Compression; + +/** + * Test class for {@link ProtoBufParser} and {@link ProtoBufRecord} + * @author Taylor Smock + * @since xxx + */ +class ProtoBufTest { + /** + * Test simple message. + * Check that a simple message is readable + * @throws IOException - if an IO error occurs + */ + @Test + void testSimpleMessage() throws IOException { + ProtoBufParser parser = new ProtoBufParser(new byte[] {(byte) 0x08, (byte) 0x96, (byte) 0x01}); + ProtoBufRecord record = new ProtoBufRecord(parser); + assertEquals(WireType.VARINT, record.getType()); + assertEquals(150, record.asUnsignedVarInt().intValue()); + } + + /** + * Test reading tile from Mapillary ( 14/3251/6258 ) + * @throws IOException if there is a problem reading the file + */ + @Test + void testRead_14_3251_6258() throws IOException { + File vectorTile = Paths.get(TestUtils.getTestDataRoot(), "pbf", "6258.mvt").toFile(); + InputStream inputStream = Compression.getUncompressedFileInputStream(vectorTile); + Collection records = new ProtoBufParser(inputStream).allRecords(); + assertEquals(2, records.size()); + List layers = new ArrayList<>(); + for (ProtoBufRecord record : records) { + if (record.getField() == Layer.LAYER_FIELD) { + layers.add(new Layer(record.getBytes())); + } else { + fail(MessageFormat.format("Invalid field {0}", record.getField())); + } + } + Layer mapillarySequences = layers.get(0); + Layer mapillaryPictures = layers.get(1); + assertEquals("mapillary-sequences", mapillarySequences.getName()); + assertEquals("mapillary-images", mapillaryPictures.getName()); + assertEquals(8192, mapillarySequences.getExtent()); + assertEquals(8192, mapillaryPictures.getExtent()); + + assertEquals(1, mapillarySequences.getFeatures().stream().filter(feature -> feature.getId() == 241083111).count()); + Feature testSequence = mapillarySequences.getFeatures().stream().filter(feature -> feature.getId() == 241083111).findAny().orElse(null); + assertEquals("jgxkXqVFM4jepMG3vP5Q9A", testSequence.getTags().get("key")); + assertEquals("C15Ul6qVMfQFlzRcmQCLcA", testSequence.getTags().get("ikey")); + assertEquals("x0hTY8cakpy0m3ui1GaG1A", testSequence.getTags().get("userkey")); + assertEquals(Long.valueOf(1565196718638L), Long.valueOf(testSequence.getTags().get("captured_at"))); + assertEquals(0, Integer.parseInt(testSequence.getTags().get("pano"))); + } +}