Index: test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java =================================================================== --- test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java (nonexistent) +++ test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java (working copy) @@ -0,0 +1,72 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.openstreetmap.josm.TestUtils; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.Feature; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.Layer; +import org.openstreetmap.josm.io.Compression; + +/** + * Test class for {@link ProtoBufParser} and {@link ProtoBufRecord} + * @author Taylor Smock + * @since xxx + */ +class ProtoBufTest { + /** + * Test simple message. + * Check that a simple message is readable + */ + @Test + void testSimpleMessage() { + ProtoBufParser parser = new ProtoBufParser(new byte[] {(byte) 0x08, (byte) 0x96, (byte) 0x01}); + ProtoBufRecord record = new ProtoBufRecord(parser); + assertEquals(WireType.VARINT, record.getType()); + assertEquals(150, record.asUnsignedVarInt().intValue()); + } + /** + * Test reading tile from Mapillary ( 14/3251/6258 ) + * @throws IOException if there is a problem reading the file + */ + @Test + void testRead_14_3251_6258() throws IOException { + File vectorTile = Paths.get(TestUtils.getTestDataRoot(), "pbf", "6258.mvt").toFile(); + InputStream inputStream = Compression.getUncompressedFileInputStream(vectorTile); + Collection records = new ProtoBufParser(inputStream).allRecords(); + assertEquals(2, records.size()); + List layers = new ArrayList<>(); + for (ProtoBufRecord record : records) { + if (record.getField() == Layer.LAYER_FIELD) { + layers.add(new Layer(record.getBytes())); + } else { + fail(MessageFormat.format("Invalid field {0}", record.getField())); + } + } + Layer mapillarySequences = layers.get(0); + Layer mapillaryPictures = layers.get(1); + assertEquals("mapillary-sequences", mapillarySequences.getName()); + assertEquals("mapillary-images", mapillaryPictures.getName()); + assertEquals(8192, mapillarySequences.getExtent()); + assertEquals(8192, mapillaryPictures.getExtent()); + + assertEquals(1, mapillarySequences.getFeatures().stream().filter(feature -> feature.getId() == 241083111).count()); + Feature testSequence = mapillarySequences.getFeatures().stream().filter(feature -> feature.getId() == 241083111).findAny().orElse(null); + assertEquals("jgxkXqVFM4jepMG3vP5Q9A", testSequence.getTags().get("key")); + assertEquals("C15Ul6qVMfQFlzRcmQCLcA", testSequence.getTags().get("ikey")); + assertEquals("x0hTY8cakpy0m3ui1GaG1A", testSequence.getTags().get("userkey")); + assertEquals(Long.valueOf(1565196718638L), Long.valueOf(testSequence.getTags().get("captured_at"))); + assertEquals(0, Integer.parseInt(testSequence.getTags().get("pano"))); + } +} Index: src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java (working copy) @@ -0,0 +1,74 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import java.util.ArrayList; +import java.util.List; + +/** + * Parse packed values (only numerical values) + * @author Taylor Smock + * @since xxx + */ +public class ProtoBufPacked { + private final byte[] bytes; + private int location; + private final int size; + private final Number[] numbers; + /** + * Create a new ProtoBufPacked object + * @param bytes The packed bytes + */ + public ProtoBufPacked(byte[] bytes) { + this.bytes = bytes; + this.size = ProtoBufParser.convertByteArray(this.nextVarInt()).intValue(); + List numbersT = new ArrayList<>(); + while (this.location < bytes.length) { + numbersT.add(ProtoBufParser.convertByteArray(this.nextVarInt())); + } + + this.numbers = new Number[numbersT.size()]; + for (int i = 0; i < numbersT.size(); i++) { + this.numbers[i] = numbersT.get(i); + } + } + + /** + * The number of expected values + * @return The expected values + */ + public int size() { + return this.size; + } + + /** + * Get the parsed number array + * @return The number array + */ + public Number[] getArray() { + return this.numbers; + } + + /** + * Check if this is actually packed + * @return {@code true} if the size equals the array size + */ + public boolean isPacked() { + return this.numbers.length == this.size; + } + + private byte[] nextVarInt() { + List byteList = new ArrayList<>(); + while ((this.bytes[this.location] & ProtoBufParser.MOST_SIGNIFICANT_BYTE) == ProtoBufParser.MOST_SIGNIFICANT_BYTE) { + // Get rid of the leading bit (shift left 1, then shift right 1 unsigned) + byteList.add((byte) (this.bytes[this.location++] ^ ProtoBufParser.MOST_SIGNIFICANT_BYTE)); + } + // The last byte doesn't drop the most significant bit + byteList.add(this.bytes[this.location++]); + byte[] byteArray = new byte[byteList.size()]; + for (int i = 0; i < byteList.size(); i++) { + byteArray[i] = byteList.get(i); + } + + return byteArray; + } +} Index: src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java (working copy) @@ -0,0 +1,171 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.apache.commons.io.IOUtils; + +/** + * A basic Protobuf parser + * @author Taylor Smock + * @since xxx + */ +public class ProtoBufParser { + /** + * Used to get the most significant byte + */ + static final byte MOST_SIGNIFICANT_BYTE = (byte) (1 << 7); + // TODO switch to a better parser + private final byte[] bytes; + private int location; + /** + * Create a new parser + * @param bytes The bytes to parse + */ + public ProtoBufParser(byte[] bytes) { + this.bytes = bytes; + this.location = 0; + } + + /** + * Create a new parser + * @param inputStream The inputstream (will be fully read at this time) + * @throws IOException If an exception occurs in {@link IOUtils#toByteArray(InputStream)}. + */ + public ProtoBufParser(InputStream inputStream) throws IOException { + this(IOUtils.toByteArray(inputStream)); + } + + /** + * Get the "next" wiretype + * @return {@link WireType} expected + */ + public WireType next() { + // TODO is this right? + return WireType.values()[this.bytes[this.location] << 3]; + } + + /** + * Get the next byte + * @return The next byte + */ + public byte nextByte() { + return this.bytes[this.location++]; + } + + /** + * Check if there is more data to read + * @return {@code true} if there is more data to read + */ + public boolean hasNext() { + return this.bytes.length > this.location; + } + + /** + * Get the next var int ({@code WireType#VARINT}) + * @return The next var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum}) + */ + public byte[] nextVarInt() { + List byteList = new ArrayList<>(); + while ((this.bytes[this.location] & MOST_SIGNIFICANT_BYTE) == MOST_SIGNIFICANT_BYTE) { + // Get rid of the leading bit (shift left 1, then shift right 1 unsigned) + byteList.add((byte) (this.nextByte() ^ MOST_SIGNIFICANT_BYTE)); + } + // The last byte doesn't drop the most significant bit + byteList.add(this.nextByte()); + byte[] byteArray = new byte[byteList.size()]; + for (int i = 0; i < byteList.size(); i++) { + byteArray[i] = byteList.get(i); + } + + return byteArray; + } + + /** + * Get the next 32 bits ({@link WireType#THIRTY_TWO_BIT}) + * @return a byte array of the next 32 bits (4 bytes) + */ + public byte[] nextFixed32() { + // 4 bytes == 32 bits + return readNextBytes(4); + } + + /** + * Get the next 64 bits ({@link WireType#SIXTY_FOUR_BIT}) + * @return a byte array of the next 64 bits (8 bytes) + */ + public byte[] nextFixed64() { + // 8 bytes == 64 bits + return readNextBytes(8); + } + + /** + * Read an arbitrary number of bytes + * @param size The number of bytes to read + * @return a byte array of the specified size, filled with bytes read + */ + private byte[] readNextBytes(int size) { + byte[] bytesRead = new byte[size]; + for (int i = 0; i < bytesRead.length; i++) { + bytesRead[i] = this.nextByte(); + } + return bytesRead; + } + + /** + * Get the next delimited message ({@link WireType#LENGTH_DELIMITED}) + * @return The next length delimited message + */ + public byte[] nextLengthDelimited() { + int length = convertByteArray(this.nextVarInt()).intValue(); + return readNextBytes(length); + } + + /** + * Convert a byte array to a number (little endian) + * @param bytes The bytes to convert + * @return An appropriate {@link Number} class. + */ + public static Number convertByteArray(byte[] bytes) { + long number = 0; + for (int i = 0; i < bytes.length; i++) { + number += bytes[i] << 7 * i; + } + return convertLong(number); + } + + /** + * Convert a long to an appropriate {@link Number} class + * @param number The long to convert + * @return A {@link Number} + */ + public static Number convertLong(long number) { + // TODO deal with booleans + if (number <= Byte.MAX_VALUE && number >= Byte.MIN_VALUE) { + return Byte.valueOf((byte) number); + } else if (number <= Short.MAX_VALUE && number >= Short.MIN_VALUE) { + return Short.valueOf((short) number); + } else if (number <= Integer.MAX_VALUE && number >= Integer.MIN_VALUE) { + return Integer.valueOf((int) number); + } else if (number <= Long.MAX_VALUE && number >= Long.MIN_VALUE) { + return Long.valueOf(number); + } + return number; + } + + /** + * Read all records + * @return A collection of all records + */ + public Collection allRecords() { + Collection records = new ArrayList<>(); + while (this.hasNext()) { + records.add(new ProtoBufRecord(this)); + } + return records; + } +} Index: src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java (working copy) @@ -0,0 +1,143 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; + +import org.openstreetmap.josm.tools.Utils; + +/** + * A protobuf record, storing the {@link WireType}, the parsed field number, and the bytes for it. + * @author Taylor Smock + * @since xxx + */ +public class ProtoBufRecord { + private static final byte[] EMPTY_BYTES = {}; + private final WireType type; + private final int field; + private final byte[] bytes; + + /** + * Create a new Protobuf record + * @param parser The parser to use to create the record + */ + public ProtoBufRecord(ProtoBufParser parser) { + Number number = ProtoBufParser.convertByteArray(parser.nextVarInt()); + // I don't foresee having field numbers > {@code Integer#MAX_VALUE >> 3} + this.field = (int) number.longValue() >> 3; + // 7 is 111 (so last three bits) + byte wireType = (byte) (number.longValue() & 7); + this.type = Stream.of(WireType.values()).filter(wType -> wType.getTypeRepresentation() == wireType).findFirst().orElse(WireType.UNKNOWN); + + if (this.type == WireType.VARINT) { + this.bytes = parser.nextVarInt(); + } else if (this.type == WireType.SIXTY_FOUR_BIT) { + this.bytes = parser.nextFixed64(); + } else if (this.type == WireType.THIRTY_TWO_BIT) { + this.bytes = parser.nextFixed32(); + } else if (this.type == WireType.LENGTH_DELIMITED) { + this.bytes = parser.nextLengthDelimited(); + // START_GROUP and END_GROUP are currently used by Mapbox Vector Tiles + } else if (this.type == WireType.START_GROUP || this.type == WireType.END_GROUP) { + this.bytes = EMPTY_BYTES; + } else { + throw new IllegalArgumentException(tr("Unknown type: {0} for field {1}", wireType, this.field)); + } + } + + /** + * Get the field value + * @return The field value + */ + public int getField() { + return this.field; + } + + /** + * Get the WireType of the data + * @return The {@link WireType} of the data + */ + public WireType getType() { + return this.type; + } + + /** + * Get the raw bytes for this record + * @return The bytes + */ + public byte[] getBytes() { + return this.bytes; + } + + /** + * Get the var int ({@code WireType#VARINT}) + * @return The var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum}) + */ + public Number asUnsignedVarInt() { + return ProtoBufParser.convertByteArray(this.bytes); + } + + /** + * Get the signed var int ({@code WireType#VARINT}). + * These are specially encoded so that they take up less space. + * + * @return The signed var int ({@code sint32} or {@code sint64}) + */ + public Number asSignedVarInt() { + Number signed = this.asUnsignedVarInt(); + if (signed instanceof Long) { + long value = ((Long) signed).longValue(); + return Long.valueOf((value << 1) ^ (value >> 63)); + } + int value = signed.intValue(); + long number = (value << 1) ^ (value >> 31); + return ProtoBufParser.convertLong(number); + } + + /** + * Get as a double ({@link WireType#SIXTY_FOUR_BIT}) + * @return the double + */ + public double asDouble() { + long doubleNumber = ProtoBufParser.convertByteArray(asFixed64()).longValue(); + return Double.longBitsToDouble(doubleNumber); + } + + /** + * Get as a float ({@link WireType#THIRTY_TWO_BIT}) + * @return the float + */ + public float asFloat() { + int floatNumber = ProtoBufParser.convertByteArray(asFixed32()).intValue(); + return Float.intBitsToFloat(floatNumber); + } + + /** + * Get as a string ({@link WireType#LENGTH_DELIMITED}) + * @return The string (encoded as {@link StandardCharsets#UTF_8}) + */ + public String asString() { + return Utils.intern(new String(this.bytes, StandardCharsets.UTF_8)); + } + + /** + * Get as 32 bits ({@link WireType#THIRTY_TWO_BIT}) + * @return a byte array of the 32 bits (4 bytes) + */ + public byte[] asFixed32() { + // TODO verify, or just assume? + // 4 bytes == 32 bits + return this.bytes; + } + + /** + * Get as 64 bits ({@link WireType#SIXTY_FOUR_BIT}) + * @return a byte array of the 64 bits (8 bytes) + */ + public byte[] asFixed64() { + // TODO verify, or just assume? + // 8 bytes == 64 bits + return this.bytes; + } +} Index: src/org/openstreetmap/josm/data/protobuf/WireType.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/WireType.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/WireType.java (working copy) @@ -0,0 +1,46 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +/** + * The WireTypes + * @author Taylor Smock + * @since xxx + */ +public enum WireType { + /** int32, int64, uint32, uint64, sing32, sint64, bool, enum */ + VARINT(0), + /** fixed64, sfixed64, double */ + SIXTY_FOUR_BIT(1), + /** string, bytes, embedded messages, packed repeated fields */ + LENGTH_DELIMITED(2), + /** + * start groups + * @deprecated Unknown reason. Deprecated since at least 2012. + */ + @Deprecated + START_GROUP(3), + /** + * end groups + * @deprecated Unknown reason. Deprecated since at least 2012. + */ + @Deprecated + END_GROUP(4), + /** fixed32, sfixed32, float */ + THIRTY_TWO_BIT(5), + + /** For unknown WireTypes */ + UNKNOWN(Byte.MAX_VALUE); + + private final byte type; + WireType(int value) { + this.type = (byte) value; + } + + /** + * Get the type representation (byte form) + * @return The wire type byte representation + */ + public byte getTypeRepresentation() { + return this.type; + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Command.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Command.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Command.java (working copy) @@ -0,0 +1,37 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +/** + * Command integers for Mapbox Vector Tiles + * @author Taylor Smock + * @since xxx + */ +public enum Command { + MoveTo((byte) 1, (byte) 2), + LineTo((byte) 2, (byte) 2), + ClosePath((byte) 7, (byte) 0); + + private final byte id; + private final byte parameters; + + Command(byte id, byte parameters) { + this.id = id; + this.parameters = parameters; + } + + /** + * Get the command id + * @return The id + */ + public byte getId() { + return this.id; + } + + /** + * Get the number of parameters + * @return The number of parameters + */ + public byte getParameterNumber() { + return this.parameters; + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/CommandInteger.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/CommandInteger.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/CommandInteger.java (working copy) @@ -0,0 +1,27 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +import java.util.stream.Stream; + +/** + * An indicator for a command to be executed + * @author Taylor Smock + * @since xxx + */ +public class CommandInteger { + private final Command type; + private final int operations; + + /** + * Create a new command + * @param command the command (treated as an unsigned int) + */ + public CommandInteger(final int command) { + long unsigned = Integer.toUnsignedLong(command); + this.type = Stream.of(Command.values()).filter(e -> e.getId() == (unsigned & 0x7)).findAny() + .orElseThrow(InvalidMapboxVectorTileException::new); + // This is safe, since we are shifting right 3 when we converted an int to a long (for unsigned). + // So we cannot lose anything. + this.operations = (int) (unsigned >> 3); + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Feature.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Feature.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Feature.java (working copy) @@ -0,0 +1,118 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +import java.util.ArrayList; +import java.util.List; + +import org.openstreetmap.josm.data.osm.TagMap; +import org.openstreetmap.josm.data.protobuf.ProtoBufPacked; +import org.openstreetmap.josm.data.protobuf.ProtoBufParser; +import org.openstreetmap.josm.data.protobuf.ProtoBufRecord; +import org.openstreetmap.josm.tools.Utils; + +/** + * A Feature for a {@link Layer} + * @author Taylor Smock + * @since xxx + */ +public class Feature { + private static final byte ID_FIELD = 1; + private static final byte TAG_FIELD = 2; + private static final byte GEOMETRY_TYPE_FIELD = 3; + private static final byte GEOMETRY_FIELD = 4; + /** The geometry of the feature. Required. */ + private final List geometry = new ArrayList<>(); + + /** The geometry type of the feature. Required. */ + private final GeometryTypes geometryType; + + /** The tags of the feature. Optional. */ + private TagMap tags; + + /** The id of the feature. Optional. */ + // Technically, uint64 + private final long id; + + /** + * Create a new Feature + * @param layer The layer the feature is part of (required for tags) + * @param record The record to create the feature from + */ + public Feature(Layer layer, ProtoBufRecord record) { + ProtoBufParser parser = new ProtoBufParser(record.getBytes()); + long tId = 0; + GeometryTypes geometryTypeTemp = GeometryTypes.UNKNOWN; + String key = null; + while (parser.hasNext()) { + ProtoBufRecord next = new ProtoBufRecord(parser); + if (next.getField() == TAG_FIELD) { + if (tags == null) { + tags = new TagMap(); + } + // This is packed in v1 and v2 + ProtoBufPacked packed = new ProtoBufPacked(next.getBytes()); + if (packed.isPacked()) { + for (Number number : packed.getArray()) { + if (key == null) { + key = layer.getKey(number.intValue()); + } else { + Object value = layer.getValue(number.intValue()); + this.tags.put(key, Utils.intern(value.toString())); + key = null; + } + } + } else { + if (key == null) { + key = layer.getKey(next.asSignedVarInt().intValue()); + } else { + Object value = layer.getValue(next.asSignedVarInt().intValue()); + this.tags.put(key, Utils.intern(value.toString())); + key = null; + } + } + } else if (next.getField() == GEOMETRY_FIELD) { + // This is packed in v1 and v2 + ProtoBufPacked packed = new ProtoBufPacked(next.getBytes()); + // TODO fallback to non-packed + } else if (next.getField() == GEOMETRY_TYPE_FIELD) { + geometryTypeTemp = GeometryTypes.values()[next.asUnsignedVarInt().intValue()]; + } else if (next.getField() == ID_FIELD) { + tId = next.asUnsignedVarInt().longValue(); + } + } + this.id = tId; + this.geometryType = geometryTypeTemp; + } + + /** + * Get the geometry instructions + * @return The geometry + */ + public List getGeometry() { + return this.geometry; + } + + /** + * Get the geometry type + * @return The {@link GeometryTypes} + */ + public GeometryTypes getGeometryType() { + return this.geometryType; + } + + /** + * Get the id of the object + * @return The unique id in the layer, or 0. + */ + public long getId() { + return this.id; + } + + /** + * Get the tags + * @return A tag map + */ + public TagMap getTags() { + return this.tags; + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/GeometryTypes.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/GeometryTypes.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/GeometryTypes.java (working copy) @@ -0,0 +1,44 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +/** + * Geometry types used by Mapbox Vector Tiles + * @author Taylor Smock + * @since xxx + */ +public enum GeometryTypes { + /** May be ignored */ + UNKNOWN((byte) 0), + /** May be a point or a multipoint geometry. Uses only {@link Command#MoveTo}. Multiple {@link Command#MoveTo} + * indicates that it is a multi-point object. */ + POINT((byte) 1), + /** May be a line or a multiline geometry. Each line {@link Command#MoveTo} and one or more {@link Command#LineTo}. */ + LINESTRING((byte) 2), + /** May be a polygon or a multipolygon. Each ring uses a {@link Command#MoveTo}, one or more {@link Command#LineTo}, + * and one {@link Command#ClosePath} command. See {@link Ring}s. */ + POLYGON((byte) 3); + + private final byte id; + GeometryTypes(byte id) { + this.id = id; + } + + /** + * Get the id for the geometry type + * @return The id + */ + public byte getId() { + return this.id; + } + + /** + * Rings used by {@link GeometryTypes#POLYGON} + * @author Taylor Smock + */ + public enum Ring { + /** A ring that goes in the clockwise direction */ + ExteriorRing, + /** A ring that goes in the anti-clockwise direction */ + InteriorRing; + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/InvalidMapboxVectorTileException.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/InvalidMapboxVectorTileException.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/InvalidMapboxVectorTileException.java (working copy) @@ -0,0 +1,25 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +/** + * Thrown when a mapbox vector tile does not match specifications. + * + * @author Taylor Smock + * @since xxx + */ +public class InvalidMapboxVectorTileException extends RuntimeException { + /** + * Create a default {@link InvalidMapboxVectorTileException}. + */ + public InvalidMapboxVectorTileException() { + super(); + } + + /** + * Create a new {@link InvalidMapboxVectorTile} exception with a message + * @param message The message + */ + public InvalidMapboxVectorTileException(final String message) { + super(message); + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Layer.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Layer.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/Layer.java (working copy) @@ -0,0 +1,178 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; +import static org.openstreetmap.josm.tools.I18n.tr; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; + +import org.openstreetmap.josm.data.protobuf.ProtoBufParser; +import org.openstreetmap.josm.data.protobuf.ProtoBufRecord; +import org.openstreetmap.josm.data.protobuf.WireType; + +/** + * A Mapbox Vector Tile Layer + * @author Taylor Smock + * @since xxx + */ +public class Layer { + private static final class ValueFields { + static final ValueFields STRING = new ValueFields<>(1, ProtoBufRecord::asString); + static final ValueFields FLOAT = new ValueFields<>(2, ProtoBufRecord::asFloat); + static final ValueFields DOUBLE = new ValueFields<>(3, ProtoBufRecord::asDouble); + static final ValueFields INT64 = new ValueFields<>(4, ProtoBufRecord::asUnsignedVarInt); + // This may have issues if there are actual uint_values (i.e., more than {@link Long#MAX_VALUE}) + static final ValueFields UINT64 = new ValueFields<>(5, ProtoBufRecord::asUnsignedVarInt); + static final ValueFields SINT64 = new ValueFields<>(6, ProtoBufRecord::asSignedVarInt); + static final ValueFields BOOL = new ValueFields<>(7, r -> r.asUnsignedVarInt().longValue() != 0); + + public static final Collection> MAPPERS = Arrays.asList(STRING, FLOAT, DOUBLE, INT64, UINT64, SINT64, BOOL); + + private final byte field; + private final Function conversion; + private ValueFields(int field, Function conversion) { + this.field = (byte) field; + this.conversion = conversion; + } + + /** + * Get the field identifier for the value + * @return The identifier + */ + public byte getField() { + return this.field; + } + + /** + * Convert a protobuf record to a value + * @param protobufRecord The record to convert + * @return the converted value + */ + public T convertValue(ProtoBufRecord protobufRecord) { + return this.conversion.apply(protobufRecord); + } + } + + /** The field value for a layer (in {@link ProtoBufRecord#getField}) */ + public static final byte LAYER_FIELD = 3; + private static final byte VERSION_FIELD = 15; + private static final byte NAME_FIELD = 1; + private static final byte FEATURE_FIELD = 2; + private static final byte KEY_FIELD = 3; + private static final byte VALUE_FIELD = 4; + private static final byte EXTENT_FIELD = 5; + private static final int DEFAULT_EXTENT = 4096; + private static final byte DEFAULT_VERSION = 1; + /** This is technically an integer, but there are currently only two major versions (1, 2). Required. */ + private final byte version; + /** A unique name for the layer. This must be unique on a per-tile basis. Required. */ + private final String name; + + /** The extent of the tile, typically 4096. Required. */ + private final int extent; + + /** A list of unique keys. Order is important. Optional. */ + private final List keyList = new ArrayList<>(); + /** A list of unique values. Order is important. Optional. */ + private final List valueList = new ArrayList<>(); + /** The actual features of this layer in this tile */ + private final List featureCollection; + + /** + * Create a layer from a collection of records + * @param records The records to convert to a layer + */ + public Layer(Collection records) { + // Do the unique required fields first + this.version = records.stream().parallel() + .filter(record -> record.getType() == WireType.VARINT && record.getField() == VERSION_FIELD) + .map(ProtoBufRecord::asSignedVarInt).map(Number::byteValue) + .findFirst().orElse(DEFAULT_VERSION); + // Per spec, we cannot continue past this until we have checked the version number + if (this.version != 1 && this.version != 2) { + throw new IllegalArgumentException(tr("We do not understand version {0} of the vector tile specification", this.version)); + } + this.name = records.stream().parallel() + .filter(record -> record.getType() == WireType.LENGTH_DELIMITED && record.getField() == NAME_FIELD) + .map(ProtoBufRecord::asString) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException(tr("Vector tile layers must have a layer name"))); + this.extent = records.stream().parallel().filter(record -> record.getType() == WireType.VARINT && record.getField() == EXTENT_FIELD).map(ProtoBufRecord::asSignedVarInt).map(Number::intValue).findAny().orElse(DEFAULT_EXTENT); + + List features = new ArrayList<>(); + for (ProtoBufRecord record : records) { + if (record.getField() == FEATURE_FIELD) { + features.add(record); + } else if (record.getField() == KEY_FIELD) { + this.keyList.add(record.asString()); + } else if (record.getField() == VALUE_FIELD && record.getType() == WireType.LENGTH_DELIMITED) { + ProtoBufParser parser = new ProtoBufParser(record.getBytes()); + ProtoBufRecord valueRecord = new ProtoBufRecord(parser); + if (parser.hasNext()) { + throw new IllegalArgumentException(tr("Vector tile value fields are restricted to one value")); + } + this.valueList.add(ValueFields.MAPPERS.stream() + .filter(v -> v.getField() == valueRecord.getField()) + .map(v -> v.convertValue(valueRecord)).findFirst() + .orElseThrow(() -> new IllegalArgumentException(tr("Unknown field in vector tile layer value ({0})", valueRecord.getField())))); + } else if (record.getField() == VALUE_FIELD) { + throw new IllegalArgumentException(tr("We do not under a value field in the vector tile")); + } + } + this.featureCollection = features.parallelStream().map(feature -> new Feature(this, feature)).collect(Collectors.toList()); + } + + /** + * Create a new layer + * @param bytes The bytes that the layer comes from + */ + public Layer(byte[] bytes) { + this(new ProtoBufParser(bytes).allRecords()); + } + + /** + * Get the extent of the tile + * @return The layer extent + */ + public int getExtent() { + return this.extent; + } + + /** + * Get the feature on this layer + * @return the features + */ + public Collection getFeatures() { + return Collections.unmodifiableCollection(this.featureCollection); + } + + /** + * Get a specified key + * @param index The index in the key list + * @return The actual key + */ + public String getKey(int index) { + return this.keyList.get(index); + } + + /** + * Get the name of the layer + * @return The layer name + */ + public String getName() { + return this.name; + } + + /** + * Get a specified value + * @param index The index in the value list + * @return The actual value. This can be a {@link String}, {@link Boolean}, {@link Integer}, or {@link Float} value. + */ + public Object getValue(int index) { + return this.valueList.get(index); + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MVTFile.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MVTFile.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/MVTFile.java (working copy) @@ -0,0 +1,33 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +import java.util.Arrays; +import java.util.List; + +/** + * Items that MAY be used to figure out if a file or server response MAY BE a Mapbox Vector Tile + * @author Taylor Smock + * @since xxx + */ +public final class MVTFile { + /** + * Extensions for Mapbox Vector Tiles. + * This is a SHOULD, not a MUST. + */ + public static final List EXTENSION = Arrays.asList("mvt"); + + /** + * mimetypes for Mapbox Vector Tiles + * This is a SHOULD, not a MUST. + */ + public static final List MIMETYPE = Arrays.asList("application/vnd.mapbox-vector-tile"); + + /** + * The default projection. This is Web Mercator, per specification. + */ + public static final String DEFAULT_PROJECTION = "EPSG:3857"; + + private MVTFile() { + // Hide the constructor + } +} Index: src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/ParameterInteger.java =================================================================== --- src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/ParameterInteger.java (nonexistent) +++ src/org/openstreetmap/josm/data/imagery/vectortile/mapbox/ParameterInteger.java (working copy) @@ -0,0 +1,26 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.imagery.vectortile.mapbox; + +/** + * The parameters that follow the {@link CommandInteger}. + * @author Taylor Smock + * @since xxx + */ +public class ParameterInteger { + private final int value; + /** + * Create a new @link{ParameterInteger} + * @param value The value to convert to a ParameterInteger (zigzag encoded). + */ + public ParameterInteger(final int value) { + this.value = ((value >> 1) ^ (-(value & 1))); + } + + /** + * Get the value for this ParameterInteger + * @return The decoded integer value + */ + public int getValue() { + return this.value; + } +}