Index: src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java (working copy) @@ -0,0 +1,62 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import java.util.ArrayList; +import java.util.List; + +/** + * Parse packed values (only numerical values) + * + * @author Taylor Smock + * @since xxx + */ +public class ProtoBufPacked { + private final byte[] bytes; + private final Number[] numbers; + private int location; + + /** + * Create a new ProtoBufPacked object + * + * @param bytes The packed bytes + */ + public ProtoBufPacked(byte[] bytes) { + this.location = 0; + this.bytes = bytes; + List numbersT = new ArrayList<>(); + while (this.location < bytes.length) { + numbersT.add(ProtoBufParser.convertByteArray(this.nextVarInt(), ProtoBufParser.VAR_INT_BYTE_SIZE)); + } + + this.numbers = new Number[numbersT.size()]; + for (int i = 0; i < numbersT.size(); i++) { + this.numbers[i] = numbersT.get(i); + } + } + + /** + * Get the parsed number array + * + * @return The number array + */ + public Number[] getArray() { + return this.numbers; + } + + private byte[] nextVarInt() { + List byteList = new ArrayList<>(); + while ((this.bytes[this.location] & ProtoBufParser.MOST_SIGNIFICANT_BYTE) + == ProtoBufParser.MOST_SIGNIFICANT_BYTE) { + // Get rid of the leading bit (shift left 1, then shift right 1 unsigned) + byteList.add((byte) (this.bytes[this.location++] ^ ProtoBufParser.MOST_SIGNIFICANT_BYTE)); + } + // The last byte doesn't drop the most significant bit + byteList.add(this.bytes[this.location++]); + byte[] byteArray = new byte[byteList.size()]; + for (int i = 0; i < byteList.size(); i++) { + byteArray[i] = byteList.get(i); + } + + return byteArray; + } +} Index: src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java (working copy) @@ -0,0 +1,245 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.openstreetmap.josm.tools.Logging; + +/** + * A basic Protobuf parser + * + * @author Taylor Smock + * @since xxx + */ +public class ProtoBufParser implements AutoCloseable { + /** + * The default byte size (see {@link #VAR_INT_BYTE_SIZE} for var ints) + */ + public static final byte BYTE_SIZE = 8; + /** + * The byte size for var ints (since the first byte is just an indicator for if the var int is done) + */ + public static final byte VAR_INT_BYTE_SIZE = BYTE_SIZE - 1; + /** + * Used to get the most significant byte + */ + static final byte MOST_SIGNIFICANT_BYTE = (byte) (1 << 7); + /** + * Convert a byte array to a number (little endian) + * + * @param bytes The bytes to convert + * @param byteSize The size of the byte. For var ints, this is 7, for other ints, this is 8. + * @return An appropriate {@link Number} class. + */ + public static Number convertByteArray(byte[] bytes, byte byteSize) { + long number = 0; + for (int i = 0; i < bytes.length; i++) { + // Need to convert to uint64 in order to avoid bit operation from filling in 1's and overflow issues + number += Byte.toUnsignedLong(bytes[i]) << (byteSize * i); + } + return convertLong(number); + } + + /** + * Convert a long to an appropriate {@link Number} class + * + * @param number The long to convert + * @return A {@link Number} + */ + public static Number convertLong(long number) { + // TODO deal with booleans + if (number <= Byte.MAX_VALUE && number >= Byte.MIN_VALUE) { + return (byte) number; + } else if (number <= Short.MAX_VALUE && number >= Short.MIN_VALUE) { + return (short) number; + } else if (number <= Integer.MAX_VALUE && number >= Integer.MIN_VALUE) { + return (int) number; + } + return number; + } + + /** + * Decode a zig-zag encoded value + * + * @param signed The value to decode + * @return The decoded value + */ + public static Number decodeZigZag(Number signed) { + final long value = signed.longValue(); + return convertLong((value >> 1) ^ -(value & 1)); + } + + /** + * Encode a number to a zig-zag encode value + * + * @param signed The number to encode + * @return The encoded value + */ + public static Number encodeZigZag(Number signed) { + final long value = signed.longValue(); + // This boundary condition could be >= or <= or both. Tests indicate that it doesn't actually matter. + // The only difference would be the number type returned, except it is always converted to the most basic type. + final int shift = (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE ? Long.BYTES : Integer.BYTES) * 8 - 1; + return convertLong((value << 1) ^ (value >> shift)); + } + + private final InputStream inputStream; + + /** + * Create a new parser + * + * @param bytes The bytes to parse + */ + public ProtoBufParser(byte[] bytes) { + this(new ByteArrayInputStream(bytes)); + } + + /** + * Create a new parser + * + * @param inputStream The InputStream (will be fully read at this time) + */ + public ProtoBufParser(InputStream inputStream) { + if (inputStream.markSupported()) { + this.inputStream = inputStream; + } else { + this.inputStream = new BufferedInputStream(inputStream); + } + } + + /** + * Read all records + * + * @return A collection of all records + * @throws IOException - if an IO error occurs + */ + public Collection allRecords() throws IOException { + Collection records = new ArrayList<>(); + while (this.hasNext()) { + records.add(new ProtoBufRecord(this)); + } + return records; + } + + @Override + public void close() { + try { + this.inputStream.close(); + } catch (IOException e) { + Logging.error(e); + } + } + + /** + * Check if there is more data to read + * + * @return {@code true} if there is more data to read + * @throws IOException - if an IO error occurs + */ + public boolean hasNext() throws IOException { + return this.inputStream.available() > 0; + } + + /** + * Get the "next" WireType + * + * @return {@link WireType} expected + * @throws IOException - if an IO error occurs + */ + public WireType next() throws IOException { + this.inputStream.mark(16); + try { + return WireType.values()[this.inputStream.read() << 3]; + } finally { + this.inputStream.reset(); + } + } + + /** + * Get the next byte + * + * @return The next byte + * @throws IOException - if an IO error occurs + */ + public int nextByte() throws IOException { + return this.inputStream.read(); + } + + /** + * Get the next 32 bits ({@link WireType#THIRTY_TWO_BIT}) + * + * @return a byte array of the next 32 bits (4 bytes) + * @throws IOException - if an IO error occurs + */ + public byte[] nextFixed32() throws IOException { + // 4 bytes == 32 bits + return readNextBytes(4); + } + + /** + * Get the next 64 bits ({@link WireType#SIXTY_FOUR_BIT}) + * + * @return a byte array of the next 64 bits (8 bytes) + * @throws IOException - if an IO error occurs + */ + public byte[] nextFixed64() throws IOException { + // 8 bytes == 64 bits + return readNextBytes(8); + } + + /** + * Get the next delimited message ({@link WireType#LENGTH_DELIMITED}) + * + * @return The next length delimited message + * @throws IOException - if an IO error occurs + */ + public byte[] nextLengthDelimited() throws IOException { + int length = convertByteArray(this.nextVarInt(), VAR_INT_BYTE_SIZE).intValue(); + return readNextBytes(length); + } + + /** + * Get the next var int ({@code WireType#VARINT}) + * + * @return The next var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum}) + * @throws IOException - if an IO error occurs + */ + public byte[] nextVarInt() throws IOException { + List byteList = new ArrayList<>(); + int currentByte = this.nextByte(); + while ((byte) (currentByte & MOST_SIGNIFICANT_BYTE) == MOST_SIGNIFICANT_BYTE && currentByte > 0) { + // Get rid of the leading bit (shift left 1, then shift right 1 unsigned) + byteList.add((byte) (currentByte ^ MOST_SIGNIFICANT_BYTE)); + currentByte = this.nextByte(); + } + // The last byte doesn't drop the most significant bit + byteList.add((byte) currentByte); + byte[] byteArray = new byte[byteList.size()]; + for (int i = 0; i < byteList.size(); i++) { + byteArray[i] = byteList.get(i); + } + + return byteArray; + } + + /** + * Read an arbitrary number of bytes + * + * @param size The number of bytes to read + * @return a byte array of the specified size, filled with bytes read (unsigned) + * @throws IOException - if an IO error occurs + */ + private byte[] readNextBytes(int size) throws IOException { + byte[] bytesRead = new byte[size]; + for (int i = 0; i < bytesRead.length; i++) { + bytesRead[i] = (byte) this.nextByte(); + } + return bytesRead; + } +} Index: src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java (working copy) @@ -0,0 +1,152 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; + +import org.openstreetmap.josm.tools.Utils; + +/** + * A protobuf record, storing the {@link WireType}, the parsed field number, and the bytes for it. + * + * @author Taylor Smock + * @since xxx + */ +public class ProtoBufRecord implements AutoCloseable { + private static final byte[] EMPTY_BYTES = {}; + private final WireType type; + private final int field; + private byte[] bytes; + + /** + * Create a new Protobuf record + * + * @param parser The parser to use to create the record + * @throws IOException - if an IO error occurs + */ + public ProtoBufRecord(ProtoBufParser parser) throws IOException { + Number number = ProtoBufParser.convertByteArray(parser.nextVarInt(), ProtoBufParser.VAR_INT_BYTE_SIZE); + // I don't foresee having field numbers > {@code Integer#MAX_VALUE >> 3} + this.field = (int) number.longValue() >> 3; + // 7 is 111 (so last three bits) + byte wireType = (byte) (number.longValue() & 7); + this.type = Stream.of(WireType.values()).filter(wType -> wType.getTypeRepresentation() == wireType).findFirst() + .orElse(WireType.UNKNOWN); + + if (this.type == WireType.VARINT) { + this.bytes = parser.nextVarInt(); + } else if (this.type == WireType.SIXTY_FOUR_BIT) { + this.bytes = parser.nextFixed64(); + } else if (this.type == WireType.THIRTY_TWO_BIT) { + this.bytes = parser.nextFixed32(); + } else if (this.type == WireType.LENGTH_DELIMITED) { + this.bytes = parser.nextLengthDelimited(); + } else { + this.bytes = EMPTY_BYTES; + } + } + + /** + * Get as a double ({@link WireType#SIXTY_FOUR_BIT}) + * + * @return the double + */ + public double asDouble() { + long doubleNumber = ProtoBufParser.convertByteArray(asFixed64(), ProtoBufParser.BYTE_SIZE).longValue(); + return Double.longBitsToDouble(doubleNumber); + } + + /** + * Get as 32 bits ({@link WireType#THIRTY_TWO_BIT}) + * + * @return a byte array of the 32 bits (4 bytes) + */ + public byte[] asFixed32() { + // TODO verify, or just assume? + // 4 bytes == 32 bits + return this.bytes; + } + + /** + * Get as 64 bits ({@link WireType#SIXTY_FOUR_BIT}) + * + * @return a byte array of the 64 bits (8 bytes) + */ + public byte[] asFixed64() { + // TODO verify, or just assume? + // 8 bytes == 64 bits + return this.bytes; + } + + /** + * Get as a float ({@link WireType#THIRTY_TWO_BIT}) + * + * @return the float + */ + public float asFloat() { + int floatNumber = ProtoBufParser.convertByteArray(asFixed32(), ProtoBufParser.BYTE_SIZE).intValue(); + return Float.intBitsToFloat(floatNumber); + } + + /** + * Get the signed var int ({@code WireType#VARINT}). + * These are specially encoded so that they take up less space. + * + * @return The signed var int ({@code sint32} or {@code sint64}) + */ + public Number asSignedVarInt() { + final Number signed = this.asUnsignedVarInt(); + return ProtoBufParser.decodeZigZag(signed); + } + + /** + * Get as a string ({@link WireType#LENGTH_DELIMITED}) + * + * @return The string (encoded as {@link StandardCharsets#UTF_8}) + */ + public String asString() { + return Utils.intern(new String(this.bytes, StandardCharsets.UTF_8)); + } + + /** + * Get the var int ({@code WireType#VARINT}) + * + * @return The var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum}) + */ + public Number asUnsignedVarInt() { + return ProtoBufParser.convertByteArray(this.bytes, ProtoBufParser.VAR_INT_BYTE_SIZE); + } + + @Override + public void close() { + this.bytes = null; + } + + /** + * Get the raw bytes for this record + * + * @return The bytes + */ + public byte[] getBytes() { + return this.bytes; + } + + /** + * Get the field value + * + * @return The field value + */ + public int getField() { + return this.field; + } + + /** + * Get the WireType of the data + * + * @return The {@link WireType} of the data + */ + public WireType getType() { + return this.type; + } +} Index: src/org/openstreetmap/josm/data/protobuf/WireType.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/WireType.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/WireType.java (working copy) @@ -0,0 +1,61 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +/** + * The WireTypes + * + * @author Taylor Smock + * @since xxx + */ +public enum WireType { + /** + * int32, int64, uint32, uint64, sing32, sint64, bool, enum + */ + VARINT(0), + /** + * fixed64, sfixed64, double + */ + SIXTY_FOUR_BIT(1), + /** + * string, bytes, embedded messages, packed repeated fields + */ + LENGTH_DELIMITED(2), + /** + * start groups + * + * @deprecated Unknown reason. Deprecated since at least 2012. + */ + @Deprecated + START_GROUP(3), + /** + * end groups + * + * @deprecated Unknown reason. Deprecated since at least 2012. + */ + @Deprecated + END_GROUP(4), + /** + * fixed32, sfixed32, float + */ + THIRTY_TWO_BIT(5), + + /** + * For unknown WireTypes + */ + UNKNOWN(Byte.MAX_VALUE); + + private final byte type; + + WireType(int value) { + this.type = (byte) value; + } + + /** + * Get the type representation (byte form) + * + * @return The wire type byte representation + */ + public byte getTypeRepresentation() { + return this.type; + } +} Index: src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/ProtoBufPacked.java (working copy) @@ -0,0 +1,62 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import java.util.ArrayList; +import java.util.List; + +/** + * Parse packed values (only numerical values) + * + * @author Taylor Smock + * @since xxx + */ +public class ProtoBufPacked { + private final byte[] bytes; + private final Number[] numbers; + private int location; + + /** + * Create a new ProtoBufPacked object + * + * @param bytes The packed bytes + */ + public ProtoBufPacked(byte[] bytes) { + this.location = 0; + this.bytes = bytes; + List numbersT = new ArrayList<>(); + while (this.location < bytes.length) { + numbersT.add(ProtoBufParser.convertByteArray(this.nextVarInt(), ProtoBufParser.VAR_INT_BYTE_SIZE)); + } + + this.numbers = new Number[numbersT.size()]; + for (int i = 0; i < numbersT.size(); i++) { + this.numbers[i] = numbersT.get(i); + } + } + + /** + * Get the parsed number array + * + * @return The number array + */ + public Number[] getArray() { + return this.numbers; + } + + private byte[] nextVarInt() { + List byteList = new ArrayList<>(); + while ((this.bytes[this.location] & ProtoBufParser.MOST_SIGNIFICANT_BYTE) + == ProtoBufParser.MOST_SIGNIFICANT_BYTE) { + // Get rid of the leading bit (shift left 1, then shift right 1 unsigned) + byteList.add((byte) (this.bytes[this.location++] ^ ProtoBufParser.MOST_SIGNIFICANT_BYTE)); + } + // The last byte doesn't drop the most significant bit + byteList.add(this.bytes[this.location++]); + byte[] byteArray = new byte[byteList.size()]; + for (int i = 0; i < byteList.size(); i++) { + byteArray[i] = byteList.get(i); + } + + return byteArray; + } +} Index: src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/ProtoBufParser.java (working copy) @@ -0,0 +1,245 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.openstreetmap.josm.tools.Logging; + +/** + * A basic Protobuf parser + * + * @author Taylor Smock + * @since xxx + */ +public class ProtoBufParser implements AutoCloseable { + /** + * The default byte size (see {@link #VAR_INT_BYTE_SIZE} for var ints) + */ + public static final byte BYTE_SIZE = 8; + /** + * The byte size for var ints (since the first byte is just an indicator for if the var int is done) + */ + public static final byte VAR_INT_BYTE_SIZE = BYTE_SIZE - 1; + /** + * Used to get the most significant byte + */ + static final byte MOST_SIGNIFICANT_BYTE = (byte) (1 << 7); + /** + * Convert a byte array to a number (little endian) + * + * @param bytes The bytes to convert + * @param byteSize The size of the byte. For var ints, this is 7, for other ints, this is 8. + * @return An appropriate {@link Number} class. + */ + public static Number convertByteArray(byte[] bytes, byte byteSize) { + long number = 0; + for (int i = 0; i < bytes.length; i++) { + // Need to convert to uint64 in order to avoid bit operation from filling in 1's and overflow issues + number += Byte.toUnsignedLong(bytes[i]) << (byteSize * i); + } + return convertLong(number); + } + + /** + * Convert a long to an appropriate {@link Number} class + * + * @param number The long to convert + * @return A {@link Number} + */ + public static Number convertLong(long number) { + // TODO deal with booleans + if (number <= Byte.MAX_VALUE && number >= Byte.MIN_VALUE) { + return (byte) number; + } else if (number <= Short.MAX_VALUE && number >= Short.MIN_VALUE) { + return (short) number; + } else if (number <= Integer.MAX_VALUE && number >= Integer.MIN_VALUE) { + return (int) number; + } + return number; + } + + /** + * Decode a zig-zag encoded value + * + * @param signed The value to decode + * @return The decoded value + */ + public static Number decodeZigZag(Number signed) { + final long value = signed.longValue(); + return convertLong((value >> 1) ^ -(value & 1)); + } + + /** + * Encode a number to a zig-zag encode value + * + * @param signed The number to encode + * @return The encoded value + */ + public static Number encodeZigZag(Number signed) { + final long value = signed.longValue(); + // This boundary condition could be >= or <= or both. Tests indicate that it doesn't actually matter. + // The only difference would be the number type returned, except it is always converted to the most basic type. + final int shift = (value > Integer.MAX_VALUE || value < Integer.MIN_VALUE ? Long.BYTES : Integer.BYTES) * 8 - 1; + return convertLong((value << 1) ^ (value >> shift)); + } + + private final InputStream inputStream; + + /** + * Create a new parser + * + * @param bytes The bytes to parse + */ + public ProtoBufParser(byte[] bytes) { + this(new ByteArrayInputStream(bytes)); + } + + /** + * Create a new parser + * + * @param inputStream The InputStream (will be fully read at this time) + */ + public ProtoBufParser(InputStream inputStream) { + if (inputStream.markSupported()) { + this.inputStream = inputStream; + } else { + this.inputStream = new BufferedInputStream(inputStream); + } + } + + /** + * Read all records + * + * @return A collection of all records + * @throws IOException - if an IO error occurs + */ + public Collection allRecords() throws IOException { + Collection records = new ArrayList<>(); + while (this.hasNext()) { + records.add(new ProtoBufRecord(this)); + } + return records; + } + + @Override + public void close() { + try { + this.inputStream.close(); + } catch (IOException e) { + Logging.error(e); + } + } + + /** + * Check if there is more data to read + * + * @return {@code true} if there is more data to read + * @throws IOException - if an IO error occurs + */ + public boolean hasNext() throws IOException { + return this.inputStream.available() > 0; + } + + /** + * Get the "next" WireType + * + * @return {@link WireType} expected + * @throws IOException - if an IO error occurs + */ + public WireType next() throws IOException { + this.inputStream.mark(16); + try { + return WireType.values()[this.inputStream.read() << 3]; + } finally { + this.inputStream.reset(); + } + } + + /** + * Get the next byte + * + * @return The next byte + * @throws IOException - if an IO error occurs + */ + public int nextByte() throws IOException { + return this.inputStream.read(); + } + + /** + * Get the next 32 bits ({@link WireType#THIRTY_TWO_BIT}) + * + * @return a byte array of the next 32 bits (4 bytes) + * @throws IOException - if an IO error occurs + */ + public byte[] nextFixed32() throws IOException { + // 4 bytes == 32 bits + return readNextBytes(4); + } + + /** + * Get the next 64 bits ({@link WireType#SIXTY_FOUR_BIT}) + * + * @return a byte array of the next 64 bits (8 bytes) + * @throws IOException - if an IO error occurs + */ + public byte[] nextFixed64() throws IOException { + // 8 bytes == 64 bits + return readNextBytes(8); + } + + /** + * Get the next delimited message ({@link WireType#LENGTH_DELIMITED}) + * + * @return The next length delimited message + * @throws IOException - if an IO error occurs + */ + public byte[] nextLengthDelimited() throws IOException { + int length = convertByteArray(this.nextVarInt(), VAR_INT_BYTE_SIZE).intValue(); + return readNextBytes(length); + } + + /** + * Get the next var int ({@code WireType#VARINT}) + * + * @return The next var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum}) + * @throws IOException - if an IO error occurs + */ + public byte[] nextVarInt() throws IOException { + List byteList = new ArrayList<>(); + int currentByte = this.nextByte(); + while ((byte) (currentByte & MOST_SIGNIFICANT_BYTE) == MOST_SIGNIFICANT_BYTE && currentByte > 0) { + // Get rid of the leading bit (shift left 1, then shift right 1 unsigned) + byteList.add((byte) (currentByte ^ MOST_SIGNIFICANT_BYTE)); + currentByte = this.nextByte(); + } + // The last byte doesn't drop the most significant bit + byteList.add((byte) currentByte); + byte[] byteArray = new byte[byteList.size()]; + for (int i = 0; i < byteList.size(); i++) { + byteArray[i] = byteList.get(i); + } + + return byteArray; + } + + /** + * Read an arbitrary number of bytes + * + * @param size The number of bytes to read + * @return a byte array of the specified size, filled with bytes read (unsigned) + * @throws IOException - if an IO error occurs + */ + private byte[] readNextBytes(int size) throws IOException { + byte[] bytesRead = new byte[size]; + for (int i = 0; i < bytesRead.length; i++) { + bytesRead[i] = (byte) this.nextByte(); + } + return bytesRead; + } +} Index: src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/ProtoBufRecord.java (working copy) @@ -0,0 +1,152 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; + +import org.openstreetmap.josm.tools.Utils; + +/** + * A protobuf record, storing the {@link WireType}, the parsed field number, and the bytes for it. + * + * @author Taylor Smock + * @since xxx + */ +public class ProtoBufRecord implements AutoCloseable { + private static final byte[] EMPTY_BYTES = {}; + private final WireType type; + private final int field; + private byte[] bytes; + + /** + * Create a new Protobuf record + * + * @param parser The parser to use to create the record + * @throws IOException - if an IO error occurs + */ + public ProtoBufRecord(ProtoBufParser parser) throws IOException { + Number number = ProtoBufParser.convertByteArray(parser.nextVarInt(), ProtoBufParser.VAR_INT_BYTE_SIZE); + // I don't foresee having field numbers > {@code Integer#MAX_VALUE >> 3} + this.field = (int) number.longValue() >> 3; + // 7 is 111 (so last three bits) + byte wireType = (byte) (number.longValue() & 7); + this.type = Stream.of(WireType.values()).filter(wType -> wType.getTypeRepresentation() == wireType).findFirst() + .orElse(WireType.UNKNOWN); + + if (this.type == WireType.VARINT) { + this.bytes = parser.nextVarInt(); + } else if (this.type == WireType.SIXTY_FOUR_BIT) { + this.bytes = parser.nextFixed64(); + } else if (this.type == WireType.THIRTY_TWO_BIT) { + this.bytes = parser.nextFixed32(); + } else if (this.type == WireType.LENGTH_DELIMITED) { + this.bytes = parser.nextLengthDelimited(); + } else { + this.bytes = EMPTY_BYTES; + } + } + + /** + * Get as a double ({@link WireType#SIXTY_FOUR_BIT}) + * + * @return the double + */ + public double asDouble() { + long doubleNumber = ProtoBufParser.convertByteArray(asFixed64(), ProtoBufParser.BYTE_SIZE).longValue(); + return Double.longBitsToDouble(doubleNumber); + } + + /** + * Get as 32 bits ({@link WireType#THIRTY_TWO_BIT}) + * + * @return a byte array of the 32 bits (4 bytes) + */ + public byte[] asFixed32() { + // TODO verify, or just assume? + // 4 bytes == 32 bits + return this.bytes; + } + + /** + * Get as 64 bits ({@link WireType#SIXTY_FOUR_BIT}) + * + * @return a byte array of the 64 bits (8 bytes) + */ + public byte[] asFixed64() { + // TODO verify, or just assume? + // 8 bytes == 64 bits + return this.bytes; + } + + /** + * Get as a float ({@link WireType#THIRTY_TWO_BIT}) + * + * @return the float + */ + public float asFloat() { + int floatNumber = ProtoBufParser.convertByteArray(asFixed32(), ProtoBufParser.BYTE_SIZE).intValue(); + return Float.intBitsToFloat(floatNumber); + } + + /** + * Get the signed var int ({@code WireType#VARINT}). + * These are specially encoded so that they take up less space. + * + * @return The signed var int ({@code sint32} or {@code sint64}) + */ + public Number asSignedVarInt() { + final Number signed = this.asUnsignedVarInt(); + return ProtoBufParser.decodeZigZag(signed); + } + + /** + * Get as a string ({@link WireType#LENGTH_DELIMITED}) + * + * @return The string (encoded as {@link StandardCharsets#UTF_8}) + */ + public String asString() { + return Utils.intern(new String(this.bytes, StandardCharsets.UTF_8)); + } + + /** + * Get the var int ({@code WireType#VARINT}) + * + * @return The var int ({@code int32}, {@code int64}, {@code uint32}, {@code uint64}, {@code bool}, {@code enum}) + */ + public Number asUnsignedVarInt() { + return ProtoBufParser.convertByteArray(this.bytes, ProtoBufParser.VAR_INT_BYTE_SIZE); + } + + @Override + public void close() { + this.bytes = null; + } + + /** + * Get the raw bytes for this record + * + * @return The bytes + */ + public byte[] getBytes() { + return this.bytes; + } + + /** + * Get the field value + * + * @return The field value + */ + public int getField() { + return this.field; + } + + /** + * Get the WireType of the data + * + * @return The {@link WireType} of the data + */ + public WireType getType() { + return this.type; + } +} Index: src/org/openstreetmap/josm/data/protobuf/WireType.java =================================================================== --- src/org/openstreetmap/josm/data/protobuf/WireType.java (nonexistent) +++ src/org/openstreetmap/josm/data/protobuf/WireType.java (working copy) @@ -0,0 +1,61 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +/** + * The WireTypes + * + * @author Taylor Smock + * @since xxx + */ +public enum WireType { + /** + * int32, int64, uint32, uint64, sing32, sint64, bool, enum + */ + VARINT(0), + /** + * fixed64, sfixed64, double + */ + SIXTY_FOUR_BIT(1), + /** + * string, bytes, embedded messages, packed repeated fields + */ + LENGTH_DELIMITED(2), + /** + * start groups + * + * @deprecated Unknown reason. Deprecated since at least 2012. + */ + @Deprecated + START_GROUP(3), + /** + * end groups + * + * @deprecated Unknown reason. Deprecated since at least 2012. + */ + @Deprecated + END_GROUP(4), + /** + * fixed32, sfixed32, float + */ + THIRTY_TWO_BIT(5), + + /** + * For unknown WireTypes + */ + UNKNOWN(Byte.MAX_VALUE); + + private final byte type; + + WireType(int value) { + this.type = (byte) value; + } + + /** + * Get the type representation (byte form) + * + * @return The wire type byte representation + */ + public byte getTypeRepresentation() { + return this.type; + } +} Index: test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufParserTest.java =================================================================== --- test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufParserTest.java (nonexistent) +++ test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufParserTest.java (working copy) @@ -0,0 +1,51 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +/** + * Test class for {@link ProtoBufParser} + * @author Taylor Smock + * @since xxx + */ +class ProtoBufParserTest { + /** + * Check that we are appropriately converting values to the "smallest" type + */ + @Test + void testConvertLong() { + // No casting due to auto conversions + assertEquals(Byte.MAX_VALUE, ProtoBufParser.convertLong(Byte.MAX_VALUE)); + assertEquals(Byte.MIN_VALUE, ProtoBufParser.convertLong(Byte.MIN_VALUE)); + assertEquals(Short.MIN_VALUE, ProtoBufParser.convertLong(Short.MIN_VALUE)); + assertEquals(Short.MAX_VALUE, ProtoBufParser.convertLong(Short.MAX_VALUE)); + assertEquals(Integer.MAX_VALUE, ProtoBufParser.convertLong(Integer.MAX_VALUE)); + assertEquals(Integer.MIN_VALUE, ProtoBufParser.convertLong(Integer.MIN_VALUE)); + assertEquals(Long.MIN_VALUE, ProtoBufParser.convertLong(Long.MIN_VALUE)); + assertEquals(Long.MAX_VALUE, ProtoBufParser.convertLong(Long.MAX_VALUE)); + } + + /** + * Check that zig zags are appropriately encoded. + */ + @Test + void testEncodeZigZag() { + assertEquals(0, ProtoBufParser.encodeZigZag(0).byteValue()); + assertEquals(1, ProtoBufParser.encodeZigZag(-1).byteValue()); + assertEquals(2, ProtoBufParser.encodeZigZag(1).byteValue()); + assertEquals(3, ProtoBufParser.encodeZigZag(-2).byteValue()); + assertEquals(254, ProtoBufParser.encodeZigZag(Byte.MAX_VALUE).shortValue()); + assertEquals(255, ProtoBufParser.encodeZigZag(Byte.MIN_VALUE).shortValue()); + assertEquals(65_534, ProtoBufParser.encodeZigZag(Short.MAX_VALUE).intValue()); + assertEquals(65_535, ProtoBufParser.encodeZigZag(Short.MIN_VALUE).intValue()); + // These integers check a possible boundary condition (the boundary between using the 32/64 bit encoding methods) + assertEquals(4_294_967_292L, ProtoBufParser.encodeZigZag(Integer.MAX_VALUE - 1).longValue()); + assertEquals(4_294_967_293L, ProtoBufParser.encodeZigZag(Integer.MIN_VALUE + 1).longValue()); + assertEquals(4_294_967_294L, ProtoBufParser.encodeZigZag(Integer.MAX_VALUE).longValue()); + assertEquals(4_294_967_295L, ProtoBufParser.encodeZigZag(Integer.MIN_VALUE).longValue()); + assertEquals(4_294_967_296L, ProtoBufParser.encodeZigZag(Integer.MAX_VALUE + 1L).longValue()); + assertEquals(4_294_967_297L, ProtoBufParser.encodeZigZag(Integer.MIN_VALUE - 1L).longValue()); + } +} Index: test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufRecordTest.java =================================================================== --- test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufRecordTest.java (nonexistent) +++ test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufRecordTest.java (working copy) @@ -0,0 +1,30 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import static org.junit.jupiter.api.Assertions.assertEquals; + + +import java.io.IOException; + +import org.junit.jupiter.api.Test; + +/** + * Test class for specific {@link ProtoBufRecord} functionality + */ +class ProtoBufRecordTest { + @Test + void testFixed32() throws IOException { + ProtoBufParser parser = new ProtoBufParser(ProtoBufTest.toByteArray(new int[] {0x0d, 0x00, 0x00, 0x80, 0x3f})); + ProtoBufRecord thirtyTwoBit = new ProtoBufRecord(parser); + assertEquals(WireType.THIRTY_TWO_BIT, thirtyTwoBit.getType()); + assertEquals(1f, thirtyTwoBit.asFloat()); + } + + @Test + void testUnknown() throws IOException { + ProtoBufParser parser = new ProtoBufParser(ProtoBufTest.toByteArray(new int[] {0x0f, 0x00, 0x00, 0x80, 0x3f})); + ProtoBufRecord unknown = new ProtoBufRecord(parser); + assertEquals(WireType.UNKNOWN, unknown.getType()); + assertEquals(0, unknown.getBytes().length); + } +} Index: test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java =================================================================== --- test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java (nonexistent) +++ test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java (working copy) @@ -0,0 +1,211 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; + +import java.awt.geom.Ellipse2D; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import org.openstreetmap.josm.TestUtils; +import org.openstreetmap.josm.data.coor.LatLon; +import org.openstreetmap.josm.data.imagery.ImageryInfo; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.Feature; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.Layer; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MVTTile; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MapboxVectorTileSource; +import org.openstreetmap.josm.data.osm.BBox; +import org.openstreetmap.josm.data.osm.Node; +import org.openstreetmap.josm.data.osm.Way; +import org.openstreetmap.josm.data.vector.VectorDataSet; +import org.openstreetmap.josm.data.vector.VectorNode; +import org.openstreetmap.josm.data.vector.VectorWay; +import org.openstreetmap.josm.io.Compression; +import org.openstreetmap.josm.testutils.JOSMTestRules; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Test class for {@link ProtoBufParser} and {@link ProtoBufRecord} + * + * @author Taylor Smock + * @since xxx + */ +class ProtoBufTest { + /** + * Convert an int array into a byte array + * @param intArray The int array to convert (NOTE: numbers must be below 255) + * @return A byte array that can be used + */ + static byte[] toByteArray(int[] intArray) { + byte[] byteArray = new byte[intArray.length]; + for (int i = 0; i < intArray.length; i++) { + if (intArray[i] > Byte.MAX_VALUE - Byte.MIN_VALUE) { + throw new IllegalArgumentException(); + } + byteArray[i] = Integer.valueOf(intArray[i]).byteValue(); + } + return byteArray; + } + + @RegisterExtension + JOSMTestRules josmTestRules = new JOSMTestRules().preferences(); + + private Number bytesToVarInt(int... bytes) { + byte[] byteArray = new byte[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + byteArray[i] = (byte) bytes[i]; + } + return ProtoBufParser.convertByteArray(byteArray, ProtoBufParser.VAR_INT_BYTE_SIZE); + } + + /** + * Test reading tile from Mapillary ( 14/3251/6258 ) + * + * @throws IOException if there is a problem reading the file + */ + @Test + void testRead_14_3251_6258() throws IOException { + File vectorTile = Paths.get(TestUtils.getTestDataRoot(), "pbf", "mapillary", "14", "3251", "6258.mvt").toFile(); + InputStream inputStream = Compression.getUncompressedFileInputStream(vectorTile); + Collection records = new ProtoBufParser(inputStream).allRecords(); + assertEquals(2, records.size()); + List layers = new ArrayList<>(); + for (ProtoBufRecord record : records) { + if (record.getField() == Layer.LAYER_FIELD) { + layers.add(new Layer(record.getBytes())); + } else { + fail(MessageFormat.format("Invalid field {0}", record.getField())); + } + } + Layer mapillarySequences = layers.get(0); + Layer mapillaryPictures = layers.get(1); + assertEquals("mapillary-sequences", mapillarySequences.getName()); + assertEquals("mapillary-images", mapillaryPictures.getName()); + assertEquals(2048, mapillarySequences.getExtent()); + assertEquals(2048, mapillaryPictures.getExtent()); + + assertEquals(1, + mapillarySequences.getFeatures().stream().filter(feature -> feature.getId() == 241083111).count()); + Feature testSequence = mapillarySequences.getFeatures().stream().filter(feature -> feature.getId() == 241083111) + .findAny().orElse(null); + assertEquals("jgxkXqVFM4jepMG3vP5Q9A", testSequence.getTags().get("key")); + assertEquals("C15Ul6qVMfQFlzRcmQCLcA", testSequence.getTags().get("ikey")); + assertEquals("x0hTY8cakpy0m3ui1GaG1A", testSequence.getTags().get("userkey")); + assertEquals(Long.valueOf(1565196718638L), Long.valueOf(testSequence.getTags().get("captured_at"))); + assertEquals(0, Integer.parseInt(testSequence.getTags().get("pano"))); + } + + @Test + void testRead_17_26028_50060() throws IOException { + File vectorTile = Paths.get(TestUtils.getTestDataRoot(), "pbf", "openinframap", "17", "26028", "50060.pbf") + .toFile(); + InputStream inputStream = Compression.getUncompressedFileInputStream(vectorTile); + Collection records = new ProtoBufParser(inputStream).allRecords(); + List layers = new ArrayList<>(); + for (ProtoBufRecord record : records) { + if (record.getField() == Layer.LAYER_FIELD) { + layers.add(new Layer(record.getBytes())); + } else { + fail(MessageFormat.format("Invalid field {0}", record.getField())); + } + } + assertEquals(19, layers.size()); + List dataLayers = layers.stream().filter(layer -> !layer.getFeatures().isEmpty()) + .collect(Collectors.toList()); + // power_plant, power_plant_point, power_generator, power_heatmap_solar, and power_generator_area + assertEquals(5, dataLayers.size()); + + // power_generator_area was rendered incorrectly + final Layer powerGeneratorArea = dataLayers.stream() + .filter(layer -> "power_generator_area".equals(layer.getName())).findAny().orElse(null); + assertNotNull(powerGeneratorArea); + final int extent = powerGeneratorArea.getExtent(); + // 17/26028/50060 bounds + VectorDataSet vectorDataSet = new VectorDataSet(); + MVTTile vectorTile1 = new MVTTile(new MapboxVectorTileSource(new ImageryInfo("Test info", "example.org")), + 26028, 50060, 17); + vectorTile1.loadImage(Compression.getUncompressedFileInputStream(vectorTile)); + vectorDataSet.addTileData(vectorTile1); + vectorDataSet.setZoom(17); + final Way one = new Way(); + one.addNode(new Node(new LatLon(39.0687509, -108.5100816))); + one.addNode(new Node(new LatLon(39.0687509, -108.5095751))); + one.addNode(new Node(new LatLon(39.0687169, -108.5095751))); + one.addNode(new Node(new LatLon(39.0687169, -108.5100816))); + one.addNode(one.getNode(0)); + one.setOsmId(666293899, 2); + final BBox searchBBox = one.getBBox(); + searchBBox.addPrimitive(one, 0.00001); + final Collection searchedNodes = vectorDataSet.searchNodes(searchBBox); + final Collection searchedWays = vectorDataSet.searchWays(searchBBox); + assertEquals(4, searchedNodes.size()); + } + + @Test + void testReadVarInt() { + assertEquals(ProtoBufParser.convertLong(0), bytesToVarInt(0x0)); + assertEquals(ProtoBufParser.convertLong(1), bytesToVarInt(0x1)); + assertEquals(ProtoBufParser.convertLong(127), bytesToVarInt(0x7f)); + // This should b 0xff 0xff 0xff 0xff 0x07, but we drop the leading bit when reading to a byte array + Number actual = bytesToVarInt(0x7f, 0x7f, 0x7f, 0x7f, 0x07); + assertEquals(ProtoBufParser.convertLong(Integer.MAX_VALUE), actual, + MessageFormat.format("Expected {0} but got {1}", Integer.toBinaryString(Integer.MAX_VALUE), + Long.toBinaryString(actual.longValue()))); + } + + /** + * Test simple message. + * Check that a simple message is readable + * + * @throws IOException - if an IO error occurs + */ + @Test + void testSimpleMessage() throws IOException { + ProtoBufParser parser = new ProtoBufParser(new byte[] { (byte) 0x08, (byte) 0x96, (byte) 0x01 }); + ProtoBufRecord record = new ProtoBufRecord(parser); + assertEquals(WireType.VARINT, record.getType()); + assertEquals(150, record.asUnsignedVarInt().intValue()); + } + + @Test + void testSingletonMultiPoint() throws IOException { + Collection records = new ProtoBufParser(new ByteArrayInputStream(toByteArray( + new int[] { 0x1a, 0x2c, 0x78, 0x02, 0x0a, 0x03, 0x74, 0x6d, 0x70, 0x28, 0x80, 0x20, 0x1a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x22, 0x0b, 0x0a, 0x09, 0x54, 0x65, 0x73, 0x74, 0x20, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x0d, 0x18, 0x01, 0x12, 0x02, 0x00, 0x00, 0x22, 0x05, 0x09, 0xe0, 0x3e, 0x84, 0x27 }))) + .allRecords(); + List layers = new ArrayList<>(); + for (ProtoBufRecord record : records) { + if (record.getField() == Layer.LAYER_FIELD) { + layers.add(new Layer(record.getBytes())); + } else { + fail(MessageFormat.format("Invalid field {0}", record.getField())); + } + } + assertEquals(1, layers.size()); + assertEquals(1, layers.get(0).getGeometry().size()); + Ellipse2D shape = (Ellipse2D) layers.get(0).getGeometry().iterator().next().getShapes().iterator().next(); + assertEquals(4016, shape.getCenterX()); + assertEquals(2498, shape.getCenterY()); + } + + @Test + void testZigZag() { + assertEquals(0, ProtoBufParser.decodeZigZag(0).intValue()); + assertEquals(-1, ProtoBufParser.decodeZigZag(1).intValue()); + assertEquals(1, ProtoBufParser.decodeZigZag(2).intValue()); + assertEquals(-2, ProtoBufParser.decodeZigZag(3).intValue()); + } +} Index: test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java =================================================================== --- test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java (nonexistent) +++ test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufTest.java (working copy) @@ -0,0 +1,211 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.fail; + +import java.awt.geom.Ellipse2D; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Paths; +import java.text.MessageFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import org.openstreetmap.josm.TestUtils; +import org.openstreetmap.josm.data.coor.LatLon; +import org.openstreetmap.josm.data.imagery.ImageryInfo; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.Feature; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.Layer; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MVTTile; +import org.openstreetmap.josm.data.imagery.vectortile.mapbox.MapboxVectorTileSource; +import org.openstreetmap.josm.data.osm.BBox; +import org.openstreetmap.josm.data.osm.Node; +import org.openstreetmap.josm.data.osm.Way; +import org.openstreetmap.josm.data.vector.VectorDataSet; +import org.openstreetmap.josm.data.vector.VectorNode; +import org.openstreetmap.josm.data.vector.VectorWay; +import org.openstreetmap.josm.io.Compression; +import org.openstreetmap.josm.testutils.JOSMTestRules; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * Test class for {@link ProtoBufParser} and {@link ProtoBufRecord} + * + * @author Taylor Smock + * @since xxx + */ +class ProtoBufTest { + /** + * Convert an int array into a byte array + * @param intArray The int array to convert (NOTE: numbers must be below 255) + * @return A byte array that can be used + */ + static byte[] toByteArray(int[] intArray) { + byte[] byteArray = new byte[intArray.length]; + for (int i = 0; i < intArray.length; i++) { + if (intArray[i] > Byte.MAX_VALUE - Byte.MIN_VALUE) { + throw new IllegalArgumentException(); + } + byteArray[i] = Integer.valueOf(intArray[i]).byteValue(); + } + return byteArray; + } + + @RegisterExtension + JOSMTestRules josmTestRules = new JOSMTestRules().preferences(); + + private Number bytesToVarInt(int... bytes) { + byte[] byteArray = new byte[bytes.length]; + for (int i = 0; i < bytes.length; i++) { + byteArray[i] = (byte) bytes[i]; + } + return ProtoBufParser.convertByteArray(byteArray, ProtoBufParser.VAR_INT_BYTE_SIZE); + } + + /** + * Test reading tile from Mapillary ( 14/3251/6258 ) + * + * @throws IOException if there is a problem reading the file + */ + @Test + void testRead_14_3251_6258() throws IOException { + File vectorTile = Paths.get(TestUtils.getTestDataRoot(), "pbf", "mapillary", "14", "3251", "6258.mvt").toFile(); + InputStream inputStream = Compression.getUncompressedFileInputStream(vectorTile); + Collection records = new ProtoBufParser(inputStream).allRecords(); + assertEquals(2, records.size()); + List layers = new ArrayList<>(); + for (ProtoBufRecord record : records) { + if (record.getField() == Layer.LAYER_FIELD) { + layers.add(new Layer(record.getBytes())); + } else { + fail(MessageFormat.format("Invalid field {0}", record.getField())); + } + } + Layer mapillarySequences = layers.get(0); + Layer mapillaryPictures = layers.get(1); + assertEquals("mapillary-sequences", mapillarySequences.getName()); + assertEquals("mapillary-images", mapillaryPictures.getName()); + assertEquals(2048, mapillarySequences.getExtent()); + assertEquals(2048, mapillaryPictures.getExtent()); + + assertEquals(1, + mapillarySequences.getFeatures().stream().filter(feature -> feature.getId() == 241083111).count()); + Feature testSequence = mapillarySequences.getFeatures().stream().filter(feature -> feature.getId() == 241083111) + .findAny().orElse(null); + assertEquals("jgxkXqVFM4jepMG3vP5Q9A", testSequence.getTags().get("key")); + assertEquals("C15Ul6qVMfQFlzRcmQCLcA", testSequence.getTags().get("ikey")); + assertEquals("x0hTY8cakpy0m3ui1GaG1A", testSequence.getTags().get("userkey")); + assertEquals(Long.valueOf(1565196718638L), Long.valueOf(testSequence.getTags().get("captured_at"))); + assertEquals(0, Integer.parseInt(testSequence.getTags().get("pano"))); + } + + @Test + void testRead_17_26028_50060() throws IOException { + File vectorTile = Paths.get(TestUtils.getTestDataRoot(), "pbf", "openinframap", "17", "26028", "50060.pbf") + .toFile(); + InputStream inputStream = Compression.getUncompressedFileInputStream(vectorTile); + Collection records = new ProtoBufParser(inputStream).allRecords(); + List layers = new ArrayList<>(); + for (ProtoBufRecord record : records) { + if (record.getField() == Layer.LAYER_FIELD) { + layers.add(new Layer(record.getBytes())); + } else { + fail(MessageFormat.format("Invalid field {0}", record.getField())); + } + } + assertEquals(19, layers.size()); + List dataLayers = layers.stream().filter(layer -> !layer.getFeatures().isEmpty()) + .collect(Collectors.toList()); + // power_plant, power_plant_point, power_generator, power_heatmap_solar, and power_generator_area + assertEquals(5, dataLayers.size()); + + // power_generator_area was rendered incorrectly + final Layer powerGeneratorArea = dataLayers.stream() + .filter(layer -> "power_generator_area".equals(layer.getName())).findAny().orElse(null); + assertNotNull(powerGeneratorArea); + final int extent = powerGeneratorArea.getExtent(); + // 17/26028/50060 bounds + VectorDataSet vectorDataSet = new VectorDataSet(); + MVTTile vectorTile1 = new MVTTile(new MapboxVectorTileSource(new ImageryInfo("Test info", "example.org")), + 26028, 50060, 17); + vectorTile1.loadImage(Compression.getUncompressedFileInputStream(vectorTile)); + vectorDataSet.addTileData(vectorTile1); + vectorDataSet.setZoom(17); + final Way one = new Way(); + one.addNode(new Node(new LatLon(39.0687509, -108.5100816))); + one.addNode(new Node(new LatLon(39.0687509, -108.5095751))); + one.addNode(new Node(new LatLon(39.0687169, -108.5095751))); + one.addNode(new Node(new LatLon(39.0687169, -108.5100816))); + one.addNode(one.getNode(0)); + one.setOsmId(666293899, 2); + final BBox searchBBox = one.getBBox(); + searchBBox.addPrimitive(one, 0.00001); + final Collection searchedNodes = vectorDataSet.searchNodes(searchBBox); + final Collection searchedWays = vectorDataSet.searchWays(searchBBox); + assertEquals(4, searchedNodes.size()); + } + + @Test + void testReadVarInt() { + assertEquals(ProtoBufParser.convertLong(0), bytesToVarInt(0x0)); + assertEquals(ProtoBufParser.convertLong(1), bytesToVarInt(0x1)); + assertEquals(ProtoBufParser.convertLong(127), bytesToVarInt(0x7f)); + // This should b 0xff 0xff 0xff 0xff 0x07, but we drop the leading bit when reading to a byte array + Number actual = bytesToVarInt(0x7f, 0x7f, 0x7f, 0x7f, 0x07); + assertEquals(ProtoBufParser.convertLong(Integer.MAX_VALUE), actual, + MessageFormat.format("Expected {0} but got {1}", Integer.toBinaryString(Integer.MAX_VALUE), + Long.toBinaryString(actual.longValue()))); + } + + /** + * Test simple message. + * Check that a simple message is readable + * + * @throws IOException - if an IO error occurs + */ + @Test + void testSimpleMessage() throws IOException { + ProtoBufParser parser = new ProtoBufParser(new byte[] { (byte) 0x08, (byte) 0x96, (byte) 0x01 }); + ProtoBufRecord record = new ProtoBufRecord(parser); + assertEquals(WireType.VARINT, record.getType()); + assertEquals(150, record.asUnsignedVarInt().intValue()); + } + + @Test + void testSingletonMultiPoint() throws IOException { + Collection records = new ProtoBufParser(new ByteArrayInputStream(toByteArray( + new int[] { 0x1a, 0x2c, 0x78, 0x02, 0x0a, 0x03, 0x74, 0x6d, 0x70, 0x28, 0x80, 0x20, 0x1a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x22, 0x0b, 0x0a, 0x09, 0x54, 0x65, 0x73, 0x74, 0x20, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x0d, 0x18, 0x01, 0x12, 0x02, 0x00, 0x00, 0x22, 0x05, 0x09, 0xe0, 0x3e, 0x84, 0x27 }))) + .allRecords(); + List layers = new ArrayList<>(); + for (ProtoBufRecord record : records) { + if (record.getField() == Layer.LAYER_FIELD) { + layers.add(new Layer(record.getBytes())); + } else { + fail(MessageFormat.format("Invalid field {0}", record.getField())); + } + } + assertEquals(1, layers.size()); + assertEquals(1, layers.get(0).getGeometry().size()); + Ellipse2D shape = (Ellipse2D) layers.get(0).getGeometry().iterator().next().getShapes().iterator().next(); + assertEquals(4016, shape.getCenterX()); + assertEquals(2498, shape.getCenterY()); + } + + @Test + void testZigZag() { + assertEquals(0, ProtoBufParser.decodeZigZag(0).intValue()); + assertEquals(-1, ProtoBufParser.decodeZigZag(1).intValue()); + assertEquals(1, ProtoBufParser.decodeZigZag(2).intValue()); + assertEquals(-2, ProtoBufParser.decodeZigZag(3).intValue()); + } +} Index: test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufParserTest.java =================================================================== --- test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufParserTest.java (nonexistent) +++ test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufParserTest.java (working copy) @@ -0,0 +1,51 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +/** + * Test class for {@link ProtoBufParser} + * @author Taylor Smock + * @since xxx + */ +class ProtoBufParserTest { + /** + * Check that we are appropriately converting values to the "smallest" type + */ + @Test + void testConvertLong() { + // No casting due to auto conversions + assertEquals(Byte.MAX_VALUE, ProtoBufParser.convertLong(Byte.MAX_VALUE)); + assertEquals(Byte.MIN_VALUE, ProtoBufParser.convertLong(Byte.MIN_VALUE)); + assertEquals(Short.MIN_VALUE, ProtoBufParser.convertLong(Short.MIN_VALUE)); + assertEquals(Short.MAX_VALUE, ProtoBufParser.convertLong(Short.MAX_VALUE)); + assertEquals(Integer.MAX_VALUE, ProtoBufParser.convertLong(Integer.MAX_VALUE)); + assertEquals(Integer.MIN_VALUE, ProtoBufParser.convertLong(Integer.MIN_VALUE)); + assertEquals(Long.MIN_VALUE, ProtoBufParser.convertLong(Long.MIN_VALUE)); + assertEquals(Long.MAX_VALUE, ProtoBufParser.convertLong(Long.MAX_VALUE)); + } + + /** + * Check that zig zags are appropriately encoded. + */ + @Test + void testEncodeZigZag() { + assertEquals(0, ProtoBufParser.encodeZigZag(0).byteValue()); + assertEquals(1, ProtoBufParser.encodeZigZag(-1).byteValue()); + assertEquals(2, ProtoBufParser.encodeZigZag(1).byteValue()); + assertEquals(3, ProtoBufParser.encodeZigZag(-2).byteValue()); + assertEquals(254, ProtoBufParser.encodeZigZag(Byte.MAX_VALUE).shortValue()); + assertEquals(255, ProtoBufParser.encodeZigZag(Byte.MIN_VALUE).shortValue()); + assertEquals(65_534, ProtoBufParser.encodeZigZag(Short.MAX_VALUE).intValue()); + assertEquals(65_535, ProtoBufParser.encodeZigZag(Short.MIN_VALUE).intValue()); + // These integers check a possible boundary condition (the boundary between using the 32/64 bit encoding methods) + assertEquals(4_294_967_292L, ProtoBufParser.encodeZigZag(Integer.MAX_VALUE - 1).longValue()); + assertEquals(4_294_967_293L, ProtoBufParser.encodeZigZag(Integer.MIN_VALUE + 1).longValue()); + assertEquals(4_294_967_294L, ProtoBufParser.encodeZigZag(Integer.MAX_VALUE).longValue()); + assertEquals(4_294_967_295L, ProtoBufParser.encodeZigZag(Integer.MIN_VALUE).longValue()); + assertEquals(4_294_967_296L, ProtoBufParser.encodeZigZag(Integer.MAX_VALUE + 1L).longValue()); + assertEquals(4_294_967_297L, ProtoBufParser.encodeZigZag(Integer.MIN_VALUE - 1L).longValue()); + } +} Index: test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufRecordTest.java =================================================================== --- test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufRecordTest.java (nonexistent) +++ test/unit/org/openstreetmap/josm/data/protobuf/ProtoBufRecordTest.java (working copy) @@ -0,0 +1,30 @@ +// License: GPL. For details, see LICENSE file. +package org.openstreetmap.josm.data.protobuf; + +import static org.junit.jupiter.api.Assertions.assertEquals; + + +import java.io.IOException; + +import org.junit.jupiter.api.Test; + +/** + * Test class for specific {@link ProtoBufRecord} functionality + */ +class ProtoBufRecordTest { + @Test + void testFixed32() throws IOException { + ProtoBufParser parser = new ProtoBufParser(ProtoBufTest.toByteArray(new int[] {0x0d, 0x00, 0x00, 0x80, 0x3f})); + ProtoBufRecord thirtyTwoBit = new ProtoBufRecord(parser); + assertEquals(WireType.THIRTY_TWO_BIT, thirtyTwoBit.getType()); + assertEquals(1f, thirtyTwoBit.asFloat()); + } + + @Test + void testUnknown() throws IOException { + ProtoBufParser parser = new ProtoBufParser(ProtoBufTest.toByteArray(new int[] {0x0f, 0x00, 0x00, 0x80, 0x3f})); + ProtoBufRecord unknown = new ProtoBufRecord(parser); + assertEquals(WireType.UNKNOWN, unknown.getType()); + assertEquals(0, unknown.getBytes().length); + } +}