From 14bbe229974b516a82594ec6476488154e8957fa Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 25 Jul 2011 20:42:48 +0000 Subject: First version of decoding git-svn-id: https://svn.kapsi.fi/jpa/nanopb@942 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb.h | 107 ++++++++++++++++++++ pb_decode.c | 274 +++++++++++++++++++++++++++++++++++++++++++++++++++ pb_decode.h | 55 +++++++++++ tests/person.proto | 18 ++++ tests/test_decode1.c | 102 +++++++++++++++++++ tests/testperson.pb | 3 + 6 files changed, 559 insertions(+) create mode 100644 pb.h create mode 100644 pb_decode.c create mode 100644 pb_decode.h create mode 100644 tests/person.proto create mode 100644 tests/test_decode1.c create mode 100644 tests/testperson.pb diff --git a/pb.h b/pb.h new file mode 100644 index 00000000..2cae75c5 --- /dev/null +++ b/pb.h @@ -0,0 +1,107 @@ +#ifndef _PB_H_ +#define _PB_H_ + +#include +#include // size_t + +/* Lightweight input stream. + * If buf is NULL, read but don't store bytes. */ +typedef struct _pb_istream_t pb_istream_t; +struct _pb_istream_t +{ + bool (*callback)(pb_istream_t *stream, char *buf, size_t count); + void *state; // Free field for use by callback implementation + size_t bytes_left; +}; + +static inline bool pb_read(pb_istream_t *stream, char *buf, size_t count) +{ + bool status = stream->callback(stream, buf, count); + stream->bytes_left -= count; + return status; +} + +/* Lightweight output stream. */ +typedef struct _pb_ostream_t pb_ostream_t; +struct _pb_ostream_t +{ + bool (*callback)(pb_ostream_t *stream, const char *buf, size_t count); + void *state; // Free field for use by callback implementation + size_t bytes_written; +}; + +static inline bool pb_write(pb_ostream_t *stream, const char *buf, size_t count) +{ + bool status = stream->callback(stream, buf, count); + stream->bytes_written += count; + return status; +} + +/* List of possible decode/encode action types */ + +typedef enum { + // Special case. Sets boolean field to true, continues parsing the data. + PB_ACT_HAS, + + // Standard integer types + PB_ACT_UINT32, + PB_ACT_SINT32, + PB_ACT_INT32, + PB_ACT_FIXED32, + PB_ACT_SFIXED32, + PB_ACT_UINT64, + PB_ACT_SINT64, + PB_ACT_INT64, + PB_ACT_FIXED64, + PB_ACT_SFIXED64, + PB_ACT_BOOL, + + // Standard float types + PB_ACT_FLOAT, + PB_ACT_DOUBLE, + + // Constant-sized array + PB_ACT_BYTES, + + // Constant-sized array, with null termination + PB_ACT_STRING, + + // Callback function pointer in field value + PB_ACT_SUBMESSAGE, + + PB_LAST_ACT +} pb_action_t; + +// This structure is used in constants to specify struct fields. +typedef struct { + int field_number; + uint16_t offset; + pb_action_t action; + uint8_t fieldsize; +} pb_field_t; + +#define PB_LAST_FIELD {0,0,0,0} + +/* --- Types to use inside generated structures. --- */ + +// Byte array and size word. +// Note: because of variable length array, this type cannot be directly used. +// Autogenerated code declares the same type of fields but with explicit length. +typedef struct { + size_t size; + char bytes[]; +} pb_bytearray_t; + +// This structure is used for giving the callback function. +typedef struct _pb_callback_t pb_callback_t; +struct _pb_callback_t { + union { + bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void *arg); + bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, void *arg); + } funcs; + + // Free arg for use by callback + void *arg; +}; + +#endif \ No newline at end of file diff --git a/pb_decode.c b/pb_decode.c new file mode 100644 index 00000000..884cf18e --- /dev/null +++ b/pb_decode.c @@ -0,0 +1,274 @@ +/* pb_decode.c -- decode a protobuf using callback functions + * + * 2011 Petteri Aimonen + */ + +#include "pb_decode.h" + +const pb_decoder_t PB_DECODERS[PB_LAST_ACT] = { + NULL, + &pb_dec_uint32, + &pb_dec_sint32, + &pb_dec_uint32, // Cast to int32 + &pb_dec_fixed32, + &pb_dec_fixed32, // Cast to int32 + &pb_dec_uint64, + &pb_dec_sint64, + &pb_dec_uint64, // Cast to int64 + &pb_dec_fixed64, + &pb_dec_fixed64, // Cast to int64 + &pb_dec_bool, + &pb_dec_float, + &pb_dec_double, + &pb_dec_bytes, + &pb_dec_string, + &pb_dec_submessage +}; + +enum wire_type { + WT_VARINT = 0, + WT_64BIT = 1, + WT_STRING = 2, + WT_32BIT = 5 +}; + +// Note: pb_decode_varint32 is a bit un-orthodox: +// it will refuse to decode values that exceed uint32 range. +// The Google implementation would simply cast to 32 bits. +bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) +{ + char byte; + int bitpos = 0; + *dest = 0; + + while (bitpos < 32 && pb_read(stream, &byte, 1)) + { + *dest |= (byte & 0x7F) << bitpos; + bitpos += 7; + + if (!(byte & 0x80)) + return true; + } + + return false; +} + +bool pb_decode_varint64(pb_istream_t *stream, uint64_t *dest) +{ + char byte; + int bitpos = 0; + *dest = 0; + + while (bitpos < 64 && pb_read(stream, &byte, 1)) + { + *dest |= (byte & 0x7F) << bitpos; + bitpos += 7; + + if (!(byte & 0x80)) + return true; + } + + return false; +} + +bool pb_skip_varint(pb_istream_t *stream) +{ + char byte; + do + { + if (!pb_read(stream, &byte, 1)) + return false; + } while (byte & 0x80); + return true; +} + +bool pb_skip_string(pb_istream_t *stream) +{ + uint32_t length; + if (!pb_decode_varint32(stream, &length)) + return false; + + return pb_read(stream, NULL, length); +} + +bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest) +{ + while (stream->bytes_left) + { + uint32_t temp; + if (!pb_decode_varint32(stream, &temp)) + return false; + + int field_number = temp >> 3; + int wire_type = temp & 7; + + const pb_field_t *field = fields; + while (field->field_number != 0) + { + if (field->field_number != field_number) + { + field++; + continue; + } + + void *destfield = dest + field->offset; + + if (field->action == PB_ACT_HAS) + { + *(bool*)destfield = true; + field++; + continue; + } + + pb_decoder_t func = PB_DECODERS[field->action]; + if (!func(stream, field, destfield)) + return false; + + break; + } + + if (field->field_number == 0) // No match found, skip data + { + bool status = false; + switch (wire_type) + { + case WT_VARINT: + status = pb_skip_varint(stream); + break; + + case WT_64BIT: + status = pb_read(stream, NULL, 8); + break; + + case WT_STRING: + status = pb_skip_string(stream); + break; + + case WT_32BIT: + status = pb_read(stream, NULL, 4); + break; + } + + if (!status) + return false; + } + } + + return true; +} + +bool pb_dec_uint32(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + return pb_decode_varint32(stream, (uint32_t*)dest); +} + +bool pb_dec_sint32(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + uint32_t *x = (uint32_t*)dest; + bool status = pb_decode_varint32(stream, x); + *x = (*x >> 1) ^ -(int32_t)(*x & 1); + return status; +} + +bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + char bytes[4] = {0}; + bool status = pb_read(stream, bytes, 4); + *(uint32_t*)dest = + bytes[0] | ((uint32_t)bytes[1] << 8) | + ((uint32_t)bytes[2] << 16) | ((uint32_t)bytes[3] << 24); + return status; +} + +bool pb_dec_uint64(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + return pb_decode_varint64(stream, (uint64_t*)dest); +} + +bool pb_dec_sint64(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + uint64_t *x = (uint64_t*)dest; + bool status = pb_decode_varint64(stream, x); + *x = (*x >> 1) ^ -(int64_t)(*x & 1); + return status; +} + +bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + char bytes[8] = {0}; + bool status = pb_read(stream, bytes, 8); + *(uint64_t*)dest = + (uint64_t)bytes[0] | ((uint64_t)bytes[1] << 8) | + ((uint64_t)bytes[2] << 16) | ((uint64_t)bytes[3] << 24) | + ((uint64_t)bytes[4] << 32) | ((uint64_t)bytes[5] << 40) | + ((uint64_t)bytes[6] << 48) | ((uint64_t)bytes[7] << 56); + return status; +} + +bool pb_dec_bool(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + uint32_t temp = 0; + bool status = pb_decode_varint32(stream, &temp); + *(bool*)dest = !!temp; + return status; +} + +bool pb_dec_float(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + return pb_read(stream, (char*)dest, sizeof(float)); +} + +bool pb_dec_double(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + return pb_read(stream, (char*)dest, sizeof(double)); +} + +bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + pb_bytearray_t *x = (pb_bytearray_t*)dest; + uint32_t temp; + if (!pb_decode_varint32(stream, &temp)) + return false; + x->size = temp; + + if (x->size > field->fieldsize) + return false; + + return pb_read(stream, x->bytes, x->size); +} + +bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + uint32_t size; + if (!pb_decode_varint32(stream, &size)) + return false; + + if (size > field->fieldsize - 1) + return false; + + bool status = pb_read(stream, (char*)dest, size); + *((char*)dest + size) = 0; + return status; +} + +bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + pb_callback_t *x = (pb_callback_t*)dest; + + if (x->funcs.decode == NULL) + return pb_skip_string(stream); + + uint32_t size; + if (!pb_decode_varint32(stream, &size)) + return false; + + if (stream->bytes_left < size) + return false; + + // Make a limited-length istream for decoding submessage + pb_istream_t shortstream = *stream; + shortstream.bytes_left = size; + bool status = x->funcs.decode(&shortstream, field, x->arg); + stream->bytes_left -= size - shortstream.bytes_left; + return status; +} diff --git a/pb_decode.h b/pb_decode.h new file mode 100644 index 00000000..b71e20b1 --- /dev/null +++ b/pb_decode.h @@ -0,0 +1,55 @@ +#ifndef _PB_DECODE_H_ +#define _PB_DECODE_H_ + +#include +#include "pb.h" + +// Decode from stream to destination struct. +// The actual struct pointed to by dest must match the description in fields. +bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest); + +/* --- Helper functions --- + * You may want to use these from your callbacks. + */ + +bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest); +bool pb_decode_varint64(pb_istream_t *stream, uint64_t *dest); + +bool pb_skip_varint(pb_istream_t *stream); +bool pb_skip_string(pb_istream_t *stream); + +/* --- Field decoders --- + * Each decoder takes stream and field description, and a pointer to the field + * in the destination struct (dest = struct_addr + field->offset). + */ + +// Integer types. +bool pb_dec_uint32(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_sint32(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_uint64(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_sint64(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_bool(pb_istream_t *stream, const pb_field_t *field, void *dest); + +// Floating point types. Info is ignored. +bool pb_dec_float(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_double(pb_istream_t *stream, const pb_field_t *field, void *dest); + +// Byte array. Dest is pointer to +bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); + +// Null-terminated string. +bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); + +// Use callback. Dest is pointer to pb_callback_t struct. +bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest); + +typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest); + +/* --- Function pointers to field decoders --- + * Order in the array must match pb_action_t numbering. + */ +const pb_decoder_t PB_DECODERS[PB_LAST_ACT]; + +#endif diff --git a/tests/person.proto b/tests/person.proto new file mode 100644 index 00000000..ec5b4cca --- /dev/null +++ b/tests/person.proto @@ -0,0 +1,18 @@ +message Person { + required string name = 1; + required int32 id = 2; + optional string email = 3; + + enum PhoneType { + MOBILE = 0; + HOME = 1; + WORK = 2; + } + + message PhoneNumber { + required string number = 1; + optional PhoneType type = 2 [default = HOME]; + } + + repeated PhoneNumber phone = 4; +} diff --git a/tests/test_decode1.c b/tests/test_decode1.c new file mode 100644 index 00000000..dc0831a2 --- /dev/null +++ b/tests/test_decode1.c @@ -0,0 +1,102 @@ +#include +#include +#include +#include "pb_decode.h" + +/* Structures for "Person" message */ + +typedef enum { + Person_PhoneType_MOBILE = 0, + Person_PhoneType_HOME = 1, + Person_PhoneType_WORK = 2, + + _Person_PhoneType_size = 0xFFFFFFFF // Force 32-bit enum +} Person_PhoneType; + +typedef struct { + char number[40]; + bool has_type; + Person_PhoneType type; +} Person_PhoneNumber; + +typedef struct { + char name[40]; + int32_t id; + bool has_email; + char email[40]; + + pb_callback_t phone; +} Person; + +/* Field descriptions */ +#define membersize(st, m) (sizeof ((st*)0)->m) + +const pb_field_t Person_PhoneNumber_fields[] = { + {1, offsetof(Person_PhoneNumber, number), PB_ACT_STRING, membersize(Person_PhoneNumber, number)}, + {2, offsetof(Person_PhoneNumber, has_type), PB_ACT_HAS, membersize(Person_PhoneNumber, has_type)}, + {2, offsetof(Person_PhoneNumber, type), PB_ACT_UINT32, membersize(Person_PhoneNumber, type)}, + PB_LAST_FIELD +}; + +const pb_field_t Person_fields[] = { + {1, offsetof(Person, name), PB_ACT_STRING, membersize(Person, name)}, + {2, offsetof(Person, id), PB_ACT_INT32, membersize(Person, id)}, + {3, offsetof(Person, email), PB_ACT_STRING, membersize(Person, email)}, + {4, offsetof(Person, phone), PB_ACT_SUBMESSAGE, membersize(Person, phone)} +}; + +/* Default value descriptions */ +#define Person_PhoneNumber_default {"", false, Person_PhoneType_HOME}; +#define Person_default {"", 0, false, "", {{0},0}}; + +/* And now, the actual test program */ + +bool print_phonenumber(pb_istream_t *stream, const pb_field_t *field, void *arg) +{ + Person_PhoneNumber x = Person_PhoneNumber_default; + if (!pb_decode(stream, Person_PhoneNumber_fields, &x)) + return false; + + printf("PhoneNumber: number '%s' type '%d'\n", x.number, x.type); + return true; +} + +bool print_person(pb_istream_t *stream) +{ + Person x = Person_default; + x.phone.funcs.decode = &print_phonenumber; + + if (!pb_decode(stream, Person_fields, &x)) + return false; + + printf("Person: name '%s' id '%d' email '%s'\n", x.name, x.id, x.email); + return true; +} + +bool my_read(pb_istream_t *stream, char *buf, size_t count) +{ + char *source = (char*)stream->state; + + if (!stream->bytes_left) + return false; + + if (buf != NULL) + { + memcpy(buf, source, count); + } + + stream->state = source + count; + return true; +} + +int main() +{ + char buffer[512]; + size_t size = fread(buffer, 1, 512, stdin); + + pb_istream_t stream = {&my_read, buffer, size}; + if (!print_person(&stream)) + printf("Parsing failed.\n"); + + return 0; +} diff --git a/tests/testperson.pb b/tests/testperson.pb new file mode 100644 index 00000000..d1eb8cfb --- /dev/null +++ b/tests/testperson.pb @@ -0,0 +1,3 @@ + + Test Person7foobar@foobar.com" + 555-12345678 \ No newline at end of file -- cgit 1.2.3-korg From 84304b343a4b06a7918bc80ab45f08d1cae0b9cb Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 27 Jul 2011 19:22:11 +0000 Subject: Improvements, array support git-svn-id: https://svn.kapsi.fi/jpa/nanopb@943 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb.h | 162 +++++++++++++++------ pb_decode.c | 387 +++++++++++++++++++++++++++++++++++++-------------- pb_decode.h | 21 ++- tests/test_decode1.c | 79 ++++++----- 4 files changed, 455 insertions(+), 194 deletions(-) diff --git a/pb.h b/pb.h index 2cae75c5..99e4bf7e 100644 --- a/pb.h +++ b/pb.h @@ -3,19 +3,36 @@ #include #include // size_t +#include + +#ifdef __GNUC__ +// This just reduces memory requirements, but is not required. +#define pb_packed __attribute__((packed)) +#else +#define pb_packed +#endif /* Lightweight input stream. - * If buf is NULL, read but don't store bytes. */ + * If buf is NULL, read but don't store bytes. + * You have to provide a callback function for reading. + * You can use state to store your own data (e.g. buffer pointer), + * and rely on pb_read to verify that no-body reads past bytes_left. + * However, substreams may change bytes_left so don't use that to + * compute any pointers. + */ typedef struct _pb_istream_t pb_istream_t; struct _pb_istream_t { - bool (*callback)(pb_istream_t *stream, char *buf, size_t count); + bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count); void *state; // Free field for use by callback implementation size_t bytes_left; }; -static inline bool pb_read(pb_istream_t *stream, char *buf, size_t count) +static inline bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) { + if (stream->bytes_left < count) + return false; + bool status = stream->callback(stream, buf, count); stream->bytes_left -= count; return status; @@ -25,74 +42,129 @@ static inline bool pb_read(pb_istream_t *stream, char *buf, size_t count) typedef struct _pb_ostream_t pb_ostream_t; struct _pb_ostream_t { - bool (*callback)(pb_ostream_t *stream, const char *buf, size_t count); + bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); void *state; // Free field for use by callback implementation size_t bytes_written; }; -static inline bool pb_write(pb_ostream_t *stream, const char *buf, size_t count) +static inline bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) { bool status = stream->callback(stream, buf, count); stream->bytes_written += count; return status; } -/* List of possible decode/encode action types */ +/* List of possible field types + * Least-significant 4 bits tell the scalar type + * Most-significant 4 bits specify repeated/required/packed etc. + * + * INT32 and UINT32 are treated the same, as are (U)INT64 and (S)FIXED* + * These types are simply casted to correct field type when they are + * assigned to the memory pointer. + * SINT* is different, though, because it is zig-zag coded. + */ typedef enum { - // Special case. Sets boolean field to true, continues parsing the data. - PB_ACT_HAS, - // Standard integer types - PB_ACT_UINT32, - PB_ACT_SINT32, - PB_ACT_INT32, - PB_ACT_FIXED32, - PB_ACT_SFIXED32, - PB_ACT_UINT64, - PB_ACT_SINT64, - PB_ACT_INT64, - PB_ACT_FIXED64, - PB_ACT_SFIXED64, - PB_ACT_BOOL, + PB_LTYPE_UINT32 = 0x00, + PB_LTYPE_INT32 = 0x00, + PB_LTYPE_SINT32 = 0x01, + PB_LTYPE_FIXED32 = 0x02, + PB_LTYPE_SFIXED32 = 0x02, + PB_LTYPE_UINT64 = 0x03, + PB_LTYPE_INT64 = 0x03, + PB_LTYPE_SINT64 = 0x04, + PB_LTYPE_FIXED64 = 0x05, + PB_LTYPE_SFIXED64 = 0x05, + PB_LTYPE_BOOL = 0x06, + PB_LTYPE_ENUM = 0x07, // Standard float types - PB_ACT_FLOAT, - PB_ACT_DOUBLE, + PB_LTYPE_FLOAT = 0x08, + PB_LTYPE_DOUBLE = 0x09, + + // Byte array with pre-allocated buffer. + // data_size is the length of the allocated PB_BYTES_ARRAY structure. + PB_LTYPE_BYTES = 0x0A, + + // String with pre-allocated buffer. + // data_size is the maximum length. + PB_LTYPE_STRING = 0x0B, - // Constant-sized array - PB_ACT_BYTES, + // Submessage + // submsg_fields is pointer to field descriptions + PB_LTYPE_SUBMESSAGE = 0x0C, - // Constant-sized array, with null termination - PB_ACT_STRING, + ///////////// + // Modifier flags - // Callback function pointer in field value - PB_ACT_SUBMESSAGE, + // Just the basic, write data at data_offset + PB_HTYPE_REQUIRED = 0x00, - PB_LAST_ACT -} pb_action_t; + // Write true at size_offset + PB_HTYPE_OPTIONAL = 0x10, + + // Read to pre-allocated array + // Maximum number of entries is array_size, + // actual number is stored at size_offset + PB_HTYPE_ARRAY = 0x20, + + // Works for all required/optional/repeated fields. + // data_offset points to pb_callback_t structure. + // LTYPE is ignored. + PB_HTYPE_CALLBACK = 0x30 +} pb_packed pb_type_t; -// This structure is used in constants to specify struct fields. -typedef struct { - int field_number; - uint16_t offset; - pb_action_t action; - uint8_t fieldsize; -} pb_field_t; +#define PB_HTYPE(x) ((x) & 0xF0) +#define PB_LTYPE(x) ((x) & 0x0F) + +/* This structure is used in auto-generated constants + * to specify struct fields. + * You can change field sizes here if you need structures + * larger than 256 bytes or field tags larger than 256. + * The compiler should complain if your .proto has such + * structures ("initializer too large for type"). + */ +typedef struct _pb_field_t pb_field_t; +struct _pb_field_t { + uint8_t tag; + pb_type_t type; + uint8_t data_offset; // Offset of actual data or array start + uint8_t size_offset; // Offset of array size or has-boolean + uint8_t data_size; // Data size in bytes for a single item + uint8_t array_size; // Maximum number of entries in array + + // Field definitions for submessage + // OR default value for all other non-array, non-callback types + // If null, then field will zeroed. + const void *ptr; +} pb_packed; #define PB_LAST_FIELD {0,0,0,0} -/* --- Types to use inside generated structures. --- */ +// This structure is used for 'bytes' arrays. +// It has the number of bytes in the beginning, and after that an array. +#define PB_BYTES_ARRAY(buffersize) \ +struct { \ + size_t size; \ + uint8_t bytes[buffersize]; \ +} -// Byte array and size word. -// Note: because of variable length array, this type cannot be directly used. -// Autogenerated code declares the same type of fields but with explicit length. -typedef struct { - size_t size; - char bytes[]; -} pb_bytearray_t; +typedef PB_BYTES_ARRAY(1) pb_bytes_array_t; // This structure is used for giving the callback function. +// +// The decoding callback will be given a limited-length stream +// If the wire type was string, the length is the length of the string. +// If the wire type was a varint/fixed32/fixed64, the length is the length +// of the actual value. +// The function may be called multiple times (especially for repeated types, +// but also otherwise if the message happens to contain the field multiple +// times.) +// +// The encoding callback will receive the actual output stream. +// It should write all the data in one call, including the field tag and +// wire type. It can write multiple fields. typedef struct _pb_callback_t pb_callback_t; struct _pb_callback_t { union { diff --git a/pb_decode.c b/pb_decode.c index 884cf18e..acd9b204 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -1,61 +1,63 @@ -/* pb_decode.c -- decode a protobuf using callback functions +/* pb_decode.c -- decode a protobuf using minimal resources * * 2011 Petteri Aimonen */ +#include "pb.h" #include "pb_decode.h" +#include -const pb_decoder_t PB_DECODERS[PB_LAST_ACT] = { - NULL, +const pb_decoder_t PB_DECODERS[16] = { &pb_dec_uint32, &pb_dec_sint32, - &pb_dec_uint32, // Cast to int32 &pb_dec_fixed32, - &pb_dec_fixed32, // Cast to int32 &pb_dec_uint64, &pb_dec_sint64, - &pb_dec_uint64, // Cast to int64 &pb_dec_fixed64, - &pb_dec_fixed64, // Cast to int64 &pb_dec_bool, + &pb_dec_enum, + &pb_dec_float, &pb_dec_double, + &pb_dec_bytes, &pb_dec_string, &pb_dec_submessage }; -enum wire_type { - WT_VARINT = 0, - WT_64BIT = 1, - WT_STRING = 2, - WT_32BIT = 5 -}; +//////////////////////// +// Helper functions +//////////////////////// -// Note: pb_decode_varint32 is a bit un-orthodox: -// it will refuse to decode values that exceed uint32 range. -// The Google implementation would simply cast to 32 bits. -bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) +static bool buf_read(pb_istream_t *stream, uint8_t *buf, size_t count) { - char byte; - int bitpos = 0; - *dest = 0; + uint8_t *source = (uint8_t*)stream->state; - while (bitpos < 32 && pb_read(stream, &byte, 1)) - { - *dest |= (byte & 0x7F) << bitpos; - bitpos += 7; - - if (!(byte & 0x80)) - return true; - } + if (buf != NULL) + memcpy(buf, source, count); - return false; + stream->state = source + count; + return true; +} + +pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize) +{ + pb_istream_t stream = {&buf_read, buf, bufsize}; + return stream; +} + +bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) +{ + uint64_t temp; + if (!pb_decode_varint64(stream, &temp)) + return false; + *dest = temp; + return true; } bool pb_decode_varint64(pb_istream_t *stream, uint64_t *dest) { - char byte; + uint8_t byte; int bitpos = 0; *dest = 0; @@ -73,7 +75,7 @@ bool pb_decode_varint64(pb_istream_t *stream, uint64_t *dest) bool pb_skip_varint(pb_istream_t *stream) { - char byte; + uint8_t byte; do { if (!pb_read(stream, &byte, 1)) @@ -91,72 +93,235 @@ bool pb_skip_string(pb_istream_t *stream) return pb_read(stream, NULL, length); } -bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest) +/* Currently all wire type related stuff is kept hidden from + * callbacks. They shouldn't need it. It's better for performance + * to just assume the correct type and fail safely on corrupt message. + */ + +enum wire_type_t { + WT_VARINT = 0, + WT_64BIT = 1, + WT_STRING = 2, + WT_32BIT = 5 +}; + +static bool skip(pb_istream_t *stream, int wire_type) { - while (stream->bytes_left) + switch (wire_type) { - uint32_t temp; - if (!pb_decode_varint32(stream, &temp)) - return false; + case WT_VARINT: return pb_skip_varint(stream); + case WT_64BIT: return pb_read(stream, NULL, 8); + case WT_STRING: return pb_skip_string(stream); + case WT_32BIT: return pb_read(stream, NULL, 4); + default: return false; + } +} + +// Read a raw value to buffer, for the purpose of passing it to callback. +// Size is maximum size on call, and actual size on return. +static bool read_raw_value(pb_istream_t *stream, int wire_type, uint8_t *buf, size_t *size) +{ + size_t max_size = *size; + switch (wire_type) + { + case WT_VARINT: + *size = 0; + do + { + (*size)++; + if (*size > max_size) return false; + if (!pb_read(stream, buf++, 1)) return false; + } while (*buf & 0x80); + return true; + + case WT_64BIT: + *size = 8; + return pb_read(stream, buf, 8); - int field_number = temp >> 3; - int wire_type = temp & 7; + case WT_32BIT: + *size = 4; + return pb_read(stream, buf, 4); - const pb_field_t *field = fields; - while (field->field_number != 0) - { - if (field->field_number != field_number) + default: return false; + } +} + +// Decode string length from stream and return a substream with limited length +static bool make_string_substream(pb_istream_t *stream, pb_istream_t *substream) +{ + uint32_t size; + if (!pb_decode_varint32(stream, &size)) + return false; + + *substream = *stream; + if (substream->bytes_left < size) + return false; + + substream->bytes_left = size; + stream->bytes_left -= size; + return true; +} + +bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, void *dest_struct) +{ + pb_decoder_t func = PB_DECODERS[PB_LTYPE(field->type)]; + void *pData = (char*)dest_struct + field->data_offset; + void *pSize = (char*)dest_struct + field->size_offset; + + switch (PB_HTYPE(field->type)) + { + case PB_HTYPE_REQUIRED: + return func(stream, field, pData); + + case PB_HTYPE_OPTIONAL: + *(bool*)pSize = true; + return func(stream, field, pData); + + case PB_HTYPE_ARRAY: + if (wire_type == WT_STRING + && PB_LTYPE(field->type) != PB_LTYPE_BYTES + && PB_LTYPE(field->type) != PB_LTYPE_STRING + && PB_LTYPE(field->type) != PB_LTYPE_SUBMESSAGE) { - field++; - continue; + // Packed array + size_t *size = (size_t*)pSize; + pb_istream_t substream; + if (!make_string_substream(stream, &substream)) + return false; + + while (substream.bytes_left && *size < field->array_size) + { + void *pItem = pData + field->data_size * (*size); + if (!func(stream, field, pItem)) + return false; + (*size)++; + } + return (substream.bytes_left == 0); } - - void *destfield = dest + field->offset; - - if (field->action == PB_ACT_HAS) + else { - *(bool*)destfield = true; - field++; - continue; + // Repeated field + size_t *size = (size_t*)pSize; + if (*size >= field->array_size) + return false; + + void *pItem = pData + field->data_size * (*size); + (*size)++; + return func(stream, field, pItem); } - - pb_decoder_t func = PB_DECODERS[field->action]; - if (!func(stream, field, destfield)) - return false; - - break; - } - if (field->field_number == 0) // No match found, skip data - { - bool status = false; - switch (wire_type) + case PB_HTYPE_CALLBACK: + if (wire_type == WT_STRING) { - case WT_VARINT: - status = pb_skip_varint(stream); - break; + pb_callback_t *pCallback = (pb_callback_t*)pData; + pb_istream_t substream; - case WT_64BIT: - status = pb_read(stream, NULL, 8); - break; + if (!make_string_substream(stream, &substream)) + return false; - case WT_STRING: - status = pb_skip_string(stream); - break; + while (substream.bytes_left) + { + if (!pCallback->funcs.decode(&substream, field, pCallback->arg)) + return false; + } + } + else + { + // Copy the single scalar value to stack. + // This is required so that we can limit the stream length, + // which in turn allows to use same callback for packed and + // not-packed fields. + uint8_t buffer[10]; + size_t size = sizeof(buffer); + if (!read_raw_value(stream, wire_type, buffer, &size)) + return false; + pb_istream_t substream = pb_istream_from_buffer(buffer, size); - case WT_32BIT: - status = pb_read(stream, NULL, 4); - break; + pb_callback_t *pCallback = (pb_callback_t*)pData; + return pCallback->funcs.decode(&substream, field, pCallback->arg); } - if (!status) - return false; + default: + return false; + } +} + +bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + // Used to check for required fields + uint32_t fields_seen = 0; + int i; + + // Initialize size/has fields and apply default values + for (i = 0; fields[i].tag != 0; i++) + { + void *pData = (char*)dest_struct + fields[i].data_offset; + void *pSize = (char*)dest_struct + fields[i].size_offset; + if (PB_HTYPE(fields[i].type) == PB_HTYPE_OPTIONAL) + { + *(bool*)pSize = false; + } + else if (PB_HTYPE(fields[i].type) == PB_HTYPE_ARRAY) + { + *(size_t*)pSize = 0; + } + + if (PB_HTYPE(fields[i].type) != PB_HTYPE_ARRAY && + PB_HTYPE(fields[i].type) != PB_HTYPE_CALLBACK) + { + if (fields[i].ptr != NULL) + { + memcpy(pData, fields[i].ptr, fields[i].data_size); + } + else + { + memset(pData, 0, fields[i].data_size); + } + } + } + + while (stream->bytes_left) + { + uint32_t temp; + if (!pb_decode_varint32(stream, &temp)) + return false; + + int tag = temp >> 3; + int wire_type = temp & 7; + + i = 0; + while (fields[i].tag != 0 && fields[i].tag != tag) + { + i++; + } + + if (fields[i].tag == 0) // No match found, skip data + { + skip(stream, wire_type); + continue; + } + + fields_seen |= 1 << (i & 31); + + if (!decode_field(stream, wire_type, &fields[i], dest_struct)) + return false; + } + + // Check that all required fields (mod 31) were present. + for (i = 0; fields[i].tag != 0; i++) + { + if (PB_HTYPE(fields[i].type) == PB_HTYPE_REQUIRED && + !(fields_seen & (1 << (i & 31)))) + { + return false; } } return true; } +/* Field decoders */ + bool pb_dec_uint32(pb_istream_t *stream, const pb_field_t *field, void *dest) { return pb_decode_varint32(stream, (uint32_t*)dest); @@ -172,11 +337,15 @@ bool pb_dec_sint32(pb_istream_t *stream, const pb_field_t *field, void *dest) bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest) { - char bytes[4] = {0}; + uint8_t bytes[4] = {0}; bool status = pb_read(stream, bytes, 4); - *(uint32_t*)dest = - bytes[0] | ((uint32_t)bytes[1] << 8) | - ((uint32_t)bytes[2] << 16) | ((uint32_t)bytes[3] << 24); + +#ifdef __BIG_ENDIAN__ + uint8_t lebytes[4] = {bytes[3], bytes[2], bytes[1], bytes[0]}; + memcpy(dest, lebytes, 4); +#else + memcpy(dest, bytes, 4); +#endif return status; } @@ -195,13 +364,16 @@ bool pb_dec_sint64(pb_istream_t *stream, const pb_field_t *field, void *dest) bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest) { - char bytes[8] = {0}; + uint8_t bytes[8] = {0}; bool status = pb_read(stream, bytes, 8); - *(uint64_t*)dest = - (uint64_t)bytes[0] | ((uint64_t)bytes[1] << 8) | - ((uint64_t)bytes[2] << 16) | ((uint64_t)bytes[3] << 24) | - ((uint64_t)bytes[4] << 32) | ((uint64_t)bytes[5] << 40) | - ((uint64_t)bytes[6] << 48) | ((uint64_t)bytes[7] << 56); + +#ifdef __BIG_ENDIAN__ + uint8_t lebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], + bytes[3], bytes[2], bytes[1], bytes[0]}; + memcpy(dest, lebytes, 4); +#else + memcpy(dest, bytes, 4); +#endif return status; } @@ -213,25 +385,37 @@ bool pb_dec_bool(pb_istream_t *stream, const pb_field_t *field, void *dest) return status; } +bool pb_dec_enum(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + // Enum sizes can vary, copy only data_size amount of bytes. + uint32_t temp = 0; + bool status = pb_decode_varint32(stream, &temp); + memcpy(dest, &temp, field->data_size); + return status; +} + bool pb_dec_float(pb_istream_t *stream, const pb_field_t *field, void *dest) { - return pb_read(stream, (char*)dest, sizeof(float)); + return pb_read(stream, (uint8_t*)dest, sizeof(float)); } bool pb_dec_double(pb_istream_t *stream, const pb_field_t *field, void *dest) { - return pb_read(stream, (char*)dest, sizeof(double)); + return pb_read(stream, (uint8_t*)dest, sizeof(double)); } bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) { - pb_bytearray_t *x = (pb_bytearray_t*)dest; + pb_bytes_array_t *x = (pb_bytes_array_t*)dest; + uint32_t temp; if (!pb_decode_varint32(stream, &temp)) return false; x->size = temp; - if (x->size > field->fieldsize) + // Note: data_size includes the size of the x.size field, too. + // Calculate actual size starting from offset. + if (x->size > field->data_size - offsetof(pb_bytes_array_t, bytes)) return false; return pb_read(stream, x->bytes, x->size); @@ -243,32 +427,23 @@ bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest) if (!pb_decode_varint32(stream, &size)) return false; - if (size > field->fieldsize - 1) + if (size > field->data_size - 1) return false; - bool status = pb_read(stream, (char*)dest, size); - *((char*)dest + size) = 0; + bool status = pb_read(stream, (uint8_t*)dest, size); + *((uint8_t*)dest + size) = 0; return status; } bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) { - pb_callback_t *x = (pb_callback_t*)dest; + pb_istream_t substream; - if (x->funcs.decode == NULL) - return pb_skip_string(stream); - - uint32_t size; - if (!pb_decode_varint32(stream, &size)) + if (!make_string_substream(stream, &substream)) return false; - if (stream->bytes_left < size) + if (field->ptr == NULL) return false; - // Make a limited-length istream for decoding submessage - pb_istream_t shortstream = *stream; - shortstream.bytes_left = size; - bool status = x->funcs.decode(&shortstream, field, x->arg); - stream->bytes_left -= size - shortstream.bytes_left; - return status; + return pb_decode(&substream, (pb_field_t*)field->ptr, dest); } diff --git a/pb_decode.h b/pb_decode.h index b71e20b1..2f689394 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -6,12 +6,14 @@ // Decode from stream to destination struct. // The actual struct pointed to by dest must match the description in fields. -bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest); +bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); /* --- Helper functions --- - * You may want to use these from your callbacks. + * You may want to use these from your caller or callbacks. */ +pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); + bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest); bool pb_decode_varint64(pb_istream_t *stream, uint64_t *dest); @@ -20,10 +22,10 @@ bool pb_skip_string(pb_istream_t *stream); /* --- Field decoders --- * Each decoder takes stream and field description, and a pointer to the field - * in the destination struct (dest = struct_addr + field->offset). + * in the destination struct (dest = struct_addr + field->data_offset). + * For arrays, these functions are called repeatedly. */ -// Integer types. bool pb_dec_uint32(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_sint32(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest); @@ -31,25 +33,20 @@ bool pb_dec_uint64(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_sint64(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_bool(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_enum(pb_istream_t *stream, const pb_field_t *field, void *dest); -// Floating point types. Info is ignored. bool pb_dec_float(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_double(pb_istream_t *stream, const pb_field_t *field, void *dest); -// Byte array. Dest is pointer to bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); - -// Null-terminated string. bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); - -// Use callback. Dest is pointer to pb_callback_t struct. bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest); typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest); /* --- Function pointers to field decoders --- - * Order in the array must match pb_action_t numbering. + * Order in the array must match pb_action_t LTYPE numbering. */ -const pb_decoder_t PB_DECODERS[PB_LAST_ACT]; +const pb_decoder_t PB_DECODERS[16]; #endif diff --git a/tests/test_decode1.c b/tests/test_decode1.c index dc0831a2..fbdb7db6 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -9,8 +9,6 @@ typedef enum { Person_PhoneType_MOBILE = 0, Person_PhoneType_HOME = 1, Person_PhoneType_WORK = 2, - - _Person_PhoneType_size = 0xFFFFFFFF // Force 32-bit enum } Person_PhoneType; typedef struct { @@ -24,52 +22,71 @@ typedef struct { int32_t id; bool has_email; char email[40]; - - pb_callback_t phone; + size_t phone_size; + Person_PhoneNumber phone[5]; } Person; /* Field descriptions */ #define membersize(st, m) (sizeof ((st*)0)->m) +const Person_PhoneType Person_PhoneType_type_default = Person_PhoneType_HOME; + const pb_field_t Person_PhoneNumber_fields[] = { - {1, offsetof(Person_PhoneNumber, number), PB_ACT_STRING, membersize(Person_PhoneNumber, number)}, - {2, offsetof(Person_PhoneNumber, has_type), PB_ACT_HAS, membersize(Person_PhoneNumber, has_type)}, - {2, offsetof(Person_PhoneNumber, type), PB_ACT_UINT32, membersize(Person_PhoneNumber, type)}, + {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING, + offsetof(Person_PhoneNumber, number), 0, + membersize(Person_PhoneNumber, number), 0, 0}, + + {2, PB_HTYPE_OPTIONAL | PB_LTYPE_ENUM, + offsetof(Person_PhoneNumber, type), + offsetof(Person_PhoneNumber, has_type), + membersize(Person_PhoneNumber, type), 0, + &Person_PhoneType_type_default}, + PB_LAST_FIELD }; const pb_field_t Person_fields[] = { - {1, offsetof(Person, name), PB_ACT_STRING, membersize(Person, name)}, - {2, offsetof(Person, id), PB_ACT_INT32, membersize(Person, id)}, - {3, offsetof(Person, email), PB_ACT_STRING, membersize(Person, email)}, - {4, offsetof(Person, phone), PB_ACT_SUBMESSAGE, membersize(Person, phone)} + {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING, + offsetof(Person, name), 0, + membersize(Person, name), 0, 0}, + + {2, PB_HTYPE_REQUIRED | PB_LTYPE_INT32, + offsetof(Person, id), 0, + membersize(Person, id), 0, 0}, + + {3, PB_HTYPE_OPTIONAL | PB_LTYPE_STRING, + offsetof(Person, email), + offsetof(Person, has_email), + membersize(Person, email), 0, 0}, + + {4, PB_HTYPE_ARRAY | PB_LTYPE_SUBMESSAGE, + offsetof(Person, phone), + offsetof(Person, phone_size), + membersize(Person, phone[0]), + membersize(Person, phone) / membersize(Person, phone[0]), + Person_PhoneNumber_fields}, + + PB_LAST_FIELD }; -/* Default value descriptions */ -#define Person_PhoneNumber_default {"", false, Person_PhoneType_HOME}; -#define Person_default {"", 0, false, "", {{0},0}}; - /* And now, the actual test program */ -bool print_phonenumber(pb_istream_t *stream, const pb_field_t *field, void *arg) -{ - Person_PhoneNumber x = Person_PhoneNumber_default; - if (!pb_decode(stream, Person_PhoneNumber_fields, &x)) - return false; - - printf("PhoneNumber: number '%s' type '%d'\n", x.number, x.type); - return true; -} - bool print_person(pb_istream_t *stream) { - Person x = Person_default; - x.phone.funcs.decode = &print_phonenumber; + int i; + Person person; - if (!pb_decode(stream, Person_fields, &x)) + if (!pb_decode(stream, Person_fields, &person)) return false; - printf("Person: name '%s' id '%d' email '%s'\n", x.name, x.id, x.email); + printf("Person: name '%s' id '%d' email '%s'\n", person.name, person.id, person.email); + + for (i = 0; i < person.phone_size; i++) + { + Person_PhoneNumber *phone = &person.phone[i]; + printf("PhoneNumber: number '%s' type '%d'\n", phone->number, phone->type); + } + return true; } @@ -91,10 +108,10 @@ bool my_read(pb_istream_t *stream, char *buf, size_t count) int main() { - char buffer[512]; + uint8_t buffer[512]; size_t size = fread(buffer, 1, 512, stdin); - pb_istream_t stream = {&my_read, buffer, size}; + pb_istream_t stream = pb_istream_from_buffer(buffer, size); if (!print_person(&stream)) printf("Parsing failed.\n"); -- cgit 1.2.3-korg From ead3b734d8f96837ea6564b694df5618134e3cf8 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 27 Jul 2011 19:57:43 +0000 Subject: Making code ansi-compatible git-svn-id: https://svn.kapsi.fi/jpa/nanopb@944 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb.h | 127 ++++++++++++++++++++++---------------------------- pb_decode.c | 128 ++++++++++++++++++++++++++++++--------------------- pb_decode.h | 49 ++++++++++++++------ tests/Makefile | 11 +++++ tests/test_decode1.c | 2 +- 5 files changed, 176 insertions(+), 141 deletions(-) create mode 100644 tests/Makefile diff --git a/pb.h b/pb.h index 99e4bf7e..773e6450 100644 --- a/pb.h +++ b/pb.h @@ -2,57 +2,31 @@ #define _PB_H_ #include -#include // size_t +#include #include #ifdef __GNUC__ -// This just reduces memory requirements, but is not required. +/* This just reduces memory requirements, but is not required. */ #define pb_packed __attribute__((packed)) #else #define pb_packed #endif -/* Lightweight input stream. - * If buf is NULL, read but don't store bytes. - * You have to provide a callback function for reading. - * You can use state to store your own data (e.g. buffer pointer), - * and rely on pb_read to verify that no-body reads past bytes_left. - * However, substreams may change bytes_left so don't use that to - * compute any pointers. - */ -typedef struct _pb_istream_t pb_istream_t; -struct _pb_istream_t -{ - bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count); - void *state; // Free field for use by callback implementation - size_t bytes_left; -}; - -static inline bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) -{ - if (stream->bytes_left < count) - return false; - - bool status = stream->callback(stream, buf, count); - stream->bytes_left -= count; - return status; -} - /* Lightweight output stream. */ typedef struct _pb_ostream_t pb_ostream_t; struct _pb_ostream_t { bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); - void *state; // Free field for use by callback implementation + void *state; /* Free field for use by callback implementation */ size_t bytes_written; }; -static inline bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) +/*static inline bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) { bool status = stream->callback(stream, buf, count); stream->bytes_written += count; return status; -} +}*/ /* List of possible field types * Least-significant 4 bits tell the scalar type @@ -65,7 +39,11 @@ static inline bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t cou */ typedef enum { - // Standard integer types + /************************ + * Field contents types * + ************************/ + + /* Standard integer types */ PB_LTYPE_UINT32 = 0x00, PB_LTYPE_INT32 = 0x00, PB_LTYPE_SINT32 = 0x01, @@ -79,39 +57,40 @@ typedef enum { PB_LTYPE_BOOL = 0x06, PB_LTYPE_ENUM = 0x07, - // Standard float types + /* Standard float types */ PB_LTYPE_FLOAT = 0x08, PB_LTYPE_DOUBLE = 0x09, - // Byte array with pre-allocated buffer. - // data_size is the length of the allocated PB_BYTES_ARRAY structure. + /* Byte array with pre-allocated buffer. + * data_size is the length of the allocated PB_BYTES_ARRAY structure. */ PB_LTYPE_BYTES = 0x0A, - // String with pre-allocated buffer. - // data_size is the maximum length. + /* String with pre-allocated buffer. + * data_size is the maximum length. */ PB_LTYPE_STRING = 0x0B, - // Submessage - // submsg_fields is pointer to field descriptions + /* Submessage + * submsg_fields is pointer to field descriptions */ PB_LTYPE_SUBMESSAGE = 0x0C, - ///////////// - // Modifier flags + /****************** + * Modifier flags * + ******************/ - // Just the basic, write data at data_offset + /* Just the basic, write data at data_offset */ PB_HTYPE_REQUIRED = 0x00, - // Write true at size_offset + /* Write true at size_offset */ PB_HTYPE_OPTIONAL = 0x10, - // Read to pre-allocated array - // Maximum number of entries is array_size, - // actual number is stored at size_offset + /* Read to pre-allocated array + * Maximum number of entries is array_size, + * actual number is stored at size_offset */ PB_HTYPE_ARRAY = 0x20, - // Works for all required/optional/repeated fields. - // data_offset points to pb_callback_t structure. - // LTYPE is ignored. + /* Works for all required/optional/repeated fields. + * data_offset points to pb_callback_t structure. + * LTYPE is ignored. */ PB_HTYPE_CALLBACK = 0x30 } pb_packed pb_type_t; @@ -129,21 +108,21 @@ typedef struct _pb_field_t pb_field_t; struct _pb_field_t { uint8_t tag; pb_type_t type; - uint8_t data_offset; // Offset of actual data or array start - uint8_t size_offset; // Offset of array size or has-boolean - uint8_t data_size; // Data size in bytes for a single item - uint8_t array_size; // Maximum number of entries in array + uint8_t data_offset; /* Offset of actual data or array start */ + uint8_t size_offset; /* Offset of array size or has-boolean */ + uint8_t data_size; /* Data size in bytes for a single item */ + uint8_t array_size; /* Maximum number of entries in array */ - // Field definitions for submessage - // OR default value for all other non-array, non-callback types - // If null, then field will zeroed. + /* Field definitions for submessage + * OR default value for all other non-array, non-callback types + * If null, then field will zeroed. */ const void *ptr; } pb_packed; #define PB_LAST_FIELD {0,0,0,0} -// This structure is used for 'bytes' arrays. -// It has the number of bytes in the beginning, and after that an array. +/* This structure is used for 'bytes' arrays. + * It has the number of bytes in the beginning, and after that an array. */ #define PB_BYTES_ARRAY(buffersize) \ struct { \ size_t size; \ @@ -152,19 +131,23 @@ struct { \ typedef PB_BYTES_ARRAY(1) pb_bytes_array_t; -// This structure is used for giving the callback function. -// -// The decoding callback will be given a limited-length stream -// If the wire type was string, the length is the length of the string. -// If the wire type was a varint/fixed32/fixed64, the length is the length -// of the actual value. -// The function may be called multiple times (especially for repeated types, -// but also otherwise if the message happens to contain the field multiple -// times.) -// -// The encoding callback will receive the actual output stream. -// It should write all the data in one call, including the field tag and -// wire type. It can write multiple fields. +/* This structure is used for giving the callback function. + * It is stored in the message structure and filled in by the method that + * calls pb_decode. + * + * The decoding callback will be given a limited-length stream + * If the wire type was string, the length is the length of the string. + * If the wire type was a varint/fixed32/fixed64, the length is the length + * of the actual value. + * The function may be called multiple times (especially for repeated types, + * but also otherwise if the message happens to contain the field multiple + * times.) + * + * The encoding callback will receive the actual output stream. + * It should write all the data in one call, including the field tag and + * wire type. It can write multiple fields. + */ +typedef struct _pb_istream_t pb_istream_t; typedef struct _pb_callback_t pb_callback_t; struct _pb_callback_t { union { @@ -172,7 +155,7 @@ struct _pb_callback_t { bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, void *arg); } funcs; - // Free arg for use by callback + /* Free arg for use by callback */ void *arg; }; diff --git a/pb_decode.c b/pb_decode.c index acd9b204..28856d37 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -8,26 +8,37 @@ #include const pb_decoder_t PB_DECODERS[16] = { - &pb_dec_uint32, - &pb_dec_sint32, - &pb_dec_fixed32, - &pb_dec_uint64, - &pb_dec_sint64, - &pb_dec_fixed64, - &pb_dec_bool, - &pb_dec_enum, + (pb_decoder_t)&pb_dec_uint32, + (pb_decoder_t)&pb_dec_sint32, + (pb_decoder_t)&pb_dec_fixed32, + (pb_decoder_t)&pb_dec_uint64, + (pb_decoder_t)&pb_dec_sint64, + (pb_decoder_t)&pb_dec_fixed64, + (pb_decoder_t)&pb_dec_bool, + (pb_decoder_t)&pb_dec_enum, - &pb_dec_float, - &pb_dec_double, + (pb_decoder_t)&pb_dec_float, + (pb_decoder_t)&pb_dec_double, - &pb_dec_bytes, - &pb_dec_string, - &pb_dec_submessage + (pb_decoder_t)&pb_dec_bytes, + (pb_decoder_t)&pb_dec_string, + (pb_decoder_t)&pb_dec_submessage }; -//////////////////////// -// Helper functions -//////////////////////// +/************** + * pb_istream * + **************/ + +bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) +{ + bool status; + if (stream->bytes_left < count) + return false; + + status = stream->callback(stream, buf, count); + stream->bytes_left -= count; + return status; +} static bool buf_read(pb_istream_t *stream, uint8_t *buf, size_t count) { @@ -42,10 +53,17 @@ static bool buf_read(pb_istream_t *stream, uint8_t *buf, size_t count) pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize) { - pb_istream_t stream = {&buf_read, buf, bufsize}; + pb_istream_t stream; + stream.callback = &buf_read; + stream.state = buf; + stream.bytes_left = bufsize; return stream; } +/******************** + * Helper functions * + ********************/ + bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) { uint64_t temp; @@ -117,8 +135,8 @@ static bool skip(pb_istream_t *stream, int wire_type) } } -// Read a raw value to buffer, for the purpose of passing it to callback. -// Size is maximum size on call, and actual size on return. +/* Read a raw value to buffer, for the purpose of passing it to callback. + * Size is maximum size on call, and actual size on return. */ static bool read_raw_value(pb_istream_t *stream, int wire_type, uint8_t *buf, size_t *size) { size_t max_size = *size; @@ -146,7 +164,7 @@ static bool read_raw_value(pb_istream_t *stream, int wire_type, uint8_t *buf, si } } -// Decode string length from stream and return a substream with limited length +/* Decode string length from stream and return a substream with limited length */ static bool make_string_substream(pb_istream_t *stream, pb_istream_t *substream) { uint32_t size; @@ -183,7 +201,7 @@ bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, && PB_LTYPE(field->type) != PB_LTYPE_STRING && PB_LTYPE(field->type) != PB_LTYPE_SUBMESSAGE) { - // Packed array + /* Packed array */ size_t *size = (size_t*)pSize; pb_istream_t substream; if (!make_string_substream(stream, &substream)) @@ -191,7 +209,7 @@ bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, while (substream.bytes_left && *size < field->array_size) { - void *pItem = pData + field->data_size * (*size); + void *pItem = (uint8_t*)pData + field->data_size * (*size); if (!func(stream, field, pItem)) return false; (*size)++; @@ -200,12 +218,12 @@ bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, } else { - // Repeated field + /* Repeated field */ size_t *size = (size_t*)pSize; + void *pItem = (uint8_t*)pData + field->data_size * (*size); if (*size >= field->array_size) return false; - void *pItem = pData + field->data_size * (*size); (*size)++; return func(stream, field, pItem); } @@ -227,17 +245,19 @@ bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, } else { - // Copy the single scalar value to stack. - // This is required so that we can limit the stream length, - // which in turn allows to use same callback for packed and - // not-packed fields. + /* Copy the single scalar value to stack. + * This is required so that we can limit the stream length, + * which in turn allows to use same callback for packed and + * not-packed fields. */ + pb_istream_t substream; + pb_callback_t *pCallback = (pb_callback_t*)pData; uint8_t buffer[10]; size_t size = sizeof(buffer); + if (!read_raw_value(stream, wire_type, buffer, &size)) return false; - pb_istream_t substream = pb_istream_from_buffer(buffer, size); + substream = pb_istream_from_buffer(buffer, size); - pb_callback_t *pCallback = (pb_callback_t*)pData; return pCallback->funcs.decode(&substream, field, pCallback->arg); } @@ -248,11 +268,11 @@ bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { - // Used to check for required fields + /* Used to check for required fields */ uint32_t fields_seen = 0; int i; - // Initialize size/has fields and apply default values + /* Initialize size/has fields and apply default values */ for (i = 0; fields[i].tag != 0; i++) { void *pData = (char*)dest_struct + fields[i].data_offset; @@ -283,11 +303,12 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc while (stream->bytes_left) { uint32_t temp; + int tag, wire_type; if (!pb_decode_varint32(stream, &temp)) return false; - int tag = temp >> 3; - int wire_type = temp & 7; + tag = temp >> 3; + wire_type = temp & 7; i = 0; while (fields[i].tag != 0 && fields[i].tag != tag) @@ -295,7 +316,7 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc i++; } - if (fields[i].tag == 0) // No match found, skip data + if (fields[i].tag == 0) /* No match found, skip data */ { skip(stream, wire_type); continue; @@ -307,7 +328,7 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc return false; } - // Check that all required fields (mod 31) were present. + /* Check that all required fields (mod 31) were present. */ for (i = 0; fields[i].tag != 0; i++) { if (PB_HTYPE(fields[i].type) == PB_HTYPE_REQUIRED && @@ -322,12 +343,12 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc /* Field decoders */ -bool pb_dec_uint32(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool pb_dec_uint32(pb_istream_t *stream, const pb_field_t *field, uint32_t *dest) { - return pb_decode_varint32(stream, (uint32_t*)dest); + return pb_decode_varint32(stream, dest); } -bool pb_dec_sint32(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool pb_dec_sint32(pb_istream_t *stream, const pb_field_t *field, int32_t *dest) { uint32_t *x = (uint32_t*)dest; bool status = pb_decode_varint32(stream, x); @@ -335,7 +356,7 @@ bool pb_dec_sint32(pb_istream_t *stream, const pb_field_t *field, void *dest) return status; } -bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, uint32_t *dest) { uint8_t bytes[4] = {0}; bool status = pb_read(stream, bytes, 4); @@ -349,12 +370,12 @@ bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest) return status; } -bool pb_dec_uint64(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool pb_dec_uint64(pb_istream_t *stream, const pb_field_t *field, uint64_t *dest) { - return pb_decode_varint64(stream, (uint64_t*)dest); + return pb_decode_varint64(stream, dest); } -bool pb_dec_sint64(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool pb_dec_sint64(pb_istream_t *stream, const pb_field_t *field, int64_t *dest) { uint64_t *x = (uint64_t*)dest; bool status = pb_decode_varint64(stream, x); @@ -362,7 +383,7 @@ bool pb_dec_sint64(pb_istream_t *stream, const pb_field_t *field, void *dest) return status; } -bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, uint64_t *dest) { uint8_t bytes[8] = {0}; bool status = pb_read(stream, bytes, 8); @@ -377,7 +398,7 @@ bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest) return status; } -bool pb_dec_bool(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool pb_dec_bool(pb_istream_t *stream, const pb_field_t *field, bool *dest) { uint32_t temp = 0; bool status = pb_decode_varint32(stream, &temp); @@ -387,24 +408,24 @@ bool pb_dec_bool(pb_istream_t *stream, const pb_field_t *field, void *dest) bool pb_dec_enum(pb_istream_t *stream, const pb_field_t *field, void *dest) { - // Enum sizes can vary, copy only data_size amount of bytes. + /* Enum sizes can vary, copy only data_size amount of bytes. */ uint32_t temp = 0; bool status = pb_decode_varint32(stream, &temp); memcpy(dest, &temp, field->data_size); return status; } -bool pb_dec_float(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool pb_dec_float(pb_istream_t *stream, const pb_field_t *field, float *dest) { return pb_read(stream, (uint8_t*)dest, sizeof(float)); } -bool pb_dec_double(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool pb_dec_double(pb_istream_t *stream, const pb_field_t *field, double *dest) { return pb_read(stream, (uint8_t*)dest, sizeof(double)); } -bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, uint8_t *dest) { pb_bytes_array_t *x = (pb_bytes_array_t*)dest; @@ -413,24 +434,25 @@ bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) return false; x->size = temp; - // Note: data_size includes the size of the x.size field, too. - // Calculate actual size starting from offset. + /* Note: data_size includes the size of the x.size field, too. + * Calculate actual size starting from offset. */ if (x->size > field->data_size - offsetof(pb_bytes_array_t, bytes)) return false; return pb_read(stream, x->bytes, x->size); } -bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, uint8_t *dest) { uint32_t size; + bool status; if (!pb_decode_varint32(stream, &size)) return false; if (size > field->data_size - 1) return false; - bool status = pb_read(stream, (uint8_t*)dest, size); + status = pb_read(stream, (uint8_t*)dest, size); *((uint8_t*)dest + size) = 0; return status; } diff --git a/pb_decode.h b/pb_decode.h index 2f689394..ac4d1f70 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -4,16 +4,35 @@ #include #include "pb.h" -// Decode from stream to destination struct. -// The actual struct pointed to by dest must match the description in fields. +/* Lightweight input stream. + * If buf is NULL, read but don't store bytes. + * You can to provide a callback function for reading or use + * pb_istream_from_buffer. + * + * You can use state to store your own data (e.g. buffer pointer), + * and rely on pb_read to verify that no-body reads past bytes_left. + * However, substreams may change bytes_left so don't use that to + * compute any pointers. + */ +struct _pb_istream_t +{ + bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count); + void *state; /* Free field for use by callback implementation */ + size_t bytes_left; +}; + +pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); +bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); + +/* Decode from stream to destination struct. + * The actual struct pointed to by dest must match the description in fields. + */ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); /* --- Helper functions --- * You may want to use these from your caller or callbacks. */ -pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); - bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest); bool pb_decode_varint64(pb_istream_t *stream, uint64_t *dest); @@ -26,20 +45,20 @@ bool pb_skip_string(pb_istream_t *stream); * For arrays, these functions are called repeatedly. */ -bool pb_dec_uint32(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_sint32(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_uint64(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_sint64(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_bool(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_uint32(pb_istream_t *stream, const pb_field_t *field, uint32_t *dest); +bool pb_dec_sint32(pb_istream_t *stream, const pb_field_t *field, int32_t *dest); +bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, uint32_t *dest); +bool pb_dec_uint64(pb_istream_t *stream, const pb_field_t *field, uint64_t *dest); +bool pb_dec_sint64(pb_istream_t *stream, const pb_field_t *field, int64_t *dest); +bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, uint64_t *dest); +bool pb_dec_bool(pb_istream_t *stream, const pb_field_t *field, bool *dest); bool pb_dec_enum(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_float(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_double(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_float(pb_istream_t *stream, const pb_field_t *field, float *dest); +bool pb_dec_double(pb_istream_t *stream, const pb_field_t *field, double *dest); -bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, uint8_t *dest); +bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, uint8_t *dest); bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest); typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest); diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 00000000..da4030e2 --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,11 @@ +CFLAGS=-ansi -pedantic -Wall -I .. -g -O0 +DEPS=../pb_decode.c ../pb_decode.h ../pb.h + +all: test_decode1 + +clean: + rm -f test_decode1 + +test_decode1: test_decode1.c $(DEPS) + $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c + diff --git a/tests/test_decode1.c b/tests/test_decode1.c index fbdb7db6..20ea65e4 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -8,7 +8,7 @@ typedef enum { Person_PhoneType_MOBILE = 0, Person_PhoneType_HOME = 1, - Person_PhoneType_WORK = 2, + Person_PhoneType_WORK = 2 } Person_PhoneType; typedef struct { -- cgit 1.2.3-korg From 42e5fcc0bee38d183e1d00b4cecb1a5c831ed601 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 27 Jul 2011 20:01:50 +0000 Subject: git-svn-id: https://svn.kapsi.fi/jpa/nanopb@945 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- tests/Makefile | 2 ++ tests/test_decode1.c | 16 ---------------- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index da4030e2..3053fa4a 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -9,3 +9,5 @@ clean: test_decode1: test_decode1.c $(DEPS) $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c +fuzztest: test_decode1 + I=1; while cat /dev/urandom | ./test_decode1 > /dev/null; do I=$(($I+1)); echo -en "\r$I"; done \ No newline at end of file diff --git a/tests/test_decode1.c b/tests/test_decode1.c index 20ea65e4..8623fbdb 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -90,22 +90,6 @@ bool print_person(pb_istream_t *stream) return true; } -bool my_read(pb_istream_t *stream, char *buf, size_t count) -{ - char *source = (char*)stream->state; - - if (!stream->bytes_left) - return false; - - if (buf != NULL) - { - memcpy(buf, source, count); - } - - stream->state = source + count; - return true; -} - int main() { uint8_t buffer[512]; -- cgit 1.2.3-korg From d9238da690d5eda9cb5139aa26734dff5ded2285 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 27 Jul 2011 20:06:17 +0000 Subject: bugfix git-svn-id: https://svn.kapsi.fi/jpa/nanopb@946 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb_decode.c | 4 ++-- tests/Makefile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 28856d37..a722e893 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -148,8 +148,8 @@ static bool read_raw_value(pb_istream_t *stream, int wire_type, uint8_t *buf, si { (*size)++; if (*size > max_size) return false; - if (!pb_read(stream, buf++, 1)) return false; - } while (*buf & 0x80); + if (!pb_read(stream, buf, 1)) return false; + } while (*buf++ & 0x80); return true; case WT_64BIT: diff --git a/tests/Makefile b/tests/Makefile index 3053fa4a..d357c674 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -10,4 +10,4 @@ test_decode1: test_decode1.c $(DEPS) $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c fuzztest: test_decode1 - I=1; while cat /dev/urandom | ./test_decode1 > /dev/null; do I=$(($I+1)); echo -en "\r$I"; done \ No newline at end of file + bash -c 'I=1; while cat /dev/urandom | ./test_decode1 > /dev/null; do I=$$(($$I+1)); echo -en "\r$$I"; done' \ No newline at end of file -- cgit 1.2.3-korg From b9ca72e6af6a3a74a5a1f581540e5004ab72ea2c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 28 Jul 2011 14:54:03 +0000 Subject: git-svn-id: https://svn.kapsi.fi/jpa/nanopb@947 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb.h | 35 ++++++--------- pb_decode.c | 125 +++++++++++++++------------------------------------ pb_decode.h | 16 ++----- tests/test_decode1.c | 4 +- 4 files changed, 56 insertions(+), 124 deletions(-) diff --git a/pb.h b/pb.h index 773e6450..370109a1 100644 --- a/pb.h +++ b/pb.h @@ -43,35 +43,28 @@ typedef enum { * Field contents types * ************************/ - /* Standard integer types */ - PB_LTYPE_UINT32 = 0x00, - PB_LTYPE_INT32 = 0x00, - PB_LTYPE_SINT32 = 0x01, - PB_LTYPE_FIXED32 = 0x02, - PB_LTYPE_SFIXED32 = 0x02, - PB_LTYPE_UINT64 = 0x03, - PB_LTYPE_INT64 = 0x03, - PB_LTYPE_SINT64 = 0x04, - PB_LTYPE_FIXED64 = 0x05, - PB_LTYPE_SFIXED64 = 0x05, - PB_LTYPE_BOOL = 0x06, - PB_LTYPE_ENUM = 0x07, + /* Numeric types */ + PB_LTYPE_VARINT = 0x00, /* int32, uint32, int64, uint64, bool, enum */ + PB_LTYPE_SVARINT = 0x01, /* sint32, sint64 */ + PB_LTYPE_FIXED = 0x02, /* fixed32, sfixed32, fixed64, sfixed64, float, double */ - /* Standard float types */ - PB_LTYPE_FLOAT = 0x08, - PB_LTYPE_DOUBLE = 0x09, + /* Marker for last packable field type. */ + PB_LTYPE_LAST_PACKABLE = 0x02, /* Byte array with pre-allocated buffer. * data_size is the length of the allocated PB_BYTES_ARRAY structure. */ - PB_LTYPE_BYTES = 0x0A, + PB_LTYPE_BYTES = 0x03, /* String with pre-allocated buffer. * data_size is the maximum length. */ - PB_LTYPE_STRING = 0x0B, + PB_LTYPE_STRING = 0x04, /* Submessage * submsg_fields is pointer to field descriptions */ - PB_LTYPE_SUBMESSAGE = 0x0C, + PB_LTYPE_SUBMESSAGE = 0x05, + + /* Number of declared LTYPES */ + PB_LTYPES_COUNT = 6, /****************** * Modifier flags * @@ -108,8 +101,8 @@ typedef struct _pb_field_t pb_field_t; struct _pb_field_t { uint8_t tag; pb_type_t type; - uint8_t data_offset; /* Offset of actual data or array start */ - uint8_t size_offset; /* Offset of array size or has-boolean */ + uint8_t data_offset; /* Offset of field data, relative to previous field. */ + int8_t size_offset; /* Offset of array size or has-boolean, relative to data */ uint8_t data_size; /* Data size in bytes for a single item */ uint8_t array_size; /* Maximum number of entries in array */ diff --git a/pb_decode.c b/pb_decode.c index a722e893..a9ae7350 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -7,18 +7,10 @@ #include "pb_decode.h" #include -const pb_decoder_t PB_DECODERS[16] = { - (pb_decoder_t)&pb_dec_uint32, - (pb_decoder_t)&pb_dec_sint32, - (pb_decoder_t)&pb_dec_fixed32, - (pb_decoder_t)&pb_dec_uint64, - (pb_decoder_t)&pb_dec_sint64, - (pb_decoder_t)&pb_dec_fixed64, - (pb_decoder_t)&pb_dec_bool, - (pb_decoder_t)&pb_dec_enum, - - (pb_decoder_t)&pb_dec_float, - (pb_decoder_t)&pb_dec_double, +const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { + (pb_decoder_t)&pb_dec_varint, + (pb_decoder_t)&pb_dec_svarint, + (pb_decoder_t)&pb_dec_fixed, (pb_decoder_t)&pb_dec_bytes, (pb_decoder_t)&pb_dec_string, @@ -67,10 +59,9 @@ pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize) bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) { uint64_t temp; - if (!pb_decode_varint64(stream, &temp)) - return false; + bool status = pb_decode_varint64(stream, &temp); *dest = temp; - return true; + return status; } bool pb_decode_varint64(pb_istream_t *stream, uint64_t *dest) @@ -197,9 +188,7 @@ bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, case PB_HTYPE_ARRAY: if (wire_type == WT_STRING - && PB_LTYPE(field->type) != PB_LTYPE_BYTES - && PB_LTYPE(field->type) != PB_LTYPE_STRING - && PB_LTYPE(field->type) != PB_LTYPE_SUBMESSAGE) + && PB_LTYPE(field->type) <= PB_LTYPE_LAST_PACKABLE) { /* Packed array */ size_t *size = (size_t*)pSize; @@ -242,6 +231,7 @@ bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, if (!pCallback->funcs.decode(&substream, field, pCallback->arg)) return false; } + return true; } else { @@ -280,23 +270,16 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc if (PB_HTYPE(fields[i].type) == PB_HTYPE_OPTIONAL) { *(bool*)pSize = false; - } - else if (PB_HTYPE(fields[i].type) == PB_HTYPE_ARRAY) - { - *(size_t*)pSize = 0; - } - - if (PB_HTYPE(fields[i].type) != PB_HTYPE_ARRAY && - PB_HTYPE(fields[i].type) != PB_HTYPE_CALLBACK) - { + + /* Initialize to default value */ if (fields[i].ptr != NULL) - { memcpy(pData, fields[i].ptr, fields[i].data_size); - } else - { memset(pData, 0, fields[i].data_size); - } + } + else if (PB_HTYPE(fields[i].type) == PB_HTYPE_ARRAY) + { + *(size_t*)pSize = 0; } } @@ -343,88 +326,52 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc /* Field decoders */ -bool pb_dec_uint32(pb_istream_t *stream, const pb_field_t *field, uint32_t *dest) -{ - return pb_decode_varint32(stream, dest); -} - -bool pb_dec_sint32(pb_istream_t *stream, const pb_field_t *field, int32_t *dest) -{ - uint32_t *x = (uint32_t*)dest; - bool status = pb_decode_varint32(stream, x); - *x = (*x >> 1) ^ -(int32_t)(*x & 1); - return status; -} - -bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, uint32_t *dest) +/* Copy destsize bytes from src so that values are casted properly. + * On little endian machine, copy first n bytes of src + * On big endian machine, copy last n bytes of src + * srcsize must always be larger than destsize + */ +static void endian_copy(void *dest, void *src, size_t destsize, size_t srcsize) { - uint8_t bytes[4] = {0}; - bool status = pb_read(stream, bytes, 4); - #ifdef __BIG_ENDIAN__ - uint8_t lebytes[4] = {bytes[3], bytes[2], bytes[1], bytes[0]}; - memcpy(dest, lebytes, 4); + memcpy(dest, (char*)src + (srcsize - destsize), destsize); #else - memcpy(dest, bytes, 4); + memcpy(dest, src, destsize); #endif - return status; } -bool pb_dec_uint64(pb_istream_t *stream, const pb_field_t *field, uint64_t *dest) +bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) { - return pb_decode_varint64(stream, dest); + uint64_t temp; + bool status = pb_decode_varint64(stream, &temp); + endian_copy(dest, &temp, field->data_size, sizeof(temp)); + return status; } -bool pb_dec_sint64(pb_istream_t *stream, const pb_field_t *field, int64_t *dest) +bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) { - uint64_t *x = (uint64_t*)dest; - bool status = pb_decode_varint64(stream, x); - *x = (*x >> 1) ^ -(int64_t)(*x & 1); + uint64_t temp; + bool status = pb_decode_varint64(stream, &temp); + temp = (temp >> 1) ^ -(int64_t)(temp & 1); + endian_copy(dest, &temp, field->data_size, sizeof(temp)); return status; } -bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, uint64_t *dest) +bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint8_t bytes[8] = {0}; - bool status = pb_read(stream, bytes, 8); + bool status = pb_read(stream, bytes, field->data_size); #ifdef __BIG_ENDIAN__ uint8_t lebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], bytes[3], bytes[2], bytes[1], bytes[0]}; - memcpy(dest, lebytes, 4); + endian_copy(dest, lebytes, field->data_size, 8); #else - memcpy(dest, bytes, 4); + endian_copy(dest, bytes, field->data_size, 8); #endif return status; } -bool pb_dec_bool(pb_istream_t *stream, const pb_field_t *field, bool *dest) -{ - uint32_t temp = 0; - bool status = pb_decode_varint32(stream, &temp); - *(bool*)dest = !!temp; - return status; -} - -bool pb_dec_enum(pb_istream_t *stream, const pb_field_t *field, void *dest) -{ - /* Enum sizes can vary, copy only data_size amount of bytes. */ - uint32_t temp = 0; - bool status = pb_decode_varint32(stream, &temp); - memcpy(dest, &temp, field->data_size); - return status; -} - -bool pb_dec_float(pb_istream_t *stream, const pb_field_t *field, float *dest) -{ - return pb_read(stream, (uint8_t*)dest, sizeof(float)); -} - -bool pb_dec_double(pb_istream_t *stream, const pb_field_t *field, double *dest) -{ - return pb_read(stream, (uint8_t*)dest, sizeof(double)); -} - bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, uint8_t *dest) { pb_bytes_array_t *x = (pb_bytes_array_t*)dest; diff --git a/pb_decode.h b/pb_decode.h index ac4d1f70..448b55db 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -45,17 +45,9 @@ bool pb_skip_string(pb_istream_t *stream); * For arrays, these functions are called repeatedly. */ -bool pb_dec_uint32(pb_istream_t *stream, const pb_field_t *field, uint32_t *dest); -bool pb_dec_sint32(pb_istream_t *stream, const pb_field_t *field, int32_t *dest); -bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, uint32_t *dest); -bool pb_dec_uint64(pb_istream_t *stream, const pb_field_t *field, uint64_t *dest); -bool pb_dec_sint64(pb_istream_t *stream, const pb_field_t *field, int64_t *dest); -bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, uint64_t *dest); -bool pb_dec_bool(pb_istream_t *stream, const pb_field_t *field, bool *dest); -bool pb_dec_enum(pb_istream_t *stream, const pb_field_t *field, void *dest); - -bool pb_dec_float(pb_istream_t *stream, const pb_field_t *field, float *dest); -bool pb_dec_double(pb_istream_t *stream, const pb_field_t *field, double *dest); +bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, uint8_t *dest); bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, uint8_t *dest); @@ -66,6 +58,6 @@ typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void /* --- Function pointers to field decoders --- * Order in the array must match pb_action_t LTYPE numbering. */ -const pb_decoder_t PB_DECODERS[16]; +const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT]; #endif diff --git a/tests/test_decode1.c b/tests/test_decode1.c index 8623fbdb..4e0d7117 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -36,7 +36,7 @@ const pb_field_t Person_PhoneNumber_fields[] = { offsetof(Person_PhoneNumber, number), 0, membersize(Person_PhoneNumber, number), 0, 0}, - {2, PB_HTYPE_OPTIONAL | PB_LTYPE_ENUM, + {2, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, offsetof(Person_PhoneNumber, type), offsetof(Person_PhoneNumber, has_type), membersize(Person_PhoneNumber, type), 0, @@ -50,7 +50,7 @@ const pb_field_t Person_fields[] = { offsetof(Person, name), 0, membersize(Person, name), 0, 0}, - {2, PB_HTYPE_REQUIRED | PB_LTYPE_INT32, + {2, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, offsetof(Person, id), 0, membersize(Person, id), 0, 0}, -- cgit 1.2.3-korg From d7863b7424b1be5da33e9d632365f3e2c0538461 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 28 Jul 2011 15:38:05 +0000 Subject: iter git-svn-id: https://svn.kapsi.fi/jpa/nanopb@948 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb_decode.c | 151 +++++++++++++++++++++++++++++++++++---------------- tests/test_decode1.c | 14 ++--- 2 files changed, 110 insertions(+), 55 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index a9ae7350..58d64805 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -72,7 +72,7 @@ bool pb_decode_varint64(pb_istream_t *stream, uint64_t *dest) while (bitpos < 64 && pb_read(stream, &byte, 1)) { - *dest |= (byte & 0x7F) << bitpos; + *dest |= (uint64_t)(byte & 0x7F) << bitpos; bitpos += 7; if (!(byte & 0x80)) @@ -171,35 +171,86 @@ static bool make_string_substream(pb_istream_t *stream, pb_istream_t *substream) return true; } -bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, void *dest_struct) +/* Iterator for pb_field_t list */ +typedef struct { + const pb_field_t *start; + const pb_field_t *current; + int field_index; + void *dest_struct; + void *pData; + void *pSize; +} pb_field_iterator_t; + +static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct) +{ + iter->start = iter->current = fields; + iter->field_index = 0; + iter->pData = dest_struct; + iter->dest_struct = dest_struct; +} + +static bool pb_field_next(pb_field_iterator_t *iter) +{ + bool notwrapped = true; + iter->current++; + iter->field_index++; + if (iter->current->tag == 0) + { + iter->current = iter->start; + iter->field_index = 0; + iter->pData = iter->dest_struct; + notwrapped = false; + } + + iter->pData = (char*)iter->pData + iter->current->data_offset; + iter->pSize = (char*)iter->pData + iter->current->size_offset; + return notwrapped; +} + +static bool pb_field_find(pb_field_iterator_t *iter, int tag) +{ + int start = iter->field_index; + + do { + if (iter->current->tag == tag) + return true; + pb_field_next(iter); + } while (iter->field_index != start); + + return false; +} + +/************************* + * Decode a single field * + *************************/ + +bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_t *iter) { - pb_decoder_t func = PB_DECODERS[PB_LTYPE(field->type)]; - void *pData = (char*)dest_struct + field->data_offset; - void *pSize = (char*)dest_struct + field->size_offset; + pb_decoder_t func = PB_DECODERS[PB_LTYPE(iter->current->type)]; - switch (PB_HTYPE(field->type)) + switch (PB_HTYPE(iter->current->type)) { case PB_HTYPE_REQUIRED: - return func(stream, field, pData); + return func(stream, iter->current, iter->pData); case PB_HTYPE_OPTIONAL: - *(bool*)pSize = true; - return func(stream, field, pData); + *(bool*)iter->pSize = true; + return func(stream, iter->current, iter->pData); case PB_HTYPE_ARRAY: if (wire_type == WT_STRING - && PB_LTYPE(field->type) <= PB_LTYPE_LAST_PACKABLE) + && PB_LTYPE(iter->current->type) <= PB_LTYPE_LAST_PACKABLE) { /* Packed array */ - size_t *size = (size_t*)pSize; + size_t *size = (size_t*)iter->pSize; pb_istream_t substream; if (!make_string_substream(stream, &substream)) return false; - while (substream.bytes_left && *size < field->array_size) + while (substream.bytes_left && *size < iter->current->array_size) { - void *pItem = (uint8_t*)pData + field->data_size * (*size); - if (!func(stream, field, pItem)) + void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size); + if (!func(stream, iter->current, pItem)) return false; (*size)++; } @@ -208,19 +259,19 @@ bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, else { /* Repeated field */ - size_t *size = (size_t*)pSize; - void *pItem = (uint8_t*)pData + field->data_size * (*size); - if (*size >= field->array_size) + size_t *size = (size_t*)iter->pSize; + void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size); + if (*size >= iter->current->array_size) return false; (*size)++; - return func(stream, field, pItem); + return func(stream, iter->current, pItem); } case PB_HTYPE_CALLBACK: if (wire_type == WT_STRING) { - pb_callback_t *pCallback = (pb_callback_t*)pData; + pb_callback_t *pCallback = (pb_callback_t*)iter->pData; pb_istream_t substream; if (!make_string_substream(stream, &substream)) @@ -228,7 +279,7 @@ bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, while (substream.bytes_left) { - if (!pCallback->funcs.decode(&substream, field, pCallback->arg)) + if (!pCallback->funcs.decode(&substream, iter->current, pCallback->arg)) return false; } return true; @@ -240,7 +291,7 @@ bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, * which in turn allows to use same callback for packed and * not-packed fields. */ pb_istream_t substream; - pb_callback_t *pCallback = (pb_callback_t*)pData; + pb_callback_t *pCallback = (pb_callback_t*)iter->pData; uint8_t buffer[10]; size_t size = sizeof(buffer); @@ -248,7 +299,7 @@ bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, return false; substream = pb_istream_from_buffer(buffer, size); - return pCallback->funcs.decode(&substream, field, pCallback->arg); + return pCallback->funcs.decode(&substream, iter->current, pCallback->arg); } default: @@ -256,32 +307,42 @@ bool decode_field(pb_istream_t *stream, int wire_type, const pb_field_t *field, } } +/********************* + * Decode all fields * + *********************/ + bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { - /* Used to check for required fields */ - uint32_t fields_seen = 0; + uint32_t fields_seen = 0; /* Used to check for required fields */ + pb_field_iterator_t iter; int i; + if (fields[0].tag == 0) + { + /* No fields -> nothing to do */ + return pb_read(stream, NULL, stream->bytes_left); + } + + pb_field_init(&iter, fields, dest_struct); + /* Initialize size/has fields and apply default values */ - for (i = 0; fields[i].tag != 0; i++) + do { - void *pData = (char*)dest_struct + fields[i].data_offset; - void *pSize = (char*)dest_struct + fields[i].size_offset; - if (PB_HTYPE(fields[i].type) == PB_HTYPE_OPTIONAL) + if (PB_HTYPE(iter.current->type) == PB_HTYPE_OPTIONAL) { - *(bool*)pSize = false; + *(bool*)iter.pSize = false; /* Initialize to default value */ - if (fields[i].ptr != NULL) - memcpy(pData, fields[i].ptr, fields[i].data_size); + if (iter.current->ptr != NULL) + memcpy(iter.pData, iter.current->ptr, iter.current->data_size); else - memset(pData, 0, fields[i].data_size); + memset(iter.pData, 0, iter.current->data_size); } - else if (PB_HTYPE(fields[i].type) == PB_HTYPE_ARRAY) + else if (PB_HTYPE(iter.current->type) == PB_HTYPE_ARRAY) { - *(size_t*)pSize = 0; + *(size_t*)iter.pSize = 0; } - } + } while (pb_field_next(&iter)); while (stream->bytes_left) { @@ -293,21 +354,16 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc tag = temp >> 3; wire_type = temp & 7; - i = 0; - while (fields[i].tag != 0 && fields[i].tag != tag) - { - i++; - } - - if (fields[i].tag == 0) /* No match found, skip data */ + if (!pb_field_find(&iter, tag)) { + /* No match found, skip data */ skip(stream, wire_type); continue; } - fields_seen |= 1 << (i & 31); + fields_seen |= 1 << (iter.field_index & 31); - if (!decode_field(stream, wire_type, &fields[i], dest_struct)) + if (!decode_field(stream, wire_type, &iter)) return false; } @@ -359,17 +415,16 @@ bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest) { +#ifdef __BIG_ENDIAN__ uint8_t bytes[8] = {0}; bool status = pb_read(stream, bytes, field->data_size); - -#ifdef __BIG_ENDIAN__ uint8_t lebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], bytes[3], bytes[2], bytes[1], bytes[0]}; endian_copy(dest, lebytes, field->data_size, 8); + return status; #else - endian_copy(dest, bytes, field->data_size, 8); + return pb_read(stream, (uint8_t*)dest, field->data_size); #endif - return status; } bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, uint8_t *dest) diff --git a/tests/test_decode1.c b/tests/test_decode1.c index 4e0d7117..a507b0dc 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -37,8 +37,8 @@ const pb_field_t Person_PhoneNumber_fields[] = { membersize(Person_PhoneNumber, number), 0, 0}, {2, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, - offsetof(Person_PhoneNumber, type), - offsetof(Person_PhoneNumber, has_type), + offsetof(Person_PhoneNumber, type) - offsetof(Person_PhoneNumber, number), + (int)offsetof(Person_PhoneNumber, has_type) - (int)offsetof(Person_PhoneNumber, type), membersize(Person_PhoneNumber, type), 0, &Person_PhoneType_type_default}, @@ -51,17 +51,17 @@ const pb_field_t Person_fields[] = { membersize(Person, name), 0, 0}, {2, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, - offsetof(Person, id), 0, + offsetof(Person, id) - offsetof(Person, name), 0, membersize(Person, id), 0, 0}, {3, PB_HTYPE_OPTIONAL | PB_LTYPE_STRING, - offsetof(Person, email), - offsetof(Person, has_email), + offsetof(Person, email) - offsetof(Person, id), + (int)offsetof(Person, has_email) - (int)offsetof(Person, email), membersize(Person, email), 0, 0}, {4, PB_HTYPE_ARRAY | PB_LTYPE_SUBMESSAGE, - offsetof(Person, phone), - offsetof(Person, phone_size), + offsetof(Person, phone) - offsetof(Person, email), + (int)offsetof(Person, phone_size) - (int)offsetof(Person, phone), membersize(Person, phone[0]), membersize(Person, phone) / membersize(Person, phone[0]), Person_PhoneNumber_fields}, -- cgit 1.2.3-korg From f8364310d3ca85d2cf59019bb4bc5e9ff4c52dc3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 30 Jul 2011 09:59:08 +0000 Subject: unittests (some) git-svn-id: https://svn.kapsi.fi/jpa/nanopb@949 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb.h | 9 +++- pb_decode.c | 7 +-- tests/Makefile | 12 +++-- tests/decode_unittests.c | 111 +++++++++++++++++++++++++++++++++++++++++++++++ tests/test_decode1.c | 30 ++++++------- 5 files changed, 145 insertions(+), 24 deletions(-) create mode 100644 tests/decode_unittests.c diff --git a/pb.h b/pb.h index 370109a1..c075cdec 100644 --- a/pb.h +++ b/pb.h @@ -112,8 +112,6 @@ struct _pb_field_t { const void *ptr; } pb_packed; -#define PB_LAST_FIELD {0,0,0,0} - /* This structure is used for 'bytes' arrays. * It has the number of bytes in the beginning, and after that an array. */ #define PB_BYTES_ARRAY(buffersize) \ @@ -152,4 +150,11 @@ struct _pb_callback_t { void *arg; }; +/* These macros are used to declare pb_field_t's in the constant array. */ +#define pb_membersize(st, m) (sizeof ((st*)0)->m) +#define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0])) +#define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2)) +#define PB_LAST_FIELD {0,0,0,0} + + #endif \ No newline at end of file diff --git a/pb_decode.c b/pb_decode.c index 58d64805..63008dd5 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -23,13 +23,14 @@ const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) { - bool status; if (stream->bytes_left < count) return false; - status = stream->callback(stream, buf, count); + if (!stream->callback(stream, buf, count)) + return false; + stream->bytes_left -= count; - return status; + return true; } static bool buf_read(pb_istream_t *stream, uint8_t *buf, size_t count) diff --git a/tests/Makefile b/tests/Makefile index d357c674..ddf91ebb 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,13 +1,17 @@ -CFLAGS=-ansi -pedantic -Wall -I .. -g -O0 +CFLAGS=-ansi -Wall -Werror -I .. -g -O0 DEPS=../pb_decode.c ../pb_decode.h ../pb.h +TESTS=test_decode1 decode_unittests -all: test_decode1 +all: $(TESTS) clean: rm -f test_decode1 -test_decode1: test_decode1.c $(DEPS) +%: %.c $(DEPS) $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c -fuzztest: test_decode1 +run_unittests: decode_unittests + ./decode_unittests + +run_fuzztest: test_decode1 bash -c 'I=1; while cat /dev/urandom | ./test_decode1 > /dev/null; do I=$$(($$I+1)); echo -en "\r$$I"; done' \ No newline at end of file diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c new file mode 100644 index 00000000..746b7e96 --- /dev/null +++ b/tests/decode_unittests.c @@ -0,0 +1,111 @@ +#include +#include +#include "pb_decode.h" + +#define COMMENT(x) printf("\n----" x "----\n"); +#define STR(x) #x +#define STR2(x) STR(x) +#define TEST(x) \ + if (!(x)) { \ + fprintf(stderr, __FILE__ ":" STR2(__LINE__) " FAILED:" #x "\n"); \ + status = 1; \ + } else { \ + printf("OK: " #x "\n"); \ + } + +#define S(x) pb_istream_from_buffer((uint8_t*)x, sizeof(x)) + +bool stream_callback(pb_istream_t *stream, uint8_t *buf, size_t count) +{ + if (stream->state != NULL) + return false; /* Simulate error */ + + if (buf != NULL) + memset(buf, 'x', count); + return true; +} + +int main() +{ + int status = 0; + + { + uint8_t buffer1[] = "foobartest1234"; + uint8_t buffer2[sizeof(buffer1)]; + pb_istream_t stream = pb_istream_from_buffer(buffer1, sizeof(buffer1)); + + COMMENT("Test pb_read and pb_istream_t"); + TEST(pb_read(&stream, buffer2, 6)) + TEST(memcmp(buffer2, "foobar", 6) == 0) + TEST(stream.bytes_left == sizeof(buffer1) - 6) + TEST(pb_read(&stream, buffer2 + 6, stream.bytes_left)) + TEST(memcmp(buffer1, buffer2, sizeof(buffer1)) == 0) + TEST(stream.bytes_left == 0) + TEST(!pb_read(&stream, buffer2, 1)) + } + + { + uint8_t buffer[20]; + pb_istream_t stream = {&stream_callback, NULL, 20}; + + COMMENT("Test pb_read with custom callback"); + TEST(pb_read(&stream, buffer, 5)) + TEST(memcmp(buffer, "xxxxx", 5) == 0) + TEST(!pb_read(&stream, buffer, 50)) + stream.state = (void*)1; /* Simulated error return from callback */ + TEST(!pb_read(&stream, buffer, 5)) + stream.state = NULL; + TEST(pb_read(&stream, buffer, 15)) + } + + { + pb_istream_t s; + uint32_t u; + int32_t i; + + COMMENT("Test pb_decode_varint32"); + TEST((s = S("\x00"), pb_decode_varint32(&s, &u) && u == 0)); + TEST((s = S("\x01"), pb_decode_varint32(&s, &u) && u == 1)); + TEST((s = S("\xAC\x02"), pb_decode_varint32(&s, &u) && u == 300)); + TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint32(&s, &u) && u == UINT32_MAX)); + TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint32(&s, (uint32_t*)&i) && i == -1)); + } + + { + pb_istream_t s; + uint64_t u; + int64_t i; + + COMMENT("Test pb_decode_varint64"); + TEST((s = S("\x00"), pb_decode_varint64(&s, &u) && u == 0)); + TEST((s = S("\x01"), pb_decode_varint64(&s, &u) && u == 1)); + TEST((s = S("\xAC\x02"), pb_decode_varint64(&s, &u) && u == 300)); + TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint64(&s, &u) && u == UINT32_MAX)); + TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint64(&s, (uint64_t*)&i) && i == UINT32_MAX)); + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), + pb_decode_varint64(&s, (uint64_t*)&i) && i == -1)); + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), + pb_decode_varint64(&s, &u) && u == UINT64_MAX)); + } + + { + pb_istream_t s; + COMMENT("Test pb_skip_varint"); + TEST((s = S("\x00""foobar"), pb_skip_varint(&s) && s.bytes_left == 7)) + TEST((s = S("\xAC\x02""foobar"), pb_skip_varint(&s) && s.bytes_left == 7)) + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01""foobar"), + pb_skip_varint(&s) && s.bytes_left == 7)) + } + + { + pb_istream_t s; + COMMENT("Test pb_skip_string") + TEST((s = S("\x00""foobar"), pb_skip_string(&s) && s.bytes_left == 7)) + TEST((s = S("\x04""testfoobar"), pb_skip_string(&s) && s.bytes_left == 7)) + } + + if (status != 0) + fprintf(stdout, "\n\nSome tests FAILED!\n"); + + return status; +} diff --git a/tests/test_decode1.c b/tests/test_decode1.c index a507b0dc..4b34022f 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -27,20 +27,20 @@ typedef struct { } Person; /* Field descriptions */ -#define membersize(st, m) (sizeof ((st*)0)->m) -const Person_PhoneType Person_PhoneType_type_default = Person_PhoneType_HOME; + +const Person_PhoneType Person_PhoneNumber_type_default = Person_PhoneType_HOME; const pb_field_t Person_PhoneNumber_fields[] = { {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING, offsetof(Person_PhoneNumber, number), 0, - membersize(Person_PhoneNumber, number), 0, 0}, + pb_membersize(Person_PhoneNumber, number), 0, 0}, {2, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, - offsetof(Person_PhoneNumber, type) - offsetof(Person_PhoneNumber, number), - (int)offsetof(Person_PhoneNumber, has_type) - (int)offsetof(Person_PhoneNumber, type), - membersize(Person_PhoneNumber, type), 0, - &Person_PhoneType_type_default}, + pb_delta(Person_PhoneNumber, type, number), + pb_delta(Person_PhoneNumber, has_type, type), + pb_membersize(Person_PhoneNumber, type), 0, + &Person_PhoneNumber_type_default}, PB_LAST_FIELD }; @@ -48,22 +48,22 @@ const pb_field_t Person_PhoneNumber_fields[] = { const pb_field_t Person_fields[] = { {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING, offsetof(Person, name), 0, - membersize(Person, name), 0, 0}, + pb_membersize(Person, name), 0, 0}, {2, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, - offsetof(Person, id) - offsetof(Person, name), 0, - membersize(Person, id), 0, 0}, + pb_delta(Person, id, name), 0, + pb_membersize(Person, id), 0, 0}, {3, PB_HTYPE_OPTIONAL | PB_LTYPE_STRING, offsetof(Person, email) - offsetof(Person, id), - (int)offsetof(Person, has_email) - (int)offsetof(Person, email), - membersize(Person, email), 0, 0}, + pb_delta(Person, has_email, email), + pb_membersize(Person, email), 0, 0}, {4, PB_HTYPE_ARRAY | PB_LTYPE_SUBMESSAGE, offsetof(Person, phone) - offsetof(Person, email), - (int)offsetof(Person, phone_size) - (int)offsetof(Person, phone), - membersize(Person, phone[0]), - membersize(Person, phone) / membersize(Person, phone[0]), + pb_delta(Person, phone_size, phone), + pb_membersize(Person, phone[0]), + pb_arraysize(Person, phone), Person_PhoneNumber_fields}, PB_LAST_FIELD -- cgit 1.2.3-korg From 3959290bc77ae26772bc107128b0a4edd3930361 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 31 Jul 2011 12:55:09 +0000 Subject: First version of header generator git-svn-id: https://svn.kapsi.fi/jpa/nanopb@950 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- generator/nanopb.proto | 18 +++ generator/nanopb_generator.py | 344 ++++++++++++++++++++++++++++++++++++++++++ generator/nanopb_pb2.py | 71 +++++++++ pb.h | 2 +- tests/Makefile | 2 +- tests/decode_unittests.c | 25 +++ tests/person.proto | 11 +- tests/test_decode1.c | 73 +-------- 8 files changed, 470 insertions(+), 76 deletions(-) create mode 100644 generator/nanopb.proto create mode 100644 generator/nanopb_generator.py create mode 100644 generator/nanopb_pb2.py diff --git a/generator/nanopb.proto b/generator/nanopb.proto new file mode 100644 index 00000000..813d5b73 --- /dev/null +++ b/generator/nanopb.proto @@ -0,0 +1,18 @@ +// Custom options for defining: +// - Maximum size of string/bytes +// - Maximum number of elements in array +// +// These are used by nanopb to generate statically allocable structures +// for memory-limited environments. + +import "google/protobuf/descriptor.proto"; + +message NanoPBOptions { + optional int32 max_size = 1; + optional int32 max_count = 2; +} + +extend google.protobuf.FieldOptions { + optional NanoPBOptions nanopb = 52001; +} + diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py new file mode 100644 index 00000000..b182d01f --- /dev/null +++ b/generator/nanopb_generator.py @@ -0,0 +1,344 @@ +'''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' + +import google.protobuf.descriptor_pb2 as descriptor +import nanopb_pb2 +import os.path + +# Values are tuple (c type, pb ltype) +FieldD = descriptor.FieldDescriptorProto +datatypes = { + FieldD.TYPE_BOOL: ('bool', 'PB_LTYPE_VARINT'), + FieldD.TYPE_DOUBLE: ('double', 'PB_LTYPE_FIXED'), + FieldD.TYPE_FIXED32: ('uint32_t', 'PB_LTYPE_FIXED'), + FieldD.TYPE_FIXED64: ('uint64_t', 'PB_LTYPE_FIXED'), + FieldD.TYPE_FLOAT: ('float', 'PB_LTYPE_FIXED'), + FieldD.TYPE_INT32: ('int32_t', 'PB_LTYPE_VARINT'), + FieldD.TYPE_INT64: ('int64_t', 'PB_LTYPE_VARINT'), + FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED'), + FieldD.TYPE_SFIXED64: ('int64_t', 'PB_LTYPE_FIXED'), + FieldD.TYPE_SINT32: ('int32_t', 'PB_LTYPE_SVARINT'), + FieldD.TYPE_SINT64: ('int64_t', 'PB_LTYPE_SVARINT'), + FieldD.TYPE_UINT32: ('uint32_t', 'PB_LTYPE_VARINT'), + FieldD.TYPE_UINT64: ('uint64_t', 'PB_LTYPE_VARINT') +} + +class Names: + '''Keeps a set of nested names and formats them to C identifier. + You can subclass this with your own implementation. + ''' + def __init__(self, parts = ()): + if isinstance(parts, Names): + parts = parts.parts + self.parts = tuple(parts) + + def __str__(self): + return '_'.join(self.parts) + + def __add__(self, other): + if isinstance(other, (str, unicode)): + return Names(self.parts + (other,)) + elif isinstance(other, tuple): + return Names(self.parts + other) + else: + raise ValueError("Name parts should be of type str") + +def names_from_type_name(type_name): + '''Parse Names() from FieldDescriptorProto type_name''' + if type_name[0] != '.': + raise NotImplementedError("Lookup of non-absolute type names is not supported") + return Names(type_name[1:].split('.')) + +class Enum: + def __init__(self, names, desc): + '''desc is EnumDescriptorProto''' + self.names = names + desc.name + self.values = [(self.names + x.name, x.number) for x in desc.value] + + def __str__(self): + result = 'typedef enum {\n' + result += ',\n'.join([" %s = %d" % x for x in self.values]) + result += '\n} %s;' % self.names + return result + +class Field: + def __init__(self, struct_name, desc): + '''desc is FieldDescriptorProto''' + self.tag = desc.number + self.struct_name = struct_name + self.name = desc.name + self.default = None + self.max_size = None + self.max_count = None + self.array_decl = "" + + # Parse nanopb-specific field options + if desc.options.HasExtension(nanopb_pb2.nanopb): + ext = desc.options.Extensions[nanopb_pb2.nanopb] + if ext.HasField("max_size"): + self.max_size = ext.max_size + if ext.HasField("max_count"): + self.max_count = ext.max_count + + if desc.HasField('default_value'): + self.default = desc.default_value + + # Decide HTYPE + # HTYPE is the high-order nibble of nanopb field description, + # defining whether value is required/optional/repeated. + is_callback = False + if desc.label == FieldD.LABEL_REQUIRED: + self.htype = 'PB_HTYPE_REQUIRED' + elif desc.label == FieldD.LABEL_OPTIONAL: + self.htype = 'PB_HTYPE_OPTIONAL' + elif desc.label == FieldD.LABEL_REPEATED: + if self.max_count is None: + is_callback = True + else: + self.htype = 'PB_HTYPE_ARRAY' + self.array_decl = '[%d]' % self.max_count + else: + raise NotImplementedError(desc.label) + + # Decide LTYPE and CTYPE + # LTYPE is the low-order nibble of nanopb field description, + # defining how to decode an individual value. + # CTYPE is the name of the c type to use in the struct. + if datatypes.has_key(desc.type): + self.ctype, self.ltype = datatypes[desc.type] + elif desc.type == FieldD.TYPE_ENUM: + self.ltype = 'PB_LTYPE_VARINT' + self.ctype = names_from_type_name(desc.type_name) + self.default = Names(self.ctype) + self.default + elif desc.type == FieldD.TYPE_STRING: + self.ltype = 'PB_LTYPE_STRING' + if self.max_size is None: + is_callback = True + else: + self.ctype = 'char' + self.array_decl += '[%d]' % self.max_size + elif desc.type == FieldD.TYPE_BYTES: + self.ltype = 'PB_LTYPE_BYTES' + if self.max_size is None: + is_callback = True + else: + self.ctype = 'PB_BYTES_ARRAY(%d)' % self.max_size + elif desc.type == FieldD.TYPE_MESSAGE: + self.ltype = 'PB_LTYPE_SUBMESSAGE' + self.ctype = names_from_type_name(desc.type_name) + else: + raise NotImplementedError(desc.type) + + if is_callback: + self.htype = 'PB_HTYPE_CALLBACK' + self.ctype = 'pb_callback_t' + self.array_decl = '' + + def __cmp__(self, other): + return cmp(self.tag, other.tag) + + def __str__(self): + if self.htype == 'PB_HTYPE_OPTIONAL': + result = ' bool has_' + self.name + ';\n' + elif self.htype == 'PB_HTYPE_ARRAY': + result = ' size_t ' + self.name + '_count;\n' + else: + result = '' + result += ' %s %s%s;' % (self.ctype, self.name, self.array_decl) + return result + + def default_decl(self): + '''Return definition for this field's default value.''' + if self.default is None: + return None + + if self.ltype == 'PB_LTYPE_STRING': + ctype = 'char' + if self.max_size is None: + array_decl = '[]' + else: + array_decl = '[%d]' % self.max_size + default = self.default.encode('string_escape') + default = default.replace('"', '\\"') + default = '"' + default + '"' + elif self.ltype == 'PB_LTYPE_BYTES': + data = self.default.decode('string_escape') + data = ['0x%02x' % ord(c) for c in data] + + if self.max_size is None: + ctype = 'PB_BYTES_ARRAY(%d)' % len(data) + else: + ctype = 'PB_BYTES_ARRAY(%d)' % self.max_size + + default = '{%d, {%s}}' % (len(data), ','.join(data)) + array_decl = '' + else: + ctype, default = self.ctype, self.default + array_decl = '' + + return 'const %s %s_default%s = %s;' % (ctype, self.struct_name + self.name, array_decl, default) + + def pb_field_t(self, prev_field_name): + '''Return the pb_field_t initializer to use in the constant array. + prev_field_name is the name of the previous field or None. + ''' + result = ' {%d, ' % self.tag + result += self.htype + if self.ltype is not None: + result += ' | ' + self.ltype + result += ',\n' + + if prev_field_name is None: + result += ' offsetof(%s, %s),' % (self.struct_name, self.name) + else: + result += ' pb_delta(%s, %s, %s),' % (self.struct_name, self.name, prev_field_name) + + if self.htype == 'PB_HTYPE_OPTIONAL': + result += '\n pb_delta(%s, has_%s, %s),' % (self.struct_name, self.name, self.name) + elif self.htype == 'PB_HTYPE_ARRAY': + result += '\n pb_delta(%s, %s_count, %s),' % (self.struct_name, self.name, self.name) + else: + result += ' 0,' + + + if self.htype == 'PB_HTYPE_ARRAY': + result += '\n pb_membersize(%s, %s[0]),' % (self.struct_name, self.name) + result += ('\n pb_membersize(%s, %s) / pb_membersize(%s, %s[0]),' + % (self.struct_name, self.name, self.struct_name, self.name)) + else: + result += '\n pb_membersize(%s, %s),' % (self.struct_name, self.name) + result += ' 0,' + + if self.ltype == 'PB_LTYPE_SUBMESSAGE': + result += '\n &%s_fields}' % self.ctype + elif self.default is None or self.htype == 'PB_HTYPE_CALLBACK': + result += ' 0}' + else: + result += '\n &%s_default}' % (self.struct_name + self.name) + + return result + +class Message: + def __init__(self, names, desc): + self.name = names + self.fields = [Field(self.name, f) for f in desc.field] + self.ordered_fields = self.fields[:] + self.ordered_fields.sort() + + def __cmp__(self, other): + '''Sort messages so that submessages are declared before the message + that uses them. + ''' + if self.refers_to(other.name): + return 1 + elif other.refers_to(self.name): + return -1 + else: + return 0 + + def refers_to(self, name): + '''Returns True if this message uses the specified type as field type.''' + for field in self.fields: + if str(field.ctype) == str(name): + return True + return False + + def __str__(self): + result = 'typedef struct {\n' + result += '\n'.join([str(f) for f in self.fields]) + result += '\n} %s;' % self.name + return result + + def default_decl(self): + result = "" + for field in self.fields: + default = field.default_decl() + if default is not None: + result += default + '\n' + return result + + def pb_field_t(self): + result = 'const pb_field_t %s_fields[] = {\n' % self.name + + prev = None + for field in self.ordered_fields: + result += field.pb_field_t(prev) + result += ',\n\n' + prev = field.name + + result = result[:-3] + '\n};' + return result + +def iterate_messages(desc, names = Names()): + '''Recursively find all messages. For each, yield name, DescriptorProto.''' + if hasattr(desc, 'message_type'): + submsgs = desc.message_type + else: + submsgs = desc.nested_type + + for submsg in submsgs: + sub_names = names + submsg.name + yield sub_names, submsg + + for x in iterate_messages(submsg, sub_names): + yield x + +def process_file(fdesc): + '''Takes a FileDescriptorProto and generate content for its header file. + Generates strings, which should be concatenated and stored to file. + ''' + + yield '/* Automatically generated nanopb header */\n' + yield '#include \n\n' + + enums = [] + messages = [] + + for enum in fdesc.enum_type: + enums.append(Enum(Names(), enum)) + + for names, message in iterate_messages(fdesc): + for enum in message.enum_type: + enums.append(Enum(names, enum)) + + messages.append(Message(names, message)) + + yield '/* Enum definitions */\n' + for enum in enums: + yield str(enum) + '\n\n' + + yield '/* Struct definitions */\n' + messages.sort() + for msg in messages: + yield str(msg) + '\n\n' + + yield '/* Default values for struct fields */\n' + for msg in messages: + yield msg.default_decl() + yield '\n' + + yield '/* Struct field encoding specification for nanopb */\n' + for msg in messages: + yield msg.pb_field_t() + '\n\n' + +if __name__ == '__main__': + import sys + import os.path + + if len(sys.argv) != 2: + print "Usage: " + sys.argv[0] + " file.pb" + print "where file.pb has been compiled from .proto by:" + print "protoc -ofile.pb file.proto" + print "Output fill be written to file.h" + sys.exit(1) + + data = open(sys.argv[1]).read() + fdesc = descriptor.FileDescriptorSet.FromString(data) + + destfile = os.path.splitext(sys.argv[1])[0] + '.h' + + print "Writing to " + destfile + + destfile = open(destfile, 'w') + + for part in process_file(fdesc.file[0]): + destfile.write(part) diff --git a/generator/nanopb_pb2.py b/generator/nanopb_pb2.py new file mode 100644 index 00000000..eb926de9 --- /dev/null +++ b/generator/nanopb_pb2.py @@ -0,0 +1,71 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! + +from google.protobuf import descriptor +from google.protobuf import message +from google.protobuf import reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + +DESCRIPTOR = descriptor.FileDescriptor( + name='nanopb.proto', + package='', + serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"4\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05:?\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xa1\x96\x03 \x01(\x0b\x32\x0e.NanoPBOptions') + + +NANOPB_FIELD_NUMBER = 52001 +nanopb = descriptor.FieldDescriptor( + name='nanopb', full_name='nanopb', index=0, + number=52001, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=True, extension_scope=None, + options=None) + + +_NANOPBOPTIONS = descriptor.Descriptor( + name='NanoPBOptions', + full_name='NanoPBOptions', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='max_size', full_name='NanoPBOptions.max_size', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='max_count', full_name='NanoPBOptions.max_count', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=50, + serialized_end=102, +) + +import google.protobuf.descriptor_pb2 + + +class NanoPBOptions(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + DESCRIPTOR = _NANOPBOPTIONS + + # @@protoc_insertion_point(class_scope:NanoPBOptions) + +nanopb.message_type = _NANOPBOPTIONS +google.protobuf.descriptor_pb2.FieldOptions.RegisterExtension(nanopb) +# @@protoc_insertion_point(module_scope) diff --git a/pb.h b/pb.h index c075cdec..27ee11fd 100644 --- a/pb.h +++ b/pb.h @@ -120,7 +120,7 @@ struct { \ uint8_t bytes[buffersize]; \ } -typedef PB_BYTES_ARRAY(1) pb_bytes_array_t; +typedef PB_BYTES_ARRAY() pb_bytes_array_t; /* This structure is used for giving the callback function. * It is stored in the message structure and filled in by the method that diff --git a/tests/Makefile b/tests/Makefile index ddf91ebb..35c6f977 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,5 +1,5 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 -DEPS=../pb_decode.c ../pb_decode.h ../pb.h +DEPS=../pb_decode.c ../pb_decode.h ../pb.h person.h TESTS=test_decode1 decode_unittests all: $(TESTS) diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c index 746b7e96..7d3b13e5 100644 --- a/tests/decode_unittests.c +++ b/tests/decode_unittests.c @@ -104,6 +104,31 @@ int main() TEST((s = S("\x04""testfoobar"), pb_skip_string(&s) && s.bytes_left == 7)) } + { + pb_istream_t s = S("\x01\xFF\xFF\x03"); + pb_field_t f = {1, PB_LTYPE_VARINT, 0, 0, 4, 0, 0}; + uint32_t d; + COMMENT("Test pb_dec_varint using uint32_t") + TEST(pb_dec_varint(&s, &f, &d) && d == 1) + + /* Verify that no more than data_size is written. */ + d = 0; + f.data_size = 1; + TEST(pb_dec_varint(&s, &f, &d) && d == 0xFF) + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 4, 0, 0}; + int32_t d; + + COMMENT("Test pb_dec_svarint using int32_t") + TEST((s = S("\x01"), pb_dec_svarint(&s, &f, &d) && d == -1)) + TEST((s = S("\x02"), pb_dec_svarint(&s, &f, &d) && d == 1)) + TEST((s = S("\xfe\xff\xff\xff\x0f"), pb_dec_svarint(&s, &f, &d) && d == INT32_MAX)) + TEST((s = S("\xff\xff\xff\xff\x0f"), pb_dec_svarint(&s, &f, &d) && d == INT32_MIN)) + } + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); diff --git a/tests/person.proto b/tests/person.proto index ec5b4cca..01b2d4e1 100644 --- a/tests/person.proto +++ b/tests/person.proto @@ -1,7 +1,10 @@ +import "nanopb.proto"; + message Person { - required string name = 1; + required string name = 1 [(nanopb).max_size = 40]; required int32 id = 2; - optional string email = 3; + optional string email = 3 [(nanopb).max_size = 40]; + optional bytes test = 5 [default = "abc\x00\x01\x02"]; enum PhoneType { MOBILE = 0; @@ -10,9 +13,9 @@ message Person { } message PhoneNumber { - required string number = 1; + required string number = 1 [(nanopb).max_size = 40]; optional PhoneType type = 2 [default = HOME]; } - repeated PhoneNumber phone = 4; + repeated PhoneNumber phone = 4 [(nanopb).max_count = 5]; } diff --git a/tests/test_decode1.c b/tests/test_decode1.c index 4b34022f..362c404a 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -1,75 +1,8 @@ #include #include #include -#include "pb_decode.h" - -/* Structures for "Person" message */ - -typedef enum { - Person_PhoneType_MOBILE = 0, - Person_PhoneType_HOME = 1, - Person_PhoneType_WORK = 2 -} Person_PhoneType; - -typedef struct { - char number[40]; - bool has_type; - Person_PhoneType type; -} Person_PhoneNumber; - -typedef struct { - char name[40]; - int32_t id; - bool has_email; - char email[40]; - size_t phone_size; - Person_PhoneNumber phone[5]; -} Person; - -/* Field descriptions */ - - -const Person_PhoneType Person_PhoneNumber_type_default = Person_PhoneType_HOME; - -const pb_field_t Person_PhoneNumber_fields[] = { - {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING, - offsetof(Person_PhoneNumber, number), 0, - pb_membersize(Person_PhoneNumber, number), 0, 0}, - - {2, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, - pb_delta(Person_PhoneNumber, type, number), - pb_delta(Person_PhoneNumber, has_type, type), - pb_membersize(Person_PhoneNumber, type), 0, - &Person_PhoneNumber_type_default}, - - PB_LAST_FIELD -}; - -const pb_field_t Person_fields[] = { - {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING, - offsetof(Person, name), 0, - pb_membersize(Person, name), 0, 0}, - - {2, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, - pb_delta(Person, id, name), 0, - pb_membersize(Person, id), 0, 0}, - - {3, PB_HTYPE_OPTIONAL | PB_LTYPE_STRING, - offsetof(Person, email) - offsetof(Person, id), - pb_delta(Person, has_email, email), - pb_membersize(Person, email), 0, 0}, - - {4, PB_HTYPE_ARRAY | PB_LTYPE_SUBMESSAGE, - offsetof(Person, phone) - offsetof(Person, email), - pb_delta(Person, phone_size, phone), - pb_membersize(Person, phone[0]), - pb_arraysize(Person, phone), - Person_PhoneNumber_fields}, - - PB_LAST_FIELD -}; - -/* And now, the actual test program */ +#include +#include "person.h" bool print_person(pb_istream_t *stream) { @@ -81,7 +14,7 @@ bool print_person(pb_istream_t *stream) printf("Person: name '%s' id '%d' email '%s'\n", person.name, person.id, person.email); - for (i = 0; i < person.phone_size; i++) + for (i = 0; i < person.phone_count; i++) { Person_PhoneNumber *phone = &person.phone[i]; printf("PhoneNumber: number '%s' type '%d'\n", phone->number, phone->type); -- cgit 1.2.3-korg From a8d0172507d59b73b95f766aa7644147fd060f20 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 4 Aug 2011 16:49:32 +0000 Subject: Encoder git-svn-id: https://svn.kapsi.fi/jpa/nanopb@951 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- generator/nanopb_generator.py | 118 +++++++++++---- pb.h | 43 +++--- pb_decode.c | 57 ++++---- pb_decode.h | 36 +++-- pb_encode.c | 326 ++++++++++++++++++++++++++++++++++++++++++ pb_encode.h | 69 +++++++++ tests/Makefile | 8 +- tests/person.proto | 2 +- tests/test_decode1.c | 5 +- tests/test_encode1.c | 23 +++ 10 files changed, 583 insertions(+), 104 deletions(-) create mode 100644 pb_encode.c create mode 100644 pb_encode.h create mode 100644 tests/test_encode1.c diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index b182d01f..6aff3155 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -121,7 +121,7 @@ class Field: if self.max_size is None: is_callback = True else: - self.ctype = 'PB_BYTES_ARRAY(%d)' % self.max_size + self.ctype = self.struct_name + self.name + 't' elif desc.type == FieldD.TYPE_MESSAGE: self.ltype = 'PB_LTYPE_SUBMESSAGE' self.ctype = names_from_type_name(desc.type_name) @@ -146,7 +146,18 @@ class Field: result += ' %s %s%s;' % (self.ctype, self.name, self.array_decl) return result - def default_decl(self): + def types(self): + '''Return definitions for any special types this field might need.''' + if self.ltype == 'PB_LTYPE_BYTES' and self.max_size is not None: + result = 'typedef struct {\n' + result += ' size_t size;\n' + result += ' uint8_t bytes[%d];\n' % self.max_size + result += '} %s;\n' % self.ctype + else: + result = None + return result + + def default_decl(self, declaration_only = False): '''Return definition for this field's default value.''' if self.default is None: return None @@ -154,7 +165,7 @@ class Field: if self.ltype == 'PB_LTYPE_STRING': ctype = 'char' if self.max_size is None: - array_decl = '[]' + return None # Not implemented else: array_decl = '[%d]' % self.max_size default = self.default.encode('string_escape') @@ -165,9 +176,9 @@ class Field: data = ['0x%02x' % ord(c) for c in data] if self.max_size is None: - ctype = 'PB_BYTES_ARRAY(%d)' % len(data) + return None # Not implemented else: - ctype = 'PB_BYTES_ARRAY(%d)' % self.max_size + ctype = self.ctype default = '{%d, {%s}}' % (len(data), ','.join(data)) array_decl = '' @@ -175,7 +186,10 @@ class Field: ctype, default = self.ctype, self.default array_decl = '' - return 'const %s %s_default%s = %s;' % (ctype, self.struct_name + self.name, array_decl, default) + if declaration_only: + return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl) + else: + return 'const %s %s_default%s = %s;' % (ctype, self.struct_name + self.name, array_decl, default) def pb_field_t(self, prev_field_name): '''Return the pb_field_t initializer to use in the constant array. @@ -244,20 +258,32 @@ class Message: def __str__(self): result = 'typedef struct {\n' - result += '\n'.join([str(f) for f in self.fields]) + result += '\n'.join([str(f) for f in self.ordered_fields]) result += '\n} %s;' % self.name return result - def default_decl(self): + def types(self): + result = "" + for field in self.fields: + types = field.types() + if types is not None: + result += types + '\n' + return result + + def default_decl(self, declaration_only = False): result = "" for field in self.fields: - default = field.default_decl() + default = field.default_decl(declaration_only) if default is not None: result += default + '\n' return result - def pb_field_t(self): - result = 'const pb_field_t %s_fields[] = {\n' % self.name + def fields_declaration(self): + result = 'extern const pb_field_t %s_fields[%d];' % (self.name, len(self.fields)) + return result + + def fields_definition(self): + result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, len(self.fields)) prev = None for field in self.ordered_fields: @@ -282,13 +308,8 @@ def iterate_messages(desc, names = Names()): for x in iterate_messages(submsg, sub_names): yield x -def process_file(fdesc): - '''Takes a FileDescriptorProto and generate content for its header file. - Generates strings, which should be concatenated and stored to file. - ''' - - yield '/* Automatically generated nanopb header */\n' - yield '#include \n\n' +def parse_file(fdesc): + '''Takes a FileDescriptorProto and returns tuple (enum, messages).''' enums = [] messages = [] @@ -297,10 +318,23 @@ def process_file(fdesc): enums.append(Enum(Names(), enum)) for names, message in iterate_messages(fdesc): + messages.append(Message(names, message)) for enum in message.enum_type: enums.append(Enum(names, enum)) - - messages.append(Message(names, message)) + + return enums, messages + +def generate_header(headername, enums, messages): + '''Generate content for a header file. + Generates strings, which should be concatenated and stored to file. + ''' + + yield '/* Automatically generated nanopb header */\n' + + symbol = headername.replace('.', '_').upper() + yield '#ifndef _PB_%s_\n' % symbol + yield '#define _PB_%s_\n' % symbol + yield '#include \n\n' yield '/* Enum definitions */\n' for enum in enums: @@ -309,17 +343,34 @@ def process_file(fdesc): yield '/* Struct definitions */\n' messages.sort() for msg in messages: + yield msg.types() yield str(msg) + '\n\n' yield '/* Default values for struct fields */\n' for msg in messages: - yield msg.default_decl() + yield msg.default_decl(True) yield '\n' yield '/* Struct field encoding specification for nanopb */\n' for msg in messages: - yield msg.pb_field_t() + '\n\n' + yield msg.fields_declaration() + '\n' + + yield '\n#endif\n' + +def generate_source(headername, enums, messages): + '''Generate content for a source file.''' + + yield '/* Automatically generated nanopb constant definitions */\n' + yield '#include "%s"\n\n' % headername + + for msg in messages: + yield msg.default_decl(False) + yield '\n\n' + + for msg in messages: + yield msg.fields_definition() + '\n\n' + if __name__ == '__main__': import sys import os.path @@ -328,17 +379,26 @@ if __name__ == '__main__': print "Usage: " + sys.argv[0] + " file.pb" print "where file.pb has been compiled from .proto by:" print "protoc -ofile.pb file.proto" - print "Output fill be written to file.h" + print "Output fill be written to file.h and file.c" sys.exit(1) data = open(sys.argv[1]).read() fdesc = descriptor.FileDescriptorSet.FromString(data) + enums, messages = parse_file(fdesc.file[0]) - destfile = os.path.splitext(sys.argv[1])[0] + '.h' - - print "Writing to " + destfile + noext = os.path.splitext(sys.argv[1])[0] + headername = noext + '.h' + sourcename = noext + '.c' + headerbasename = os.path.basename(headername) - destfile = open(destfile, 'w') + print "Writing to " + headername + " and " + sourcename - for part in process_file(fdesc.file[0]): - destfile.write(part) + header = open(headername, 'w') + for part in generate_header(headerbasename, enums, messages): + header.write(part) + + source = open(sourcename, 'w') + for part in generate_source(headerbasename, enums, messages): + source.write(part) + + \ No newline at end of file diff --git a/pb.h b/pb.h index 27ee11fd..46d7c2ce 100644 --- a/pb.h +++ b/pb.h @@ -12,21 +12,13 @@ #define pb_packed #endif -/* Lightweight output stream. */ -typedef struct _pb_ostream_t pb_ostream_t; -struct _pb_ostream_t -{ - bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); - void *state; /* Free field for use by callback implementation */ - size_t bytes_written; -}; - -/*static inline bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) -{ - bool status = stream->callback(stream, buf, count); - stream->bytes_written += count; - return status; -}*/ +/* Wire types */ +typedef enum { + PB_WT_VARINT = 0, + PB_WT_64BIT = 1, + PB_WT_STRING = 2, + PB_WT_32BIT = 5 +} pb_wire_type_t; /* List of possible field types * Least-significant 4 bits tell the scalar type @@ -83,7 +75,8 @@ typedef enum { /* Works for all required/optional/repeated fields. * data_offset points to pb_callback_t structure. - * LTYPE is ignored. */ + * LTYPE should be 0 (it is ignored, but sometimes + * used to speculatively index an array). */ PB_HTYPE_CALLBACK = 0x30 } pb_packed pb_type_t; @@ -113,14 +106,13 @@ struct _pb_field_t { } pb_packed; /* This structure is used for 'bytes' arrays. - * It has the number of bytes in the beginning, and after that an array. */ -#define PB_BYTES_ARRAY(buffersize) \ -struct { \ - size_t size; \ - uint8_t bytes[buffersize]; \ -} - -typedef PB_BYTES_ARRAY() pb_bytes_array_t; + * It has the number of bytes in the beginning, and after that an array. + * Note that actual structs used will have a different length of bytes array. + */ +typedef struct { + size_t size; + uint8_t bytes[1]; +} pb_bytes_array_t; /* This structure is used for giving the callback function. * It is stored in the message structure and filled in by the method that @@ -139,11 +131,12 @@ typedef PB_BYTES_ARRAY() pb_bytes_array_t; * wire type. It can write multiple fields. */ typedef struct _pb_istream_t pb_istream_t; +typedef struct _pb_ostream_t pb_ostream_t; typedef struct _pb_callback_t pb_callback_t; struct _pb_callback_t { union { bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void *arg); - bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, void *arg); + bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, const void *arg); } funcs; /* Free arg for use by callback */ diff --git a/pb_decode.c b/pb_decode.c index 63008dd5..df053314 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -7,14 +7,19 @@ #include "pb_decode.h" #include -const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { - (pb_decoder_t)&pb_dec_varint, - (pb_decoder_t)&pb_dec_svarint, - (pb_decoder_t)&pb_dec_fixed, +typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest); + +/* --- Function pointers to field decoders --- + * Order in the array must match pb_action_t LTYPE numbering. + */ +static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { + &pb_dec_varint, + &pb_dec_svarint, + &pb_dec_fixed, - (pb_decoder_t)&pb_dec_bytes, - (pb_decoder_t)&pb_dec_string, - (pb_decoder_t)&pb_dec_submessage + &pb_dec_bytes, + &pb_dec_string, + &pb_dec_submessage }; /************** @@ -108,33 +113,27 @@ bool pb_skip_string(pb_istream_t *stream) * to just assume the correct type and fail safely on corrupt message. */ -enum wire_type_t { - WT_VARINT = 0, - WT_64BIT = 1, - WT_STRING = 2, - WT_32BIT = 5 -}; - static bool skip(pb_istream_t *stream, int wire_type) { switch (wire_type) { - case WT_VARINT: return pb_skip_varint(stream); - case WT_64BIT: return pb_read(stream, NULL, 8); - case WT_STRING: return pb_skip_string(stream); - case WT_32BIT: return pb_read(stream, NULL, 4); + case PB_WT_VARINT: return pb_skip_varint(stream); + case PB_WT_64BIT: return pb_read(stream, NULL, 8); + case PB_WT_STRING: return pb_skip_string(stream); + case PB_WT_32BIT: return pb_read(stream, NULL, 4); default: return false; } } -/* Read a raw value to buffer, for the purpose of passing it to callback. - * Size is maximum size on call, and actual size on return. */ -static bool read_raw_value(pb_istream_t *stream, int wire_type, uint8_t *buf, size_t *size) +/* Read a raw value to buffer, for the purpose of passing it to callback as + * a substream. Size is maximum size on call, and actual size on return. + */ +static bool read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, uint8_t *buf, size_t *size) { size_t max_size = *size; switch (wire_type) { - case WT_VARINT: + case PB_WT_VARINT: *size = 0; do { @@ -144,11 +143,11 @@ static bool read_raw_value(pb_istream_t *stream, int wire_type, uint8_t *buf, si } while (*buf++ & 0x80); return true; - case WT_64BIT: + case PB_WT_64BIT: *size = 8; return pb_read(stream, buf, 8); - case WT_32BIT: + case PB_WT_32BIT: *size = 4; return pb_read(stream, buf, 4); @@ -239,7 +238,7 @@ bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_t *iter return func(stream, iter->current, iter->pData); case PB_HTYPE_ARRAY: - if (wire_type == WT_STRING + if (wire_type == PB_WT_STRING && PB_LTYPE(iter->current->type) <= PB_LTYPE_LAST_PACKABLE) { /* Packed array */ @@ -270,7 +269,7 @@ bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_t *iter } case PB_HTYPE_CALLBACK: - if (wire_type == WT_STRING) + if (wire_type == PB_WT_STRING) { pb_callback_t *pCallback = (pb_callback_t*)iter->pData; pb_istream_t substream; @@ -419,7 +418,7 @@ bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest) #ifdef __BIG_ENDIAN__ uint8_t bytes[8] = {0}; bool status = pb_read(stream, bytes, field->data_size); - uint8_t lebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], + uint8_t bebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], bytes[3], bytes[2], bytes[1], bytes[0]}; endian_copy(dest, lebytes, field->data_size, 8); return status; @@ -428,7 +427,7 @@ bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest) #endif } -bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, uint8_t *dest) +bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) { pb_bytes_array_t *x = (pb_bytes_array_t*)dest; @@ -445,7 +444,7 @@ bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, uint8_t *dest) return pb_read(stream, x->bytes, x->size); } -bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, uint8_t *dest) +bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint32_t size; bool status; diff --git a/pb_decode.h b/pb_decode.h index 448b55db..011efdd9 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -1,18 +1,32 @@ #ifndef _PB_DECODE_H_ #define _PB_DECODE_H_ +/* pb_decode.h: Functions to decode protocol buffers. Depends on pb_decode.c. + * The main function is pb_decode. You will also need to create an input + * stream, which is easiest to do with pb_istream_t. + * + * You also need structures and their corresponding pb_field_t descriptions. + * These are usually generated from .proto-files with a script. + */ + #include #include "pb.h" /* Lightweight input stream. - * If buf is NULL, read but don't store bytes. - * You can to provide a callback function for reading or use + * You can provide a callback function for reading or use * pb_istream_from_buffer. * - * You can use state to store your own data (e.g. buffer pointer), + * Rules for callback: + * 1) Return false on IO errors. This will cause decoding to abort. + * + * 2) If buf is NULL, read but don't store bytes ("skip input"). + * + * 3) You can use state to store your own data (e.g. buffer pointer), * and rely on pb_read to verify that no-body reads past bytes_left. - * However, substreams may change bytes_left so don't use that to - * compute any pointers. + * + * 4) Your callback may be used with substreams, in which case bytes_left + * is different than from the main stream. Don't use bytes_left to compute + * any pointers. */ struct _pb_istream_t { @@ -25,6 +39,7 @@ pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); /* Decode from stream to destination struct. + * Returns true on success, false on any failure. * The actual struct pointed to by dest must match the description in fields. */ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); @@ -49,15 +64,8 @@ bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, uint8_t *dest); -bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, uint8_t *dest); +bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest); -typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest); - -/* --- Function pointers to field decoders --- - * Order in the array must match pb_action_t LTYPE numbering. - */ -const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT]; - #endif diff --git a/pb_encode.c b/pb_encode.c new file mode 100644 index 00000000..ae185c1a --- /dev/null +++ b/pb_encode.c @@ -0,0 +1,326 @@ +/* pb_encode.c -- encode a protobuf using minimal resources + * + * 2011 Petteri Aimonen + */ + +#include "pb.h" +#include "pb_encode.h" +#include + +typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, const void *src); + +/* --- Function pointers to field encoders --- + * Order in the array must match pb_action_t LTYPE numbering. + */ +static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { + &pb_enc_varint, + &pb_enc_svarint, + &pb_enc_fixed, + + &pb_enc_bytes, + &pb_enc_string, + &pb_enc_submessage +}; + +/* pb_ostream_t implementation */ + +static bool buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) +{ + uint8_t *dest = (uint8_t*)stream->state; + memcpy(dest, buf, count); + stream->state = dest + count; + return true; +} + +pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize) +{ + pb_ostream_t stream; + stream.callback = &buf_write; + stream.state = buf; + stream.max_size = bufsize; + stream.bytes_written = 0; + return stream; +} + +bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) +{ + stream->bytes_written += count; + + if (stream->callback == NULL) + return true; + + if (stream->bytes_written > stream->max_size) + return false; + + return stream->callback(stream, buf, count); +} + +/* Main encoding stuff */ + +static bool encode_array(pb_ostream_t *stream, const pb_field_t *field, + const void *pData, size_t count, pb_encoder_t func) +{ + int i; + const void *p; + size_t size; + + if (PB_LTYPE(field->type) < PB_LTYPE_LAST_PACKABLE) + { + if (!pb_encode_tag(stream, PB_WT_STRING, field->tag)) + return false; + + /* Determine the total size of packed array. */ + if (PB_LTYPE(field->type) == PB_LTYPE_FIXED) + { + size = field->data_size * count; + } + else + { + pb_ostream_t sizestream = {0}; + p = pData; + for (i = 0; i < count; i++) + { + if (!func(&sizestream, field, p)) + return false; + p = (const char*)p + field->data_size; + } + size = sizestream.bytes_written; + } + + pb_encode_varint(stream, size); + + if (stream->callback == NULL) + return pb_write(stream, NULL, size); /* Just sizing.. */ + + /* Write the data */ + p = pData; + for (i = 0; i < count; i++) + { + if (!func(stream, field, p)) + return false; + p = (const char*)p + field->data_size; + } + } + else + { + p = pData; + for (i = 0; i < count; i++) + { + if (!pb_encode_tag_for_field(stream, field)) + return false; + if (!func(stream, field, p)) + return false; + p = (const char*)p + field->data_size; + } + } + + return true; +} + +bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) +{ + const pb_field_t *field = fields; + const void *pData = src_struct; + const void *pSize; + + while (field->tag != 0) + { + pData = (const char*)pData + field->data_offset; + pSize = (const char*)pData + field->size_offset; + + pb_encoder_t func = PB_ENCODERS[PB_LTYPE(field->type)]; + + switch (PB_HTYPE(field->type)) + { + case PB_HTYPE_REQUIRED: + if (!pb_encode_tag_for_field(stream, field)) + return false; + if (!func(stream, field, pData)) + return false; + break; + + case PB_HTYPE_OPTIONAL: + if (*(bool*)pSize) + { + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!func(stream, field, pData)) + return false; + } + break; + + case PB_HTYPE_ARRAY: + if (!encode_array(stream, field, pData, *(size_t*)pSize, func)) + return false; + break; + + case PB_HTYPE_CALLBACK: + { + pb_callback_t *callback = (pb_callback_t*)pData; + if (callback->funcs.encode != NULL) + { + if (!callback->funcs.encode(stream, field, callback->arg)) + return false; + } + break; + } + } + + field++; + } + + return true; +} + +/* Helper functions */ +bool pb_encode_varint(pb_ostream_t *stream, uint64_t value) +{ + uint8_t buffer[10]; + int i = 0; + + if (value == 0) + return pb_write(stream, (uint8_t*)&value, 1); + + while (value) + { + buffer[i] = (value & 0x7F) | 0x80; + value >>= 7; + i++; + } + buffer[i-1] &= 0x7F; /* Unset top bit on last byte */ + + return pb_write(stream, buffer, i); +} + +bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number) +{ + int tag = wiretype | (field_number << 3); + return pb_encode_varint(stream, tag); +} + +bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field) +{ + pb_wire_type_t wiretype; + switch (PB_LTYPE(field->type)) + { + case PB_LTYPE_VARINT: + case PB_LTYPE_SVARINT: + wiretype = PB_WT_VARINT; + break; + + case PB_LTYPE_FIXED: + if (field->data_size == 4) + wiretype = PB_WT_32BIT; + else if (field->data_size == 8) + wiretype = PB_WT_64BIT; + else + return false; + break; + + case PB_LTYPE_BYTES: + case PB_LTYPE_STRING: + case PB_LTYPE_SUBMESSAGE: + wiretype = PB_WT_STRING; + break; + + default: + return false; + } + + return pb_encode_tag(stream, wiretype, field->tag); +} + +bool pb_encode_string(pb_ostream_t *stream, uint8_t *buffer, size_t size) +{ + if (!pb_encode_varint(stream, size)) + return false; + + return pb_write(stream, buffer, size); +} + +/* Field encoders */ + +/* Copy srcsize bytes from src so that values are casted properly. + * On little endian machine, copy to start of dest + * On big endian machine, copy to end of dest + * destsize must always be larger than srcsize + * + * Note: This is the reverse of the endian_copy in pb_decode.c. + */ +static void endian_copy(void *dest, const void *src, size_t destsize, size_t srcsize) +{ +#ifdef __BIG_ENDIAN__ + memcpy((char*)dest + (destsize - srcsize), src, srcsize); +#else + memcpy(dest, src, srcsize); +#endif +} + +bool pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src) +{ + uint64_t value = 0; + endian_copy(&value, src, sizeof(value), field->data_size); + return pb_encode_varint(stream, value); +} + +bool pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src) +{ + uint64_t value = 0; + uint64_t zigzagged; + uint64_t mask; + endian_copy(&value, src, sizeof(value), field->data_size); + + mask = 0x80 << (field->data_size * 8); + zigzagged = (value & ~mask) << 1; + if (value & mask) zigzagged |= 1; + + return pb_encode_varint(stream, value); +} + +bool pb_enc_fixed(pb_ostream_t *stream, const pb_field_t *field, const void *src) +{ + #ifdef __BIG_ENDIAN__ + uint8_t bytes[8] = {0}; + endian_copy(bytes, src, sizeof(bytes), field->data_size); + uint8_t lebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], + bytes[3], bytes[2], bytes[1], bytes[0]}; + return pb_write(stream, lebytes, field->data_size); + #else + return pb_write(stream, (uint8_t*)src, field->data_size); + #endif +} + +bool pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) +{ + pb_bytes_array_t *bytes = (pb_bytes_array_t*)src; + return pb_encode_string(stream, bytes->bytes, bytes->size); +} + +bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src) +{ + return pb_encode_string(stream, (uint8_t*)src, strlen((char*)src)); +} + +bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src) +{ + pb_ostream_t sizestream = {0}; + size_t size; + + if (field->ptr == NULL) + return false; + + if (!pb_encode(&sizestream, (pb_field_t*)field->ptr, src)) + return false; + + size = sizestream.bytes_written; + + if (!pb_encode_varint(stream, size)) + return false; + + if (stream->callback == NULL) + return pb_write(stream, NULL, size); /* Just sizing */ + + return pb_encode(stream, (pb_field_t*)field->ptr, src); +} + diff --git a/pb_encode.h b/pb_encode.h new file mode 100644 index 00000000..95f3483a --- /dev/null +++ b/pb_encode.h @@ -0,0 +1,69 @@ +#ifndef _PB_ENCODE_H_ +#define _PB_ENCODE_H_ + +/* pb_encode.h: Functions to encode protocol buffers. Depends on pb_encode.c. + * The main function is pb_encode. You also need an output stream, structures + * and their field descriptions (just like with pb_decode). + */ + +#include +#include "pb.h" + +/* Lightweight output stream. + * You can provide callback for writing or use pb_ostream_from_buffer. + * + * Alternatively, callback can be NULL in which case the stream will just + * count the number of bytes that would have been written. In this case + * max_size is not checked. + * + * Rules for callback: + * 1) Return false on IO errors. This will cause decoding to abort. + * + * 2) You can use state to store your own data (e.g. buffer pointer). + * + * 3) pb_write will update bytes_written before your callback runs. + * + * 4) Your callback will be always used with the same pb_ostream_t. + * There are no substreams when encoding. + */ +struct _pb_ostream_t +{ + bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); + void *state; /* Free field for use by callback implementation */ + size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ + size_t bytes_written; +}; + +pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize); +bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); + +/* Encode struct to given output stream. + * Returns true on success, false on any failure. + * The actual struct pointed to by src_struct must match the description in fields. + * All required fields in the struct are assumed to have been filled in. + */ +bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); + +/* --- Helper functions --- + * You may want to use these from your caller or callbacks. + */ + +bool pb_encode_varint(pb_ostream_t *stream, uint64_t value); +bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number); +bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field); +bool pb_encode_string(pb_ostream_t *stream, uint8_t *buffer, size_t size); + +/* --- Field encoders --- + * Each encoder writes the content for the field. + * The tag/wire type has been written already. + */ + +bool pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src); +bool pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); +bool pb_enc_fixed(pb_ostream_t *stream, const pb_field_t *field, const void *src); + +bool pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src); +bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src); +bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src); + +#endif \ No newline at end of file diff --git a/tests/Makefile b/tests/Makefile index 35c6f977..7450e1ed 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,14 +1,14 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 -DEPS=../pb_decode.c ../pb_decode.h ../pb.h person.h -TESTS=test_decode1 decode_unittests +DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h person.h +TESTS=test_decode1 test_encode1 decode_unittests all: $(TESTS) clean: - rm -f test_decode1 + rm -f $(TESTS) %: %.c $(DEPS) - $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c + $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c run_unittests: decode_unittests ./decode_unittests diff --git a/tests/person.proto b/tests/person.proto index 01b2d4e1..5befb076 100644 --- a/tests/person.proto +++ b/tests/person.proto @@ -4,7 +4,7 @@ message Person { required string name = 1 [(nanopb).max_size = 40]; required int32 id = 2; optional string email = 3 [(nanopb).max_size = 40]; - optional bytes test = 5 [default = "abc\x00\x01\x02"]; + optional bytes test = 5 [default="\x00\x01\x02", (nanopb).max_size = 20]; enum PhoneType { MOBILE = 0; diff --git a/tests/test_decode1.c b/tests/test_decode1.c index 362c404a..cc4688dc 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -1,9 +1,10 @@ #include -#include -#include #include #include "person.h" +/* This test has only one source file anyway.. */ +#include "person.c" + bool print_person(pb_istream_t *stream) { int i; diff --git a/tests/test_encode1.c b/tests/test_encode1.c new file mode 100644 index 00000000..b4998f47 --- /dev/null +++ b/tests/test_encode1.c @@ -0,0 +1,23 @@ +#include +#include +#include "person.h" + +/* This test has only one source file anyway.. */ +#include "person.c" + +bool callback(pb_ostream_t *stream, const uint8_t *buf, size_t count) +{ + return fwrite(buf, 1, count, stdout) == count; +} + +int main() +{ + Person person = {"Test Person 99", 99, true, "test@person.com", + 1, {{"555-12345678", true, Person_PhoneType_MOBILE}}}; + + pb_ostream_t stream = {&callback, 0, SIZE_MAX, 0}; + + pb_encode(&stream, Person_fields, &person); + + return 0; +} -- cgit 1.2.3-korg From 0c5bc4caf9cfb221822fead442b462be2f47b97b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 10 Aug 2011 17:54:15 +0000 Subject: Unittests for encode git-svn-id: https://svn.kapsi.fi/jpa/nanopb@952 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb_encode.c | 33 +++++++------ pb_encode.h | 6 ++- tests/Makefile | 9 ++-- tests/decode_unittests.c | 26 +++++----- tests/encode_unittests.c | 123 +++++++++++++++++++++++++++++++++++++++++++++++ tests/unittests.h | 14 ++++++ 6 files changed, 179 insertions(+), 32 deletions(-) create mode 100644 tests/encode_unittests.c create mode 100644 tests/unittests.h diff --git a/pb_encode.c b/pb_encode.c index ae185c1a..63ca2889 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -44,15 +44,17 @@ pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize) bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) { - stream->bytes_written += count; - - if (stream->callback == NULL) - return true; - - if (stream->bytes_written > stream->max_size) - return false; + if (stream->callback != NULL) + { + if (stream->bytes_written + count > stream->max_size) + return false; + + if (!stream->callback(stream, buf, count)) + return false; + } - return stream->callback(stream, buf, count); + stream->bytes_written += count; + return true; } /* Main encoding stuff */ @@ -231,7 +233,7 @@ bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field) return pb_encode_tag(stream, wiretype, field->tag); } -bool pb_encode_string(pb_ostream_t *stream, uint8_t *buffer, size_t size) +bool pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size) { if (!pb_encode_varint(stream, size)) return false; @@ -268,14 +270,17 @@ bool pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *s { uint64_t value = 0; uint64_t zigzagged; - uint64_t mask; + uint64_t signbitmask, xormask; endian_copy(&value, src, sizeof(value), field->data_size); - mask = 0x80 << (field->data_size * 8); - zigzagged = (value & ~mask) << 1; - if (value & mask) zigzagged |= 1; + signbitmask = (uint64_t)0x80 << (field->data_size * 8 - 8); + xormask = ((uint64_t)-1) >> (64 - field->data_size * 8); + if (value & signbitmask) + zigzagged = ((value ^ xormask) << 1) | 1; + else + zigzagged = value << 1; - return pb_encode_varint(stream, value); + return pb_encode_varint(stream, zigzagged); } bool pb_enc_fixed(pb_ostream_t *stream, const pb_field_t *field, const void *src) diff --git a/pb_encode.h b/pb_encode.h index 95f3483a..17ba5b9a 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -21,7 +21,7 @@ * * 2) You can use state to store your own data (e.g. buffer pointer). * - * 3) pb_write will update bytes_written before your callback runs. + * 3) pb_write will update bytes_written after your callback runs. * * 4) Your callback will be always used with the same pb_ostream_t. * There are no substreams when encoding. @@ -50,8 +50,10 @@ bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_ bool pb_encode_varint(pb_ostream_t *stream, uint64_t value); bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number); +/* Encode tag based on LTYPE and field number defined in the field structure. */ bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field); -bool pb_encode_string(pb_ostream_t *stream, uint8_t *buffer, size_t size); +/* Write length as varint and then the contents of buffer. */ +bool pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size); /* --- Field encoders --- * Each encoder writes the content for the field. diff --git a/tests/Makefile b/tests/Makefile index 7450e1ed..84c035e5 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,8 +1,8 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 -DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h person.h -TESTS=test_decode1 test_encode1 decode_unittests +DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h person.h unittests.h +TESTS=test_decode1 test_encode1 decode_unittests encode_unittests -all: $(TESTS) +all: $(TESTS) run_unittests clean: rm -f $(TESTS) @@ -10,8 +10,9 @@ clean: %: %.c $(DEPS) $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c -run_unittests: decode_unittests +run_unittests: decode_unittests encode_unittests ./decode_unittests + ./encode_unittests run_fuzztest: test_decode1 bash -c 'I=1; while cat /dev/urandom | ./test_decode1 > /dev/null; do I=$$(($$I+1)); echo -en "\r$$I"; done' \ No newline at end of file diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c index 7d3b13e5..ff4c6b46 100644 --- a/tests/decode_unittests.c +++ b/tests/decode_unittests.c @@ -1,17 +1,7 @@ #include #include #include "pb_decode.h" - -#define COMMENT(x) printf("\n----" x "----\n"); -#define STR(x) #x -#define STR2(x) STR(x) -#define TEST(x) \ - if (!(x)) { \ - fprintf(stderr, __FILE__ ":" STR2(__LINE__) " FAILED:" #x "\n"); \ - status = 1; \ - } else { \ - printf("OK: " #x "\n"); \ - } +#include "unittests.h" #define S(x) pb_istream_from_buffer((uint8_t*)x, sizeof(x)) @@ -114,7 +104,7 @@ int main() /* Verify that no more than data_size is written. */ d = 0; f.data_size = 1; - TEST(pb_dec_varint(&s, &f, &d) && d == 0xFF) + TEST(pb_dec_varint(&s, &f, &d) && (d == 0xFF || d == 0xFF000000)) } { @@ -129,6 +119,18 @@ int main() TEST((s = S("\xff\xff\xff\xff\x0f"), pb_dec_svarint(&s, &f, &d) && d == INT32_MIN)) } + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 8, 0, 0}; + uint64_t d; + + COMMENT("Test pb_dec_svarint using uint64_t") + TEST((s = S("\x01"), pb_dec_svarint(&s, &f, &d) && d == -1)) + TEST((s = S("\x02"), pb_dec_svarint(&s, &f, &d) && d == 1)) + TEST((s = S("\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_dec_svarint(&s, &f, &d) && d == INT64_MAX)) + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_dec_svarint(&s, &f, &d) && d == INT64_MIN)) + } + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); diff --git a/tests/encode_unittests.c b/tests/encode_unittests.c new file mode 100644 index 00000000..645dd21a --- /dev/null +++ b/tests/encode_unittests.c @@ -0,0 +1,123 @@ +#include +#include +#include "pb_encode.h" +#include "unittests.h" + +bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) +{ + /* Allow only 'x' to be written */ + while (count--) + { + if (*buf++ != 'x') + return false; + } + return true; +} + +/* Check that expression x writes data y. + * Y is a string, which may contain null bytes. Null terminator is ignored. + */ +#define WRITES(x, y) \ +memset(buffer, 0xAA, sizeof(buffer)), \ +s = pb_ostream_from_buffer(buffer, sizeof(buffer)), \ +(x) && \ +memcmp(buffer, y, sizeof(y) - 1) == 0 && \ +buffer[sizeof(y) - 1] == 0xAA + +int main() +{ + int status = 0; + + { + uint8_t buffer1[] = "foobartest1234"; + uint8_t buffer2[sizeof(buffer1)]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer2, sizeof(buffer1)); + + COMMENT("Test pb_write and pb_ostream_t"); + TEST(pb_write(&stream, buffer1, sizeof(buffer1))); + TEST(memcmp(buffer1, buffer2, sizeof(buffer1)) == 0); + TEST(!pb_write(&stream, buffer1, 1)); + TEST(stream.bytes_written == sizeof(buffer1)); + } + + { + uint8_t buffer1[] = "xxxxxxx"; + pb_ostream_t stream = {&streamcallback, 0, SIZE_MAX, 0}; + + COMMENT("Test pb_write with custom callback"); + TEST(pb_write(&stream, buffer1, 5)); + buffer1[0] = 'a'; + TEST(!pb_write(&stream, buffer1, 5)); + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + + COMMENT("Test pb_encode_varint") + TEST(WRITES(pb_encode_varint(&s, 0), "\0")); + TEST(WRITES(pb_encode_varint(&s, 1), "\1")); + TEST(WRITES(pb_encode_varint(&s, 0x7F), "\x7F")); + TEST(WRITES(pb_encode_varint(&s, 0x80), "\x80\x01")); + TEST(WRITES(pb_encode_varint(&s, UINT32_MAX), "\xFF\xFF\xFF\xFF\x0F")); + TEST(WRITES(pb_encode_varint(&s, UINT64_MAX), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01")); + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + + COMMENT("Test pb_encode_tag") + TEST(WRITES(pb_encode_tag(&s, PB_WT_STRING, 5), "\x2A")); + TEST(WRITES(pb_encode_tag(&s, PB_WT_VARINT, 99), "\x98\x06")); + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + pb_field_t field = {10, PB_LTYPE_SVARINT}; + + COMMENT("Test pb_encode_tag_for_field") + TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x50")); + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + + COMMENT("Test pb_encode_string") + TEST(WRITES(pb_encode_string(&s, (const uint8_t*)"abcd", 4), "\x04""abcd")); + TEST(WRITES(pb_encode_string(&s, (const uint8_t*)"abcd\x00", 5), "\x05""abcd\x00")); + TEST(WRITES(pb_encode_string(&s, (const uint8_t*)"", 0), "\x00")); + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + uint8_t value = 1; + int8_t svalue = -1; + int32_t max = INT32_MAX; + int32_t min = INT32_MIN; + int64_t lmax = INT64_MAX; + int64_t lmin = INT64_MIN; + pb_field_t field = {1, PB_LTYPE_VARINT, 0, 0, sizeof(value)}; + + COMMENT("Test pb_enc_varint and pb_enc_svarint") + TEST(WRITES(pb_enc_varint(&s, &field, &value), "\x01")); + TEST(WRITES(pb_enc_svarint(&s, &field, &svalue), "\x01")); + TEST(WRITES(pb_enc_svarint(&s, &field, &value), "\x02")); + + field.data_size = sizeof(max); + TEST(WRITES(pb_enc_svarint(&s, &field, &max), "\xfe\xff\xff\xff\x0f")); + TEST(WRITES(pb_enc_svarint(&s, &field, &min), "\xff\xff\xff\xff\x0f")); + + field.data_size = sizeof(lmax); + TEST(WRITES(pb_enc_svarint(&s, &field, &lmax), "\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01")); + TEST(WRITES(pb_enc_svarint(&s, &field, &lmin), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01")); + } + + if (status != 0) + fprintf(stdout, "\n\nSome tests FAILED!\n"); + + return status; +} diff --git a/tests/unittests.h b/tests/unittests.h new file mode 100644 index 00000000..c2b470ad --- /dev/null +++ b/tests/unittests.h @@ -0,0 +1,14 @@ +#include + +#define COMMENT(x) printf("\n----" x "----\n"); +#define STR(x) #x +#define STR2(x) STR(x) +#define TEST(x) \ + if (!(x)) { \ + fprintf(stderr, "\033[31;1mFAILED:\033[22;39m " __FILE__ ":" STR2(__LINE__) " " #x "\n"); \ + status = 1; \ + } else { \ + printf("\033[32;1mOK:\033[22;39m " #x "\n"); \ + } + + -- cgit 1.2.3-korg From 09f92bafa59460ea4597c557e469e982386c9e3b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 10 Aug 2011 20:08:06 +0000 Subject: Started writing documentation git-svn-id: https://svn.kapsi.fi/jpa/nanopb@953 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/Makefile | 4 + docs/index.rst | 92 +++++++++++++++++++++++ docs/lsr.css | 231 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ pb.h | 2 +- 4 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 docs/Makefile create mode 100644 docs/index.rst create mode 100644 docs/lsr.css diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..e4cac27d --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,4 @@ +all: index.html + +%.html: %.rst + rst2html --stylesheet=lsr.css --link-stylesheet $< $@ \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..93b06e86 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,92 @@ +============================================= +Nanopb: Protocol Buffers with small code size +============================================= + +Nanopb is an ANSI-C library for encoding and decoding messages in Google's `Protocol Buffers`__ format with minimal requirements for RAM and code space. +It is primarily suitable for 32-bit microcontrollers. + +__ http://code.google.com/apis/protocolbuffers/ + +Overall structure +================= + +For the runtime program, you always need *pb.h* for type declarations. +Depending on whether you want to encode, decode or both, you also need *pb_encode.h/c* or *pb_decode.h/c*. + +The high-level encoding and decoding functions take an array of *pb_field_t* structures, which describes the fields of a message structure. Usually you want these autogenerated from a *.proto* file. The tool string *nanopb_generator.py* accomplishes this. + +So a typical project might include these files: + +1) Nanopb runtime library: + - pb.h + - pb_decode.h and pb_decode.c + - pb_encode.h and pb_encode.c +2) Protocol description (you can have many): + - person.proto + - person.c (autogenerated, contains initializers for const arrays) + - person.h (autogenerated, contains type declarations) + +Features and limitations +======================== + +**Features** + +#) Pure C runtime +#) Small code size (2–10 kB depending on processor) +#) Small ram usage (typically 200 bytes) +#) Allows specifying maximum size for strings and arrays, so that they can be allocated statically. +#) No malloc needed: everything is stored on the stack. +#) You can use either encoder or decoder alone to cut the code size in half. + +**Limitations** + +#) User must provide callbacks when decoding arrays or strings without maximum size. +#) Some speed has been sacrificed for code size. For example varint calculations are always done in 64 bits. +#) Encoding is focused on writing to streams. For memory buffers only it could be made more efficient. +#) The deprecated Protocol Buffers feature called "groups" is not supported. + +Getting started +=============== + +For starters, consider this simple message:: + + message Example { + required int32 value = 1; + } + +Save this in *example.proto* and run it through *nanopb_generate.py*. You +should now have in *example.h*:: + + typedef struct { + int32_t value; + } Example; + + extern const pb_field_t Example_fields[2]; + +Now in your main program do this to encode a message:: + + Example mymessage = {42}; + uint8_t buffer[10]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + pb_encode(&stream, Example_fields, &mymessage); + +After that, buffer will contain the encoded message. +The number of bytes in the message is stored in *stream.bytes_written*. +You can feed the message to *protoc --decode=Example example.proto* to verify its validity. + +Library reference +================= + +**Encoding** + +**Decoding** + +**Specifying field options** + +**Generated code** + +Wishlist +======== +#) A specialized encoder for encoding to a memory buffer. Should serialize in reverse order to avoid having to determine submessage size beforehand. +#) A cleaner rewrite of the source generator. +#) Better performance for 16- and 8-bit platforms. diff --git a/docs/lsr.css b/docs/lsr.css new file mode 100644 index 00000000..758b231b --- /dev/null +++ b/docs/lsr.css @@ -0,0 +1,231 @@ +/* +Author: Peter Parente +Date: 2008/01/22 +Version: 1.0 (modified) +Copyright: This stylesheet has been placed in the public domain - free to edit and use for all uses. +*/ + +body { + font: 100% sans-serif; + background: #ffffff; + color: black; + margin: 2em; + padding: 0em 2em; +} + +p.topic-title { + font-weight: bold; +} + +table.docinfo { + text-align: left; + margin: 2em 0em; +} + +a[href] { + color: #436976; + background-color: transparent; +} + +a.toc-backref { + text-decoration: none; +} + +h1 a[href] { + color: #003a6b; + text-decoration: none; + background-color: transparent; +} + +a.strong { + font-weight: bold; +} + +img { + margin: 0; + border: 0; +} + +p { + margin: 0.5em 0 1em 0; + line-height: 1.5em; +} + +p a:visited { + color: purple; + background-color: transparent; +} + +p a:active { + color: red; + background-color: transparent; +} + +a:hover { + text-decoration: none; +} + +p img { + border: 0; + margin: 0; +} + +p.rubric { + font-weight: bold; + font-style: italic; +} + +em { + font-style: normal; + font-family: monospace; + font-weight: bold; +} + +pre { + border-left: 3px double #aaa; + padding-left: 10px; +} + +h1.title { + color: #003a6b; + font-size: 250%; + margin-bottom: 0em; +} + +h2.subtitle { + color: #003a6b; + border-bottom: 0px; +} + +h1, h2, h3, h4, h5, h6 { + color: #555; + background-color: transparent; + margin: 0em; + padding-top: 0.5em; +} + +h1 { + font-size: 160%; + margin-bottom: 0.5em; + border-bottom: 2px solid #aaa; +} + +h2 { + font-size: 140%; + margin-bottom: 0.5em; + border-bottom: 1px solid #aaa; +} + +h3 { + font-size: 130%; + margin-bottom: 0.5em; +} + +h4 { + font-size: 110%; + font-weight: bold; + margin-bottom: 0.5em; +} + +h5 { + font-size: 105%; + font-weight: bold; + margin-bottom: 0.5em; +} + +h6 { + font-size: 100%; + font-weight: bold; + margin-bottom: 0.5em; +} + +dt { + font-style: italic; +} + +dd { + margin-bottom: 1.5em; +} + +div.admonition, div.note, div.tip, div.caution, div.important { + margin: 2em 2em; + padding: 0em 1em; + border-top: 1px solid #aaa; + border-left: 1px solid #aaa; + border-bottom: 2px solid #555; + border-right: 2px solid #555; +} + +div.important { + background: transparent url('../images/important.png') 10px 2px no-repeat; +} + +div.caution { + background: transparent url('../images/caution.png') 10px 2px no-repeat; +} + +div.note { + background: transparent url('../images/note.png') 10px 2px no-repeat; +} + +div.tip { + background: transparent url('../images/tip.png') 10px 2px no-repeat; +} + +div.admonition-example { + background: transparent url('../images/tip.png') 10px 2px no-repeat; +} + +div.admonition-critical-example { + background: transparent url('../images/important.png') 10px 2px no-repeat; +} + +p.admonition-title { + font-weight: bold; + border-bottom: 1px solid #aaa; + padding-left: 30px; +} + +table.docutils { + text-align: left; + border: 1px solid gray; + border-collapse: collapse; + width: 100%; + margin: 1.5em 0em; +} + +table.docutils caption { + font-style: italic; +} + +table.docutils td, table.docutils th { + padding: 0.25em 0.5em; +} + +table.docutils th { + background-color: #dddddd; +} + +div.sidebar { + width: 33%; + float: right; + margin: 0em 2em; + padding: 0em 1em; + border-top: 1px solid #aaa; + border-left: 1px solid #aaa; + border-bottom: 2px solid #555; + border-right: 2px solid #555; +} + +p.sidebar-title { + margin-bottom: 0em; + color: #003a6b; + border-bottom: 1px solid #aaa; + font-weight: bold; +} + +p.sidebar-subtitle { + margin-top: 0em; + font-style: italic; + color: #003a6b; +} diff --git a/pb.h b/pb.h index 46d7c2ce..1d32ee9b 100644 --- a/pb.h +++ b/pb.h @@ -12,7 +12,7 @@ #define pb_packed #endif -/* Wire types */ +/* Wire types. Library user needs these only in encoder callbacks. */ typedef enum { PB_WT_VARINT = 0, PB_WT_64BIT = 1, -- cgit 1.2.3-korg From 6dfba365b00175eae7e8b83aaf5d29ce190fd9eb Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 11 Aug 2011 19:22:36 +0000 Subject: Documenting and improving stream behaviour git-svn-id: https://svn.kapsi.fi/jpa/nanopb@954 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/Makefile | 2 +- docs/concepts.rst | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++++ docs/encoding.rst | 39 +++++++++++++++++++++ docs/index.rst | 2 +- docs/lsr.css | 8 ++--- pb_decode.c | 2 +- pb_encode.c | 26 +++++++++++--- pb_encode.h | 6 ++-- tests/test_decode1.c | 24 ++++++++++--- tests/test_encode1.c | 7 ++-- 10 files changed, 193 insertions(+), 21 deletions(-) create mode 100644 docs/concepts.rst create mode 100644 docs/encoding.rst diff --git a/docs/Makefile b/docs/Makefile index e4cac27d..80b61b62 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,4 +1,4 @@ -all: index.html +all: index.html encoding.html %.html: %.rst rst2html --stylesheet=lsr.css --link-stylesheet $< $@ \ No newline at end of file diff --git a/docs/concepts.rst b/docs/concepts.rst new file mode 100644 index 00000000..30daed01 --- /dev/null +++ b/docs/concepts.rst @@ -0,0 +1,98 @@ +====================== +Nanopb: Basic concepts +====================== + +The things outlined here are common to both the encoder and the decoder part. + +Return values and error handling +================================ + +Most functions in nanopb return *bool*. *True* means success, *false* means failure. + +Because code size is of the essence, nanopb doesn't give any information about the cause of the error. However, there are few possible sources of errors: + +1) Running out of memory. Because everything is allocated from the stack, nanopb can't detect this itself. Encoding or decoding the same type of a message always takes the same amount of stack space. Therefore, if it works once, it works always. +2) Invalid field description. These are usually stored as constants, so if it works under the debugger, it always does. +3) IO errors in your own stream callbacks. Because encoding/decoding stops at the first error, you can overwrite the *state* field in the struct and store your own error code there. +4) Errors in your callback functions. You can use the state field in the callback structure. +5) Exceeding the max_size or bytes_left of a stream. +6) Exceeding the max_size of a string or array field +7) Invalid protocol buffers binary message. It's not like you could recover from it anyway, so a simple failure should be enough. + +In my opinion, it is enough that 1) and 2) can be resolved using a debugger. + +However, you may be interested which of the remaining conditions caused the error. For 3) and 4), you can check the state. If you have to detect 5) and 6), you should convert the fields to callback type. Any remaining problem is of type 7). + +Streams +======= + +Nanopb uses streams for accessing the data in encoded format. +The stream abstraction is very lightweight, and consists of a structure (*pb_ostream_t* or *pb_istream_t*) which contains a pointer to a callback function. + +There are a few generic rules for callback functions: + +#) Return false on IO errors. The encoding or decoding process will abort immediately. +#) Use state to store your own data, such as a file descriptor. +#) *bytes_written* and *bytes_left* are updated by *pb_write* and *pb_read*. Don't touch them. +#) Your callback may be used with substreams. In this case *bytes_left*, *bytes_written* and *max_size* have smaller values than the original stream. Don't use these values to calculate pointers. + +Output streams +-------------- + +:: + + struct _pb_ostream_t + { + bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); + void *state; + size_t max_size; + size_t bytes_written; + }; + +The *callback* for output stream may be NULL, in which case the stream simply counts the number of bytes written. In this case, *max_size* is ignored. + +Otherwise, if *bytes_written* + bytes_to_be_written is larger than *max_size*, *pb_write* returns false before doing anything else. If you don't want to limit the size of the stream, pass SIZE_MAX. + +Most commonly you want to initialize *bytes_written* to 0. It doesn't matter to the library, though. + +**Example 1:** + +This is the way to get the size of the message without storing it anywhere:: + + Person myperson = ...; + pb_ostream_t sizestream = {0}; + pb_encode(&sizestream, Person_fields, &myperson); + printf("Encoded size is %d\n", sizestream.bytes_written); + +**Example 2:** + +Writing to stdout:: + + bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) + { + FILE *file = (FILE*) stream->state; + return fwrite(buf, 1, count, file) == count; + } + + pb_ostream_t stdoutstream = {&streamcallback, stdout, SIZE_MAX, 0}; + +Input streams +------------- +For input streams, there are a few extra rules: +#) If buf is NULL, read from stream but don't store the data. This is used to skip unknown input. +#) You don't need to know the length of the message in advance. After getting EOF error when reading, set bytes_left to 0 and return false. Pb_decode will detect this and if the EOF was in a proper position, it will return true. + +:: + + struct _pb_istream_t + { + bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count); + void *state; + size_t bytes_left; + }; + +The *callback* must always be a function pointer. + +*Bytes_left* is an upper limit on the number of bytes that will be read. You can use SIZE_MAX if your callback handles EOF as described above. + +**Example** \ No newline at end of file diff --git a/docs/encoding.rst b/docs/encoding.rst new file mode 100644 index 00000000..e4e0cd7f --- /dev/null +++ b/docs/encoding.rst @@ -0,0 +1,39 @@ +========================= +Nanopb: Encoding messages +========================= + +The basic way to encode messages is to: + +1) Write a callback function for whatever stream you want to write the message to. +2) Fill a structure with your data. +3) Call pb_encode with the stream, a pointer to *const pb_field_t* array and a pointer to your structure. + +A few extra steps are necessary if you need to know the size of the message beforehand, or if you have dynamically sized fields. + +Output streams +============== + +This is the contents of *pb_ostream_t* structure:: + + typedef struct _pb_ostream_t pb_ostream_t; + struct _pb_ostream_t + { + bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); + void *state; + size_t max_size; + size_t bytes_written; + }; + +This, combined with the pb_write function, provides a light-weight abstraction +for whatever destination you want to write data to. + +*callback* should be a pointer to your callback function. These are the rules for it: + +1) Return false on IO errors. This will cause encoding to abort. + * + * 2) You can use state to store your own data (e.g. buffer pointer). + * + * 3) pb_write will update bytes_written after your callback runs. + * + * 4) Substreams will modify max_size and bytes_written. Don't use them to + * calculate any pointers. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 93b06e86..5a5cc826 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -89,4 +89,4 @@ Wishlist ======== #) A specialized encoder for encoding to a memory buffer. Should serialize in reverse order to avoid having to determine submessage size beforehand. #) A cleaner rewrite of the source generator. -#) Better performance for 16- and 8-bit platforms. +#) Better performance for 16- and 8-bit platforms: use smaller datatypes where possible. diff --git a/docs/lsr.css b/docs/lsr.css index 758b231b..081bf068 100644 --- a/docs/lsr.css +++ b/docs/lsr.css @@ -88,7 +88,7 @@ pre { h1.title { color: #003a6b; - font-size: 250%; + font-size: 180%; margin-bottom: 0em; } @@ -105,19 +105,19 @@ h1, h2, h3, h4, h5, h6 { } h1 { - font-size: 160%; + font-size: 150%; margin-bottom: 0.5em; border-bottom: 2px solid #aaa; } h2 { - font-size: 140%; + font-size: 130%; margin-bottom: 0.5em; border-bottom: 1px solid #aaa; } h3 { - font-size: 130%; + font-size: 120%; margin-bottom: 0.5em; } diff --git a/pb_decode.c b/pb_decode.c index df053314..e2888f22 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -349,7 +349,7 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc uint32_t temp; int tag, wire_type; if (!pb_decode_varint32(stream, &temp)) - return false; + return stream->bytes_left == 0; /* Was it EOF? */ tag = temp >> 3; wire_type = temp & 7; diff --git a/pb_encode.c b/pb_encode.c index 63ca2889..19a531cf 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -309,16 +309,17 @@ bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *sr bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - pb_ostream_t sizestream = {0}; + pb_ostream_t substream = {0}; size_t size; + bool status; if (field->ptr == NULL) return false; - if (!pb_encode(&sizestream, (pb_field_t*)field->ptr, src)) + if (!pb_encode(&substream, (pb_field_t*)field->ptr, src)) return false; - size = sizestream.bytes_written; + size = substream.bytes_written; if (!pb_encode_varint(stream, size)) return false; @@ -326,6 +327,23 @@ bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void if (stream->callback == NULL) return pb_write(stream, NULL, size); /* Just sizing */ - return pb_encode(stream, (pb_field_t*)field->ptr, src); + if (stream->bytes_written + size > stream->max_size) + return false; + + /* Use a substream to verify that a callback doesn't write more than + * what it did the first time. */ + substream.callback = stream->callback; + substream.state = stream->state; + substream.max_size = size; + substream.bytes_written = 0; + + status = pb_encode(stream, (pb_field_t*)field->ptr, src); + + stream->bytes_written += substream.bytes_written; + + if (substream.bytes_written != size) + return false; + + return status; } diff --git a/pb_encode.h b/pb_encode.h index 17ba5b9a..b341602d 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -17,14 +17,14 @@ * max_size is not checked. * * Rules for callback: - * 1) Return false on IO errors. This will cause decoding to abort. + * 1) Return false on IO errors. This will cause encoding to abort. * * 2) You can use state to store your own data (e.g. buffer pointer). * * 3) pb_write will update bytes_written after your callback runs. * - * 4) Your callback will be always used with the same pb_ostream_t. - * There are no substreams when encoding. + * 4) Substreams will modify max_size and bytes_written. Don't use them to + * calculate any pointers. */ struct _pb_ostream_t { diff --git a/tests/test_decode1.c b/tests/test_decode1.c index cc4688dc..a2c7f428 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -24,12 +24,28 @@ bool print_person(pb_istream_t *stream) return true; } -int main() +bool callback(pb_istream_t *stream, uint8_t *buf, size_t count) { - uint8_t buffer[512]; - size_t size = fread(buffer, 1, 512, stdin); + FILE *file = (FILE*)stream->state; + bool status; + + if (buf == NULL) + { + while (count-- && fgetc(file) != EOF); + return count == 0; + } + + status = (fread(buf, 1, count, file) == count); - pb_istream_t stream = pb_istream_from_buffer(buffer, size); + if (feof(file)) + stream->bytes_left = 0; + + return status; +} + +int main() +{ + pb_istream_t stream = {&callback, stdin, SIZE_MAX}; if (!print_person(&stream)) printf("Parsing failed.\n"); diff --git a/tests/test_encode1.c b/tests/test_encode1.c index b4998f47..99be7cb5 100644 --- a/tests/test_encode1.c +++ b/tests/test_encode1.c @@ -5,9 +5,10 @@ /* This test has only one source file anyway.. */ #include "person.c" -bool callback(pb_ostream_t *stream, const uint8_t *buf, size_t count) +bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) { - return fwrite(buf, 1, count, stdout) == count; + FILE *file = (FILE*) stream->state; + return fwrite(buf, 1, count, file) == count; } int main() @@ -15,7 +16,7 @@ int main() Person person = {"Test Person 99", 99, true, "test@person.com", 1, {{"555-12345678", true, Person_PhoneType_MOBILE}}}; - pb_ostream_t stream = {&callback, 0, SIZE_MAX, 0}; + pb_ostream_t stream = {&streamcallback, stdout, SIZE_MAX, 0}; pb_encode(&stream, Person_fields, &person); -- cgit 1.2.3-korg From 842d52633d650286ce62490362f8dfa356e17800 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 14 Aug 2011 20:11:05 +0000 Subject: More documentation, small improvements git-svn-id: https://svn.kapsi.fi/jpa/nanopb@955 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/Makefile | 2 +- docs/concepts.rst | 139 ++++++++++++--- docs/encoding.rst | 31 +--- docs/index.rst | 1 + docs/lsr.css | 15 +- docs/reference.rst | 385 ++++++++++++++++++++++++++++++++++++++++++ generator/nanopb_generator.py | 8 +- pb.h | 24 ++- pb_decode.c | 21 ++- pb_decode.h | 3 +- pb_encode.c | 4 +- tests/Makefile | 4 + tests/decode_unittests.c | 29 +--- 13 files changed, 562 insertions(+), 104 deletions(-) create mode 100644 docs/reference.rst diff --git a/docs/Makefile b/docs/Makefile index 80b61b62..e0ddc37d 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,4 +1,4 @@ -all: index.html encoding.html +all: index.html encoding.html concepts.html reference.html %.html: %.rst rst2html --stylesheet=lsr.css --link-stylesheet $< $@ \ No newline at end of file diff --git a/docs/concepts.rst b/docs/concepts.rst index 30daed01..f2475aec 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -4,24 +4,9 @@ Nanopb: Basic concepts The things outlined here are common to both the encoder and the decoder part. -Return values and error handling -================================ - -Most functions in nanopb return *bool*. *True* means success, *false* means failure. - -Because code size is of the essence, nanopb doesn't give any information about the cause of the error. However, there are few possible sources of errors: - -1) Running out of memory. Because everything is allocated from the stack, nanopb can't detect this itself. Encoding or decoding the same type of a message always takes the same amount of stack space. Therefore, if it works once, it works always. -2) Invalid field description. These are usually stored as constants, so if it works under the debugger, it always does. -3) IO errors in your own stream callbacks. Because encoding/decoding stops at the first error, you can overwrite the *state* field in the struct and store your own error code there. -4) Errors in your callback functions. You can use the state field in the callback structure. -5) Exceeding the max_size or bytes_left of a stream. -6) Exceeding the max_size of a string or array field -7) Invalid protocol buffers binary message. It's not like you could recover from it anyway, so a simple failure should be enough. +.. sectnum:: -In my opinion, it is enough that 1) and 2) can be resolved using a debugger. - -However, you may be interested which of the remaining conditions caused the error. For 3) and 4), you can check the state. If you have to detect 5) and 6), you should convert the fields to callback type. Any remaining problem is of type 7). +.. contents:: Streams ======= @@ -33,7 +18,7 @@ There are a few generic rules for callback functions: #) Return false on IO errors. The encoding or decoding process will abort immediately. #) Use state to store your own data, such as a file descriptor. -#) *bytes_written* and *bytes_left* are updated by *pb_write* and *pb_read*. Don't touch them. +#) *bytes_written* and *bytes_left* are updated by pb_write and pb_read. #) Your callback may be used with substreams. In this case *bytes_left*, *bytes_written* and *max_size* have smaller values than the original stream. Don't use these values to calculate pointers. Output streams @@ -51,9 +36,7 @@ Output streams The *callback* for output stream may be NULL, in which case the stream simply counts the number of bytes written. In this case, *max_size* is ignored. -Otherwise, if *bytes_written* + bytes_to_be_written is larger than *max_size*, *pb_write* returns false before doing anything else. If you don't want to limit the size of the stream, pass SIZE_MAX. - -Most commonly you want to initialize *bytes_written* to 0. It doesn't matter to the library, though. +Otherwise, if *bytes_written* + bytes_to_be_written is larger than *max_size*, pb_write returns false before doing anything else. If you don't want to limit the size of the stream, pass SIZE_MAX. **Example 1:** @@ -68,21 +51,22 @@ This is the way to get the size of the message without storing it anywhere:: Writing to stdout:: - bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) + bool callback(pb_ostream_t *stream, const uint8_t *buf, size_t count) { FILE *file = (FILE*) stream->state; return fwrite(buf, 1, count, file) == count; } - pb_ostream_t stdoutstream = {&streamcallback, stdout, SIZE_MAX, 0}; + pb_ostream_t stdoutstream = {&callback, stdout, SIZE_MAX, 0}; Input streams ------------- For input streams, there are a few extra rules: + #) If buf is NULL, read from stream but don't store the data. This is used to skip unknown input. #) You don't need to know the length of the message in advance. After getting EOF error when reading, set bytes_left to 0 and return false. Pb_decode will detect this and if the EOF was in a proper position, it will return true. -:: +Here is the structure:: struct _pb_istream_t { @@ -91,8 +75,109 @@ For input streams, there are a few extra rules: size_t bytes_left; }; -The *callback* must always be a function pointer. +The *callback* must always be a function pointer. *Bytes_left* is an upper limit on the number of bytes that will be read. You can use SIZE_MAX if your callback handles EOF as described above. + +**Example:** + +This function binds an input stream to stdin: + +:: + + bool callback(pb_istream_t *stream, uint8_t *buf, size_t count) + { + FILE *file = (FILE*)stream->state; + bool status; + + if (buf == NULL) + { + while (count-- && fgetc(file) != EOF); + return count == 0; + } + + status = (fread(buf, 1, count, file) == count); + + if (feof(file)) + stream->bytes_left = 0; + + return status; + } + + pb_istream_t stdinstream = {&callback, stdin, SIZE_MAX}; + +Data types +========== + +Most Protocol Buffers datatypes have directly corresponding C datatypes, such as int32 is int32_t, float is float and bool is bool. However, the variable-length datatypes are more complex: + +1) Strings, bytes and repeated fields of any type map to callback functions by default. +2) If there is a special option *(nanopb).max_size* specified in the .proto file, string maps to null-terminated char array and bytes map to a structure containing a char array and a size field. +3) If there is a special option *(nanopb).max_count* specified on a repeated field, it maps to an array of whatever type is being repeated. Another field will be created for the actual number of entries stored. + +=============================================================================== ======================= + field in .proto autogenerated in .h +=============================================================================== ======================= +required string name = 1; pb_callback_t name; +required string name = 1 [(nanopb).max_size = 40]; char name[40]; +repeated string name = 1 [(nanopb).max_size = 40]; pb_callback_t name; +repeated string name = 1 [(nanopb).max_size = 40, (nanopb).max_count = 5]; | size_t name_count; + | char name[5][40]; +required bytes data = 1 [(nanopb).max_size = 40]; | typedef struct { + | size_t size; + | uint8_t bytes[40]; + | } Person_data_t; + | Person_data_t data; +=============================================================================== ======================= + +The maximum lengths are checked in runtime. If string/bytes/array exceeds the allocated length, *pb_decode* will return false. + +For more information about callbacks, see the `Encoding` and `Decoding` sections. + +Field description array +======================= + +For using the *pb_encode* and *pb_decode* functions, you need an array of pb_field_t constants describing the structure you wish to encode. This description is usually autogenerated from .proto file. + +:: + + message PhoneNumber { + required string number = 1 [(nanopb).max_size = 40]; + optional PhoneType type = 2 [default = HOME]; + } + +:: + + const pb_field_t Person_PhoneNumber_fields[3] = { + {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING, + offsetof(Person_PhoneNumber, number), 0, + pb_membersize(Person_PhoneNumber, number), 0, 0}, + + {2, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, + pb_delta(Person_PhoneNumber, type, number), + pb_delta(Person_PhoneNumber, has_type, type), + pb_membersize(Person_PhoneNumber, type), 0, + &Person_PhoneNumber_type_default}, + + PB_LAST_FIELD + }; + +For more information about the format, see the `Generated code` section. + + +Return values and error handling +================================ + +Most functions in nanopb return bool: *true* means success, *false* means failure. If this is enough for you, skip this section. + +For simplicity, nanopb doesn't define it's own error codes. This might be added if there is a compelling need for it. You can however deduce something about the error causes: + +1) Running out of memory. Because everything is allocated from the stack, nanopb can't detect this itself. Encoding or decoding the same type of a message always takes the same amount of stack space. Therefore, if it works once, it works always. +2) Invalid field description. These are usually stored as constants, so if it works under the debugger, it always does. +3) IO errors in your own stream callbacks. Because encoding/decoding stops at the first error, you can overwrite the *state* field in the struct and store your own error code there. +4) Errors that happen in your callback functions. You can use the state field in the callback structure. +5) Exceeding the max_size or bytes_left of a stream. +6) Exceeding the max_size of a string or array field +7) Invalid protocol buffers binary message. It's not like you could recover from it anyway, so a simple failure should be enough. -*Bytes_left* is an upper limit on the number of bytes that will be read. You can use SIZE_MAX if your callback handles EOF as described above. +In my opinion, it is enough that 1. and 2. can be resolved using a debugger. -**Example** \ No newline at end of file +However, you may be interested which of the remaining conditions caused the error. For 3. and 4., you can set and check the state. If you have to detect 5. and 6., you should convert the fields to callback type. Any remaining problem is of type 7. diff --git a/docs/encoding.rst b/docs/encoding.rst index e4e0cd7f..3f673f35 100644 --- a/docs/encoding.rst +++ b/docs/encoding.rst @@ -4,36 +4,15 @@ Nanopb: Encoding messages The basic way to encode messages is to: -1) Write a callback function for whatever stream you want to write the message to. +1) Create an `output stream`_. 2) Fill a structure with your data. -3) Call pb_encode with the stream, a pointer to *const pb_field_t* array and a pointer to your structure. +3) Call *pb_encode* with the stream, a pointer to *const pb_field_t* array and a pointer to your structure. A few extra steps are necessary if you need to know the size of the message beforehand, or if you have dynamically sized fields. -Output streams -============== +.. _`output stream`: concepts.html#output-streams -This is the contents of *pb_ostream_t* structure:: +Function: pb_encode +=================== - typedef struct _pb_ostream_t pb_ostream_t; - struct _pb_ostream_t - { - bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); - void *state; - size_t max_size; - size_t bytes_written; - }; -This, combined with the pb_write function, provides a light-weight abstraction -for whatever destination you want to write data to. - -*callback* should be a pointer to your callback function. These are the rules for it: - -1) Return false on IO errors. This will cause encoding to abort. - * - * 2) You can use state to store your own data (e.g. buffer pointer). - * - * 3) pb_write will update bytes_written after your callback runs. - * - * 4) Substreams will modify max_size and bytes_written. Don't use them to - * calculate any pointers. \ No newline at end of file diff --git a/docs/index.rst b/docs/index.rst index 5a5cc826..0ae6f0fd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -44,6 +44,7 @@ Features and limitations #) Some speed has been sacrificed for code size. For example varint calculations are always done in 64 bits. #) Encoding is focused on writing to streams. For memory buffers only it could be made more efficient. #) The deprecated Protocol Buffers feature called "groups" is not supported. +#) Fields in the generated structs are ordered by the tag number, instead of the natural ordering in .proto file. Getting started =============== diff --git a/docs/lsr.css b/docs/lsr.css index 081bf068..81badb23 100644 --- a/docs/lsr.css +++ b/docs/lsr.css @@ -83,7 +83,8 @@ em { pre { border-left: 3px double #aaa; - padding-left: 10px; + padding: 5px 10px; + background-color: #f6f6f6; } h1.title { @@ -202,14 +203,18 @@ table.docutils td, table.docutils th { padding: 0.25em 0.5em; } +th.field-name { + text-align: right; + width: 15em; +} + table.docutils th { - background-color: #dddddd; + font-family: monospace; + background-color: #f6f6f6; } div.sidebar { - width: 33%; - float: right; - margin: 0em 2em; + margin: 0em 2em 2em 0em; padding: 0em 1em; border-top: 1px solid #aaa; border-left: 1px solid #aaa; diff --git a/docs/reference.rst b/docs/reference.rst new file mode 100644 index 00000000..4c8c874e --- /dev/null +++ b/docs/reference.rst @@ -0,0 +1,385 @@ +===================== +Nanopb: API reference +===================== + +.. contents :: + +pb.h +==== + +pb_type_t +--------- +Defines the encoder/decoder behaviour that should be used for a field. :: + + typedef enum { ... } pb_type_t; + +The low-order byte of the enumeration values defines the function that can be used for encoding and decoding the field data: + +==================== ===== ================================================ +LTYPE identifier Value Storage format +==================== ===== ================================================ +PB_LTYPE_VARINT 0x00 Integer. +PB_LTYPE_SVARINT 0x01 Integer, zigzag encoded. +PB_LTYPE_FIXED 0x02 Integer or floating point. +PB_LTYPE_BYTES 0x03 Structure with *size_t* field and byte array. +PB_LTYPE_STRING 0x04 Null-terminated string. +PB_LTYPE_SUBMESSAGE 0x05 Submessage structure. +==================== ===== ================================================ + +The high-order byte defines whether the field is required, optional, repeated or callback: + +==================== ===== ================================================ +HTYPE identifier Value Field handling +==================== ===== ================================================ +PB_HTYPE_REQUIRED 0x00 Verify that field exists in decoded message. +PB_HTYPE_OPTIONAL 0x10 Use separate *has_* boolean to specify + whether the field is present. +PB_HTYPE_ARRAY 0x20 A repeated field with preallocated array. +PB_HTYPE_CALLBACK 0x30 A field with dynamic storage size, data is + actually a pointer to a structure containing a + callback function. +==================== ===== ================================================ + +pb_field_t +---------- +Describes a single structure field with memory position in relation to others. :: + + typedef struct _pb_field_t pb_field_t; + struct _pb_field_t { + uint8_t tag; + pb_type_t type; + uint8_t data_offset; + int8_t size_offset; + uint8_t data_size; + uint8_t array_size; + const void *ptr; + } pb_packed; + +:tag: Tag number of the field or 0 to terminate a list of fields. +:type: LTYPE and HTYPE of the field. +:data_offset: Offset of field data, relative to the end of the previous field. +:size_offset: Offset of *bool* flag for optional fields or *size_t* count for arrays, relative to field data. +:data_size: Size of a single data entry, in bytes. +:array_size: Maximum number of entries in an array, if it is an array type. +:ptr: Pointer to default value for optional fields, or to submessage description for PB_LTYPE_SUBMESSAGE. + +pb_encode.h +=========== + +pb_ostream_from_buffer +---------------------- +Constructs an output stream for writing into a memory buffer. This is just a helper function, it doesn't do anything you couldn't do yourself in a callback function. It uses an internal callback that stores the pointer in stream *state* field. :: + + pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize); + +:buf: Memory buffer to write into. +:bufsize: Maximum number of bytes to write. +:returns: An output stream. + +After writing, you can check *stream.bytes_written* to find out how much valid data there is in the buffer. + +pb_write +-------- +Writes data to an output stream. Always use this function, instead of trying to call stream callback manually. :: + + bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); + +:stream: Output stream to write to. +:buf: Pointer to buffer with the data to be written. +:count: Number of bytes to write. +:returns: True on success, false if maximum length is exceeded or an IO error happens. + +If an error happens, *bytes_written* is not incremented. Depending on the callback used, calling pb_write again after it has failed once may be dangerous. Nanopb itself never does this, instead it returns the error to user application. The builtin pb_ostream_from_buffer is safe to call again after failed write. + +pb_encode +--------- +Encodes the contents of a structure as a protocol buffers message and writes it to output stream. :: + + bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); + +:stream: Output stream to write to. +:fields: A field description array, usually autogenerated. +:src_struct: Pointer to the data that will be serialized. +:returns: True on success, false on IO error, on detectable errors in field description, or if a field encoder returns false. + +Normally pb_encode simply walks through the fields description array and serializes each field in turn. However, submessages must be serialized twice: first to calculate their size and then to actually write them to output. This causes some constraints for callback fields, which must return the same data on every call. + +pb_encode_varint +---------------- +Encodes an unsigned integer in the varint_ format. :: + + bool pb_encode_varint(pb_ostream_t *stream, uint64_t value); + +:stream: Output stream to write to. 1-10 bytes will be written. +:value: Value to encode. +:returns: True on success, false on IO error. + +.. _varint: http://code.google.com/apis/protocolbuffers/docs/encoding.html#varints + +pb_encode_tag +------------- +Starts a field in the Protocol Buffers binary format: encodes the field number and the wire type of the data. :: + + bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number); + +:stream: Output stream to write to. 1-5 bytes will be written. +:wiretype: PB_WT_VARINT, PB_WT_64BIT, PB_WT_STRING or PB_WT_32BIT +:field_number: Identifier for the field, defined in the .proto file. +:returns: True on success, false on IO error. + +pb_encode_tag_for_field +----------------------- +Same as `pb_encode_tag`_, except takes the parameters from a *pb_field_t* structure. :: + + bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field); + +:stream: Output stream to write to. 1-5 bytes will be written. +:field: Field description structure. Usually autogenerated. +:returns: True on success, false on IO error or unknown field type. + +This function only considers the LTYPE of the field. You can use it from your field callbacks, because the source generator writes correct LTYPE also for callback type fields. + +pb_encode_string +---------------- +Writes the length of a string as varint and then contents of the string. Used for writing fields with wire type PB_WT_STRING. :: + + bool pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size); + +:stream: Output stream to write to. +:buffer: Pointer to string data. +:size: Number of bytes in the string. +:returns: True on success, false on IO error. + +.. sidebar:: Field encoders + + The functions with names beginning with *pb_enc_* are called field encoders. Each PB_LTYPE has an own field encoder, which handles translating from C data into Protocol Buffers data. + + By using the *data_size* in the field description and by taking advantage of C casting rules, it has been possible to combine many data types to a single LTYPE. For example, *int32*, *uint32*, *int64*, *uint64*, *bool* and *enum* are all handled by *pb_enc_varint*. + + Each field encoder only encodes the contents of the field. The tag must be encoded separately with `pb_encode_tag_for_field`_. + + You can use the field encoders from your callbacks. + +pb_enc_varint +------------- +Field encoder for PB_LTYPE_VARINT. Takes the first *field->data_size* bytes from src, casts them as *uint64_t* and calls `pb_encode_varint`_. :: + + bool pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src); + +:stream: Output stream to write to. +:field: Field description structure. Only *data_size* matters. +:src: Pointer to start of the field data. +:returns: True on success, false on IO error. + +pb_enc_svarint +-------------- +Field encoder for PB_LTYPE_SVARINT. Similar to `pb_enc_varint`_, except first zig-zag encodes the value for more efficient negative number encoding. :: + + bool pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); + +(parameters are the same as for `pb_enc_varint`_) + +The number is considered negative if the high-order bit of the value is set. On big endian computers, it is the highest bit of *\*src*. On little endian computers, it is the highest bit of *\*(src + field->data_size - 1)*. + +pb_enc_fixed +------------ +Field encoder for PB_LTYPE_FIXED. Writes the data in little endian order. On big endian computers, reverses the order of bytes. :: + + bool pb_enc_fixed(pb_ostream_t *stream, const pb_field_t *field, const void *src); + +(parameters are the same as for `pb_enc_varint`_) + +The same function is used for both integers, floats and doubles. This break encoding of double values on architectures where they are mixed endian (primarily some arm processors with hardware FPU). + +pb_enc_bytes +------------ +Field encoder for PB_LTYPE_BYTES. Just calls `pb_encode_string`_. :: + + bool pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src); + +:stream: Output stream to write to. +:field: Not used. +:src: Pointer to a structure similar to pb_bytes_array_t. +:returns: True on success, false on IO error. + +This function expects a pointer to a structure with a *size_t* field at start, and a variable sized byte array after it. The platform-specific field offset is inferred from *pb_bytes_array_t*, which has a byte array of size 1. + +pb_enc_string +------------- +Field encoder for PB_LTYPE_STRING. Determines size of string with strlen() and then calls `pb_encode_string`_. :: + + bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src); + +:stream: Output stream to write to. +:field: Not used. +:src: Pointer to a null-terminated string. +:returns: True on success, false on IO error. + +pb_enc_submessage +----------------- +Field encoder for PB_LTYPE_SUBMESSAGE. Calls `pb_encode`_ to perform the actual encoding. :: + + bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src); + +:stream: Output stream to write to. +:field: Field description structure. The *ptr* field must be a pointer to a field description array for the submessage. +:src: Pointer to the structure where submessage data is. +:returns: True on success, false on IO errors, pb_encode errors or if submessage size changes between calls. + +In Protocol Buffers format, the submessage size must be written before the submessage contents. Therefore, this function has to encode the submessage twice in order to know the size beforehand. + +If the submessage contains callback fields, the callback function might misbehave and write out a different amount of data on the second call. This situation is recognized and *false* is returned, but it is up to the caller to ensure that the receiver of the message does not interpret it as valid data. + +pb_decode.h +=========== + +pb_istream_from_buffer +---------------------- +Helper function for creating an input stream that reads data from a memory buffer. :: + + pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); + +:buf: Pointer to byte array to read from. +:bufsize: Size of the byte array. +:returns: An input stream ready to use. + +pb_read +------- +Read data from input stream. Always use this function, don't try to call the stream callback directly. :: + + bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); + +:stream: Input stream to read from. +:buf: Buffer to store the data to, or NULL to just read data without storing it anywhere. +:count: Number of bytes to read. +:returns: True on success, false if *stream->bytes_left* is less than *count* or if an IO error occurs. + +End of file is signalled by *stream->bytes_left* being zero after pb_read returns false. + +pb_decode_varint +---------------- +Read and decode a varint_ encoded integer. :: + + bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest); + +:stream: Input stream to read from. 1-10 bytes will be read. +:dest: Storage for the decoded integer. Value is undefined on error. +:returns: True on success, false if value exceeds uint64_t range or an IO error happens. + +pb_skip_varint +-------------- +Skip a varint_ encoded integer without decoding it. :: + + bool pb_skip_varint(pb_istream_t *stream); + +:stream: Input stream to read from. Will read 1 byte at a time until the MSB is clear. +:returns: True on success, false on IO error. + +pb_skip_string +-------------- +Skip a varint-length-prefixed string. This means skipping a value with wire type PB_WT_STRING. :: + + bool pb_skip_string(pb_istream_t *stream); + +:stream: Input stream to read from. +:returns: True on success, false on IO error or length exceeding uint32_t. + +pb_decode +--------- +Read and decode all fields of a structure. Reads until EOF on input stream. :: + + bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +:stream: Input stream to read from. +:fields: A field description array. Usually autogenerated. +:dest_struct: Pointer to structure where data will be stored. +:returns: True on success, false on IO error, on detectable errors in field description, if a field encoder returns false or if a required field is missing. + +In Protocol Buffers binary format, EOF is only allowed between fields. If it happens anywhere else, pb_decode will return *false*. + +In addition to EOF, the pb_decode implementation supports terminating a message with a 0 byte. This is compatible with the official Protocol Buffers because 0 is never a valid field tag. + +For optional fields, this function applies the default value and sets *has_* to false if the field is not present. + +Because of memory concerns, the detection of missing required fields is not perfect if the structure contains more than 32 fields. + +.. sidebar:: Field decoders + + The functions with names beginning with *pb_dec_* are called field decoders. Each PB_LTYPE has an own field decoder, which handles translating from Protocol Buffers data to C data. + + Each field decoder reads and decodes a single value. For arrays, the decoder is called repeatedly. + + You can use the decoders from your callbacks. + +pb_dec_varint +------------- +Field decoder for PB_LTYPE_VARINT. :: + + bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) + +:stream: Input stream to read from. 1-10 bytes will be read. +:field: Field description structure. Only *field->data_size* matters. +:dest: Pointer to destination integer. Must have size of *field->data_size* bytes. +:returns: True on success, false on IO errors or if `pb_decode_varint`_ fails. + +This function first calls `pb_decode_varint`_. It then copies the first bytes of the 64-bit result value to *dest*, or on big endian architectures, the last bytes. + +pb_dec_svarint +-------------- +Field decoder for PB_LTYPE_SVARINT. Similar to `pb_dec_varint`_, except that it performs zigzag-decoding on the value. :: + + bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest); + +(parameters are the same as `pb_dec_varint`_) + +pb_dec_fixed +------------ +Field decoder for PB_LTYPE_FIXED. :: + + bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest); + +(parameters are the same as `pb_dec_varint`_) + +This function reads *field->data_size* bytes from the input stream. +On big endian architectures, it then reverses the order of the bytes. +Finally, it writes the bytes to *dest*. + +pb_dec_bytes +------------ +Field decoder for PB_LTYPE_BYTES. Reads a length-prefixed block of bytes. :: + + bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); + +:stream: Input stream to read from. +:field: Field description structure. Only *field->data_size* matters. +:dest: Pointer to a structure similar to pb_bytes_array_t. +:returns: True on success, false on IO error or if length exceeds the array size. + +This function expects a pointer to a structure with a *size_t* field at start, and a variable sized byte array after it. It will deduce the maximum size of the array from *field->data_size*. + +pb_dec_string +------------- +Field decoder for PB_LTYPE_STRING. Reads a length-prefixed string. :: + + bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); + +:stream: Input stream to read from. +:field: Field description structure. Only *field->data_size* matters. +:dest: Pointer to a character array of size *field->data_size*. +:returns: True on success, false on IO error or if length exceeds the array size. + +This function null-terminates the string when successful. On error, the contents of the destination array is undefined. + +pb_dec_submessage +----------------- +Field decoder for PB_LTYPE_SUBMESSAGE. Calls `pb_decode`_ to perform the actual decoding. :: + + bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) + +:stream: Input stream to read from. +:field: Field description structure. Only *field->ptr* matters. +:dest: Pointer to the destination structure. +:returns: True on success, false on IO error or if `pb_decode`_ fails. + +The *field->ptr* should be a pointer to *pb_field_t* array describing the submessage. + diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 6aff3155..e62d04f1 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -204,7 +204,7 @@ class Field: if prev_field_name is None: result += ' offsetof(%s, %s),' % (self.struct_name, self.name) else: - result += ' pb_delta(%s, %s, %s),' % (self.struct_name, self.name, prev_field_name) + result += ' pb_delta_end(%s, %s, %s),' % (self.struct_name, self.name, prev_field_name) if self.htype == 'PB_HTYPE_OPTIONAL': result += '\n pb_delta(%s, has_%s, %s),' % (self.struct_name, self.name, self.name) @@ -279,11 +279,11 @@ class Message: return result def fields_declaration(self): - result = 'extern const pb_field_t %s_fields[%d];' % (self.name, len(self.fields)) + result = 'extern const pb_field_t %s_fields[%d];' % (self.name, len(self.fields) + 1) return result def fields_definition(self): - result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, len(self.fields)) + result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, len(self.fields) + 1) prev = None for field in self.ordered_fields: @@ -291,7 +291,7 @@ class Message: result += ',\n\n' prev = field.name - result = result[:-3] + '\n};' + result += ' PB_LAST_FIELD\n};' return result def iterate_messages(desc, names = Names()): diff --git a/pb.h b/pb.h index 1d32ee9b..e6037b81 100644 --- a/pb.h +++ b/pb.h @@ -1,6 +1,11 @@ #ifndef _PB_H_ #define _PB_H_ +/* pb.h: Common parts for nanopb library. + * Most of these are quite low-level stuff. For the high-level interface, + * see pb_encode.h or pb_decode.h + */ + #include #include #include @@ -12,15 +17,7 @@ #define pb_packed #endif -/* Wire types. Library user needs these only in encoder callbacks. */ -typedef enum { - PB_WT_VARINT = 0, - PB_WT_64BIT = 1, - PB_WT_STRING = 2, - PB_WT_32BIT = 5 -} pb_wire_type_t; - -/* List of possible field types +/* List of possible field types. These are used in the autogenerated code. * Least-significant 4 bits tell the scalar type * Most-significant 4 bits specify repeated/required/packed etc. * @@ -143,10 +140,19 @@ struct _pb_callback_t { void *arg; }; +/* Wire types. Library user needs these only in encoder callbacks. */ +typedef enum { + PB_WT_VARINT = 0, + PB_WT_64BIT = 1, + PB_WT_STRING = 2, + PB_WT_32BIT = 5 +} pb_wire_type_t; + /* These macros are used to declare pb_field_t's in the constant array. */ #define pb_membersize(st, m) (sizeof ((st*)0)->m) #define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0])) #define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2)) +#define pb_delta_end(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) #define PB_LAST_FIELD {0,0,0,0} diff --git a/pb_decode.c b/pb_decode.c index e2888f22..379d134c 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -62,15 +62,15 @@ pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize) * Helper functions * ********************/ -bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) +static bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) { uint64_t temp; - bool status = pb_decode_varint64(stream, &temp); + bool status = pb_decode_varint(stream, &temp); *dest = temp; return status; } -bool pb_decode_varint64(pb_istream_t *stream, uint64_t *dest) +bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest) { uint8_t byte; int bitpos = 0; @@ -108,7 +108,7 @@ bool pb_skip_string(pb_istream_t *stream) return pb_read(stream, NULL, length); } -/* Currently all wire type related stuff is kept hidden from +/* Currently the wire type related stuff is kept hidden from * callbacks. They shouldn't need it. It's better for performance * to just assume the correct type and fail safely on corrupt message. */ @@ -192,6 +192,7 @@ static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, v static bool pb_field_next(pb_field_iterator_t *iter) { bool notwrapped = true; + size_t prev_size = iter->current->data_size * iter->current->array_size; iter->current++; iter->field_index++; if (iter->current->tag == 0) @@ -199,10 +200,11 @@ static bool pb_field_next(pb_field_iterator_t *iter) iter->current = iter->start; iter->field_index = 0; iter->pData = iter->dest_struct; + prev_size = 0; notwrapped = false; } - iter->pData = (char*)iter->pData + iter->current->data_offset; + iter->pData = (char*)iter->pData + prev_size + iter->current->data_offset; iter->pSize = (char*)iter->pData + iter->current->size_offset; return notwrapped; } @@ -224,7 +226,7 @@ static bool pb_field_find(pb_field_iterator_t *iter, int tag) * Decode a single field * *************************/ -bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_t *iter) +static bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_t *iter) { pb_decoder_t func = PB_DECODERS[PB_LTYPE(iter->current->type)]; @@ -351,6 +353,9 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc if (!pb_decode_varint32(stream, &temp)) return stream->bytes_left == 0; /* Was it EOF? */ + if (temp == 0) + return true; /* Special feature: allow 0-terminated messages. */ + tag = temp >> 3; wire_type = temp & 7; @@ -399,7 +404,7 @@ static void endian_copy(void *dest, void *src, size_t destsize, size_t srcsize) bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint64_t temp; - bool status = pb_decode_varint64(stream, &temp); + bool status = pb_decode_varint(stream, &temp); endian_copy(dest, &temp, field->data_size, sizeof(temp)); return status; } @@ -407,7 +412,7 @@ bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint64_t temp; - bool status = pb_decode_varint64(stream, &temp); + bool status = pb_decode_varint(stream, &temp); temp = (temp >> 1) ^ -(int64_t)(temp & 1); endian_copy(dest, &temp, field->data_size, sizeof(temp)); return status; diff --git a/pb_decode.h b/pb_decode.h index 011efdd9..2d4e5863 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -48,8 +48,7 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc * You may want to use these from your caller or callbacks. */ -bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest); -bool pb_decode_varint64(pb_istream_t *stream, uint64_t *dest); +bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest); bool pb_skip_varint(pb_istream_t *stream); bool pb_skip_string(pb_istream_t *stream); diff --git a/pb_encode.c b/pb_encode.c index 19a531cf..188d768d 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -124,11 +124,13 @@ bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_ const pb_field_t *field = fields; const void *pData = src_struct; const void *pSize; + size_t prev_size = 0; while (field->tag != 0) { - pData = (const char*)pData + field->data_offset; + pData = (const char*)pData + prev_size + field->data_offset; pSize = (const char*)pData + field->size_offset; + prev_size = field->data_size * field->array_size; pb_encoder_t func = PB_ENCODERS[PB_LTYPE(field->type)]; diff --git a/tests/Makefile b/tests/Makefile index 84c035e5..caed5bad 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -10,6 +10,10 @@ clean: %: %.c $(DEPS) $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c +person.h: person.proto + protoc -I. -I../generator -I/usr/include -operson.pb $< + python ../generator/nanopb_generator.py person.pb + run_unittests: decode_unittests encode_unittests ./decode_unittests ./encode_unittests diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c index ff4c6b46..ac51d491 100644 --- a/tests/decode_unittests.c +++ b/tests/decode_unittests.c @@ -48,34 +48,21 @@ int main() TEST(pb_read(&stream, buffer, 15)) } - { - pb_istream_t s; - uint32_t u; - int32_t i; - - COMMENT("Test pb_decode_varint32"); - TEST((s = S("\x00"), pb_decode_varint32(&s, &u) && u == 0)); - TEST((s = S("\x01"), pb_decode_varint32(&s, &u) && u == 1)); - TEST((s = S("\xAC\x02"), pb_decode_varint32(&s, &u) && u == 300)); - TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint32(&s, &u) && u == UINT32_MAX)); - TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint32(&s, (uint32_t*)&i) && i == -1)); - } - { pb_istream_t s; uint64_t u; int64_t i; - COMMENT("Test pb_decode_varint64"); - TEST((s = S("\x00"), pb_decode_varint64(&s, &u) && u == 0)); - TEST((s = S("\x01"), pb_decode_varint64(&s, &u) && u == 1)); - TEST((s = S("\xAC\x02"), pb_decode_varint64(&s, &u) && u == 300)); - TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint64(&s, &u) && u == UINT32_MAX)); - TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint64(&s, (uint64_t*)&i) && i == UINT32_MAX)); + COMMENT("Test pb_decode_varint"); + TEST((s = S("\x00"), pb_decode_varint(&s, &u) && u == 0)); + TEST((s = S("\x01"), pb_decode_varint(&s, &u) && u == 1)); + TEST((s = S("\xAC\x02"), pb_decode_varint(&s, &u) && u == 300)); + TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint(&s, &u) && u == UINT32_MAX)); + TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint(&s, (uint64_t*)&i) && i == UINT32_MAX)); TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), - pb_decode_varint64(&s, (uint64_t*)&i) && i == -1)); + pb_decode_varint(&s, (uint64_t*)&i) && i == -1)); TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), - pb_decode_varint64(&s, &u) && u == UINT64_MAX)); + pb_decode_varint(&s, &u) && u == UINT64_MAX)); } { -- cgit 1.2.3-korg From 2cefaeaf61f38f1566293e53b4708e9ceff2d945 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 16 Aug 2011 17:28:59 +0000 Subject: Docs git-svn-id: https://svn.kapsi.fi/jpa/nanopb@956 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/Makefile | 2 +- docs/concepts.rst | 105 ++++++++++++++++++++++++++++++++++++++++++++++++----- docs/encoding.rst | 18 --------- docs/index.rst | 23 +++++------- docs/lsr.css | 2 +- docs/menu.rst | 10 +++++ docs/reference.rst | 46 ++++++++++++++++++++++- tests/Makefile | 2 +- 8 files changed, 162 insertions(+), 46 deletions(-) delete mode 100644 docs/encoding.rst create mode 100644 docs/menu.rst diff --git a/docs/Makefile b/docs/Makefile index e0ddc37d..d1d9587d 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,4 +1,4 @@ -all: index.html encoding.html concepts.html reference.html +all: index.html concepts.html reference.html %.html: %.rst rst2html --stylesheet=lsr.css --link-stylesheet $< $@ \ No newline at end of file diff --git a/docs/concepts.rst b/docs/concepts.rst index f2475aec..58620a85 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -2,12 +2,42 @@ Nanopb: Basic concepts ====================== -The things outlined here are common to both the encoder and the decoder part. +.. include :: menu.rst -.. sectnum:: +The things outlined here are common to both the encoder and the decoder part. .. contents:: +Proto files +=========== +All Protocol Buffers implementations use .proto files to describe the message format. +The point of these files is to be a portable interface description language. + +Compiling .proto files for nanopb +--------------------------------- +Nanopb uses the Google's protoc compiler to parse the .proto file, and then a python script to generate the C header and source code from it:: + + user@host:~$ protoc -omessage.pb message.proto + user@host:~$ python ../generator/nanopb_generator.py message.pb + Writing to message.h and message.c + user@host:~$ + +Compiling .proto files with nanopb options +------------------------------------------ +Nanopb defines two extensions for message fields described in .proto files: *max_size* and *max_count*. +These are the maximum size of a string and maximum count of items in an array:: + + required string name = 1 [(nanopb).max_size = 40]; + repeated PhoneNumber phone = 4 [(nanopb).max_count = 5]; + +To use these extensions, you need to place an import statement in the beginning of the file:: + + import "nanopb.proto"; + +This file, in turn, requires the file *google/protobuf/descriptor.proto*. This is usually installed under */usr/include*. Therefore, to compile a .proto file which uses options, use a protoc command similar to:: + + protoc -I/usr/include -Inanopb/generator -I. -omessage.pb message.proto + Streams ======= @@ -130,21 +160,78 @@ required bytes data = 1 [(nanopb).max_size = 40]; The maximum lengths are checked in runtime. If string/bytes/array exceeds the allocated length, *pb_decode* will return false. -For more information about callbacks, see the `Encoding` and `Decoding` sections. +Field callbacks +=============== +When a field has dynamic length, nanopb cannot statically allocate storage for it. Instead, it allows you to handle the field in whatever way you want, using a callback function. + +The `pb_callback_t`_ structure contains a function pointer and a *void* pointer you can use for passing data to the callback. The actual behavior of the callback function is different in encoding and decoding modes. + +.. _`pb_callback_t`: reference.html#pb-callback-t + +Encoding callbacks +------------------ +:: + + bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, const void *arg); + +When encoding, the callback should write out complete fields, including the wire type and field number tag. It can write as many or as few fields as it likes. For example, if you want to write out an array as *repeated* field, you should do it all in a single call. + +The callback may be called multiple times during a single call to `pb_encode`_. It must produce the same amount of data every time. + +.. _`pb_encode`: reference.html#pb-encode + +This callback writes out a dynamically sized string:: + + bool write_string(pb_ostream_t *stream, const pb_field_t *field, const void *arg) + { + char *str = get_string_from_somewhere(); + if (!pb_encode_tag_for_field(stream, field)) + return false; + + return pb_encode_string(stream, (uint8_t*)str, strlen(str)); + } + +Decoding callbacks +------------------ +:: + + bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void *arg); + +When decoding, the callback receives a length-limited substring that reads the contents of a single field. The field tag has already been read. + +The callback will be called multiple times for repeated fields. For packed fields, you can either read multiple values until the stream ends, or leave it to `pb_decode`_ to call your function over and over until all values have been read. + +.. _`pb_decode`: reference.html#pb-decode + +This callback reads multiple integers and prints them:: + + bool read_ints(pb_istream_t *stream, const pb_field_t *field, void *arg) + { + while (stream.bytes_left) + { + uint64_t value; + if (!pb_decode_varint(stream, &value)) + return false; + printf("%lld\n", value); + } + return true; + } Field description array ======================= For using the *pb_encode* and *pb_decode* functions, you need an array of pb_field_t constants describing the structure you wish to encode. This description is usually autogenerated from .proto file. -:: +For example this submessage in the Person.proto file:: - message PhoneNumber { - required string number = 1 [(nanopb).max_size = 40]; - optional PhoneType type = 2 [default = HOME]; + message Person { + message PhoneNumber { + required string number = 1 [(nanopb).max_size = 40]; + optional PhoneType type = 2 [default = HOME]; + } } -:: +generates this field description array for the structure *Person_PhoneNumber*:: const pb_field_t Person_PhoneNumber_fields[3] = { {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING, @@ -160,8 +247,6 @@ For using the *pb_encode* and *pb_decode* functions, you need an array of pb_fie PB_LAST_FIELD }; -For more information about the format, see the `Generated code` section. - Return values and error handling ================================ diff --git a/docs/encoding.rst b/docs/encoding.rst deleted file mode 100644 index 3f673f35..00000000 --- a/docs/encoding.rst +++ /dev/null @@ -1,18 +0,0 @@ -========================= -Nanopb: Encoding messages -========================= - -The basic way to encode messages is to: - -1) Create an `output stream`_. -2) Fill a structure with your data. -3) Call *pb_encode* with the stream, a pointer to *const pb_field_t* array and a pointer to your structure. - -A few extra steps are necessary if you need to know the size of the message beforehand, or if you have dynamically sized fields. - -.. _`output stream`: concepts.html#output-streams - -Function: pb_encode -=================== - - diff --git a/docs/index.rst b/docs/index.rst index 0ae6f0fd..93f04c6c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -2,6 +2,8 @@ Nanopb: Protocol Buffers with small code size ============================================= +.. include :: menu.rst + Nanopb is an ANSI-C library for encoding and decoding messages in Google's `Protocol Buffers`__ format with minimal requirements for RAM and code space. It is primarily suitable for 32-bit microcontrollers. @@ -55,8 +57,12 @@ For starters, consider this simple message:: required int32 value = 1; } -Save this in *example.proto* and run it through *nanopb_generate.py*. You -should now have in *example.h*:: +Save this in *example.proto* and compile it:: + + user@host:~$ protoc -omessage.pb message.proto + user@host:~$ python ../generator/nanopb_generator.py message.pb + +You should now have in *example.h*:: typedef struct { int32_t value; @@ -75,19 +81,8 @@ After that, buffer will contain the encoded message. The number of bytes in the message is stored in *stream.bytes_written*. You can feed the message to *protoc --decode=Example example.proto* to verify its validity. -Library reference -================= - -**Encoding** - -**Decoding** - -**Specifying field options** - -**Generated code** - Wishlist ======== #) A specialized encoder for encoding to a memory buffer. Should serialize in reverse order to avoid having to determine submessage size beforehand. -#) A cleaner rewrite of the source generator. +#) A cleaner rewrite of the Python-based source generator. #) Better performance for 16- and 8-bit platforms: use smaller datatypes where possible. diff --git a/docs/lsr.css b/docs/lsr.css index 81badb23..86053252 100644 --- a/docs/lsr.css +++ b/docs/lsr.css @@ -214,7 +214,7 @@ table.docutils th { } div.sidebar { - margin: 0em 2em 2em 0em; + margin: 2em 2em 2em 0em; padding: 0em 1em; border-top: 1px solid #aaa; border-left: 1px solid #aaa; diff --git a/docs/menu.rst b/docs/menu.rst new file mode 100644 index 00000000..4c643bc8 --- /dev/null +++ b/docs/menu.rst @@ -0,0 +1,10 @@ +.. sidebar :: Documentation index + + 1) `Overview`_ + 2) `Concepts`_ + 3) `API reference`_ + +.. _`Overview`: index.html +.. _`Concepts`: concepts.html +.. _`API reference`: reference.html + diff --git a/docs/reference.rst b/docs/reference.rst index 4c8c874e..b8f9454e 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -2,6 +2,8 @@ Nanopb: API reference ===================== +.. include :: menu.rst + .. contents :: pb.h @@ -42,7 +44,7 @@ PB_HTYPE_CALLBACK 0x30 A field with dynamic storage size, data is pb_field_t ---------- -Describes a single structure field with memory position in relation to others. :: +Describes a single structure field with memory position in relation to others. The descriptions are usually autogenerated. :: typedef struct _pb_field_t pb_field_t; struct _pb_field_t { @@ -63,6 +65,48 @@ Describes a single structure field with memory position in relation to others. : :array_size: Maximum number of entries in an array, if it is an array type. :ptr: Pointer to default value for optional fields, or to submessage description for PB_LTYPE_SUBMESSAGE. +The *uint8_t* datatypes limit the maximum size of a single item to 255 bytes and arrays to 255 items. Compiler will warn "Initializer too large for type" if the limits are exceeded. The types can be changed to larger ones if necessary. + +pb_bytes_array_t +---------------- +An byte array with a field for storing the length:: + + typedef struct { + size_t size; + uint8_t bytes[1]; + } pb_bytes_array_t; + +In an actual array, the length of *bytes* may be different. + +pb_callback_t +------------- +Part of a message structure, for fields with type PB_HTYPE_CALLBACK:: + + typedef struct _pb_callback_t pb_callback_t; + struct _pb_callback_t { + union { + bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void *arg); + bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, const void *arg); + } funcs; + + void *arg; + }; + +The *arg* is passed to the callback when calling. It can be used to store any information that the callback might need. + +When calling `pb_encode`_, *funcs.encode* must be set, and similarly when calling `pb_decode`_, *funcs.decode* must be set. The function pointers are stored in the same memory location but are of incompatible types. + +pb_wire_type_t +-------------- +Protocol Buffers wire types. These are used with `pb_encode_tag`_. :: + + typedef enum { + PB_WT_VARINT = 0, + PB_WT_64BIT = 1, + PB_WT_STRING = 2, + PB_WT_32BIT = 5 + } pb_wire_type_t; + pb_encode.h =========== diff --git a/tests/Makefile b/tests/Makefile index caed5bad..4993297e 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -10,7 +10,7 @@ clean: %: %.c $(DEPS) $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c -person.h: person.proto +%.h: %.proto protoc -I. -I../generator -I/usr/include -operson.pb $< python ../generator/nanopb_generator.py person.pb -- cgit 1.2.3-korg From 7f53c3f7484679c1e38607f59542d43be1387650 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 17 Aug 2011 19:03:06 +0000 Subject: Example git-svn-id: https://svn.kapsi.fi/jpa/nanopb@957 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/concepts.rst | 4 +- docs/index.rst | 2 +- docs/reference.rst | 2 +- example/Makefile | 11 ++++ example/fileproto.proto | 26 ++++++++ example/server.c | 149 ++++++++++++++++++++++++++++++++++++++++++ generator/nanopb_generator.py | 8 +-- pb.h | 2 + pb_decode.c | 33 ++++++++-- pb_encode.c | 7 +- tests/Makefile | 18 +++-- tests/person.proto | 3 +- tests/test_decode1.c | 28 ++++++-- tests/test_encode1.c | 3 - 14 files changed, 263 insertions(+), 33 deletions(-) create mode 100644 example/Makefile create mode 100644 example/fileproto.proto create mode 100644 example/server.c diff --git a/docs/concepts.rst b/docs/concepts.rst index 58620a85..fac9061f 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -164,7 +164,7 @@ Field callbacks =============== When a field has dynamic length, nanopb cannot statically allocate storage for it. Instead, it allows you to handle the field in whatever way you want, using a callback function. -The `pb_callback_t`_ structure contains a function pointer and a *void* pointer you can use for passing data to the callback. The actual behavior of the callback function is different in encoding and decoding modes. +The `pb_callback_t`_ structure contains a function pointer and a *void* pointer you can use for passing data to the callback. If the function pointer is NULL, the field will be skipped. The actual behavior of the callback function is different in encoding and decoding modes. .. _`pb_callback_t`: reference.html#pb-callback-t @@ -176,7 +176,7 @@ Encoding callbacks When encoding, the callback should write out complete fields, including the wire type and field number tag. It can write as many or as few fields as it likes. For example, if you want to write out an array as *repeated* field, you should do it all in a single call. -The callback may be called multiple times during a single call to `pb_encode`_. It must produce the same amount of data every time. +If the callback is used in a submessage, it will be called multiple times during a single call to `pb_encode`_. It must produce the same amount of data every time. If the callback is directly in the main message, it is called only once. .. _`pb_encode`: reference.html#pb-encode diff --git a/docs/index.rst b/docs/index.rst index 93f04c6c..ea891237 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -42,7 +42,7 @@ Features and limitations **Limitations** -#) User must provide callbacks when decoding arrays or strings without maximum size. +#) User must provide callbacks when decoding arrays or strings without maximum size. Malloc support could be added as a separate module. #) Some speed has been sacrificed for code size. For example varint calculations are always done in 64 bits. #) Encoding is focused on writing to streams. For memory buffers only it could be made more efficient. #) The deprecated Protocol Buffers feature called "groups" is not supported. diff --git a/docs/reference.rst b/docs/reference.rst index b8f9454e..2812958f 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -94,7 +94,7 @@ Part of a message structure, for fields with type PB_HTYPE_CALLBACK:: The *arg* is passed to the callback when calling. It can be used to store any information that the callback might need. -When calling `pb_encode`_, *funcs.encode* must be set, and similarly when calling `pb_decode`_, *funcs.decode* must be set. The function pointers are stored in the same memory location but are of incompatible types. +When calling `pb_encode`_, *funcs.encode* is used, and similarly when calling `pb_decode`_, *funcs.decode* is used. The function pointers are stored in the same memory location but are of incompatible types. You can set the function pointer to NULL to skip the field. pb_wire_type_t -------------- diff --git a/example/Makefile b/example/Makefile new file mode 100644 index 00000000..0f08f2d7 --- /dev/null +++ b/example/Makefile @@ -0,0 +1,11 @@ +CFLAGS=-ansi -Wall -Werror -I .. -g -O0 +DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h + +all: server + +%: %.c $(DEPS) fileproto.h + $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c fileproto.c + +fileproto.h: fileproto.proto ../generator/nanopb_generator.py + protoc -I. -I../generator -I/usr/include -ofileproto.pb $< + python ../generator/nanopb_generator.py fileproto.pb diff --git a/example/fileproto.proto b/example/fileproto.proto new file mode 100644 index 00000000..e2786b15 --- /dev/null +++ b/example/fileproto.proto @@ -0,0 +1,26 @@ +import "nanopb.proto"; + +// This defines protocol for a simple server that lists files. +// +// If you come from high-level programming background, the hardcoded +// maximum lengths may disgust you. However, if your microcontroller only +// has a few kB of ram to begin with, setting reasonable limits for +// filenames is ok. +// +// On the other hand, using the callback interface, it is not necessary +// to set a limit on the number of files in the response. + +message ListFilesRequest { + optional string path = 1 [default = "/", (nanopb).max_size = 128]; +} + +message FileInfo { + required uint64 inode = 1; + required string name = 2 [(nanopb).max_size = 128]; +} + +message ListFilesResponse { + optional bool path_error = 1 [default = false]; + repeated FileInfo file = 2; +} + diff --git a/example/server.c b/example/server.c new file mode 100644 index 00000000..a671f4cb --- /dev/null +++ b/example/server.c @@ -0,0 +1,149 @@ +/* This is a simple TCP server that listens on port 1234 and provides lists + * of files to clients, using a protocol defined in file_server.proto. + * + * It directly deserializes and serializes messages from network, minimizing + * memory use. + * + * For flexibility, this example is implemented using posix api. + * In a real embedded system you would typically use some other kind of + * a communication and filesystem layer. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "fileproto.h" + +bool write_callback(pb_ostream_t *stream, const uint8_t *buf, size_t count) +{ + int fd = *(int*)stream->state; + return send(fd, buf, count, 0) == count; +} + +bool read_callback(pb_istream_t *stream, uint8_t *buf, size_t count) +{ + int fd = *(int*)stream->state; + + if (buf == NULL) + { + /* Well, this is a really inefficient way to skip input. */ + /* It is only used when there are unknown fields. */ + char dummy; + while (count-- && recv(fd, &dummy, 1, 0) == 1); + return count == 0; + } + + return recv(fd, buf, count, MSG_WAITALL) == count; +} + +bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +{ + DIR *dir = (DIR*) arg; + struct dirent *file; + FileInfo fileinfo; + + while ((file = readdir(dir)) != NULL) + { + fileinfo.inode = file->d_ino; + strncpy(fileinfo.name, file->d_name, sizeof(fileinfo.name)); + fileinfo.name[sizeof(fileinfo.name) - 1] = '\0'; + + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!pb_enc_submessage(stream, field, &fileinfo)) + return false; + } + + return true; +} + +void handle_connection(int connfd) +{ + ListFilesRequest request; + ListFilesResponse response; + pb_istream_t input = {&read_callback, &connfd, SIZE_MAX}; + pb_ostream_t output = {&write_callback, &connfd, SIZE_MAX, 0}; + DIR *directory; + + if (!pb_decode(&input, ListFilesRequest_fields, &request)) + { + printf("Decoding failed.\n"); + return; + } + + directory = opendir(request.path); + + printf("Listing directory: %s\n", request.path); + + if (directory == NULL) + { + perror("opendir"); + + response.has_path_error = true; + response.path_error = true; + response.file.funcs.encode = NULL; + } + else + { + response.has_path_error = false; + response.file.funcs.encode = &listdir_callback; + response.file.arg = directory; + } + + if (!pb_encode(&output, ListFilesResponse_fields, &response)) + { + printf("Encoding failed.\n"); + } +} + +int main(int argc, char **argv) +{ + int listenfd, connfd; + struct sockaddr_in servaddr; + + listenfd = socket(AF_INET, SOCK_STREAM, 0); + + memset(&servaddr, 0, sizeof(servaddr)); + servaddr.sin_family = AF_INET; + servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + servaddr.sin_port = htons(1234); + if (bind(listenfd, (struct sockaddr*)&servaddr, sizeof(servaddr)) != 0) + { + perror("bind"); + return 1; + } + + if (listen(listenfd, 5) != 0) + { + perror("listen"); + return 1; + } + + for(;;) + { + connfd = accept(listenfd, NULL, NULL); + + if (connfd < 0) + { + perror("accept"); + return 1; + } + + printf("Got connection.\n"); + + handle_connection(connfd); + + printf("Closing connection.\n"); + + close(connfd); + } +} diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index e62d04f1..a6d38c0b 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -124,7 +124,7 @@ class Field: self.ctype = self.struct_name + self.name + 't' elif desc.type == FieldD.TYPE_MESSAGE: self.ltype = 'PB_LTYPE_SUBMESSAGE' - self.ctype = names_from_type_name(desc.type_name) + self.ctype = self.submsgname = names_from_type_name(desc.type_name) else: raise NotImplementedError(desc.type) @@ -167,8 +167,8 @@ class Field: if self.max_size is None: return None # Not implemented else: - array_decl = '[%d]' % self.max_size - default = self.default.encode('string_escape') + array_decl = '[%d]' % (self.max_size + 1) + default = str(self.default).encode('string_escape') default = default.replace('"', '\\"') default = '"' + default + '"' elif self.ltype == 'PB_LTYPE_BYTES': @@ -223,7 +223,7 @@ class Field: result += ' 0,' if self.ltype == 'PB_LTYPE_SUBMESSAGE': - result += '\n &%s_fields}' % self.ctype + result += '\n &%s_fields}' % self.submsgname elif self.default is None or self.htype == 'PB_HTYPE_CALLBACK': result += ' 0}' else: diff --git a/pb.h b/pb.h index e6037b81..2c6a2520 100644 --- a/pb.h +++ b/pb.h @@ -126,6 +126,8 @@ typedef struct { * The encoding callback will receive the actual output stream. * It should write all the data in one call, including the field tag and * wire type. It can write multiple fields. + * + * The callback can be null if you want to skip a field. */ typedef struct _pb_istream_t pb_istream_t; typedef struct _pb_ostream_t pb_ostream_t; diff --git a/pb_decode.c b/pb_decode.c index 379d134c..2cde54c2 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -185,14 +185,19 @@ static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, v { iter->start = iter->current = fields; iter->field_index = 0; - iter->pData = dest_struct; + iter->pData = dest_struct + iter->current->data_offset; + iter->pSize = (char*)iter->pData + iter->current->size_offset; iter->dest_struct = dest_struct; } static bool pb_field_next(pb_field_iterator_t *iter) { bool notwrapped = true; - size_t prev_size = iter->current->data_size * iter->current->array_size; + size_t prev_size = iter->current->data_size; + + if (PB_HTYPE(iter->current->type) == PB_HTYPE_ARRAY) + prev_size *= iter->current->array_size; + iter->current++; iter->field_index++; if (iter->current->tag == 0) @@ -271,9 +276,14 @@ static bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_ } case PB_HTYPE_CALLBACK: + { + pb_callback_t *pCallback = (pb_callback_t*)iter->pData; + + if (pCallback->funcs.decode == NULL) + return skip(stream, wire_type); + if (wire_type == PB_WT_STRING) { - pb_callback_t *pCallback = (pb_callback_t*)iter->pData; pb_istream_t substream; if (!make_string_substream(stream, &substream)) @@ -293,7 +303,6 @@ static bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_ * which in turn allows to use same callback for packed and * not-packed fields. */ pb_istream_t substream; - pb_callback_t *pCallback = (pb_callback_t*)iter->pData; uint8_t buffer[10]; size_t size = sizeof(buffer); @@ -303,7 +312,8 @@ static bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_ return pCallback->funcs.decode(&substream, iter->current, pCallback->arg); } - + } + default: return false; } @@ -344,6 +354,10 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc { *(size_t*)iter.pSize = 0; } + else if (PB_HTYPE(iter.current->type) == PB_HTYPE_REQUIRED) + { + memset(iter.pData, 0, iter.current->data_size); + } } while (pb_field_next(&iter)); while (stream->bytes_left) @@ -351,10 +365,15 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc uint32_t temp; int tag, wire_type; if (!pb_decode_varint32(stream, &temp)) - return stream->bytes_left == 0; /* Was it EOF? */ + { + if (stream->bytes_left == 0) + break; /* It was EOF */ + else + return false; /* It was error */ + } if (temp == 0) - return true; /* Special feature: allow 0-terminated messages. */ + break; /* Special feature: allow 0-terminated messages. */ tag = temp >> 3; wire_type = temp & 7; diff --git a/pb_encode.c b/pb_encode.c index 188d768d..2e740347 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -130,7 +130,10 @@ bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_ { pData = (const char*)pData + prev_size + field->data_offset; pSize = (const char*)pData + field->size_offset; - prev_size = field->data_size * field->array_size; + + prev_size = field->data_size; + if (PB_HTYPE(field->type) == PB_HTYPE_ARRAY) + prev_size *= field->array_size; pb_encoder_t func = PB_ENCODERS[PB_LTYPE(field->type)]; @@ -339,7 +342,7 @@ bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void substream.max_size = size; substream.bytes_written = 0; - status = pb_encode(stream, (pb_field_t*)field->ptr, src); + status = pb_encode(&substream, (pb_field_t*)field->ptr, src); stream->bytes_written += substream.bytes_written; diff --git a/tests/Makefile b/tests/Makefile index 4993297e..243dbd17 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -2,21 +2,27 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h person.h unittests.h TESTS=test_decode1 test_encode1 decode_unittests encode_unittests -all: $(TESTS) run_unittests +all: $(TESTS) run_unittests breakpoints clean: rm -f $(TESTS) %: %.c $(DEPS) - $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c + $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c person.c -%.h: %.proto +person.h: person.proto protoc -I. -I../generator -I/usr/include -operson.pb $< python ../generator/nanopb_generator.py person.pb -run_unittests: decode_unittests encode_unittests - ./decode_unittests - ./encode_unittests +breakpoints: ../*.c *.c + grep -n 'return false;' $^ | cut -d: -f-2 | xargs -n 1 echo b > $@ + +run_unittests: decode_unittests encode_unittests test_encode1 test_decode1 + ./decode_unittests > /dev/null + ./encode_unittests > /dev/null + + [ "`./test_encode1 | ./test_decode1`" = \ + "`./test_encode1 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] run_fuzztest: test_decode1 bash -c 'I=1; while cat /dev/urandom | ./test_decode1 > /dev/null; do I=$$(($$I+1)); echo -en "\r$$I"; done' \ No newline at end of file diff --git a/tests/person.proto b/tests/person.proto index 5befb076..dafcf934 100644 --- a/tests/person.proto +++ b/tests/person.proto @@ -4,8 +4,7 @@ message Person { required string name = 1 [(nanopb).max_size = 40]; required int32 id = 2; optional string email = 3 [(nanopb).max_size = 40]; - optional bytes test = 5 [default="\x00\x01\x02", (nanopb).max_size = 20]; - + enum PhoneType { MOBILE = 0; HOME = 1; diff --git a/tests/test_decode1.c b/tests/test_decode1.c index a2c7f428..b7698efc 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -2,9 +2,6 @@ #include #include "person.h" -/* This test has only one source file anyway.. */ -#include "person.c" - bool print_person(pb_istream_t *stream) { int i; @@ -13,12 +10,33 @@ bool print_person(pb_istream_t *stream) if (!pb_decode(stream, Person_fields, &person)) return false; - printf("Person: name '%s' id '%d' email '%s'\n", person.name, person.id, person.email); + printf("name: \"%s\"\n", person.name); + printf("id: %d\n", person.id); + + if (person.has_email) + printf("email: \"%s\"\n", person.email); for (i = 0; i < person.phone_count; i++) { Person_PhoneNumber *phone = &person.phone[i]; - printf("PhoneNumber: number '%s' type '%d'\n", phone->number, phone->type); + printf("phone {\n"); + printf(" number: \"%s\"\n", phone->number); + + switch (phone->type) + { + case Person_PhoneType_WORK: + printf(" type: WORK\n"); + break; + + case Person_PhoneType_HOME: + printf(" type: HOME\n"); + break; + + case Person_PhoneType_MOBILE: + printf(" type: MOBILE\n"); + break; + } + printf("}\n"); } return true; diff --git a/tests/test_encode1.c b/tests/test_encode1.c index 99be7cb5..ac13df35 100644 --- a/tests/test_encode1.c +++ b/tests/test_encode1.c @@ -2,9 +2,6 @@ #include #include "person.h" -/* This test has only one source file anyway.. */ -#include "person.c" - bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) { FILE *file = (FILE*) stream->state; -- cgit 1.2.3-korg From 494fbd91e4e5574a4cf8dbe69b3f80a08e97e85b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 22 Aug 2011 15:22:41 +0000 Subject: example client git-svn-id: https://svn.kapsi.fi/jpa/nanopb@958 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- example/Makefile | 4 +- example/client.c | 116 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ example/common.c | 49 +++++++++++++++++++++++ example/common.h | 9 +++++ example/server.c | 30 +++----------- 5 files changed, 182 insertions(+), 26 deletions(-) create mode 100644 example/client.c create mode 100644 example/common.c create mode 100644 example/common.h diff --git a/example/Makefile b/example/Makefile index 0f08f2d7..efc4a8de 100644 --- a/example/Makefile +++ b/example/Makefile @@ -1,10 +1,10 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h -all: server +all: server client %: %.c $(DEPS) fileproto.h - $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c fileproto.c + $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c fileproto.c common.c fileproto.h: fileproto.proto ../generator/nanopb_generator.py protoc -I. -I../generator -I/usr/include -ofileproto.pb $< diff --git a/example/client.c b/example/client.c new file mode 100644 index 00000000..f0520574 --- /dev/null +++ b/example/client.c @@ -0,0 +1,116 @@ +/* This is a simple TCP client that connects to port 1234 and prints a list + * of files in a given directory. + * + * It directly deserializes and serializes messages from network, minimizing + * memory use. + * + * For flexibility, this example is implemented using posix api. + * In a real embedded system you would typically use some other kind of + * a communication and filesystem layer. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "fileproto.h" +#include "common.h" + +bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void *arg) +{ + FileInfo fileinfo; + + if (!pb_decode(stream, FileInfo_fields, &fileinfo)) + return false; + + printf("%-10lld %s\n", fileinfo.inode, fileinfo.name); + + return true; +} + +bool listdir(int fd, char *path) +{ + ListFilesRequest request; + ListFilesResponse response; + pb_istream_t input = pb_istream_from_socket(fd); + pb_ostream_t output = pb_ostream_from_socket(fd); + uint8_t zero = 0; + + if (path == NULL) + { + request.has_path = false; + } + else + { + request.has_path = true; + if (strlen(path) + 1 > sizeof(request.path)) + { + fprintf(stderr, "Too long path.\n"); + return false; + } + + strcpy(request.path, path); + } + + if (!pb_encode(&output, ListFilesRequest_fields, &request)) + { + fprintf(stderr, "Encoding failed.\n"); + return false; + } + + /* We signal the end of request with a 0 tag. */ + pb_write(&output, &zero, 1); + + response.file.funcs.decode = &printfile_callback; + + if (!pb_decode(&input, ListFilesResponse_fields, &response)) + { + fprintf(stderr, "Decoding failed.\n"); + return false; + } + + if (response.path_error) + { + fprintf(stderr, "Server reported error.\n"); + return false; + } + + return true; +} + +int main(int argc, char **argv) +{ + int sockfd; + struct sockaddr_in servaddr; + char *path = NULL; + + if (argc > 1) + path = argv[1]; + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + + memset(&servaddr, 0, sizeof(servaddr)); + servaddr.sin_family = AF_INET; + servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + servaddr.sin_port = htons(1234); + + if (connect(sockfd, (struct sockaddr *)&servaddr, sizeof(servaddr)) != 0) + { + perror("connect"); + return 1; + } + + if (!listdir(sockfd, path)) + return 2; + + close(sockfd); + + return 0; +} diff --git a/example/common.c b/example/common.c new file mode 100644 index 00000000..9d93219b --- /dev/null +++ b/example/common.c @@ -0,0 +1,49 @@ +/* Simple binding of nanopb streams to TCP sockets. + */ + +#include +#include +#include +#include + +#include "common.h" + +static bool write_callback(pb_ostream_t *stream, const uint8_t *buf, size_t count) +{ + int fd = (int)stream->state; + return send(fd, buf, count, 0) == count; +} + +static bool read_callback(pb_istream_t *stream, uint8_t *buf, size_t count) +{ + int fd = (int)stream->state; + int result; + + if (buf == NULL) + { + /* Well, this is a really inefficient way to skip input. */ + /* It is only used when there are unknown fields. */ + char dummy; + while (count-- && recv(fd, &dummy, 1, 0) == 1); + return count == 0; + } + + result = recv(fd, buf, count, MSG_WAITALL); + + if (result == 0) + stream->bytes_left = 0; /* EOF */ + + return result == count; +} + +pb_ostream_t pb_ostream_from_socket(int fd) +{ + pb_ostream_t stream = {&write_callback, (void*)fd, SIZE_MAX, 0}; + return stream; +} + +pb_istream_t pb_istream_from_socket(int fd) +{ + pb_istream_t stream = {&read_callback, (void*)fd, SIZE_MAX}; + return stream; +} diff --git a/example/common.h b/example/common.h new file mode 100644 index 00000000..8dab3b7c --- /dev/null +++ b/example/common.h @@ -0,0 +1,9 @@ +#ifndef _PB_EXAMPLE_COMMON_H_ +#define _PB_EXAMPLE_COMMON_H_ + +#include + +pb_ostream_t pb_ostream_from_socket(int fd); +pb_istream_t pb_istream_from_socket(int fd); + +#endif \ No newline at end of file diff --git a/example/server.c b/example/server.c index a671f4cb..04f88f0f 100644 --- a/example/server.c +++ b/example/server.c @@ -21,28 +21,7 @@ #include #include "fileproto.h" - -bool write_callback(pb_ostream_t *stream, const uint8_t *buf, size_t count) -{ - int fd = *(int*)stream->state; - return send(fd, buf, count, 0) == count; -} - -bool read_callback(pb_istream_t *stream, uint8_t *buf, size_t count) -{ - int fd = *(int*)stream->state; - - if (buf == NULL) - { - /* Well, this is a really inefficient way to skip input. */ - /* It is only used when there are unknown fields. */ - char dummy; - while (count-- && recv(fd, &dummy, 1, 0) == 1); - return count == 0; - } - - return recv(fd, buf, count, MSG_WAITALL) == count; -} +#include "common.h" bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, const void *arg) { @@ -70,8 +49,8 @@ void handle_connection(int connfd) { ListFilesRequest request; ListFilesResponse response; - pb_istream_t input = {&read_callback, &connfd, SIZE_MAX}; - pb_ostream_t output = {&write_callback, &connfd, SIZE_MAX, 0}; + pb_istream_t input = pb_istream_from_socket(connfd); + pb_ostream_t output = pb_ostream_from_socket(connfd); DIR *directory; if (!pb_decode(&input, ListFilesRequest_fields, &request)) @@ -109,9 +88,12 @@ int main(int argc, char **argv) { int listenfd, connfd; struct sockaddr_in servaddr; + int reuse = 1; listenfd = socket(AF_INET, SOCK_STREAM, 0); + setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + memset(&servaddr, 0, sizeof(servaddr)); servaddr.sin_family = AF_INET; servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); -- cgit 1.2.3-korg From 5a9f85b87615078868d3b394370a30b550186cce Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 22 Aug 2011 18:57:23 +0000 Subject: unittests, change to PB_LTYPE_BYTES data size git-svn-id: https://svn.kapsi.fi/jpa/nanopb@959 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/index.rst | 1 + docs/reference.rst | 4 +-- generator/nanopb_generator.py | 3 +++ pb_decode.c | 6 ++--- tests/Makefile | 22 +++++++++++----- tests/decode_unittests.c | 59 ++++++++++++++++++++++++++++++++++++++----- 6 files changed, 77 insertions(+), 18 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index ea891237..f2f8dcfd 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,6 +47,7 @@ Features and limitations #) Encoding is focused on writing to streams. For memory buffers only it could be made more efficient. #) The deprecated Protocol Buffers feature called "groups" is not supported. #) Fields in the generated structs are ordered by the tag number, instead of the natural ordering in .proto file. +#) Unknown fields are not preserved when decoding and re-encoding a message. Getting started =============== diff --git a/docs/reference.rst b/docs/reference.rst index 2812958f..2fc3d481 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -61,7 +61,7 @@ Describes a single structure field with memory position in relation to others. T :type: LTYPE and HTYPE of the field. :data_offset: Offset of field data, relative to the end of the previous field. :size_offset: Offset of *bool* flag for optional fields or *size_t* count for arrays, relative to field data. -:data_size: Size of a single data entry, in bytes. +:data_size: Size of a single data entry, in bytes. For PB_LTYPE_BYTES, the size of the byte array inside the containing structure. :array_size: Maximum number of entries in an array, if it is an array type. :ptr: Pointer to default value for optional fields, or to submessage description for PB_LTYPE_SUBMESSAGE. @@ -339,7 +339,7 @@ Read and decode all fields of a structure. Reads until EOF on input stream. :: :dest_struct: Pointer to structure where data will be stored. :returns: True on success, false on IO error, on detectable errors in field description, if a field encoder returns false or if a required field is missing. -In Protocol Buffers binary format, EOF is only allowed between fields. If it happens anywhere else, pb_decode will return *false*. +In Protocol Buffers binary format, EOF is only allowed between fields. If it happens anywhere else, pb_decode will return *false*. If pb_decode returns false, you cannot trust any of the data in the structure. In addition to EOF, the pb_decode implementation supports terminating a message with a 0 byte. This is compatible with the official Protocol Buffers because 0 is never a valid field tag. diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index a6d38c0b..28268512 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -218,6 +218,9 @@ class Field: result += '\n pb_membersize(%s, %s[0]),' % (self.struct_name, self.name) result += ('\n pb_membersize(%s, %s) / pb_membersize(%s, %s[0]),' % (self.struct_name, self.name, self.struct_name, self.name)) + elif self.ltype == 'PB_LTYPE_BYTES': + result += '\n pb_membersize(%s, bytes),' % self.ctype + result += ' 0,' else: result += '\n pb_membersize(%s, %s),' % (self.struct_name, self.name) result += ' 0,' diff --git a/pb_decode.c b/pb_decode.c index 2cde54c2..9208bdac 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -73,7 +73,7 @@ static bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest) { uint8_t byte; - int bitpos = 0; + uint8_t bitpos = 0; *dest = 0; while (bitpos < 64 && pb_read(stream, &byte, 1)) @@ -460,9 +460,7 @@ bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) return false; x->size = temp; - /* Note: data_size includes the size of the x.size field, too. - * Calculate actual size starting from offset. */ - if (x->size > field->data_size - offsetof(pb_bytes_array_t, bytes)) + if (x->size > field->data_size) return false; return pb_read(stream, x->bytes, x->size); diff --git a/tests/Makefile b/tests/Makefile index 243dbd17..115364b6 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,22 +1,32 @@ -CFLAGS=-ansi -Wall -Werror -I .. -g -O0 -DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h person.h unittests.h +CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage +LDFLAGS=--coverage +DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.h unittests.h TESTS=test_decode1 test_encode1 decode_unittests encode_unittests all: $(TESTS) run_unittests breakpoints clean: - rm -f $(TESTS) + rm -f $(TESTS) *.o *.gcda *.gcno -%: %.c $(DEPS) - $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c person.c +%.o: %.c $(DEPS) + $(CC) $(CFLAGS) -c -o $@ $< -person.h: person.proto +test_decode1: test_decode1.o ../pb_decode.o person.o +test_encode1: test_encode1.o ../pb_encode.o person.o +decode_unittests: decode_unittests.o ../pb_decode.o person.o +encode_unittests: encode_unittests.o ../pb_encode.o person.o + +person.c person.h: person.proto protoc -I. -I../generator -I/usr/include -operson.pb $< python ../generator/nanopb_generator.py person.pb breakpoints: ../*.c *.c grep -n 'return false;' $^ | cut -d: -f-2 | xargs -n 1 echo b > $@ +coverage: + gcov -o .. ../pb_encode.c + gcov -o .. ../pb_decode.c + run_unittests: decode_unittests encode_unittests test_encode1 test_decode1 ./decode_unittests > /dev/null ./encode_unittests > /dev/null diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c index ac51d491..e733379d 100644 --- a/tests/decode_unittests.c +++ b/tests/decode_unittests.c @@ -3,7 +3,7 @@ #include "pb_decode.h" #include "unittests.h" -#define S(x) pb_istream_from_buffer((uint8_t*)x, sizeof(x)) +#define S(x) pb_istream_from_buffer((uint8_t*)x, sizeof(x) - 1) bool stream_callback(pb_istream_t *stream, uint8_t *buf, size_t count) { @@ -68,17 +68,17 @@ int main() { pb_istream_t s; COMMENT("Test pb_skip_varint"); - TEST((s = S("\x00""foobar"), pb_skip_varint(&s) && s.bytes_left == 7)) - TEST((s = S("\xAC\x02""foobar"), pb_skip_varint(&s) && s.bytes_left == 7)) + TEST((s = S("\x00""foobar"), pb_skip_varint(&s) && s.bytes_left == 6)) + TEST((s = S("\xAC\x02""foobar"), pb_skip_varint(&s) && s.bytes_left == 6)) TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01""foobar"), - pb_skip_varint(&s) && s.bytes_left == 7)) + pb_skip_varint(&s) && s.bytes_left == 6)) } { pb_istream_t s; COMMENT("Test pb_skip_string") - TEST((s = S("\x00""foobar"), pb_skip_string(&s) && s.bytes_left == 7)) - TEST((s = S("\x04""testfoobar"), pb_skip_string(&s) && s.bytes_left == 7)) + TEST((s = S("\x00""foobar"), pb_skip_string(&s) && s.bytes_left == 6)) + TEST((s = S("\x04""testfoobar"), pb_skip_string(&s) && s.bytes_left == 6)) } { @@ -118,6 +118,53 @@ int main() TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_dec_svarint(&s, &f, &d) && d == INT64_MIN)) } + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_FIXED, 0, 0, 4, 0, 0}; + float d; + + COMMENT("Test pb_dec_fixed using float (failures here may be caused by imperfect rounding)") + TEST((s = S("\x00\x00\x00\x00"), pb_dec_fixed(&s, &f, &d) && d == 0.0f)) + TEST((s = S("\x00\x00\xc6\x42"), pb_dec_fixed(&s, &f, &d) && d == 99.0f)) + TEST((s = S("\x4e\x61\x3c\xcb"), pb_dec_fixed(&s, &f, &d) && d == -12345678.0f)) + TEST((s = S("\x00"), !pb_dec_fixed(&s, &f, &d) && d == -12345678.0f)) + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_FIXED, 0, 0, 8, 0, 0}; + double d; + + COMMENT("Test pb_dec_fixed using double (failures here may be caused by imperfect rounding)") + TEST((s = S("\x00\x00\x00\x00\x00\x00\x00\x00"), pb_dec_fixed(&s, &f, &d) && d == 0.0)) + TEST((s = S("\x00\x00\x00\x00\x00\xc0\x58\x40"), pb_dec_fixed(&s, &f, &d) && d == 99.0)) + TEST((s = S("\x00\x00\x00\xc0\x29\x8c\x67\xc1"), pb_dec_fixed(&s, &f, &d) && d == -12345678.0f)) + } + + { + pb_istream_t s; + struct { size_t size; uint8_t bytes[5]; } d; + pb_field_t f = {1, PB_LTYPE_BYTES, 0, 0, 5, 0, 0}; + + COMMENT("Test pb_dec_bytes") + TEST((s = S("\x00"), pb_dec_bytes(&s, &f, &d) && d.size == 0)) + TEST((s = S("\x01\xFF"), pb_dec_bytes(&s, &f, &d) && d.size == 1 && d.bytes[0] == 0xFF)) + TEST((s = S("\x06xxxxxx"), !pb_dec_bytes(&s, &f, &d))) + TEST((s = S("\x05xxxxx"), pb_dec_bytes(&s, &f, &d) && d.size == 5)) + TEST((s = S("\x05xxxx"), !pb_dec_bytes(&s, &f, &d))) + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_STRING, 0, 0, 5, 0, 0}; + char d[5]; + + COMMENT("Test pb_dec_string") + TEST((s = S("\x00"), pb_dec_string(&s, &f, &d) && d[0] == '\0')) + TEST((s = S("\x04xyzz"), pb_dec_string(&s, &f, &d) && strcmp(d, "xyzz") == 0)) + TEST((s = S("\x05xyzzy"), !pb_dec_string(&s, &f, &d))) + } + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); -- cgit 1.2.3-korg From 5490163200131e4d2af9676f22d13a611ed2b7b3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 23 Aug 2011 09:59:18 +0000 Subject: More unittests git-svn-id: https://svn.kapsi.fi/jpa/nanopb@960 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/index.rst | 7 +++ pb_decode.c | 5 +- tests/Makefile | 21 ++++++--- tests/decode_unittests.c | 117 +++++++++++++++++++++++++++++++++++++++++++++++ tests/encode_unittests.c | 47 +++++++++++++++++++ 5 files changed, 188 insertions(+), 9 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index f2f8dcfd..76580364 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -48,6 +48,7 @@ Features and limitations #) The deprecated Protocol Buffers feature called "groups" is not supported. #) Fields in the generated structs are ordered by the tag number, instead of the natural ordering in .proto file. #) Unknown fields are not preserved when decoding and re-encoding a message. +#) Numeric arrays are always encoded as packed, even if not marked as packed in .proto. This causes incompatibility with decoders that do not support packed format. Getting started =============== @@ -82,6 +83,12 @@ After that, buffer will contain the encoded message. The number of bytes in the message is stored in *stream.bytes_written*. You can feed the message to *protoc --decode=Example example.proto* to verify its validity. +Debugging and testing +===================== +Extensive unittests are included under the *tests* folder. Just type *make* there to run the tests. + +This also generates a file called *breakpoints* which includes all lines returning *false* in nanopb. You can use this in gdb by typing *source breakpoints*, after which gdb will break on first nanopb error. + Wishlist ======== #) A specialized encoder for encoding to a memory buffer. Should serialize in reverse order to avoid having to determine submessage size beforehand. diff --git a/pb_decode.c b/pb_decode.c index 9208bdac..26924472 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -257,7 +257,7 @@ static bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_ while (substream.bytes_left && *size < iter->current->array_size) { void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size); - if (!func(stream, iter->current, pItem)) + if (!func(&substream, iter->current, pItem)) return false; (*size)++; } @@ -381,7 +381,8 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc if (!pb_field_find(&iter, tag)) { /* No match found, skip data */ - skip(stream, wire_type); + if (!skip(stream, wire_type)) + return false; continue; } diff --git a/tests/Makefile b/tests/Makefile index 115364b6..38bdb25f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -11,10 +11,15 @@ clean: %.o: %.c $(DEPS) $(CC) $(CFLAGS) -c -o $@ $< -test_decode1: test_decode1.o ../pb_decode.o person.o -test_encode1: test_encode1.o ../pb_encode.o person.o -decode_unittests: decode_unittests.o ../pb_decode.o person.o -encode_unittests: encode_unittests.o ../pb_encode.o person.o +pb_encode.o: ../pb_encode.c $(DEPS) + $(CC) $(CFLAGS) -c -o $@ $< +pb_decode.o: ../pb_decode.c $(DEPS) + $(CC) $(CFLAGS) -c -o $@ $< + +test_decode1: test_decode1.o pb_decode.o person.o +test_encode1: test_encode1.o pb_encode.o person.o +decode_unittests: decode_unittests.o pb_decode.o person.o +encode_unittests: encode_unittests.o pb_encode.o person.o person.c person.h: person.proto protoc -I. -I../generator -I/usr/include -operson.pb $< @@ -23,11 +28,13 @@ person.c person.h: person.proto breakpoints: ../*.c *.c grep -n 'return false;' $^ | cut -d: -f-2 | xargs -n 1 echo b > $@ -coverage: - gcov -o .. ../pb_encode.c - gcov -o .. ../pb_decode.c +coverage: run_unittests + gcov pb_encode.gcda + gcov pb_decode.gcda run_unittests: decode_unittests encode_unittests test_encode1 test_decode1 + rm -f *.gcda + ./decode_unittests > /dev/null ./encode_unittests > /dev/null diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c index e733379d..85dd8d58 100644 --- a/tests/decode_unittests.c +++ b/tests/decode_unittests.c @@ -15,6 +15,43 @@ bool stream_callback(pb_istream_t *stream, uint8_t *buf, size_t count) return true; } +typedef struct { size_t data_count; int32_t data[10]; } IntegerArray; +const pb_field_t IntegerArray_fields[] = { + {1, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, offsetof(IntegerArray, data), + pb_delta(IntegerArray, data_count, data), + pb_membersize(IntegerArray, data[0]), + pb_membersize(IntegerArray, data) / pb_membersize(IntegerArray, data[0])}, + + PB_LAST_FIELD +}; + +typedef struct { pb_callback_t data; } CallbackArray; +const pb_field_t CallbackArray_fields[] = { + {1, PB_HTYPE_CALLBACK | PB_LTYPE_VARINT, offsetof(CallbackArray, data), + 0, pb_membersize(CallbackArray, data), 0}, + + PB_LAST_FIELD +}; + +/* Verifies that the stream passed to callback matches the byte array pointed to by arg. */ +bool callback_check(pb_istream_t *stream, const pb_field_t *field, void *arg) +{ + int i; + uint8_t byte; + pb_bytes_array_t *ref = (pb_bytes_array_t*) arg; + + for (i = 0; i < ref->size; i++) + { + if (!pb_read(stream, &byte, 1)) + return false; + + if (byte != ref->bytes[i]) + return false; + } + + return true; +} + int main() { int status = 0; @@ -72,6 +109,7 @@ int main() TEST((s = S("\xAC\x02""foobar"), pb_skip_varint(&s) && s.bytes_left == 6)) TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01""foobar"), pb_skip_varint(&s) && s.bytes_left == 6)) + TEST((s = S("\xFF"), !pb_skip_varint(&s))) } { @@ -79,6 +117,8 @@ int main() COMMENT("Test pb_skip_string") TEST((s = S("\x00""foobar"), pb_skip_string(&s) && s.bytes_left == 6)) TEST((s = S("\x04""testfoobar"), pb_skip_string(&s) && s.bytes_left == 6)) + TEST((s = S("\x04"), !pb_skip_string(&s))) + TEST((s = S("\xFF"), !pb_skip_string(&s))) } { @@ -165,6 +205,83 @@ int main() TEST((s = S("\x05xyzzy"), !pb_dec_string(&s, &f, &d))) } + { + pb_istream_t s; + IntegerArray dest; + + COMMENT("Testing pb_decode with repeated int32 field") + TEST((s = S(""), pb_decode(&s, IntegerArray_fields, &dest) && dest.data_count == 0)) + TEST((s = S("\x08\x01\x08\x02"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 2 && dest.data[0] == 1 && dest.data[1] == 2)) + s = S("\x08\x01\x08\x02\x08\x03\x08\x04\x08\x05\x08\x06\x08\x07\x08\x08\x08\x09\x08\x0A"); + TEST(pb_decode(&s, IntegerArray_fields, &dest) && dest.data_count == 10 && dest.data[9] == 10) + s = S("\x08\x01\x08\x02\x08\x03\x08\x04\x08\x05\x08\x06\x08\x07\x08\x08\x08\x09\x08\x0A\x08\x0B"); + TEST(!pb_decode(&s, IntegerArray_fields, &dest)) + } + + { + pb_istream_t s; + IntegerArray dest; + + COMMENT("Testing pb_decode with packed int32 field") + TEST((s = S("\x0A\x01\x01"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 1 && dest.data[0] == 1)) + TEST((s = S("\x0A\x0A\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 10 && dest.data[0] == 1 && dest.data[9] == 10)) + TEST((s = S("\x0A\x0B\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B"), !pb_decode(&s, IntegerArray_fields, &dest))) + + /* Test invalid wire data */ + TEST((s = S("\x0A\xFF"), !pb_decode(&s, IntegerArray_fields, &dest))) + TEST((s = S("\x0A\x01"), !pb_decode(&s, IntegerArray_fields, &dest))) + } + + { + pb_istream_t s; + IntegerArray dest; + + COMMENT("Testing pb_decode with unknown fields") + TEST((s = S("\x18\x0F\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 1 && dest.data[0] == 1)) + TEST((s = S("\x19\x00\x00\x00\x00\x00\x00\x00\x00\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 1 && dest.data[0] == 1)) + TEST((s = S("\x1A\x00\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 1 && dest.data[0] == 1)) + TEST((s = S("\x1B\x08\x01"), !pb_decode(&s, IntegerArray_fields, &dest))) + TEST((s = S("\x1D\x00\x00\x00\x00\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 1 && dest.data[0] == 1)) + } + + { + pb_istream_t s; + CallbackArray dest; + struct { size_t size; uint8_t bytes[10]; } ref; + dest.data.funcs.decode = &callback_check; + dest.data.arg = &ref; + + COMMENT("Testing pb_decode with callbacks") + /* Single varint */ + ref.size = 1; ref.bytes[0] = 0x55; + TEST((s = S("\x08\x55"), pb_decode(&s, CallbackArray_fields, &dest))) + /* Packed varint */ + ref.size = 3; ref.bytes[0] = ref.bytes[1] = ref.bytes[2] = 0x55; + TEST((s = S("\x0A\x03\x55\x55\x55"), pb_decode(&s, CallbackArray_fields, &dest))) + /* Packed varint with loop */ + ref.size = 1; ref.bytes[0] = 0x55; + TEST((s = S("\x0A\x03\x55\x55\x55"), pb_decode(&s, CallbackArray_fields, &dest))) + /* Single fixed32 */ + ref.size = 4; ref.bytes[0] = ref.bytes[1] = ref.bytes[2] = ref.bytes[3] = 0xAA; + TEST((s = S("\x0D\xAA\xAA\xAA\xAA"), pb_decode(&s, CallbackArray_fields, &dest))) + /* Single fixed64 */ + ref.size = 8; memset(ref.bytes, 0xAA, 8); + TEST((s = S("\x09\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"), pb_decode(&s, CallbackArray_fields, &dest))) + /* Unsupported field type */ + TEST((s = S("\x0B\x00"), !pb_decode(&s, CallbackArray_fields, &dest))) + + /* Just make sure that our test function works */ + ref.size = 1; ref.bytes[0] = 0x56; + TEST((s = S("\x08\x55"), !pb_decode(&s, CallbackArray_fields, &dest))) + } + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); diff --git a/tests/encode_unittests.c b/tests/encode_unittests.c index 645dd21a..ee7f7a34 100644 --- a/tests/encode_unittests.c +++ b/tests/encode_unittests.c @@ -116,6 +116,53 @@ int main() TEST(WRITES(pb_enc_svarint(&s, &field, &lmin), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01")); } + { + uint8_t buffer[30]; + pb_ostream_t s; + pb_field_t field = {1, PB_LTYPE_FIXED, 0, 0, sizeof(float)}; + float fvalue; + double dvalue; + + COMMENT("Test pb_enc_fixed using float") + fvalue = 0.0f; + TEST(WRITES(pb_enc_fixed(&s, &field, &fvalue), "\x00\x00\x00\x00")) + fvalue = 99.0f; + TEST(WRITES(pb_enc_fixed(&s, &field, &fvalue), "\x00\x00\xc6\x42")) + fvalue = -12345678.0f; + TEST(WRITES(pb_enc_fixed(&s, &field, &fvalue), "\x4e\x61\x3c\xcb")) + + COMMENT("Test pb_enc_fixed using double") + field.data_size = sizeof(double); + dvalue = 0.0; + TEST(WRITES(pb_enc_fixed(&s, &field, &dvalue), "\x00\x00\x00\x00\x00\x00\x00\x00")) + dvalue = 99.0; + TEST(WRITES(pb_enc_fixed(&s, &field, &dvalue), "\x00\x00\x00\x00\x00\xc0\x58\x40")) + dvalue = -12345678.0; + TEST(WRITES(pb_enc_fixed(&s, &field, &dvalue), "\x00\x00\x00\xc0\x29\x8c\x67\xc1")) + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + struct { size_t size; uint8_t bytes[5]; } value = {5, {'x', 'y', 'z', 'z', 'y'}}; + + COMMENT("Test pb_enc_bytes") + TEST(WRITES(pb_enc_bytes(&s, NULL, &value), "\x05xyzzy")) + value.size = 0; + TEST(WRITES(pb_enc_bytes(&s, NULL, &value), "\x00")) + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + char value[] = "xyzzy"; + + COMMENT("Test pb_enc_string") + TEST(WRITES(pb_enc_string(&s, NULL, &value), "\x05xyzzy")) + value[0] = '\0'; + TEST(WRITES(pb_enc_string(&s, NULL, &value), "\x00")) + } + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); -- cgit 1.2.3-korg From 8b635924ba6b199e792e22bf56e30af9ba37ecdb Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 23 Aug 2011 13:33:43 +0000 Subject: check return values git-svn-id: https://svn.kapsi.fi/jpa/nanopb@961 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb_decode.c | 46 +++++++++++++++++++++++++++------------------- pb_encode.c | 41 +++++++++++++++++++++++++---------------- 2 files changed, 52 insertions(+), 35 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 26924472..0e55cbfb 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -3,11 +3,19 @@ * 2011 Petteri Aimonen */ + +#ifdef __GNUC__ +/* Verify that we remember to check all return values for proper error propagation */ +#define checkreturn __attribute__((warn_unused_result)) +#else +#define checkreturn +#endif + #include "pb.h" #include "pb_decode.h" #include -typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest); +typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn; /* --- Function pointers to field decoders --- * Order in the array must match pb_action_t LTYPE numbering. @@ -26,7 +34,7 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { * pb_istream * **************/ -bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) +bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) { if (stream->bytes_left < count) return false; @@ -38,7 +46,7 @@ bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) return true; } -static bool buf_read(pb_istream_t *stream, uint8_t *buf, size_t count) +static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count) { uint8_t *source = (uint8_t*)stream->state; @@ -62,7 +70,7 @@ pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize) * Helper functions * ********************/ -static bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) +static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) { uint64_t temp; bool status = pb_decode_varint(stream, &temp); @@ -70,7 +78,7 @@ static bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) return status; } -bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest) +bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) { uint8_t byte; uint8_t bitpos = 0; @@ -88,7 +96,7 @@ bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest) return false; } -bool pb_skip_varint(pb_istream_t *stream) +bool checkreturn pb_skip_varint(pb_istream_t *stream) { uint8_t byte; do @@ -99,7 +107,7 @@ bool pb_skip_varint(pb_istream_t *stream) return true; } -bool pb_skip_string(pb_istream_t *stream) +bool checkreturn pb_skip_string(pb_istream_t *stream) { uint32_t length; if (!pb_decode_varint32(stream, &length)) @@ -113,7 +121,7 @@ bool pb_skip_string(pb_istream_t *stream) * to just assume the correct type and fail safely on corrupt message. */ -static bool skip(pb_istream_t *stream, int wire_type) +static bool checkreturn skip(pb_istream_t *stream, int wire_type) { switch (wire_type) { @@ -128,7 +136,7 @@ static bool skip(pb_istream_t *stream, int wire_type) /* Read a raw value to buffer, for the purpose of passing it to callback as * a substream. Size is maximum size on call, and actual size on return. */ -static bool read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, uint8_t *buf, size_t *size) +static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, uint8_t *buf, size_t *size) { size_t max_size = *size; switch (wire_type) @@ -156,7 +164,7 @@ static bool read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, uint8 } /* Decode string length from stream and return a substream with limited length */ -static bool make_string_substream(pb_istream_t *stream, pb_istream_t *substream) +static bool checkreturn make_string_substream(pb_istream_t *stream, pb_istream_t *substream) { uint32_t size; if (!pb_decode_varint32(stream, &size)) @@ -214,7 +222,7 @@ static bool pb_field_next(pb_field_iterator_t *iter) return notwrapped; } -static bool pb_field_find(pb_field_iterator_t *iter, int tag) +static bool checkreturn pb_field_find(pb_field_iterator_t *iter, int tag) { int start = iter->field_index; @@ -231,7 +239,7 @@ static bool pb_field_find(pb_field_iterator_t *iter, int tag) * Decode a single field * *************************/ -static bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_t *iter) +static bool checkreturn decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_t *iter) { pb_decoder_t func = PB_DECODERS[PB_LTYPE(iter->current->type)]; @@ -323,7 +331,7 @@ static bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_ * Decode all fields * *********************/ -bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { uint32_t fields_seen = 0; /* Used to check for required fields */ pb_field_iterator_t iter; @@ -421,7 +429,7 @@ static void endian_copy(void *dest, void *src, size_t destsize, size_t srcsize) #endif } -bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint64_t temp; bool status = pb_decode_varint(stream, &temp); @@ -429,7 +437,7 @@ bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) return status; } -bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint64_t temp; bool status = pb_decode_varint(stream, &temp); @@ -438,7 +446,7 @@ bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) return status; } -bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool checkreturn pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest) { #ifdef __BIG_ENDIAN__ uint8_t bytes[8] = {0}; @@ -452,7 +460,7 @@ bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest) #endif } -bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) { pb_bytes_array_t *x = (pb_bytes_array_t*)dest; @@ -467,7 +475,7 @@ bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) return pb_read(stream, x->bytes, x->size); } -bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint32_t size; bool status; @@ -482,7 +490,7 @@ bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest) return status; } -bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) { pb_istream_t substream; diff --git a/pb_encode.c b/pb_encode.c index 2e740347..2a4d8e11 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -7,7 +7,15 @@ #include "pb_encode.h" #include -typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, const void *src); +#ifdef __GNUC__ +/* Verify that we remember to check all return values for proper error propagation */ +#define checkreturn __attribute__((warn_unused_result)) +#else +#define checkreturn +#endif + + +typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, const void *src) checkreturn; /* --- Function pointers to field encoders --- * Order in the array must match pb_action_t LTYPE numbering. @@ -24,7 +32,7 @@ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { /* pb_ostream_t implementation */ -static bool buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) +static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) { uint8_t *dest = (uint8_t*)stream->state; memcpy(dest, buf, count); @@ -42,7 +50,7 @@ pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize) return stream; } -bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) +bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) { if (stream->callback != NULL) { @@ -59,7 +67,7 @@ bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) /* Main encoding stuff */ -static bool encode_array(pb_ostream_t *stream, const pb_field_t *field, +static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, const void *pData, size_t count, pb_encoder_t func) { int i; @@ -89,7 +97,8 @@ static bool encode_array(pb_ostream_t *stream, const pb_field_t *field, size = sizestream.bytes_written; } - pb_encode_varint(stream, size); + if (!pb_encode_varint(stream, size)) + return false; if (stream->callback == NULL) return pb_write(stream, NULL, size); /* Just sizing.. */ @@ -119,7 +128,7 @@ static bool encode_array(pb_ostream_t *stream, const pb_field_t *field, return true; } -bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) +bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) { const pb_field_t *field = fields; const void *pData = src_struct; @@ -181,7 +190,7 @@ bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_ } /* Helper functions */ -bool pb_encode_varint(pb_ostream_t *stream, uint64_t value) +bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value) { uint8_t buffer[10]; int i = 0; @@ -200,13 +209,13 @@ bool pb_encode_varint(pb_ostream_t *stream, uint64_t value) return pb_write(stream, buffer, i); } -bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number) +bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number) { int tag = wiretype | (field_number << 3); return pb_encode_varint(stream, tag); } -bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field) +bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field) { pb_wire_type_t wiretype; switch (PB_LTYPE(field->type)) @@ -238,7 +247,7 @@ bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field) return pb_encode_tag(stream, wiretype, field->tag); } -bool pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size) +bool checkreturn pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size) { if (!pb_encode_varint(stream, size)) return false; @@ -264,14 +273,14 @@ static void endian_copy(void *dest, const void *src, size_t destsize, size_t src #endif } -bool pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src) +bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src) { uint64_t value = 0; endian_copy(&value, src, sizeof(value), field->data_size); return pb_encode_varint(stream, value); } -bool pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src) +bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src) { uint64_t value = 0; uint64_t zigzagged; @@ -288,7 +297,7 @@ bool pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *s return pb_encode_varint(stream, zigzagged); } -bool pb_enc_fixed(pb_ostream_t *stream, const pb_field_t *field, const void *src) +bool checkreturn pb_enc_fixed(pb_ostream_t *stream, const pb_field_t *field, const void *src) { #ifdef __BIG_ENDIAN__ uint8_t bytes[8] = {0}; @@ -301,18 +310,18 @@ bool pb_enc_fixed(pb_ostream_t *stream, const pb_field_t *field, const void *src #endif } -bool pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) +bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) { pb_bytes_array_t *bytes = (pb_bytes_array_t*)src; return pb_encode_string(stream, bytes->bytes, bytes->size); } -bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src) +bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src) { return pb_encode_string(stream, (uint8_t*)src, strlen((char*)src)); } -bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src) +bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src) { pb_ostream_t substream = {0}; size_t size; -- cgit 1.2.3-korg From 5e27a83f9c0373893a62f2d98287d25fd6414f86 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 23 Aug 2011 15:57:20 +0000 Subject: Allocated extension number git-svn-id: https://svn.kapsi.fi/jpa/nanopb@962 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- generator/nanopb.proto | 10 +++++++++- generator/nanopb_pb2.py | 6 +++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/generator/nanopb.proto b/generator/nanopb.proto index 813d5b73..2610cd5c 100644 --- a/generator/nanopb.proto +++ b/generator/nanopb.proto @@ -12,7 +12,15 @@ message NanoPBOptions { optional int32 max_count = 2; } +// Protocol Buffers extension number registry +// -------------------------------- +// Project: Nanopb +// Contact: Petteri Aimonen +// Web site: http://kapsi.fi/~jpa/nanopb +// Extensions: 1010 (all types) +// -------------------------------- + extend google.protobuf.FieldOptions { - optional NanoPBOptions nanopb = 52001; + optional NanoPBOptions nanopb = 1010; } diff --git a/generator/nanopb_pb2.py b/generator/nanopb_pb2.py index eb926de9..f2fbeefd 100644 --- a/generator/nanopb_pb2.py +++ b/generator/nanopb_pb2.py @@ -10,13 +10,13 @@ from google.protobuf import descriptor_pb2 DESCRIPTOR = descriptor.FileDescriptor( name='nanopb.proto', package='', - serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"4\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05:?\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xa1\x96\x03 \x01(\x0b\x32\x0e.NanoPBOptions') + serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"4\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions') -NANOPB_FIELD_NUMBER = 52001 +NANOPB_FIELD_NUMBER = 1010 nanopb = descriptor.FieldDescriptor( name='nanopb', full_name='nanopb', index=0, - number=52001, type=11, cpp_type=10, label=1, + number=1010, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=True, extension_scope=None, -- cgit 1.2.3-korg From 7780da38e88027370009afda5a9b6dc93a4168d4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 23 Aug 2011 15:57:40 +0000 Subject: license git-svn-id: https://svn.kapsi.fi/jpa/nanopb@963 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- LICENSE | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..c7e973e6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,11 @@ +Copyright (c) 2011 Petteri Aimonen + +This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source distribution. \ No newline at end of file -- cgit 1.2.3-korg From 9966a8c9b680ada47b6bd5fe96405c0abaf773e4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 23 Aug 2011 15:57:47 +0000 Subject: more unittests git-svn-id: https://svn.kapsi.fi/jpa/nanopb@964 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb_decode.c | 9 +++------ tests/decode_unittests.c | 12 ++++++++++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 0e55cbfb..6c6b1d9e 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -337,17 +337,14 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void pb_field_iterator_t iter; int i; - if (fields[0].tag == 0) - { - /* No fields -> nothing to do */ - return pb_read(stream, NULL, stream->bytes_left); - } - pb_field_init(&iter, fields, dest_struct); /* Initialize size/has fields and apply default values */ do { + if (iter.current->tag == 0) + continue; + if (PB_HTYPE(iter.current->type) == PB_HTYPE_OPTIONAL) { *(bool*)iter.pSize = false; diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c index 85dd8d58..6f2b77e5 100644 --- a/tests/decode_unittests.c +++ b/tests/decode_unittests.c @@ -282,6 +282,18 @@ int main() TEST((s = S("\x08\x55"), !pb_decode(&s, CallbackArray_fields, &dest))) } + { + pb_istream_t s; + IntegerArray dest; + + COMMENT("Testing pb_decode message termination") + TEST((s = S(""), pb_decode(&s, IntegerArray_fields, &dest))) + TEST((s = S("\x00"), pb_decode(&s, IntegerArray_fields, &dest))) + TEST((s = S("\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest))) + TEST((s = S("\x08\x01\x00"), pb_decode(&s, IntegerArray_fields, &dest))) + TEST((s = S("\x08"), !pb_decode(&s, IntegerArray_fields, &dest))) + } + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); -- cgit 1.2.3-korg From 64ac18c8848d3071a16065c84fca3b5e7210ce95 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 23 Aug 2011 16:16:33 +0000 Subject: Changed autogenerated file naming from foo.c to foo.pb.c git-svn-id: https://svn.kapsi.fi/jpa/nanopb@965 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- example/Makefile | 9 ++++++--- example/client.c | 2 +- example/server.c | 2 +- generator/nanopb_generator.py | 6 +++--- pb_encode.c | 7 +++++++ tests/Makefile | 20 +++++++++++--------- tests/decode_unittests.c | 21 +++------------------ tests/encode_unittests.c | 22 ++++++++++++++++++++++ 8 files changed, 54 insertions(+), 35 deletions(-) diff --git a/example/Makefile b/example/Makefile index efc4a8de..14dd9fa4 100644 --- a/example/Makefile +++ b/example/Makefile @@ -3,9 +3,12 @@ DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h all: server client -%: %.c $(DEPS) fileproto.h - $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c fileproto.c common.c +clean: + rm -f server client fileproto.pb.c fileproto.pb.h -fileproto.h: fileproto.proto ../generator/nanopb_generator.py +%: %.c $(DEPS) fileproto.pb.h fileproto.pb.c + $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c fileproto.pb.c common.c + +fileproto.pb.c fileproto.pb.h: fileproto.proto ../generator/nanopb_generator.py protoc -I. -I../generator -I/usr/include -ofileproto.pb $< python ../generator/nanopb_generator.py fileproto.pb diff --git a/example/client.c b/example/client.c index f0520574..95112e4c 100644 --- a/example/client.c +++ b/example/client.c @@ -20,7 +20,7 @@ #include #include -#include "fileproto.h" +#include "fileproto.pb.h" #include "common.h" bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void *arg) diff --git a/example/server.c b/example/server.c index 04f88f0f..aba0667b 100644 --- a/example/server.c +++ b/example/server.c @@ -20,7 +20,7 @@ #include #include -#include "fileproto.h" +#include "fileproto.pb.h" #include "common.h" bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, const void *arg) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 28268512..f09be346 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -382,7 +382,7 @@ if __name__ == '__main__': print "Usage: " + sys.argv[0] + " file.pb" print "where file.pb has been compiled from .proto by:" print "protoc -ofile.pb file.proto" - print "Output fill be written to file.h and file.c" + print "Output fill be written to file.pb.h and file.pb.c" sys.exit(1) data = open(sys.argv[1]).read() @@ -390,8 +390,8 @@ if __name__ == '__main__': enums, messages = parse_file(fdesc.file[0]) noext = os.path.splitext(sys.argv[1])[0] - headername = noext + '.h' - sourcename = noext + '.c' + headername = noext + '.pb.h' + sourcename = noext + '.pb.c' headerbasename = os.path.basename(headername) print "Writing to " + headername + " and " + sourcename diff --git a/pb_encode.c b/pb_encode.c index 2a4d8e11..d333cfd5 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -67,6 +67,10 @@ bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count /* Main encoding stuff */ +/* Callbacks don't need this function because they usually know the data type + * without examining the field structure. + * Therefore it is static for now. + */ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, const void *pData, size_t count, pb_encoder_t func) { @@ -74,6 +78,9 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie const void *p; size_t size; + if (count == 0) + return true; + if (PB_LTYPE(field->type) < PB_LTYPE_LAST_PACKABLE) { if (!pb_encode_tag(stream, PB_WT_STRING, field->tag)) diff --git a/tests/Makefile b/tests/Makefile index 38bdb25f..5035af8f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,12 +1,12 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage LDFLAGS=--coverage -DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.h unittests.h +DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h unittests.h unittestproto.pb.h TESTS=test_decode1 test_encode1 decode_unittests encode_unittests all: $(TESTS) run_unittests breakpoints clean: - rm -f $(TESTS) *.o *.gcda *.gcno + rm -f $(TESTS) person.pb* *.o *.gcda *.gcno %.o: %.c $(DEPS) $(CC) $(CFLAGS) -c -o $@ $< @@ -16,14 +16,16 @@ pb_encode.o: ../pb_encode.c $(DEPS) pb_decode.o: ../pb_decode.c $(DEPS) $(CC) $(CFLAGS) -c -o $@ $< -test_decode1: test_decode1.o pb_decode.o person.o -test_encode1: test_encode1.o pb_encode.o person.o -decode_unittests: decode_unittests.o pb_decode.o person.o -encode_unittests: encode_unittests.o pb_encode.o person.o +test_decode1: test_decode1.o pb_decode.o person.pb.o +test_encode1: test_encode1.o pb_encode.o person.pb.o +decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o +encode_unittests: encode_unittests.o pb_encode.o unittestproto.pb.o -person.c person.h: person.proto - protoc -I. -I../generator -I/usr/include -operson.pb $< - python ../generator/nanopb_generator.py person.pb +%.pb: %.proto + protoc -I. -I../generator -I/usr/include -o$@ $< + +%.pb.c %.pb.h: %.pb + python ../generator/nanopb_generator.py $< breakpoints: ../*.c *.c grep -n 'return false;' $^ | cut -d: -f-2 | xargs -n 1 echo b > $@ diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c index 6f2b77e5..006ad900 100644 --- a/tests/decode_unittests.c +++ b/tests/decode_unittests.c @@ -2,6 +2,7 @@ #include #include "pb_decode.h" #include "unittests.h" +#include "unittestproto.pb.h" #define S(x) pb_istream_from_buffer((uint8_t*)x, sizeof(x) - 1) @@ -15,24 +16,6 @@ bool stream_callback(pb_istream_t *stream, uint8_t *buf, size_t count) return true; } -typedef struct { size_t data_count; int32_t data[10]; } IntegerArray; -const pb_field_t IntegerArray_fields[] = { - {1, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, offsetof(IntegerArray, data), - pb_delta(IntegerArray, data_count, data), - pb_membersize(IntegerArray, data[0]), - pb_membersize(IntegerArray, data) / pb_membersize(IntegerArray, data[0])}, - - PB_LAST_FIELD -}; - -typedef struct { pb_callback_t data; } CallbackArray; -const pb_field_t CallbackArray_fields[] = { - {1, PB_HTYPE_CALLBACK | PB_LTYPE_VARINT, offsetof(CallbackArray, data), - 0, pb_membersize(CallbackArray, data), 0}, - - PB_LAST_FIELD -}; - /* Verifies that the stream passed to callback matches the byte array pointed to by arg. */ bool callback_check(pb_istream_t *stream, const pb_field_t *field, void *arg) { @@ -224,6 +207,8 @@ int main() IntegerArray dest; COMMENT("Testing pb_decode with packed int32 field") + TEST((s = S("\x0A\x00"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 0)) TEST((s = S("\x0A\x01\x01"), pb_decode(&s, IntegerArray_fields, &dest) && dest.data_count == 1 && dest.data[0] == 1)) TEST((s = S("\x0A\x0A\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A"), pb_decode(&s, IntegerArray_fields, &dest) diff --git a/tests/encode_unittests.c b/tests/encode_unittests.c index ee7f7a34..5578a0ab 100644 --- a/tests/encode_unittests.c +++ b/tests/encode_unittests.c @@ -2,6 +2,7 @@ #include #include "pb_encode.h" #include "unittests.h" +#include "unittestproto.pb.h" bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) { @@ -163,6 +164,27 @@ int main() TEST(WRITES(pb_enc_string(&s, NULL, &value), "\x00")) } + { + uint8_t buffer[10]; + pb_ostream_t s; + IntegerArray msg = {5, {1, 2, 3, 4, 5}}; + + COMMENT("Test pb_encode with int32 array") + + TEST(WRITES(pb_encode(&s, IntegerArray_fields, &msg), "\x0A\x05\x01\x02\x03\x04\x05")) + + msg.data_count = 0; + TEST(WRITES(pb_encode(&s, IntegerArray_fields, &msg), "")) + + msg.data_count = 10; + TEST(!pb_encode(&s, IntegerArray_fields, &msg)) + } + + { + + + } + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); -- cgit 1.2.3-korg From a3534170212675e0c7d3d89e23838e25f3664316 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 23 Aug 2011 18:50:09 +0000 Subject: More unittests git-svn-id: https://svn.kapsi.fi/jpa/nanopb@966 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/concepts.rst | 8 +++-- docs/lsr.css | 6 +++- docs/reference.rst | 12 +++++++ pb_encode.c | 2 +- tests/encode_unittests.c | 88 +++++++++++++++++++++++++++++++++++++++++++++++ tests/unittestproto.proto | 28 +++++++++++++++ 6 files changed, 140 insertions(+), 4 deletions(-) create mode 100644 tests/unittestproto.proto diff --git a/docs/concepts.rst b/docs/concepts.rst index fac9061f..e6076401 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -176,9 +176,13 @@ Encoding callbacks When encoding, the callback should write out complete fields, including the wire type and field number tag. It can write as many or as few fields as it likes. For example, if you want to write out an array as *repeated* field, you should do it all in a single call. -If the callback is used in a submessage, it will be called multiple times during a single call to `pb_encode`_. It must produce the same amount of data every time. If the callback is directly in the main message, it is called only once. +Usually you can use `pb_encode_tag_for_field`_ to encode the wire type and tag number of the field. However, if you want to encode a repeated field as a packed array, you must call `pb_encode_tag`_ instead to specify a wire type of *PB_WT_STRING*. + +If the callback is used in a submessage, it will be called multiple times during a single call to `pb_encode`_. In this case, it must produce the same amount of data every time. If the callback is directly in the main message, it is called only once. .. _`pb_encode`: reference.html#pb-encode +.. _`pb_encode_tag_for_field`: reference.html#pb-encode-tag-for-field +.. _`pb_encode_tag`: reference.html#pb-encode-tag This callback writes out a dynamically sized string:: @@ -207,7 +211,7 @@ This callback reads multiple integers and prints them:: bool read_ints(pb_istream_t *stream, const pb_field_t *field, void *arg) { - while (stream.bytes_left) + while (stream->bytes_left) { uint64_t value; if (!pb_decode_varint(stream, &value)) diff --git a/docs/lsr.css b/docs/lsr.css index 86053252..429bce51 100644 --- a/docs/lsr.css +++ b/docs/lsr.css @@ -191,7 +191,6 @@ table.docutils { text-align: left; border: 1px solid gray; border-collapse: collapse; - width: 100%; margin: 1.5em 0em; } @@ -211,6 +210,11 @@ th.field-name { table.docutils th { font-family: monospace; background-color: #f6f6f6; + vertical-align: middle; +} + +table.field-list { + border: none; } div.sidebar { diff --git a/docs/reference.rst b/docs/reference.rst index 2fc3d481..55ade5e3 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -37,6 +37,7 @@ PB_HTYPE_REQUIRED 0x00 Verify that field exists in decoded message. PB_HTYPE_OPTIONAL 0x10 Use separate *has_* boolean to specify whether the field is present. PB_HTYPE_ARRAY 0x20 A repeated field with preallocated array. + Separate *_count* for number of items. PB_HTYPE_CALLBACK 0x30 A field with dynamic storage size, data is actually a pointer to a structure containing a callback function. @@ -183,6 +184,17 @@ Same as `pb_encode_tag`_, except takes the parameters from a *pb_field_t* struct This function only considers the LTYPE of the field. You can use it from your field callbacks, because the source generator writes correct LTYPE also for callback type fields. +Wire type mapping is as follows: + +========================= ============ +LTYPEs Wire type +========================= ============ +VARINT, SVARINT PB_WT_VARINT +FIXED with data_size == 8 PB_WT_64BIT +STRING, BYTES, SUBMESSAGE PB_WT_STRING +FIXED with data_size == 4 PB_WT_32BIT +========================= ============ + pb_encode_string ---------------- Writes the length of a string as varint and then contents of the string. Used for writing fields with wire type PB_WT_STRING. :: diff --git a/pb_encode.c b/pb_encode.c index d333cfd5..e83e0685 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -81,7 +81,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie if (count == 0) return true; - if (PB_LTYPE(field->type) < PB_LTYPE_LAST_PACKABLE) + if (PB_LTYPE(field->type) <= PB_LTYPE_LAST_PACKABLE) { if (!pb_encode_tag(stream, PB_WT_STRING, field->tag)) return false; diff --git a/tests/encode_unittests.c b/tests/encode_unittests.c index 5578a0ab..b8664d9c 100644 --- a/tests/encode_unittests.c +++ b/tests/encode_unittests.c @@ -15,6 +15,24 @@ bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) return true; } +bool fieldcallback(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +{ + int value = 0x55; + if (!pb_encode_tag_for_field(stream, field)) + return false; + return pb_encode_varint(stream, value); +} + +bool crazyfieldcallback(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +{ + /* This callback writes different amount of data the second time. */ + uint32_t *state = (uint32_t*)arg; + *state <<= 8; + if (!pb_encode_tag_for_field(stream, field)) + return false; + return pb_encode_varint(stream, *state); +} + /* Check that expression x writes data y. * Y is a string, which may contain null bytes. Null terminator is ignored. */ @@ -80,6 +98,17 @@ int main() COMMENT("Test pb_encode_tag_for_field") TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x50")); + + field.type = PB_LTYPE_FIXED; + field.data_size = 8; + TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x51")); + + field.type = PB_LTYPE_STRING; + TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x52")); + + field.type = PB_LTYPE_FIXED; + field.data_size = 4; + TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x55")); } { @@ -181,8 +210,67 @@ int main() } { + uint8_t buffer[10]; + pb_ostream_t s; + FloatArray msg = {1, {99.0f}}; + + COMMENT("Test pb_encode with float array") + + TEST(WRITES(pb_encode(&s, FloatArray_fields, &msg), + "\x0A\x04\x00\x00\xc6\x42")) + + msg.data_count = 0; + TEST(WRITES(pb_encode(&s, FloatArray_fields, &msg), "")) + + msg.data_count = 3; + TEST(!pb_encode(&s, FloatArray_fields, &msg)) + } + + { + uint8_t buffer[10]; + pb_ostream_t s; + CallbackArray msg; + + msg.data.funcs.encode = &fieldcallback; + + COMMENT("Test pb_encode with callback field.") + TEST(WRITES(pb_encode(&s, CallbackArray_fields, &msg), "\x08\x55")) + } + + { + uint8_t buffer[10]; + pb_ostream_t s; + IntegerContainer msg = {{5, {1,2,3,4,5}}}; + + COMMENT("Test pb_encode with packed array in a submessage.") + TEST(WRITES(pb_encode(&s, IntegerContainer_fields, &msg), + "\x0A\x07\x0A\x05\x01\x02\x03\x04\x05")) + } + + { + uint8_t buffer[10]; + pb_ostream_t s; + CallbackContainer msg; + CallbackContainerContainer msg2; + uint32_t state = 1; + + msg.submsg.data.funcs.encode = &fieldcallback; + msg2.submsg.submsg.data.funcs.encode = &fieldcallback; + + COMMENT("Test pb_encode with callback field in a submessage.") + TEST(WRITES(pb_encode(&s, CallbackContainer_fields, &msg), "\x0A\x02\x08\x55")) + TEST(WRITES(pb_encode(&s, CallbackContainerContainer_fields, &msg2), + "\x0A\x04\x0A\x02\x08\x55")) + /* Misbehaving callback */ + msg.submsg.data.funcs.encode = &crazyfieldcallback; + msg.submsg.data.arg = &state; + msg2.submsg.submsg.data.funcs.encode = &crazyfieldcallback; + msg2.submsg.submsg.data.arg = &state; + TEST(!pb_encode(&s, CallbackContainer_fields, &msg)) + state = 1; + TEST(!pb_encode(&s, CallbackContainerContainer_fields, &msg2)) } if (status != 0) diff --git a/tests/unittestproto.proto b/tests/unittestproto.proto new file mode 100644 index 00000000..c8a39dd6 --- /dev/null +++ b/tests/unittestproto.proto @@ -0,0 +1,28 @@ +import 'nanopb.proto'; + +message IntegerArray { + repeated int32 data = 1 [(nanopb).max_count = 10]; +} + +message FloatArray { + repeated float data = 1 [(nanopb).max_count = 10]; +} + +message CallbackArray { + // We cheat a bit and use this message for testing other types, too. + // Nanopb does not care about the actual defined data type for callback + // fields. + repeated int32 data = 1; +} + +message IntegerContainer { + required IntegerArray submsg = 1; +} + +message CallbackContainer { + required CallbackArray submsg = 1; +} + +message CallbackContainerContainer { + required CallbackContainer submsg = 1; +} -- cgit 1.2.3-korg From 9cb63dd2667360e165739ab06a1aeb68d4007a24 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 24 Aug 2011 12:13:24 +0000 Subject: wrong file name git-svn-id: https://svn.kapsi.fi/jpa/nanopb@967 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- tests/encode_unittests.c | 2 +- tests/test_decode1.c | 2 +- tests/test_encode1.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/encode_unittests.c b/tests/encode_unittests.c index b8664d9c..926a2545 100644 --- a/tests/encode_unittests.c +++ b/tests/encode_unittests.c @@ -262,7 +262,7 @@ int main() TEST(WRITES(pb_encode(&s, CallbackContainerContainer_fields, &msg2), "\x0A\x04\x0A\x02\x08\x55")) - /* Misbehaving callback */ + /* Misbehaving callback: varying output between calls */ msg.submsg.data.funcs.encode = &crazyfieldcallback; msg.submsg.data.arg = &state; msg2.submsg.submsg.data.funcs.encode = &crazyfieldcallback; diff --git a/tests/test_decode1.c b/tests/test_decode1.c index b7698efc..def45a67 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -1,6 +1,6 @@ #include #include -#include "person.h" +#include "person.pb.h" bool print_person(pb_istream_t *stream) { diff --git a/tests/test_encode1.c b/tests/test_encode1.c index ac13df35..412a271e 100644 --- a/tests/test_encode1.c +++ b/tests/test_encode1.c @@ -1,6 +1,6 @@ #include #include -#include "person.h" +#include "person.pb.h" bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) { -- cgit 1.2.3-korg From 4c76c9c398213db1c82caf5d4a366035eecc1c93 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 24 Aug 2011 12:14:44 +0000 Subject: makefile fix git-svn-id: https://svn.kapsi.fi/jpa/nanopb@968 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- tests/Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 5035af8f..807da64a 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -8,6 +8,7 @@ all: $(TESTS) run_unittests breakpoints clean: rm -f $(TESTS) person.pb* *.o *.gcda *.gcno +%.o: %.c %.o: %.c $(DEPS) $(CC) $(CFLAGS) -c -o $@ $< @@ -44,4 +45,4 @@ run_unittests: decode_unittests encode_unittests test_encode1 test_decode1 "`./test_encode1 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] run_fuzztest: test_decode1 - bash -c 'I=1; while cat /dev/urandom | ./test_decode1 > /dev/null; do I=$$(($$I+1)); echo -en "\r$$I"; done' \ No newline at end of file + bash -c 'I=1; while cat /dev/urandom | ./test_decode1 > /dev/null; do I=$$(($$I+1)); echo -en "\r$$I"; done' -- cgit 1.2.3-korg From 646e3c4944ae8e49e92c2ff14765648b1acbe0c3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 24 Aug 2011 12:57:16 +0000 Subject: documentation git-svn-id: https://svn.kapsi.fi/jpa/nanopb@969 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- LICENSE | 2 +- docs/index.rst | 27 ++++++++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/LICENSE b/LICENSE index c7e973e6..ac41e52a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2011 Petteri Aimonen +Copyright (c) 2011 Petteri Aimonen This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. diff --git a/docs/index.rst b/docs/index.rst index 76580364..3279ba6f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,10 +21,10 @@ So a typical project might include these files: 1) Nanopb runtime library: - pb.h - - pb_decode.h and pb_decode.c - - pb_encode.h and pb_encode.c + - pb_decode.h and pb_decode.c (needed for decoding messages) + - pb_encode.h and pb_encode.c (needed for encoding messages) 2) Protocol description (you can have many): - - person.proto + - person.proto (just an example) - person.c (autogenerated, contains initializers for const arrays) - person.h (autogenerated, contains type declarations) @@ -59,12 +59,12 @@ For starters, consider this simple message:: required int32 value = 1; } -Save this in *example.proto* and compile it:: +Save this in *message.proto* and compile it:: user@host:~$ protoc -omessage.pb message.proto - user@host:~$ python ../generator/nanopb_generator.py message.pb + user@host:~$ python nanopb/generator/nanopb_generator.py message.pb -You should now have in *example.h*:: +You should now have in *message.pb.h*:: typedef struct { int32_t value; @@ -81,7 +81,20 @@ Now in your main program do this to encode a message:: After that, buffer will contain the encoded message. The number of bytes in the message is stored in *stream.bytes_written*. -You can feed the message to *protoc --decode=Example example.proto* to verify its validity. +You can feed the message to *protoc --decode=Example message.proto* to verify its validity. + +For complete examples of the simple cases, see *tests/test_decode1.c* and *tests/test_encode1.c*. For an example with network interface, see the *example* subdirectory. + +Compiler requirements +===================== +Nanopb should compile with most ansi-C compatible compilers. It however requires a few header files to be available: + +#) *string.h*, with these functions: *strlen*, *memcpy*, *memset* +#) *stdint.h*, for definitions of *int32_t* etc. +#) *stddef.h*, for definition of *size_t* +#) *stdbool.h*, for definition of *bool* + +If these header files do not come with your compiler, you should be able to find suitable replacements online. Mostly the requirements are very simple, just a few basic functions and typedefs. Debugging and testing ===================== -- cgit 1.2.3-korg From 9af96cd669f7f9189cdedcef110e7dbc1d526857 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 24 Aug 2011 13:52:08 +0000 Subject: Generator bugfixes git-svn-id: https://svn.kapsi.fi/jpa/nanopb@970 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- generator/nanopb_generator.py | 70 ++++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 24 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index f09be346..67c422d9 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -108,7 +108,8 @@ class Field: elif desc.type == FieldD.TYPE_ENUM: self.ltype = 'PB_LTYPE_VARINT' self.ctype = names_from_type_name(desc.type_name) - self.default = Names(self.ctype) + self.default + if self.default is not None: + self.default = self.ctype + self.default elif desc.type == FieldD.TYPE_STRING: self.ltype = 'PB_LTYPE_STRING' if self.max_size is None: @@ -218,7 +219,7 @@ class Field: result += '\n pb_membersize(%s, %s[0]),' % (self.struct_name, self.name) result += ('\n pb_membersize(%s, %s) / pb_membersize(%s, %s[0]),' % (self.struct_name, self.name, self.struct_name, self.name)) - elif self.ltype == 'PB_LTYPE_BYTES': + elif self.htype != 'PB_HTYPE_CALLBACK' and self.ltype == 'PB_LTYPE_BYTES': result += '\n pb_membersize(%s, bytes),' % self.ctype result += ' 0,' else: @@ -240,24 +241,10 @@ class Message: self.fields = [Field(self.name, f) for f in desc.field] self.ordered_fields = self.fields[:] self.ordered_fields.sort() - - def __cmp__(self, other): - '''Sort messages so that submessages are declared before the message - that uses them. - ''' - if self.refers_to(other.name): - return 1 - elif other.refers_to(self.name): - return -1 - else: - return 0 - - def refers_to(self, name): - '''Returns True if this message uses the specified type as field type.''' - for field in self.fields: - if str(field.ctype) == str(name): - return True - return False + + def get_dependencies(self): + '''Get list of type names that this structure refers to.''' + return [str(field.ctype) for field in self.fields] def __str__(self): result = 'typedef struct {\n' @@ -317,16 +304,52 @@ def parse_file(fdesc): enums = [] messages = [] + if fdesc.package: + base_name = Names(fdesc.package.split('.')) + else: + base_name = Names() + for enum in fdesc.enum_type: - enums.append(Enum(Names(), enum)) + enums.append(Enum(base_name, enum)) - for names, message in iterate_messages(fdesc): + for names, message in iterate_messages(fdesc, base_name): messages.append(Message(names, message)) for enum in message.enum_type: enums.append(Enum(names, enum)) return enums, messages +def toposort2(data): + '''Topological sort. + From http://code.activestate.com/recipes/577413-topological-sort/ + This function is under the MIT license. + ''' + for k, v in data.items(): + v.discard(k) # Ignore self dependencies + extra_items_in_deps = reduce(set.union, data.values()) - set(data.keys()) + data.update({item:set() for item in extra_items_in_deps}) + while True: + ordered = set(item for item,dep in data.items() if not dep) + if not ordered: + break + for item in sorted(ordered): + yield item + data = {item: (dep - ordered) for item,dep in data.items() + if item not in ordered} + assert not data, "A cyclic dependency exists amongst %r" % data + +def sort_dependencies(messages): + '''Sort a list of Messages based on dependencies.''' + dependencies = {} + message_by_name = {} + for message in messages: + dependencies[str(message.name)] = set(message.get_dependencies()) + message_by_name[str(message.name)] = message + + for msgname in toposort2(dependencies): + if msgname in message_by_name: + yield message_by_name[msgname] + def generate_header(headername, enums, messages): '''Generate content for a header file. Generates strings, which should be concatenated and stored to file. @@ -344,8 +367,7 @@ def generate_header(headername, enums, messages): yield str(enum) + '\n\n' yield '/* Struct definitions */\n' - messages.sort() - for msg in messages: + for msg in sort_dependencies(messages): yield msg.types() yield str(msg) + '\n\n' -- cgit 1.2.3-korg From fab52deda9f08eed9ba7ab24113400b31474cc48 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 24 Aug 2011 18:23:05 +0000 Subject: Make fuzz test runnable again :) git-svn-id: https://svn.kapsi.fi/jpa/nanopb@971 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- tests/test_decode1.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_decode1.c b/tests/test_decode1.c index def45a67..b46d0d5a 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -63,7 +63,10 @@ bool callback(pb_istream_t *stream, uint8_t *buf, size_t count) int main() { - pb_istream_t stream = {&callback, stdin, SIZE_MAX}; + /* Maximum size is specified to prevent infinite length messages from + * hanging this in the fuzz test. + */ + pb_istream_t stream = {&callback, stdin, 10000}; if (!print_person(&stream)) printf("Parsing failed.\n"); -- cgit 1.2.3-korg From 16040007143f30904813986778b1b3e1d687e40e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 24 Aug 2011 19:03:28 +0000 Subject: Cyclic messages not supported - thanks to Josh for pointing this out. git-svn-id: https://svn.kapsi.fi/jpa/nanopb@972 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 3279ba6f..1022a1b2 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -49,6 +49,7 @@ Features and limitations #) Fields in the generated structs are ordered by the tag number, instead of the natural ordering in .proto file. #) Unknown fields are not preserved when decoding and re-encoding a message. #) Numeric arrays are always encoded as packed, even if not marked as packed in .proto. This causes incompatibility with decoders that do not support packed format. +#) Cyclic references between messages are not supported. They could be supported in callback-mode if there was an option in the generator to set the mode. Getting started =============== -- cgit 1.2.3-korg From 35814517ad9da0edad3905269ea41facfb120f3c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 1 Sep 2011 08:30:48 +0000 Subject: Python 2.6 compatibility for the generator git-svn-id: https://svn.kapsi.fi/jpa/nanopb@973 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- generator/nanopb_generator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 67c422d9..22331f0d 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -327,15 +327,15 @@ def toposort2(data): for k, v in data.items(): v.discard(k) # Ignore self dependencies extra_items_in_deps = reduce(set.union, data.values()) - set(data.keys()) - data.update({item:set() for item in extra_items_in_deps}) + data.update(dict([(item, set()) for item in extra_items_in_deps])) while True: ordered = set(item for item,dep in data.items() if not dep) if not ordered: break for item in sorted(ordered): yield item - data = {item: (dep - ordered) for item,dep in data.items() - if item not in ordered} + data = dict([(item, (dep - ordered)) for item,dep in data.items() + if item not in ordered]) assert not data, "A cyclic dependency exists amongst %r" % data def sort_dependencies(messages): @@ -426,4 +426,4 @@ if __name__ == '__main__': for part in generate_source(headerbasename, enums, messages): source.write(part) - \ No newline at end of file + -- cgit 1.2.3-korg From d4abb63c052dc9d4c23ce72e498b4c7483f3b7cb Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 12 Sep 2011 18:53:33 +0000 Subject: Tests for callback fields git-svn-id: https://svn.kapsi.fi/jpa/nanopb@974 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/concepts.rst | 2 +- tests/Makefile | 9 +++++++-- tests/callbacks.proto | 16 ++++++++++++++++ tests/test_decode1.c | 4 ++++ tests/test_decode_callbacks.c | 44 +++++++++++++++++++++++++++++++++++++++++++ tests/test_encode1.c | 4 ++++ tests/test_encode_callbacks.c | 33 ++++++++++++++++++++++++++++++++ 7 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 tests/callbacks.proto create mode 100644 tests/test_decode_callbacks.c create mode 100644 tests/test_encode_callbacks.c diff --git a/docs/concepts.rst b/docs/concepts.rst index e6076401..c4e5476e 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -201,7 +201,7 @@ Decoding callbacks bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void *arg); -When decoding, the callback receives a length-limited substring that reads the contents of a single field. The field tag has already been read. +When decoding, the callback receives a length-limited substring that reads the contents of a single field. The field tag has already been read. For *string* and *bytes*, the length value has already been parsed, and is available at *stream->bytes_left*. The callback will be called multiple times for repeated fields. For packed fields, you can either read multiple values until the stream ends, or leave it to `pb_decode`_ to call your function over and over until all values have been read. diff --git a/tests/Makefile b/tests/Makefile index 807da64a..30bce647 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,6 +1,6 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage LDFLAGS=--coverage -DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h unittests.h unittestproto.pb.h +DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h TESTS=test_decode1 test_encode1 decode_unittests encode_unittests all: $(TESTS) run_unittests breakpoints @@ -19,6 +19,8 @@ pb_decode.o: ../pb_decode.c $(DEPS) test_decode1: test_decode1.o pb_decode.o person.pb.o test_encode1: test_encode1.o pb_encode.o person.pb.o +test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o +test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o encode_unittests: encode_unittests.o pb_encode.o unittestproto.pb.o @@ -35,7 +37,7 @@ coverage: run_unittests gcov pb_encode.gcda gcov pb_decode.gcda -run_unittests: decode_unittests encode_unittests test_encode1 test_decode1 +run_unittests: decode_unittests encode_unittests test_encode1 test_decode1 test_encode_callbacks test_decode_callbacks rm -f *.gcda ./decode_unittests > /dev/null @@ -43,6 +45,9 @@ run_unittests: decode_unittests encode_unittests test_encode1 test_decode1 [ "`./test_encode1 | ./test_decode1`" = \ "`./test_encode1 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] + + [ "`./test_encode_callbacks | ./test_decode_callbacks`" = \ + "`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ] run_fuzztest: test_decode1 bash -c 'I=1; while cat /dev/urandom | ./test_decode1 > /dev/null; do I=$$(($$I+1)); echo -en "\r$$I"; done' diff --git a/tests/callbacks.proto b/tests/callbacks.proto new file mode 100644 index 00000000..7bc79002 --- /dev/null +++ b/tests/callbacks.proto @@ -0,0 +1,16 @@ +/* Todo: write tests for the rest of these fields, currently only stringvalue + * is tested. + */ + +message SubMessage { + optional int32 int32value = 1; +} + +message TestMessage { + optional string stringvalue = 1; + optional int32 int32value = 2; + optional fixed32 fixed32value = 3; + optional fixed64 fixed64value = 4; + optional SubMessage submsg = 5; +} + diff --git a/tests/test_decode1.c b/tests/test_decode1.c index b46d0d5a..d0cc427a 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -1,3 +1,7 @@ +/* A very simple decoding test case, using person.proto. + * Produces output compatible with protoc --decode. + */ + #include #include #include "person.pb.h" diff --git a/tests/test_decode_callbacks.c b/tests/test_decode_callbacks.c new file mode 100644 index 00000000..1c8d43a5 --- /dev/null +++ b/tests/test_decode_callbacks.c @@ -0,0 +1,44 @@ +/* Decoding testcase for callback fields. + * Run e.g. ./test_encode_callbacks | ./test_decode_callbacks + */ + +#include +#include +#include "callbacks.pb.h" + +bool print_string(pb_istream_t *stream, const pb_field_t *field, void *arg) +{ + uint8_t buffer[1024]; + + /* We could read block-by-block to avoid the large buffer... */ + if (stream->bytes_left > sizeof(buffer)) + return false; + + if (!pb_read(stream, buffer, stream->bytes_left)) + return false; + + /* Print the string, in format comparable with protoc --decode. */ + printf("%s: \"%s\"\n", (char*)arg, buffer); + return true; +} + +int main() +{ + uint8_t buffer[1024]; + size_t length = fread(buffer, 1, 1024, stdin); + pb_istream_t stream = pb_istream_from_buffer(buffer, length); + + /* Note: empty initializer list initializes the struct with all-0. + * This is recommended so that unused callbacks are set to NULL instead + * of crashing at runtime. + */ + TestMessage testmessage = {}; + + testmessage.stringvalue.funcs.decode = &print_string; + testmessage.stringvalue.arg = "stringvalue"; + + if (!pb_decode(&stream, TestMessage_fields, &testmessage)) + return 1; + + return 0; +} \ No newline at end of file diff --git a/tests/test_encode1.c b/tests/test_encode1.c index 412a271e..df1ec4fc 100644 --- a/tests/test_encode1.c +++ b/tests/test_encode1.c @@ -1,3 +1,7 @@ +/* A very simple encoding test case using person.proto. + * Just puts constant data in the fields. + */ + #include #include #include "person.pb.h" diff --git a/tests/test_encode_callbacks.c b/tests/test_encode_callbacks.c new file mode 100644 index 00000000..da2ee28e --- /dev/null +++ b/tests/test_encode_callbacks.c @@ -0,0 +1,33 @@ +/* Encoding testcase for callback fields */ + +#include +#include +#include +#include "callbacks.pb.h" + +bool encode_string(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +{ + char *str = "Hello world!"; + + if (!pb_encode_tag_for_field(stream, field)) + return false; + + return pb_encode_string(stream, (uint8_t*)str, strlen(str)); +} + +int main() +{ + uint8_t buffer[1024]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, 1024); + TestMessage testmessage = {}; + + testmessage.stringvalue.funcs.encode = &encode_string; + + if (!pb_encode(&stream, TestMessage_fields, &testmessage)) + return 1; + + if (fwrite(buffer, stream.bytes_written, 1, stdout) != 1) + return 2; + + return 0; +} -- cgit 1.2.3-korg From fcfc99f766f278564c062922a2857c21ac9af041 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 13 Sep 2011 16:14:08 +0000 Subject: Fixed a bunch of bugs related to callback fields. Most importantly, callback fields in submessages were being overwritten with garbage, causing segfaults. Additionally, converted PB_LTYPE_FIXED to PB_LTYPE_FIXED32 and PB_LTYPE_FIXED64. This makes the interface a bit easier to use, and in addition runs faster. git-svn-id: https://svn.kapsi.fi/jpa/nanopb@975 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/reference.rst | 54 +++++++++++++++++++-------- generator/nanopb_generator.py | 8 ++-- pb.h | 13 ++++--- pb_decode.c | 85 ++++++++++++++++++++++++++++++++----------- pb_decode.h | 3 +- pb_encode.c | 44 +++++++++++++++------- pb_encode.h | 3 +- tests/Makefile | 4 +- tests/callbacks.proto | 15 ++++---- tests/decode_unittests.c | 22 +++++------ tests/encode_unittests.c | 24 +++++------- tests/test_decode_callbacks.c | 57 ++++++++++++++++++++++++++--- tests/test_encode_callbacks.c | 35 ++++++++++++++++++ 13 files changed, 263 insertions(+), 104 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 55ade5e3..71b934cf 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -62,7 +62,7 @@ Describes a single structure field with memory position in relation to others. T :type: LTYPE and HTYPE of the field. :data_offset: Offset of field data, relative to the end of the previous field. :size_offset: Offset of *bool* flag for optional fields or *size_t* count for arrays, relative to field data. -:data_size: Size of a single data entry, in bytes. For PB_LTYPE_BYTES, the size of the byte array inside the containing structure. +:data_size: Size of a single data entry, in bytes. For PB_LTYPE_BYTES, the size of the byte array inside the containing structure. For PB_HTYPE_CALLBACK, size of the C data type if known. :array_size: Maximum number of entries in an array, if it is an array type. :ptr: Pointer to default value for optional fields, or to submessage description for PB_LTYPE_SUBMESSAGE. @@ -190,9 +190,9 @@ Wire type mapping is as follows: LTYPEs Wire type ========================= ============ VARINT, SVARINT PB_WT_VARINT -FIXED with data_size == 8 PB_WT_64BIT +FIXED64 PB_WT_64BIT STRING, BYTES, SUBMESSAGE PB_WT_STRING -FIXED with data_size == 4 PB_WT_32BIT +FIXED32 PB_WT_32BIT ========================= ============ pb_encode_string @@ -214,7 +214,7 @@ Writes the length of a string as varint and then contents of the string. Used fo Each field encoder only encodes the contents of the field. The tag must be encoded separately with `pb_encode_tag_for_field`_. - You can use the field encoders from your callbacks. + You can use the field encoders from your callbacks. Just be aware that the pb_field_t passed to the callback is not directly compatible with most of the encoders. Instead, you must create a new pb_field_t structure and set the data_size according to the data type you pass to *src. pb_enc_varint ------------- @@ -237,15 +237,26 @@ Field encoder for PB_LTYPE_SVARINT. Similar to `pb_enc_varint`_, except first zi The number is considered negative if the high-order bit of the value is set. On big endian computers, it is the highest bit of *\*src*. On little endian computers, it is the highest bit of *\*(src + field->data_size - 1)*. -pb_enc_fixed ------------- -Field encoder for PB_LTYPE_FIXED. Writes the data in little endian order. On big endian computers, reverses the order of bytes. :: +pb_enc_fixed32 +-------------- +Field encoder for PB_LTYPE_FIXED32. Writes the data in little endian order. On big endian computers, reverses the order of bytes. :: - bool pb_enc_fixed(pb_ostream_t *stream, const pb_field_t *field, const void *src); + bool pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src); -(parameters are the same as for `pb_enc_varint`_) +:stream: Output stream to write to. +:field: Not used. +:src: Pointer to start of the field data. +:returns: True on success, false on IO error. + +pb_enc_fixed64 +-------------- +Field encoder for PB_LTYPE_FIXED64. Writes the data in little endian order. On big endian computers, reverses the order of bytes. :: -The same function is used for both integers, floats and doubles. This break encoding of double values on architectures where they are mixed endian (primarily some arm processors with hardware FPU). + bool pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src); + +(parameters are the same as for `pb_enc_fixed32`_) + +The same function is used for both integers and doubles. This breaks encoding of double values on architectures where they are mixed endian (primarily some arm processors with hardware FPU). pb_enc_bytes ------------ @@ -365,7 +376,7 @@ Because of memory concerns, the detection of missing required fields is not perf Each field decoder reads and decodes a single value. For arrays, the decoder is called repeatedly. - You can use the decoders from your callbacks. + You can use the decoders from your callbacks. Just be aware that the pb_field_t passed to the callback is not directly compatible with most of the field decoders. Instead, you must create a new pb_field_t structure and set the data_size according to the data type you pass to *dest. pb_dec_varint ------------- @@ -388,18 +399,29 @@ Field decoder for PB_LTYPE_SVARINT. Similar to `pb_dec_varint`_, except that it (parameters are the same as `pb_dec_varint`_) -pb_dec_fixed ------------- -Field decoder for PB_LTYPE_FIXED. :: +pb_dec_fixed32 +-------------- +Field decoder for PB_LTYPE_FIXED32. :: bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest); -(parameters are the same as `pb_dec_varint`_) +:stream: Input stream to read from. 1-10 bytes will be read. +:field: Not used. +:dest: Pointer to destination integer. Must have size of *field->data_size* bytes. +:returns: True on success, false on IO errors or if `pb_decode_varint`_ fails. -This function reads *field->data_size* bytes from the input stream. +This function reads 4 bytes from the input stream. On big endian architectures, it then reverses the order of the bytes. Finally, it writes the bytes to *dest*. +pb_dec_fixed64 +-------------- +Field decoder for PB_LTYPE_FIXED64. :: + + bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest); + +Same as `pb_dec_fixed32`_, except this reads 8 bytes. + pb_dec_bytes ------------ Field decoder for PB_LTYPE_BYTES. Reads a length-prefixed block of bytes. :: diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 22331f0d..4d5aaaca 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -8,10 +8,10 @@ import os.path FieldD = descriptor.FieldDescriptorProto datatypes = { FieldD.TYPE_BOOL: ('bool', 'PB_LTYPE_VARINT'), - FieldD.TYPE_DOUBLE: ('double', 'PB_LTYPE_FIXED'), - FieldD.TYPE_FIXED32: ('uint32_t', 'PB_LTYPE_FIXED'), - FieldD.TYPE_FIXED64: ('uint64_t', 'PB_LTYPE_FIXED'), - FieldD.TYPE_FLOAT: ('float', 'PB_LTYPE_FIXED'), + FieldD.TYPE_DOUBLE: ('double', 'PB_LTYPE_FIXED64'), + FieldD.TYPE_FIXED32: ('uint32_t', 'PB_LTYPE_FIXED32'), + FieldD.TYPE_FIXED64: ('uint64_t', 'PB_LTYPE_FIXED64'), + FieldD.TYPE_FLOAT: ('float', 'PB_LTYPE_FIXED32'), FieldD.TYPE_INT32: ('int32_t', 'PB_LTYPE_VARINT'), FieldD.TYPE_INT64: ('int64_t', 'PB_LTYPE_VARINT'), FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED'), diff --git a/pb.h b/pb.h index 2c6a2520..cd9c75d0 100644 --- a/pb.h +++ b/pb.h @@ -35,25 +35,26 @@ typedef enum { /* Numeric types */ PB_LTYPE_VARINT = 0x00, /* int32, uint32, int64, uint64, bool, enum */ PB_LTYPE_SVARINT = 0x01, /* sint32, sint64 */ - PB_LTYPE_FIXED = 0x02, /* fixed32, sfixed32, fixed64, sfixed64, float, double */ + PB_LTYPE_FIXED32 = 0x02, /* fixed32, sfixed32, float */ + PB_LTYPE_FIXED64 = 0x03, /* fixed64, sfixed64, double */ /* Marker for last packable field type. */ - PB_LTYPE_LAST_PACKABLE = 0x02, + PB_LTYPE_LAST_PACKABLE = 0x03, /* Byte array with pre-allocated buffer. * data_size is the length of the allocated PB_BYTES_ARRAY structure. */ - PB_LTYPE_BYTES = 0x03, + PB_LTYPE_BYTES = 0x04, /* String with pre-allocated buffer. * data_size is the maximum length. */ - PB_LTYPE_STRING = 0x04, + PB_LTYPE_STRING = 0x05, /* Submessage * submsg_fields is pointer to field descriptions */ - PB_LTYPE_SUBMESSAGE = 0x05, + PB_LTYPE_SUBMESSAGE = 0x06, /* Number of declared LTYPES */ - PB_LTYPES_COUNT = 6, + PB_LTYPES_COUNT = 7, /****************** * Modifier flags * diff --git a/pb_decode.c b/pb_decode.c index 6c6b1d9e..453b1cc9 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -23,7 +23,8 @@ typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { &pb_dec_varint, &pb_dec_svarint, - &pb_dec_fixed, + &pb_dec_fixed32, + &pb_dec_fixed64, &pb_dec_bytes, &pb_dec_string, @@ -163,7 +164,10 @@ static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire } } -/* Decode string length from stream and return a substream with limited length */ +/* Decode string length from stream and return a substream with limited length. + * Before disposing the substream, remember to copy the substream->state back + * to stream->state. + */ static bool checkreturn make_string_substream(pb_istream_t *stream, pb_istream_t *substream) { uint32_t size; @@ -302,6 +306,8 @@ static bool checkreturn decode_field(pb_istream_t *stream, int wire_type, pb_fie if (!pCallback->funcs.decode(&substream, iter->current, pCallback->arg)) return false; } + + stream->state = substream.state; return true; } else @@ -327,16 +333,10 @@ static bool checkreturn decode_field(pb_istream_t *stream, int wire_type, pb_fie } } -/********************* - * Decode all fields * - *********************/ - -bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +/* Initialize message fields to default values, recursively */ +static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct) { - uint32_t fields_seen = 0; /* Used to check for required fields */ pb_field_iterator_t iter; - int i; - pb_field_init(&iter, fields, dest_struct); /* Initialize size/has fields and apply default values */ @@ -345,25 +345,50 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void if (iter.current->tag == 0) continue; + /* Initialize the size field for optional/repeated fields to 0. */ if (PB_HTYPE(iter.current->type) == PB_HTYPE_OPTIONAL) { *(bool*)iter.pSize = false; - - /* Initialize to default value */ - if (iter.current->ptr != NULL) - memcpy(iter.pData, iter.current->ptr, iter.current->data_size); - else - memset(iter.pData, 0, iter.current->data_size); } else if (PB_HTYPE(iter.current->type) == PB_HTYPE_ARRAY) { *(size_t*)iter.pSize = 0; + continue; /* Array is empty, no need to initialize contents */ + } + + /* Initialize field contents to default value */ + if (PB_HTYPE(iter.current->type) == PB_HTYPE_CALLBACK) + { + continue; /* Don't overwrite callback */ + } + else if (PB_LTYPE(iter.current->type) == PB_LTYPE_SUBMESSAGE) + { + pb_message_set_to_defaults(iter.current->ptr, iter.pData); + } + else if (iter.current->ptr != NULL) + { + memcpy(iter.pData, iter.current->ptr, iter.current->data_size); } - else if (PB_HTYPE(iter.current->type) == PB_HTYPE_REQUIRED) + else { memset(iter.pData, 0, iter.current->data_size); } } while (pb_field_next(&iter)); +} + +/********************* + * Decode all fields * + *********************/ + +bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + uint32_t fields_seen = 0; /* Used to check for required fields */ + pb_field_iterator_t iter; + int i; + + pb_message_set_to_defaults(fields, dest_struct); + + pb_field_init(&iter, fields, dest_struct); while (stream->bytes_left) { @@ -443,17 +468,30 @@ bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, v return status; } -bool checkreturn pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest) +bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ +#ifdef __BIG_ENDIAN__ + uint8_t bytes[4] = {0}; + bool status = pb_read(stream, bytes, 4); + uint8_t bebytes[4] = {bytes[3], bytes[2], bytes[1], bytes[0]}; + memcpy(dest, bebytes, 4); + return status; +#else + return pb_read(stream, (uint8_t*)dest, 4); +#endif +} + +bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest) { #ifdef __BIG_ENDIAN__ uint8_t bytes[8] = {0}; - bool status = pb_read(stream, bytes, field->data_size); + bool status = pb_read(stream, bytes, 8); uint8_t bebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], bytes[3], bytes[2], bytes[1], bytes[0]}; - endian_copy(dest, lebytes, field->data_size, 8); + memcpy(dest, bebytes, 8); return status; #else - return pb_read(stream, (uint8_t*)dest, field->data_size); + return pb_read(stream, (uint8_t*)dest, 8); #endif } @@ -489,6 +527,7 @@ bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, vo bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) { + bool status; pb_istream_t substream; if (!make_string_substream(stream, &substream)) @@ -497,5 +536,7 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field if (field->ptr == NULL) return false; - return pb_decode(&substream, (pb_field_t*)field->ptr, dest); + status = pb_decode(&substream, (pb_field_t*)field->ptr, dest); + stream->state = substream.state; + return status; } diff --git a/pb_decode.h b/pb_decode.h index 2d4e5863..f12b1900 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -61,7 +61,8 @@ bool pb_skip_string(pb_istream_t *stream); bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); diff --git a/pb_encode.c b/pb_encode.c index e83e0685..58e03a68 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -23,7 +23,8 @@ typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, cons static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { &pb_enc_varint, &pb_enc_svarint, - &pb_enc_fixed, + &pb_enc_fixed32, + &pb_enc_fixed64, &pb_enc_bytes, &pb_enc_string, @@ -87,9 +88,13 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie return false; /* Determine the total size of packed array. */ - if (PB_LTYPE(field->type) == PB_LTYPE_FIXED) + if (PB_LTYPE(field->type) == PB_LTYPE_FIXED32) { - size = field->data_size * count; + size = 4 * count; + } + else if (PB_LTYPE(field->type) == PB_LTYPE_FIXED64) + { + size = 8 * count; } else { @@ -232,13 +237,12 @@ bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t wiretype = PB_WT_VARINT; break; - case PB_LTYPE_FIXED: - if (field->data_size == 4) - wiretype = PB_WT_32BIT; - else if (field->data_size == 8) - wiretype = PB_WT_64BIT; - else - return false; + case PB_LTYPE_FIXED32: + wiretype = PB_WT_32BIT; + break; + + case PB_LTYPE_FIXED64: + wiretype = PB_WT_64BIT; break; case PB_LTYPE_BYTES: @@ -304,16 +308,28 @@ bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, c return pb_encode_varint(stream, zigzagged); } -bool checkreturn pb_enc_fixed(pb_ostream_t *stream, const pb_field_t *field, const void *src) +bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src) { #ifdef __BIG_ENDIAN__ uint8_t bytes[8] = {0}; - endian_copy(bytes, src, sizeof(bytes), field->data_size); + memcpy(bytes, src, 8); uint8_t lebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], bytes[3], bytes[2], bytes[1], bytes[0]}; - return pb_write(stream, lebytes, field->data_size); + return pb_write(stream, lebytes, 8); + #else + return pb_write(stream, (uint8_t*)src, 8); + #endif +} + +bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src) +{ + #ifdef __BIG_ENDIAN__ + uint8_t bytes[4] = {0}; + memcpy(bytes, src, 4); + uint8_t lebytes[4] = {bytes[3], bytes[2], bytes[1], bytes[0]}; + return pb_write(stream, lebytes, 4); #else - return pb_write(stream, (uint8_t*)src, field->data_size); + return pb_write(stream, (uint8_t*)src, 4); #endif } diff --git a/pb_encode.h b/pb_encode.h index b341602d..864a48b7 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -62,7 +62,8 @@ bool pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size); bool pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src); bool pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); -bool pb_enc_fixed(pb_ostream_t *stream, const pb_field_t *field, const void *src); +bool pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src); +bool pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src); bool pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src); bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src); diff --git a/tests/Makefile b/tests/Makefile index 30bce647..2d5ee506 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -3,7 +3,7 @@ LDFLAGS=--coverage DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h TESTS=test_decode1 test_encode1 decode_unittests encode_unittests -all: $(TESTS) run_unittests breakpoints +all: breakpoints $(TESTS) run_unittests clean: rm -f $(TESTS) person.pb* *.o *.gcda *.gcno @@ -27,7 +27,7 @@ encode_unittests: encode_unittests.o pb_encode.o unittestproto.pb.o %.pb: %.proto protoc -I. -I../generator -I/usr/include -o$@ $< -%.pb.c %.pb.h: %.pb +%.pb.c %.pb.h: %.pb ../generator/nanopb_generator.py python ../generator/nanopb_generator.py $< breakpoints: ../*.c *.c diff --git a/tests/callbacks.proto b/tests/callbacks.proto index 7bc79002..8beeaabf 100644 --- a/tests/callbacks.proto +++ b/tests/callbacks.proto @@ -1,16 +1,15 @@ -/* Todo: write tests for the rest of these fields, currently only stringvalue - * is tested. - */ - message SubMessage { - optional int32 int32value = 1; + optional string stringvalue = 1; + repeated int32 int32value = 2; + repeated fixed32 fixed32value = 3; + repeated fixed64 fixed64value = 4; } message TestMessage { optional string stringvalue = 1; - optional int32 int32value = 2; - optional fixed32 fixed32value = 3; - optional fixed64 fixed64value = 4; + repeated int32 int32value = 2; + repeated fixed32 fixed32value = 3; + repeated fixed64 fixed64value = 4; optional SubMessage submsg = 5; } diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c index 006ad900..ab12ac33 100644 --- a/tests/decode_unittests.c +++ b/tests/decode_unittests.c @@ -143,25 +143,25 @@ int main() { pb_istream_t s; - pb_field_t f = {1, PB_LTYPE_FIXED, 0, 0, 4, 0, 0}; + pb_field_t f = {1, PB_LTYPE_FIXED32, 0, 0, 4, 0, 0}; float d; - COMMENT("Test pb_dec_fixed using float (failures here may be caused by imperfect rounding)") - TEST((s = S("\x00\x00\x00\x00"), pb_dec_fixed(&s, &f, &d) && d == 0.0f)) - TEST((s = S("\x00\x00\xc6\x42"), pb_dec_fixed(&s, &f, &d) && d == 99.0f)) - TEST((s = S("\x4e\x61\x3c\xcb"), pb_dec_fixed(&s, &f, &d) && d == -12345678.0f)) - TEST((s = S("\x00"), !pb_dec_fixed(&s, &f, &d) && d == -12345678.0f)) + COMMENT("Test pb_dec_fixed32 using float (failures here may be caused by imperfect rounding)") + TEST((s = S("\x00\x00\x00\x00"), pb_dec_fixed32(&s, &f, &d) && d == 0.0f)) + TEST((s = S("\x00\x00\xc6\x42"), pb_dec_fixed32(&s, &f, &d) && d == 99.0f)) + TEST((s = S("\x4e\x61\x3c\xcb"), pb_dec_fixed32(&s, &f, &d) && d == -12345678.0f)) + TEST((s = S("\x00"), !pb_dec_fixed32(&s, &f, &d) && d == -12345678.0f)) } { pb_istream_t s; - pb_field_t f = {1, PB_LTYPE_FIXED, 0, 0, 8, 0, 0}; + pb_field_t f = {1, PB_LTYPE_FIXED64, 0, 0, 8, 0, 0}; double d; - COMMENT("Test pb_dec_fixed using double (failures here may be caused by imperfect rounding)") - TEST((s = S("\x00\x00\x00\x00\x00\x00\x00\x00"), pb_dec_fixed(&s, &f, &d) && d == 0.0)) - TEST((s = S("\x00\x00\x00\x00\x00\xc0\x58\x40"), pb_dec_fixed(&s, &f, &d) && d == 99.0)) - TEST((s = S("\x00\x00\x00\xc0\x29\x8c\x67\xc1"), pb_dec_fixed(&s, &f, &d) && d == -12345678.0f)) + COMMENT("Test pb_dec_fixed64 using double (failures here may be caused by imperfect rounding)") + TEST((s = S("\x00\x00\x00\x00\x00\x00\x00\x00"), pb_dec_fixed64(&s, &f, &d) && d == 0.0)) + TEST((s = S("\x00\x00\x00\x00\x00\xc0\x58\x40"), pb_dec_fixed64(&s, &f, &d) && d == 99.0)) + TEST((s = S("\x00\x00\x00\xc0\x29\x8c\x67\xc1"), pb_dec_fixed64(&s, &f, &d) && d == -12345678.0f)) } { diff --git a/tests/encode_unittests.c b/tests/encode_unittests.c index 926a2545..166e6e83 100644 --- a/tests/encode_unittests.c +++ b/tests/encode_unittests.c @@ -99,15 +99,13 @@ int main() COMMENT("Test pb_encode_tag_for_field") TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x50")); - field.type = PB_LTYPE_FIXED; - field.data_size = 8; + field.type = PB_LTYPE_FIXED64; TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x51")); field.type = PB_LTYPE_STRING; TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x52")); - field.type = PB_LTYPE_FIXED; - field.data_size = 4; + field.type = PB_LTYPE_FIXED32; TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x55")); } @@ -149,26 +147,24 @@ int main() { uint8_t buffer[30]; pb_ostream_t s; - pb_field_t field = {1, PB_LTYPE_FIXED, 0, 0, sizeof(float)}; float fvalue; double dvalue; - COMMENT("Test pb_enc_fixed using float") + COMMENT("Test pb_enc_fixed32 using float") fvalue = 0.0f; - TEST(WRITES(pb_enc_fixed(&s, &field, &fvalue), "\x00\x00\x00\x00")) + TEST(WRITES(pb_enc_fixed32(&s, NULL, &fvalue), "\x00\x00\x00\x00")) fvalue = 99.0f; - TEST(WRITES(pb_enc_fixed(&s, &field, &fvalue), "\x00\x00\xc6\x42")) + TEST(WRITES(pb_enc_fixed32(&s, NULL, &fvalue), "\x00\x00\xc6\x42")) fvalue = -12345678.0f; - TEST(WRITES(pb_enc_fixed(&s, &field, &fvalue), "\x4e\x61\x3c\xcb")) + TEST(WRITES(pb_enc_fixed32(&s, NULL, &fvalue), "\x4e\x61\x3c\xcb")) - COMMENT("Test pb_enc_fixed using double") - field.data_size = sizeof(double); + COMMENT("Test pb_enc_fixed64 using double") dvalue = 0.0; - TEST(WRITES(pb_enc_fixed(&s, &field, &dvalue), "\x00\x00\x00\x00\x00\x00\x00\x00")) + TEST(WRITES(pb_enc_fixed64(&s, NULL, &dvalue), "\x00\x00\x00\x00\x00\x00\x00\x00")) dvalue = 99.0; - TEST(WRITES(pb_enc_fixed(&s, &field, &dvalue), "\x00\x00\x00\x00\x00\xc0\x58\x40")) + TEST(WRITES(pb_enc_fixed64(&s, NULL, &dvalue), "\x00\x00\x00\x00\x00\xc0\x58\x40")) dvalue = -12345678.0; - TEST(WRITES(pb_enc_fixed(&s, &field, &dvalue), "\x00\x00\x00\xc0\x29\x8c\x67\xc1")) + TEST(WRITES(pb_enc_fixed64(&s, NULL, &dvalue), "\x00\x00\x00\xc0\x29\x8c\x67\xc1")) } { diff --git a/tests/test_decode_callbacks.c b/tests/test_decode_callbacks.c index 1c8d43a5..6c0072de 100644 --- a/tests/test_decode_callbacks.c +++ b/tests/test_decode_callbacks.c @@ -8,17 +8,49 @@ bool print_string(pb_istream_t *stream, const pb_field_t *field, void *arg) { - uint8_t buffer[1024]; + uint8_t buffer[1024] = {0}; /* We could read block-by-block to avoid the large buffer... */ - if (stream->bytes_left > sizeof(buffer)) + if (stream->bytes_left > sizeof(buffer) - 1) return false; if (!pb_read(stream, buffer, stream->bytes_left)) return false; - /* Print the string, in format comparable with protoc --decode. */ - printf("%s: \"%s\"\n", (char*)arg, buffer); + /* Print the string, in format comparable with protoc --decode. + * Format comes from the arg defined in main(). + */ + printf((char*)arg, buffer); + return true; +} + +bool print_int32(pb_istream_t *stream, const pb_field_t *field, void *arg) +{ + uint64_t value; + if (!pb_decode_varint(stream, &value)) + return false; + + printf((char*)arg, (int32_t)value); + return true; +} + +bool print_fixed32(pb_istream_t *stream, const pb_field_t *field, void *arg) +{ + uint32_t value; + if (!pb_dec_fixed32(stream, NULL, &value)) + return false; + + printf((char*)arg, value); + return true; +} + +bool print_fixed64(pb_istream_t *stream, const pb_field_t *field, void *arg) +{ + uint64_t value; + if (!pb_dec_fixed64(stream, NULL, &value)) + return false; + + printf((char*)arg, value); return true; } @@ -34,8 +66,23 @@ int main() */ TestMessage testmessage = {}; + testmessage.submsg.stringvalue.funcs.decode = &print_string; + testmessage.submsg.stringvalue.arg = "submsg {\n stringvalue: \"%s\"\n"; + testmessage.submsg.int32value.funcs.decode = &print_int32; + testmessage.submsg.int32value.arg = " int32value: %d\n"; + testmessage.submsg.fixed32value.funcs.decode = &print_fixed32; + testmessage.submsg.fixed32value.arg = " fixed32value: %d\n"; + testmessage.submsg.fixed64value.funcs.decode = &print_fixed64; + testmessage.submsg.fixed64value.arg = " fixed64value: %lld\n}\n"; + testmessage.stringvalue.funcs.decode = &print_string; - testmessage.stringvalue.arg = "stringvalue"; + testmessage.stringvalue.arg = "stringvalue: \"%s\"\n"; + testmessage.int32value.funcs.decode = &print_int32; + testmessage.int32value.arg = "int32value: %d\n"; + testmessage.fixed32value.funcs.decode = &print_fixed32; + testmessage.fixed32value.arg = "fixed32value: %d\n"; + testmessage.fixed64value.funcs.decode = &print_fixed64; + testmessage.fixed64value.arg = "fixed64value: %lld\n"; if (!pb_decode(&stream, TestMessage_fields, &testmessage)) return 1; diff --git a/tests/test_encode_callbacks.c b/tests/test_encode_callbacks.c index da2ee28e..f0a046de 100644 --- a/tests/test_encode_callbacks.c +++ b/tests/test_encode_callbacks.c @@ -15,6 +15,32 @@ bool encode_string(pb_ostream_t *stream, const pb_field_t *field, const void *ar return pb_encode_string(stream, (uint8_t*)str, strlen(str)); } +bool encode_int32(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +{ + if (!pb_encode_tag_for_field(stream, field)) + return false; + + return pb_encode_varint(stream, 42); +} + +bool encode_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +{ + if (!pb_encode_tag_for_field(stream, field)) + return false; + + uint32_t value = 42; + return pb_enc_fixed32(stream, field, &value); +} + +bool encode_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +{ + if (!pb_encode_tag_for_field(stream, field)) + return false; + + uint64_t value = 42; + return pb_enc_fixed64(stream, field, &value); +} + int main() { uint8_t buffer[1024]; @@ -22,6 +48,15 @@ int main() TestMessage testmessage = {}; testmessage.stringvalue.funcs.encode = &encode_string; + testmessage.int32value.funcs.encode = &encode_int32; + testmessage.fixed32value.funcs.encode = &encode_fixed32; + testmessage.fixed64value.funcs.encode = &encode_fixed64; + + testmessage.has_submsg = true; + testmessage.submsg.stringvalue.funcs.encode = &encode_string; + testmessage.submsg.int32value.funcs.encode = &encode_int32; + testmessage.submsg.fixed32value.funcs.encode = &encode_fixed32; + testmessage.submsg.fixed64value.funcs.encode = &encode_fixed64; if (!pb_encode(&stream, TestMessage_fields, &testmessage)) return 1; -- cgit 1.2.3-korg From b2a696795e0f79e6913b79c6a8e840ce4cd44d25 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 14 Sep 2011 10:36:04 +0000 Subject: Added README git-svn-id: https://svn.kapsi.fi/jpa/nanopb@976 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- README | 9 +++++++++ docs/reference.rst | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 README diff --git a/README b/README new file mode 100644 index 00000000..0fc79c47 --- /dev/null +++ b/README @@ -0,0 +1,9 @@ +Nanopb is a small code-size Protocol Buffers implementation. + +Homepage: http://kapsi.fi/~jpa/nanopb/ + +To compile the library, you'll need these libraries: +protobuf-compiler python-protobuf libprotobuf-dev + +To run the tests, run make under the tests folder. +If it completes without error, everything is fine. diff --git a/docs/reference.rst b/docs/reference.rst index 71b934cf..d67499e1 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -214,7 +214,7 @@ Writes the length of a string as varint and then contents of the string. Used fo Each field encoder only encodes the contents of the field. The tag must be encoded separately with `pb_encode_tag_for_field`_. - You can use the field encoders from your callbacks. Just be aware that the pb_field_t passed to the callback is not directly compatible with most of the encoders. Instead, you must create a new pb_field_t structure and set the data_size according to the data type you pass to *src. + You can use the field encoders from your callbacks. Just be aware that the pb_field_t passed to the callback is not directly compatible with most of the encoders. Instead, you must create a new pb_field_t structure and set the data_size according to the data type you pass to *src*. pb_enc_varint ------------- @@ -376,7 +376,7 @@ Because of memory concerns, the detection of missing required fields is not perf Each field decoder reads and decodes a single value. For arrays, the decoder is called repeatedly. - You can use the decoders from your callbacks. Just be aware that the pb_field_t passed to the callback is not directly compatible with most of the field decoders. Instead, you must create a new pb_field_t structure and set the data_size according to the data type you pass to *dest. + You can use the decoders from your callbacks. Just be aware that the pb_field_t passed to the callback is not directly compatible with most of the field decoders. Instead, you must create a new pb_field_t structure and set the data_size according to the data type you pass to *dest*. pb_dec_varint ------------- -- cgit 1.2.3-korg From d96e9575d4eb188bf84134c25cfa515b2949dbd7 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 18 Sep 2011 16:04:45 +0000 Subject: Added comments to the simplest decode/encode examples. git-svn-id: https://svn.kapsi.fi/jpa/nanopb@979 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- tests/test_decode1.c | 17 +++++++++++++++-- tests/test_encode1.c | 14 ++++++++++---- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/tests/test_decode1.c b/tests/test_decode1.c index d0cc427a..3f02745e 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -1,11 +1,17 @@ /* A very simple decoding test case, using person.proto. * Produces output compatible with protoc --decode. + * Reads the encoded data from stdin and prints the values + * to stdout as text. + * + * Run e.g. ./test_encode1 | ./test_decode1 */ #include #include #include "person.pb.h" +/* This function is called once from main(), it handles + the decoding and printing. */ bool print_person(pb_istream_t *stream) { int i; @@ -14,6 +20,8 @@ bool print_person(pb_istream_t *stream) if (!pb_decode(stream, Person_fields, &person)) return false; + /* Now the decoding is done, rest is just to print stuff out. */ + printf("name: \"%s\"\n", person.name); printf("id: %d\n", person.id); @@ -46,6 +54,7 @@ bool print_person(pb_istream_t *stream) return true; } +/* This binds the pb_istream_t to stdin */ bool callback(pb_istream_t *stream, uint8_t *buf, size_t count) { FILE *file = (FILE*)stream->state; @@ -53,6 +62,7 @@ bool callback(pb_istream_t *stream, uint8_t *buf, size_t count) if (buf == NULL) { + /* Skipping data */ while (count-- && fgetc(file) != EOF); return count == 0; } @@ -72,7 +82,10 @@ int main() */ pb_istream_t stream = {&callback, stdin, 10000}; if (!print_person(&stream)) + { printf("Parsing failed.\n"); - - return 0; + return 1; + } else { + return 0; + } } diff --git a/tests/test_encode1.c b/tests/test_encode1.c index df1ec4fc..f46e60a6 100644 --- a/tests/test_encode1.c +++ b/tests/test_encode1.c @@ -1,11 +1,13 @@ /* A very simple encoding test case using person.proto. - * Just puts constant data in the fields. + * Just puts constant data in the fields and writes the + * data to stdout. */ #include #include #include "person.pb.h" +/* This binds the pb_ostream_t into the stdout stream */ bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) { FILE *file = (FILE*) stream->state; @@ -14,12 +16,16 @@ bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) int main() { + /* Initialize the structure with constants */ Person person = {"Test Person 99", 99, true, "test@person.com", 1, {{"555-12345678", true, Person_PhoneType_MOBILE}}}; + /* Prepare the stream, output goes directly to stdout */ pb_ostream_t stream = {&streamcallback, stdout, SIZE_MAX, 0}; - pb_encode(&stream, Person_fields, &person); - - return 0; + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, Person_fields, &person)) + return 0; /* Success */ + else + return 1; /* Failure */ } -- cgit 1.2.3-korg From c7e2d6cc2fde5a69da7bf2735f4fb33e1633b221 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 18 Sep 2011 16:10:08 +0000 Subject: Documentation outdated phrase git-svn-id: https://svn.kapsi.fi/jpa/nanopb@980 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index c4e5476e..f5e32adc 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -4,7 +4,7 @@ Nanopb: Basic concepts .. include :: menu.rst -The things outlined here are common to both the encoder and the decoder part. +The things outlined here are the underlying concepts of the nanopb design. .. contents:: -- cgit 1.2.3-korg From 85e0afd894daf851baca8ecd50ebae3cdba40e65 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 4 Nov 2011 07:22:05 +0000 Subject: Some additions to documentation git-svn-id: https://svn.kapsi.fi/jpa/nanopb@1003 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/Makefile | 8 +- docs/generator_flow.svg | 2869 +++++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 13 +- 3 files changed, 2883 insertions(+), 7 deletions(-) create mode 100644 docs/generator_flow.svg diff --git a/docs/Makefile b/docs/Makefile index d1d9587d..3c953990 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,4 +1,8 @@ -all: index.html concepts.html reference.html +all: index.html concepts.html reference.html \ + generator_flow.png + +%.png: %.svg + rsvg $< $@ %.html: %.rst - rst2html --stylesheet=lsr.css --link-stylesheet $< $@ \ No newline at end of file + rst2html --stylesheet=lsr.css --link-stylesheet $< $@ diff --git a/docs/generator_flow.svg b/docs/generator_flow.svg new file mode 100644 index 00000000..e30277a7 --- /dev/null +++ b/docs/generator_flow.svg @@ -0,0 +1,2869 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MyMessage.proto + + + pb_encode( ); + pb_decode( ); + + + nanopb_generator.py + + + + + + + + + + + + + + + + + + MyMessage.pb.c + MyMessage.pb.h + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + c + + Nanopb library + + + + Protocol Buffersmessages + + + + + + + + + + + + + + + + + + + + + Data structures + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + User application + + + diff --git a/docs/index.rst b/docs/index.rst index 1022a1b2..31d781ea 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -13,9 +13,11 @@ Overall structure ================= For the runtime program, you always need *pb.h* for type declarations. -Depending on whether you want to encode, decode or both, you also need *pb_encode.h/c* or *pb_decode.h/c*. +Depending on whether you want to encode, decode, or both, you also need *pb_encode.h/c* or *pb_decode.h/c*. -The high-level encoding and decoding functions take an array of *pb_field_t* structures, which describes the fields of a message structure. Usually you want these autogenerated from a *.proto* file. The tool string *nanopb_generator.py* accomplishes this. +The high-level encoding and decoding functions take an array of *pb_field_t* structures, which describes the fields of a message structure. Usually you want these autogenerated from a *.proto* file. The tool script *nanopb_generator.py* accomplishes this. + +.. image:: generator_flow.png So a typical project might include these files: @@ -25,8 +27,8 @@ So a typical project might include these files: - pb_encode.h and pb_encode.c (needed for encoding messages) 2) Protocol description (you can have many): - person.proto (just an example) - - person.c (autogenerated, contains initializers for const arrays) - - person.h (autogenerated, contains type declarations) + - person.pb.c (autogenerated, contains initializers for const arrays) + - person.pb.h (autogenerated, contains type declarations) Features and limitations ======================== @@ -37,7 +39,7 @@ Features and limitations #) Small code size (2–10 kB depending on processor) #) Small ram usage (typically 200 bytes) #) Allows specifying maximum size for strings and arrays, so that they can be allocated statically. -#) No malloc needed: everything is stored on the stack. +#) No malloc needed: everything can be allocated statically or on the stack. #) You can use either encoder or decoder alone to cut the code size in half. **Limitations** @@ -48,6 +50,7 @@ Features and limitations #) The deprecated Protocol Buffers feature called "groups" is not supported. #) Fields in the generated structs are ordered by the tag number, instead of the natural ordering in .proto file. #) Unknown fields are not preserved when decoding and re-encoding a message. +#) Reflection (runtime introspection) is not supported. E.g. you can't request a field by giving its name in a string. #) Numeric arrays are always encoded as packed, even if not marked as packed in .proto. This causes incompatibility with decoders that do not support packed format. #) Cyclic references between messages are not supported. They could be supported in callback-mode if there was an option in the generator to set the mode. -- cgit 1.2.3-korg From 13b6988b642e786efb5aaaba0601d25a65f80091 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 10 Nov 2011 16:19:55 +0000 Subject: Fix some compiler warnings in strict C89 mode Contributed by Michael Poole. git-svn-id: https://svn.kapsi.fi/jpa/nanopb@1004 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb_decode.c | 11 ++++++----- pb_encode.c | 5 ++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 453b1cc9..baa622d3 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -122,7 +122,7 @@ bool checkreturn pb_skip_string(pb_istream_t *stream) * to just assume the correct type and fail safely on corrupt message. */ -static bool checkreturn skip(pb_istream_t *stream, int wire_type) +static bool checkreturn skip(pb_istream_t *stream, pb_wire_type_t wire_type) { switch (wire_type) { @@ -197,7 +197,7 @@ static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, v { iter->start = iter->current = fields; iter->field_index = 0; - iter->pData = dest_struct + iter->current->data_offset; + iter->pData = (char*)dest_struct + iter->current->data_offset; iter->pSize = (char*)iter->pData + iter->current->size_offset; iter->dest_struct = dest_struct; } @@ -243,7 +243,7 @@ static bool checkreturn pb_field_find(pb_field_iterator_t *iter, int tag) * Decode a single field * *************************/ -static bool checkreturn decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_t *iter) +static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) { pb_decoder_t func = PB_DECODERS[PB_LTYPE(iter->current->type)]; @@ -393,7 +393,8 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void while (stream->bytes_left) { uint32_t temp; - int tag, wire_type; + int tag; + pb_wire_type_t wire_type; if (!pb_decode_varint32(stream, &temp)) { if (stream->bytes_left == 0) @@ -406,7 +407,7 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void break; /* Special feature: allow 0-terminated messages. */ tag = temp >> 3; - wire_type = temp & 7; + wire_type = (pb_wire_type_t)(temp & 7); if (!pb_field_find(&iter, tag)) { diff --git a/pb_encode.c b/pb_encode.c index 58e03a68..317e31ef 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -149,15 +149,14 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons while (field->tag != 0) { + pb_encoder_t func = PB_ENCODERS[PB_LTYPE(field->type)]; pData = (const char*)pData + prev_size + field->data_offset; pSize = (const char*)pData + field->size_offset; prev_size = field->data_size; if (PB_HTYPE(field->type) == PB_HTYPE_ARRAY) prev_size *= field->array_size; - - pb_encoder_t func = PB_ENCODERS[PB_LTYPE(field->type)]; - + switch (PB_HTYPE(field->type)) { case PB_HTYPE_REQUIRED: -- cgit 1.2.3-korg From a77ab47c2979d9ce6d03aee2ffe9c7a34250d86e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 30 Nov 2011 14:59:25 +0000 Subject: Added Makefile for generating nanopb_pb2.py. I still left the precompiled version in place, as it "often works" :) git-svn-id: https://svn.kapsi.fi/jpa/nanopb@1019 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- generator/Makefile | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 generator/Makefile diff --git a/generator/Makefile b/generator/Makefile new file mode 100644 index 00000000..161ef387 --- /dev/null +++ b/generator/Makefile @@ -0,0 +1,2 @@ +nanopb_pb2.py: nanopb.proto + protoc --python_out=. -I /usr/include -I . nanopb.proto -- cgit 1.2.3-korg From e66675a25d39bb5dfebea3041c8f2360bcac290c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 30 Nov 2011 15:01:59 +0000 Subject: Merged 0003-Fixed-format-specifiers.patch by Matt Kern. Fixes cross-platform issues with the length modifier in printf specifiers, most importantly %d -> %ld. git-svn-id: https://svn.kapsi.fi/jpa/nanopb@1020 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- tests/test_decode1.c | 2 +- tests/test_decode_callbacks.c | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_decode1.c b/tests/test_decode1.c index 3f02745e..b412ea85 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -23,7 +23,7 @@ bool print_person(pb_istream_t *stream) /* Now the decoding is done, rest is just to print stuff out. */ printf("name: \"%s\"\n", person.name); - printf("id: %d\n", person.id); + printf("id: %ld\n", (long)person.id); if (person.has_email) printf("email: \"%s\"\n", person.email); diff --git a/tests/test_decode_callbacks.c b/tests/test_decode_callbacks.c index 6c0072de..aaf4cdcc 100644 --- a/tests/test_decode_callbacks.c +++ b/tests/test_decode_callbacks.c @@ -30,7 +30,7 @@ bool print_int32(pb_istream_t *stream, const pb_field_t *field, void *arg) if (!pb_decode_varint(stream, &value)) return false; - printf((char*)arg, (int32_t)value); + printf((char*)arg, (long)value); return true; } @@ -40,7 +40,7 @@ bool print_fixed32(pb_istream_t *stream, const pb_field_t *field, void *arg) if (!pb_dec_fixed32(stream, NULL, &value)) return false; - printf((char*)arg, value); + printf((char*)arg, (long)value); return true; } @@ -50,7 +50,7 @@ bool print_fixed64(pb_istream_t *stream, const pb_field_t *field, void *arg) if (!pb_dec_fixed64(stream, NULL, &value)) return false; - printf((char*)arg, value); + printf((char*)arg, (long long)value); return true; } @@ -69,18 +69,18 @@ int main() testmessage.submsg.stringvalue.funcs.decode = &print_string; testmessage.submsg.stringvalue.arg = "submsg {\n stringvalue: \"%s\"\n"; testmessage.submsg.int32value.funcs.decode = &print_int32; - testmessage.submsg.int32value.arg = " int32value: %d\n"; + testmessage.submsg.int32value.arg = " int32value: %ld\n"; testmessage.submsg.fixed32value.funcs.decode = &print_fixed32; - testmessage.submsg.fixed32value.arg = " fixed32value: %d\n"; + testmessage.submsg.fixed32value.arg = " fixed32value: %ld\n"; testmessage.submsg.fixed64value.funcs.decode = &print_fixed64; testmessage.submsg.fixed64value.arg = " fixed64value: %lld\n}\n"; testmessage.stringvalue.funcs.decode = &print_string; testmessage.stringvalue.arg = "stringvalue: \"%s\"\n"; testmessage.int32value.funcs.decode = &print_int32; - testmessage.int32value.arg = "int32value: %d\n"; + testmessage.int32value.arg = "int32value: %ld\n"; testmessage.fixed32value.funcs.decode = &print_fixed32; - testmessage.fixed32value.arg = "fixed32value: %d\n"; + testmessage.fixed32value.arg = "fixed32value: %ld\n"; testmessage.fixed64value.funcs.decode = &print_fixed64; testmessage.fixed64value.arg = "fixed64value: %lld\n"; -- cgit 1.2.3-korg From 2a80ff2bd596b61ad8bca7b174f12a7dd99efaf5 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 30 Nov 2011 15:03:23 +0000 Subject: Merged 0004-Added-missing-trailing-newlines.patch by Matt Kern. Just adding newlines at the end of files to satisfy old GCC versions. git-svn-id: https://svn.kapsi.fi/jpa/nanopb@1021 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb.h | 2 +- pb_encode.h | 2 +- tests/test_decode_callbacks.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pb.h b/pb.h index cd9c75d0..ea94efc5 100644 --- a/pb.h +++ b/pb.h @@ -159,4 +159,4 @@ typedef enum { #define PB_LAST_FIELD {0,0,0,0} -#endif \ No newline at end of file +#endif diff --git a/pb_encode.h b/pb_encode.h index 864a48b7..cec39138 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -69,4 +69,4 @@ bool pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src); bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src); -#endif \ No newline at end of file +#endif diff --git a/tests/test_decode_callbacks.c b/tests/test_decode_callbacks.c index aaf4cdcc..714b7bb2 100644 --- a/tests/test_decode_callbacks.c +++ b/tests/test_decode_callbacks.c @@ -88,4 +88,4 @@ int main() return 1; return 0; -} \ No newline at end of file +} -- cgit 1.2.3-korg From ad7a0e2111aaf599466153097e4c8eebf476244d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 30 Nov 2011 15:08:32 +0000 Subject: Merged 0005-Handle-endianness-correctly.patch by Matt Kern. On big endian machines, pb_dec_fixed32 and pb_dec_fixed64 were incorrectly overwriting the result value even if reading failed. git-svn-id: https://svn.kapsi.fi/jpa/nanopb@1022 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb_decode.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index baa622d3..3992ab86 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -474,8 +474,10 @@ bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, v #ifdef __BIG_ENDIAN__ uint8_t bytes[4] = {0}; bool status = pb_read(stream, bytes, 4); - uint8_t bebytes[4] = {bytes[3], bytes[2], bytes[1], bytes[0]}; - memcpy(dest, bebytes, 4); + if (status) { + uint8_t bebytes[4] = {bytes[3], bytes[2], bytes[1], bytes[0]}; + memcpy(dest, bebytes, 4); + } return status; #else return pb_read(stream, (uint8_t*)dest, 4); @@ -487,9 +489,11 @@ bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, v #ifdef __BIG_ENDIAN__ uint8_t bytes[8] = {0}; bool status = pb_read(stream, bytes, 8); - uint8_t bebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], - bytes[3], bytes[2], bytes[1], bytes[0]}; - memcpy(dest, bebytes, 8); + if (status) { + uint8_t bebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], + bytes[3], bytes[2], bytes[1], bytes[0]}; + memcpy(dest, bebytes, 8); + } return status; #else return pb_read(stream, (uint8_t*)dest, 8); -- cgit 1.2.3-korg From 1506450b119a504259983692cca4c8cd3daddaf1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 30 Dec 2011 08:43:50 +0000 Subject: Fixed a bug related to submessage encoding into memory buffer. Stream state was not copied back from substream in pb_enc_submessage, which caused garbage output if the stream callback modified the state. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Expanded tests to cover this problem. Thanks to Paweł Pery for debugging and reporting this problem. git-svn-id: https://svn.kapsi.fi/jpa/nanopb@1089 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- pb_encode.c | 1 + tests/Makefile | 6 +++++- tests/test_decode1.c | 27 +++++++++++++++------------ tests/test_encode1.c | 27 ++++++++++++++------------- tests/test_encode2.c | 32 ++++++++++++++++++++++++++++++++ 5 files changed, 67 insertions(+), 26 deletions(-) create mode 100644 tests/test_encode2.c diff --git a/pb_encode.c b/pb_encode.c index 317e31ef..995eb3de 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -376,6 +376,7 @@ bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field status = pb_encode(&substream, (pb_field_t*)field->ptr, src); stream->bytes_written += substream.bytes_written; + stream->state = substream.state; if (substream.bytes_written != size) return false; diff --git a/tests/Makefile b/tests/Makefile index 2d5ee506..a942b786 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -19,6 +19,7 @@ pb_decode.o: ../pb_decode.c $(DEPS) test_decode1: test_decode1.o pb_decode.o person.pb.o test_encode1: test_encode1.o pb_encode.o person.pb.o +test_encode2: test_encode2.o pb_encode.o person.pb.o test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o @@ -37,7 +38,7 @@ coverage: run_unittests gcov pb_encode.gcda gcov pb_decode.gcda -run_unittests: decode_unittests encode_unittests test_encode1 test_decode1 test_encode_callbacks test_decode_callbacks +run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_decode1 test_encode_callbacks test_decode_callbacks rm -f *.gcda ./decode_unittests > /dev/null @@ -45,6 +46,9 @@ run_unittests: decode_unittests encode_unittests test_encode1 test_decode1 test_ [ "`./test_encode1 | ./test_decode1`" = \ "`./test_encode1 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] + + [ "`./test_encode2 | ./test_decode1`" = \ + "`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] [ "`./test_encode_callbacks | ./test_decode_callbacks`" = \ "`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ] diff --git a/tests/test_decode1.c b/tests/test_decode1.c index b412ea85..3b72f664 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -34,19 +34,22 @@ bool print_person(pb_istream_t *stream) printf("phone {\n"); printf(" number: \"%s\"\n", phone->number); - switch (phone->type) + if (phone->has_type) { - case Person_PhoneType_WORK: - printf(" type: WORK\n"); - break; - - case Person_PhoneType_HOME: - printf(" type: HOME\n"); - break; - - case Person_PhoneType_MOBILE: - printf(" type: MOBILE\n"); - break; + switch (phone->type) + { + case Person_PhoneType_WORK: + printf(" type: WORK\n"); + break; + + case Person_PhoneType_HOME: + printf(" type: HOME\n"); + break; + + case Person_PhoneType_MOBILE: + printf(" type: MOBILE\n"); + break; + } } printf("}\n"); } diff --git a/tests/test_encode1.c b/tests/test_encode1.c index f46e60a6..c5131e47 100644 --- a/tests/test_encode1.c +++ b/tests/test_encode1.c @@ -1,31 +1,32 @@ /* A very simple encoding test case using person.proto. - * Just puts constant data in the fields and writes the - * data to stdout. + * Just puts constant data in the fields and encodes into + * buffer, which is then written to stdout. */ #include #include #include "person.pb.h" -/* This binds the pb_ostream_t into the stdout stream */ -bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) -{ - FILE *file = (FILE*) stream->state; - return fwrite(buf, 1, count, file) == count; -} - int main() { /* Initialize the structure with constants */ Person person = {"Test Person 99", 99, true, "test@person.com", - 1, {{"555-12345678", true, Person_PhoneType_MOBILE}}}; - - /* Prepare the stream, output goes directly to stdout */ - pb_ostream_t stream = {&streamcallback, stdout, SIZE_MAX, 0}; + 3, {{"555-12345678", true, Person_PhoneType_MOBILE}, + {"99-2342", false, 0}, + {"1234-5678", true, Person_PhoneType_WORK}, + }}; + + uint8_t buffer[512]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); /* Now encode it and check if we succeeded. */ if (pb_encode(&stream, Person_fields, &person)) + { + fwrite(buffer, stream.bytes_written, 1, stdout); return 0; /* Success */ + } else + { return 1; /* Failure */ + } } diff --git a/tests/test_encode2.c b/tests/test_encode2.c new file mode 100644 index 00000000..b1105ce6 --- /dev/null +++ b/tests/test_encode2.c @@ -0,0 +1,32 @@ +/* Same as test_encode1.c, except writes directly to stdout. + */ + +#include +#include +#include "person.pb.h" + +/* This binds the pb_ostream_t into the stdout stream */ +bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) +{ + FILE *file = (FILE*) stream->state; + return fwrite(buf, 1, count, file) == count; +} + +int main() +{ + /* Initialize the structure with constants */ + Person person = {"Test Person 99", 99, true, "test@person.com", + 3, {{"555-12345678", true, Person_PhoneType_MOBILE}, + {"99-2342", false, 0}, + {"1234-5678", true, Person_PhoneType_WORK}, + }}; + + /* Prepare the stream, output goes directly to stdout */ + pb_ostream_t stream = {&streamcallback, stdout, SIZE_MAX, 0}; + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, Person_fields, &person)) + return 0; /* Success */ + else + return 1; /* Failure */ +} -- cgit 1.2.3-korg From a3f5e4d81f8d81e208f33dded6d892fbbdce90bc Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 30 Dec 2011 08:57:27 +0000 Subject: Expanding the tests to better cover decoding from memory buffer. git-svn-id: https://svn.kapsi.fi/jpa/nanopb@1090 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- tests/Makefile | 6 +++- tests/test_decode1.c | 31 ++++-------------- tests/test_decode2.c | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/test_encode1.c | 2 +- 4 files changed, 103 insertions(+), 26 deletions(-) create mode 100644 tests/test_decode2.c diff --git a/tests/Makefile b/tests/Makefile index a942b786..899a6865 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -18,6 +18,7 @@ pb_decode.o: ../pb_decode.c $(DEPS) $(CC) $(CFLAGS) -c -o $@ $< test_decode1: test_decode1.o pb_decode.o person.pb.o +test_decode2: test_decode2.o pb_decode.o person.pb.o test_encode1: test_encode1.o pb_encode.o person.pb.o test_encode2: test_encode2.o pb_encode.o person.pb.o test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o @@ -38,7 +39,7 @@ coverage: run_unittests gcov pb_encode.gcda gcov pb_decode.gcda -run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_decode1 test_encode_callbacks test_decode_callbacks +run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_decode1 test_decode2 test_encode_callbacks test_decode_callbacks rm -f *.gcda ./decode_unittests > /dev/null @@ -49,6 +50,9 @@ run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_ [ "`./test_encode2 | ./test_decode1`" = \ "`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] + + [ "`./test_encode2 | ./test_decode2`" = \ + "`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] [ "`./test_encode_callbacks | ./test_decode_callbacks`" = \ "`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ] diff --git a/tests/test_decode1.c b/tests/test_decode1.c index 3b72f664..78781dd4 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -57,33 +57,16 @@ bool print_person(pb_istream_t *stream) return true; } -/* This binds the pb_istream_t to stdin */ -bool callback(pb_istream_t *stream, uint8_t *buf, size_t count) +int main() { - FILE *file = (FILE*)stream->state; - bool status; - - if (buf == NULL) - { - /* Skipping data */ - while (count-- && fgetc(file) != EOF); - return count == 0; - } - - status = (fread(buf, 1, count, file) == count); + /* Read the data into buffer */ + uint8_t buffer[512]; + size_t count = fread(buffer, 1, sizeof(buffer), stdin); - if (feof(file)) - stream->bytes_left = 0; + /* Construct a pb_istream_t for reading from the buffer */ + pb_istream_t stream = pb_istream_from_buffer(buffer, count); - return status; -} - -int main() -{ - /* Maximum size is specified to prevent infinite length messages from - * hanging this in the fuzz test. - */ - pb_istream_t stream = {&callback, stdin, 10000}; + /* Decode and print out the stuff */ if (!print_person(&stream)) { printf("Parsing failed.\n"); diff --git a/tests/test_decode2.c b/tests/test_decode2.c new file mode 100644 index 00000000..d38e6256 --- /dev/null +++ b/tests/test_decode2.c @@ -0,0 +1,90 @@ +/* Same as test_decode1 but reads from stdin directly. + */ + +#include +#include +#include "person.pb.h" + +/* This function is called once from main(), it handles + the decoding and printing. + Ugly copy-paste from test_decode1.c. */ +bool print_person(pb_istream_t *stream) +{ + int i; + Person person; + + if (!pb_decode(stream, Person_fields, &person)) + return false; + + /* Now the decoding is done, rest is just to print stuff out. */ + + printf("name: \"%s\"\n", person.name); + printf("id: %ld\n", (long)person.id); + + if (person.has_email) + printf("email: \"%s\"\n", person.email); + + for (i = 0; i < person.phone_count; i++) + { + Person_PhoneNumber *phone = &person.phone[i]; + printf("phone {\n"); + printf(" number: \"%s\"\n", phone->number); + + if (phone->has_type) + { + switch (phone->type) + { + case Person_PhoneType_WORK: + printf(" type: WORK\n"); + break; + + case Person_PhoneType_HOME: + printf(" type: HOME\n"); + break; + + case Person_PhoneType_MOBILE: + printf(" type: MOBILE\n"); + break; + } + } + printf("}\n"); + } + + return true; +} + +/* This binds the pb_istream_t to stdin */ +bool callback(pb_istream_t *stream, uint8_t *buf, size_t count) +{ + FILE *file = (FILE*)stream->state; + bool status; + + if (buf == NULL) + { + /* Skipping data */ + while (count-- && fgetc(file) != EOF); + return count == 0; + } + + status = (fread(buf, 1, count, file) == count); + + if (feof(file)) + stream->bytes_left = 0; + + return status; +} + +int main() +{ + /* Maximum size is specified to prevent infinite length messages from + * hanging this in the fuzz test. + */ + pb_istream_t stream = {&callback, stdin, 10000}; + if (!print_person(&stream)) + { + printf("Parsing failed.\n"); + return 1; + } else { + return 0; + } +} diff --git a/tests/test_encode1.c b/tests/test_encode1.c index c5131e47..2e978296 100644 --- a/tests/test_encode1.c +++ b/tests/test_encode1.c @@ -22,7 +22,7 @@ int main() /* Now encode it and check if we succeeded. */ if (pb_encode(&stream, Person_fields, &person)) { - fwrite(buffer, stream.bytes_written, 1, stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); return 0; /* Success */ } else -- cgit 1.2.3-korg From b5fb97f6f58a1aebe4095c5e9cae5c4bd777812e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 30 Dec 2011 09:05:01 +0000 Subject: Make the fuzztest runnable again git-svn-id: https://svn.kapsi.fi/jpa/nanopb@1091 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- tests/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 899a6865..11f061a2 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -57,5 +57,5 @@ run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_ [ "`./test_encode_callbacks | ./test_decode_callbacks`" = \ "`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ] -run_fuzztest: test_decode1 - bash -c 'I=1; while cat /dev/urandom | ./test_decode1 > /dev/null; do I=$$(($$I+1)); echo -en "\r$$I"; done' +run_fuzztest: test_decode2 + bash -c 'I=1; while true; do cat /dev/urandom | ./test_decode2 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done' -- cgit 1.2.3-korg From 28aa0ce0966ab9bc93654b2c3f9ffb3aaca255bf Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 5 Jan 2012 18:35:12 +0000 Subject: Added logo for project :) git-svn-id: https://svn.kapsi.fi/jpa/nanopb@1095 e3a754e5-d11d-0410-8d38-ebb782a927b9 --- docs/Makefile | 1 + docs/logo/logo.png | Bin 0 -> 14973 bytes docs/logo/logo.svg | 1470 ++++++++++++++++++++++++++++++++++++++++++++++++ docs/logo/logo16px.png | Bin 0 -> 854 bytes docs/logo/logo48px.png | Bin 0 -> 2577 bytes 5 files changed, 1471 insertions(+) create mode 100644 docs/logo/logo.png create mode 100644 docs/logo/logo.svg create mode 100644 docs/logo/logo16px.png create mode 100644 docs/logo/logo48px.png diff --git a/docs/Makefile b/docs/Makefile index 3c953990..cfd84ac0 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -6,3 +6,4 @@ all: index.html concepts.html reference.html \ %.html: %.rst rst2html --stylesheet=lsr.css --link-stylesheet $< $@ + sed -i 's!!\n!' $@ diff --git a/docs/logo/logo.png b/docs/logo/logo.png new file mode 100644 index 00000000..0d9534fa Binary files /dev/null and b/docs/logo/logo.png differ diff --git a/docs/logo/logo.svg b/docs/logo/logo.svg new file mode 100644 index 00000000..91ab28b6 --- /dev/null +++ b/docs/logo/logo.svg @@ -0,0 +1,1470 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + image/svg+xml + + + + + + + + + + + + + + Pb + Pb + + + + + + + + + + + + + + + + + + nano + nano + + + + + diff --git a/docs/logo/logo16px.png b/docs/logo/logo16px.png new file mode 100644 index 00000000..8db0e2ef Binary files /dev/null and b/docs/logo/logo16px.png differ diff --git a/docs/logo/logo48px.png b/docs/logo/logo48px.png new file mode 100644 index 00000000..b598c011 Binary files /dev/null and b/docs/logo/logo48px.png differ -- cgit 1.2.3-korg From b519edba9d98bcd89bde389b4845b2f19a35cdf3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 6 Jan 2012 19:04:49 +0200 Subject: Linewrapping in license --- LICENSE | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/LICENSE b/LICENSE index ac41e52a..d11c9af1 100644 --- a/LICENSE +++ b/LICENSE @@ -1,11 +1,20 @@ Copyright (c) 2011 Petteri Aimonen -This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. +This software is provided 'as-is', without any express or +implied warranty. In no event will the authors be held liable +for any damages arising from the use of this software. -Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: +Permission is granted to anyone to use this software for any +purpose, including commercial applications, and to alter it and +redistribute it freely, subject to the following restrictions: -1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. +1. The origin of this software must not be misrepresented; you + must not claim that you wrote the original software. If you use + this software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. -2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. +2. Altered source versions must be plainly marked as such, and + must not be misrepresented as being the original software. -3. This notice may not be removed or altered from any source distribution. \ No newline at end of file +3. This notice may not be removed or altered from any source + distribution. -- cgit 1.2.3-korg From a1adf398052bf2c78a457c484efbc5ec0258374e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 12 Jan 2012 18:10:12 +0200 Subject: Fixed a bug in the generator that caused a compiler error on sfixed32 and sfixed64 fields. --- generator/nanopb_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 4d5aaaca..e1d99c97 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -14,8 +14,8 @@ datatypes = { FieldD.TYPE_FLOAT: ('float', 'PB_LTYPE_FIXED32'), FieldD.TYPE_INT32: ('int32_t', 'PB_LTYPE_VARINT'), FieldD.TYPE_INT64: ('int64_t', 'PB_LTYPE_VARINT'), - FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED'), - FieldD.TYPE_SFIXED64: ('int64_t', 'PB_LTYPE_FIXED'), + FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED32'), + FieldD.TYPE_SFIXED64: ('int64_t', 'PB_LTYPE_FIXED64'), FieldD.TYPE_SINT32: ('int32_t', 'PB_LTYPE_SVARINT'), FieldD.TYPE_SINT64: ('int64_t', 'PB_LTYPE_SVARINT'), FieldD.TYPE_UINT32: ('uint32_t', 'PB_LTYPE_VARINT'), -- cgit 1.2.3-korg From 0f6b615ae3395734ee9a1b35185540acad18c452 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 12 Jan 2012 19:06:33 +0200 Subject: Added an encode/decode test for 'required' fields of all types. --- tests/Makefile | 11 ++++++--- tests/alltypes.proto | 40 ++++++++++++++++++++++++++++++ tests/test_decode3.c | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/test_encode3.c | 50 +++++++++++++++++++++++++++++++++++++ 4 files changed, 168 insertions(+), 3 deletions(-) create mode 100644 tests/alltypes.proto create mode 100644 tests/test_decode3.c create mode 100644 tests/test_encode3.c diff --git a/tests/Makefile b/tests/Makefile index 11f061a2..8e2df140 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,12 +1,12 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage LDFLAGS=--coverage -DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h +DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h TESTS=test_decode1 test_encode1 decode_unittests encode_unittests all: breakpoints $(TESTS) run_unittests clean: - rm -f $(TESTS) person.pb* *.o *.gcda *.gcno + rm -f $(TESTS) person.pb* alltypes.pb* *.o *.gcda *.gcno %.o: %.c %.o: %.c $(DEPS) @@ -19,8 +19,10 @@ pb_decode.o: ../pb_decode.c $(DEPS) test_decode1: test_decode1.o pb_decode.o person.pb.o test_decode2: test_decode2.o pb_decode.o person.pb.o +test_decode3: test_decode3.o pb_decode.o alltypes.pb.o test_encode1: test_encode1.o pb_encode.o person.pb.o test_encode2: test_encode2.o pb_encode.o person.pb.o +test_encode3: test_encode3.o pb_encode.o alltypes.pb.o test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o @@ -39,7 +41,7 @@ coverage: run_unittests gcov pb_encode.gcda gcov pb_decode.gcda -run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_decode1 test_decode2 test_encode_callbacks test_decode_callbacks +run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks rm -f *.gcda ./decode_unittests > /dev/null @@ -57,5 +59,8 @@ run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_ [ "`./test_encode_callbacks | ./test_decode_callbacks`" = \ "`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ] + ./test_encode3 | ./test_decode3 + ./test_encode3 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null + run_fuzztest: test_decode2 bash -c 'I=1; while true; do cat /dev/urandom | ./test_decode2 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done' diff --git a/tests/alltypes.proto b/tests/alltypes.proto new file mode 100644 index 00000000..744a0fca --- /dev/null +++ b/tests/alltypes.proto @@ -0,0 +1,40 @@ +import "nanopb.proto"; + +message SubMessage { + required string substuff1 = 1 [(nanopb).max_size = 16]; + required int32 substuff2 = 2; +} + +enum MyEnum { + First = 1; + Second = 2; + Truth = 42; +} + +message AllTypes { + required int32 req_int32 = 1; + required int64 req_int64 = 2; + required uint32 req_uint32 = 3; + required uint64 req_uint64 = 4; + required sint32 req_sint32 = 5; + required sint64 req_sint64 = 6; + required bool req_bool = 7; + + required fixed32 req_fixed32 = 8; + required sfixed32 req_sfixed32= 9; + required float req_float = 10; + + required fixed64 req_fixed64 = 11; + required sfixed64 req_sfixed64= 12; + required double req_double = 13; + + required string req_string = 14 [(nanopb).max_size = 16]; + required bytes req_bytes = 15 [(nanopb).max_size = 16]; + required SubMessage req_submsg = 16; + required MyEnum req_enum = 17; + + // Just to make sure that the size of the fields has been calculated + // properly, i.e. otherwise a bug in last field might not be detected. + required int32 end = 99; +} + diff --git a/tests/test_decode3.c b/tests/test_decode3.c new file mode 100644 index 00000000..a7106dee --- /dev/null +++ b/tests/test_decode3.c @@ -0,0 +1,70 @@ +/* Tests the decoding of all types. Currently only in the 'required' variety. + * This is the counterpart of test_encode3. + * Run e.g. ./test_encode3 | ./test_decode3 + */ + +#include +#include +#include +#include "alltypes.pb.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed.\n"); \ + return false; \ + } + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_alltypes(pb_istream_t *stream) +{ + AllTypes alltypes = {}; + + if (!pb_decode(stream, AllTypes_fields, &alltypes)) + return false; + + TEST(alltypes.req_int32 == 1001); + TEST(alltypes.req_int64 == 1002); + TEST(alltypes.req_uint32 == 1003); + TEST(alltypes.req_uint64 == 1004); + TEST(alltypes.req_sint32 == 1005); + TEST(alltypes.req_sint64 == 1006); + TEST(alltypes.req_bool == true); + + TEST(alltypes.req_fixed32 == 1008); + TEST(alltypes.req_sfixed32 == 1009); + TEST(alltypes.req_float == 1010.0f); + + TEST(alltypes.req_fixed64 == 1011); + TEST(alltypes.req_sfixed64 == 1012); + TEST(alltypes.req_double == 1013.0f); + + TEST(strcmp(alltypes.req_string, "1014") == 0); + TEST(alltypes.req_bytes.size == 4); + TEST(memcmp(alltypes.req_bytes.bytes, "1015", 4) == 0); + TEST(strcmp(alltypes.req_submsg.substuff1, "1016") == 0); + TEST(alltypes.req_submsg.substuff2 == 1016); + TEST(alltypes.req_enum == MyEnum_Truth); + + TEST(alltypes.end == 1099); + + return true; +} + +int main() +{ + /* Read the data into buffer */ + uint8_t buffer[512]; + size_t count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + pb_istream_t stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_alltypes(&stream)) + { + printf("Parsing failed.\n"); + return 1; + } else { + return 0; + } +} diff --git a/tests/test_encode3.c b/tests/test_encode3.c new file mode 100644 index 00000000..5e94be51 --- /dev/null +++ b/tests/test_encode3.c @@ -0,0 +1,50 @@ +/* Attempts to test all the datatypes supported by ProtoBuf. + * Currently only tests the 'required' variety. + */ + +#include +#include +#include "alltypes.pb.h" + +int main() +{ + /* Initialize the structure with constants */ + AllTypes alltypes = { + 1001, + 1002, + 1003, + 1004, + 1005, + 1006, + true, + + 1008, + 1009, + 1010.0f, + + 1011, + 1012, + 1013.0, + + "1014", + {4, "1015"}, + {"1016", 1016}, + MyEnum_Truth, + + 1099 + }; + + uint8_t buffer[512]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + return 1; /* Failure */ + } +} -- cgit 1.2.3-korg From 113bd7ee878ac2284c8c049fdb8dc2d2bd19f016 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 12 Jan 2012 19:08:05 +0200 Subject: Fixed issue 1 reported by Erik Rosen: The size of non-callback bytes-fields was miscalculated, which caused all following fields in a message to contain garbage. Previous commit contains a testcase for this. This fix changes the generated message description. If your protocol uses bytes-fields, you should regenerate *.pb.c. --- docs/concepts.rst | 5 ++++- generator/nanopb_generator.py | 3 --- pb_decode.c | 4 +++- tests/decode_unittests.c | 12 ++++++++++-- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index f5e32adc..1d0fe08f 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -158,7 +158,10 @@ required bytes data = 1 [(nanopb).max_size = 40]; | Person_data_t data; =============================================================================== ======================= -The maximum lengths are checked in runtime. If string/bytes/array exceeds the allocated length, *pb_decode* will return false. +The maximum lengths are checked in runtime. If string/bytes/array exceeds the allocated length, *pb_decode* will return false. + +Note: for the *bytes* datatype, the field length checking may not be exact. +The compiler may add some padding to the *pb_bytes_t* structure, and the nanopb runtime doesn't know how much of the structure size is padding. Therefore it uses the whole length of the structure for storing data, which is not very smart but shouldn't cause problems. In practise, this means that if you specify *(nanopb).max_size=5* on a *bytes* field, you may be able to store 6 bytes there. For the *string* field type, the length limit is exact. Field callbacks =============== diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index e1d99c97..2ceafc7e 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -219,9 +219,6 @@ class Field: result += '\n pb_membersize(%s, %s[0]),' % (self.struct_name, self.name) result += ('\n pb_membersize(%s, %s) / pb_membersize(%s, %s[0]),' % (self.struct_name, self.name, self.struct_name, self.name)) - elif self.htype != 'PB_HTYPE_CALLBACK' and self.ltype == 'PB_LTYPE_BYTES': - result += '\n pb_membersize(%s, bytes),' % self.ctype - result += ' 0,' else: result += '\n pb_membersize(%s, %s),' % (self.struct_name, self.name) result += ' 0,' diff --git a/pb_decode.c b/pb_decode.c index 3992ab86..1e2fea0a 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -509,7 +509,8 @@ bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, voi return false; x->size = temp; - if (x->size > field->data_size) + /* Check length, noting the space taken by the size_t header. */ + if (x->size > field->data_size - offsetof(pb_bytes_array_t, bytes)) return false; return pb_read(stream, x->bytes, x->size); @@ -522,6 +523,7 @@ bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, vo if (!pb_decode_varint32(stream, &size)) return false; + /* Check length, noting the null terminator */ if (size > field->data_size - 1) return false; diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c index ab12ac33..6ba6d4f2 100644 --- a/tests/decode_unittests.c +++ b/tests/decode_unittests.c @@ -167,14 +167,22 @@ int main() { pb_istream_t s; struct { size_t size; uint8_t bytes[5]; } d; - pb_field_t f = {1, PB_LTYPE_BYTES, 0, 0, 5, 0, 0}; + pb_field_t f = {1, PB_LTYPE_BYTES, 0, 0, sizeof(d), 0, 0}; COMMENT("Test pb_dec_bytes") TEST((s = S("\x00"), pb_dec_bytes(&s, &f, &d) && d.size == 0)) TEST((s = S("\x01\xFF"), pb_dec_bytes(&s, &f, &d) && d.size == 1 && d.bytes[0] == 0xFF)) - TEST((s = S("\x06xxxxxx"), !pb_dec_bytes(&s, &f, &d))) TEST((s = S("\x05xxxxx"), pb_dec_bytes(&s, &f, &d) && d.size == 5)) TEST((s = S("\x05xxxx"), !pb_dec_bytes(&s, &f, &d))) + + /* Note: the size limit on bytes-fields is not strictly obeyed, as + * the compiler may add some padding to the struct. Using this padding + * is not a very good thing to do, but it is difficult to avoid when + * we use only a single uint8_t to store the size of the field. + * Therefore this tests against a 10-byte string, while otherwise even + * 6 bytes should error out. + */ + TEST((s = S("\x10xxxxxxxxxx"), !pb_dec_bytes(&s, &f, &d))) } { -- cgit 1.2.3-korg From b36a1a259ac065fcd21f4871c8c1e1db429e32bb Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 23 Jan 2012 18:13:26 +0200 Subject: Improved documentation on field decoders. --- docs/reference.rst | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index d67499e1..31f4e9fd 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -376,7 +376,12 @@ Because of memory concerns, the detection of missing required fields is not perf Each field decoder reads and decodes a single value. For arrays, the decoder is called repeatedly. - You can use the decoders from your callbacks. Just be aware that the pb_field_t passed to the callback is not directly compatible with most of the field decoders. Instead, you must create a new pb_field_t structure and set the data_size according to the data type you pass to *dest*. + You can use the decoders from your callbacks. Just be aware that the pb_field_t passed to the callback is not directly compatible + with the *varint* field decoders. Instead, you must create a new pb_field_t structure and set the data_size according to the data type + you pass to *dest*, e.g. *field.data_size = sizeof(int);*. Other fields in the *pb_field_t* don't matter. + + The field decoder interface is a bit messy as a result of the interface required inside the nanopb library. + Eventually they may be replaced by separate wrapper functions with a more friendly interface. pb_dec_varint ------------- @@ -403,12 +408,12 @@ pb_dec_fixed32 -------------- Field decoder for PB_LTYPE_FIXED32. :: - bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest); + bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest); -:stream: Input stream to read from. 1-10 bytes will be read. +:stream: Input stream to read from. 4 bytes will be read. :field: Not used. -:dest: Pointer to destination integer. Must have size of *field->data_size* bytes. -:returns: True on success, false on IO errors or if `pb_decode_varint`_ fails. +:dest: Pointer to destination *int32_t*, *uint32_t* or *float*. +:returns: True on success, false on IO errors. This function reads 4 bytes from the input stream. On big endian architectures, it then reverses the order of the bytes. @@ -420,6 +425,11 @@ Field decoder for PB_LTYPE_FIXED64. :: bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest); +:stream: Input stream to read from. 8 bytes will be read. +:field: Not used. +:dest: Pointer to destination *int64_t*, *uint64_t* or *double*. +:returns: True on success, false on IO errors. + Same as `pb_dec_fixed32`_, except this reads 8 bytes. pb_dec_bytes @@ -428,6 +438,9 @@ Field decoder for PB_LTYPE_BYTES. Reads a length-prefixed block of bytes. :: bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); +**Note:** This is an internal function that is not useful in decoder callbacks. To read bytes fields in callbacks, use +*stream->bytes_left* and `pb_read`_. + :stream: Input stream to read from. :field: Field description structure. Only *field->data_size* matters. :dest: Pointer to a structure similar to pb_bytes_array_t. @@ -441,6 +454,9 @@ Field decoder for PB_LTYPE_STRING. Reads a length-prefixed string. :: bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); +**Note:** This is an internal function that is not useful in decoder callbacks. To read string fields in callbacks, use +*stream->bytes_left* and `pb_read`_. + :stream: Input stream to read from. :field: Field description structure. Only *field->data_size* matters. :dest: Pointer to a character array of size *field->data_size*. @@ -454,6 +470,9 @@ Field decoder for PB_LTYPE_SUBMESSAGE. Calls `pb_decode`_ to perform the actual bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) +**Note:** This is an internal function that is not useful in decoder callbacks. To read submessage fields in callbacks, use +`pb_decode`_ directly. + :stream: Input stream to read from. :field: Field description structure. Only *field->ptr* matters. :dest: Pointer to the destination structure. -- cgit 1.2.3-korg From f6b08404fa9408e719785f89ee69df8e2457e158 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 30 Jan 2012 10:36:17 +0200 Subject: Fixed nanopb_generator.py to read the input file in binary mode. --- generator/nanopb_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 2ceafc7e..337b32a5 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -404,7 +404,7 @@ if __name__ == '__main__': print "Output fill be written to file.pb.h and file.pb.c" sys.exit(1) - data = open(sys.argv[1]).read() + data = open(sys.argv[1], 'rb').read() fdesc = descriptor.FileDescriptorSet.FromString(data) enums, messages = parse_file(fdesc.file[0]) -- cgit 1.2.3-korg From 0cdc623050ac8c76f3a8dbd14675a3b29f18542d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 15 Feb 2012 17:34:48 +0200 Subject: Modified nanopb_generator.py to generate includes for other .proto files. Implementation was suggested by extremeblue99. Fixes issue 4. --- generator/nanopb_generator.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 337b32a5..2cc92f01 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -347,7 +347,7 @@ def sort_dependencies(messages): if msgname in message_by_name: yield message_by_name[msgname] -def generate_header(headername, enums, messages): +def generate_header(dependencies, headername, enums, messages): '''Generate content for a header file. Generates strings, which should be concatenated and stored to file. ''' @@ -359,6 +359,11 @@ def generate_header(headername, enums, messages): yield '#define _PB_%s_\n' % symbol yield '#include \n\n' + for dependency in dependencies: + noext = os.path.splitext(dependency)[0] + yield '#include "%s.pb.h"\n' % noext + yield '\n' + yield '/* Enum definitions */\n' for enum in enums: yield str(enum) + '\n\n' @@ -415,8 +420,13 @@ if __name__ == '__main__': print "Writing to " + headername + " and " + sourcename + # List of .proto files that should not be included in the C header file + # even if they are mentioned in the source .proto. + excludes = ['nanopb.proto'] + dependencies = [d for d in fdesc.file[0].dependency if d not in excludes] + header = open(headername, 'w') - for part in generate_header(headerbasename, enums, messages): + for part in generate_header(dependencies, headerbasename, enums, messages): header.write(part) source = open(sourcename, 'w') -- cgit 1.2.3-korg From 9fbe9a5de30c3326bd7015e91c5ba634df49ee25 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 1 Mar 2012 13:46:52 +0200 Subject: Refactoring the field encoder interface. Replaced the confusing pb_enc_* functions with new pb_encode_* functions that have a cleaner interface. Updated documentation. Got rid of the endian_copy stuff in pb_encode.c, instead using C casts to do it automatically. This makes the code safer and also reduces binary size by about 5%. Fixes Issue 6. --- docs/reference.rst | 124 +++++++++++------------------- example/server.c | 2 +- pb_encode.c | 170 +++++++++++++++++++++++------------------- pb_encode.h | 47 ++++++++++-- tests/encode_unittests.c | 5 +- tests/test_encode_callbacks.c | 4 +- 6 files changed, 179 insertions(+), 173 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 31f4e9fd..4021c764 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -149,17 +149,13 @@ Encodes the contents of a structure as a protocol buffers message and writes it Normally pb_encode simply walks through the fields description array and serializes each field in turn. However, submessages must be serialized twice: first to calculate their size and then to actually write them to output. This causes some constraints for callback fields, which must return the same data on every call. -pb_encode_varint ----------------- -Encodes an unsigned integer in the varint_ format. :: +.. sidebar:: Encoding fields manually - bool pb_encode_varint(pb_ostream_t *stream, uint64_t value); + The functions with names *pb_encode_\** are used when dealing with callback fields. The typical reason for using callbacks is to have an array of unlimited size. In that case, `pb_encode`_ will call your callback function, which in turn will call *pb_encode_\** functions repeatedly to write out values. -:stream: Output stream to write to. 1-10 bytes will be written. -:value: Value to encode. -:returns: True on success, false on IO error. + The tag of a field must be encoded separately with `pb_encode_tag_for_field`_. After that, you can call exactly one of the content-writing functions to encode the payload of the field. For repeated fields, you can repeat this process multiple times. -.. _varint: http://code.google.com/apis/protocolbuffers/docs/encoding.html#varints + Writing packed arrays is a little bit more involved: you need to use `pb_encode_tag` and specify `PB_WT_STRING` as the wire type. Then you need to know exactly how much data you are going to write, and use `pb_encode_varint`_ to write out the number of bytes before writing the actual data. Substreams can be used to determine the number of bytes beforehand; see `pb_encode_submessage`_ source code for an example. pb_encode_tag ------------- @@ -169,7 +165,7 @@ Starts a field in the Protocol Buffers binary format: encodes the field number a :stream: Output stream to write to. 1-5 bytes will be written. :wiretype: PB_WT_VARINT, PB_WT_64BIT, PB_WT_STRING or PB_WT_32BIT -:field_number: Identifier for the field, defined in the .proto file. +:field_number: Identifier for the field, defined in the .proto file. You can get it from field->tag. :returns: True on success, false on IO error. pb_encode_tag_for_field @@ -195,107 +191,71 @@ STRING, BYTES, SUBMESSAGE PB_WT_STRING FIXED32 PB_WT_32BIT ========================= ============ -pb_encode_string +pb_encode_varint ---------------- -Writes the length of a string as varint and then contents of the string. Used for writing fields with wire type PB_WT_STRING. :: - - bool pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size); +Encodes a signed or unsigned integer in the varint_ format. Works for fields of type `bool`, `enum`, `int32`, `int64`, `uint32` and `uint64`:: -:stream: Output stream to write to. -:buffer: Pointer to string data. -:size: Number of bytes in the string. -:returns: True on success, false on IO error. - -.. sidebar:: Field encoders - - The functions with names beginning with *pb_enc_* are called field encoders. Each PB_LTYPE has an own field encoder, which handles translating from C data into Protocol Buffers data. - - By using the *data_size* in the field description and by taking advantage of C casting rules, it has been possible to combine many data types to a single LTYPE. For example, *int32*, *uint32*, *int64*, *uint64*, *bool* and *enum* are all handled by *pb_enc_varint*. - - Each field encoder only encodes the contents of the field. The tag must be encoded separately with `pb_encode_tag_for_field`_. - - You can use the field encoders from your callbacks. Just be aware that the pb_field_t passed to the callback is not directly compatible with most of the encoders. Instead, you must create a new pb_field_t structure and set the data_size according to the data type you pass to *src*. - -pb_enc_varint -------------- -Field encoder for PB_LTYPE_VARINT. Takes the first *field->data_size* bytes from src, casts them as *uint64_t* and calls `pb_encode_varint`_. :: - - bool pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src); + bool pb_encode_varint(pb_ostream_t *stream, uint64_t value); -:stream: Output stream to write to. -:field: Field description structure. Only *data_size* matters. -:src: Pointer to start of the field data. +:stream: Output stream to write to. 1-10 bytes will be written. +:value: Value to encode. Just cast e.g. int32_t directly to uint64_t. :returns: True on success, false on IO error. -pb_enc_svarint --------------- -Field encoder for PB_LTYPE_SVARINT. Similar to `pb_enc_varint`_, except first zig-zag encodes the value for more efficient negative number encoding. :: +.. _varint: http://code.google.com/apis/protocolbuffers/docs/encoding.html#varints - bool pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); +pb_encode_svarint +----------------- +Encodes a signed integer in the 'zig-zagged' format. Works for fields of type `sint32` and `sint64`:: -(parameters are the same as for `pb_enc_varint`_) + bool pb_encode_svarint(pb_ostream_t *stream, int64_t value); -The number is considered negative if the high-order bit of the value is set. On big endian computers, it is the highest bit of *\*src*. On little endian computers, it is the highest bit of *\*(src + field->data_size - 1)*. +(parameters are the same as for `pb_encode_varint`_ -pb_enc_fixed32 --------------- -Field encoder for PB_LTYPE_FIXED32. Writes the data in little endian order. On big endian computers, reverses the order of bytes. :: +pb_encode_string +---------------- +Writes the length of a string as varint and then contents of the string. Works for fields of type `bytes` and `string`:: - bool pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src); + bool pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size); :stream: Output stream to write to. -:field: Not used. -:src: Pointer to start of the field data. +:buffer: Pointer to string data. +:size: Number of bytes in the string. Pass `strlen(s)` for strings. :returns: True on success, false on IO error. -pb_enc_fixed64 --------------- -Field encoder for PB_LTYPE_FIXED64. Writes the data in little endian order. On big endian computers, reverses the order of bytes. :: - - bool pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src); - -(parameters are the same as for `pb_enc_fixed32`_) - -The same function is used for both integers and doubles. This breaks encoding of double values on architectures where they are mixed endian (primarily some arm processors with hardware FPU). - -pb_enc_bytes ------------- -Field encoder for PB_LTYPE_BYTES. Just calls `pb_encode_string`_. :: - - bool pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src); +pb_encode_fixed32 +----------------- +Writes 4 bytes to stream and swaps bytes on big-endian architectures. Works for fields of type `fixed32`, `sfixed32` and `float`:: -:stream: Output stream to write to. -:field: Not used. -:src: Pointer to a structure similar to pb_bytes_array_t. -:returns: True on success, false on IO error. + bool pb_encode_fixed32(pb_ostream_t *stream, const void *value); -This function expects a pointer to a structure with a *size_t* field at start, and a variable sized byte array after it. The platform-specific field offset is inferred from *pb_bytes_array_t*, which has a byte array of size 1. +:stream: Output stream to write to. +:value: Pointer to a 4-bytes large C variable, for example `uint32_t foo;`. +:returns: True on success, false on IO error. -pb_enc_string -------------- -Field encoder for PB_LTYPE_STRING. Determines size of string with strlen() and then calls `pb_encode_string`_. :: +pb_encode_fixed64 +----------------- +Writes 8 bytes to stream and swaps bytes on big-endian architecture. Works for fields of type `fixed64`, `sfixed64` and `double`:: - bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src); + bool pb_encode_fixed64(pb_ostream_t *stream, const void *value); -:stream: Output stream to write to. -:field: Not used. -:src: Pointer to a null-terminated string. -:returns: True on success, false on IO error. +:stream: Output stream to write to. +:value: Pointer to a 8-bytes large C variable, for example `uint64_t foo;`. +:returns: True on success, false on IO error. -pb_enc_submessage ------------------ -Field encoder for PB_LTYPE_SUBMESSAGE. Calls `pb_encode`_ to perform the actual encoding. :: +pb_encode_submessage +-------------------- +Encodes a submessage field, including the size header for it. Works for fields of any message type:: - bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src); + bool pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); :stream: Output stream to write to. -:field: Field description structure. The *ptr* field must be a pointer to a field description array for the submessage. +:fields: Pointer to the autogenerated field description array for the submessage type, e.g. `MyMessage_fields`. :src: Pointer to the structure where submessage data is. :returns: True on success, false on IO errors, pb_encode errors or if submessage size changes between calls. In Protocol Buffers format, the submessage size must be written before the submessage contents. Therefore, this function has to encode the submessage twice in order to know the size beforehand. -If the submessage contains callback fields, the callback function might misbehave and write out a different amount of data on the second call. This situation is recognized and *false* is returned, but it is up to the caller to ensure that the receiver of the message does not interpret it as valid data. +If the submessage contains callback fields, the callback function might misbehave and write out a different amount of data on the second call. This situation is recognized and *false* is returned, but garbage will be written to the output before the problem is detected. pb_decode.h =========== diff --git a/example/server.c b/example/server.c index aba0667b..9f27906d 100644 --- a/example/server.c +++ b/example/server.c @@ -38,7 +38,7 @@ bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, const void if (!pb_encode_tag_for_field(stream, field)) return false; - if (!pb_enc_submessage(stream, field, &fileinfo)) + if (!pb_encode_submessage(stream, FileInfo_fields, &fileinfo)) return false; } diff --git a/pb_encode.c b/pb_encode.c index 995eb3de..804c1a6a 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -3,6 +3,7 @@ * 2011 Petteri Aimonen */ +#define NANOPB_INTERNALS #include "pb.h" #include "pb_encode.h" #include @@ -220,6 +221,40 @@ bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value) return pb_write(stream, buffer, i); } +bool checkreturn pb_encode_svarint(pb_ostream_t *stream, int64_t value) +{ + uint64_t zigzagged; + if (value < 0) + zigzagged = ~(value << 1); + else + zigzagged = value << 1; + + return pb_encode_varint(stream, zigzagged); +} + +bool checkreturn pb_encode_fixed32(pb_ostream_t *stream, const void *value) +{ + #ifdef __BIG_ENDIAN__ + uint8_t *bytes = value; + uint8_t lebytes[4] = {bytes[3], bytes[2], bytes[1], bytes[0]}; + return pb_write(stream, lebytes, 4); + #else + return pb_write(stream, (uint8_t*)value, 4); + #endif +} + +bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value) +{ + #ifdef __BIG_ENDIAN__ + uint8_t *bytes[8] = value; + uint8_t lebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], + bytes[3], bytes[2], bytes[1], bytes[0]}; + return pb_write(stream, lebytes, 8); + #else + return pb_write(stream, (uint8_t*)value, 8); + #endif +} + bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number) { int tag = wiretype | (field_number << 3); @@ -265,71 +300,85 @@ bool checkreturn pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, s return pb_write(stream, buffer, size); } -/* Field encoders */ - -/* Copy srcsize bytes from src so that values are casted properly. - * On little endian machine, copy to start of dest - * On big endian machine, copy to end of dest - * destsize must always be larger than srcsize - * - * Note: This is the reverse of the endian_copy in pb_decode.c. - */ -static void endian_copy(void *dest, const void *src, size_t destsize, size_t srcsize) +bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) { -#ifdef __BIG_ENDIAN__ - memcpy((char*)dest + (destsize - srcsize), src, srcsize); -#else - memcpy(dest, src, srcsize); -#endif + /* First calculate the message size using a non-writing substream. */ + pb_ostream_t substream = {0}; + size_t size; + bool status; + + if (!pb_encode(&substream, fields, src_struct)) + return false; + + size = substream.bytes_written; + + if (!pb_encode_varint(stream, size)) + return false; + + if (stream->callback == NULL) + return pb_write(stream, NULL, size); /* Just sizing */ + + if (stream->bytes_written + size > stream->max_size) + return false; + + /* Use a substream to verify that a callback doesn't write more than + * what it did the first time. */ + substream.callback = stream->callback; + substream.state = stream->state; + substream.max_size = size; + substream.bytes_written = 0; + + status = pb_encode(&substream, fields, src_struct); + + stream->bytes_written += substream.bytes_written; + stream->state = substream.state; + + if (substream.bytes_written != size) + return false; + + return status; } +/* Field encoders */ + bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src) { uint64_t value = 0; - endian_copy(&value, src, sizeof(value), field->data_size); + + switch (field->data_size) + { + case 1: value = *(uint8_t*)src; break; + case 2: value = *(uint16_t*)src; break; + case 4: value = *(uint32_t*)src; break; + case 8: value = *(uint64_t*)src; break; + default: return false; + } + return pb_encode_varint(stream, value); } bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src) { uint64_t value = 0; - uint64_t zigzagged; - uint64_t signbitmask, xormask; - endian_copy(&value, src, sizeof(value), field->data_size); - signbitmask = (uint64_t)0x80 << (field->data_size * 8 - 8); - xormask = ((uint64_t)-1) >> (64 - field->data_size * 8); - if (value & signbitmask) - zigzagged = ((value ^ xormask) << 1) | 1; - else - zigzagged = value << 1; + switch (field->data_size) + { + case 4: value = *(int32_t*)src; break; + case 8: value = *(int64_t*)src; break; + default: return false; + } - return pb_encode_varint(stream, zigzagged); + return pb_encode_svarint(stream, value); } bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - #ifdef __BIG_ENDIAN__ - uint8_t bytes[8] = {0}; - memcpy(bytes, src, 8); - uint8_t lebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], - bytes[3], bytes[2], bytes[1], bytes[0]}; - return pb_write(stream, lebytes, 8); - #else - return pb_write(stream, (uint8_t*)src, 8); - #endif + return pb_encode_fixed64(stream, src); } bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - #ifdef __BIG_ENDIAN__ - uint8_t bytes[4] = {0}; - memcpy(bytes, src, 4); - uint8_t lebytes[4] = {bytes[3], bytes[2], bytes[1], bytes[0]}; - return pb_write(stream, lebytes, 4); - #else - return pb_write(stream, (uint8_t*)src, 4); - #endif + return pb_encode_fixed32(stream, src); } bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) @@ -345,42 +394,9 @@ bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, co bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - pb_ostream_t substream = {0}; - size_t size; - bool status; - if (field->ptr == NULL) return false; - if (!pb_encode(&substream, (pb_field_t*)field->ptr, src)) - return false; - - size = substream.bytes_written; - - if (!pb_encode_varint(stream, size)) - return false; - - if (stream->callback == NULL) - return pb_write(stream, NULL, size); /* Just sizing */ - - if (stream->bytes_written + size > stream->max_size) - return false; - - /* Use a substream to verify that a callback doesn't write more than - * what it did the first time. */ - substream.callback = stream->callback; - substream.state = stream->state; - substream.max_size = size; - substream.bytes_written = 0; - - status = pb_encode(&substream, (pb_field_t*)field->ptr, src); - - stream->bytes_written += substream.bytes_written; - stream->state = substream.state; - - if (substream.bytes_written != size) - return false; - - return status; + return pb_encode_submessage(stream, (pb_field_t*)field->ptr, src); } diff --git a/pb_encode.h b/pb_encode.h index cec39138..59ec554b 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -48,25 +48,56 @@ bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_ * You may want to use these from your caller or callbacks. */ -bool pb_encode_varint(pb_ostream_t *stream, uint64_t value); -bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number); -/* Encode tag based on LTYPE and field number defined in the field structure. */ +/* Encode field header based on LTYPE and field number defined in the field structure. + * Call this from the callback before writing out field contents. */ bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field); -/* Write length as varint and then the contents of buffer. */ + +/* Encode field header by manually specifing wire type. You need to use this if + * you want to write out packed arrays from a callback field. */ +bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number); + +/* Encode an integer in the varint format. + * This works for bool, enum, int32, int64, uint32 and uint64 field types. */ +bool pb_encode_varint(pb_ostream_t *stream, uint64_t value); + +/* Encode an integer in the zig-zagged svarint format. + * This works for sint32 and sint64. */ +bool pb_encode_svarint(pb_ostream_t *stream, int64_t value); + +/* Encode a string or bytes type field. For strings, pass strlen(s) as size. */ bool pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size); -/* --- Field encoders --- - * Each encoder writes the content for the field. - * The tag/wire type has been written already. +/* Encode a fixed32, sfixed32 or float value. + * You need to pass a pointer to a 4-byte wide C variable. */ +bool pb_encode_fixed32(pb_ostream_t *stream, const void *value); + +/* Encode a fixed64, sfixed64 or double value. + * You need to pass a pointer to a 8-byte wide C variable. */ +bool pb_encode_fixed64(pb_ostream_t *stream, const void *value); + +/* Encode a submessage field. + * You need to pass the pb_field_t array and pointer to struct, just like with pb_encode(). + * This internally encodes the submessage twice, first to calculate message size and then to actually write it out. + */ +bool pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); + +/* --- Internal functions --- + * These functions are not terribly useful for the average library user, but + * are exported to make the unit testing and extending nanopb easier. */ +#ifdef NANOPB_INTERNALS bool pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src); bool pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); bool pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src); bool pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src); - bool pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src); bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src); +#endif + +/* This function is not recommended for new programs. Use pb_encode_submessage() + * instead, it has the same functionality with a less confusing interface. */ bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src); + #endif diff --git a/tests/encode_unittests.c b/tests/encode_unittests.c index 166e6e83..9cdbc66e 100644 --- a/tests/encode_unittests.c +++ b/tests/encode_unittests.c @@ -1,3 +1,5 @@ +#define NANOPB_INTERNALS + #include #include #include "pb_encode.h" @@ -123,7 +125,6 @@ int main() uint8_t buffer[30]; pb_ostream_t s; uint8_t value = 1; - int8_t svalue = -1; int32_t max = INT32_MAX; int32_t min = INT32_MIN; int64_t lmax = INT64_MAX; @@ -132,8 +133,6 @@ int main() COMMENT("Test pb_enc_varint and pb_enc_svarint") TEST(WRITES(pb_enc_varint(&s, &field, &value), "\x01")); - TEST(WRITES(pb_enc_svarint(&s, &field, &svalue), "\x01")); - TEST(WRITES(pb_enc_svarint(&s, &field, &value), "\x02")); field.data_size = sizeof(max); TEST(WRITES(pb_enc_svarint(&s, &field, &max), "\xfe\xff\xff\xff\x0f")); diff --git a/tests/test_encode_callbacks.c b/tests/test_encode_callbacks.c index f0a046de..7fa017f9 100644 --- a/tests/test_encode_callbacks.c +++ b/tests/test_encode_callbacks.c @@ -29,7 +29,7 @@ bool encode_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *a return false; uint32_t value = 42; - return pb_enc_fixed32(stream, field, &value); + return pb_encode_fixed32(stream, &value); } bool encode_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *arg) @@ -38,7 +38,7 @@ bool encode_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *a return false; uint64_t value = 42; - return pb_enc_fixed64(stream, field, &value); + return pb_encode_fixed64(stream, &value); } int main() -- cgit 1.2.3-korg From d1ca88d20ec1b205752546a40ef520a392d0002f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 18 Apr 2012 20:15:36 +0300 Subject: Fixing compiler warnings, mostly related to unused parameters. Thanks to David Hotham for the patch. Fixes issue 8. --- pb.h | 5 +++++ pb_decode.c | 7 +++++-- pb_encode.c | 8 ++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/pb.h b/pb.h index ea94efc5..502958be 100644 --- a/pb.h +++ b/pb.h @@ -17,6 +17,11 @@ #define pb_packed #endif +/* Handly macro for suppressing unreferenced-parameter compiler warnings. */ +#ifndef UNUSED +#define UNUSED(x) (void)(x) +#endif + /* List of possible field types. These are used in the autogenerated code. * Least-significant 4 bits tell the scalar type * Most-significant 4 bits specify repeated/required/packed etc. diff --git a/pb_decode.c b/pb_decode.c index 1e2fea0a..fd234884 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -75,7 +75,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) { uint64_t temp; bool status = pb_decode_varint(stream, &temp); - *dest = temp; + *dest = (uint32_t)temp; return status; } @@ -448,6 +448,7 @@ static void endian_copy(void *dest, void *src, size_t destsize, size_t srcsize) #ifdef __BIG_ENDIAN__ memcpy(dest, (char*)src + (srcsize - destsize), destsize); #else + UNUSED(srcsize); memcpy(dest, src, destsize); #endif } @@ -480,6 +481,7 @@ bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, v } return status; #else + UNUSED(field); return pb_read(stream, (uint8_t*)dest, 4); #endif } @@ -496,6 +498,7 @@ bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, v } return status; #else + UNUSED(field); return pb_read(stream, (uint8_t*)dest, 8); #endif } @@ -524,7 +527,7 @@ bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, vo return false; /* Check length, noting the null terminator */ - if (size > field->data_size - 1) + if (size + 1 > field->data_size) return false; status = pb_read(stream, (uint8_t*)dest, size); diff --git a/pb_encode.c b/pb_encode.c index 804c1a6a..18173735 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -76,7 +76,7 @@ bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, const void *pData, size_t count, pb_encoder_t func) { - int i; + size_t i; const void *p; size_t size; @@ -212,7 +212,7 @@ bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value) while (value) { - buffer[i] = (value & 0x7F) | 0x80; + buffer[i] = (uint8_t)((value & 0x7F) | 0x80); value >>= 7; i++; } @@ -373,22 +373,26 @@ bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, c bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src) { + UNUSED(field); return pb_encode_fixed64(stream, src); } bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src) { + UNUSED(field); return pb_encode_fixed32(stream, src); } bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) { pb_bytes_array_t *bytes = (pb_bytes_array_t*)src; + UNUSED(field); return pb_encode_string(stream, bytes->bytes, bytes->size); } bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src) { + UNUSED(field); return pb_encode_string(stream, (uint8_t*)src, strlen((char*)src)); } -- cgit 1.2.3-korg From b94329088691538004f745a28e568d5ae1fba456 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 19 May 2012 21:15:52 +0300 Subject: Fix 64-bitness warnings in the example. Fixes issues 9 and 10. --- example/client.c | 2 +- example/common.c | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/example/client.c b/example/client.c index 95112e4c..9ad9c8c9 100644 --- a/example/client.c +++ b/example/client.c @@ -30,7 +30,7 @@ bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void *arg if (!pb_decode(stream, FileInfo_fields, &fileinfo)) return false; - printf("%-10lld %s\n", fileinfo.inode, fileinfo.name); + printf("%-10lld %s\n", (long long)fileinfo.inode, fileinfo.name); return true; } diff --git a/example/common.c b/example/common.c index 9d93219b..b27ccae2 100644 --- a/example/common.c +++ b/example/common.c @@ -10,13 +10,13 @@ static bool write_callback(pb_ostream_t *stream, const uint8_t *buf, size_t count) { - int fd = (int)stream->state; + int fd = (intptr_t)stream->state; return send(fd, buf, count, 0) == count; } static bool read_callback(pb_istream_t *stream, uint8_t *buf, size_t count) { - int fd = (int)stream->state; + int fd = (intptr_t)stream->state; int result; if (buf == NULL) @@ -38,12 +38,12 @@ static bool read_callback(pb_istream_t *stream, uint8_t *buf, size_t count) pb_ostream_t pb_ostream_from_socket(int fd) { - pb_ostream_t stream = {&write_callback, (void*)fd, SIZE_MAX, 0}; + pb_ostream_t stream = {&write_callback, (void*)(intptr_t)fd, SIZE_MAX, 0}; return stream; } pb_istream_t pb_istream_from_socket(int fd) { - pb_istream_t stream = {&read_callback, (void*)fd, SIZE_MAX}; + pb_istream_t stream = {&read_callback, (void*)(intptr_t)fd, SIZE_MAX}; return stream; } -- cgit 1.2.3-korg From f9501ca18589ff7d1983a9d79400776062ae4b71 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 19 May 2012 21:25:10 +0300 Subject: Add better error messages in the generator when libraries cannot be imported. Fixes issue #5. --- generator/nanopb_generator.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 2cc92f01..39153839 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,27 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -import google.protobuf.descriptor_pb2 as descriptor -import nanopb_pb2 +try: + import google.protobuf.descriptor_pb2 as descriptor +except: + print + print "*************************************************************" + print "*** Could not import the Google protobuf Python libraries ***" + print "*** Try installing package 'python-protobuf' or similar. ***" + print "*************************************************************" + print + raise + +try: + import nanopb_pb2 +except: + print + print "***************************************************************" + print "*** Could not import the precompiled nanopb_pb2.py. ***" + print "*** Run 'make' in the 'generator' folder to update the file.***" + print "***************************************************************" + print + raise + import os.path # Values are tuple (c type, pb ltype) -- cgit 1.2.3-korg From 5af2c97ecd71af46cc45baefb7a7041b5b904efc Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 12 Jun 2012 17:51:08 +0300 Subject: Add descriptor.proto into the #include exclusion list --- generator/nanopb_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 39153839..d41efb88 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -442,7 +442,7 @@ if __name__ == '__main__': # List of .proto files that should not be included in the C header file # even if they are mentioned in the source .proto. - excludes = ['nanopb.proto'] + excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] dependencies = [d for d in fdesc.file[0].dependency if d not in excludes] header = open(headername, 'w') -- cgit 1.2.3-korg From 7e1059628c1d67f96c00781cf2f57c915feadde1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 13 Jun 2012 21:43:40 +0300 Subject: Fix non-constant initializer errors with some compilers. Fixes issue #13. Thanks to Kevin Worth for reporting. --- pb_decode.c | 19 ++++++++++++++----- pb_encode.c | 17 ++++++++++++++--- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index fd234884..eace906f 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -476,8 +476,11 @@ bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, v uint8_t bytes[4] = {0}; bool status = pb_read(stream, bytes, 4); if (status) { - uint8_t bebytes[4] = {bytes[3], bytes[2], bytes[1], bytes[0]}; - memcpy(dest, bebytes, 4); + uint8_t *d = (uint8_t*)dest; + d[0] = bytes[3]; + d[1] = bytes[2]; + d[2] = bytes[1]; + d[3] = bytes[0]; } return status; #else @@ -492,9 +495,15 @@ bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, v uint8_t bytes[8] = {0}; bool status = pb_read(stream, bytes, 8); if (status) { - uint8_t bebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], - bytes[3], bytes[2], bytes[1], bytes[0]}; - memcpy(dest, bebytes, 8); + uint8_t *d = (uint8_t*)dest; + d[0] = bytes[7]; + d[1] = bytes[6]; + d[2] = bytes[5]; + d[3] = bytes[4]; + d[4] = bytes[3]; + d[5] = bytes[2]; + d[6] = bytes[1]; + d[7] = bytes[0]; } return status; #else diff --git a/pb_encode.c b/pb_encode.c index 18173735..c6cccf21 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -236,7 +236,11 @@ bool checkreturn pb_encode_fixed32(pb_ostream_t *stream, const void *value) { #ifdef __BIG_ENDIAN__ uint8_t *bytes = value; - uint8_t lebytes[4] = {bytes[3], bytes[2], bytes[1], bytes[0]}; + uint8_t lebytes[4]; + lebytes[0] = bytes[3]; + lebytes[1] = bytes[2]; + lebytes[2] = bytes[1]; + lebytes[3] = bytes[0]; return pb_write(stream, lebytes, 4); #else return pb_write(stream, (uint8_t*)value, 4); @@ -247,8 +251,15 @@ bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value) { #ifdef __BIG_ENDIAN__ uint8_t *bytes[8] = value; - uint8_t lebytes[8] = {bytes[7], bytes[6], bytes[5], bytes[4], - bytes[3], bytes[2], bytes[1], bytes[0]}; + uint8_t lebytes[8]; + lebytes[0] = bytes[7]; + lebytes[1] = bytes[6]; + lebytes[2] = bytes[5]; + lebytes[3] = bytes[4]; + lebytes[4] = bytes[3]; + lebytes[5] = bytes[2]; + lebytes[6] = bytes[1]; + lebytes[7] = bytes[0]; return pb_write(stream, lebytes, 8); #else return pb_write(stream, (uint8_t*)value, 8); -- cgit 1.2.3-korg From e18352d50678005c9dbb3ac76913555f5317c81c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 16 Jun 2012 14:07:37 +0300 Subject: Added new functions to public interface in pb_decode.h. pb_decode_tag and pb_skip_field allow manually iterating the fields in a message. --- docs/reference.rst | 47 ++++++++++++++++++++++++++++++++++++----------- pb_decode.c | 52 ++++++++++++++++++++++++++++++++++------------------ pb_decode.h | 3 +++ 3 files changed, 73 insertions(+), 29 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 4021c764..8846ce1b 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -283,6 +283,25 @@ Read data from input stream. Always use this function, don't try to call the str End of file is signalled by *stream->bytes_left* being zero after pb_read returns false. +pb_decode +--------- +Read and decode all fields of a structure. Reads until EOF on input stream. :: + + bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +:stream: Input stream to read from. +:fields: A field description array. Usually autogenerated. +:dest_struct: Pointer to structure where data will be stored. +:returns: True on success, false on IO error, on detectable errors in field description, if a field encoder returns false or if a required field is missing. + +In Protocol Buffers binary format, EOF is only allowed between fields. If it happens anywhere else, pb_decode will return *false*. If pb_decode returns false, you cannot trust any of the data in the structure. + +In addition to EOF, the pb_decode implementation supports terminating a message with a 0 byte. This is compatible with the official Protocol Buffers because 0 is never a valid field tag. + +For optional fields, this function applies the default value and sets *has_* to false if the field is not present. + +Because of memory concerns, the detection of missing required fields is not perfect if the structure contains more than 32 fields. + pb_decode_varint ---------------- Read and decode a varint_ encoded integer. :: @@ -311,24 +330,30 @@ Skip a varint-length-prefixed string. This means skipping a value with wire type :stream: Input stream to read from. :returns: True on success, false on IO error or length exceeding uint32_t. -pb_decode ---------- -Read and decode all fields of a structure. Reads until EOF on input stream. :: +pb_decode_tag +------------- +Decode the tag that comes before field in the protobuf encoding:: - bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + bool pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, int *tag, bool *eof); :stream: Input stream to read from. -:fields: A field description array. Usually autogenerated. -:dest_struct: Pointer to structure where data will be stored. -:returns: True on success, false on IO error, on detectable errors in field description, if a field encoder returns false or if a required field is missing. +:wire_type: Pointer to variable where to store the wire type of the field. +:tag: Pointer to variable where to store the tag of the field. +:eof: Pointer to variable where to store end-of-file status. +:returns: True on success, false on error or EOF. -In Protocol Buffers binary format, EOF is only allowed between fields. If it happens anywhere else, pb_decode will return *false*. If pb_decode returns false, you cannot trust any of the data in the structure. +When the message (stream) ends, this function will return false and set *eof* to true. On other +errors, *eof* will be set to false. -In addition to EOF, the pb_decode implementation supports terminating a message with a 0 byte. This is compatible with the official Protocol Buffers because 0 is never a valid field tag. +pb_skip_field +------------- +Remove the data for a field from the stream, without actually decoding it:: -For optional fields, this function applies the default value and sets *has_* to false if the field is not present. + bool pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type); -Because of memory concerns, the detection of missing required fields is not perfect if the structure contains more than 32 fields. +:stream: Input stream to read from. +:wire_type: Type of field to skip. +:returns: True on success, false on IO error. .. sidebar:: Field decoders diff --git a/pb_decode.c b/pb_decode.c index eace906f..50a11c40 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -117,12 +117,33 @@ bool checkreturn pb_skip_string(pb_istream_t *stream) return pb_read(stream, NULL, length); } -/* Currently the wire type related stuff is kept hidden from - * callbacks. They shouldn't need it. It's better for performance - * to just assume the correct type and fail safely on corrupt message. - */ +bool checkreturn pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, int *tag, bool *eof) +{ + uint32_t temp; + *eof = false; + *wire_type = 0; + *tag = 0; + + if (!pb_decode_varint32(stream, &temp)) + { + if (stream->bytes_left == 0) + *eof = true; + + return false; + } + + if (temp == 0) + { + *eof = true; /* Special feature: allow 0-terminated messages. */ + return false; + } + + *tag = temp >> 3; + *wire_type = (pb_wire_type_t)(temp & 7); + return true; +} -static bool checkreturn skip(pb_istream_t *stream, pb_wire_type_t wire_type) +bool checkreturn pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type) { switch (wire_type) { @@ -292,7 +313,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t pb_callback_t *pCallback = (pb_callback_t*)iter->pData; if (pCallback->funcs.decode == NULL) - return skip(stream, wire_type); + return pb_skip_field(stream, wire_type); if (wire_type == PB_WT_STRING) { @@ -392,27 +413,22 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void while (stream->bytes_left) { - uint32_t temp; int tag; pb_wire_type_t wire_type; - if (!pb_decode_varint32(stream, &temp)) + bool eof; + + if (!pb_decode_tag(stream, &wire_type, &tag, &eof)) { - if (stream->bytes_left == 0) - break; /* It was EOF */ + if (eof) + break; else - return false; /* It was error */ + return false; } - if (temp == 0) - break; /* Special feature: allow 0-terminated messages. */ - - tag = temp >> 3; - wire_type = (pb_wire_type_t)(temp & 7); - if (!pb_field_find(&iter, tag)) { /* No match found, skip data */ - if (!skip(stream, wire_type)) + if (!pb_skip_field(stream, wire_type)) return false; continue; } diff --git a/pb_decode.h b/pb_decode.h index f12b1900..0abb342c 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -48,6 +48,9 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc * You may want to use these from your caller or callbacks. */ +bool pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, int *tag, bool *eof); +bool pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type); + bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest); bool pb_skip_varint(pb_istream_t *stream); -- cgit 1.2.3-korg From 0f1d5cca59a2fddcf6bc627bb35e207bf3889547 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 16 Jun 2012 14:08:40 +0300 Subject: Added example on how to handle unions. --- example_unions/Makefile | 17 ++++++++ example_unions/decode.c | 92 +++++++++++++++++++++++++++++++++++++++++ example_unions/encode.c | 85 +++++++++++++++++++++++++++++++++++++ example_unions/unionproto.proto | 30 ++++++++++++++ 4 files changed, 224 insertions(+) create mode 100644 example_unions/Makefile create mode 100644 example_unions/decode.c create mode 100644 example_unions/encode.c create mode 100644 example_unions/unionproto.proto diff --git a/example_unions/Makefile b/example_unions/Makefile new file mode 100644 index 00000000..29514ca9 --- /dev/null +++ b/example_unions/Makefile @@ -0,0 +1,17 @@ +CFLAGS=-ansi -Wall -Werror -I .. -g -O0 +DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h + +all: encode decode + ./encode 1 | ./decode + ./encode 2 | ./decode + ./encode 3 | ./decode + +clean: + rm -f encode unionproto.pb.h unionproto.pb.c + +%: %.c $(DEPS) unionproto.pb.h unionproto.pb.c + $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c unionproto.pb.c + +unionproto.pb.h unionproto.pb.c: unionproto.proto ../generator/nanopb_generator.py + protoc -I. -I../generator -I/usr/include -ounionproto.pb $< + python ../generator/nanopb_generator.py unionproto.pb diff --git a/example_unions/decode.c b/example_unions/decode.c new file mode 100644 index 00000000..b20df84e --- /dev/null +++ b/example_unions/decode.c @@ -0,0 +1,92 @@ +/* This program reads a message from stdin, detects its type and decodes it. + */ + +#include +#include +#include + +#include +#include "unionproto.pb.h" + +/* This function reads manually the first tag from the stream and finds the + * corresponding message type. It doesn't yet decode the actual message. + * + * Returns a pointer to the MsgType_fields array, as an identifier for the + * message type. Returns null if the tag is of unknown type or an error occurs. + */ +const pb_field_t* decode_unionmessage_type(pb_istream_t *stream) +{ + pb_wire_type_t wire_type; + int tag; + bool eof; + + while (pb_decode_tag(stream, &wire_type, &tag, &eof)) + { + if (wire_type == PB_WT_STRING) + { + const pb_field_t *field; + for (field = UnionMessage_fields; field->tag != 0; field++) + { + if (field->tag == tag && (field->type & PB_LTYPE_SUBMESSAGE)) + { + /* Found our field. */ + return field->ptr; + } + } + } + + /* Wasn't our field.. */ + pb_skip_field(stream, wire_type); + } + + return NULL; +} + +bool decode_unionmessage_contents(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + pb_field_t field = {}; /* NB: Could get rid of this wrapper by fixing issue #2. */ + field.ptr = fields; + + return pb_dec_submessage(stream, &field, dest_struct); +} + +int main() +{ + /* Read the data into buffer */ + uint8_t buffer[512]; + size_t count = fread(buffer, 1, sizeof(buffer), stdin); + pb_istream_t stream = pb_istream_from_buffer(buffer, count); + + const pb_field_t *type = decode_unionmessage_type(&stream); + bool status = false; + + if (type == MsgType1_fields) + { + MsgType1 msg = {}; + status = decode_unionmessage_contents(&stream, MsgType1_fields, &msg); + printf("Got MsgType1: %d\n", msg.value); + } + else if (type == MsgType2_fields) + { + MsgType2 msg = {}; + status = decode_unionmessage_contents(&stream, MsgType2_fields, &msg); + printf("Got MsgType2: %s\n", msg.value ? "true" : "false"); + } + else if (type == MsgType3_fields) + { + MsgType3 msg = {}; + status = decode_unionmessage_contents(&stream, MsgType3_fields, &msg); + printf("Got MsgType3: %d %d\n", msg.value1, msg.value2); + } + + if (!status) + { + printf("Decoding failed.\n"); + return 1; + } + + return 0; +} + + + diff --git a/example_unions/encode.c b/example_unions/encode.c new file mode 100644 index 00000000..e124bf91 --- /dev/null +++ b/example_unions/encode.c @@ -0,0 +1,85 @@ +/* This program takes a command line argument and encodes a message in + * one of MsgType1, MsgType2 or MsgType3. + */ + +#include +#include +#include + +#include +#include "unionproto.pb.h" + +/* This function is the core of the union encoding process. It handles + * the top-level pb_field_t array manually, in order to encode a correct + * field tag before the message. The pointer to MsgType_fields array is + * used as an unique identifier for the message type. + */ +bool encode_unionmessage(pb_ostream_t *stream, const pb_field_t messagetype[], const void *message) +{ + const pb_field_t *field; + for (field = UnionMessage_fields; field->tag != 0; field++) + { + if (field->ptr == messagetype) + { + /* This is our field, encode the message using it. */ + if (!pb_encode_tag_for_field(stream, field)) + return false; + + return pb_encode_submessage(stream, messagetype, message); + } + } + + /* Didn't find the field for messagetype */ + return false; +} + +int main(int argc, char **argv) +{ + if (argc != 2) + { + fprintf(stderr, "Usage: %s (1|2|3)\n", argv[0]); + return 1; + } + + uint8_t buffer[512]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + bool status = false; + int msgtype = atoi(argv[1]); + if (msgtype == 1) + { + /* Send message of type 1 */ + MsgType1 msg = {42}; + status = encode_unionmessage(&stream, MsgType1_fields, &msg); + } + else if (msgtype == 2) + { + /* Send message of type 2 */ + MsgType2 msg = {true}; + status = encode_unionmessage(&stream, MsgType2_fields, &msg); + } + else if (msgtype == 3) + { + /* Send message of type 3 */ + MsgType3 msg = {3, 1415}; + status = encode_unionmessage(&stream, MsgType3_fields, &msg); + } + else + { + fprintf(stderr, "Unknown message type: %d\n", msgtype); + return 2; + } + + if (!status) + { + fprintf(stderr, "Encoding failed!\n"); + return 3; + } + else + { + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } +} + + diff --git a/example_unions/unionproto.proto b/example_unions/unionproto.proto new file mode 100644 index 00000000..d7c9de2d --- /dev/null +++ b/example_unions/unionproto.proto @@ -0,0 +1,30 @@ +// This is an example of how to handle 'union' style messages +// with nanopb, without allocating memory for all the message types. +// +// There is no official type in Protocol Buffers for describing unions, +// but they are commonly implemented by filling out exactly one of +// several optional fields. + +message MsgType1 +{ + required int32 value = 1; +} + +message MsgType2 +{ + required bool value = 1; +} + +message MsgType3 +{ + required int32 value1 = 1; + required int32 value2 = 2; +} + +message UnionMessage +{ + optional MsgType1 msg1 = 1; + optional MsgType2 msg2 = 2; + optional MsgType3 msg3 = 3; +} + -- cgit 1.2.3-korg From 3a919ddc5e13091f7212d9369e1794c038d6cdfe Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 20 Jun 2012 21:31:23 +0300 Subject: Fix error when .proto contains no Messages (e.g. just enums). Thanks to Paul Fertser for reporting this bug. --- generator/nanopb_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index d41efb88..1e7e1f98 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -343,7 +343,7 @@ def toposort2(data): ''' for k, v in data.items(): v.discard(k) # Ignore self dependencies - extra_items_in_deps = reduce(set.union, data.values()) - set(data.keys()) + extra_items_in_deps = reduce(set.union, data.values(), set()) - set(data.keys()) data.update(dict([(item, set()) for item in extra_items_in_deps])) while True: ordered = set(item for item,dep in data.items() if not dep) -- cgit 1.2.3-korg From c07e576de86644d278a987550e389b000d8311d0 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 25 Jun 2012 21:45:40 +0300 Subject: Note about __BIG_ENDIAN__ compilation option --- docs/reference.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/reference.rst b/docs/reference.rst index 8846ce1b..8c1af773 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -6,6 +6,12 @@ Nanopb: API reference .. contents :: +Compilation options +=================== +The following options can be specified using -D switch given to the C compiler: + +:__BIG_ENDIAN__: Set this if your platform stores integers and floats in big-endian format. Mixed-endian systems (different layout for ints and floats) are currently not supported. + pb.h ==== -- cgit 1.2.3-korg From 99375a132fe91bd3a333eeacff7888c7b814a30b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 25 Jun 2012 22:08:05 +0300 Subject: Documented NANOPB_INTERNALS compilation option --- docs/reference.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference.rst b/docs/reference.rst index 8c1af773..42e7c768 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -11,6 +11,7 @@ Compilation options The following options can be specified using -D switch given to the C compiler: :__BIG_ENDIAN__: Set this if your platform stores integers and floats in big-endian format. Mixed-endian systems (different layout for ints and floats) are currently not supported. +:NANOPB_INTERNALS: Set this to expose the field encoder functions that are hidden since nanopb-0.1.3. pb.h ==== -- cgit 1.2.3-korg From 7bcf7ef5796c66125a2a4df1317b55a27adedd89 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 25 Jun 2012 22:17:13 +0300 Subject: Fixed formatting in docs --- docs/reference.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 42e7c768..f4957804 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -10,8 +10,11 @@ Compilation options =================== The following options can be specified using -D switch given to the C compiler: -:__BIG_ENDIAN__: Set this if your platform stores integers and floats in big-endian format. Mixed-endian systems (different layout for ints and floats) are currently not supported. -:NANOPB_INTERNALS: Set this to expose the field encoder functions that are hidden since nanopb-0.1.3. +================== ============================================================================================== +__BIG_ENDIAN__ Set this if your platform stores integers and floats in big-endian format. + Mixed-endian systems (different layout for ints and floats) are currently not supported. +NANOPB_INTERNALS Set this to expose the field encoder functions that are hidden since nanopb-0.1.3. +================== ============================================================================================== pb.h ==== -- cgit 1.2.3-korg From 95eb4a549981dc556dd30c5d76a6b437bb5ed06d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 30 Jun 2012 18:10:08 +0300 Subject: Improve the detection of missing required fields. Now the limit of tracked fields is configurable at compile-time using PB_MAX_REQUIRED_FIELDS. Added related test and updated documentation. Fixes issue #18. --- docs/reference.rst | 13 ++--- pb.h | 6 ++ pb_decode.c | 40 ++++++++----- tests/Makefile | 7 ++- tests/missing_fields.proto | 138 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_missing_fields.c | 49 ++++++++++++++++ 6 files changed, 229 insertions(+), 24 deletions(-) create mode 100644 tests/missing_fields.proto create mode 100644 tests/test_missing_fields.c diff --git a/docs/reference.rst b/docs/reference.rst index f4957804..81251fe6 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -10,11 +10,12 @@ Compilation options =================== The following options can be specified using -D switch given to the C compiler: -================== ============================================================================================== -__BIG_ENDIAN__ Set this if your platform stores integers and floats in big-endian format. - Mixed-endian systems (different layout for ints and floats) are currently not supported. -NANOPB_INTERNALS Set this to expose the field encoder functions that are hidden since nanopb-0.1.3. -================== ============================================================================================== +============================ ============================================================================================== +__BIG_ENDIAN__ Set this if your platform stores integers and floats in big-endian format. + Mixed-endian systems (different layout for ints and floats) are currently not supported. +NANOPB_INTERNALS Set this to expose the field encoder functions that are hidden since nanopb-0.1.3. +PB_MAX_REQUIRED_FIELDS Maximum number of required fields to check for presence. Default value is 64. +============================ ============================================================================================== pb.h ==== @@ -310,8 +311,6 @@ In addition to EOF, the pb_decode implementation supports terminating a message For optional fields, this function applies the default value and sets *has_* to false if the field is not present. -Because of memory concerns, the detection of missing required fields is not perfect if the structure contains more than 32 fields. - pb_decode_varint ---------------- Read and decode a varint_ encoded integer. :: diff --git a/pb.h b/pb.h index 502958be..fc74dbd4 100644 --- a/pb.h +++ b/pb.h @@ -22,6 +22,12 @@ #define UNUSED(x) (void)(x) #endif +/* Number of required fields to keep track of + * (change here or on compiler command line). */ +#ifndef PB_MAX_REQUIRED_FIELDS +#define PB_MAX_REQUIRED_FIELDS 64 +#endif + /* List of possible field types. These are used in the autogenerated code. * Least-significant 4 bits tell the scalar type * Most-significant 4 bits specify repeated/required/packed etc. diff --git a/pb_decode.c b/pb_decode.c index 50a11c40..458d5f1e 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -206,18 +206,20 @@ static bool checkreturn make_string_substream(pb_istream_t *stream, pb_istream_t /* Iterator for pb_field_t list */ typedef struct { - const pb_field_t *start; - const pb_field_t *current; - int field_index; - void *dest_struct; - void *pData; - void *pSize; + const pb_field_t *start; /* Start of the pb_field_t array */ + const pb_field_t *current; /* Current position of the iterator */ + int field_index; /* Zero-based index of the field. */ + int required_field_index; /* Zero-based index that counts only the required fields */ + void *dest_struct; /* Pointer to the destination structure to decode to */ + void *pData; /* Pointer where to store current field value */ + void *pSize; /* Pointer where to store the size of current array field */ } pb_field_iterator_t; static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct) { iter->start = iter->current = fields; iter->field_index = 0; + iter->required_field_index = 0; iter->pData = (char*)dest_struct + iter->current->data_offset; iter->pSize = (char*)iter->pData + iter->current->size_offset; iter->dest_struct = dest_struct; @@ -231,12 +233,16 @@ static bool pb_field_next(pb_field_iterator_t *iter) if (PB_HTYPE(iter->current->type) == PB_HTYPE_ARRAY) prev_size *= iter->current->array_size; + if (PB_HTYPE(iter->current->type) == PB_HTYPE_REQUIRED) + iter->required_field_index++; + iter->current++; iter->field_index++; if (iter->current->tag == 0) { iter->current = iter->start; iter->field_index = 0; + iter->required_field_index = 0; iter->pData = iter->dest_struct; prev_size = 0; notwrapped = false; @@ -403,9 +409,8 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { - uint32_t fields_seen = 0; /* Used to check for required fields */ + uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {}; /* Used to check for required fields */ pb_field_iterator_t iter; - int i; pb_message_set_to_defaults(fields, dest_struct); @@ -433,21 +438,26 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void continue; } - fields_seen |= 1 << (iter.field_index & 31); + if (PB_HTYPE(iter.current->type) == PB_HTYPE_REQUIRED + && iter.required_field_index < PB_MAX_REQUIRED_FIELDS) + { + fields_seen[iter.required_field_index >> 3] |= 1 << (iter.required_field_index & 7); + } if (!decode_field(stream, wire_type, &iter)) return false; } - /* Check that all required fields (mod 31) were present. */ - for (i = 0; fields[i].tag != 0; i++) - { - if (PB_HTYPE(fields[i].type) == PB_HTYPE_REQUIRED && - !(fields_seen & (1 << (i & 31)))) + /* Check that all required fields were present. */ + pb_field_init(&iter, fields, dest_struct); + do { + if (PB_HTYPE(iter.current->type) == PB_HTYPE_REQUIRED && + iter.required_field_index < PB_MAX_REQUIRED_FIELDS && + !(fields_seen[iter.required_field_index >> 3] & (1 << (iter.required_field_index & 7)))) { return false; } - } + } while (pb_field_next(&iter)); return true; } diff --git a/tests/Makefile b/tests/Makefile index 8e2df140..24e9ba62 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,6 +1,6 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage LDFLAGS=--coverage -DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h +DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h missing_fields.pb.h TESTS=test_decode1 test_encode1 decode_unittests encode_unittests all: breakpoints $(TESTS) run_unittests @@ -25,6 +25,7 @@ test_encode2: test_encode2.o pb_encode.o person.pb.o test_encode3: test_encode3.o pb_encode.o alltypes.pb.o test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o +test_missing_fields: test_missing_fields.o pb_encode.o pb_decode.o missing_fields.pb.o decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o encode_unittests: encode_unittests.o pb_encode.o unittestproto.pb.o @@ -41,7 +42,7 @@ coverage: run_unittests gcov pb_encode.gcda gcov pb_decode.gcda -run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks +run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields rm -f *.gcda ./decode_unittests > /dev/null @@ -61,6 +62,8 @@ run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_ ./test_encode3 | ./test_decode3 ./test_encode3 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null + + ./test_missing_fields run_fuzztest: test_decode2 bash -c 'I=1; while true; do cat /dev/urandom | ./test_decode2 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done' diff --git a/tests/missing_fields.proto b/tests/missing_fields.proto new file mode 100644 index 00000000..cbb23ba1 --- /dev/null +++ b/tests/missing_fields.proto @@ -0,0 +1,138 @@ +/* Test for one missing field among many */ + +message AllFields +{ + required int32 field1 = 1; + required int32 field2 = 2; + required int32 field3 = 3; + required int32 field4 = 4; + required int32 field5 = 5; + required int32 field6 = 6; + required int32 field7 = 7; + required int32 field8 = 8; + required int32 field9 = 9; + required int32 field10 = 10; + required int32 field11 = 11; + required int32 field12 = 12; + required int32 field13 = 13; + required int32 field14 = 14; + required int32 field15 = 15; + required int32 field16 = 16; + required int32 field17 = 17; + required int32 field18 = 18; + required int32 field19 = 19; + required int32 field20 = 20; + required int32 field21 = 21; + required int32 field22 = 22; + required int32 field23 = 23; + required int32 field24 = 24; + required int32 field25 = 25; + required int32 field26 = 26; + required int32 field27 = 27; + required int32 field28 = 28; + required int32 field29 = 29; + required int32 field30 = 30; + required int32 field31 = 31; + required int32 field32 = 32; + required int32 field33 = 33; + required int32 field34 = 34; + required int32 field35 = 35; + required int32 field36 = 36; + required int32 field37 = 37; + required int32 field38 = 38; + required int32 field39 = 39; + required int32 field40 = 40; + required int32 field41 = 41; + required int32 field42 = 42; + required int32 field43 = 43; + required int32 field44 = 44; + required int32 field45 = 45; + required int32 field46 = 46; + required int32 field47 = 47; + required int32 field48 = 48; + required int32 field49 = 49; + required int32 field50 = 50; + required int32 field51 = 51; + required int32 field52 = 52; + required int32 field53 = 53; + required int32 field54 = 54; + required int32 field55 = 55; + required int32 field56 = 56; + required int32 field57 = 57; + required int32 field58 = 58; + required int32 field59 = 59; + required int32 field60 = 60; + required int32 field61 = 61; + required int32 field62 = 62; + required int32 field63 = 63; + required int32 field64 = 64; +} + +message MissingField +{ + required int32 field1 = 1; + required int32 field2 = 2; + required int32 field3 = 3; + required int32 field4 = 4; + required int32 field5 = 5; + required int32 field6 = 6; + required int32 field7 = 7; + required int32 field8 = 8; + required int32 field9 = 9; + required int32 field10 = 10; + required int32 field11 = 11; + required int32 field12 = 12; + required int32 field13 = 13; + required int32 field14 = 14; + required int32 field15 = 15; + required int32 field16 = 16; + required int32 field17 = 17; + required int32 field18 = 18; + required int32 field19 = 19; + required int32 field20 = 20; + required int32 field21 = 21; + required int32 field22 = 22; + required int32 field23 = 23; + required int32 field24 = 24; + required int32 field25 = 25; + required int32 field26 = 26; + required int32 field27 = 27; + required int32 field28 = 28; + required int32 field29 = 29; + required int32 field30 = 30; + required int32 field31 = 31; + required int32 field32 = 32; + required int32 field33 = 33; + required int32 field34 = 34; + required int32 field35 = 35; + required int32 field36 = 36; + required int32 field37 = 37; + required int32 field38 = 38; + required int32 field39 = 39; + required int32 field40 = 40; + required int32 field41 = 41; + required int32 field42 = 42; + required int32 field43 = 43; + required int32 field44 = 44; + required int32 field45 = 45; + required int32 field46 = 46; + required int32 field47 = 47; + required int32 field48 = 48; + required int32 field49 = 49; + required int32 field50 = 50; + required int32 field51 = 51; + required int32 field52 = 52; + required int32 field53 = 53; + required int32 field54 = 54; + required int32 field55 = 55; + required int32 field56 = 56; + required int32 field57 = 57; + required int32 field58 = 58; + required int32 field59 = 59; + required int32 field60 = 60; + required int32 field61 = 61; + required int32 field62 = 62; +/* required int32 field63 = 63; */ + required int32 field64 = 64; +} + diff --git a/tests/test_missing_fields.c b/tests/test_missing_fields.c new file mode 100644 index 00000000..46cd7d96 --- /dev/null +++ b/tests/test_missing_fields.c @@ -0,0 +1,49 @@ +/* Checks that missing required fields are detected properly */ + +#include +#include +#include +#include "missing_fields.pb.h" + +int main() +{ + uint8_t buffer[512] = {}; + + /* Create a message with one missing field */ + { + MissingField msg = {}; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + if (!pb_encode(&stream, MissingField_fields, &msg)) + { + printf("Encode failed.\n"); + return 1; + } + } + + /* Test that it decodes properly if we don't require that field */ + { + MissingField msg = {}; + pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer)); + + if (!pb_decode(&stream, MissingField_fields, &msg)) + { + printf("Decode failed.\n"); + return 2; + } + } + + /* Test that it does *not* decode properly if we require the field */ + { + AllFields msg = {}; + pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer)); + + if (pb_decode(&stream, AllFields_fields, &msg)) + { + printf("Decode didn't detect missing field.\n"); + return 3; + } + } + + return 0; /* All ok */ +} + -- cgit 1.2.3-korg From 67add3259a6429cb4afc1bdb6345e8c057a55ca8 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 30 Jun 2012 18:23:18 +0300 Subject: Warn if PB_MAX_REQUIRED_FIELDS is not large enough. --- generator/nanopb_generator.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 1e7e1f98..730c0aab 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -402,6 +402,16 @@ def generate_header(dependencies, headername, enums, messages): for msg in messages: yield msg.fields_declaration() + '\n' + count_required_fields = lambda m: len([f for f in msg.fields if f.htype == 'PB_HTYPE_REQUIRED']) + largest_msg = max(messages, key = count_required_fields) + largest_count = count_required_fields(largest_msg) + if largest_count > 64: + yield '\n/* Check that missing required fields will be properly detected */\n' + yield '#if PB_MAX_REQUIRED_FIELDS < %d\n' % largest_count + yield '#warning Properly detecting missing required fields in %s requires \\\n' % largest_msg.name + yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count + yield '#endif\n' + yield '\n#endif\n' def generate_source(headername, enums, messages): -- cgit 1.2.3-korg From 78086cc27d746a425f3f1130e822275bdb623090 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 30 Jun 2012 19:28:49 +0300 Subject: Add PB_MANY_FIELDS option for supporting fields > 255. Add generator warning if this is necessary. Fixes issue #14. --- docs/reference.rst | 4 +++- generator/nanopb_generator.py | 38 ++++++++++++++++++++++++++++++++++++++ pb.h | 15 +++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/docs/reference.rst b/docs/reference.rst index 81251fe6..aefc25f7 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -15,6 +15,8 @@ __BIG_ENDIAN__ Set this if your platform stores integers and flo Mixed-endian systems (different layout for ints and floats) are currently not supported. NANOPB_INTERNALS Set this to expose the field encoder functions that are hidden since nanopb-0.1.3. PB_MAX_REQUIRED_FIELDS Maximum number of required fields to check for presence. Default value is 64. +PB_MANY_FIELDS Add support for tag numbers > 255 and fields larger than 255 bytes or 255 array entries. + Increases code size 9 bytes per each field. Compiler error will tell if you need this. ============================ ============================================================================================== pb.h @@ -77,7 +79,7 @@ Describes a single structure field with memory position in relation to others. T :array_size: Maximum number of entries in an array, if it is an array type. :ptr: Pointer to default value for optional fields, or to submessage description for PB_LTYPE_SUBMESSAGE. -The *uint8_t* datatypes limit the maximum size of a single item to 255 bytes and arrays to 255 items. Compiler will warn "Initializer too large for type" if the limits are exceeded. The types can be changed to larger ones if necessary. +The *uint8_t* datatypes limit the maximum size of a single item to 255 bytes and arrays to 255 items. Compiler will give error if the values are too large. The types can be changed to larger ones by defining *PB_MANY_FIELDS*. pb_bytes_array_t ---------------- diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 730c0aab..405fedaf 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -251,6 +251,20 @@ class Field: result += '\n &%s_default}' % (self.struct_name + self.name) return result + + def needs_32bit_pb_field_t(self): + '''Determine if this field needs 32bit pb_field_t structure to compile properly. + Returns True, False or a C-expression for assert.''' + if self.tag > 255 or self.max_size > 255: + return True + + if self.ltype == 'PB_LTYPE_SUBMESSAGE': + if self.htype == 'PB_HTYPE_ARRAY': + return 'pb_membersize(%s, %s[0]) > 255' % (self.struct_name, self.name) + else: + return 'pb_membersize(%s, %s) > 255' % (self.struct_name, self.name) + + return False class Message: def __init__(self, names, desc): @@ -412,6 +426,30 @@ def generate_header(dependencies, headername, enums, messages): yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count yield '#endif\n' + worst = False + worst_field = '' + for msg in messages: + for field in msg.fields: + status = field.needs_32bit_pb_field_t() + if status == True: + worst = True + worst_field = str(field.struct_name) + '.' + str(field.name) + elif status != False: + if worst == False: + worst = status + elif worst != True: + worst += ' || ' + status + + if worst != False: + yield '\n/* Check that field information fits in pb_field_t */\n' + yield '#ifndef PB_MANY_FIELDS\n' + if worst == True: + yield '#error Field descriptor for %s is too large. Define PB_MANY_FIELDS to fix this.\n' % worst_field + else: + yield 'STATIC_ASSERT(!(%s), YOU_MUST_DEFINE_PB_MANY_FIELDS)\n' % worst + yield '#endif\n' + + # End of header yield '\n#endif\n' def generate_source(headername, enums, messages): diff --git a/pb.h b/pb.h index fc74dbd4..a81e9ef4 100644 --- a/pb.h +++ b/pb.h @@ -22,6 +22,11 @@ #define UNUSED(x) (void)(x) #endif +/* Compile-time assertion, used for checking compatible compilation options. */ +#ifndef STATIC_ASSERT +#define STATIC_ASSERT(COND,MSG) typedef char static_assertion_##MSG[(COND)?1:-1]; +#endif + /* Number of required fields to keep track of * (change here or on compiler command line). */ #ifndef PB_MAX_REQUIRED_FIELDS @@ -101,12 +106,22 @@ typedef enum { */ typedef struct _pb_field_t pb_field_t; struct _pb_field_t { + +#ifndef PB_MANY_FIELDS uint8_t tag; pb_type_t type; uint8_t data_offset; /* Offset of field data, relative to previous field. */ int8_t size_offset; /* Offset of array size or has-boolean, relative to data */ uint8_t data_size; /* Data size in bytes for a single item */ uint8_t array_size; /* Maximum number of entries in array */ +#else + uint32_t tag; + pb_type_t type; + uint8_t data_offset; + int8_t size_offset; + uint32_t data_size; + uint32_t array_size; +#endif /* Field definitions for submessage * OR default value for all other non-array, non-callback types -- cgit 1.2.3-korg From 9b1e1b440ab6a21bacab939b9c7bef0fa4ca5c90 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 1 Jul 2012 10:15:37 +0300 Subject: Replace PB_MANY_FIELDS with PB_FIELD_16BIT and PB_FIELD_32BIT. This allows more precise control over the memory use vs. field size. --- docs/reference.rst | 17 ++++++++---- generator/nanopb_generator.py | 63 +++++++++++++++++++++++++------------------ pb.h | 19 ++++++++++--- 3 files changed, 65 insertions(+), 34 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index aefc25f7..3331c6d3 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -10,14 +10,21 @@ Compilation options =================== The following options can be specified using -D switch given to the C compiler: -============================ ============================================================================================== +============================ ================================================================================================ __BIG_ENDIAN__ Set this if your platform stores integers and floats in big-endian format. Mixed-endian systems (different layout for ints and floats) are currently not supported. NANOPB_INTERNALS Set this to expose the field encoder functions that are hidden since nanopb-0.1.3. -PB_MAX_REQUIRED_FIELDS Maximum number of required fields to check for presence. Default value is 64. -PB_MANY_FIELDS Add support for tag numbers > 255 and fields larger than 255 bytes or 255 array entries. +PB_MAX_REQUIRED_FIELDS Maximum number of required fields to check for presence. Default value is 64. Increases stack + usage 1 byte per every 8 fields. Compiler warning will tell if you need this. +PB_FIELD_16BIT Add support for tag numbers > 255 and fields larger than 255 bytes or 255 array entries. + Increases code size 3 bytes per each field. Compiler error will tell if you need this. +PB_FIELD_32BIT Add support for tag numbers > 65535 and fields larger than 65535 bytes or 65535 array entries. Increases code size 9 bytes per each field. Compiler error will tell if you need this. -============================ ============================================================================================== +============================ ================================================================================================ + +The PB_MAX_REQUIRED_FIELDS, PB_FIELD_16BIT and PB_FIELD_32BIT settings allow raising some datatype limits to suit larger messages. +Their need is recognized automatically by C-preprocessor #if-directives in the generated .pb.h files. The default setting is to use +the smallest datatypes (least resources used). pb.h ==== @@ -79,7 +86,7 @@ Describes a single structure field with memory position in relation to others. T :array_size: Maximum number of entries in an array, if it is an array type. :ptr: Pointer to default value for optional fields, or to submessage description for PB_LTYPE_SUBMESSAGE. -The *uint8_t* datatypes limit the maximum size of a single item to 255 bytes and arrays to 255 items. Compiler will give error if the values are too large. The types can be changed to larger ones by defining *PB_MANY_FIELDS*. +The *uint8_t* datatypes limit the maximum size of a single item to 255 bytes and arrays to 255 items. Compiler will give error if the values are too large. The types can be changed to larger ones by defining *PB_FIELD_16BIT*. pb_bytes_array_t ---------------- diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 405fedaf..9e2a30ed 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -252,19 +252,17 @@ class Field: return result - def needs_32bit_pb_field_t(self): - '''Determine if this field needs 32bit pb_field_t structure to compile properly. - Returns True, False or a C-expression for assert.''' - if self.tag > 255 or self.max_size > 255: - return True - + def largest_field_value(self): + '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly. + Returns numeric value or a C-expression for assert.''' if self.ltype == 'PB_LTYPE_SUBMESSAGE': if self.htype == 'PB_HTYPE_ARRAY': - return 'pb_membersize(%s, %s[0]) > 255' % (self.struct_name, self.name) + return 'pb_membersize(%s, %s[0])' % (self.struct_name, self.name) else: - return 'pb_membersize(%s, %s) > 255' % (self.struct_name, self.name) - - return False + return 'pb_membersize(%s, %s)' % (self.struct_name, self.name) + + return max(self.tag, self.max_size, self.max_count) + class Message: def __init__(self, names, desc): @@ -426,28 +424,41 @@ def generate_header(dependencies, headername, enums, messages): yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count yield '#endif\n' - worst = False + worst = 0 worst_field = '' + checks = [] for msg in messages: for field in msg.fields: - status = field.needs_32bit_pb_field_t() - if status == True: - worst = True + status = field.largest_field_value() + if isinstance(status, (str, unicode)): + checks.append(status) + elif status > worst: + worst = status worst_field = str(field.struct_name) + '.' + str(field.name) - elif status != False: - if worst == False: - worst = status - elif worst != True: - worst += ' || ' + status - if worst != False: + if worst > 255 or checks: yield '\n/* Check that field information fits in pb_field_t */\n' - yield '#ifndef PB_MANY_FIELDS\n' - if worst == True: - yield '#error Field descriptor for %s is too large. Define PB_MANY_FIELDS to fix this.\n' % worst_field - else: - yield 'STATIC_ASSERT(!(%s), YOU_MUST_DEFINE_PB_MANY_FIELDS)\n' % worst - yield '#endif\n' + yield '/* (Largest message has %d fields' % worst + if checks: yield ' and submessages have to be checked at compile-time.' + yield ') */\n' + + if worst < 65536: + yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n' + if worst > 255: + yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field + else: + assertion = ' && '.join(str(c) + ' < 256' for c in checks) + yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT)\n' % assertion + yield '#endif\n\n' + + if worst > 65535 or checks: + yield '#if !defined(PB_FIELD_32BIT)\n' + if worst > 65535: + yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field + else: + assertion = ' && '.join(str(c) + ' < 65536' for c in checks) + yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT)\n' % assertion + yield '#endif\n' # End of header yield '\n#endif\n' diff --git a/pb.h b/pb.h index a81e9ef4..d030d9e3 100644 --- a/pb.h +++ b/pb.h @@ -33,6 +33,11 @@ #define PB_MAX_REQUIRED_FIELDS 64 #endif +#if PB_MAX_REQUIRED_FIELDS < 64 +#warning You should not lower PB_MAX_REQUIRED_FIELDS from the default value (64). \ + The automatic checks against too low value will not be active. +#endif + /* List of possible field types. These are used in the autogenerated code. * Least-significant 4 bits tell the scalar type * Most-significant 4 bits specify repeated/required/packed etc. @@ -99,21 +104,29 @@ typedef enum { /* This structure is used in auto-generated constants * to specify struct fields. - * You can change field sizes here if you need structures + * You can change field sizes if you need structures * larger than 256 bytes or field tags larger than 256. * The compiler should complain if your .proto has such - * structures ("initializer too large for type"). + * structures. Fix that by defining PB_FIELD_16BIT or + * PB_FIELD_32BIT. */ typedef struct _pb_field_t pb_field_t; struct _pb_field_t { -#ifndef PB_MANY_FIELDS +#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) uint8_t tag; pb_type_t type; uint8_t data_offset; /* Offset of field data, relative to previous field. */ int8_t size_offset; /* Offset of array size or has-boolean, relative to data */ uint8_t data_size; /* Data size in bytes for a single item */ uint8_t array_size; /* Maximum number of entries in array */ +#elif defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) + uint16_t tag; + pb_type_t type; + uint8_t data_offset; + int8_t size_offset; + uint16_t data_size; + uint16_t array_size; #else uint32_t tag; pb_type_t type; -- cgit 1.2.3-korg From 7c5bb6541a820dd4721358226ca3815d94736863 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Mon, 2 Jul 2012 14:00:45 -0700 Subject: Cast enum vaules to integers and cast them back where appropraite to prevent mixed enumeration type compiler warnings --- generator/nanopb_generator.py | 6 +++--- pb.h | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 9e2a30ed..1925167f 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -217,10 +217,10 @@ class Field: prev_field_name is the name of the previous field or None. ''' result = ' {%d, ' % self.tag - result += self.htype + result += '(pb_type_t) ((int) ' + self.htype if self.ltype is not None: - result += ' | ' + self.ltype - result += ',\n' + result += ' | (int) ' + self.ltype + result += '),\n' if prev_field_name is None: result += ' offsetof(%s, %s),' % (self.struct_name, self.name) diff --git a/pb.h b/pb.h index d030d9e3..61696a4f 100644 --- a/pb.h +++ b/pb.h @@ -195,7 +195,7 @@ typedef enum { #define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0])) #define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2)) #define pb_delta_end(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) -#define PB_LAST_FIELD {0,0,0,0} +#define PB_LAST_FIELD {0,(pb_type_t) 0,0,0} #endif -- cgit 1.2.3-korg From 72cca8d7838402eb436f1403d4c29dee46db62a1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 5 Jul 2012 18:19:38 +0300 Subject: Replace #warning with the standard #error. --- generator/nanopb_generator.py | 7 ++----- pb.h | 3 +-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 1925167f..1923cc1c 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -420,8 +420,8 @@ def generate_header(dependencies, headername, enums, messages): if largest_count > 64: yield '\n/* Check that missing required fields will be properly detected */\n' yield '#if PB_MAX_REQUIRED_FIELDS < %d\n' % largest_count - yield '#warning Properly detecting missing required fields in %s requires \\\n' % largest_msg.name - yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count + yield '#error Properly detecting missing required fields in %s requires \\\n' % largest_msg.name + yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count yield '#endif\n' worst = 0 @@ -438,9 +438,6 @@ def generate_header(dependencies, headername, enums, messages): if worst > 255 or checks: yield '\n/* Check that field information fits in pb_field_t */\n' - yield '/* (Largest message has %d fields' % worst - if checks: yield ' and submessages have to be checked at compile-time.' - yield ') */\n' if worst < 65536: yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n' diff --git a/pb.h b/pb.h index 61696a4f..4983d069 100644 --- a/pb.h +++ b/pb.h @@ -34,8 +34,7 @@ #endif #if PB_MAX_REQUIRED_FIELDS < 64 -#warning You should not lower PB_MAX_REQUIRED_FIELDS from the default value (64). \ - The automatic checks against too low value will not be active. +#error You should not lower PB_MAX_REQUIRED_FIELDS from the default value (64). #endif /* List of possible field types. These are used in the autogenerated code. -- cgit 1.2.3-korg From efef38cf7869f11750fba1dfb965e93b42d5d49e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 5 Jul 2012 18:24:11 +0300 Subject: Fix some typos in __BIG_ENDIAN__ code --- pb_encode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pb_encode.c b/pb_encode.c index c6cccf21..faaeac9d 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -235,7 +235,7 @@ bool checkreturn pb_encode_svarint(pb_ostream_t *stream, int64_t value) bool checkreturn pb_encode_fixed32(pb_ostream_t *stream, const void *value) { #ifdef __BIG_ENDIAN__ - uint8_t *bytes = value; + const uint8_t *bytes = value; uint8_t lebytes[4]; lebytes[0] = bytes[3]; lebytes[1] = bytes[2]; @@ -250,7 +250,7 @@ bool checkreturn pb_encode_fixed32(pb_ostream_t *stream, const void *value) bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value) { #ifdef __BIG_ENDIAN__ - uint8_t *bytes[8] = value; + const uint8_t *bytes = value; uint8_t lebytes[8]; lebytes[0] = bytes[7]; lebytes[1] = bytes[6]; -- cgit 1.2.3-korg From 01a155689828574047a8377d595a0ad038c48917 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 5 Jul 2012 18:27:07 +0300 Subject: Apparently some compilers don't want to automatically cast size_t to uint64_t. --- pb_encode.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pb_encode.c b/pb_encode.c index faaeac9d..be909ec2 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -110,7 +110,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie size = sizestream.bytes_written; } - if (!pb_encode_varint(stream, size)) + if (!pb_encode_varint(stream, (uint64_t)size)) return false; if (stream->callback == NULL) @@ -269,7 +269,7 @@ bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value) bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number) { int tag = wiretype | (field_number << 3); - return pb_encode_varint(stream, tag); + return pb_encode_varint(stream, (uint64_t)tag); } bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field) @@ -305,7 +305,7 @@ bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t bool checkreturn pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size) { - if (!pb_encode_varint(stream, size)) + if (!pb_encode_varint(stream, (uint64_t)size)) return false; return pb_write(stream, buffer, size); @@ -323,7 +323,7 @@ bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fie size = substream.bytes_written; - if (!pb_encode_varint(stream, size)) + if (!pb_encode_varint(stream, (uint64_t)size)) return false; if (stream->callback == NULL) -- cgit 1.2.3-korg From d8bddabb8372d1c684d7cd7cf4bb04131acb9ff1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 5 Jul 2012 20:02:06 +0300 Subject: Fix bug with .proto without messages (again), and add a test case for it. --- generator/nanopb_generator.py | 19 ++++++++++--------- tests/Makefile | 3 ++- tests/no_messages.proto | 7 +++++++ 3 files changed, 19 insertions(+), 10 deletions(-) create mode 100644 tests/no_messages.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 1923cc1c..b9018aeb 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -414,15 +414,16 @@ def generate_header(dependencies, headername, enums, messages): for msg in messages: yield msg.fields_declaration() + '\n' - count_required_fields = lambda m: len([f for f in msg.fields if f.htype == 'PB_HTYPE_REQUIRED']) - largest_msg = max(messages, key = count_required_fields) - largest_count = count_required_fields(largest_msg) - if largest_count > 64: - yield '\n/* Check that missing required fields will be properly detected */\n' - yield '#if PB_MAX_REQUIRED_FIELDS < %d\n' % largest_count - yield '#error Properly detecting missing required fields in %s requires \\\n' % largest_msg.name - yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count - yield '#endif\n' + if messages: + count_required_fields = lambda m: len([f for f in msg.fields if f.htype == 'PB_HTYPE_REQUIRED']) + largest_msg = max(messages, key = count_required_fields) + largest_count = count_required_fields(largest_msg) + if largest_count > 64: + yield '\n/* Check that missing required fields will be properly detected */\n' + yield '#if PB_MAX_REQUIRED_FIELDS < %d\n' % largest_count + yield '#error Properly detecting missing required fields in %s requires \\\n' % largest_msg.name + yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count + yield '#endif\n' worst = 0 worst_field = '' diff --git a/tests/Makefile b/tests/Makefile index 24e9ba62..8c063f30 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,7 +1,7 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage LDFLAGS=--coverage DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h missing_fields.pb.h -TESTS=test_decode1 test_encode1 decode_unittests encode_unittests +TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages all: breakpoints $(TESTS) run_unittests @@ -28,6 +28,7 @@ test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o test_missing_fields: test_missing_fields.o pb_encode.o pb_decode.o missing_fields.pb.o decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o encode_unittests: encode_unittests.o pb_encode.o unittestproto.pb.o +test_no_messages: no_messages.pb.h no_messages.pb.c no_messages.pb.o %.pb: %.proto protoc -I. -I../generator -I/usr/include -o$@ $< diff --git a/tests/no_messages.proto b/tests/no_messages.proto new file mode 100644 index 00000000..279216b0 --- /dev/null +++ b/tests/no_messages.proto @@ -0,0 +1,7 @@ +/* Test that a file without any messages works. */ + +enum Test { + First = 1; +} + + -- cgit 1.2.3-korg From 10b5da12dc9757b345a9d6f9daeef9098731ff69 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 18 Jul 2012 21:09:13 +0300 Subject: Fixed a few compiler warnings, added check. Main code is now compiled (for tests) with -pedantic -Wextra. The test programs are not as strictly bound, but this should improve the chances that atleast the core library compiles with most compilers without warnings. --- pb_decode.c | 2 +- pb_encode.c | 4 ++-- tests/Makefile | 7 +++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 458d5f1e..e183be7d 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -409,7 +409,7 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { - uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {}; /* Used to check for required fields */ + uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {0}; /* Used to check for required fields */ pb_field_iterator_t iter; pb_message_set_to_defaults(fields, dest_struct); diff --git a/pb_encode.c b/pb_encode.c index be909ec2..8868bb64 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -99,7 +99,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie } else { - pb_ostream_t sizestream = {0}; + pb_ostream_t sizestream = {0,0,0,0}; p = pData; for (i = 0; i < count; i++) { @@ -314,7 +314,7 @@ bool checkreturn pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, s bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) { /* First calculate the message size using a non-writing substream. */ - pb_ostream_t substream = {0}; + pb_ostream_t substream = {0,0,0,0}; size_t size; bool status; diff --git a/tests/Makefile b/tests/Makefile index 8c063f30..5a5d94ab 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -3,6 +3,9 @@ LDFLAGS=--coverage DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h missing_fields.pb.h TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages +# More strict checks for the core part of nanopb +CFLAGS_CORE=-pedantic -Wextra + all: breakpoints $(TESTS) run_unittests clean: @@ -13,9 +16,9 @@ clean: $(CC) $(CFLAGS) -c -o $@ $< pb_encode.o: ../pb_encode.c $(DEPS) - $(CC) $(CFLAGS) -c -o $@ $< + $(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< pb_decode.o: ../pb_decode.c $(DEPS) - $(CC) $(CFLAGS) -c -o $@ $< + $(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< test_decode1: test_decode1.o pb_decode.o person.pb.o test_decode2: test_decode2.o pb_decode.o person.pb.o -- cgit 1.2.3-korg From 8524de39ce2e4c369ed1051c7ed2e4cd43ad321d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 18 Jul 2012 21:21:07 +0300 Subject: Fix an unsigned vs. signed warning on some compiler. --- pb_decode.c | 4 ++-- pb_decode.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index e183be7d..cada00c5 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -117,7 +117,7 @@ bool checkreturn pb_skip_string(pb_istream_t *stream) return pb_read(stream, NULL, length); } -bool checkreturn pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, int *tag, bool *eof) +bool checkreturn pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, uint32_t *tag, bool *eof) { uint32_t temp; *eof = false; @@ -418,7 +418,7 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void while (stream->bytes_left) { - int tag; + uint32_t tag; pb_wire_type_t wire_type; bool eof; diff --git a/pb_decode.h b/pb_decode.h index 0abb342c..7045c5d6 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -48,7 +48,7 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc * You may want to use these from your caller or callbacks. */ -bool pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, int *tag, bool *eof); +bool pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, uint32_t *tag, bool *eof); bool pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type); bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest); -- cgit 1.2.3-korg From 0a5b6852ec34a9b954503e707f5fe71df37638ce Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 19 Jul 2012 09:05:36 +0300 Subject: Additional unsigned vs. signed fix for tag --- pb_decode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pb_decode.c b/pb_decode.c index cada00c5..18941cbe 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -253,7 +253,7 @@ static bool pb_field_next(pb_field_iterator_t *iter) return notwrapped; } -static bool checkreturn pb_field_find(pb_field_iterator_t *iter, int tag) +static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag) { int start = iter->field_index; -- cgit 1.2.3-korg From 5703ad0c55634e91157a965bbfa089a4396f3cb5 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 31 Jul 2012 19:10:33 +0300 Subject: Extend 'alltypes' testcase to cover repeated fields. --- tests/alltypes.proto | 24 ++++++++++++++++ tests/test_decode3.c | 27 ++++++++++++++++++ tests/test_encode3.c | 77 ++++++++++++++++++++++++++++++++++++---------------- 3 files changed, 104 insertions(+), 24 deletions(-) diff --git a/tests/alltypes.proto b/tests/alltypes.proto index 744a0fca..edaaa621 100644 --- a/tests/alltypes.proto +++ b/tests/alltypes.proto @@ -6,6 +6,7 @@ message SubMessage { } enum MyEnum { + Zero = 0; First = 1; Second = 2; Truth = 42; @@ -33,6 +34,29 @@ message AllTypes { required SubMessage req_submsg = 16; required MyEnum req_enum = 17; + + repeated int32 rep_int32 = 21 [(nanopb).max_count = 5]; + repeated int64 rep_int64 = 22 [(nanopb).max_count = 5]; + repeated uint32 rep_uint32 = 23 [(nanopb).max_count = 5]; + repeated uint64 rep_uint64 = 24 [(nanopb).max_count = 5]; + repeated sint32 rep_sint32 = 25 [(nanopb).max_count = 5]; + repeated sint64 rep_sint64 = 26 [(nanopb).max_count = 5]; + repeated bool rep_bool = 27 [(nanopb).max_count = 5]; + + repeated fixed32 rep_fixed32 = 28 [(nanopb).max_count = 5]; + repeated sfixed32 rep_sfixed32= 29 [(nanopb).max_count = 5]; + repeated float rep_float = 30 [(nanopb).max_count = 5]; + + repeated fixed64 rep_fixed64 = 31 [(nanopb).max_count = 5]; + repeated sfixed64 rep_sfixed64= 32 [(nanopb).max_count = 5]; + repeated double rep_double = 33 [(nanopb).max_count = 5]; + + repeated string rep_string = 34 [(nanopb).max_size = 16, (nanopb).max_count = 5]; + repeated bytes rep_bytes = 35 [(nanopb).max_size = 16, (nanopb).max_count = 5]; + repeated SubMessage rep_submsg = 36 [(nanopb).max_count = 5]; + repeated MyEnum rep_enum = 37 [(nanopb).max_count = 5]; + + // Just to make sure that the size of the fields has been calculated // properly, i.e. otherwise a bug in last field might not be detected. required int32 end = 99; diff --git a/tests/test_decode3.c b/tests/test_decode3.c index a7106dee..30a8afa7 100644 --- a/tests/test_decode3.c +++ b/tests/test_decode3.c @@ -45,6 +45,33 @@ bool check_alltypes(pb_istream_t *stream) TEST(alltypes.req_submsg.substuff2 == 1016); TEST(alltypes.req_enum == MyEnum_Truth); + TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == 2001 && alltypes.rep_int32[0] == 0); + TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == 2002 && alltypes.rep_int64[0] == 0); + TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); + TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); + TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == 2005 && alltypes.rep_sint32[0] == 0); + TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == 2006 && alltypes.rep_sint64[0] == 0); + TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); + + TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); + TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == 2009 && alltypes.rep_sfixed32[0] == 0); + TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); + + TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); + TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == 2012 && alltypes.rep_sfixed64[0] == 0); + TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); + + TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); + TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); + TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); + + TEST(alltypes.rep_submsg_count == 5); + TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); + TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); + + TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); + + TEST(alltypes.end == 1099); return true; diff --git a/tests/test_encode3.c b/tests/test_encode3.c index 5e94be51..8128ea4a 100644 --- a/tests/test_encode3.c +++ b/tests/test_encode3.c @@ -3,36 +3,65 @@ */ #include +#include #include #include "alltypes.pb.h" int main() { /* Initialize the structure with constants */ - AllTypes alltypes = { - 1001, - 1002, - 1003, - 1004, - 1005, - 1006, - true, - - 1008, - 1009, - 1010.0f, - - 1011, - 1012, - 1013.0, - - "1014", - {4, "1015"}, - {"1016", 1016}, - MyEnum_Truth, - - 1099 - }; + AllTypes alltypes = {0}; + + alltypes.req_int32 = 1001; + alltypes.req_int64 = 1002; + alltypes.req_uint32 = 1003; + alltypes.req_uint64 = 1004; + alltypes.req_sint32 = 1005; + alltypes.req_sint64 = 1006; + alltypes.req_bool = true; + + alltypes.req_fixed32 = 1008; + alltypes.req_sfixed32 = 1009; + alltypes.req_float = 1010.0f; + + alltypes.req_fixed64 = 1011; + alltypes.req_sfixed64 = 1012; + alltypes.req_double = 1013.0; + + strcpy(alltypes.req_string, "1014"); + alltypes.req_bytes.size = 4; + memcpy(alltypes.req_bytes.bytes, "1015", 4); + strcpy(alltypes.req_submsg.substuff1, "1016"); + alltypes.req_submsg.substuff2 = 1016; + alltypes.req_enum = MyEnum_Truth; + + alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = 2001; + alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = 2002; + alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; + alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004; + alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = 2005; + alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = 2006; + alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; + + alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; + alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = 2009; + alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; + + alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011; + alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = 2012; + alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0; + + alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); + alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4; + memcpy(alltypes.rep_bytes[4].bytes, "2015", 4); + + alltypes.rep_submsg_count = 5; + strcpy(alltypes.rep_submsg[4].substuff1, "2016"); + alltypes.rep_submsg[4].substuff2 = 2016; + + alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; + + alltypes.end = 1099; uint8_t buffer[512]; pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); -- cgit 1.2.3-korg From b582bc9bf699ee76f8634c3c7b7aa1ec0a0afb08 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 31 Jul 2012 19:12:57 +0300 Subject: Fix bug in decoder with packed arrays. Update issue 23 Status: FixedInGit --- pb_decode.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pb_decode.c b/pb_decode.c index 18941cbe..6d3a0b6c 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -288,6 +288,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t && PB_LTYPE(iter->current->type) <= PB_LTYPE_LAST_PACKABLE) { /* Packed array */ + bool status; size_t *size = (size_t*)iter->pSize; pb_istream_t substream; if (!make_string_substream(stream, &substream)) @@ -300,7 +301,9 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t return false; (*size)++; } - return (substream.bytes_left == 0); + status = (substream.bytes_left == 0); + stream->state = substream.state; + return status; } else { -- cgit 1.2.3-korg From 1aa61f108a1cb1b24219bf0fc0dc52f72c310e89 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 4 Aug 2012 14:34:19 +0300 Subject: Fix signedness warning in example_unions. --- example_unions/decode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_unions/decode.c b/example_unions/decode.c index b20df84e..edd568cb 100644 --- a/example_unions/decode.c +++ b/example_unions/decode.c @@ -17,7 +17,7 @@ const pb_field_t* decode_unionmessage_type(pb_istream_t *stream) { pb_wire_type_t wire_type; - int tag; + uint32_t tag; bool eof; while (pb_decode_tag(stream, &wire_type, &tag, &eof)) -- cgit 1.2.3-korg From 8d5086f052f4282f22d350ac44f6699f3bdf7a06 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 9 Aug 2012 16:15:23 +0300 Subject: Declare warn_unused_result only on GCC >= 3.4.0. Thanks to cea.max.simoes for bug report & fix. Update issue 25 Status: FixedInGit --- pb_decode.c | 10 +++++----- pb_encode.c | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 6d3a0b6c..5ef40ecc 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -3,12 +3,12 @@ * 2011 Petteri Aimonen */ - -#ifdef __GNUC__ -/* Verify that we remember to check all return values for proper error propagation */ -#define checkreturn __attribute__((warn_unused_result)) +/* The warn_unused_result attribute appeared first in gcc-3.4.0 */ +#if !defined(__GNUC__) || ( __GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4) + #define checkreturn #else -#define checkreturn + /* Verify that we remember to check all return values for proper error propagation */ + #define checkreturn __attribute__((warn_unused_result)) #endif #include "pb.h" diff --git a/pb_encode.c b/pb_encode.c index 8868bb64..3fc03811 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -8,14 +8,14 @@ #include "pb_encode.h" #include -#ifdef __GNUC__ -/* Verify that we remember to check all return values for proper error propagation */ -#define checkreturn __attribute__((warn_unused_result)) +/* The warn_unused_result attribute appeared first in gcc-3.4.0 */ +#if !defined(__GNUC__) || ( __GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4) + #define checkreturn #else -#define checkreturn + /* Verify that we remember to check all return values for proper error propagation */ + #define checkreturn __attribute__((warn_unused_result)) #endif - typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, const void *src) checkreturn; /* --- Function pointers to field encoders --- -- cgit 1.2.3-korg From 07f24f9b7022086187f60f52e786d0dbdbd3b7f1 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 16 Aug 2012 11:29:08 -0700 Subject: Fix casting issues --- pb_decode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 5ef40ecc..48068325 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -121,7 +121,7 @@ bool checkreturn pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, { uint32_t temp; *eof = false; - *wire_type = 0; + *wire_type = (pb_wire_type_t) 0; *tag = 0; if (!pb_decode_varint32(stream, &temp)) @@ -393,7 +393,7 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str } else if (PB_LTYPE(iter.current->type) == PB_LTYPE_SUBMESSAGE) { - pb_message_set_to_defaults(iter.current->ptr, iter.pData); + pb_message_set_to_defaults((const pb_field_t *) iter.current->ptr, iter.pData); } else if (iter.current->ptr != NULL) { -- cgit 1.2.3-korg From 86257a2a7000875c8ec1e2eef59de258400ac724 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 24 Aug 2012 19:25:29 +0300 Subject: Fix a confusing statement in a comment. --- pb_decode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pb_decode.h b/pb_decode.h index 7045c5d6..192326eb 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -3,7 +3,7 @@ /* pb_decode.h: Functions to decode protocol buffers. Depends on pb_decode.c. * The main function is pb_decode. You will also need to create an input - * stream, which is easiest to do with pb_istream_t. + * stream, which is easiest to do with pb_istream_from_buffer(). * * You also need structures and their corresponding pb_field_t descriptions. * These are usually generated from .proto-files with a script. -- cgit 1.2.3-korg From dc2da0edc568b29361479fb7405c96b1a13442cf Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 24 Aug 2012 19:35:17 +0300 Subject: Change the substream implementation in pb_decode. This makes it unnecessary to copy back the state, and also relaxes the requirements on callbacks (bytes_left will always be valid). It decreases code size by a few bytes, but may be just slightly slower. --- pb_decode.c | 29 +++++++++++++---------------- pb_decode.h | 4 ---- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 48068325..9df4a894 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -185,22 +185,26 @@ static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire } } -/* Decode string length from stream and return a substream with limited length. - * Before disposing the substream, remember to copy the substream->state back - * to stream->state. - */ +/* Decode string length from stream and return a substream with limited length. */ +static bool substream_callback(pb_istream_t *stream, uint8_t *buf, size_t count) +{ + pb_istream_t *parent = (pb_istream_t*)stream->state; + return pb_read(parent, buf, count); +} + static bool checkreturn make_string_substream(pb_istream_t *stream, pb_istream_t *substream) { uint32_t size; if (!pb_decode_varint32(stream, &size)) return false; - *substream = *stream; - if (substream->bytes_left < size) + if (stream->bytes_left < size) return false; + substream->callback = &substream_callback; + substream->state = stream; substream->bytes_left = size; - stream->bytes_left -= size; + return true; } @@ -288,7 +292,6 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t && PB_LTYPE(iter->current->type) <= PB_LTYPE_LAST_PACKABLE) { /* Packed array */ - bool status; size_t *size = (size_t*)iter->pSize; pb_istream_t substream; if (!make_string_substream(stream, &substream)) @@ -301,9 +304,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t return false; (*size)++; } - status = (substream.bytes_left == 0); - stream->state = substream.state; - return status; + return (substream.bytes_left == 0); } else { @@ -337,7 +338,6 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t return false; } - stream->state = substream.state; return true; } else @@ -575,7 +575,6 @@ bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, vo bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) { - bool status; pb_istream_t substream; if (!make_string_substream(stream, &substream)) @@ -584,7 +583,5 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field if (field->ptr == NULL) return false; - status = pb_decode(&substream, (pb_field_t*)field->ptr, dest); - stream->state = substream.state; - return status; + return pb_decode(&substream, (pb_field_t*)field->ptr, dest); } diff --git a/pb_decode.h b/pb_decode.h index 192326eb..9cc67e8e 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -23,10 +23,6 @@ * * 3) You can use state to store your own data (e.g. buffer pointer), * and rely on pb_read to verify that no-body reads past bytes_left. - * - * 4) Your callback may be used with substreams, in which case bytes_left - * is different than from the main stream. Don't use bytes_left to compute - * any pointers. */ struct _pb_istream_t { -- cgit 1.2.3-korg From 2941e90e041b0d7872a4ea458e77dcc787ca6ff2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 24 Aug 2012 20:23:25 +0300 Subject: Reorganize the field decoder interface. This makes the field decoding functions more intuitive to use. The old interface is still present if you specify NANOPB_INTERNALS. Update issue 2 Status: FixedInGit --- docs/reference.rst | 118 ++++++++++-------------------------- example_unions/decode.c | 7 ++- pb_decode.c | 137 +++++++++++++++++++++++++----------------- pb_decode.h | 34 +++++++++-- tests/decode_unittests.c | 2 + tests/test_decode_callbacks.c | 4 +- 6 files changed, 150 insertions(+), 152 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 3331c6d3..7dd08edd 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -320,16 +320,6 @@ In addition to EOF, the pb_decode implementation supports terminating a message For optional fields, this function applies the default value and sets *has_* to false if the field is not present. -pb_decode_varint ----------------- -Read and decode a varint_ encoded integer. :: - - bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest); - -:stream: Input stream to read from. 1-10 bytes will be read. -:dest: Storage for the decoded integer. Value is undefined on error. -:returns: True on success, false if value exceeds uint64_t range or an IO error happens. - pb_skip_varint -------------- Skip a varint_ encoded integer without decoding it. :: @@ -373,48 +363,41 @@ Remove the data for a field from the stream, without actually decoding it:: :wire_type: Type of field to skip. :returns: True on success, false on IO error. -.. sidebar:: Field decoders +.. sidebar:: Decoding fields manually - The functions with names beginning with *pb_dec_* are called field decoders. Each PB_LTYPE has an own field decoder, which handles translating from Protocol Buffers data to C data. + The functions with names beginning with *pb_decode_* are used when dealing with callback fields. The typical reason for using callbacks is to have an array of unlimited size. In that case, `pb_decode`_ will call your callback function repeatedly, which can then store the values into e.g. filesystem in the order received in. - Each field decoder reads and decodes a single value. For arrays, the decoder is called repeatedly. + For decoding numeric (including enumerated and boolean) values, use `pb_decode_varint`_, `pb_decode_svarint`_, `pb_decode_fixed32`_ and `pb_decode_fixed64`_. They take a pointer to a 32- or 64-bit C variable, which you may then cast to smaller datatype for storage. - You can use the decoders from your callbacks. Just be aware that the pb_field_t passed to the callback is not directly compatible - with the *varint* field decoders. Instead, you must create a new pb_field_t structure and set the data_size according to the data type - you pass to *dest*, e.g. *field.data_size = sizeof(int);*. Other fields in the *pb_field_t* don't matter. + For decoding strings and bytes fields, the length has already been decoded. You can therefore check the total length in *stream->state* and read the data using `pb_read`_. - The field decoder interface is a bit messy as a result of the interface required inside the nanopb library. - Eventually they may be replaced by separate wrapper functions with a more friendly interface. + Finally, for decoding submessages in a callback, simply use `pb_decode`_ and pass it the *SubMessage_fields* descriptor array. -pb_dec_varint -------------- -Field decoder for PB_LTYPE_VARINT. :: +pb_decode_varint +---------------- +Read and decode a varint_ encoded integer. :: - bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) + bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest); :stream: Input stream to read from. 1-10 bytes will be read. -:field: Field description structure. Only *field->data_size* matters. -:dest: Pointer to destination integer. Must have size of *field->data_size* bytes. -:returns: True on success, false on IO errors or if `pb_decode_varint`_ fails. - -This function first calls `pb_decode_varint`_. It then copies the first bytes of the 64-bit result value to *dest*, or on big endian architectures, the last bytes. +:dest: Storage for the decoded integer. Value is undefined on error. +:returns: True on success, false if value exceeds uint64_t range or an IO error happens. -pb_dec_svarint --------------- -Field decoder for PB_LTYPE_SVARINT. Similar to `pb_dec_varint`_, except that it performs zigzag-decoding on the value. :: +pb_decode_svarint +----------------- +Similar to `pb_decode_varint`_, except that it performs zigzag-decoding on the value. This corresponds to the Protocol Buffers *sint32* and *sint64* datatypes. :: - bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest); + bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest); -(parameters are the same as `pb_dec_varint`_) +(parameters are the same as `pb_decode_varint`_) -pb_dec_fixed32 --------------- -Field decoder for PB_LTYPE_FIXED32. :: +pb_decode_fixed32 +----------------- +Decode a *fixed32*, *sfixed32* or *float* value. :: - bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest); + bool pb_decode_fixed32(pb_istream_t *stream, void *dest); :stream: Input stream to read from. 4 bytes will be read. -:field: Not used. :dest: Pointer to destination *int32_t*, *uint32_t* or *float*. :returns: True on success, false on IO errors. @@ -422,9 +405,9 @@ This function reads 4 bytes from the input stream. On big endian architectures, it then reverses the order of the bytes. Finally, it writes the bytes to *dest*. -pb_dec_fixed64 --------------- -Field decoder for PB_LTYPE_FIXED64. :: +pb_decode_fixed64 +----------------- +Decode a *fixed64*, *sfixed64* or *double* value. :: bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest); @@ -433,53 +416,16 @@ Field decoder for PB_LTYPE_FIXED64. :: :dest: Pointer to destination *int64_t*, *uint64_t* or *double*. :returns: True on success, false on IO errors. -Same as `pb_dec_fixed32`_, except this reads 8 bytes. - -pb_dec_bytes ------------- -Field decoder for PB_LTYPE_BYTES. Reads a length-prefixed block of bytes. :: - - bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); +Same as `pb_decode_fixed32`_, except this reads 8 bytes. -**Note:** This is an internal function that is not useful in decoder callbacks. To read bytes fields in callbacks, use -*stream->bytes_left* and `pb_read`_. - -:stream: Input stream to read from. -:field: Field description structure. Only *field->data_size* matters. -:dest: Pointer to a structure similar to pb_bytes_array_t. -:returns: True on success, false on IO error or if length exceeds the array size. - -This function expects a pointer to a structure with a *size_t* field at start, and a variable sized byte array after it. It will deduce the maximum size of the array from *field->data_size*. - -pb_dec_string -------------- -Field decoder for PB_LTYPE_STRING. Reads a length-prefixed string. :: +pb_make_string_substream +------------------------ +Decode the length for a field with wire type *PB_WT_STRING* and create a substream for reading the data. :: - bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); - -**Note:** This is an internal function that is not useful in decoder callbacks. To read string fields in callbacks, use -*stream->bytes_left* and `pb_read`_. - -:stream: Input stream to read from. -:field: Field description structure. Only *field->data_size* matters. -:dest: Pointer to a character array of size *field->data_size*. -:returns: True on success, false on IO error or if length exceeds the array size. - -This function null-terminates the string when successful. On error, the contents of the destination array is undefined. - -pb_dec_submessage ------------------ -Field decoder for PB_LTYPE_SUBMESSAGE. Calls `pb_decode`_ to perform the actual decoding. :: - - bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) - -**Note:** This is an internal function that is not useful in decoder callbacks. To read submessage fields in callbacks, use -`pb_decode`_ directly. - -:stream: Input stream to read from. -:field: Field description structure. Only *field->ptr* matters. -:dest: Pointer to the destination structure. -:returns: True on success, false on IO error or if `pb_decode`_ fails. + bool pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream); -The *field->ptr* should be a pointer to *pb_field_t* array describing the submessage. +:stream: Original input stream to read the length and data from. +:substream: New substream that has limited length. Filled in by the function. +:returns: True on success, false if reading the length fails. +This function uses `pb_decode_varint`_ to read an integer from the stream. This is interpreted as a number of bytes, and the substream is set up so that its `bytes_left` is initially the same as the length. The substream has a wrapper callback that in turn reads from the parent stream. diff --git a/example_unions/decode.c b/example_unions/decode.c index edd568cb..a7cc7814 100644 --- a/example_unions/decode.c +++ b/example_unions/decode.c @@ -44,10 +44,11 @@ const pb_field_t* decode_unionmessage_type(pb_istream_t *stream) bool decode_unionmessage_contents(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { - pb_field_t field = {}; /* NB: Could get rid of this wrapper by fixing issue #2. */ - field.ptr = fields; + pb_istream_t substream; + if (!pb_make_string_substream(stream, &substream)) + return false; - return pb_dec_submessage(stream, &field, dest_struct); + return pb_decode(&substream, fields, dest_struct); } int main() diff --git a/pb_decode.c b/pb_decode.c index 9df4a894..59efb3d6 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -11,6 +11,7 @@ #define checkreturn __attribute__((warn_unused_result)) #endif +#define NANOPB_INTERNALS #include "pb.h" #include "pb_decode.h" #include @@ -192,7 +193,7 @@ static bool substream_callback(pb_istream_t *stream, uint8_t *buf, size_t count) return pb_read(parent, buf, count); } -static bool checkreturn make_string_substream(pb_istream_t *stream, pb_istream_t *substream) +bool checkreturn pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream) { uint32_t size; if (!pb_decode_varint32(stream, &size)) @@ -294,7 +295,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t /* Packed array */ size_t *size = (size_t*)iter->pSize; pb_istream_t substream; - if (!make_string_substream(stream, &substream)) + if (!pb_make_string_substream(stream, &substream)) return false; while (substream.bytes_left && *size < iter->current->array_size) @@ -329,7 +330,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t { pb_istream_t substream; - if (!make_string_substream(stream, &substream)) + if (!pb_make_string_substream(stream, &substream)) return false; while (substream.bytes_left) @@ -467,78 +468,104 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void /* Field decoders */ -/* Copy destsize bytes from src so that values are casted properly. - * On little endian machine, copy first n bytes of src - * On big endian machine, copy last n bytes of src - * srcsize must always be larger than destsize - */ -static void endian_copy(void *dest, void *src, size_t destsize, size_t srcsize) +bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest) { -#ifdef __BIG_ENDIAN__ - memcpy(dest, (char*)src + (srcsize - destsize), destsize); -#else - UNUSED(srcsize); - memcpy(dest, src, destsize); -#endif + uint64_t value; + if (!pb_decode_varint(stream, &value)) + return false; + + if (value & 1) + *dest = ~(value >> 1); + else + *dest = value >> 1; + + return true; +} + +bool pb_decode_fixed32(pb_istream_t *stream, void *dest) +{ + #ifdef __BIG_ENDIAN__ + uint8_t *bytes = (uint8_t*)dest; + uint8_t lebytes[4]; + + if (!pb_read(stream, lebytes, 4)) + return false; + + bytes[0] = lebytes[3]; + bytes[1] = lebytes[2]; + bytes[2] = lebytes[1]; + bytes[3] = lebytes[0]; + return true; + #else + return pb_read(stream, (uint8_t*)dest, 4); + #endif +} + +bool pb_decode_fixed64(pb_istream_t *stream, void *dest) +{ + #ifdef __BIG_ENDIAN__ + uint8_t *bytes = (uint8_t*)dest; + uint8_t lebytes[8]; + + if (!pb_read(stream, lebytes, 8)) + return false; + + bytes[0] = lebytes[7]; + bytes[1] = lebytes[6]; + bytes[2] = lebytes[5]; + bytes[3] = lebytes[4]; + bytes[4] = lebytes[3]; + bytes[5] = lebytes[2]; + bytes[6] = lebytes[1]; + bytes[7] = lebytes[0]; + return true; + #else + return pb_read(stream, (uint8_t*)dest, 8); + #endif } bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) { - uint64_t temp; - bool status = pb_decode_varint(stream, &temp); - endian_copy(dest, &temp, field->data_size, sizeof(temp)); + uint64_t value; + bool status = pb_decode_varint(stream, &value); + + switch (field->data_size) + { + case 1: *(uint8_t*)dest = value; break; + case 2: *(uint16_t*)dest = value; break; + case 4: *(uint32_t*)dest = value; break; + case 8: *(uint64_t*)dest = value; break; + default: return false; + } + return status; } bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) { - uint64_t temp; - bool status = pb_decode_varint(stream, &temp); - temp = (temp >> 1) ^ -(int64_t)(temp & 1); - endian_copy(dest, &temp, field->data_size, sizeof(temp)); + int64_t value; + bool status = pb_decode_svarint(stream, &value); + + switch (field->data_size) + { + case 4: *(int32_t*)dest = value; break; + case 8: *(int64_t*)dest = value; break; + default: return false; + } + return status; } bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest) { -#ifdef __BIG_ENDIAN__ - uint8_t bytes[4] = {0}; - bool status = pb_read(stream, bytes, 4); - if (status) { - uint8_t *d = (uint8_t*)dest; - d[0] = bytes[3]; - d[1] = bytes[2]; - d[2] = bytes[1]; - d[3] = bytes[0]; - } - return status; -#else UNUSED(field); - return pb_read(stream, (uint8_t*)dest, 4); -#endif + return pb_decode_fixed32(stream, dest); } bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest) { -#ifdef __BIG_ENDIAN__ - uint8_t bytes[8] = {0}; - bool status = pb_read(stream, bytes, 8); - if (status) { - uint8_t *d = (uint8_t*)dest; - d[0] = bytes[7]; - d[1] = bytes[6]; - d[2] = bytes[5]; - d[3] = bytes[4]; - d[4] = bytes[3]; - d[5] = bytes[2]; - d[6] = bytes[1]; - d[7] = bytes[0]; - } - return status; -#else UNUSED(field); - return pb_read(stream, (uint8_t*)dest, 8); -#endif + return pb_decode_fixed64(stream, dest); } bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) @@ -577,7 +604,7 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field { pb_istream_t substream; - if (!make_string_substream(stream, &substream)) + if (!pb_make_string_substream(stream, &substream)) return false; if (field->ptr == NULL) diff --git a/pb_decode.h b/pb_decode.h index 9cc67e8e..27535c10 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -44,20 +44,38 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc * You may want to use these from your caller or callbacks. */ +/* Decode the tag for the next field in the stream. Gives the wire type and + * field tag. At end of the message, returns false and sets eof to true. */ bool pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, uint32_t *tag, bool *eof); + +/* Skip the field payload data, given the wire type. */ bool pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type); +/* Decode an integer in the varint format. This works for bool, enum, int32, + * int64, uint32 and uint64 field types. */ bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest); -bool pb_skip_varint(pb_istream_t *stream); -bool pb_skip_string(pb_istream_t *stream); +/* Decode an integer in the zig-zagged svarint format. This works for sint32 + * and sint64. */ +bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest); + +/* Decode a fixed32, sfixed32 or float value. You need to pass a pointer to + * a 4-byte wide C variable. */ +bool pb_decode_fixed32(pb_istream_t *stream, void *dest); + +/* Decode a fixed64, sfixed64 or double value. You need to pass a pointer to + * a 8-byte wide C variable. */ +bool pb_decode_fixed64(pb_istream_t *stream, void *dest); -/* --- Field decoders --- - * Each decoder takes stream and field description, and a pointer to the field - * in the destination struct (dest = struct_addr + field->data_offset). - * For arrays, these functions are called repeatedly. +/* Make a limited-length substream for reading a PB_WT_STRING field. */ +bool pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream); + +/* --- Internal functions --- + * These functions are not terribly useful for the average library user, but + * are exported to make the unit testing and extending nanopb easier. */ +#ifdef NANOPB_INTERNALS bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest); @@ -67,4 +85,8 @@ bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest); +bool pb_skip_varint(pb_istream_t *stream); +bool pb_skip_string(pb_istream_t *stream); +#endif + #endif diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c index 6ba6d4f2..039c9fa4 100644 --- a/tests/decode_unittests.c +++ b/tests/decode_unittests.c @@ -1,3 +1,5 @@ +#define NANOPB_INTERNALS + #include #include #include "pb_decode.h" diff --git a/tests/test_decode_callbacks.c b/tests/test_decode_callbacks.c index 714b7bb2..95824d1a 100644 --- a/tests/test_decode_callbacks.c +++ b/tests/test_decode_callbacks.c @@ -37,7 +37,7 @@ bool print_int32(pb_istream_t *stream, const pb_field_t *field, void *arg) bool print_fixed32(pb_istream_t *stream, const pb_field_t *field, void *arg) { uint32_t value; - if (!pb_dec_fixed32(stream, NULL, &value)) + if (!pb_decode_fixed32(stream, &value)) return false; printf((char*)arg, (long)value); @@ -47,7 +47,7 @@ bool print_fixed32(pb_istream_t *stream, const pb_field_t *field, void *arg) bool print_fixed64(pb_istream_t *stream, const pb_field_t *field, void *arg) { uint64_t value; - if (!pb_dec_fixed64(stream, NULL, &value)) + if (!pb_decode_fixed64(stream, &value)) return false; printf((char*)arg, (long long)value); -- cgit 1.2.3-korg From 9383f305dc42eeca8b043c5c6dff3fe2e3e8f4f2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 24 Aug 2012 20:43:21 +0300 Subject: Revert "Change the substream implementation in pb_decode." This reverts commit dc2da0edc568b29361479fb7405c96b1a13442cf. Add pb_close_string_substream() for copying back the state. This makes adding error messages easier in the future, as also them need to be propagated back from the substream. --- example_unions/decode.c | 5 ++++- pb_decode.c | 33 ++++++++++++++++++++------------- pb_decode.h | 5 +++++ 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/example_unions/decode.c b/example_unions/decode.c index a7cc7814..d40cd8ce 100644 --- a/example_unions/decode.c +++ b/example_unions/decode.c @@ -45,10 +45,13 @@ const pb_field_t* decode_unionmessage_type(pb_istream_t *stream) bool decode_unionmessage_contents(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { pb_istream_t substream; + bool status; if (!pb_make_string_substream(stream, &substream)) return false; - return pb_decode(&substream, fields, dest_struct); + status = pb_decode(&substream, fields, dest_struct); + pb_close_string_substream(stream, &substream); + return status; } int main() diff --git a/pb_decode.c b/pb_decode.c index 59efb3d6..8f72f776 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -186,29 +186,29 @@ static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire } } -/* Decode string length from stream and return a substream with limited length. */ -static bool substream_callback(pb_istream_t *stream, uint8_t *buf, size_t count) -{ - pb_istream_t *parent = (pb_istream_t*)stream->state; - return pb_read(parent, buf, count); -} - +/* Decode string length from stream and return a substream with limited length. + * Remember to close the substream using pb_close_string_substream(). + */ bool checkreturn pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream) { uint32_t size; if (!pb_decode_varint32(stream, &size)) return false; - if (stream->bytes_left < size) + *substream = *stream; + if (substream->bytes_left < size) return false; - substream->callback = &substream_callback; - substream->state = stream; substream->bytes_left = size; - + stream->bytes_left -= size; return true; } +void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) +{ + stream->state = substream->state; +} + /* Iterator for pb_field_t list */ typedef struct { const pb_field_t *start; /* Start of the pb_field_t array */ @@ -293,6 +293,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t && PB_LTYPE(iter->current->type) <= PB_LTYPE_LAST_PACKABLE) { /* Packed array */ + bool status; size_t *size = (size_t*)iter->pSize; pb_istream_t substream; if (!pb_make_string_substream(stream, &substream)) @@ -305,7 +306,9 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t return false; (*size)++; } - return (substream.bytes_left == 0); + status = (substream.bytes_left == 0); + pb_close_string_substream(stream, &substream); + return status; } else { @@ -339,6 +342,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t return false; } + pb_close_string_substream(stream, &substream); return true; } else @@ -602,6 +606,7 @@ bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, vo bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) { + bool status; pb_istream_t substream; if (!pb_make_string_substream(stream, &substream)) @@ -610,5 +615,7 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field if (field->ptr == NULL) return false; - return pb_decode(&substream, (pb_field_t*)field->ptr, dest); + status = pb_decode(&substream, (pb_field_t*)field->ptr, dest); + pb_close_string_substream(stream, &substream); + return status; } diff --git a/pb_decode.h b/pb_decode.h index 27535c10..2880c070 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -23,6 +23,10 @@ * * 3) You can use state to store your own data (e.g. buffer pointer), * and rely on pb_read to verify that no-body reads past bytes_left. + * + * 4) Your callback may be used with substreams, in which case bytes_left + * is different than from the main stream. Don't use bytes_left to compute + * any pointers. */ struct _pb_istream_t { @@ -69,6 +73,7 @@ bool pb_decode_fixed64(pb_istream_t *stream, void *dest); /* Make a limited-length substream for reading a PB_WT_STRING field. */ bool pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream); +void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream); /* --- Internal functions --- * These functions are not terribly useful for the average library user, but -- cgit 1.2.3-korg From ea57f74741b5b5ab3ab4a3e81d8b61811417b4c6 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 24 Aug 2012 20:51:29 +0300 Subject: Document pb_close_string_substream. --- docs/reference.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/reference.rst b/docs/reference.rst index 7dd08edd..3a6e11a4 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -428,4 +428,16 @@ Decode the length for a field with wire type *PB_WT_STRING* and create a substre :substream: New substream that has limited length. Filled in by the function. :returns: True on success, false if reading the length fails. -This function uses `pb_decode_varint`_ to read an integer from the stream. This is interpreted as a number of bytes, and the substream is set up so that its `bytes_left` is initially the same as the length. The substream has a wrapper callback that in turn reads from the parent stream. +This function uses `pb_decode_varint`_ to read an integer from the stream. This is interpreted as a number of bytes, and the substream is set up so that its `bytes_left` is initially the same as the length, and its callback function and state the same as the parent stream. + +pb_close_string_substream +------------------------ +Close the substream created with `pb_make_string_substream`_. :: + + void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream); + +:stream: Original input stream to read the length and data from. +:substream: Substream to close + +This function copies back the state from the substream to the parent stream. +It must be called after done with the substream. -- cgit 1.2.3-korg From 0fb5e5e068326b23493952619d7efb640cb37377 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 24 Aug 2012 21:22:20 +0300 Subject: Implement error messages in the decoder side. Update issue 7 Status: Started --- docs/concepts.rst | 14 +++++--------- docs/reference.rst | 4 +++- example/client.c | 2 +- example/server.c | 2 +- example_unions/decode.c | 2 +- pb.h | 18 ++++++++++++++++++ pb_decode.c | 30 +++++++++++++++--------------- pb_decode.h | 4 ++++ tests/test_decode1.c | 2 +- tests/test_decode2.c | 2 +- tests/test_decode3.c | 2 +- tests/test_missing_fields.c | 2 +- 12 files changed, 52 insertions(+), 32 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 1d0fe08f..122d29cf 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -258,18 +258,14 @@ generates this field description array for the structure *Person_PhoneNumber*:: Return values and error handling ================================ -Most functions in nanopb return bool: *true* means success, *false* means failure. If this is enough for you, skip this section. +Most functions in nanopb return bool: *true* means success, *false* means failure. There is also some support for error messages for debugging purposes: the error messages go in *stream->errmsg*. -For simplicity, nanopb doesn't define it's own error codes. This might be added if there is a compelling need for it. You can however deduce something about the error causes: +The error messages help in guessing what is the underlying cause of the error. The most common error conditions are: 1) Running out of memory. Because everything is allocated from the stack, nanopb can't detect this itself. Encoding or decoding the same type of a message always takes the same amount of stack space. Therefore, if it works once, it works always. 2) Invalid field description. These are usually stored as constants, so if it works under the debugger, it always does. -3) IO errors in your own stream callbacks. Because encoding/decoding stops at the first error, you can overwrite the *state* field in the struct and store your own error code there. -4) Errors that happen in your callback functions. You can use the state field in the callback structure. +3) IO errors in your own stream callbacks. +4) Errors that happen in your callback functions. 5) Exceeding the max_size or bytes_left of a stream. 6) Exceeding the max_size of a string or array field -7) Invalid protocol buffers binary message. It's not like you could recover from it anyway, so a simple failure should be enough. - -In my opinion, it is enough that 1. and 2. can be resolved using a debugger. - -However, you may be interested which of the remaining conditions caused the error. For 3. and 4., you can set and check the state. If you have to detect 5. and 6., you should convert the fields to callback type. Any remaining problem is of type 7. +7) Invalid protocol buffers binary message. diff --git a/docs/reference.rst b/docs/reference.rst index 3a6e11a4..ec9aec5d 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -20,6 +20,8 @@ PB_FIELD_16BIT Add support for tag numbers > 255 and fields larg Increases code size 3 bytes per each field. Compiler error will tell if you need this. PB_FIELD_32BIT Add support for tag numbers > 65535 and fields larger than 65535 bytes or 65535 array entries. Increases code size 9 bytes per each field. Compiler error will tell if you need this. +PB_NO_ERRMSG Disables the support for error messages; only error information is the true/false return value. + Decreases the code size by a few hundred bytes. ============================ ================================================================================================ The PB_MAX_REQUIRED_FIELDS, PB_FIELD_16BIT and PB_FIELD_32BIT settings allow raising some datatype limits to suit larger messages. @@ -431,7 +433,7 @@ Decode the length for a field with wire type *PB_WT_STRING* and create a substre This function uses `pb_decode_varint`_ to read an integer from the stream. This is interpreted as a number of bytes, and the substream is set up so that its `bytes_left` is initially the same as the length, and its callback function and state the same as the parent stream. pb_close_string_substream ------------------------- +------------------------- Close the substream created with `pb_make_string_substream`_. :: void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream); diff --git a/example/client.c b/example/client.c index 9ad9c8c9..edc83948 100644 --- a/example/client.c +++ b/example/client.c @@ -72,7 +72,7 @@ bool listdir(int fd, char *path) if (!pb_decode(&input, ListFilesResponse_fields, &response)) { - fprintf(stderr, "Decoding failed.\n"); + fprintf(stderr, "Decode failed: %s\n", PB_GET_ERROR(&input)); return false; } diff --git a/example/server.c b/example/server.c index 9f27906d..346e9fb9 100644 --- a/example/server.c +++ b/example/server.c @@ -55,7 +55,7 @@ void handle_connection(int connfd) if (!pb_decode(&input, ListFilesRequest_fields, &request)) { - printf("Decoding failed.\n"); + printf("Decode failed: %s\n", PB_GET_ERROR(&input)); return; } diff --git a/example_unions/decode.c b/example_unions/decode.c index d40cd8ce..b9f4af55 100644 --- a/example_unions/decode.c +++ b/example_unions/decode.c @@ -85,7 +85,7 @@ int main() if (!status) { - printf("Decoding failed.\n"); + printf("Decode failed: %s\n", PB_GET_ERROR(&stream)); return 1; } diff --git a/pb.h b/pb.h index 4983d069..85f4421c 100644 --- a/pb.h +++ b/pb.h @@ -196,5 +196,23 @@ typedef enum { #define pb_delta_end(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) #define PB_LAST_FIELD {0,(pb_type_t) 0,0,0} +/* These macros are used for giving out error messages. + * They are mostly a debugging aid; the main error information + * is the true/false return value from functions. + * Some code space can be saved by disabling the error + * messages if not used. + */ +#ifdef PB_NO_ERRMSG +#define PB_RETURN_ERROR(stream,msg) return false +#define PB_GET_ERROR(stream) "(errmsg disabled)" +#else +#define PB_RETURN_ERROR(stream,msg) \ + do {\ + if ((stream)->errmsg == NULL) \ + (stream)->errmsg = (msg); \ + return false; \ + } while(0) +#define PB_GET_ERROR(stream) ((stream)->errmsg ? (stream)->errmsg : "(none)") +#endif #endif diff --git a/pb_decode.c b/pb_decode.c index 8f72f776..9d65504c 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -39,10 +39,10 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) { if (stream->bytes_left < count) - return false; + PB_RETURN_ERROR(stream, "end-of-stream"); if (!stream->callback(stream, buf, count)) - return false; + PB_RETURN_ERROR(stream, "io error"); stream->bytes_left -= count; return true; @@ -95,7 +95,7 @@ bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) return true; } - return false; + PB_RETURN_ERROR(stream, "varint overflow"); } bool checkreturn pb_skip_varint(pb_istream_t *stream) @@ -152,7 +152,7 @@ bool checkreturn pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type) case PB_WT_64BIT: return pb_read(stream, NULL, 8); case PB_WT_STRING: return pb_skip_string(stream); case PB_WT_32BIT: return pb_read(stream, NULL, 4); - default: return false; + default: PB_RETURN_ERROR(stream, "invalid wire_type"); } } @@ -182,7 +182,7 @@ static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire *size = 4; return pb_read(stream, buf, 4); - default: return false; + default: PB_RETURN_ERROR(stream, "invalid wire_type"); } } @@ -197,7 +197,7 @@ bool checkreturn pb_make_string_substream(pb_istream_t *stream, pb_istream_t *su *substream = *stream; if (substream->bytes_left < size) - return false; + PB_RETURN_ERROR(stream, "parent stream too short"); substream->bytes_left = size; stream->bytes_left -= size; @@ -316,7 +316,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t size_t *size = (size_t*)iter->pSize; void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size); if (*size >= iter->current->array_size) - return false; + PB_RETURN_ERROR(stream, "array overflow"); (*size)++; return func(stream, iter->current, pItem); @@ -339,7 +339,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t while (substream.bytes_left) { if (!pCallback->funcs.decode(&substream, iter->current, pCallback->arg)) - return false; + PB_RETURN_ERROR(stream, "callback failed"); } pb_close_string_substream(stream, &substream); @@ -364,7 +364,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t } default: - return false; + PB_RETURN_ERROR(stream, "invalid field type"); } } @@ -463,7 +463,7 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void iter.required_field_index < PB_MAX_REQUIRED_FIELDS && !(fields_seen[iter.required_field_index >> 3] & (1 << (iter.required_field_index & 7)))) { - return false; + PB_RETURN_ERROR(stream, "missing required field"); } } while (pb_field_next(&iter)); @@ -539,7 +539,7 @@ bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, vo case 2: *(uint16_t*)dest = value; break; case 4: *(uint32_t*)dest = value; break; case 8: *(uint64_t*)dest = value; break; - default: return false; + default: PB_RETURN_ERROR(stream, "invalid data_size"); } return status; @@ -554,7 +554,7 @@ bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, v { case 4: *(int32_t*)dest = value; break; case 8: *(int64_t*)dest = value; break; - default: return false; + default: PB_RETURN_ERROR(stream, "invalid data_size"); } return status; @@ -583,7 +583,7 @@ bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, voi /* Check length, noting the space taken by the size_t header. */ if (x->size > field->data_size - offsetof(pb_bytes_array_t, bytes)) - return false; + PB_RETURN_ERROR(stream, "bytes overflow"); return pb_read(stream, x->bytes, x->size); } @@ -597,7 +597,7 @@ bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, vo /* Check length, noting the null terminator */ if (size + 1 > field->data_size) - return false; + PB_RETURN_ERROR(stream, "string overflow"); status = pb_read(stream, (uint8_t*)dest, size); *((uint8_t*)dest + size) = 0; @@ -613,7 +613,7 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field return false; if (field->ptr == NULL) - return false; + PB_RETURN_ERROR(stream, "invalid field descriptor"); status = pb_decode(&substream, (pb_field_t*)field->ptr, dest); pb_close_string_substream(stream, &substream); diff --git a/pb_decode.h b/pb_decode.h index 2880c070..ad45efbf 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -33,6 +33,10 @@ struct _pb_istream_t bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count); void *state; /* Free field for use by callback implementation */ size_t bytes_left; + +#ifndef PB_NO_ERRMSG + const char *errmsg; +#endif }; pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); diff --git a/tests/test_decode1.c b/tests/test_decode1.c index 78781dd4..64eb4689 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -69,7 +69,7 @@ int main() /* Decode and print out the stuff */ if (!print_person(&stream)) { - printf("Parsing failed.\n"); + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); return 1; } else { return 0; diff --git a/tests/test_decode2.c b/tests/test_decode2.c index d38e6256..762b2b3f 100644 --- a/tests/test_decode2.c +++ b/tests/test_decode2.c @@ -82,7 +82,7 @@ int main() pb_istream_t stream = {&callback, stdin, 10000}; if (!print_person(&stream)) { - printf("Parsing failed.\n"); + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); return 1; } else { return 0; diff --git a/tests/test_decode3.c b/tests/test_decode3.c index 30a8afa7..4f55b558 100644 --- a/tests/test_decode3.c +++ b/tests/test_decode3.c @@ -89,7 +89,7 @@ int main() /* Decode and print out the stuff */ if (!check_alltypes(&stream)) { - printf("Parsing failed.\n"); + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); return 1; } else { return 0; diff --git a/tests/test_missing_fields.c b/tests/test_missing_fields.c index 46cd7d96..27741847 100644 --- a/tests/test_missing_fields.c +++ b/tests/test_missing_fields.c @@ -27,7 +27,7 @@ int main() if (!pb_decode(&stream, MissingField_fields, &msg)) { - printf("Decode failed.\n"); + printf("Decode failed: %s\n", PB_GET_ERROR(&stream)); return 2; } } -- cgit 1.2.3-korg From 440da6959f36022185fc465590b53bd10acda862 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 24 Aug 2012 21:32:42 +0300 Subject: Add note that stream callbacks must read the whole requested length. Fixes issue #22. --- docs/concepts.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/concepts.rst b/docs/concepts.rst index 122d29cf..d3261146 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -50,6 +50,7 @@ There are a few generic rules for callback functions: #) Use state to store your own data, such as a file descriptor. #) *bytes_written* and *bytes_left* are updated by pb_write and pb_read. #) Your callback may be used with substreams. In this case *bytes_left*, *bytes_written* and *max_size* have smaller values than the original stream. Don't use these values to calculate pointers. +#) Always read or write the full requested length of data. For example, POSIX *recv()* needs the *MSG_WAITALL* parameter to accomplish this. Output streams -------------- -- cgit 1.2.3-korg From f1d7640fe1be0f150f604c72108ea516222c2505 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 26 Aug 2012 09:56:11 +0300 Subject: Add pb_decode_noinit and use it from pb_dec_submessage. This avoids double initialization when decoding nested submessages. Update issue 28 Status: FixedInGit --- docs/reference.rst | 10 ++++++++++ pb_decode.c | 12 ++++++++---- pb_decode.h | 6 ++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index ec9aec5d..371c155a 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -322,6 +322,16 @@ In addition to EOF, the pb_decode implementation supports terminating a message For optional fields, this function applies the default value and sets *has_* to false if the field is not present. +pb_decode_noinit +---------------- +Same as `pb_decode`_, except does not apply the default values to fields. :: + + bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +(parameters are the same as for `pb_decode`_.) + +The destination structure should be filled with zeros before calling this function. Doing a *memset* manually can be slightly faster than using `pb_decode`_ if you don't need any default values. + pb_skip_varint -------------- Skip a varint_ encoded integer without decoding it. :: diff --git a/pb_decode.c b/pb_decode.c index 9d65504c..601dc742 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -415,13 +415,11 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str * Decode all fields * *********************/ -bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {0}; /* Used to check for required fields */ pb_field_iterator_t iter; - pb_message_set_to_defaults(fields, dest_struct); - pb_field_init(&iter, fields, dest_struct); while (stream->bytes_left) @@ -470,6 +468,12 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void return true; } +bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + pb_message_set_to_defaults(fields, dest_struct); + return pb_decode_noinit(stream, fields, dest_struct); +} + /* Field decoders */ bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest) @@ -615,7 +619,7 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field if (field->ptr == NULL) PB_RETURN_ERROR(stream, "invalid field descriptor"); - status = pb_decode(&substream, (pb_field_t*)field->ptr, dest); + status = pb_decode_noinit(&substream, (pb_field_t*)field->ptr, dest); pb_close_string_substream(stream, &substream); return status; } diff --git a/pb_decode.h b/pb_decode.h index ad45efbf..483665ee 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -48,6 +48,12 @@ bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); */ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); +/* Same as pb_decode, except does not initialize the destination structure + * to default values. This is slightly faster if you need no default values + * and just do memset(struct, 0, sizeof(struct)) yourself. + */ +bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + /* --- Helper functions --- * You may want to use these from your caller or callbacks. */ -- cgit 1.2.3-korg From 1463e687e36c8dd404d33c6ef1cba61b574adc1e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 26 Aug 2012 10:05:46 +0300 Subject: Revert "Add pb_decode_noinit and use it from pb_dec_submessage." The patch breaks default values inside submessage arrays (I think). Have to add test cases and check back. This reverts commit f1d7640fe1be0f150f604c72108ea516222c2505. --- docs/reference.rst | 10 ---------- pb_decode.c | 12 ++++-------- pb_decode.h | 6 ------ 3 files changed, 4 insertions(+), 24 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 371c155a..ec9aec5d 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -322,16 +322,6 @@ In addition to EOF, the pb_decode implementation supports terminating a message For optional fields, this function applies the default value and sets *has_* to false if the field is not present. -pb_decode_noinit ----------------- -Same as `pb_decode`_, except does not apply the default values to fields. :: - - bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); - -(parameters are the same as for `pb_decode`_.) - -The destination structure should be filled with zeros before calling this function. Doing a *memset* manually can be slightly faster than using `pb_decode`_ if you don't need any default values. - pb_skip_varint -------------- Skip a varint_ encoded integer without decoding it. :: diff --git a/pb_decode.c b/pb_decode.c index 601dc742..9d65504c 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -415,11 +415,13 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str * Decode all fields * *********************/ -bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {0}; /* Used to check for required fields */ pb_field_iterator_t iter; + pb_message_set_to_defaults(fields, dest_struct); + pb_field_init(&iter, fields, dest_struct); while (stream->bytes_left) @@ -468,12 +470,6 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ return true; } -bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) -{ - pb_message_set_to_defaults(fields, dest_struct); - return pb_decode_noinit(stream, fields, dest_struct); -} - /* Field decoders */ bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest) @@ -619,7 +615,7 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field if (field->ptr == NULL) PB_RETURN_ERROR(stream, "invalid field descriptor"); - status = pb_decode_noinit(&substream, (pb_field_t*)field->ptr, dest); + status = pb_decode(&substream, (pb_field_t*)field->ptr, dest); pb_close_string_substream(stream, &substream); return status; } diff --git a/pb_decode.h b/pb_decode.h index 483665ee..ad45efbf 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -48,12 +48,6 @@ bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); */ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); -/* Same as pb_decode, except does not initialize the destination structure - * to default values. This is slightly faster if you need no default values - * and just do memset(struct, 0, sizeof(struct)) yourself. - */ -bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); - /* --- Helper functions --- * You may want to use these from your caller or callbacks. */ -- cgit 1.2.3-korg From a7a6cdad8885b48a648ad69c9c0a3f01e0707934 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 26 Aug 2012 10:57:51 +0300 Subject: Added alltypes-testcases for optional fields and default values. --- tests/Makefile | 3 +- tests/alltypes.proto | 25 +++++++++++- tests/test_decode3.c | 109 ++++++++++++++++++++++++++++++++++++++++++++++++--- tests/test_encode3.c | 55 ++++++++++++++++++++++++-- 4 files changed, 180 insertions(+), 12 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 5a5d94ab..5014221e 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -65,7 +65,8 @@ run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_ "`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ] ./test_encode3 | ./test_decode3 - ./test_encode3 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null + ./test_encode3 1 | ./test_decode3 1 + ./test_encode3 1 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null ./test_missing_fields diff --git a/tests/alltypes.proto b/tests/alltypes.proto index edaaa621..b9003a79 100644 --- a/tests/alltypes.proto +++ b/tests/alltypes.proto @@ -1,8 +1,9 @@ import "nanopb.proto"; message SubMessage { - required string substuff1 = 1 [(nanopb).max_size = 16]; - required int32 substuff2 = 2; + required string substuff1 = 1 [(nanopb).max_size = 16, default = "1"]; + required int32 substuff2 = 2 [default = 2]; + optional fixed32 substuff3 = 3 [default = 3]; } enum MyEnum { @@ -56,7 +57,27 @@ message AllTypes { repeated SubMessage rep_submsg = 36 [(nanopb).max_count = 5]; repeated MyEnum rep_enum = 37 [(nanopb).max_count = 5]; + optional int32 opt_int32 = 41 [default = 4041]; + optional int64 opt_int64 = 42 [default = 4042]; + optional uint32 opt_uint32 = 43 [default = 4043]; + optional uint64 opt_uint64 = 44 [default = 4044]; + optional sint32 opt_sint32 = 45 [default = 4045]; + optional sint64 opt_sint64 = 46 [default = 4046]; + optional bool opt_bool = 47 [default = false]; + optional fixed32 opt_fixed32 = 48 [default = 4048]; + optional sfixed32 opt_sfixed32= 49 [default = 4049]; + optional float opt_float = 50 [default = 4050]; + + optional fixed64 opt_fixed64 = 51 [default = 4051]; + optional sfixed64 opt_sfixed64= 52 [default = 4052]; + optional double opt_double = 53 [default = 4053]; + + optional string opt_string = 54 [(nanopb).max_size = 16, default = "4054"]; + optional bytes opt_bytes = 55 [(nanopb).max_size = 16, default = "4055"]; + optional SubMessage opt_submsg = 56; + optional MyEnum opt_enum = 57 [default = Second]; + // Just to make sure that the size of the fields has been calculated // properly, i.e. otherwise a bug in last field might not be detected. required int32 end = 99; diff --git a/tests/test_decode3.c b/tests/test_decode3.c index 4f55b558..4b7ee9e2 100644 --- a/tests/test_decode3.c +++ b/tests/test_decode3.c @@ -1,10 +1,11 @@ -/* Tests the decoding of all types. Currently only in the 'required' variety. +/* Tests the decoding of all types. * This is the counterpart of test_encode3. * Run e.g. ./test_encode3 | ./test_decode3 */ #include #include +#include #include #include "alltypes.pb.h" @@ -15,9 +16,12 @@ /* This function is called once from main(), it handles the decoding and checks the fields. */ -bool check_alltypes(pb_istream_t *stream) +bool check_alltypes(pb_istream_t *stream, int mode) { - AllTypes alltypes = {}; + AllTypes alltypes; + + /* Fill with garbage to better detect initialization errors */ + memset(&alltypes, 0xAA, sizeof(alltypes)); if (!pb_decode(stream, AllTypes_fields, &alltypes)) return false; @@ -43,6 +47,7 @@ bool check_alltypes(pb_istream_t *stream) TEST(memcmp(alltypes.req_bytes.bytes, "1015", 4) == 0); TEST(strcmp(alltypes.req_submsg.substuff1, "1016") == 0); TEST(alltypes.req_submsg.substuff2 == 1016); + TEST(alltypes.req_submsg.substuff3 == 3); TEST(alltypes.req_enum == MyEnum_Truth); TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == 2001 && alltypes.rep_int32[0] == 0); @@ -68,26 +73,118 @@ bool check_alltypes(pb_istream_t *stream) TEST(alltypes.rep_submsg_count == 5); TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); + TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3); TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); + if (mode == 0) + { + /* Expect default values */ + TEST(alltypes.has_opt_int32 == false); + TEST(alltypes.opt_int32 == 4041); + TEST(alltypes.has_opt_int64 == false); + TEST(alltypes.opt_int64 == 4042); + TEST(alltypes.has_opt_uint32 == false); + TEST(alltypes.opt_uint32 == 4043); + TEST(alltypes.has_opt_uint64 == false); + TEST(alltypes.opt_uint64 == 4044); + TEST(alltypes.has_opt_sint32 == false); + TEST(alltypes.opt_sint32 == 4045); + TEST(alltypes.has_opt_sint64 == false); + TEST(alltypes.opt_sint64 == 4046); + TEST(alltypes.has_opt_bool == false); + TEST(alltypes.opt_bool == false); + + TEST(alltypes.has_opt_fixed32 == false); + TEST(alltypes.opt_fixed32 == 4048); + TEST(alltypes.has_opt_sfixed32 == false); + TEST(alltypes.opt_sfixed32 == 4049); + TEST(alltypes.has_opt_float == false); + TEST(alltypes.opt_float == 4050.0f); + + TEST(alltypes.has_opt_fixed64 == false); + TEST(alltypes.opt_fixed64 == 4051); + TEST(alltypes.has_opt_sfixed64 == false); + TEST(alltypes.opt_sfixed64 == 4052); + TEST(alltypes.has_opt_double == false); + TEST(alltypes.opt_double == 4053.0); + + TEST(alltypes.has_opt_string == false); + TEST(strcmp(alltypes.opt_string, "4054") == 0); + TEST(alltypes.has_opt_bytes == false); + TEST(alltypes.opt_bytes.size == 4); + TEST(memcmp(alltypes.opt_bytes.bytes, "4055", 4) == 0); + TEST(alltypes.has_opt_submsg == false); + TEST(strcmp(alltypes.opt_submsg.substuff1, "1") == 0); + TEST(alltypes.opt_submsg.substuff2 == 2); + TEST(alltypes.opt_submsg.substuff3 == 3); + TEST(alltypes.has_opt_enum == false); + TEST(alltypes.opt_enum == MyEnum_Second); + } + else + { + /* Expect filled-in values */ + TEST(alltypes.has_opt_int32 == true); + TEST(alltypes.opt_int32 == 3041); + TEST(alltypes.has_opt_int64 == true); + TEST(alltypes.opt_int64 == 3042); + TEST(alltypes.has_opt_uint32 == true); + TEST(alltypes.opt_uint32 == 3043); + TEST(alltypes.has_opt_uint64 == true); + TEST(alltypes.opt_uint64 == 3044); + TEST(alltypes.has_opt_sint32 == true); + TEST(alltypes.opt_sint32 == 3045); + TEST(alltypes.has_opt_sint64 == true); + TEST(alltypes.opt_sint64 == 3046); + TEST(alltypes.has_opt_bool == true); + TEST(alltypes.opt_bool == true); + + TEST(alltypes.has_opt_fixed32 == true); + TEST(alltypes.opt_fixed32 == 3048); + TEST(alltypes.has_opt_sfixed32 == true); + TEST(alltypes.opt_sfixed32 == 3049); + TEST(alltypes.has_opt_float == true); + TEST(alltypes.opt_float == 3050.0f); + + TEST(alltypes.has_opt_fixed64 == true); + TEST(alltypes.opt_fixed64 == 3051); + TEST(alltypes.has_opt_sfixed64 == true); + TEST(alltypes.opt_sfixed64 == 3052); + TEST(alltypes.has_opt_double == true); + TEST(alltypes.opt_double == 3053.0); + + TEST(alltypes.has_opt_string == true); + TEST(strcmp(alltypes.opt_string, "3054") == 0); + TEST(alltypes.has_opt_bytes == true); + TEST(alltypes.opt_bytes.size == 4); + TEST(memcmp(alltypes.opt_bytes.bytes, "3055", 4) == 0); + TEST(alltypes.has_opt_submsg == true); + TEST(strcmp(alltypes.opt_submsg.substuff1, "3056") == 0); + TEST(alltypes.opt_submsg.substuff2 == 3056); + TEST(alltypes.opt_submsg.substuff3 == 3); + TEST(alltypes.has_opt_enum == true); + TEST(alltypes.opt_enum == MyEnum_Truth); + } TEST(alltypes.end == 1099); return true; } -int main() +int main(int argc, char **argv) { + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + /* Read the data into buffer */ - uint8_t buffer[512]; + uint8_t buffer[1024]; size_t count = fread(buffer, 1, sizeof(buffer), stdin); /* Construct a pb_istream_t for reading from the buffer */ pb_istream_t stream = pb_istream_from_buffer(buffer, count); /* Decode and print out the stuff */ - if (!check_alltypes(&stream)) + if (!check_alltypes(&stream, mode)) { printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); return 1; diff --git a/tests/test_encode3.c b/tests/test_encode3.c index 8128ea4a..1a48f06b 100644 --- a/tests/test_encode3.c +++ b/tests/test_encode3.c @@ -1,14 +1,16 @@ /* Attempts to test all the datatypes supported by ProtoBuf. - * Currently only tests the 'required' variety. */ #include +#include #include #include #include "alltypes.pb.h" -int main() +int main(int argc, char **argv) { + int mode = (argc > 1) ? atoi(argv[1]) : 0; + /* Initialize the structure with constants */ AllTypes alltypes = {0}; @@ -58,12 +60,58 @@ int main() alltypes.rep_submsg_count = 5; strcpy(alltypes.rep_submsg[4].substuff1, "2016"); alltypes.rep_submsg[4].substuff2 = 2016; + alltypes.rep_submsg[4].has_substuff3 = true; + alltypes.rep_submsg[4].substuff3 = 2016; alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; + if (mode != 0) + { + /* Fill in values for optional fields */ + alltypes.has_opt_int32 = true; + alltypes.opt_int32 = 3041; + alltypes.has_opt_int64 = true; + alltypes.opt_int64 = 3042; + alltypes.has_opt_uint32 = true; + alltypes.opt_uint32 = 3043; + alltypes.has_opt_uint64 = true; + alltypes.opt_uint64 = 3044; + alltypes.has_opt_sint32 = true; + alltypes.opt_sint32 = 3045; + alltypes.has_opt_sint64 = true; + alltypes.opt_sint64 = 3046; + alltypes.has_opt_bool = true; + alltypes.opt_bool = true; + + alltypes.has_opt_fixed32 = true; + alltypes.opt_fixed32 = 3048; + alltypes.has_opt_sfixed32 = true; + alltypes.opt_sfixed32 = 3049; + alltypes.has_opt_float = true; + alltypes.opt_float = 3050.0f; + + alltypes.has_opt_fixed64 = true; + alltypes.opt_fixed64 = 3051; + alltypes.has_opt_sfixed64 = true; + alltypes.opt_sfixed64 = 3052; + alltypes.has_opt_double = true; + alltypes.opt_double = 3053.0; + + alltypes.has_opt_string = true; + strcpy(alltypes.opt_string, "3054"); + alltypes.has_opt_bytes = true; + alltypes.opt_bytes.size = 4; + memcpy(alltypes.opt_bytes.bytes, "3055", 4); + alltypes.has_opt_submsg = true; + strcpy(alltypes.opt_submsg.substuff1, "3056"); + alltypes.opt_submsg.substuff2 = 3056; + alltypes.has_opt_enum = true; + alltypes.opt_enum = MyEnum_Truth; + } + alltypes.end = 1099; - uint8_t buffer[512]; + uint8_t buffer[1024]; pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); /* Now encode it and check if we succeeded. */ @@ -74,6 +122,7 @@ int main() } else { + fprintf(stderr, "Encoding failed!\n"); return 1; /* Failure */ } } -- cgit 1.2.3-korg From 160f02e4d0f8f404492ffd4d7611ba4478190da2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 26 Aug 2012 09:56:11 +0300 Subject: Add pb_decode_noinit and use it from pb_dec_submessage. This avoids double initialization when decoding nested submessages. Fixes an issue with submessage arrays that was present in previous version of this patch. Update issue 28 Status: FixedInGit --- docs/reference.rst | 10 ++++++++++ pb_decode.c | 12 ++++++++---- pb_decode.h | 6 ++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index ec9aec5d..371c155a 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -322,6 +322,16 @@ In addition to EOF, the pb_decode implementation supports terminating a message For optional fields, this function applies the default value and sets *has_* to false if the field is not present. +pb_decode_noinit +---------------- +Same as `pb_decode`_, except does not apply the default values to fields. :: + + bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +(parameters are the same as for `pb_decode`_.) + +The destination structure should be filled with zeros before calling this function. Doing a *memset* manually can be slightly faster than using `pb_decode`_ if you don't need any default values. + pb_skip_varint -------------- Skip a varint_ encoded integer without decoding it. :: diff --git a/pb_decode.c b/pb_decode.c index 9d65504c..601dc742 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -415,13 +415,11 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str * Decode all fields * *********************/ -bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {0}; /* Used to check for required fields */ pb_field_iterator_t iter; - pb_message_set_to_defaults(fields, dest_struct); - pb_field_init(&iter, fields, dest_struct); while (stream->bytes_left) @@ -470,6 +468,12 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void return true; } +bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + pb_message_set_to_defaults(fields, dest_struct); + return pb_decode_noinit(stream, fields, dest_struct); +} + /* Field decoders */ bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest) @@ -615,7 +619,7 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field if (field->ptr == NULL) PB_RETURN_ERROR(stream, "invalid field descriptor"); - status = pb_decode(&substream, (pb_field_t*)field->ptr, dest); + status = pb_decode_noinit(&substream, (pb_field_t*)field->ptr, dest); pb_close_string_substream(stream, &substream); return status; } diff --git a/pb_decode.h b/pb_decode.h index ad45efbf..483665ee 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -48,6 +48,12 @@ bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); */ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); +/* Same as pb_decode, except does not initialize the destination structure + * to default values. This is slightly faster if you need no default values + * and just do memset(struct, 0, sizeof(struct)) yourself. + */ +bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + /* --- Helper functions --- * You may want to use these from your caller or callbacks. */ -- cgit 1.2.3-korg From a06dba6e49a7fc5fbc1e476539566e3f407be908 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 26 Aug 2012 15:21:20 +0300 Subject: Fix warnings with -Wcast-qual. Add test for C++ compile. Update issue 27 Status: FixedInGit --- pb_decode.c | 9 ++++++++- pb_encode.c | 28 ++++++++++++++-------------- tests/Makefile | 10 ++++++++-- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 601dc742..2235280e 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -612,6 +612,7 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field { bool status; pb_istream_t substream; + const pb_field_t* submsg_fields = (const pb_field_t*)field->ptr; if (!pb_make_string_substream(stream, &substream)) return false; @@ -619,7 +620,13 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field if (field->ptr == NULL) PB_RETURN_ERROR(stream, "invalid field descriptor"); - status = pb_decode_noinit(&substream, (pb_field_t*)field->ptr, dest); + /* New array entries need to be initialized, while required and optional + * submessages have already been initialized in the top-level pb_decode. */ + if (PB_HTYPE(field->type) == PB_HTYPE_ARRAY) + status = pb_decode(&substream, submsg_fields, dest); + else + status = pb_decode_noinit(&substream, submsg_fields, dest); + pb_close_string_substream(stream, &substream); return status; } diff --git a/pb_encode.c b/pb_encode.c index 3fc03811..9a6ad99f 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -168,7 +168,7 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons break; case PB_HTYPE_OPTIONAL: - if (*(bool*)pSize) + if (*(const bool*)pSize) { if (!pb_encode_tag_for_field(stream, field)) return false; @@ -179,13 +179,13 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons break; case PB_HTYPE_ARRAY: - if (!encode_array(stream, field, pData, *(size_t*)pSize, func)) + if (!encode_array(stream, field, pData, *(const size_t*)pSize, func)) return false; break; case PB_HTYPE_CALLBACK: { - pb_callback_t *callback = (pb_callback_t*)pData; + const pb_callback_t *callback = (const pb_callback_t*)pData; if (callback->funcs.encode != NULL) { if (!callback->funcs.encode(stream, field, callback->arg)) @@ -243,7 +243,7 @@ bool checkreturn pb_encode_fixed32(pb_ostream_t *stream, const void *value) lebytes[3] = bytes[0]; return pb_write(stream, lebytes, 4); #else - return pb_write(stream, (uint8_t*)value, 4); + return pb_write(stream, (const uint8_t*)value, 4); #endif } @@ -262,7 +262,7 @@ bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value) lebytes[7] = bytes[0]; return pb_write(stream, lebytes, 8); #else - return pb_write(stream, (uint8_t*)value, 8); + return pb_write(stream, (const uint8_t*)value, 8); #endif } @@ -358,10 +358,10 @@ bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, co switch (field->data_size) { - case 1: value = *(uint8_t*)src; break; - case 2: value = *(uint16_t*)src; break; - case 4: value = *(uint32_t*)src; break; - case 8: value = *(uint64_t*)src; break; + case 1: value = *(const uint8_t*)src; break; + case 2: value = *(const uint16_t*)src; break; + case 4: value = *(const uint32_t*)src; break; + case 8: value = *(const uint64_t*)src; break; default: return false; } @@ -374,8 +374,8 @@ bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, c switch (field->data_size) { - case 4: value = *(int32_t*)src; break; - case 8: value = *(int64_t*)src; break; + case 4: value = *(const int32_t*)src; break; + case 8: value = *(const int64_t*)src; break; default: return false; } @@ -396,7 +396,7 @@ bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, c bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - pb_bytes_array_t *bytes = (pb_bytes_array_t*)src; + const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src; UNUSED(field); return pb_encode_string(stream, bytes->bytes, bytes->size); } @@ -404,7 +404,7 @@ bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, con bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src) { UNUSED(field); - return pb_encode_string(stream, (uint8_t*)src, strlen((char*)src)); + return pb_encode_string(stream, (const uint8_t*)src, strlen((const char*)src)); } bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src) @@ -412,6 +412,6 @@ bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field if (field->ptr == NULL) return false; - return pb_encode_submessage(stream, (pb_field_t*)field->ptr, src); + return pb_encode_submessage(stream, (const pb_field_t*)field->ptr, src); } diff --git a/tests/Makefile b/tests/Makefile index 5014221e..3810c95f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -4,7 +4,7 @@ DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests. TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages # More strict checks for the core part of nanopb -CFLAGS_CORE=-pedantic -Wextra +CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wlogical-op all: breakpoints $(TESTS) run_unittests @@ -20,6 +20,12 @@ pb_encode.o: ../pb_encode.c $(DEPS) pb_decode.o: ../pb_decode.c $(DEPS) $(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< +pb_encode.cxx.o: ../pb_encode.c $(DEPS) + $(CXX) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< +pb_decode.cxx.o: ../pb_decode.c $(DEPS) + $(CXX) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< + +test_cxxcompile: pb_encode.cxx.o pb_decode.cxx.o test_decode1: test_decode1.o pb_decode.o person.pb.o test_decode2: test_decode2.o pb_decode.o person.pb.o test_decode3: test_decode3.o pb_decode.o alltypes.pb.o @@ -46,7 +52,7 @@ coverage: run_unittests gcov pb_encode.gcda gcov pb_decode.gcda -run_unittests: decode_unittests encode_unittests test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields +run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields rm -f *.gcda ./decode_unittests > /dev/null -- cgit 1.2.3-korg From f187aad63db2ff8e1a82508f4a43988b43e10f54 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 2 Sep 2012 19:54:35 +0300 Subject: Fix -Wextra warning in generated .pb.c files. Compile the generated files with the same extra-strict settings as the core, in order to detect problems in tests. Update issue 32 Status: FixedInGit --- pb.h | 2 +- tests/Makefile | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pb.h b/pb.h index 85f4421c..f894a9a5 100644 --- a/pb.h +++ b/pb.h @@ -194,7 +194,7 @@ typedef enum { #define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0])) #define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2)) #define pb_delta_end(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) -#define PB_LAST_FIELD {0,(pb_type_t) 0,0,0} +#define PB_LAST_FIELD {0,(pb_type_t) 0,0,0,0,0,0} /* These macros are used for giving out error messages. * They are mostly a debugging aid; the main error information diff --git a/tests/Makefile b/tests/Makefile index 3810c95f..e4c67c59 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -11,6 +11,9 @@ all: breakpoints $(TESTS) run_unittests clean: rm -f $(TESTS) person.pb* alltypes.pb* *.o *.gcda *.gcno +%.pb.o: %.pb.c %.pb.h + $(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< + %.o: %.c %.o: %.c $(DEPS) $(CC) $(CFLAGS) -c -o $@ $< -- cgit 1.2.3-korg From 07d44cf632bb3d1a4fb87c44e422ab9af42f2717 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 2 Sep 2012 20:14:22 +0300 Subject: Clean up one (uint64_t) cast. --- pb_encode.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pb_encode.c b/pb_encode.c index 9a6ad99f..a8873828 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -268,8 +268,8 @@ bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value) bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number) { - int tag = wiretype | (field_number << 3); - return pb_encode_varint(stream, (uint64_t)tag); + uint64_t tag = wiretype | (field_number << 3); + return pb_encode_varint(stream, tag); } bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field) -- cgit 1.2.3-korg From 11e0918b5e8832fef167a260b4afde2dc972dcf7 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 2 Sep 2012 20:49:08 +0300 Subject: Created a CHANGELOG file for easy reference. --- CHANGELOG | 37 +++++++++++++++++++++++++++++++++++++ README | 2 ++ 2 files changed, 39 insertions(+) create mode 100644 CHANGELOG diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 00000000..1effb161 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,37 @@ +nanopb-0.1.6 + Reorganize the field decoder interface (issue 2) + Improve performance in submessage decoding (issue 28) + Implement error messages in the decoder side (issue 7) + Extended testcases (alltypes test is now complete). + Fix some compiler warnings (issues 25, 26, 27, 32). + +nanopb-0.1.5 + Fix bug in decoder with packed arrays (issue 23). + Extended testcases. + Fix some compiler warnings. + +nanopb-0.1.4 + Add compile-time options for easy-to-use >255 field support. + Improve the detection of missing required fields. + Added example on how to handle union messages. + Fix generator error with .proto without messages. + Fix problems that stopped the code from compiling with some compilers. + Fix some compiler warnings. + +nanopb-0.1.3 + Refactor the field encoder interface. + Improve generator error messages (issue 5) + Add descriptor.proto into the #include exclusion list + Fix some compiler warnings. + +nanopb-0.1.2 + Make the generator to generate include for other .proto files (issue 4). + Fixed generator not working on Windows (issue 3) + +nanopb-0.1.1 + Fixed bug in encoder with 'bytes' fields (issue 1). + Fixed a bug in the generator that caused a compiler error on sfixed32 and sfixed64 fields. + Extended testcases. + +nanopb-0.1.0 + First stable release. diff --git a/README b/README index 0fc79c47..2a12b4f5 100644 --- a/README +++ b/README @@ -5,5 +5,7 @@ Homepage: http://kapsi.fi/~jpa/nanopb/ To compile the library, you'll need these libraries: protobuf-compiler python-protobuf libprotobuf-dev +The only runtime dependencies are memset() and memcpy(). + To run the tests, run make under the tests folder. If it completes without error, everything is fine. -- cgit 1.2.3-korg From be78e3b4d0ff9839817b0eeef5d3e75056b53af1 Mon Sep 17 00:00:00 2001 From: dch Date: Sun, 2 Sep 2012 21:24:19 +0100 Subject: Fix build warnings on MS compilers Update issue 33 Status: FixedInGit --- pb_decode.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 2235280e..65db511c 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -539,9 +539,9 @@ bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, vo switch (field->data_size) { - case 1: *(uint8_t*)dest = value; break; - case 2: *(uint16_t*)dest = value; break; - case 4: *(uint32_t*)dest = value; break; + case 1: *(uint8_t*)dest = (uint8_t)value; break; + case 2: *(uint16_t*)dest = (uint16_t)value; break; + case 4: *(uint32_t*)dest = (uint32_t)value; break; case 8: *(uint64_t*)dest = value; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); } @@ -556,7 +556,7 @@ bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, v switch (field->data_size) { - case 4: *(int32_t*)dest = value; break; + case 4: *(int32_t*)dest = (int32_t)value; break; case 8: *(int64_t*)dest = value; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); } -- cgit 1.2.3-korg From b214de4e1e53f14354db36b8f199db6177a63744 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 3 Sep 2012 17:35:14 +0300 Subject: Enable -Wconversion for core and fix the warnings. This should help avoid issue 33 in the future. --- pb.h | 9 ++++++--- pb_decode.c | 8 ++++---- pb_encode.c | 10 +++++----- pb_encode.h | 2 +- tests/Makefile | 2 +- 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/pb.h b/pb.h index f894a9a5..dce8df30 100644 --- a/pb.h +++ b/pb.h @@ -75,6 +75,7 @@ typedef enum { /* Number of declared LTYPES */ PB_LTYPES_COUNT = 7, + PB_LTYPE_MASK = 0x0F, /****************** * Modifier flags * @@ -95,11 +96,13 @@ typedef enum { * data_offset points to pb_callback_t structure. * LTYPE should be 0 (it is ignored, but sometimes * used to speculatively index an array). */ - PB_HTYPE_CALLBACK = 0x30 + PB_HTYPE_CALLBACK = 0x30, + + PB_HTYPE_MASK = 0xF0 } pb_packed pb_type_t; -#define PB_HTYPE(x) ((x) & 0xF0) -#define PB_LTYPE(x) ((x) & 0x0F) +#define PB_HTYPE(x) ((x) & PB_HTYPE_MASK) +#define PB_LTYPE(x) ((x) & PB_LTYPE_MASK) /* This structure is used in auto-generated constants * to specify struct fields. diff --git a/pb_decode.c b/pb_decode.c index 65db511c..86dec4b7 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -83,7 +83,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) { uint8_t byte; - uint8_t bitpos = 0; + int bitpos = 0; *dest = 0; while (bitpos < 64 && pb_read(stream, &byte, 1)) @@ -447,7 +447,7 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ if (PB_HTYPE(iter.current->type) == PB_HTYPE_REQUIRED && iter.required_field_index < PB_MAX_REQUIRED_FIELDS) { - fields_seen[iter.required_field_index >> 3] |= 1 << (iter.required_field_index & 7); + fields_seen[iter.required_field_index >> 3] |= (uint8_t)(1 << (iter.required_field_index & 7)); } if (!decode_field(stream, wire_type, &iter)) @@ -483,9 +483,9 @@ bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest) return false; if (value & 1) - *dest = ~(value >> 1); + *dest = (int64_t)(~(value >> 1)); else - *dest = value >> 1; + *dest = (int64_t)(value >> 1); return true; } diff --git a/pb_encode.c b/pb_encode.c index a8873828..5d1965d1 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -205,7 +205,7 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value) { uint8_t buffer[10]; - int i = 0; + size_t i = 0; if (value == 0) return pb_write(stream, (uint8_t*)&value, 1); @@ -225,9 +225,9 @@ bool checkreturn pb_encode_svarint(pb_ostream_t *stream, int64_t value) { uint64_t zigzagged; if (value < 0) - zigzagged = ~(value << 1); + zigzagged = (uint64_t)(~(value << 1)); else - zigzagged = value << 1; + zigzagged = (uint64_t)(value << 1); return pb_encode_varint(stream, zigzagged); } @@ -266,7 +266,7 @@ bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value) #endif } -bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number) +bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, uint32_t field_number) { uint64_t tag = wiretype | (field_number << 3); return pb_encode_varint(stream, tag); @@ -370,7 +370,7 @@ bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, co bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - uint64_t value = 0; + int64_t value = 0; switch (field->data_size) { diff --git a/pb_encode.h b/pb_encode.h index 59ec554b..af6cc3c5 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -54,7 +54,7 @@ bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field); /* Encode field header by manually specifing wire type. You need to use this if * you want to write out packed arrays from a callback field. */ -bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number); +bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, uint32_t field_number); /* Encode an integer in the varint format. * This works for bool, enum, int32, int64, uint32 and uint64 field types. */ diff --git a/tests/Makefile b/tests/Makefile index e4c67c59..9b02817f 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -4,7 +4,7 @@ DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests. TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages # More strict checks for the core part of nanopb -CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wlogical-op +CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wlogical-op -Wconversion all: breakpoints $(TESTS) run_unittests -- cgit 1.2.3-korg From 3f563792adb8448a55c3b4f16f11a3b0d37dac37 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 18 Sep 2012 16:48:43 +0300 Subject: Add buffer size check in test_decode1.c example. This check gives a better error message in case you test stuff and have a message longer than 512 bytes. Update issue 34 Status: FixedInGit --- tests/test_decode1.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_decode1.c b/tests/test_decode1.c index 64eb4689..56bbd8f8 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -63,6 +63,12 @@ int main() uint8_t buffer[512]; size_t count = fread(buffer, 1, sizeof(buffer), stdin); + if (!feof(stdin)) + { + printf("Message does not fit in buffer\n"); + return 1; + } + /* Construct a pb_istream_t for reading from the buffer */ pb_istream_t stream = pb_istream_from_buffer(buffer, count); -- cgit 1.2.3-korg From 900c8dd1252afe2b2474b852ae48dcb46e100505 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 14 Oct 2012 12:27:08 +0300 Subject: Add tests for negative values in the alltypes testcase. --- tests/test_decode3.c | 24 ++++++++++++------------ tests/test_encode3.c | 24 ++++++++++++------------ 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/tests/test_decode3.c b/tests/test_decode3.c index 4b7ee9e2..cd1b154b 100644 --- a/tests/test_decode3.c +++ b/tests/test_decode3.c @@ -26,20 +26,20 @@ bool check_alltypes(pb_istream_t *stream, int mode) if (!pb_decode(stream, AllTypes_fields, &alltypes)) return false; - TEST(alltypes.req_int32 == 1001); - TEST(alltypes.req_int64 == 1002); + TEST(alltypes.req_int32 == -1001); + TEST(alltypes.req_int64 == -1002); TEST(alltypes.req_uint32 == 1003); TEST(alltypes.req_uint64 == 1004); - TEST(alltypes.req_sint32 == 1005); - TEST(alltypes.req_sint64 == 1006); + TEST(alltypes.req_sint32 == -1005); + TEST(alltypes.req_sint64 == -1006); TEST(alltypes.req_bool == true); TEST(alltypes.req_fixed32 == 1008); - TEST(alltypes.req_sfixed32 == 1009); + TEST(alltypes.req_sfixed32 == -1009); TEST(alltypes.req_float == 1010.0f); TEST(alltypes.req_fixed64 == 1011); - TEST(alltypes.req_sfixed64 == 1012); + TEST(alltypes.req_sfixed64 == -1012); TEST(alltypes.req_double == 1013.0f); TEST(strcmp(alltypes.req_string, "1014") == 0); @@ -50,20 +50,20 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.req_submsg.substuff3 == 3); TEST(alltypes.req_enum == MyEnum_Truth); - TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == 2001 && alltypes.rep_int32[0] == 0); - TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == 2002 && alltypes.rep_int64[0] == 0); + TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); + TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); - TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == 2005 && alltypes.rep_sint32[0] == 0); - TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == 2006 && alltypes.rep_sint64[0] == 0); + TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); + TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0); TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); - TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == 2009 && alltypes.rep_sfixed32[0] == 0); + TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); - TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == 2012 && alltypes.rep_sfixed64[0] == 0); + TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0); TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); diff --git a/tests/test_encode3.c b/tests/test_encode3.c index 1a48f06b..6db37fdc 100644 --- a/tests/test_encode3.c +++ b/tests/test_encode3.c @@ -14,20 +14,20 @@ int main(int argc, char **argv) /* Initialize the structure with constants */ AllTypes alltypes = {0}; - alltypes.req_int32 = 1001; - alltypes.req_int64 = 1002; + alltypes.req_int32 = -1001; + alltypes.req_int64 = -1002; alltypes.req_uint32 = 1003; alltypes.req_uint64 = 1004; - alltypes.req_sint32 = 1005; - alltypes.req_sint64 = 1006; + alltypes.req_sint32 = -1005; + alltypes.req_sint64 = -1006; alltypes.req_bool = true; alltypes.req_fixed32 = 1008; - alltypes.req_sfixed32 = 1009; + alltypes.req_sfixed32 = -1009; alltypes.req_float = 1010.0f; alltypes.req_fixed64 = 1011; - alltypes.req_sfixed64 = 1012; + alltypes.req_sfixed64 = -1012; alltypes.req_double = 1013.0; strcpy(alltypes.req_string, "1014"); @@ -37,20 +37,20 @@ int main(int argc, char **argv) alltypes.req_submsg.substuff2 = 1016; alltypes.req_enum = MyEnum_Truth; - alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = 2001; - alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = 2002; + alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; + alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002; alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004; - alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = 2005; - alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = 2006; + alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005; + alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006; alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; - alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = 2009; + alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009; alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011; - alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = 2012; + alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012; alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0; alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); -- cgit 1.2.3-korg From dcab39a41c0a403db38860c22426075e6ae9f25d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 18 Oct 2012 19:45:28 +0300 Subject: Remove the "buf = NULL" => skip requirement from pb_istream_t callbacks. Rationale: it's easy to implement the callback wrong. Doing so introduces io errors when unknown fields are present in the input. If code is not tested with unknown fields, these bugs can remain hidden for long time. Added a special case for the memory buffer stream, where it gives a small speed benefit. Added testcase for skipping fields with test_decode2 implementation. Update issue 37 Status: FixedInGit --- docs/concepts.rst | 3 +-- example/common.c | 9 --------- pb_decode.c | 37 ++++++++++++++++++++++++++----------- pb_decode.h | 6 ++---- tests/Makefile | 3 +++ tests/person_with_extra_field.pb | Bin 0 -> 90 bytes tests/person_with_extra_field.txt | 3 +++ tests/test_decode2.c | 7 ------- 8 files changed, 35 insertions(+), 33 deletions(-) create mode 100644 tests/person_with_extra_field.pb create mode 100644 tests/person_with_extra_field.txt diff --git a/docs/concepts.rst b/docs/concepts.rst index d3261146..355af25e 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -92,9 +92,8 @@ Writing to stdout:: Input streams ------------- -For input streams, there are a few extra rules: +For input streams, there is one extra rule: -#) If buf is NULL, read from stream but don't store the data. This is used to skip unknown input. #) You don't need to know the length of the message in advance. After getting EOF error when reading, set bytes_left to 0 and return false. Pb_decode will detect this and if the EOF was in a proper position, it will return true. Here is the structure:: diff --git a/example/common.c b/example/common.c index b27ccae2..04a5aa85 100644 --- a/example/common.c +++ b/example/common.c @@ -19,15 +19,6 @@ static bool read_callback(pb_istream_t *stream, uint8_t *buf, size_t count) int fd = (intptr_t)stream->state; int result; - if (buf == NULL) - { - /* Well, this is a really inefficient way to skip input. */ - /* It is only used when there are unknown fields. */ - char dummy; - while (count-- && recv(fd, &dummy, 1, 0) == 1); - return count == 0; - } - result = recv(fd, buf, count, MSG_WAITALL); if (result == 0) diff --git a/pb_decode.c b/pb_decode.c index 86dec4b7..8e01fd7a 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -36,26 +36,41 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { * pb_istream * **************/ -bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) +static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count) { - if (stream->bytes_left < count) - PB_RETURN_ERROR(stream, "end-of-stream"); + uint8_t *source = (uint8_t*)stream->state; - if (!stream->callback(stream, buf, count)) - PB_RETURN_ERROR(stream, "io error"); + if (buf != NULL) + memcpy(buf, source, count); - stream->bytes_left -= count; + stream->state = source + count; return true; } -static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count) +bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) { - uint8_t *source = (uint8_t*)stream->state; + if (buf == NULL && stream->callback != buf_read) + { + /* Skip input bytes */ + uint8_t tmp[16]; + while (count > 16) + { + if (!pb_read(stream, tmp, 16)) + return false; + + count -= 16; + } + + return pb_read(stream, tmp, count); + } + + if (stream->bytes_left < count) + PB_RETURN_ERROR(stream, "end-of-stream"); - if (buf != NULL) - memcpy(buf, source, count); + if (!stream->callback(stream, buf, count)) + PB_RETURN_ERROR(stream, "io error"); - stream->state = source + count; + stream->bytes_left -= count; return true; } diff --git a/pb_decode.h b/pb_decode.h index 483665ee..2be92050 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -19,12 +19,10 @@ * Rules for callback: * 1) Return false on IO errors. This will cause decoding to abort. * - * 2) If buf is NULL, read but don't store bytes ("skip input"). - * - * 3) You can use state to store your own data (e.g. buffer pointer), + * 2) You can use state to store your own data (e.g. buffer pointer), * and rely on pb_read to verify that no-body reads past bytes_left. * - * 4) Your callback may be used with substreams, in which case bytes_left + * 3) Your callback may be used with substreams, in which case bytes_left * is different than from the main stream. Don't use bytes_left to compute * any pointers. */ diff --git a/tests/Makefile b/tests/Makefile index 9b02817f..73efbe63 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -70,6 +70,9 @@ run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 te [ "`./test_encode2 | ./test_decode2`" = \ "`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] + [ "`./test_decode2 < person_with_extra_field.pb`" = \ + "`cat person_with_extra_field.txt`" ] + [ "`./test_encode_callbacks | ./test_decode_callbacks`" = \ "`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ] diff --git a/tests/person_with_extra_field.pb b/tests/person_with_extra_field.pb new file mode 100644 index 00000000..00d153cb Binary files /dev/null and b/tests/person_with_extra_field.pb differ diff --git a/tests/person_with_extra_field.txt b/tests/person_with_extra_field.txt new file mode 100644 index 00000000..fae9f87d --- /dev/null +++ b/tests/person_with_extra_field.txt @@ -0,0 +1,3 @@ +name: "Test Person 99" +id: 99 +email: "test@person.com" diff --git a/tests/test_decode2.c b/tests/test_decode2.c index 762b2b3f..2142977e 100644 --- a/tests/test_decode2.c +++ b/tests/test_decode2.c @@ -59,13 +59,6 @@ bool callback(pb_istream_t *stream, uint8_t *buf, size_t count) FILE *file = (FILE*)stream->state; bool status; - if (buf == NULL) - { - /* Skipping data */ - while (count-- && fgetc(file) != EOF); - return count == 0; - } - status = (fread(buf, 1, count, file) == count); if (feof(file)) -- cgit 1.2.3-korg From bffd3a9accd748f20ad4a2d75942893684b90d81 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 21 Oct 2012 21:31:20 +0300 Subject: Improve the person_with_extra_field test. Added a field after the extra field to verify it's also ok. --- tests/Makefile | 2 +- tests/person_with_extra_field.pb | Bin 90 -> 90 bytes tests/person_with_extra_field.txt | 3 --- 3 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 tests/person_with_extra_field.txt diff --git a/tests/Makefile b/tests/Makefile index 73efbe63..76561759 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -71,7 +71,7 @@ run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 te "`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] [ "`./test_decode2 < person_with_extra_field.pb`" = \ - "`cat person_with_extra_field.txt`" ] + "`./test_encode2 | ./test_decode2`" ] [ "`./test_encode_callbacks | ./test_decode_callbacks`" = \ "`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ] diff --git a/tests/person_with_extra_field.pb b/tests/person_with_extra_field.pb index 00d153cb..ced3057c 100644 Binary files a/tests/person_with_extra_field.pb and b/tests/person_with_extra_field.pb differ diff --git a/tests/person_with_extra_field.txt b/tests/person_with_extra_field.txt deleted file mode 100644 index fae9f87d..00000000 --- a/tests/person_with_extra_field.txt +++ /dev/null @@ -1,3 +0,0 @@ -name: "Test Person 99" -id: 99 -email: "test@person.com" -- cgit 1.2.3-korg From c3fa362653480159dff15c074f59c5fab301dd0e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 29 Oct 2012 16:56:45 +0200 Subject: Give names to generated structures to allow forward declaration. Update issue 39 Status: FixedInGit --- generator/nanopb_generator.py | 4 ++-- pb.h | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index b9018aeb..1c214222 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -75,7 +75,7 @@ class Enum: self.values = [(self.names + x.name, x.number) for x in desc.value] def __str__(self): - result = 'typedef enum {\n' + result = 'typedef enum _%s {\n' % self.names result += ',\n'.join([" %s = %d" % x for x in self.values]) result += '\n} %s;' % self.names return result @@ -276,7 +276,7 @@ class Message: return [str(field.ctype) for field in self.fields] def __str__(self): - result = 'typedef struct {\n' + result = 'typedef struct _%s {\n' % self.name result += '\n'.join([str(f) for f in self.ordered_fields]) result += '\n} %s;' % self.name return result diff --git a/pb.h b/pb.h index dce8df30..72b41963 100644 --- a/pb.h +++ b/pb.h @@ -148,10 +148,12 @@ struct _pb_field_t { * It has the number of bytes in the beginning, and after that an array. * Note that actual structs used will have a different length of bytes array. */ -typedef struct { +struct _pb_bytes_array_t { size_t size; uint8_t bytes[1]; -} pb_bytes_array_t; +}; + +typedef struct _pb_bytes_array_t pb_bytes_array_t; /* This structure is used for giving the callback function. * It is stored in the message structure and filled in by the method that -- cgit 1.2.3-korg From 9e0ee92f0a42ce2c5c9d4bf4f1d7d822caf1c561 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 29 Oct 2012 17:22:02 +0200 Subject: Use optparse in nanopb_generator.py --- generator/nanopb_generator.py | 116 +++++++++++++++++++++++++++++++----------- 1 file changed, 85 insertions(+), 31 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 1c214222..6ce91cf4 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -22,6 +22,16 @@ except: print raise + + + + + +# --------------------------------------------------------------------------- +# Generation of single fields +# --------------------------------------------------------------------------- + + import os.path # Values are tuple (c type, pb ltype) @@ -264,6 +274,15 @@ class Field: return max(self.tag, self.max_size, self.max_count) + + + + +# --------------------------------------------------------------------------- +# Generation of messages (structures) +# --------------------------------------------------------------------------- + + class Message: def __init__(self, names, desc): self.name = names @@ -313,6 +332,16 @@ class Message: result += ' PB_LAST_FIELD\n};' return result + + + + + +# --------------------------------------------------------------------------- +# Processing of entire .proto files +# --------------------------------------------------------------------------- + + def iterate_messages(desc, names = Names()): '''Recursively find all messages. For each, yield name, DescriptorProto.''' if hasattr(desc, 'message_type'): @@ -475,39 +504,64 @@ def generate_source(headername, enums, messages): for msg in messages: yield msg.fields_definition() + '\n\n' -if __name__ == '__main__': - import sys - import os.path - - if len(sys.argv) != 2: - print "Usage: " + sys.argv[0] + " file.pb" - print "where file.pb has been compiled from .proto by:" - print "protoc -ofile.pb file.proto" - print "Output fill be written to file.pb.h and file.pb.c" - sys.exit(1) - - data = open(sys.argv[1], 'rb').read() - fdesc = descriptor.FileDescriptorSet.FromString(data) - enums, messages = parse_file(fdesc.file[0]) - - noext = os.path.splitext(sys.argv[1])[0] - headername = noext + '.pb.h' - sourcename = noext + '.pb.c' - headerbasename = os.path.basename(headername) - - print "Writing to " + headername + " and " + sourcename + + +# --------------------------------------------------------------------------- +# Command line interface +# --------------------------------------------------------------------------- + +import sys +import os.path +from optparse import OptionParser + +optparser = OptionParser( + usage = "Usage: nanopb_generator.py [options] file.pb ...", + epilog = "Compile file.pb from file.proto by: 'protoc -ofile.pb file.proto'. " + + "Output will be written to file.pb.h and file.pb.c.") +optparser.add_option("-x", dest="exclude", metavar="FILE", action="append", default=[], + help="Exclude file from generated #include list.") +optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, + help="Don't print anything except errors.") + +def process(filenames, options): + '''Process the files given on the command line.''' - # List of .proto files that should not be included in the C header file - # even if they are mentioned in the source .proto. - excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] - dependencies = [d for d in fdesc.file[0].dependency if d not in excludes] + if not filenames: + optparser.print_help() + return False - header = open(headername, 'w') - for part in generate_header(dependencies, headerbasename, enums, messages): - header.write(part) + for filename in filenames: + data = open(filename, 'rb').read() + fdesc = descriptor.FileDescriptorSet.FromString(data) + enums, messages = parse_file(fdesc.file[0]) + + noext = os.path.splitext(filename)[0] + headername = noext + '.pb.h' + sourcename = noext + '.pb.c' + headerbasename = os.path.basename(headername) + + if not options.quiet: + print "Writing to " + headername + " and " + sourcename + + # List of .proto files that should not be included in the C header file + # even if they are mentioned in the source .proto. + excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + dependencies = [d for d in fdesc.file[0].dependency if d not in excludes] + + header = open(headername, 'w') + for part in generate_header(dependencies, headerbasename, enums, messages): + header.write(part) - source = open(sourcename, 'w') - for part in generate_source(headerbasename, enums, messages): - source.write(part) + source = open(sourcename, 'w') + for part in generate_source(headerbasename, enums, messages): + source.write(part) + return True + +if __name__ == '__main__': + options, filenames = optparser.parse_args() + status = process(filenames, options) + + if not status: + sys.exit(1) -- cgit 1.2.3-korg From 28b0136ea4dcd045f0422d16a25b7d82b0d2aaee Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 29 Oct 2012 18:20:15 +0200 Subject: Improve .proto options parsing. Options can now be defined on command line, file, message or in field scope. Update issue 12 Status: Started --- generator/nanopb.proto | 8 +++++ generator/nanopb_generator.py | 73 +++++++++++++++++++++++++++++++++---------- generator/nanopb_pb2.py | 29 ++++++++++++++--- tests/Makefile | 10 +++++- tests/options.expected | 3 ++ tests/options.proto | 28 +++++++++++++++++ 6 files changed, 129 insertions(+), 22 deletions(-) create mode 100644 tests/options.expected create mode 100644 tests/options.proto diff --git a/generator/nanopb.proto b/generator/nanopb.proto index 2610cd5c..a377f63b 100644 --- a/generator/nanopb.proto +++ b/generator/nanopb.proto @@ -20,6 +20,14 @@ message NanoPBOptions { // Extensions: 1010 (all types) // -------------------------------- +extend google.protobuf.FileOptions { + optional NanoPBOptions nanopb_fileopt = 1010; +} + +extend google.protobuf.MessageOptions { + optional NanoPBOptions nanopb_msgopt = 1010; +} + extend google.protobuf.FieldOptions { optional NanoPBOptions nanopb = 1010; } diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 6ce91cf4..69a9eab7 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -79,7 +79,7 @@ def names_from_type_name(type_name): return Names(type_name[1:].split('.')) class Enum: - def __init__(self, names, desc): + def __init__(self, names, desc, enum_options): '''desc is EnumDescriptorProto''' self.names = names + desc.name self.values = [(self.names + x.name, x.number) for x in desc.value] @@ -91,7 +91,7 @@ class Enum: return result class Field: - def __init__(self, struct_name, desc): + def __init__(self, struct_name, desc, field_options): '''desc is FieldDescriptorProto''' self.tag = desc.number self.struct_name = struct_name @@ -101,13 +101,12 @@ class Field: self.max_count = None self.array_decl = "" - # Parse nanopb-specific field options - if desc.options.HasExtension(nanopb_pb2.nanopb): - ext = desc.options.Extensions[nanopb_pb2.nanopb] - if ext.HasField("max_size"): - self.max_size = ext.max_size - if ext.HasField("max_count"): - self.max_count = ext.max_count + # Parse field options + if field_options.HasField("max_size"): + self.max_size = field_options.max_size + + if field_options.HasField("max_count"): + self.max_count = field_options.max_count if desc.HasField('default_value'): self.default = desc.default_value @@ -284,9 +283,9 @@ class Field: class Message: - def __init__(self, names, desc): + def __init__(self, names, desc, message_options): self.name = names - self.fields = [Field(self.name, f) for f in desc.field] + self.fields = [Field(self.name, f, get_nanopb_suboptions(f, message_options)) for f in desc.field] self.ordered_fields = self.fields[:] self.ordered_fields.sort() @@ -356,7 +355,7 @@ def iterate_messages(desc, names = Names()): for x in iterate_messages(submsg, sub_names): yield x -def parse_file(fdesc): +def parse_file(fdesc, file_options): '''Takes a FileDescriptorProto and returns tuple (enum, messages).''' enums = [] @@ -368,12 +367,13 @@ def parse_file(fdesc): base_name = Names() for enum in fdesc.enum_type: - enums.append(Enum(base_name, enum)) + enums.append(Enum(base_name, enum, file_options)) for names, message in iterate_messages(fdesc, base_name): - messages.append(Message(names, message)) + message_options = get_nanopb_suboptions(message, file_options) + messages.append(Message(names, message, message_options)) for enum in message.enum_type: - enums.append(Enum(names, enum)) + enums.append(Enum(names, enum, message_options)) return enums, messages @@ -513,6 +513,7 @@ def generate_source(headername, enums, messages): import sys import os.path from optparse import OptionParser +import google.protobuf.text_format as text_format optparser = OptionParser( usage = "Usage: nanopb_generator.py [options] file.pb ...", @@ -522,6 +523,30 @@ optparser.add_option("-x", dest="exclude", metavar="FILE", action="append", defa help="Exclude file from generated #include list.") optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Don't print anything except errors.") +optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, + help="Print more information.") +optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[], + help="Set generator option (max_size, max_count etc.).") + +def get_nanopb_suboptions(subdesc, options): + '''Get copy of options, and merge information from subdesc.''' + new_options = nanopb_pb2.NanoPBOptions() + new_options.CopyFrom(options) + + if isinstance(subdesc.options, descriptor.FieldOptions): + ext_type = nanopb_pb2.nanopb + elif isinstance(subdesc.options, descriptor.FileOptions): + ext_type = nanopb_pb2.nanopb_fileopt + elif isinstance(subdesc.options, descriptor.MessageOptions): + ext_type = nanopb_pb2.nanopb_msgopt + else: + raise Exception("Unknown options type") + + if subdesc.options.HasExtension(ext_type): + ext = subdesc.options.Extensions[ext_type] + new_options.MergeFrom(ext) + + return new_options def process(filenames, options): '''Process the files given on the command line.''' @@ -530,10 +555,24 @@ def process(filenames, options): optparser.print_help() return False + if options.quiet: + options.verbose = False + + toplevel_options = nanopb_pb2.NanoPBOptions() + for s in options.settings: + text_format.Merge(s, toplevel_options) + for filename in filenames: data = open(filename, 'rb').read() fdesc = descriptor.FileDescriptorSet.FromString(data) - enums, messages = parse_file(fdesc.file[0]) + + file_options = get_nanopb_suboptions(fdesc.file[0], toplevel_options) + + if options.verbose: + print "Options for " + filename + ":" + print text_format.MessageToString(file_options) + + enums, messages = parse_file(fdesc.file[0], file_options) noext = os.path.splitext(filename)[0] headername = noext + '.pb.h' @@ -545,7 +584,7 @@ def process(filenames, options): # List of .proto files that should not be included in the C header file # even if they are mentioned in the source .proto. - excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude dependencies = [d for d in fdesc.file[0].dependency if d not in excludes] header = open(headername, 'w') diff --git a/generator/nanopb_pb2.py b/generator/nanopb_pb2.py index f2fbeefd..09378194 100644 --- a/generator/nanopb_pb2.py +++ b/generator/nanopb_pb2.py @@ -7,15 +7,33 @@ from google.protobuf import descriptor_pb2 # @@protoc_insertion_point(imports) +import google.protobuf.descriptor_pb2 + DESCRIPTOR = descriptor.FileDescriptor( name='nanopb.proto', package='', - serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"4\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions') + serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"4\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions') +NANOPB_FILEOPT_FIELD_NUMBER = 1010 +nanopb_fileopt = descriptor.FieldDescriptor( + name='nanopb_fileopt', full_name='nanopb_fileopt', index=0, + number=1010, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=True, extension_scope=None, + options=None) +NANOPB_MSGOPT_FIELD_NUMBER = 1010 +nanopb_msgopt = descriptor.FieldDescriptor( + name='nanopb_msgopt', full_name='nanopb_msgopt', index=1, + number=1010, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=True, extension_scope=None, + options=None) NANOPB_FIELD_NUMBER = 1010 nanopb = descriptor.FieldDescriptor( - name='nanopb', full_name='nanopb', index=0, + name='nanopb', full_name='nanopb', index=2, number=1010, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -57,8 +75,7 @@ _NANOPBOPTIONS = descriptor.Descriptor( serialized_end=102, ) -import google.protobuf.descriptor_pb2 - +DESCRIPTOR.message_types_by_name['NanoPBOptions'] = _NANOPBOPTIONS class NanoPBOptions(message.Message): __metaclass__ = reflection.GeneratedProtocolMessageType @@ -66,6 +83,10 @@ class NanoPBOptions(message.Message): # @@protoc_insertion_point(class_scope:NanoPBOptions) +nanopb_fileopt.message_type = _NANOPBOPTIONS +google.protobuf.descriptor_pb2.FileOptions.RegisterExtension(nanopb_fileopt) +nanopb_msgopt.message_type = _NANOPBOPTIONS +google.protobuf.descriptor_pb2.MessageOptions.RegisterExtension(nanopb_msgopt) nanopb.message_type = _NANOPBOPTIONS google.protobuf.descriptor_pb2.FieldOptions.RegisterExtension(nanopb) # @@protoc_insertion_point(module_scope) diff --git a/tests/Makefile b/tests/Makefile index 76561759..434819c3 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -55,7 +55,7 @@ coverage: run_unittests gcov pb_encode.gcda gcov pb_decode.gcda -run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields +run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields test_options rm -f *.gcda ./decode_unittests > /dev/null @@ -82,5 +82,13 @@ run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 te ./test_missing_fields +test_options: options.pb.h options.expected + for p in $$(grep . options.expected); do \ + if ! grep -qF "$$p" $<; then \ + echo Expected: $$p; \ + exit 1; \ + fi \ + done + run_fuzztest: test_decode2 bash -c 'I=1; while true; do cat /dev/urandom | ./test_decode2 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done' diff --git a/tests/options.expected b/tests/options.expected new file mode 100644 index 00000000..e184cf91 --- /dev/null +++ b/tests/options.expected @@ -0,0 +1,3 @@ +char filesize[20]; +char msgsize[30]; +char fieldsize[40]; diff --git a/tests/options.proto b/tests/options.proto new file mode 100644 index 00000000..73edf2ba --- /dev/null +++ b/tests/options.proto @@ -0,0 +1,28 @@ +/* Test nanopb option parsing. + * options.expected lists the patterns that are searched for in the output. + */ + +import "nanopb.proto"; + +// File level options +option (nanopb_fileopt).max_size = 20; + +message Message1 +{ + required string filesize = 1; +} + +// Message level options +message Message2 +{ + option (nanopb_msgopt).max_size = 30; + required string msgsize = 1; +} + +// Field level options +message Message3 +{ + required string fieldsize = 1 [(nanopb).max_size = 40]; +} + + -- cgit 1.2.3-korg From 0ee4bb96b1e53d16a29869864eff87c8934894ae Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 29 Oct 2012 18:34:24 +0200 Subject: Allow defining field type in .proto. Update issue 30 Status: FixedInGit --- generator/nanopb.proto | 7 +++++++ generator/nanopb_generator.py | 21 +++++++++++++++------ generator/nanopb_pb2.py | 41 +++++++++++++++++++++++++++++++++++++++-- tests/options.expected | 2 ++ tests/options.proto | 5 +++++ 5 files changed, 68 insertions(+), 8 deletions(-) diff --git a/generator/nanopb.proto b/generator/nanopb.proto index a377f63b..e7bea139 100644 --- a/generator/nanopb.proto +++ b/generator/nanopb.proto @@ -7,9 +7,16 @@ import "google/protobuf/descriptor.proto"; +enum FieldType { + FT_DEFAULT = 0; // Automatically decide field type, generate static field if possible. + FT_CALLBACK = 1; // Always generate a callback field. + FT_STATIC = 2; // Generate a static field or raise an exception if not possible. +} + message NanoPBOptions { optional int32 max_size = 1; optional int32 max_count = 2; + optional FieldType type = 3 [default = FT_DEFAULT]; } // Protocol Buffers extension number registry diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 69a9eab7..860f3742 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -110,18 +110,18 @@ class Field: if desc.HasField('default_value'): self.default = desc.default_value - + # Decide HTYPE # HTYPE is the high-order nibble of nanopb field description, # defining whether value is required/optional/repeated. - is_callback = False + can_be_static = True if desc.label == FieldD.LABEL_REQUIRED: self.htype = 'PB_HTYPE_REQUIRED' elif desc.label == FieldD.LABEL_OPTIONAL: self.htype = 'PB_HTYPE_OPTIONAL' elif desc.label == FieldD.LABEL_REPEATED: if self.max_count is None: - is_callback = True + can_be_static = False else: self.htype = 'PB_HTYPE_ARRAY' self.array_decl = '[%d]' % self.max_count @@ -142,14 +142,14 @@ class Field: elif desc.type == FieldD.TYPE_STRING: self.ltype = 'PB_LTYPE_STRING' if self.max_size is None: - is_callback = True + can_be_static = False else: self.ctype = 'char' self.array_decl += '[%d]' % self.max_size elif desc.type == FieldD.TYPE_BYTES: self.ltype = 'PB_LTYPE_BYTES' if self.max_size is None: - is_callback = True + can_be_static = False else: self.ctype = self.struct_name + self.name + 't' elif desc.type == FieldD.TYPE_MESSAGE: @@ -158,7 +158,16 @@ class Field: else: raise NotImplementedError(desc.type) - if is_callback: + if field_options.type == nanopb_pb2.FT_DEFAULT: + if can_be_static: + field_options.type = nanopb_pb2.FT_STATIC + else: + field_options.type = nanopb_pb2.FT_CALLBACK + + if field_options.type == nanopb_pb2.FT_STATIC and not can_be_static: + raise Exception("Field %s is defined as static, but max_size or max_count is not given." % self.name) + + if field_options.type == nanopb_pb2.FT_CALLBACK: self.htype = 'PB_HTYPE_CALLBACK' self.ctype = 'pb_callback_t' self.array_decl = '' diff --git a/generator/nanopb_pb2.py b/generator/nanopb_pb2.py index 09378194..502726a5 100644 --- a/generator/nanopb_pb2.py +++ b/generator/nanopb_pb2.py @@ -12,8 +12,37 @@ import google.protobuf.descriptor_pb2 DESCRIPTOR = descriptor.FileDescriptor( name='nanopb.proto', package='', - serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"4\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions') + serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"Z\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT*;\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions') +_FIELDTYPE = descriptor.EnumDescriptor( + name='FieldType', + full_name='FieldType', + filename=None, + file=DESCRIPTOR, + values=[ + descriptor.EnumValueDescriptor( + name='FT_DEFAULT', index=0, number=0, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='FT_CALLBACK', index=1, number=1, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='FT_STATIC', index=2, number=2, + options=None, + type=None), + ], + containing_type=None, + options=None, + serialized_start=142, + serialized_end=201, +) + + +FT_DEFAULT = 0 +FT_CALLBACK = 1 +FT_STATIC = 2 NANOPB_FILEOPT_FIELD_NUMBER = 1010 nanopb_fileopt = descriptor.FieldDescriptor( @@ -62,6 +91,13 @@ _NANOPBOPTIONS = descriptor.Descriptor( message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='type', full_name='NanoPBOptions.type', index=2, + number=3, type=14, cpp_type=8, label=1, + has_default_value=True, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -72,9 +108,10 @@ _NANOPBOPTIONS = descriptor.Descriptor( is_extendable=False, extension_ranges=[], serialized_start=50, - serialized_end=102, + serialized_end=140, ) +_NANOPBOPTIONS.fields_by_name['type'].enum_type = _FIELDTYPE DESCRIPTOR.message_types_by_name['NanoPBOptions'] = _NANOPBOPTIONS class NanoPBOptions(message.Message): diff --git a/tests/options.expected b/tests/options.expected index e184cf91..ef74a00d 100644 --- a/tests/options.expected +++ b/tests/options.expected @@ -1,3 +1,5 @@ char filesize[20]; char msgsize[30]; char fieldsize[40]; +pb_callback_t int32_callback; + diff --git a/tests/options.proto b/tests/options.proto index 73edf2ba..6ba7c070 100644 --- a/tests/options.proto +++ b/tests/options.proto @@ -25,4 +25,9 @@ message Message3 required string fieldsize = 1 [(nanopb).max_size = 40]; } +// Forced callback field +message Message4 +{ + required int32 int32_callback = 1 [(nanopb).type = FT_CALLBACK]; +} -- cgit 1.2.3-korg From db1eefc24bd9d1181dee35683c5fe12329f7d969 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 29 Oct 2012 18:55:49 +0200 Subject: Add option to use short names for enum values. Update issue 38 Status: FixedInGit --- generator/nanopb.proto | 13 +++++++++++++ generator/nanopb_generator.py | 12 ++++++++++-- generator/nanopb_pb2.py | 27 ++++++++++++++++++++++----- tests/Makefile | 6 +++--- tests/options.expected | 8 ++++---- tests/options.proto | 6 ++++++ 6 files changed, 58 insertions(+), 14 deletions(-) diff --git a/generator/nanopb.proto b/generator/nanopb.proto index e7bea139..e548a137 100644 --- a/generator/nanopb.proto +++ b/generator/nanopb.proto @@ -14,9 +14,17 @@ enum FieldType { } message NanoPBOptions { + // Allocated size for 'bytes' and 'string' fields. optional int32 max_size = 1; + + // Allocated number of entries in arrays ('repeated' fields) optional int32 max_count = 2; + + // Force type of field (callback or static allocation) optional FieldType type = 3 [default = FT_DEFAULT]; + + // Use long names for enums, i.e. EnumName_EnumValue. + optional bool long_names = 4 [default = true]; } // Protocol Buffers extension number registry @@ -35,7 +43,12 @@ extend google.protobuf.MessageOptions { optional NanoPBOptions nanopb_msgopt = 1010; } +extend google.protobuf.EnumOptions { + optional NanoPBOptions nanopb_enumopt = 1010; +} + extend google.protobuf.FieldOptions { optional NanoPBOptions nanopb = 1010; } + diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 860f3742..2267238c 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -81,7 +81,12 @@ def names_from_type_name(type_name): class Enum: def __init__(self, names, desc, enum_options): '''desc is EnumDescriptorProto''' - self.names = names + desc.name + + if enum_options.long_names: + self.names = names + desc.name + else: + self.names = names + self.values = [(self.names + x.name, x.number) for x in desc.value] def __str__(self): @@ -376,7 +381,8 @@ def parse_file(fdesc, file_options): base_name = Names() for enum in fdesc.enum_type: - enums.append(Enum(base_name, enum, file_options)) + enum_options = get_nanopb_suboptions(enum, file_options) + enums.append(Enum(base_name, enum, enum_options)) for names, message in iterate_messages(fdesc, base_name): message_options = get_nanopb_suboptions(message, file_options) @@ -548,6 +554,8 @@ def get_nanopb_suboptions(subdesc, options): ext_type = nanopb_pb2.nanopb_fileopt elif isinstance(subdesc.options, descriptor.MessageOptions): ext_type = nanopb_pb2.nanopb_msgopt + elif isinstance(subdesc.options, descriptor.EnumOptions): + ext_type = nanopb_pb2.nanopb_enumopt else: raise Exception("Unknown options type") diff --git a/generator/nanopb_pb2.py b/generator/nanopb_pb2.py index 502726a5..c48a412e 100644 --- a/generator/nanopb_pb2.py +++ b/generator/nanopb_pb2.py @@ -12,7 +12,7 @@ import google.protobuf.descriptor_pb2 DESCRIPTOR = descriptor.FileDescriptor( name='nanopb.proto', package='', - serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"Z\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT*;\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions') + serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"t\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true*;\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions') _FIELDTYPE = descriptor.EnumDescriptor( name='FieldType', @@ -35,8 +35,8 @@ _FIELDTYPE = descriptor.EnumDescriptor( ], containing_type=None, options=None, - serialized_start=142, - serialized_end=201, + serialized_start=168, + serialized_end=227, ) @@ -60,9 +60,17 @@ nanopb_msgopt = descriptor.FieldDescriptor( message_type=None, enum_type=None, containing_type=None, is_extension=True, extension_scope=None, options=None) +NANOPB_ENUMOPT_FIELD_NUMBER = 1010 +nanopb_enumopt = descriptor.FieldDescriptor( + name='nanopb_enumopt', full_name='nanopb_enumopt', index=2, + number=1010, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=True, extension_scope=None, + options=None) NANOPB_FIELD_NUMBER = 1010 nanopb = descriptor.FieldDescriptor( - name='nanopb', full_name='nanopb', index=2, + name='nanopb', full_name='nanopb', index=3, number=1010, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -98,6 +106,13 @@ _NANOPBOPTIONS = descriptor.Descriptor( message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='long_names', full_name='NanoPBOptions.long_names', index=3, + number=4, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=True, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -108,7 +123,7 @@ _NANOPBOPTIONS = descriptor.Descriptor( is_extendable=False, extension_ranges=[], serialized_start=50, - serialized_end=140, + serialized_end=166, ) _NANOPBOPTIONS.fields_by_name['type'].enum_type = _FIELDTYPE @@ -124,6 +139,8 @@ nanopb_fileopt.message_type = _NANOPBOPTIONS google.protobuf.descriptor_pb2.FileOptions.RegisterExtension(nanopb_fileopt) nanopb_msgopt.message_type = _NANOPBOPTIONS google.protobuf.descriptor_pb2.MessageOptions.RegisterExtension(nanopb_msgopt) +nanopb_enumopt.message_type = _NANOPBOPTIONS +google.protobuf.descriptor_pb2.EnumOptions.RegisterExtension(nanopb_enumopt) nanopb.message_type = _NANOPBOPTIONS google.protobuf.descriptor_pb2.FieldOptions.RegisterExtension(nanopb) # @@protoc_insertion_point(module_scope) diff --git a/tests/Makefile b/tests/Makefile index 434819c3..99e27ffd 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -83,9 +83,9 @@ run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 te ./test_missing_fields test_options: options.pb.h options.expected - for p in $$(grep . options.expected); do \ - if ! grep -qF "$$p" $<; then \ - echo Expected: $$p; \ + cat options.expected | while read -r p; do \ + if ! grep -q "$$p" $<; then \ + echo Expected: "$$p"; \ exit 1; \ fi \ done diff --git a/tests/options.expected b/tests/options.expected index ef74a00d..960daeee 100644 --- a/tests/options.expected +++ b/tests/options.expected @@ -1,5 +1,5 @@ -char filesize[20]; -char msgsize[30]; -char fieldsize[40]; +char filesize\[20\]; +char msgsize\[30\]; +char fieldsize\[40\]; pb_callback_t int32_callback; - +[^_]EnumValue1 = 1 diff --git a/tests/options.proto b/tests/options.proto index 6ba7c070..e296bcd1 100644 --- a/tests/options.proto +++ b/tests/options.proto @@ -31,3 +31,9 @@ message Message4 required int32 int32_callback = 1 [(nanopb).type = FT_CALLBACK]; } +// Short enum names +enum Enum1 +{ + option (nanopb_enumopt).long_names = false; + EnumValue1 = 1; +} -- cgit 1.2.3-korg From 3aaa4ad8c290af87d9bb9e24447f9cd6f9a79eab Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 29 Oct 2012 19:03:19 +0200 Subject: Add extern "C" to header files. Update issue 35 Status: FixedInGit --- generator/nanopb_generator.py | 9 ++++++++- pb_decode.h | 8 ++++++++ pb_encode.h | 7 +++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 2267238c..bd164e03 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -438,7 +438,10 @@ def generate_header(dependencies, headername, enums, messages): for dependency in dependencies: noext = os.path.splitext(dependency)[0] yield '#include "%s.pb.h"\n' % noext - yield '\n' + + yield '#ifdef __cplusplus\n' + yield 'extern "C" {\n' + yield '#endif\n\n' yield '/* Enum definitions */\n' for enum in enums: @@ -502,6 +505,10 @@ def generate_header(dependencies, headername, enums, messages): yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT)\n' % assertion yield '#endif\n' + yield '\n#ifdef __cplusplus\n' + yield '} /* extern "C" */\n' + yield '#endif\n' + # End of header yield '\n#endif\n' diff --git a/pb_decode.h b/pb_decode.h index 2be92050..e9f8ced8 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -12,6 +12,10 @@ #include #include "pb.h" +#ifdef __cplusplus +extern "C" { +#endif + /* Lightweight input stream. * You can provide a callback function for reading or use * pb_istream_from_buffer. @@ -102,4 +106,8 @@ bool pb_skip_varint(pb_istream_t *stream); bool pb_skip_string(pb_istream_t *stream); #endif +#ifdef __cplusplus +} /* extern "C" */ +#endif + #endif diff --git a/pb_encode.h b/pb_encode.h index af6cc3c5..85a82973 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -9,6 +9,10 @@ #include #include "pb.h" +#ifdef __cplusplus +extern "C" { +#endif + /* Lightweight output stream. * You can provide callback for writing or use pb_ostream_from_buffer. * @@ -99,5 +103,8 @@ bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *sr * instead, it has the same functionality with a less confusing interface. */ bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src); +#ifdef __cplusplus +} /* extern "C" */ +#endif #endif -- cgit 1.2.3-korg From 08391f35eeeac9583c9ae5010e5e37ca2a5bc0a4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 29 Oct 2012 19:15:34 +0200 Subject: Add nanopb version number to generated files. tools/set_version.sh is used to update the values. Update issue 36 Status: FixedInGit --- generator/nanopb_generator.py | 5 ++++- pb.h | 2 ++ tools/set_version.sh | 10 ++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) create mode 100755 tools/set_version.sh diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index bd164e03..dadad646 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,4 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' +nanopb_version = "nanopb-0.1.7-dev" try: import google.protobuf.descriptor_pb2 as descriptor @@ -31,7 +32,7 @@ except: # Generation of single fields # --------------------------------------------------------------------------- - +import time import os.path # Values are tuple (c type, pb ltype) @@ -429,6 +430,7 @@ def generate_header(dependencies, headername, enums, messages): ''' yield '/* Automatically generated nanopb header */\n' + yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) symbol = headername.replace('.', '_').upper() yield '#ifndef _PB_%s_\n' % symbol @@ -516,6 +518,7 @@ def generate_source(headername, enums, messages): '''Generate content for a source file.''' yield '/* Automatically generated nanopb constant definitions */\n' + yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) yield '#include "%s"\n\n' % headername for msg in messages: diff --git a/pb.h b/pb.h index 72b41963..cfcf6f75 100644 --- a/pb.h +++ b/pb.h @@ -6,6 +6,8 @@ * see pb_encode.h or pb_decode.h */ +#define NANOPB_VERSION nanopb-0.1.7-dev + #include #include #include diff --git a/tools/set_version.sh b/tools/set_version.sh new file mode 100755 index 00000000..e15a859d --- /dev/null +++ b/tools/set_version.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Run this from the top directory of nanopb tree. +# e.g. user@localhost:~/nanopb$ tools/set_version.sh nanopb-0.1.9-dev +# It sets the version number in pb.h and generator/nanopb_generator.py. + +sed -i -e 's/nanopb_version\s*=\s*"[^"]*"/nanopb_version = "'$1'"/' generator/nanopb_generator.py +sed -i -e 's/#define\s*NANOPB_VERSION\s*.*/#define NANOPB_VERSION '$1'/' pb.h + + -- cgit 1.2.3-korg From ad9a885644d0c3e61eecd796ea227edded96a2b5 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 29 Oct 2012 19:33:33 +0200 Subject: Document new generator options --- CHANGELOG | 10 ++++++++++ docs/concepts.rst | 14 ++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 1effb161..c7202896 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,13 @@ +nanopb-0.1.7 (not yet released) + Remove "skip" mode from pb_istream_t callbacks. Example implementation had a bug. (issue 37) + Add option to use shorter names for enum values (issue 38) + Improve options support in generator (issues 12, 30) + Add nanopb version number to generated files (issue 36) + Add extern "C" to generated headers (issue 35) + Add names for structs to allow forward declaration (issue 39) + Add buffer size check in example (issue 34) + Fix build warnings on MS compilers (issue 33) + nanopb-0.1.6 Reorganize the field decoder interface (issue 2) Improve performance in submessage decoding (issue 28) diff --git a/docs/concepts.rst b/docs/concepts.rst index 355af25e..b18c5057 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -38,6 +38,20 @@ This file, in turn, requires the file *google/protobuf/descriptor.proto*. This i protoc -I/usr/include -Inanopb/generator -I. -omessage.pb message.proto +The options can be defined in file, message and field scopes:: + + option (nanopb_fileopt).max_size = 20; // File scope + message Message + { + option (nanopb_msgopt).max_size = 30; // Message scope + required string fieldsize = 1 [(nanopb).max_size = 40]; // Field scope + } + +It is also possible to give the options on command line, but then they will affect the whole file. For example:: + + user@host:~$ python ../generator/nanopb_generator.py -s 'max_size: 20' message.pb + + Streams ======= -- cgit 1.2.3-korg From 92bb37b07420d0d3b7cb1fe62f3cb6fcc84d8bbf Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 11 Nov 2012 11:16:01 +0200 Subject: Publishing nanopb-0.1.7 --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index dadad646..36e6bf9b 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.1.7-dev" +nanopb_version = "nanopb-0.1.7" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index cfcf6f75..1be8d4ed 100644 --- a/pb.h +++ b/pb.h @@ -6,7 +6,7 @@ * see pb_encode.h or pb_decode.h */ -#define NANOPB_VERSION nanopb-0.1.7-dev +#define NANOPB_VERSION nanopb-0.1.7 #include #include -- cgit 1.2.3-korg From fc6f56b2bd3b8a04884c55017144a85f3e69af1f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 11 Nov 2012 11:19:17 +0200 Subject: Setting version to 0.1.8-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 36e6bf9b..9e6f2e2b 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.1.7" +nanopb_version = "nanopb-0.1.8-dev" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index 1be8d4ed..dafdfd51 100644 --- a/pb.h +++ b/pb.h @@ -6,7 +6,7 @@ * see pb_encode.h or pb_decode.h */ -#define NANOPB_VERSION nanopb-0.1.7 +#define NANOPB_VERSION nanopb-0.1.8-dev #include #include -- cgit 1.2.3-korg From 332a9ee95c38db4537d0a33e58fef617d3e1d589 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 11 Nov 2012 11:20:34 +0200 Subject: Fix changelog --- CHANGELOG | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index c7202896..ecb13e02 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,4 @@ -nanopb-0.1.7 (not yet released) +nanopb-0.1.7 Remove "skip" mode from pb_istream_t callbacks. Example implementation had a bug. (issue 37) Add option to use shorter names for enum values (issue 38) Improve options support in generator (issues 12, 30) -- cgit 1.2.3-korg From 02ecee2de89f3225a6cd4bf87b0e4285bb1d06eb Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 14 Nov 2012 00:20:44 +0200 Subject: Fix naming of enums when long_names=false. Modify test case to check that options.pb.o compiles. Update issue 42 Status: FixedInGit Update issue 43 Status: FixedInGit --- generator/nanopb_generator.py | 21 ++++++++++++++++++--- tests/Makefile | 2 +- tests/options.expected | 2 +- tests/options.proto | 6 ++++++ 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 9e6f2e2b..68cc800c 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -73,6 +73,9 @@ class Names: else: raise ValueError("Name parts should be of type str") + def __eq__(self, other): + return isinstance(other, Names) and self.parts == other.parts + def names_from_type_name(type_name): '''Parse Names() from FieldDescriptorProto type_name''' if type_name[0] != '.': @@ -83,12 +86,15 @@ class Enum: def __init__(self, names, desc, enum_options): '''desc is EnumDescriptorProto''' + self.options = enum_options + self.names = names + desc.name + if enum_options.long_names: - self.names = names + desc.name + self.values = [(self.names + x.name, x.number) for x in desc.value] else: - self.names = names + self.values = [(x.name, x.number) for x in desc.value] - self.values = [(self.names + x.name, x.number) for x in desc.value] + self.value_longnames = [names + desc.name + x.name for x in desc.value] def __str__(self): result = 'typedef enum _%s {\n' % self.names @@ -391,6 +397,15 @@ def parse_file(fdesc, file_options): for enum in message.enum_type: enums.append(Enum(names, enum, message_options)) + # Fix field default values where enum short names are used. + for enum in enums: + if not enum.options.long_names: + for message in messages: + for field in message.fields: + if field.default in enum.value_longnames: + idx = enum.value_longnames.index(field.default) + field.default = enum.values[idx][0] + return enums, messages def toposort2(data): diff --git a/tests/Makefile b/tests/Makefile index 99e27ffd..1f2be716 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -82,7 +82,7 @@ run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 te ./test_missing_fields -test_options: options.pb.h options.expected +test_options: options.pb.h options.expected options.pb.o cat options.expected | while read -r p; do \ if ! grep -q "$$p" $<; then \ echo Expected: "$$p"; \ diff --git a/tests/options.expected b/tests/options.expected index 960daeee..61d9805e 100644 --- a/tests/options.expected +++ b/tests/options.expected @@ -2,4 +2,4 @@ char filesize\[20\]; char msgsize\[30\]; char fieldsize\[40\]; pb_callback_t int32_callback; -[^_]EnumValue1 = 1 +\sEnumValue1 = 1 diff --git a/tests/options.proto b/tests/options.proto index e296bcd1..7eb2eb30 100644 --- a/tests/options.proto +++ b/tests/options.proto @@ -36,4 +36,10 @@ enum Enum1 { option (nanopb_enumopt).long_names = false; EnumValue1 = 1; + EnumValue2 = 2; +} + +message EnumTest +{ + required Enum1 field = 1 [default = EnumValue2]; } -- cgit 1.2.3-korg From 0abb764b1816e6a986c10c30444530e11d112cb1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 14 Nov 2012 00:43:51 +0200 Subject: Fix naming of nested Enums with short names --- generator/nanopb_generator.py | 7 ++++--- tests/options.expected | 1 + tests/options.proto | 11 +++++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 68cc800c..6aac1b8c 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -92,9 +92,9 @@ class Enum: if enum_options.long_names: self.values = [(self.names + x.name, x.number) for x in desc.value] else: - self.values = [(x.name, x.number) for x in desc.value] + self.values = [(names + x.name, x.number) for x in desc.value] - self.value_longnames = [names + desc.name + x.name for x in desc.value] + self.value_longnames = [self.names + x.name for x in desc.value] def __str__(self): result = 'typedef enum _%s {\n' % self.names @@ -395,7 +395,8 @@ def parse_file(fdesc, file_options): message_options = get_nanopb_suboptions(message, file_options) messages.append(Message(names, message, message_options)) for enum in message.enum_type: - enums.append(Enum(names, enum, message_options)) + enum_options = get_nanopb_suboptions(enum, message_options) + enums.append(Enum(names, enum, enum_options)) # Fix field default values where enum short names are used. for enum in enums: diff --git a/tests/options.expected b/tests/options.expected index 61d9805e..93501430 100644 --- a/tests/options.expected +++ b/tests/options.expected @@ -3,3 +3,4 @@ char msgsize\[30\]; char fieldsize\[40\]; pb_callback_t int32_callback; \sEnumValue1 = 1 +Message5_EnumValue1 diff --git a/tests/options.proto b/tests/options.proto index 7eb2eb30..413d21b5 100644 --- a/tests/options.proto +++ b/tests/options.proto @@ -43,3 +43,14 @@ message EnumTest { required Enum1 field = 1 [default = EnumValue2]; } + +// Short enum names inside message +message Message5 +{ + enum Enum2 + { + option (nanopb_enumopt).long_names = false; + EnumValue1 = 1; + } + required Enum2 field = 1 [default = EnumValue1]; +} -- cgit 1.2.3-korg From cc29958d34e43ed889536ad4a723fce819544785 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Sun, 11 Nov 2012 22:48:21 +0000 Subject: Fix STATIC_ASSERT macro when using multiple .proto files. The __COUNTER__ macro (used for generating unique names) is at least supported by gcc, clang and Visual Studio. With this change test_compiles.c is compilable, since no more typedefs are redefined. Compilers/Preprocessors not supporting __COUNTER__ error's are still possible which are hopfully handled by the usage of __LINE__ in most sittuations. Added unit test for the problem. --- generator/nanopb_generator.py | 8 ++++++-- pb.h | 4 +++- tests/Makefile | 7 +++++-- tests/callbacks2.proto | 9 +++++++++ tests/test_compiles.c | 13 +++++++++++++ 5 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 tests/callbacks2.proto create mode 100644 tests/test_compiles.c diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 6aac1b8c..2a3cab28 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -493,7 +493,9 @@ def generate_header(dependencies, headername, enums, messages): worst = 0 worst_field = '' checks = [] + checks_msgnames = [] for msg in messages: + checks_msgnames.append(msg.name) for field in msg.fields: status = field.largest_field_value() if isinstance(status, (str, unicode)): @@ -511,7 +513,8 @@ def generate_header(dependencies, headername, enums, messages): yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field else: assertion = ' && '.join(str(c) + ' < 256' for c in checks) - yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT)\n' % assertion + msgs = '_'.join(str(n) for n in checks_msgnames) + yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) yield '#endif\n\n' if worst > 65535 or checks: @@ -520,7 +523,8 @@ def generate_header(dependencies, headername, enums, messages): yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field else: assertion = ' && '.join(str(c) + ' < 65536' for c in checks) - yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT)\n' % assertion + msgs = '_'.join(str(n) for n in checks_msgnames) + yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) yield '#endif\n' yield '\n#ifdef __cplusplus\n' diff --git a/pb.h b/pb.h index dafdfd51..22b10d89 100644 --- a/pb.h +++ b/pb.h @@ -26,7 +26,9 @@ /* Compile-time assertion, used for checking compatible compilation options. */ #ifndef STATIC_ASSERT -#define STATIC_ASSERT(COND,MSG) typedef char static_assertion_##MSG[(COND)?1:-1]; +#define STATIC_ASSERT(COND,MSG) typedef char STATIC_ASSERT_MSG(MSG, __LINE__, __COUNTER__)[(COND)?1:-1]; +#define STATIC_ASSERT_MSG(MSG, LINE, COUNTER) STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) +#define STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) static_assertion_##MSG##LINE##COUNTER #endif /* Number of required fields to keep track of diff --git a/tests/Makefile b/tests/Makefile index 1f2be716..f3a64d14 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,7 +1,9 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage LDFLAGS=--coverage -DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h missing_fields.pb.h +DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks2.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h missing_fields.pb.h TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages +TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages test_compiles + # More strict checks for the core part of nanopb CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wlogical-op -Wconversion @@ -35,6 +37,7 @@ test_decode3: test_decode3.o pb_decode.o alltypes.pb.o test_encode1: test_encode1.o pb_encode.o person.pb.o test_encode2: test_encode2.o pb_encode.o person.pb.o test_encode3: test_encode3.o pb_encode.o alltypes.pb.o +test_compiles: test_compiles.o pb_encode.o callbacks2.pb.o callbacks.pb.o test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o test_missing_fields: test_missing_fields.o pb_encode.o pb_decode.o missing_fields.pb.o @@ -55,7 +58,7 @@ coverage: run_unittests gcov pb_encode.gcda gcov pb_decode.gcda -run_unittests: decode_unittests encode_unittests test_cxxcompile test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields test_options +run_unittests: decode_unittests encode_unittests test_cxxcompile test_compiles test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields test_options rm -f *.gcda ./decode_unittests > /dev/null diff --git a/tests/callbacks2.proto b/tests/callbacks2.proto new file mode 100644 index 00000000..9a55e15a --- /dev/null +++ b/tests/callbacks2.proto @@ -0,0 +1,9 @@ +// Test if including generated header file for this file + implicit include of +// callbacks.pb.h still compiles. Used with test_compiles.c. +import "callbacks.proto"; + +message Callback2Message { + required TestMessage tstmsg = 1; + required SubMessage submsg = 2; +} + diff --git a/tests/test_compiles.c b/tests/test_compiles.c new file mode 100644 index 00000000..cb4e16d3 --- /dev/null +++ b/tests/test_compiles.c @@ -0,0 +1,13 @@ +/* + * Tests if still compile if typedefs are redfefined in STATIC_ASSERTS when + * proto file includes another poto file + */ + +#include +#include +#include "callbacks2.pb.h" + +int main() +{ + return 0; +} -- cgit 1.2.3-korg From 59788e2aabe41d46164184d4c31fd877397b62ee Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 16 Nov 2012 09:33:11 +0200 Subject: Rename test_compiles.c to test_multiple_files.c --- pb.h | 4 +++- tests/Makefile | 6 +++--- tests/test_compiles.c | 13 ------------- tests/test_multiple_files.c | 13 +++++++++++++ 4 files changed, 19 insertions(+), 17 deletions(-) delete mode 100644 tests/test_compiles.c create mode 100644 tests/test_multiple_files.c diff --git a/pb.h b/pb.h index 22b10d89..5e9fad21 100644 --- a/pb.h +++ b/pb.h @@ -24,7 +24,9 @@ #define UNUSED(x) (void)(x) #endif -/* Compile-time assertion, used for checking compatible compilation options. */ +/* Compile-time assertion, used for checking compatible compilation options. + * If this fails on your compiler for some reason, use #define STATIC_ASSERT + * to disable it. */ #ifndef STATIC_ASSERT #define STATIC_ASSERT(COND,MSG) typedef char STATIC_ASSERT_MSG(MSG, __LINE__, __COUNTER__)[(COND)?1:-1]; #define STATIC_ASSERT_MSG(MSG, LINE, COUNTER) STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) diff --git a/tests/Makefile b/tests/Makefile index f3a64d14..a21e3c43 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -2,7 +2,7 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage LDFLAGS=--coverage DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks2.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h missing_fields.pb.h TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages -TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages test_compiles +TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages test_multiple_files # More strict checks for the core part of nanopb @@ -37,7 +37,7 @@ test_decode3: test_decode3.o pb_decode.o alltypes.pb.o test_encode1: test_encode1.o pb_encode.o person.pb.o test_encode2: test_encode2.o pb_encode.o person.pb.o test_encode3: test_encode3.o pb_encode.o alltypes.pb.o -test_compiles: test_compiles.o pb_encode.o callbacks2.pb.o callbacks.pb.o +test_multiple_files: test_multiple_files.o pb_encode.o callbacks2.pb.o callbacks.pb.o test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o test_missing_fields: test_missing_fields.o pb_encode.o pb_decode.o missing_fields.pb.o @@ -58,7 +58,7 @@ coverage: run_unittests gcov pb_encode.gcda gcov pb_decode.gcda -run_unittests: decode_unittests encode_unittests test_cxxcompile test_compiles test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields test_options +run_unittests: decode_unittests encode_unittests test_cxxcompile test_multiple_files test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields test_options rm -f *.gcda ./decode_unittests > /dev/null diff --git a/tests/test_compiles.c b/tests/test_compiles.c deleted file mode 100644 index cb4e16d3..00000000 --- a/tests/test_compiles.c +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Tests if still compile if typedefs are redfefined in STATIC_ASSERTS when - * proto file includes another poto file - */ - -#include -#include -#include "callbacks2.pb.h" - -int main() -{ - return 0; -} diff --git a/tests/test_multiple_files.c b/tests/test_multiple_files.c new file mode 100644 index 00000000..cb4e16d3 --- /dev/null +++ b/tests/test_multiple_files.c @@ -0,0 +1,13 @@ +/* + * Tests if still compile if typedefs are redfefined in STATIC_ASSERTS when + * proto file includes another poto file + */ + +#include +#include +#include "callbacks2.pb.h" + +int main() +{ + return 0; +} -- cgit 1.2.3-korg From 434dcbb2eeed391ad418c7d311bd8d6df0cdc5d7 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 16 Nov 2012 09:51:23 +0200 Subject: Select compilation options based on $(CC) in tests/Makefile. Makes 'make CC=clang' work. Based on patch submitted by Steffen Siering. Update issue 40: Status: FixedInGit --- tests/Makefile | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index a21e3c43..06062127 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,12 +1,20 @@ -CFLAGS=-ansi -Wall -Werror -I .. -g -O0 --coverage -LDFLAGS=--coverage +CFLAGS=-ansi -Wall -Werror -I .. -g -O0 DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks2.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h missing_fields.pb.h TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages test_multiple_files # More strict checks for the core part of nanopb -CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wlogical-op -Wconversion +CC_VERSION=$(shell $(CC) -v 2>&1) +CFLAGS_CORE= +ifneq (,$(findstring gcc,$(CC_VERSION))) + CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wlogical-op -Wconversion + CFLAGS+=--coverage + LDFLAGS+=--coverage +endif +ifneq (,$(findstring clang,$(CC_VERSION))) + CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wconversion +endif all: breakpoints $(TESTS) run_unittests -- cgit 1.2.3-korg From 1f8fb1f1ed3dbb606241db191bc3c0b5683fd4a8 Mon Sep 17 00:00:00 2001 From: Steffen Siering Date: Sun, 11 Nov 2012 23:02:26 +0000 Subject: Use TESTS variable to define dependencies for run_unittests --- tests/Makefile | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 06062127..58b84a3b 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,8 +1,13 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 -DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h callbacks2.pb.h callbacks.pb.h unittests.h unittestproto.pb.h alltypes.pb.h missing_fields.pb.h -TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages -TESTS=test_decode1 test_encode1 decode_unittests encode_unittests test_no_messages test_multiple_files - +DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h \ + callbacks2.pb.h callbacks.pb.h unittests.h unittestproto.pb.h \ + alltypes.pb.h missing_fields.pb.h +TESTS= decode_unittests encode_unittests \ + test_decode1 test_decode2 test_decode3 \ + test_encode1 test_encode2 test_encode3 \ + test_decode_callbacks test_encode_callbacks \ + test_missing_fields test_no_messages \ + test_multiple_files test_cxxcompile test_options # More strict checks for the core part of nanopb CC_VERSION=$(shell $(CC) -v 2>&1) @@ -19,7 +24,7 @@ endif all: breakpoints $(TESTS) run_unittests clean: - rm -f $(TESTS) person.pb* alltypes.pb* *.o *.gcda *.gcno + rm -f $(TESTS) person.pb* alltypes.pb* *.o *.gcda *.gcno *.pb.h *.pb.c %.pb.o: %.pb.c %.pb.h $(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< @@ -66,7 +71,7 @@ coverage: run_unittests gcov pb_encode.gcda gcov pb_decode.gcda -run_unittests: decode_unittests encode_unittests test_cxxcompile test_multiple_files test_encode1 test_encode2 test_encode3 test_decode1 test_decode2 test_decode3 test_encode_callbacks test_decode_callbacks test_missing_fields test_options +run_unittests: $(TESTS) rm -f *.gcda ./decode_unittests > /dev/null -- cgit 1.2.3-korg From 068de05c51b7b46e33ef87b8d78c9dda6704abc8 Mon Sep 17 00:00:00 2001 From: Jens Steinhauser Date: Tue, 27 Nov 2012 00:19:55 +0100 Subject: Complete initialization of pb_istream_t. Because PB_RETURN_ERROR checks if the 'errmsg' member is NULL before assigning to it, error messages would get lost. --- pb_decode.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pb_decode.c b/pb_decode.c index 8e01fd7a..c9652afb 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -80,6 +80,9 @@ pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize) stream.callback = &buf_read; stream.state = buf; stream.bytes_left = bufsize; +#ifndef PB_NO_ERRMSG + stream.errmsg = NULL; +#endif return stream; } -- cgit 1.2.3-korg From 871e5be9dd6bb69e2bee2d8e23d27b89285c8f8b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 6 Dec 2012 11:13:53 +0200 Subject: Fix small error in field callback documentation. Update issue 44 Status: FixedInGit --- docs/reference.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference.rst b/docs/reference.rst index 371c155a..2b10c2ee 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -381,7 +381,7 @@ Remove the data for a field from the stream, without actually decoding it:: For decoding numeric (including enumerated and boolean) values, use `pb_decode_varint`_, `pb_decode_svarint`_, `pb_decode_fixed32`_ and `pb_decode_fixed64`_. They take a pointer to a 32- or 64-bit C variable, which you may then cast to smaller datatype for storage. - For decoding strings and bytes fields, the length has already been decoded. You can therefore check the total length in *stream->state* and read the data using `pb_read`_. + For decoding strings and bytes fields, the length has already been decoded. You can therefore check the total length in *stream->bytes_left* and read the data using `pb_read`_. Finally, for decoding submessages in a callback, simply use `pb_decode`_ and pass it the *SubMessage_fields* descriptor array. -- cgit 1.2.3-korg From 09ec60cadf2436c864f2626e3726a35c2d8394bb Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 13 Dec 2012 21:29:57 +0200 Subject: Publishing nanopb-0.1.8 --- CHANGELOG | 6 ++++++ generator/nanopb_generator.py | 2 +- pb.h | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index ecb13e02..fa849235 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,9 @@ +nanopb-0.1.8 + Fix bugs in the enum short names introduced in 0.1.7 (issues 42, 43) + Fix STATIC_ASSERT macro when using multiple .proto files. (issue 41) + Fix missing initialization of istream.errmsg + Make tests/Makefile work for non-gcc compilers (issue 40) + nanopb-0.1.7 Remove "skip" mode from pb_istream_t callbacks. Example implementation had a bug. (issue 37) Add option to use shorter names for enum values (issue 38) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 2a3cab28..feb7f14d 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.1.8-dev" +nanopb_version = "nanopb-0.1.8" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index 5e9fad21..14d9c1ca 100644 --- a/pb.h +++ b/pb.h @@ -6,7 +6,7 @@ * see pb_encode.h or pb_decode.h */ -#define NANOPB_VERSION nanopb-0.1.8-dev +#define NANOPB_VERSION nanopb-0.1.8 #include #include -- cgit 1.2.3-korg From bb5dc04584f4e147cf3c892ab004ecd36ba7642b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 13 Dec 2012 21:32:37 +0200 Subject: Setting version to 0.1.9-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index feb7f14d..bffdb353 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.1.8" +nanopb_version = "nanopb-0.1.9-dev" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index 14d9c1ca..fbdf185e 100644 --- a/pb.h +++ b/pb.h @@ -6,7 +6,7 @@ * see pb_encode.h or pb_decode.h */ -#define NANOPB_VERSION nanopb-0.1.8 +#define NANOPB_VERSION nanopb-0.1.9-dev #include #include -- cgit 1.2.3-korg From 93ffe14a0ae32eeda8275f48106e76ecc650168d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 10 Jan 2013 17:31:33 +0200 Subject: Cleanup: get rid of unnecessary spaces --- generator/nanopb.proto | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/generator/nanopb.proto b/generator/nanopb.proto index e548a137..94ce7d9c 100644 --- a/generator/nanopb.proto +++ b/generator/nanopb.proto @@ -28,12 +28,12 @@ message NanoPBOptions { } // Protocol Buffers extension number registry -// -------------------------------- -// Project: Nanopb -// Contact: Petteri Aimonen -// Web site: http://kapsi.fi/~jpa/nanopb -// Extensions: 1010 (all types) -// -------------------------------- +// -------------------------------- +// Project: Nanopb +// Contact: Petteri Aimonen +// Web site: http://kapsi.fi/~jpa/nanopb +// Extensions: 1010 (all types) +// -------------------------------- extend google.protobuf.FileOptions { optional NanoPBOptions nanopb_fileopt = 1010; -- cgit 1.2.3-korg From d2c1604d6d239cdab6b21f3daf6634eb112ae318 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 10 Jan 2013 17:32:27 +0200 Subject: Add generator option for packed structs. Usage is: message Foo { option (nanopb_msgopt).packed_struct = true; ... } Valid also in file scope. Update issue 49 Status: FixedInGit --- generator/nanopb.proto | 3 +++ generator/nanopb_generator.py | 8 +++++++- generator/nanopb_pb2.py | 17 ++++++++++++----- tests/options.expected | 1 + tests/options.proto | 8 ++++++++ 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/generator/nanopb.proto b/generator/nanopb.proto index 94ce7d9c..b9100abd 100644 --- a/generator/nanopb.proto +++ b/generator/nanopb.proto @@ -25,6 +25,9 @@ message NanoPBOptions { // Use long names for enums, i.e. EnumName_EnumValue. optional bool long_names = 4 [default = true]; + + // Add 'packed' attribute to generated structs. + optional bool packed_struct = 5 [default = false]; } // Protocol Buffers extension number registry diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index bffdb353..a583b3f6 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -307,6 +307,7 @@ class Message: def __init__(self, names, desc, message_options): self.name = names self.fields = [Field(self.name, f, get_nanopb_suboptions(f, message_options)) for f in desc.field] + self.packed = message_options.packed_struct self.ordered_fields = self.fields[:] self.ordered_fields.sort() @@ -317,7 +318,12 @@ class Message: def __str__(self): result = 'typedef struct _%s {\n' % self.name result += '\n'.join([str(f) for f in self.ordered_fields]) - result += '\n} %s;' % self.name + result += '\n}' + + if self.packed: + result += ' pb_packed' + + result += ' %s;' % self.name return result def types(self): diff --git a/generator/nanopb_pb2.py b/generator/nanopb_pb2.py index c48a412e..d08b82ab 100644 --- a/generator/nanopb_pb2.py +++ b/generator/nanopb_pb2.py @@ -12,7 +12,7 @@ import google.protobuf.descriptor_pb2 DESCRIPTOR = descriptor.FileDescriptor( name='nanopb.proto', package='', - serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"t\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true*;\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions') + serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*;\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions') _FIELDTYPE = descriptor.EnumDescriptor( name='FieldType', @@ -35,8 +35,8 @@ _FIELDTYPE = descriptor.EnumDescriptor( ], containing_type=None, options=None, - serialized_start=168, - serialized_end=227, + serialized_start=199, + serialized_end=258, ) @@ -113,6 +113,13 @@ _NANOPBOPTIONS = descriptor.Descriptor( message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, options=None), + descriptor.FieldDescriptor( + name='packed_struct', full_name='NanoPBOptions.packed_struct', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), ], extensions=[ ], @@ -122,8 +129,8 @@ _NANOPBOPTIONS = descriptor.Descriptor( options=None, is_extendable=False, extension_ranges=[], - serialized_start=50, - serialized_end=166, + serialized_start=51, + serialized_end=197, ) _NANOPBOPTIONS.fields_by_name['type'].enum_type = _FIELDTYPE diff --git a/tests/options.expected b/tests/options.expected index 93501430..e6179a27 100644 --- a/tests/options.expected +++ b/tests/options.expected @@ -4,3 +4,4 @@ char fieldsize\[40\]; pb_callback_t int32_callback; \sEnumValue1 = 1 Message5_EnumValue1 +} pb_packed my_packed_struct; diff --git a/tests/options.proto b/tests/options.proto index 413d21b5..3e9383cd 100644 --- a/tests/options.proto +++ b/tests/options.proto @@ -54,3 +54,11 @@ message Message5 } required Enum2 field = 1 [default = EnumValue1]; } + +// Packed structure +message my_packed_struct +{ + option (nanopb_msgopt).packed_struct = true; + optional int32 myfield = 1; +} + -- cgit 1.2.3-korg From b9baec6b4c3a04136ea0430109bc7bc1eda12685 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 13 Jan 2013 18:44:15 +0200 Subject: Add a test for the backwards compatibility of generated files. It is not necessary to maintain full compatibility of generated files for all of eternity, but this test will warn us if there is a need to regenerate the files. --- tests/Makefile | 6 +- tests/bc_alltypes.pb.c | 326 +++++++++++++++++++++++++++++++++++++++++++++++++ tests/bc_alltypes.pb.h | 155 +++++++++++++++++++++++ tests/bc_decode.c | 197 ++++++++++++++++++++++++++++++ tests/bc_encode.c | 131 ++++++++++++++++++++ 5 files changed, 814 insertions(+), 1 deletion(-) create mode 100644 tests/bc_alltypes.pb.c create mode 100644 tests/bc_alltypes.pb.h create mode 100644 tests/bc_decode.c create mode 100644 tests/bc_encode.c diff --git a/tests/Makefile b/tests/Makefile index 58b84a3b..63830999 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -7,7 +7,8 @@ TESTS= decode_unittests encode_unittests \ test_encode1 test_encode2 test_encode3 \ test_decode_callbacks test_encode_callbacks \ test_missing_fields test_no_messages \ - test_multiple_files test_cxxcompile test_options + test_multiple_files test_cxxcompile test_options \ + bc_encode bc_decode # More strict checks for the core part of nanopb CC_VERSION=$(shell $(CC) -v 2>&1) @@ -57,6 +58,8 @@ test_missing_fields: test_missing_fields.o pb_encode.o pb_decode.o missing_field decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o encode_unittests: encode_unittests.o pb_encode.o unittestproto.pb.o test_no_messages: no_messages.pb.h no_messages.pb.c no_messages.pb.o +bc_encode: bc_alltypes.pb.o pb_encode.o bc_encode.o +bc_decode: bc_alltypes.pb.o pb_decode.o bc_decode.o %.pb: %.proto protoc -I. -I../generator -I/usr/include -o$@ $< @@ -95,6 +98,7 @@ run_unittests: $(TESTS) ./test_encode3 | ./test_decode3 ./test_encode3 1 | ./test_decode3 1 ./test_encode3 1 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null + ./bc_encode | ./bc_decode ./test_missing_fields diff --git a/tests/bc_alltypes.pb.c b/tests/bc_alltypes.pb.c new file mode 100644 index 00000000..322b634e --- /dev/null +++ b/tests/bc_alltypes.pb.c @@ -0,0 +1,326 @@ +/* Automatically generated nanopb constant definitions */ +#include "bc_alltypes.pb.h" + +const char SubMessage_substuff1_default[17] = "1"; +const int32_t SubMessage_substuff2_default = 2; +const uint32_t SubMessage_substuff3_default = 3; +const int32_t AllTypes_opt_int32_default = 4041; +const int64_t AllTypes_opt_int64_default = 4042; +const uint32_t AllTypes_opt_uint32_default = 4043; +const uint64_t AllTypes_opt_uint64_default = 4044; +const int32_t AllTypes_opt_sint32_default = 4045; +const int64_t AllTypes_opt_sint64_default = 4046; +const bool AllTypes_opt_bool_default = false; +const uint32_t AllTypes_opt_fixed32_default = 4048; +const int32_t AllTypes_opt_sfixed32_default = 4049; +const float AllTypes_opt_float_default = 4050; +const uint64_t AllTypes_opt_fixed64_default = 4051; +const int64_t AllTypes_opt_sfixed64_default = 4052; +const double AllTypes_opt_double_default = 4053; +const char AllTypes_opt_string_default[17] = "4054"; +const AllTypes_opt_bytes_t AllTypes_opt_bytes_default = {4, {0x34,0x30,0x35,0x35}}; +const MyEnum AllTypes_opt_enum_default = MyEnum_Second; + + +const pb_field_t SubMessage_fields[4] = { + {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING, + offsetof(SubMessage, substuff1), 0, + pb_membersize(SubMessage, substuff1), 0, + &SubMessage_substuff1_default}, + + {2, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, + pb_delta_end(SubMessage, substuff2, substuff1), 0, + pb_membersize(SubMessage, substuff2), 0, + &SubMessage_substuff2_default}, + + {3, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED32, + pb_delta_end(SubMessage, substuff3, substuff2), + pb_delta(SubMessage, has_substuff3, substuff3), + pb_membersize(SubMessage, substuff3), 0, + &SubMessage_substuff3_default}, + + PB_LAST_FIELD +}; + +const pb_field_t AllTypes_fields[53] = { + {1, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, + offsetof(AllTypes, req_int32), 0, + pb_membersize(AllTypes, req_int32), 0, 0}, + + {2, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, req_int64, req_int32), 0, + pb_membersize(AllTypes, req_int64), 0, 0}, + + {3, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, req_uint32, req_int64), 0, + pb_membersize(AllTypes, req_uint32), 0, 0}, + + {4, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, req_uint64, req_uint32), 0, + pb_membersize(AllTypes, req_uint64), 0, 0}, + + {5, PB_HTYPE_REQUIRED | PB_LTYPE_SVARINT, + pb_delta_end(AllTypes, req_sint32, req_uint64), 0, + pb_membersize(AllTypes, req_sint32), 0, 0}, + + {6, PB_HTYPE_REQUIRED | PB_LTYPE_SVARINT, + pb_delta_end(AllTypes, req_sint64, req_sint32), 0, + pb_membersize(AllTypes, req_sint64), 0, 0}, + + {7, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, req_bool, req_sint64), 0, + pb_membersize(AllTypes, req_bool), 0, 0}, + + {8, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED32, + pb_delta_end(AllTypes, req_fixed32, req_bool), 0, + pb_membersize(AllTypes, req_fixed32), 0, 0}, + + {9, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED32, + pb_delta_end(AllTypes, req_sfixed32, req_fixed32), 0, + pb_membersize(AllTypes, req_sfixed32), 0, 0}, + + {10, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED32, + pb_delta_end(AllTypes, req_float, req_sfixed32), 0, + pb_membersize(AllTypes, req_float), 0, 0}, + + {11, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED64, + pb_delta_end(AllTypes, req_fixed64, req_float), 0, + pb_membersize(AllTypes, req_fixed64), 0, 0}, + + {12, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED64, + pb_delta_end(AllTypes, req_sfixed64, req_fixed64), 0, + pb_membersize(AllTypes, req_sfixed64), 0, 0}, + + {13, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED64, + pb_delta_end(AllTypes, req_double, req_sfixed64), 0, + pb_membersize(AllTypes, req_double), 0, 0}, + + {14, PB_HTYPE_REQUIRED | PB_LTYPE_STRING, + pb_delta_end(AllTypes, req_string, req_double), 0, + pb_membersize(AllTypes, req_string), 0, 0}, + + {15, PB_HTYPE_REQUIRED | PB_LTYPE_BYTES, + pb_delta_end(AllTypes, req_bytes, req_string), 0, + pb_membersize(AllTypes, req_bytes), 0, 0}, + + {16, PB_HTYPE_REQUIRED | PB_LTYPE_SUBMESSAGE, + pb_delta_end(AllTypes, req_submsg, req_bytes), 0, + pb_membersize(AllTypes, req_submsg), 0, + &SubMessage_fields}, + + {17, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, req_enum, req_submsg), 0, + pb_membersize(AllTypes, req_enum), 0, 0}, + + {21, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, rep_int32, req_enum), + pb_delta(AllTypes, rep_int32_count, rep_int32), + pb_membersize(AllTypes, rep_int32[0]), + pb_membersize(AllTypes, rep_int32) / pb_membersize(AllTypes, rep_int32[0]), 0}, + + {22, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, rep_int64, rep_int32), + pb_delta(AllTypes, rep_int64_count, rep_int64), + pb_membersize(AllTypes, rep_int64[0]), + pb_membersize(AllTypes, rep_int64) / pb_membersize(AllTypes, rep_int64[0]), 0}, + + {23, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, rep_uint32, rep_int64), + pb_delta(AllTypes, rep_uint32_count, rep_uint32), + pb_membersize(AllTypes, rep_uint32[0]), + pb_membersize(AllTypes, rep_uint32) / pb_membersize(AllTypes, rep_uint32[0]), 0}, + + {24, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, rep_uint64, rep_uint32), + pb_delta(AllTypes, rep_uint64_count, rep_uint64), + pb_membersize(AllTypes, rep_uint64[0]), + pb_membersize(AllTypes, rep_uint64) / pb_membersize(AllTypes, rep_uint64[0]), 0}, + + {25, PB_HTYPE_ARRAY | PB_LTYPE_SVARINT, + pb_delta_end(AllTypes, rep_sint32, rep_uint64), + pb_delta(AllTypes, rep_sint32_count, rep_sint32), + pb_membersize(AllTypes, rep_sint32[0]), + pb_membersize(AllTypes, rep_sint32) / pb_membersize(AllTypes, rep_sint32[0]), 0}, + + {26, PB_HTYPE_ARRAY | PB_LTYPE_SVARINT, + pb_delta_end(AllTypes, rep_sint64, rep_sint32), + pb_delta(AllTypes, rep_sint64_count, rep_sint64), + pb_membersize(AllTypes, rep_sint64[0]), + pb_membersize(AllTypes, rep_sint64) / pb_membersize(AllTypes, rep_sint64[0]), 0}, + + {27, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, rep_bool, rep_sint64), + pb_delta(AllTypes, rep_bool_count, rep_bool), + pb_membersize(AllTypes, rep_bool[0]), + pb_membersize(AllTypes, rep_bool) / pb_membersize(AllTypes, rep_bool[0]), 0}, + + {28, PB_HTYPE_ARRAY | PB_LTYPE_FIXED32, + pb_delta_end(AllTypes, rep_fixed32, rep_bool), + pb_delta(AllTypes, rep_fixed32_count, rep_fixed32), + pb_membersize(AllTypes, rep_fixed32[0]), + pb_membersize(AllTypes, rep_fixed32) / pb_membersize(AllTypes, rep_fixed32[0]), 0}, + + {29, PB_HTYPE_ARRAY | PB_LTYPE_FIXED32, + pb_delta_end(AllTypes, rep_sfixed32, rep_fixed32), + pb_delta(AllTypes, rep_sfixed32_count, rep_sfixed32), + pb_membersize(AllTypes, rep_sfixed32[0]), + pb_membersize(AllTypes, rep_sfixed32) / pb_membersize(AllTypes, rep_sfixed32[0]), 0}, + + {30, PB_HTYPE_ARRAY | PB_LTYPE_FIXED32, + pb_delta_end(AllTypes, rep_float, rep_sfixed32), + pb_delta(AllTypes, rep_float_count, rep_float), + pb_membersize(AllTypes, rep_float[0]), + pb_membersize(AllTypes, rep_float) / pb_membersize(AllTypes, rep_float[0]), 0}, + + {31, PB_HTYPE_ARRAY | PB_LTYPE_FIXED64, + pb_delta_end(AllTypes, rep_fixed64, rep_float), + pb_delta(AllTypes, rep_fixed64_count, rep_fixed64), + pb_membersize(AllTypes, rep_fixed64[0]), + pb_membersize(AllTypes, rep_fixed64) / pb_membersize(AllTypes, rep_fixed64[0]), 0}, + + {32, PB_HTYPE_ARRAY | PB_LTYPE_FIXED64, + pb_delta_end(AllTypes, rep_sfixed64, rep_fixed64), + pb_delta(AllTypes, rep_sfixed64_count, rep_sfixed64), + pb_membersize(AllTypes, rep_sfixed64[0]), + pb_membersize(AllTypes, rep_sfixed64) / pb_membersize(AllTypes, rep_sfixed64[0]), 0}, + + {33, PB_HTYPE_ARRAY | PB_LTYPE_FIXED64, + pb_delta_end(AllTypes, rep_double, rep_sfixed64), + pb_delta(AllTypes, rep_double_count, rep_double), + pb_membersize(AllTypes, rep_double[0]), + pb_membersize(AllTypes, rep_double) / pb_membersize(AllTypes, rep_double[0]), 0}, + + {34, PB_HTYPE_ARRAY | PB_LTYPE_STRING, + pb_delta_end(AllTypes, rep_string, rep_double), + pb_delta(AllTypes, rep_string_count, rep_string), + pb_membersize(AllTypes, rep_string[0]), + pb_membersize(AllTypes, rep_string) / pb_membersize(AllTypes, rep_string[0]), 0}, + + {35, PB_HTYPE_ARRAY | PB_LTYPE_BYTES, + pb_delta_end(AllTypes, rep_bytes, rep_string), + pb_delta(AllTypes, rep_bytes_count, rep_bytes), + pb_membersize(AllTypes, rep_bytes[0]), + pb_membersize(AllTypes, rep_bytes) / pb_membersize(AllTypes, rep_bytes[0]), 0}, + + {36, PB_HTYPE_ARRAY | PB_LTYPE_SUBMESSAGE, + pb_delta_end(AllTypes, rep_submsg, rep_bytes), + pb_delta(AllTypes, rep_submsg_count, rep_submsg), + pb_membersize(AllTypes, rep_submsg[0]), + pb_membersize(AllTypes, rep_submsg) / pb_membersize(AllTypes, rep_submsg[0]), + &SubMessage_fields}, + + {37, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, rep_enum, rep_submsg), + pb_delta(AllTypes, rep_enum_count, rep_enum), + pb_membersize(AllTypes, rep_enum[0]), + pb_membersize(AllTypes, rep_enum) / pb_membersize(AllTypes, rep_enum[0]), 0}, + + {41, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, opt_int32, rep_enum), + pb_delta(AllTypes, has_opt_int32, opt_int32), + pb_membersize(AllTypes, opt_int32), 0, + &AllTypes_opt_int32_default}, + + {42, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, opt_int64, opt_int32), + pb_delta(AllTypes, has_opt_int64, opt_int64), + pb_membersize(AllTypes, opt_int64), 0, + &AllTypes_opt_int64_default}, + + {43, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, opt_uint32, opt_int64), + pb_delta(AllTypes, has_opt_uint32, opt_uint32), + pb_membersize(AllTypes, opt_uint32), 0, + &AllTypes_opt_uint32_default}, + + {44, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, opt_uint64, opt_uint32), + pb_delta(AllTypes, has_opt_uint64, opt_uint64), + pb_membersize(AllTypes, opt_uint64), 0, + &AllTypes_opt_uint64_default}, + + {45, PB_HTYPE_OPTIONAL | PB_LTYPE_SVARINT, + pb_delta_end(AllTypes, opt_sint32, opt_uint64), + pb_delta(AllTypes, has_opt_sint32, opt_sint32), + pb_membersize(AllTypes, opt_sint32), 0, + &AllTypes_opt_sint32_default}, + + {46, PB_HTYPE_OPTIONAL | PB_LTYPE_SVARINT, + pb_delta_end(AllTypes, opt_sint64, opt_sint32), + pb_delta(AllTypes, has_opt_sint64, opt_sint64), + pb_membersize(AllTypes, opt_sint64), 0, + &AllTypes_opt_sint64_default}, + + {47, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, opt_bool, opt_sint64), + pb_delta(AllTypes, has_opt_bool, opt_bool), + pb_membersize(AllTypes, opt_bool), 0, + &AllTypes_opt_bool_default}, + + {48, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED32, + pb_delta_end(AllTypes, opt_fixed32, opt_bool), + pb_delta(AllTypes, has_opt_fixed32, opt_fixed32), + pb_membersize(AllTypes, opt_fixed32), 0, + &AllTypes_opt_fixed32_default}, + + {49, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED32, + pb_delta_end(AllTypes, opt_sfixed32, opt_fixed32), + pb_delta(AllTypes, has_opt_sfixed32, opt_sfixed32), + pb_membersize(AllTypes, opt_sfixed32), 0, + &AllTypes_opt_sfixed32_default}, + + {50, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED32, + pb_delta_end(AllTypes, opt_float, opt_sfixed32), + pb_delta(AllTypes, has_opt_float, opt_float), + pb_membersize(AllTypes, opt_float), 0, + &AllTypes_opt_float_default}, + + {51, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED64, + pb_delta_end(AllTypes, opt_fixed64, opt_float), + pb_delta(AllTypes, has_opt_fixed64, opt_fixed64), + pb_membersize(AllTypes, opt_fixed64), 0, + &AllTypes_opt_fixed64_default}, + + {52, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED64, + pb_delta_end(AllTypes, opt_sfixed64, opt_fixed64), + pb_delta(AllTypes, has_opt_sfixed64, opt_sfixed64), + pb_membersize(AllTypes, opt_sfixed64), 0, + &AllTypes_opt_sfixed64_default}, + + {53, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED64, + pb_delta_end(AllTypes, opt_double, opt_sfixed64), + pb_delta(AllTypes, has_opt_double, opt_double), + pb_membersize(AllTypes, opt_double), 0, + &AllTypes_opt_double_default}, + + {54, PB_HTYPE_OPTIONAL | PB_LTYPE_STRING, + pb_delta_end(AllTypes, opt_string, opt_double), + pb_delta(AllTypes, has_opt_string, opt_string), + pb_membersize(AllTypes, opt_string), 0, + &AllTypes_opt_string_default}, + + {55, PB_HTYPE_OPTIONAL | PB_LTYPE_BYTES, + pb_delta_end(AllTypes, opt_bytes, opt_string), + pb_delta(AllTypes, has_opt_bytes, opt_bytes), + pb_membersize(AllTypes, opt_bytes), 0, + &AllTypes_opt_bytes_default}, + + {56, PB_HTYPE_OPTIONAL | PB_LTYPE_SUBMESSAGE, + pb_delta_end(AllTypes, opt_submsg, opt_bytes), + pb_delta(AllTypes, has_opt_submsg, opt_submsg), + pb_membersize(AllTypes, opt_submsg), 0, + &SubMessage_fields}, + + {57, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, opt_enum, opt_submsg), + pb_delta(AllTypes, has_opt_enum, opt_enum), + pb_membersize(AllTypes, opt_enum), 0, + &AllTypes_opt_enum_default}, + + {99, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, + pb_delta_end(AllTypes, end, opt_enum), 0, + pb_membersize(AllTypes, end), 0, 0}, + + PB_LAST_FIELD +}; + diff --git a/tests/bc_alltypes.pb.h b/tests/bc_alltypes.pb.h new file mode 100644 index 00000000..fe0b8f76 --- /dev/null +++ b/tests/bc_alltypes.pb.h @@ -0,0 +1,155 @@ +/* Automatically generated nanopb header */ +/* This is a file generated using nanopb-0.1.1. + * It is used as a part of test suite in order to detect any + * incompatible changes made to the generator in future versions. + */ +#ifndef _PB_BC_ALLTYPES_PB_H_ +#define _PB_BC_ALLTYPES_PB_H_ +#include + +/* Enum definitions */ +typedef enum { + MyEnum_Zero = 0, + MyEnum_First = 1, + MyEnum_Second = 2, + MyEnum_Truth = 42 +} MyEnum; + +/* Struct definitions */ +typedef struct { + char substuff1[16]; + int32_t substuff2; + bool has_substuff3; + uint32_t substuff3; +} SubMessage; + +typedef struct { + size_t size; + uint8_t bytes[16]; +} AllTypes_req_bytes_t; + +typedef struct { + size_t size; + uint8_t bytes[16]; +} AllTypes_rep_bytes_t; + +typedef struct { + size_t size; + uint8_t bytes[16]; +} AllTypes_opt_bytes_t; + +typedef struct { + int32_t req_int32; + int64_t req_int64; + uint32_t req_uint32; + uint64_t req_uint64; + int32_t req_sint32; + int64_t req_sint64; + bool req_bool; + uint32_t req_fixed32; + int32_t req_sfixed32; + float req_float; + uint64_t req_fixed64; + int64_t req_sfixed64; + double req_double; + char req_string[16]; + AllTypes_req_bytes_t req_bytes; + SubMessage req_submsg; + MyEnum req_enum; + size_t rep_int32_count; + int32_t rep_int32[5]; + size_t rep_int64_count; + int64_t rep_int64[5]; + size_t rep_uint32_count; + uint32_t rep_uint32[5]; + size_t rep_uint64_count; + uint64_t rep_uint64[5]; + size_t rep_sint32_count; + int32_t rep_sint32[5]; + size_t rep_sint64_count; + int64_t rep_sint64[5]; + size_t rep_bool_count; + bool rep_bool[5]; + size_t rep_fixed32_count; + uint32_t rep_fixed32[5]; + size_t rep_sfixed32_count; + int32_t rep_sfixed32[5]; + size_t rep_float_count; + float rep_float[5]; + size_t rep_fixed64_count; + uint64_t rep_fixed64[5]; + size_t rep_sfixed64_count; + int64_t rep_sfixed64[5]; + size_t rep_double_count; + double rep_double[5]; + size_t rep_string_count; + char rep_string[5][16]; + size_t rep_bytes_count; + AllTypes_rep_bytes_t rep_bytes[5]; + size_t rep_submsg_count; + SubMessage rep_submsg[5]; + size_t rep_enum_count; + MyEnum rep_enum[5]; + bool has_opt_int32; + int32_t opt_int32; + bool has_opt_int64; + int64_t opt_int64; + bool has_opt_uint32; + uint32_t opt_uint32; + bool has_opt_uint64; + uint64_t opt_uint64; + bool has_opt_sint32; + int32_t opt_sint32; + bool has_opt_sint64; + int64_t opt_sint64; + bool has_opt_bool; + bool opt_bool; + bool has_opt_fixed32; + uint32_t opt_fixed32; + bool has_opt_sfixed32; + int32_t opt_sfixed32; + bool has_opt_float; + float opt_float; + bool has_opt_fixed64; + uint64_t opt_fixed64; + bool has_opt_sfixed64; + int64_t opt_sfixed64; + bool has_opt_double; + double opt_double; + bool has_opt_string; + char opt_string[16]; + bool has_opt_bytes; + AllTypes_opt_bytes_t opt_bytes; + bool has_opt_submsg; + SubMessage opt_submsg; + bool has_opt_enum; + MyEnum opt_enum; + int32_t end; +} AllTypes; + +/* Default values for struct fields */ +extern const char SubMessage_substuff1_default[17]; +extern const int32_t SubMessage_substuff2_default; +extern const uint32_t SubMessage_substuff3_default; +extern const int32_t AllTypes_opt_int32_default; +extern const int64_t AllTypes_opt_int64_default; +extern const uint32_t AllTypes_opt_uint32_default; +extern const uint64_t AllTypes_opt_uint64_default; +extern const int32_t AllTypes_opt_sint32_default; +extern const int64_t AllTypes_opt_sint64_default; +extern const bool AllTypes_opt_bool_default; +extern const uint32_t AllTypes_opt_fixed32_default; +extern const int32_t AllTypes_opt_sfixed32_default; +extern const float AllTypes_opt_float_default; +extern const uint64_t AllTypes_opt_fixed64_default; +extern const int64_t AllTypes_opt_sfixed64_default; +extern const double AllTypes_opt_double_default; +extern const char AllTypes_opt_string_default[17]; +extern const AllTypes_opt_bytes_t AllTypes_opt_bytes_default; +extern const MyEnum AllTypes_opt_enum_default; + +/* Struct field encoding specification for nanopb */ +extern const pb_field_t SubMessage_fields[4]; +extern const pb_field_t AllTypes_fields[53]; + +#endif diff --git a/tests/bc_decode.c b/tests/bc_decode.c new file mode 100644 index 00000000..b74172fd --- /dev/null +++ b/tests/bc_decode.c @@ -0,0 +1,197 @@ +/* Tests the decoding of all types. + * This is a backwards-compatibility test, using bc_alltypes.pb.h. + * It is similar to test_decode3, but duplicated in order to allow + * test_decode3 to test any new features introduced later. + * + * Run e.g. ./bc_encode | ./bc_decode + */ + +#include +#include +#include +#include +#include "bc_alltypes.pb.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed.\n"); \ + return false; \ + } + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_alltypes(pb_istream_t *stream, int mode) +{ + AllTypes alltypes; + + /* Fill with garbage to better detect initialization errors */ + memset(&alltypes, 0xAA, sizeof(alltypes)); + + if (!pb_decode(stream, AllTypes_fields, &alltypes)) + return false; + + TEST(alltypes.req_int32 == -1001); + TEST(alltypes.req_int64 == -1002); + TEST(alltypes.req_uint32 == 1003); + TEST(alltypes.req_uint64 == 1004); + TEST(alltypes.req_sint32 == -1005); + TEST(alltypes.req_sint64 == -1006); + TEST(alltypes.req_bool == true); + + TEST(alltypes.req_fixed32 == 1008); + TEST(alltypes.req_sfixed32 == -1009); + TEST(alltypes.req_float == 1010.0f); + + TEST(alltypes.req_fixed64 == 1011); + TEST(alltypes.req_sfixed64 == -1012); + TEST(alltypes.req_double == 1013.0f); + + TEST(strcmp(alltypes.req_string, "1014") == 0); + TEST(alltypes.req_bytes.size == 4); + TEST(memcmp(alltypes.req_bytes.bytes, "1015", 4) == 0); + TEST(strcmp(alltypes.req_submsg.substuff1, "1016") == 0); + TEST(alltypes.req_submsg.substuff2 == 1016); + TEST(alltypes.req_submsg.substuff3 == 3); + TEST(alltypes.req_enum == MyEnum_Truth); + + TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); + TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); + TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); + TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); + TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); + TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0); + TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); + + TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); + TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); + TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); + + TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); + TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0); + TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); + + TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); + TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); + TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); + + TEST(alltypes.rep_submsg_count == 5); + TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); + TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); + TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3); + + TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); + + if (mode == 0) + { + /* Expect default values */ + TEST(alltypes.has_opt_int32 == false); + TEST(alltypes.opt_int32 == 4041); + TEST(alltypes.has_opt_int64 == false); + TEST(alltypes.opt_int64 == 4042); + TEST(alltypes.has_opt_uint32 == false); + TEST(alltypes.opt_uint32 == 4043); + TEST(alltypes.has_opt_uint64 == false); + TEST(alltypes.opt_uint64 == 4044); + TEST(alltypes.has_opt_sint32 == false); + TEST(alltypes.opt_sint32 == 4045); + TEST(alltypes.has_opt_sint64 == false); + TEST(alltypes.opt_sint64 == 4046); + TEST(alltypes.has_opt_bool == false); + TEST(alltypes.opt_bool == false); + + TEST(alltypes.has_opt_fixed32 == false); + TEST(alltypes.opt_fixed32 == 4048); + TEST(alltypes.has_opt_sfixed32 == false); + TEST(alltypes.opt_sfixed32 == 4049); + TEST(alltypes.has_opt_float == false); + TEST(alltypes.opt_float == 4050.0f); + + TEST(alltypes.has_opt_fixed64 == false); + TEST(alltypes.opt_fixed64 == 4051); + TEST(alltypes.has_opt_sfixed64 == false); + TEST(alltypes.opt_sfixed64 == 4052); + TEST(alltypes.has_opt_double == false); + TEST(alltypes.opt_double == 4053.0); + + TEST(alltypes.has_opt_string == false); + TEST(strcmp(alltypes.opt_string, "4054") == 0); + TEST(alltypes.has_opt_bytes == false); + TEST(alltypes.opt_bytes.size == 4); + TEST(memcmp(alltypes.opt_bytes.bytes, "4055", 4) == 0); + TEST(alltypes.has_opt_submsg == false); + TEST(strcmp(alltypes.opt_submsg.substuff1, "1") == 0); + TEST(alltypes.opt_submsg.substuff2 == 2); + TEST(alltypes.opt_submsg.substuff3 == 3); + TEST(alltypes.has_opt_enum == false); + TEST(alltypes.opt_enum == MyEnum_Second); + } + else + { + /* Expect filled-in values */ + TEST(alltypes.has_opt_int32 == true); + TEST(alltypes.opt_int32 == 3041); + TEST(alltypes.has_opt_int64 == true); + TEST(alltypes.opt_int64 == 3042); + TEST(alltypes.has_opt_uint32 == true); + TEST(alltypes.opt_uint32 == 3043); + TEST(alltypes.has_opt_uint64 == true); + TEST(alltypes.opt_uint64 == 3044); + TEST(alltypes.has_opt_sint32 == true); + TEST(alltypes.opt_sint32 == 3045); + TEST(alltypes.has_opt_sint64 == true); + TEST(alltypes.opt_sint64 == 3046); + TEST(alltypes.has_opt_bool == true); + TEST(alltypes.opt_bool == true); + + TEST(alltypes.has_opt_fixed32 == true); + TEST(alltypes.opt_fixed32 == 3048); + TEST(alltypes.has_opt_sfixed32 == true); + TEST(alltypes.opt_sfixed32 == 3049); + TEST(alltypes.has_opt_float == true); + TEST(alltypes.opt_float == 3050.0f); + + TEST(alltypes.has_opt_fixed64 == true); + TEST(alltypes.opt_fixed64 == 3051); + TEST(alltypes.has_opt_sfixed64 == true); + TEST(alltypes.opt_sfixed64 == 3052); + TEST(alltypes.has_opt_double == true); + TEST(alltypes.opt_double == 3053.0); + + TEST(alltypes.has_opt_string == true); + TEST(strcmp(alltypes.opt_string, "3054") == 0); + TEST(alltypes.has_opt_bytes == true); + TEST(alltypes.opt_bytes.size == 4); + TEST(memcmp(alltypes.opt_bytes.bytes, "3055", 4) == 0); + TEST(alltypes.has_opt_submsg == true); + TEST(strcmp(alltypes.opt_submsg.substuff1, "3056") == 0); + TEST(alltypes.opt_submsg.substuff2 == 3056); + TEST(alltypes.opt_submsg.substuff3 == 3); + TEST(alltypes.has_opt_enum == true); + TEST(alltypes.opt_enum == MyEnum_Truth); + } + + TEST(alltypes.end == 1099); + + return true; +} + +int main(int argc, char **argv) +{ + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Read the data into buffer */ + uint8_t buffer[1024]; + size_t count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + pb_istream_t stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_alltypes(&stream, mode)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} diff --git a/tests/bc_encode.c b/tests/bc_encode.c new file mode 100644 index 00000000..e84f0908 --- /dev/null +++ b/tests/bc_encode.c @@ -0,0 +1,131 @@ +/* Attempts to test all the datatypes supported by ProtoBuf. + * This is a backwards-compatibility test, using bc_alltypes.pb.h. + * It is similar to test_encode3, but duplicated in order to allow + * test_encode3 to test any new features introduced later. + */ + +#include +#include +#include +#include +#include "bc_alltypes.pb.h" + +int main(int argc, char **argv) +{ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Initialize the structure with constants */ + AllTypes alltypes = {0}; + + alltypes.req_int32 = -1001; + alltypes.req_int64 = -1002; + alltypes.req_uint32 = 1003; + alltypes.req_uint64 = 1004; + alltypes.req_sint32 = -1005; + alltypes.req_sint64 = -1006; + alltypes.req_bool = true; + + alltypes.req_fixed32 = 1008; + alltypes.req_sfixed32 = -1009; + alltypes.req_float = 1010.0f; + + alltypes.req_fixed64 = 1011; + alltypes.req_sfixed64 = -1012; + alltypes.req_double = 1013.0; + + strcpy(alltypes.req_string, "1014"); + alltypes.req_bytes.size = 4; + memcpy(alltypes.req_bytes.bytes, "1015", 4); + strcpy(alltypes.req_submsg.substuff1, "1016"); + alltypes.req_submsg.substuff2 = 1016; + alltypes.req_enum = MyEnum_Truth; + + alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; + alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002; + alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; + alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004; + alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005; + alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006; + alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; + + alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; + alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009; + alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; + + alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011; + alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012; + alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0; + + alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); + alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4; + memcpy(alltypes.rep_bytes[4].bytes, "2015", 4); + + alltypes.rep_submsg_count = 5; + strcpy(alltypes.rep_submsg[4].substuff1, "2016"); + alltypes.rep_submsg[4].substuff2 = 2016; + alltypes.rep_submsg[4].has_substuff3 = true; + alltypes.rep_submsg[4].substuff3 = 2016; + + alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; + + if (mode != 0) + { + /* Fill in values for optional fields */ + alltypes.has_opt_int32 = true; + alltypes.opt_int32 = 3041; + alltypes.has_opt_int64 = true; + alltypes.opt_int64 = 3042; + alltypes.has_opt_uint32 = true; + alltypes.opt_uint32 = 3043; + alltypes.has_opt_uint64 = true; + alltypes.opt_uint64 = 3044; + alltypes.has_opt_sint32 = true; + alltypes.opt_sint32 = 3045; + alltypes.has_opt_sint64 = true; + alltypes.opt_sint64 = 3046; + alltypes.has_opt_bool = true; + alltypes.opt_bool = true; + + alltypes.has_opt_fixed32 = true; + alltypes.opt_fixed32 = 3048; + alltypes.has_opt_sfixed32 = true; + alltypes.opt_sfixed32 = 3049; + alltypes.has_opt_float = true; + alltypes.opt_float = 3050.0f; + + alltypes.has_opt_fixed64 = true; + alltypes.opt_fixed64 = 3051; + alltypes.has_opt_sfixed64 = true; + alltypes.opt_sfixed64 = 3052; + alltypes.has_opt_double = true; + alltypes.opt_double = 3053.0; + + alltypes.has_opt_string = true; + strcpy(alltypes.opt_string, "3054"); + alltypes.has_opt_bytes = true; + alltypes.opt_bytes.size = 4; + memcpy(alltypes.opt_bytes.bytes, "3055", 4); + alltypes.has_opt_submsg = true; + strcpy(alltypes.opt_submsg.substuff1, "3056"); + alltypes.opt_submsg.substuff2 = 3056; + alltypes.has_opt_enum = true; + alltypes.opt_enum = MyEnum_Truth; + } + + alltypes.end = 1099; + + uint8_t buffer[1024]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed!\n"); + return 1; /* Failure */ + } +} -- cgit 1.2.3-korg From d32d3226580baf52f60fb7dee3921d8cc9aab67c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 14 Jan 2013 17:30:49 +0200 Subject: Add field type FT_IGNORE to generator. This allows ignoring fields that are unnecessary or too large for an embedded system using nanopb, while allowing them to remain in the .proto for other platforms. Update issue 51 Status: FixedInGit --- generator/nanopb.proto | 1 + generator/nanopb_generator.py | 8 +++++++- generator/nanopb_pb2.py | 9 +++++++-- tests/options.proto | 9 +++++++++ 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/generator/nanopb.proto b/generator/nanopb.proto index b9100abd..bc40f290 100644 --- a/generator/nanopb.proto +++ b/generator/nanopb.proto @@ -11,6 +11,7 @@ enum FieldType { FT_DEFAULT = 0; // Automatically decide field type, generate static field if possible. FT_CALLBACK = 1; // Always generate a callback field. FT_STATIC = 2; // Generate a static field or raise an exception if not possible. + FT_IGNORE = 3; // Ignore the field completely. } message NanoPBOptions { diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index a583b3f6..9e85a11d 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -306,7 +306,13 @@ class Field: class Message: def __init__(self, names, desc, message_options): self.name = names - self.fields = [Field(self.name, f, get_nanopb_suboptions(f, message_options)) for f in desc.field] + self.fields = [] + + for f in desc.field: + field_options = get_nanopb_suboptions(f, message_options) + if field_options.type != nanopb_pb2.FT_IGNORE: + self.fields.append(Field(self.name, f, field_options)) + self.packed = message_options.packed_struct self.ordered_fields = self.fields[:] self.ordered_fields.sort() diff --git a/generator/nanopb_pb2.py b/generator/nanopb_pb2.py index d08b82ab..4ba18b26 100644 --- a/generator/nanopb_pb2.py +++ b/generator/nanopb_pb2.py @@ -12,7 +12,7 @@ import google.protobuf.descriptor_pb2 DESCRIPTOR = descriptor.FileDescriptor( name='nanopb.proto', package='', - serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*;\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions') + serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*J\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions') _FIELDTYPE = descriptor.EnumDescriptor( name='FieldType', @@ -32,17 +32,22 @@ _FIELDTYPE = descriptor.EnumDescriptor( name='FT_STATIC', index=2, number=2, options=None, type=None), + descriptor.EnumValueDescriptor( + name='FT_IGNORE', index=3, number=3, + options=None, + type=None), ], containing_type=None, options=None, serialized_start=199, - serialized_end=258, + serialized_end=273, ) FT_DEFAULT = 0 FT_CALLBACK = 1 FT_STATIC = 2 +FT_IGNORE = 3 NANOPB_FILEOPT_FIELD_NUMBER = 1010 nanopb_fileopt = descriptor.FieldDescriptor( diff --git a/tests/options.proto b/tests/options.proto index 3e9383cd..b5badcfd 100644 --- a/tests/options.proto +++ b/tests/options.proto @@ -62,3 +62,12 @@ message my_packed_struct optional int32 myfield = 1; } +// Message with ignored field +// Note: doesn't really test if the field is missing in the output, +// but atleast tests that the output compiles. +message Message6 +{ + required int32 field1 = 1; + optional int32 field2 = 2 [(nanopb).type = FT_IGNORE]; +} + -- cgit 1.2.3-korg From 88eba4bc27908659c6c8d203b5f16220a3736fb2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 16 Jan 2013 16:28:51 +0200 Subject: Fix bug with error messages. Update issue 52 Status: FixedInGit --- pb_decode.c | 1 + 1 file changed, 1 insertion(+) diff --git a/pb_decode.c b/pb_decode.c index c9652afb..6b6066e8 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -225,6 +225,7 @@ bool checkreturn pb_make_string_substream(pb_istream_t *stream, pb_istream_t *su void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) { stream->state = substream->state; + stream->errmsg = substream->errmsg; } /* Iterator for pb_field_t list */ -- cgit 1.2.3-korg From 8e840cc81aa29c6cebf66e252cf8445abb2b03f9 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 16 Jan 2013 16:31:45 +0200 Subject: Put previous commit (issue 52) inside #ifndef --- pb_decode.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pb_decode.c b/pb_decode.c index 6b6066e8..96e0c43b 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -225,7 +225,10 @@ bool checkreturn pb_make_string_substream(pb_istream_t *stream, pb_istream_t *su void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) { stream->state = substream->state; + +#ifdef PB_NO_ERRMSG stream->errmsg = substream->errmsg; +#endif } /* Iterator for pb_field_t list */ -- cgit 1.2.3-korg From eab4151a99f0c2fc2f41c51356d04898255ccbfe Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 16 Jan 2013 16:32:48 +0200 Subject: Oops, typoed #ifdef; again fixing previous commit. --- pb_decode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pb_decode.c b/pb_decode.c index 96e0c43b..7c68b5a2 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -226,7 +226,7 @@ void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) { stream->state = substream->state; -#ifdef PB_NO_ERRMSG +#ifndef PB_NO_ERRMSG stream->errmsg = substream->errmsg; #endif } -- cgit 1.2.3-korg From 2392d255749715ad337d3f5e23d3de7f2065e3dd Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 21 Jan 2013 21:18:38 +0200 Subject: Fix misleading comment (current generator sets LTYPE always). --- pb.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pb.h b/pb.h index fbdf185e..b19dfcfe 100644 --- a/pb.h +++ b/pb.h @@ -100,8 +100,8 @@ typedef enum { /* Works for all required/optional/repeated fields. * data_offset points to pb_callback_t structure. - * LTYPE should be 0 (it is ignored, but sometimes - * used to speculatively index an array). */ + * LTYPE should be valid or 0 (it is ignored, but + * sometimes used to speculatively index an array). */ PB_HTYPE_CALLBACK = 0x30, PB_HTYPE_MASK = 0xF0 -- cgit 1.2.3-korg From e4b55179d1f53c42916f5e62fb83789973bf4b01 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 29 Jan 2013 22:10:37 +0200 Subject: Add an example of handling doubles on AVR platform. --- example_avr_double/Makefile | 22 ++++++ example_avr_double/README.txt | 22 ++++++ example_avr_double/decode_double.c | 33 +++++++++ example_avr_double/double_conversion.c | 123 +++++++++++++++++++++++++++++++++ example_avr_double/double_conversion.h | 26 +++++++ example_avr_double/doubleproto.proto | 13 ++++ example_avr_double/encode_double.c | 25 +++++++ example_avr_double/test_conversions.c | 56 +++++++++++++++ 8 files changed, 320 insertions(+) create mode 100644 example_avr_double/Makefile create mode 100644 example_avr_double/README.txt create mode 100644 example_avr_double/decode_double.c create mode 100644 example_avr_double/double_conversion.c create mode 100644 example_avr_double/double_conversion.h create mode 100644 example_avr_double/doubleproto.proto create mode 100644 example_avr_double/encode_double.c create mode 100644 example_avr_double/test_conversions.c diff --git a/example_avr_double/Makefile b/example_avr_double/Makefile new file mode 100644 index 00000000..74300fc6 --- /dev/null +++ b/example_avr_double/Makefile @@ -0,0 +1,22 @@ +CFLAGS=-Wall -Werror -I .. -g -O0 +DEPS=double_conversion.c ../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h + +all: run_tests + +clean: + rm -f test_conversions encode_double decode_double doubleproto.pb.c doubleproto.pb.h + +test_conversions: test_conversions.c double_conversion.c + $(CC) $(CFLAGS) -o $@ $^ + +%: %.c $(DEPS) doubleproto.pb.h doubleproto.pb.c + $(CC) $(CFLAGS) -o $@ $< double_conversion.c ../pb_decode.c ../pb_encode.c doubleproto.pb.c + +doubleproto.pb.c doubleproto.pb.h: doubleproto.proto ../generator/nanopb_generator.py + protoc -I. -I../generator -I/usr/include -odoubleproto.pb $< + python ../generator/nanopb_generator.py doubleproto.pb + +run_tests: test_conversions encode_double decode_double + ./test_conversions + ./encode_double | ./decode_double + diff --git a/example_avr_double/README.txt b/example_avr_double/README.txt new file mode 100644 index 00000000..0090d729 --- /dev/null +++ b/example_avr_double/README.txt @@ -0,0 +1,22 @@ +Some processors/compilers, such as AVR-GCC, do not support the double +datatype. Instead, they have sizeof(double) == 4. Because protocol +binary format uses the double encoding directly, this causes trouble +if the protocol in .proto requires double fields. + +This directory contains a solution to this problem. It uses uint64_t +to store the raw wire values, because its size is correct on all +platforms. The file double_conversion.c provides functions that +convert these values to/from floats, without relying on compiler +support. + +To use this method, you need to make two modifications to your code: + +1) Change all 'double' fields into 'fixed64' in the .proto. + +2) Whenever writing to a 'double' field, use float_to_double(). + +3) Whenever reading a 'double' field, use double_to_float(). + +The conversion routines should be as accurate as the float datatype can +be. Furthermore, they should handle all special values (NaN, inf, denormalized +numbers) correctly. There are testcases in test_conversions.c. diff --git a/example_avr_double/decode_double.c b/example_avr_double/decode_double.c new file mode 100644 index 00000000..5802eca7 --- /dev/null +++ b/example_avr_double/decode_double.c @@ -0,0 +1,33 @@ +/* Decodes a double value into a float variable. + * Used to read double values with AVR code, which doesn't support double directly. + */ + +#include +#include +#include "double_conversion.h" +#include "doubleproto.pb.h" + +int main() +{ + uint8_t buffer[32]; + size_t count = fread(buffer, 1, sizeof(buffer), stdin); + pb_istream_t stream = pb_istream_from_buffer(buffer, count); + + AVRDoubleMessage message; + pb_decode(&stream, AVRDoubleMessage_fields, &message); + + float v1 = double_to_float(message.field1); + float v2 = double_to_float(message.field2); + + printf("Values: %f %f\n", v1, v2); + + if (v1 == 1234.5678f && + v2 == 0.00001f) + { + return 0; + } + else + { + return 1; + } +} diff --git a/example_avr_double/double_conversion.c b/example_avr_double/double_conversion.c new file mode 100644 index 00000000..cf79b9a0 --- /dev/null +++ b/example_avr_double/double_conversion.c @@ -0,0 +1,123 @@ +/* Conversion routines for platforms that do not support 'double' directly. */ + +#include "double_conversion.h" +#include + +typedef union { + float f; + uint32_t i; +} conversion_t; + +/* Note: IEE 754 standard specifies float formats as follows: + * Single precision: sign, 8-bit exp, 23-bit frac. + * Double precision: sign, 11-bit exp, 52-bit frac. + */ + +uint64_t float_to_double(float value) +{ + conversion_t in; + in.f = value; + uint8_t sign; + int16_t exponent; + uint64_t mantissa; + + /* Decompose input value */ + sign = (in.i >> 31) & 1; + exponent = ((in.i >> 23) & 0xFF) - 127; + mantissa = in.i & 0x7FFFFF; + + if (exponent == 128) + { + /* Special value (NaN etc.) */ + exponent = 1024; + } + else if (exponent == -127) + { + if (!mantissa) + { + /* Zero */ + exponent = -1023; + } + else + { + /* Denormalized */ + mantissa <<= 1; + while (!(mantissa & 0x800000)) + { + mantissa <<= 1; + exponent--; + } + mantissa &= 0x7FFFFF; + } + } + + /* Combine fields */ + mantissa <<= 29; + mantissa |= (uint64_t)(exponent + 1023) << 52; + mantissa |= (uint64_t)sign << 63; + + return mantissa; +} + +float double_to_float(uint64_t value) +{ + uint8_t sign; + int16_t exponent; + uint32_t mantissa; + conversion_t out; + + /* Decompose input value */ + sign = (value >> 63) & 1; + exponent = ((value >> 52) & 0x7FF) - 1023; + mantissa = (value >> 28) & 0xFFFFFF; /* Highest 24 bits */ + + /* Figure if value is in range representable by floats. */ + if (exponent == 1024) + { + /* Special value */ + exponent = 128; + } + else if (exponent > 127) + { + /* Too large */ + if (sign) + return -INFINITY; + else + return INFINITY; + } + else if (exponent < -150) + { + /* Too small */ + if (sign) + return -0.0f; + else + return 0.0f; + } + else if (exponent < -126) + { + /* Denormalized */ + mantissa |= 0x1000000; + mantissa >>= (-126 - exponent); + exponent = -127; + } + + /* Round off mantissa */ + mantissa = (mantissa + 1) >> 1; + + /* Check if mantissa went over 2.0 */ + if (mantissa & 0x800000) + { + exponent += 1; + mantissa &= 0x7FFFFF; + mantissa >>= 1; + } + + /* Combine fields */ + out.i = mantissa; + out.i |= (uint32_t)(exponent + 127) << 23; + out.i |= (uint32_t)sign << 31; + + return out.f; +} + + diff --git a/example_avr_double/double_conversion.h b/example_avr_double/double_conversion.h new file mode 100644 index 00000000..62b6a8ae --- /dev/null +++ b/example_avr_double/double_conversion.h @@ -0,0 +1,26 @@ +/* AVR-GCC does not have real double datatype. Instead its double + * is equal to float, i.e. 32 bit value. If you need to communicate + * with other systems that use double in their .proto files, you + * need to do some conversion. + * + * These functions use bitwise operations to mangle floats into doubles + * and then store them in uint64_t datatype. + */ + +#ifndef DOUBLE_CONVERSION +#define DOUBLE_CONVERSION + +#include + +/* Convert native 4-byte float into a 8-byte double. */ +extern uint64_t float_to_double(float value); + +/* Convert 8-byte double into native 4-byte float. + * Values are rounded to nearest, 0.5 away from zero. + * Overflowing values are converted to Inf or -Inf. + */ +extern float double_to_float(uint64_t value); + + +#endif + diff --git a/example_avr_double/doubleproto.proto b/example_avr_double/doubleproto.proto new file mode 100644 index 00000000..d8b7f2db --- /dev/null +++ b/example_avr_double/doubleproto.proto @@ -0,0 +1,13 @@ +// A message containing doubles, as used by other applications. +message DoubleMessage { + required double field1 = 1; + required double field2 = 2; +} + +// A message containing doubles, but redefined using uint64_t. +// For use in AVR code. +message AVRDoubleMessage { + required fixed64 field1 = 1; + required fixed64 field2 = 2; +} + diff --git a/example_avr_double/encode_double.c b/example_avr_double/encode_double.c new file mode 100644 index 00000000..cd532d46 --- /dev/null +++ b/example_avr_double/encode_double.c @@ -0,0 +1,25 @@ +/* Encodes a float value into a double on the wire. + * Used to emit doubles from AVR code, which doesn't support double directly. + */ + +#include +#include +#include "double_conversion.h" +#include "doubleproto.pb.h" + +int main() +{ + AVRDoubleMessage message = { + float_to_double(1234.5678f), + float_to_double(0.00001f) + }; + + uint8_t buffer[32]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + pb_encode(&stream, AVRDoubleMessage_fields, &message); + fwrite(buffer, 1, stream.bytes_written, stdout); + + return 0; +} + diff --git a/example_avr_double/test_conversions.c b/example_avr_double/test_conversions.c new file mode 100644 index 00000000..22620a6a --- /dev/null +++ b/example_avr_double/test_conversions.c @@ -0,0 +1,56 @@ +#include "double_conversion.h" +#include +#include + +static const double testvalues[] = { + 0.0, -0.0, 0.1, -0.1, + M_PI, -M_PI, 123456.789, -123456.789, + INFINITY, -INFINITY, NAN, INFINITY - INFINITY, + 1e38, -1e38, 1e39, -1e39, + 1e-38, -1e-38, 1e-39, -1e-39, + 3.14159e-37,-3.14159e-37, 3.14159e-43, -3.14159e-43, + 1e-60, -1e-60, 1e-45, -1e-45, + 0.99999999999999, -0.99999999999999, 127.999999999999, -127.999999999999 +}; + +#define TESTVALUES_COUNT (sizeof(testvalues)/sizeof(testvalues[0])) + +int main() +{ + int status = 0; + int i; + for (i = 0; i < TESTVALUES_COUNT; i++) + { + double orig = testvalues[i]; + float expected_float = (float)orig; + double expected_double = (double)expected_float; + + float got_float = double_to_float(*(uint64_t*)&orig); + uint64_t got_double = float_to_double(got_float); + + uint32_t e1 = *(uint32_t*)&expected_float; + uint32_t g1 = *(uint32_t*)&got_float; + uint64_t e2 = *(uint64_t*)&expected_double; + uint64_t g2 = got_double; + + if (g1 != e1) + { + printf("%3d double_to_float fail: %08x != %08x\n", i, g1, e1); + status = 1; + } + + if (g2 != e2) + { + printf("%3d float_to_double fail: %016llx != %016llx\n", i, + (unsigned long long)g2, + (unsigned long long)e2); + status = 1; + } + } + + return status; +} + + + + -- cgit 1.2.3-korg From 38ced18639f1e2236e9949071e3ddc148cf2ad73 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 5 Feb 2013 11:47:37 +0200 Subject: Add Java package name to nanopb.proto. This fixes problems when interoperating with Java. Fix suggested by trepidacious: https://groups.google.com/forum/#!topic/nanopb/hZgrj6h9OnY --- generator/nanopb.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/generator/nanopb.proto b/generator/nanopb.proto index bc40f290..40662522 100644 --- a/generator/nanopb.proto +++ b/generator/nanopb.proto @@ -7,6 +7,8 @@ import "google/protobuf/descriptor.proto"; +option java_package = "fi.kapsi.koti.jpa.nanopb"; + enum FieldType { FT_DEFAULT = 0; // Automatically decide field type, generate static field if possible. FT_CALLBACK = 1; // Always generate a callback field. -- cgit 1.2.3-korg From c372ebc665540df2578e869e73405b3d309bfc48 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 5 Feb 2013 22:06:36 +0200 Subject: Performance improvement: replace memcpy with loop. In the pb_istream_from_buffer and pb_ostream_from_buffer, memcpy was used to transfer values to the buffer. For the common case of count = 1-10 bytes, a simple loop is faster. --- pb_decode.c | 7 +++++-- pb_encode.c | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 7c68b5a2..6ddde773 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -39,11 +39,14 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count) { uint8_t *source = (uint8_t*)stream->state; + stream->state = source + count; if (buf != NULL) - memcpy(buf, source, count); + { + while (count--) + *buf++ = *source++; + } - stream->state = source + count; return true; } diff --git a/pb_encode.c b/pb_encode.c index 5d1965d1..95223cb6 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -37,8 +37,11 @@ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) { uint8_t *dest = (uint8_t*)stream->state; - memcpy(dest, buf, count); stream->state = dest + count; + + while (count--) + *dest++ = *buf++; + return true; } -- cgit 1.2.3-korg From 39b8a5e2bbd5da85f23b48280e81a5ce6672b09d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 5 Feb 2013 22:39:32 +0200 Subject: Make pb_decode_varint32 a separate implementation. This avoids doing 64-bit arithmetic for 32-bit varint decodings. It does increase the code size somewhat. Results for ARM Cortex-M3: -10% execution time, +1% code size, -2% ram usage. --- pb_decode.c | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 6ddde773..9faceca2 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -95,10 +95,20 @@ pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize) static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) { - uint64_t temp; - bool status = pb_decode_varint(stream, &temp); - *dest = (uint32_t)temp; - return status; + uint8_t byte; + int bitpos = 0; + *dest = 0; + + while (bitpos < 32 && pb_read(stream, &byte, 1)) + { + *dest |= (uint32_t)(byte & 0x7F) << bitpos; + bitpos += 7; + + if (!(byte & 0x80)) + return true; + } + + PB_RETURN_ERROR(stream, "varint overflow"); } bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) -- cgit 1.2.3-korg From 4ba6a3027d7a5d8c17abeb031622389f8be234fe Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 6 Feb 2013 20:54:25 +0200 Subject: Add compile-time option PB_BUFFER_ONLY. This allows slight optimizations if only memory buffer support (as opposed to stream callbacks) is wanted. On ARM difference is -12% execution time, -4% code size when enabled. --- docs/reference.rst | 2 ++ pb_decode.c | 11 +++++++++++ pb_decode.h | 9 +++++++++ pb_encode.c | 11 ++++++++++- pb_encode.h | 10 ++++++++++ tests/Makefile | 20 ++++++++++++++++++-- 6 files changed, 60 insertions(+), 3 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 2b10c2ee..fee1b702 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -22,6 +22,8 @@ PB_FIELD_32BIT Add support for tag numbers > 65535 and fields la Increases code size 9 bytes per each field. Compiler error will tell if you need this. PB_NO_ERRMSG Disables the support for error messages; only error information is the true/false return value. Decreases the code size by a few hundred bytes. +PB_BUFFER_ONLY Disables the support for custom streams. Only supports encoding to memory buffers. + Speeds up execution and decreases code size slightly. ============================ ================================================================================================ The PB_MAX_REQUIRED_FIELDS, PB_FIELD_16BIT and PB_FIELD_32BIT settings allow raising some datatype limits to suit larger messages. diff --git a/pb_decode.c b/pb_decode.c index 9faceca2..b664fe14 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -52,6 +52,7 @@ static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t coun bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) { +#ifndef PB_BUFFER_ONLY if (buf == NULL && stream->callback != buf_read) { /* Skip input bytes */ @@ -66,12 +67,18 @@ bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) return pb_read(stream, tmp, count); } +#endif if (stream->bytes_left < count) PB_RETURN_ERROR(stream, "end-of-stream"); +#ifndef PB_BUFFER_ONLY if (!stream->callback(stream, buf, count)) PB_RETURN_ERROR(stream, "io error"); +#else + if (!buf_read(stream, buf, count)) + return false; +#endif stream->bytes_left -= count; return true; @@ -80,7 +87,11 @@ bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize) { pb_istream_t stream; +#ifdef PB_BUFFER_ONLY + stream.callback = NULL; +#else stream.callback = &buf_read; +#endif stream.state = buf; stream.bytes_left = bufsize; #ifndef PB_NO_ERRMSG diff --git a/pb_decode.h b/pb_decode.h index e9f8ced8..35de0a22 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -32,7 +32,16 @@ extern "C" { */ struct _pb_istream_t { +#ifdef PB_BUFFER_ONLY + /* Callback pointer is not used in buffer-only configuration. + * Having an int pointer here allows binary compatibility but + * gives an error if someone tries to assign callback function. + */ + int *callback; +#else bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count); +#endif + void *state; /* Free field for use by callback implementation */ size_t bytes_left; diff --git a/pb_encode.c b/pb_encode.c index 95223cb6..bba5dc19 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -48,7 +48,11 @@ static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize) { pb_ostream_t stream; +#ifdef PB_BUFFER_ONLY + stream.callback = (void*)1; /* Just some marker value */ +#else stream.callback = &buf_write; +#endif stream.state = buf; stream.max_size = bufsize; stream.bytes_written = 0; @@ -61,9 +65,14 @@ bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count { if (stream->bytes_written + count > stream->max_size) return false; - + +#ifdef PB_BUFFER_ONLY + if (!buf_write(stream, buf, count)) + return false; +#else if (!stream->callback(stream, buf, count)) return false; +#endif } stream->bytes_written += count; diff --git a/pb_encode.h b/pb_encode.h index 85a82973..69b88e86 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -32,7 +32,17 @@ extern "C" { */ struct _pb_ostream_t { +#ifdef PB_BUFFER_ONLY + /* Callback pointer is not used in buffer-only configuration. + * Having an int pointer here allows binary compatibility but + * gives an error if someone tries to assign callback function. + * Also, NULL pointer marks a 'sizing stream' that does not + * write anything. + */ + int *callback; +#else bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); +#endif void *state; /* Free field for use by callback implementation */ size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ size_t bytes_written; diff --git a/tests/Makefile b/tests/Makefile index 63830999..38f10eae 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -3,8 +3,8 @@ DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h \ callbacks2.pb.h callbacks.pb.h unittests.h unittestproto.pb.h \ alltypes.pb.h missing_fields.pb.h TESTS= decode_unittests encode_unittests \ - test_decode1 test_decode2 test_decode3 \ - test_encode1 test_encode2 test_encode3 \ + test_decode1 test_decode2 test_decode3 test_decode3_buf \ + test_encode1 test_encode2 test_encode3 test_encode3_buf \ test_decode_callbacks test_encode_callbacks \ test_missing_fields test_no_messages \ test_multiple_files test_cxxcompile test_options \ @@ -39,11 +39,26 @@ pb_encode.o: ../pb_encode.c $(DEPS) pb_decode.o: ../pb_decode.c $(DEPS) $(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< +# Test for compilability with c++ compiler + pb_encode.cxx.o: ../pb_encode.c $(DEPS) $(CXX) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< pb_decode.cxx.o: ../pb_decode.c $(DEPS) $(CXX) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< +# Test for PB_BUF_ONLY compilation option + +pb_encode.buf.o: ../pb_encode.c $(DEPS) + $(CC) -DPB_BUFFER_ONLY $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< +pb_decode.buf.o: ../pb_decode.c $(DEPS) + $(CC) -DPB_BUFFER_ONLY $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< +%.buf.o: %.c $(DEPS) + $(CC) -DPB_BUFFER_ONLY $(CFLAGS) -c -o $@ $< +test_encode3_buf: test_encode3.buf.o pb_encode.buf.o alltypes.pb.o + $(CC) $(LDFLAGS) $^ -o $@ +test_decode3_buf: test_decode3.buf.o pb_decode.buf.o alltypes.pb.o + $(CC) $(LDFLAGS) $^ -o $@ + test_cxxcompile: pb_encode.cxx.o pb_decode.cxx.o test_decode1: test_decode1.o pb_decode.o person.pb.o test_decode2: test_decode2.o pb_decode.o person.pb.o @@ -98,6 +113,7 @@ run_unittests: $(TESTS) ./test_encode3 | ./test_decode3 ./test_encode3 1 | ./test_decode3 1 ./test_encode3 1 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null + ./test_encode3_buf 1 | ./test_decode3_buf 1 ./bc_encode | ./bc_decode ./test_missing_fields -- cgit 1.2.3-korg From d23939d688b36bf9b474ab9077d4a42b69b8442d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 6 Feb 2013 21:44:40 +0200 Subject: Avoid unnecessary looping in required fields check. Results for ARM: -6% execution time, -1% code size --- pb_decode.c | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index b664fe14..7c3407ce 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -501,15 +501,34 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ } /* Check that all required fields were present. */ - pb_field_init(&iter, fields, dest_struct); - do { - if (PB_HTYPE(iter.current->type) == PB_HTYPE_REQUIRED && - iter.required_field_index < PB_MAX_REQUIRED_FIELDS && - !(fields_seen[iter.required_field_index >> 3] & (1 << (iter.required_field_index & 7)))) + { + /* First figure out the number of required fields by + * seeking to the end of the field array. Usually we + * are already close to end after decoding. + */ + int req_field_count; + uint8_t last_type; + int i; + do { + req_field_count = iter.required_field_index; + last_type = iter.current->type; + } while (pb_field_next(&iter)); + + /* Fixup if last field was also required. */ + if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED) + req_field_count++; + + /* Check the whole bytes */ + for (i = 0; i < (req_field_count >> 3); i++) { - PB_RETURN_ERROR(stream, "missing required field"); + if (fields_seen[i] != 0xFF) + PB_RETURN_ERROR(stream, "missing required field"); } - } while (pb_field_next(&iter)); + + /* Check the remaining bits */ + if (fields_seen[req_field_count >> 3] != (0xFF >> (8 - (req_field_count & 7)))) + PB_RETURN_ERROR(stream, "missing required field"); + } return true; } -- cgit 1.2.3-korg From 4f379364b31fd608f531290e7fb126870769df97 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 6 Feb 2013 22:11:02 +0200 Subject: Improve the pb_decode_varint implementations. Results for ARM: -4% execution time, +1% code size --- pb_decode.c | 58 ++++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 18 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 7c3407ce..51df32f8 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -107,37 +107,59 @@ pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize) static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) { uint8_t byte; - int bitpos = 0; - *dest = 0; + uint32_t result; + + if (!pb_read(stream, &byte, 1)) + return false; - while (bitpos < 32 && pb_read(stream, &byte, 1)) + if (!(byte & 0x80)) { - *dest |= (uint32_t)(byte & 0x7F) << bitpos; - bitpos += 7; - - if (!(byte & 0x80)) - return true; + /* Quick case, 1 byte value */ + result = byte; } - - PB_RETURN_ERROR(stream, "varint overflow"); + else + { + /* Multibyte case */ + int bitpos = 7; + result = byte & 0x7F; + + do + { + if (bitpos >= 32) + PB_RETURN_ERROR(stream, "varint overflow"); + + if (!pb_read(stream, &byte, 1)) + return false; + + result |= (uint32_t)(byte & 0x7F) << bitpos; + bitpos += 7; + } while (byte & 0x80); + } + + *dest = result; + return true; } bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) { uint8_t byte; int bitpos = 0; - *dest = 0; + uint64_t result = 0; - while (bitpos < 64 && pb_read(stream, &byte, 1)) + do { - *dest |= (uint64_t)(byte & 0x7F) << bitpos; - bitpos += 7; + if (bitpos >= 64) + PB_RETURN_ERROR(stream, "varint overflow"); - if (!(byte & 0x80)) - return true; - } + if (!pb_read(stream, &byte, 1)) + return false; + + result |= (uint64_t)(byte & 0x7F) << bitpos; + bitpos += 7; + } while (byte & 0x80); - PB_RETURN_ERROR(stream, "varint overflow"); + *dest = result; + return true; } bool checkreturn pb_skip_varint(pb_istream_t *stream) -- cgit 1.2.3-korg From c3729599b0542770eff6259836c840e7a8bb3aab Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 6 Feb 2013 22:18:52 +0200 Subject: Use unsigned datatypes where appropriate. --- pb_decode.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 51df32f8..edcbb182 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -120,7 +120,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) else { /* Multibyte case */ - int bitpos = 7; + uint8_t bitpos = 7; result = byte & 0x7F; do @@ -132,7 +132,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) return false; result |= (uint32_t)(byte & 0x7F) << bitpos; - bitpos += 7; + bitpos = (uint8_t)(bitpos + 7); } while (byte & 0x80); } @@ -143,7 +143,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) { uint8_t byte; - int bitpos = 0; + uint8_t bitpos = 0; uint64_t result = 0; do @@ -155,7 +155,7 @@ bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) return false; result |= (uint64_t)(byte & 0x7F) << bitpos; - bitpos += 7; + bitpos = (uint8_t)(bitpos + 7); } while (byte & 0x80); *dest = result; @@ -281,8 +281,8 @@ void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) typedef struct { const pb_field_t *start; /* Start of the pb_field_t array */ const pb_field_t *current; /* Current position of the iterator */ - int field_index; /* Zero-based index of the field. */ - int required_field_index; /* Zero-based index that counts only the required fields */ + unsigned field_index; /* Zero-based index of the field. */ + unsigned required_field_index; /* Zero-based index that counts only the required fields */ void *dest_struct; /* Pointer to the destination structure to decode to */ void *pData; /* Pointer where to store current field value */ void *pSize; /* Pointer where to store the size of current array field */ @@ -328,7 +328,7 @@ static bool pb_field_next(pb_field_iterator_t *iter) static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag) { - int start = iter->field_index; + unsigned start = iter->field_index; do { if (iter->current->tag == tag) @@ -528,9 +528,9 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ * seeking to the end of the field array. Usually we * are already close to end after decoding. */ - int req_field_count; + unsigned req_field_count; uint8_t last_type; - int i; + unsigned i; do { req_field_count = iter.required_field_index; last_type = iter.current->type; -- cgit 1.2.3-korg From 4b7ddabbcf2b4e272675edbd20fd64fe921c99cf Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 7 Feb 2013 17:19:53 +0200 Subject: Fix compiler warning on MSVC (issue #57) --- pb_decode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pb_decode.c b/pb_decode.c index edcbb182..b25f3f55 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -529,7 +529,7 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ * are already close to end after decoding. */ unsigned req_field_count; - uint8_t last_type; + pb_type_t last_type; unsigned i; do { req_field_count = iter.required_field_index; -- cgit 1.2.3-korg From 47b10ec0ab370475c51294707c0fe6e160046973 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 7 Feb 2013 17:40:17 +0200 Subject: Sanitize filenames before putting them in #ifndef. Update issue 50 Status: FixedInGit --- generator/nanopb_generator.py | 12 +++++++++++- tests/Makefile | 3 ++- tests/funny-proto+name.proto | 0 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 tests/funny-proto+name.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 9e85a11d..d35a4252 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -452,6 +452,16 @@ def sort_dependencies(messages): if msgname in message_by_name: yield message_by_name[msgname] +def make_identifier(headername): + '''Make #ifndef identifier that contains uppercase A-Z and digits 0-9''' + result = "" + for c in headername.upper(): + if c.isalnum(): + result += c + else: + result += '_' + return result + def generate_header(dependencies, headername, enums, messages): '''Generate content for a header file. Generates strings, which should be concatenated and stored to file. @@ -460,7 +470,7 @@ def generate_header(dependencies, headername, enums, messages): yield '/* Automatically generated nanopb header */\n' yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) - symbol = headername.replace('.', '_').upper() + symbol = make_identifier(headername) yield '#ifndef _PB_%s_\n' % symbol yield '#define _PB_%s_\n' % symbol yield '#include \n\n' diff --git a/tests/Makefile b/tests/Makefile index 38f10eae..7385fe35 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -6,7 +6,7 @@ TESTS= decode_unittests encode_unittests \ test_decode1 test_decode2 test_decode3 test_decode3_buf \ test_encode1 test_encode2 test_encode3 test_encode3_buf \ test_decode_callbacks test_encode_callbacks \ - test_missing_fields test_no_messages \ + test_missing_fields test_no_messages test_funny_name \ test_multiple_files test_cxxcompile test_options \ bc_encode bc_decode @@ -73,6 +73,7 @@ test_missing_fields: test_missing_fields.o pb_encode.o pb_decode.o missing_field decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o encode_unittests: encode_unittests.o pb_encode.o unittestproto.pb.o test_no_messages: no_messages.pb.h no_messages.pb.c no_messages.pb.o +test_funny_name: funny-proto+name.pb.h funny-proto+name.pb.o bc_encode: bc_alltypes.pb.o pb_encode.o bc_encode.o bc_decode: bc_alltypes.pb.o pb_decode.o bc_decode.o diff --git a/tests/funny-proto+name.proto b/tests/funny-proto+name.proto new file mode 100644 index 00000000..e69de29b -- cgit 1.2.3-korg From e7bf063abc2ca3a46c778e0194f0e8d4ef1a5168 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 7 Feb 2013 17:48:50 +0200 Subject: Add check for sizeof(double) == 8. Update issue 54 Status: FixedInGit --- generator/nanopb_generator.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index d35a4252..48239c18 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -512,6 +512,7 @@ def generate_header(dependencies, headername, enums, messages): yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count yield '#endif\n' + # Add checks for numeric limits worst = 0 worst_field = '' checks = [] @@ -549,6 +550,20 @@ def generate_header(dependencies, headername, enums, messages): yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) yield '#endif\n' + # Add check for sizeof(double) + has_double = False + for msg in messages: + for field in msg.fields: + if field.ctype == 'double': + has_double = True + + if has_double: + yield '\n' + yield '/* On some platforms (such as AVR), double is really float.\n' + yield ' * These are not directly supported by nanopb, but see example_avr_double.\n' + yield ' */\n' + yield 'STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n' + yield '\n#ifdef __cplusplus\n' yield '} /* extern "C" */\n' yield '#endif\n' -- cgit 1.2.3-korg From c1bd1a6ad3479088fef09c02e39b70bea0a718ae Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 7 Feb 2013 17:56:52 +0200 Subject: Fix error message bugs with packed arrays. Error messages were not propagated correctly with PB_HTYPE_ARRAY. Error status (boolean return value) was correct. Update issue 56 Status: FixedInGit --- pb_decode.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index b25f3f55..d4741130 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -361,7 +361,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t && PB_LTYPE(iter->current->type) <= PB_LTYPE_LAST_PACKABLE) { /* Packed array */ - bool status; + bool status = true; size_t *size = (size_t*)iter->pSize; pb_istream_t substream; if (!pb_make_string_substream(stream, &substream)) @@ -371,11 +371,17 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t { void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size); if (!func(&substream, iter->current, pItem)) - return false; + { + status = false; + break; + } (*size)++; } - status = (substream.bytes_left == 0); pb_close_string_substream(stream, &substream); + + if (substream.bytes_left != 0) + PB_RETURN_ERROR(stream, "array overflow"); + return status; } else -- cgit 1.2.3-korg From 22e0a78e5e406b0c1aed3ab46a0dbbd7d9493296 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 7 Feb 2013 18:06:42 +0200 Subject: Update changelog --- CHANGELOG | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index fa849235..772770ac 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,14 @@ +git master + Fixed error message bugs (issues 52, 56) + Sanitize #ifndef filename (issue 50) + Performance improvements + Add compile-time option PB_BUFFER_ONLY + Add Java package name to nanopb.proto + Check for sizeof(double) == 8 (issue 54) + Added generator option to ignore some fields. (issue 51) + Added generator option to make message structs packed. (issue 49) + Add more test cases. + nanopb-0.1.8 Fix bugs in the enum short names introduced in 0.1.7 (issues 42, 43) Fix STATIC_ASSERT macro when using multiple .proto files. (issue 41) -- cgit 1.2.3-korg From c1a355b23e3aa57bab901f0efd0604030ac50a4c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 11 Feb 2013 21:54:24 +0200 Subject: Set version to 0.2.0-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 48239c18..1683b013 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.1.9-dev" +nanopb_version = "0.2.0-dev" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index b19dfcfe..e28db6af 100644 --- a/pb.h +++ b/pb.h @@ -6,7 +6,7 @@ * see pb_encode.h or pb_decode.h */ -#define NANOPB_VERSION nanopb-0.1.9-dev +#define NANOPB_VERSION 0.2.0-dev #include #include -- cgit 1.2.3-korg From ec4a7a0ccef07266cdd3e309e1d3b7233413142c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 11 Feb 2013 21:55:55 +0200 Subject: Replace pb_type_t enum with #defines. See issue #57. --- pb.h | 108 +++++++++++++++++++++++++++++++++---------------------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/pb.h b/pb.h index e28db6af..29aed850 100644 --- a/pb.h +++ b/pb.h @@ -6,7 +6,7 @@ * see pb_encode.h or pb_decode.h */ -#define NANOPB_VERSION 0.2.0-dev +#define NANOPB_VERSION nanopb-0.2.0-dev #include #include @@ -53,59 +53,59 @@ * SINT* is different, though, because it is zig-zag coded. */ -typedef enum { - /************************ - * Field contents types * - ************************/ - - /* Numeric types */ - PB_LTYPE_VARINT = 0x00, /* int32, uint32, int64, uint64, bool, enum */ - PB_LTYPE_SVARINT = 0x01, /* sint32, sint64 */ - PB_LTYPE_FIXED32 = 0x02, /* fixed32, sfixed32, float */ - PB_LTYPE_FIXED64 = 0x03, /* fixed64, sfixed64, double */ - - /* Marker for last packable field type. */ - PB_LTYPE_LAST_PACKABLE = 0x03, - - /* Byte array with pre-allocated buffer. - * data_size is the length of the allocated PB_BYTES_ARRAY structure. */ - PB_LTYPE_BYTES = 0x04, - - /* String with pre-allocated buffer. - * data_size is the maximum length. */ - PB_LTYPE_STRING = 0x05, - - /* Submessage - * submsg_fields is pointer to field descriptions */ - PB_LTYPE_SUBMESSAGE = 0x06, - - /* Number of declared LTYPES */ - PB_LTYPES_COUNT = 7, - PB_LTYPE_MASK = 0x0F, - - /****************** - * Modifier flags * - ******************/ - - /* Just the basic, write data at data_offset */ - PB_HTYPE_REQUIRED = 0x00, - - /* Write true at size_offset */ - PB_HTYPE_OPTIONAL = 0x10, - - /* Read to pre-allocated array - * Maximum number of entries is array_size, - * actual number is stored at size_offset */ - PB_HTYPE_ARRAY = 0x20, - - /* Works for all required/optional/repeated fields. - * data_offset points to pb_callback_t structure. - * LTYPE should be valid or 0 (it is ignored, but - * sometimes used to speculatively index an array). */ - PB_HTYPE_CALLBACK = 0x30, - - PB_HTYPE_MASK = 0xF0 -} pb_packed pb_type_t; +typedef uint8_t pb_type_t; + +/************************ + * Field contents types * + ************************/ + +/* Numeric types */ +#define PB_LTYPE_VARINT 0x00 /* int32, uint32, int64, uint64, bool, enum */ +#define PB_LTYPE_SVARINT 0x01 /* sint32, sint64 */ +#define PB_LTYPE_FIXED32 0x02 /* fixed32, sfixed32, float */ +#define PB_LTYPE_FIXED64 0x03 /* fixed64, sfixed64, double */ + +/* Marker for last packable field type. */ +#define PB_LTYPE_LAST_PACKABLE 0x03 + +/* Byte array with pre-allocated buffer. + * data_size is the length of the allocated PB_BYTES_ARRAY structure. */ +#define PB_LTYPE_BYTES 0x04 + +/* String with pre-allocated buffer. + * data_size is the maximum length. */ +#define PB_LTYPE_STRING 0x05 + +/* Submessage + * submsg_fields is pointer to field descriptions */ +#define PB_LTYPE_SUBMESSAGE 0x06 + +/* Number of declared LTYPES */ +#define PB_LTYPES_COUNT 7 +#define PB_LTYPE_MASK 0x0F + +/****************** + * Modifier flags * + ******************/ + +/* Just the basic, write data at data_offset */ +#define PB_HTYPE_REQUIRED 0x00 + +/* Write true at size_offset */ +#define PB_HTYPE_OPTIONAL 0x10 + +/* Read to pre-allocated array + * Maximum number of entries is array_size, + * actual number is stored at size_offset */ +#define PB_HTYPE_ARRAY 0x20 + +/* Works for all required/optional/repeated fields. + * data_offset points to pb_callback_t structure. + * LTYPE should be valid or 0 (it is ignored, but + * sometimes used to speculatively index an array). */ +#define PB_HTYPE_CALLBACK 0x30 + +#define PB_HTYPE_MASK 0xF0 #define PB_HTYPE(x) ((x) & PB_HTYPE_MASK) #define PB_LTYPE(x) ((x) & PB_LTYPE_MASK) -- cgit 1.2.3-korg From 9d3d7b57303c6d6feaa1f7e79d7a09d8b5bacfa2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 11 Feb 2013 22:03:14 +0200 Subject: Add gitignore. This is a bit bloated gitignore file. Having binaries in a separate build directory would be cleaner, but I don't bother to make that change now. --- .gitignore | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..dec8b00e --- /dev/null +++ b/.gitignore @@ -0,0 +1,39 @@ +*.gcda +*.gcno +*.gcov +*.o +*.pb.c +*.pb.h +*.pb +*~ +*.tar.gz +julkaisu.txt +docs/*.html +docs/generator_flow.png +example/client +example/server +example_avr_double/decode_double +example_avr_double/encode_double +example_avr_double/test_conversions +example_unions/decode +example_unions/encode +generator/nanopb_pb2.pyc +tests/decode_unittests +tests/encode_unittests +tests/test_compiles +tests/test_decode1 +tests/test_decode2 +tests/test_decode3 +tests/test_decode3_buf +tests/test_decode_callbacks +tests/test_encode1 +tests/test_encode2 +tests/test_encode3 +tests/test_encode3_buf +tests/test_encode_callbacks +tests/test_missing_fields +tests/test_multiple_files +tests/bc_decode +tests/bc_encode +tests/breakpoints + -- cgit 1.2.3-korg From 4cc3372b0356f2f4db2353ed3374a59155e85736 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 13 Feb 2013 21:12:09 +0200 Subject: Publishing nanopb-0.1.9 --- CHANGELOG | 2 +- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 772770ac..655acc89 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,4 @@ -git master +nanopb-0.1.9 Fixed error message bugs (issues 52, 56) Sanitize #ifndef filename (issue 50) Performance improvements diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 48239c18..3e042154 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.1.9-dev" +nanopb_version = "nanopb-0.1.9" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index b19dfcfe..daba1277 100644 --- a/pb.h +++ b/pb.h @@ -6,7 +6,7 @@ * see pb_encode.h or pb_decode.h */ -#define NANOPB_VERSION nanopb-0.1.9-dev +#define NANOPB_VERSION nanopb-0.1.9 #include #include -- cgit 1.2.3-korg From 5b536d40a11edb3161f2f19609c2b51899c354b0 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 13 Feb 2013 21:13:35 +0200 Subject: Setting version to 0.1.9.1-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 3e042154..ca81adf5 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.1.9" +nanopb_version = "nanopb-0.1.9.1-dev" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index daba1277..bca31022 100644 --- a/pb.h +++ b/pb.h @@ -6,7 +6,7 @@ * see pb_encode.h or pb_decode.h */ -#define NANOPB_VERSION nanopb-0.1.9 +#define NANOPB_VERSION nanopb-0.1.9.1-dev #include #include -- cgit 1.2.3-korg From 258ba8335dd2b54ee9ce934782239f357bca3581 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Feb 2013 00:10:47 +0200 Subject: Reformat generated .pb.c files using macros. This has the following advantages: 1) Easier to modify pb_field_t encoding 2) Simpler generator logic 3) Tidier looking, easier to read .pb.c files Update issue 58 Status: FixedInGit --- generator/nanopb_generator.py | 149 ++++++++++++++++++------------------------ pb.h | 77 +++++++++++++++++++++- 2 files changed, 139 insertions(+), 87 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 1683b013..9ecc8f3e 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -35,22 +35,22 @@ except: import time import os.path -# Values are tuple (c type, pb ltype) +# Values are tuple (c type, pb type) FieldD = descriptor.FieldDescriptorProto datatypes = { - FieldD.TYPE_BOOL: ('bool', 'PB_LTYPE_VARINT'), - FieldD.TYPE_DOUBLE: ('double', 'PB_LTYPE_FIXED64'), - FieldD.TYPE_FIXED32: ('uint32_t', 'PB_LTYPE_FIXED32'), - FieldD.TYPE_FIXED64: ('uint64_t', 'PB_LTYPE_FIXED64'), - FieldD.TYPE_FLOAT: ('float', 'PB_LTYPE_FIXED32'), - FieldD.TYPE_INT32: ('int32_t', 'PB_LTYPE_VARINT'), - FieldD.TYPE_INT64: ('int64_t', 'PB_LTYPE_VARINT'), - FieldD.TYPE_SFIXED32: ('int32_t', 'PB_LTYPE_FIXED32'), - FieldD.TYPE_SFIXED64: ('int64_t', 'PB_LTYPE_FIXED64'), - FieldD.TYPE_SINT32: ('int32_t', 'PB_LTYPE_SVARINT'), - FieldD.TYPE_SINT64: ('int64_t', 'PB_LTYPE_SVARINT'), - FieldD.TYPE_UINT32: ('uint32_t', 'PB_LTYPE_VARINT'), - FieldD.TYPE_UINT64: ('uint64_t', 'PB_LTYPE_VARINT') + FieldD.TYPE_BOOL: ('bool', 'BOOL'), + FieldD.TYPE_DOUBLE: ('double', 'DOUBLE'), + FieldD.TYPE_FIXED32: ('uint32_t', 'FIXED32'), + FieldD.TYPE_FIXED64: ('uint64_t', 'FIXED64'), + FieldD.TYPE_FLOAT: ('float', 'FLOAT'), + FieldD.TYPE_INT32: ('int32_t', 'INT32'), + FieldD.TYPE_INT64: ('int64_t', 'INT64'), + FieldD.TYPE_SFIXED32: ('int32_t', 'SFIXED32'), + FieldD.TYPE_SFIXED64: ('int64_t', 'SFIXED64'), + FieldD.TYPE_SINT32: ('int32_t', 'SINT32'), + FieldD.TYPE_SINT64: ('int64_t', 'SINT64'), + FieldD.TYPE_UINT32: ('uint32_t', 'UINT32'), + FieldD.TYPE_UINT64: ('uint64_t', 'UINT64') } class Names: @@ -123,49 +123,44 @@ class Field: if desc.HasField('default_value'): self.default = desc.default_value - # Decide HTYPE - # HTYPE is the high-order nibble of nanopb field description, - # defining whether value is required/optional/repeated. + # Check field rules, i.e. required/optional/repeated. can_be_static = True if desc.label == FieldD.LABEL_REQUIRED: - self.htype = 'PB_HTYPE_REQUIRED' + self.rules = 'REQUIRED' elif desc.label == FieldD.LABEL_OPTIONAL: - self.htype = 'PB_HTYPE_OPTIONAL' + self.rules = 'OPTIONAL' elif desc.label == FieldD.LABEL_REPEATED: + self.rules = 'REPEATED' if self.max_count is None: can_be_static = False else: - self.htype = 'PB_HTYPE_ARRAY' self.array_decl = '[%d]' % self.max_count else: raise NotImplementedError(desc.label) - # Decide LTYPE and CTYPE - # LTYPE is the low-order nibble of nanopb field description, - # defining how to decode an individual value. - # CTYPE is the name of the c type to use in the struct. + # Decide the C data type to use in the struct. if datatypes.has_key(desc.type): - self.ctype, self.ltype = datatypes[desc.type] + self.ctype, self.pbtype = datatypes[desc.type] elif desc.type == FieldD.TYPE_ENUM: - self.ltype = 'PB_LTYPE_VARINT' + self.pbtype = 'ENUM' self.ctype = names_from_type_name(desc.type_name) if self.default is not None: self.default = self.ctype + self.default elif desc.type == FieldD.TYPE_STRING: - self.ltype = 'PB_LTYPE_STRING' + self.pbtype = 'STRING' if self.max_size is None: can_be_static = False else: self.ctype = 'char' self.array_decl += '[%d]' % self.max_size elif desc.type == FieldD.TYPE_BYTES: - self.ltype = 'PB_LTYPE_BYTES' + self.pbtype = 'BYTES' if self.max_size is None: can_be_static = False else: self.ctype = self.struct_name + self.name + 't' elif desc.type == FieldD.TYPE_MESSAGE: - self.ltype = 'PB_LTYPE_SUBMESSAGE' + self.pbtype = 'MESSAGE' self.ctype = self.submsgname = names_from_type_name(desc.type_name) else: raise NotImplementedError(desc.type) @@ -179,18 +174,22 @@ class Field: if field_options.type == nanopb_pb2.FT_STATIC and not can_be_static: raise Exception("Field %s is defined as static, but max_size or max_count is not given." % self.name) - if field_options.type == nanopb_pb2.FT_CALLBACK: - self.htype = 'PB_HTYPE_CALLBACK' + if field_options.type == nanopb_pb2.FT_STATIC: + self.allocation = 'STATIC' + elif field_options.type == nanopb_pb2.FT_CALLBACK: + self.allocation = 'CALLBACK' self.ctype = 'pb_callback_t' self.array_decl = '' + else: + raise NotImplementedError(field_options.type) def __cmp__(self, other): return cmp(self.tag, other.tag) def __str__(self): - if self.htype == 'PB_HTYPE_OPTIONAL': + if self.rules == 'OPTIONAL': result = ' bool has_' + self.name + ';\n' - elif self.htype == 'PB_HTYPE_ARRAY': + elif self.rules == 'REPEATED' and self.allocation == 'STATIC': result = ' size_t ' + self.name + '_count;\n' else: result = '' @@ -199,7 +198,7 @@ class Field: def types(self): '''Return definitions for any special types this field might need.''' - if self.ltype == 'PB_LTYPE_BYTES' and self.max_size is not None: + if self.pbtype == 'BYTES' and self.allocation == 'STATIC': result = 'typedef struct {\n' result += ' size_t size;\n' result += ' uint8_t bytes[%d];\n' % self.max_size @@ -212,30 +211,25 @@ class Field: '''Return definition for this field's default value.''' if self.default is None: return None + + ctype, default = self.ctype, self.default + array_decl = '' - if self.ltype == 'PB_LTYPE_STRING': - ctype = 'char' - if self.max_size is None: + if self.pbtype == 'STRING': + if self.allocation != 'STATIC': return None # Not implemented - else: - array_decl = '[%d]' % (self.max_size + 1) + + array_decl = '[%d]' % self.max_size default = str(self.default).encode('string_escape') default = default.replace('"', '\\"') default = '"' + default + '"' - elif self.ltype == 'PB_LTYPE_BYTES': + elif self.pbtype == 'BYTES': + if self.allocation != 'STATIC': + return None # Not implemented + data = self.default.decode('string_escape') data = ['0x%02x' % ord(c) for c in data] - - if self.max_size is None: - return None # Not implemented - else: - ctype = self.ctype - default = '{%d, {%s}}' % (len(data), ','.join(data)) - array_decl = '' - else: - ctype, default = self.ctype, self.default - array_decl = '' if declaration_only: return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl) @@ -246,47 +240,30 @@ class Field: '''Return the pb_field_t initializer to use in the constant array. prev_field_name is the name of the previous field or None. ''' - result = ' {%d, ' % self.tag - result += '(pb_type_t) ((int) ' + self.htype - if self.ltype is not None: - result += ' | (int) ' + self.ltype - result += '),\n' - - if prev_field_name is None: - result += ' offsetof(%s, %s),' % (self.struct_name, self.name) - else: - result += ' pb_delta_end(%s, %s, %s),' % (self.struct_name, self.name, prev_field_name) - - if self.htype == 'PB_HTYPE_OPTIONAL': - result += '\n pb_delta(%s, has_%s, %s),' % (self.struct_name, self.name, self.name) - elif self.htype == 'PB_HTYPE_ARRAY': - result += '\n pb_delta(%s, %s_count, %s),' % (self.struct_name, self.name, self.name) - else: - result += ' 0,' - - - if self.htype == 'PB_HTYPE_ARRAY': - result += '\n pb_membersize(%s, %s[0]),' % (self.struct_name, self.name) - result += ('\n pb_membersize(%s, %s) / pb_membersize(%s, %s[0]),' - % (self.struct_name, self.name, self.struct_name, self.name)) - else: - result += '\n pb_membersize(%s, %s),' % (self.struct_name, self.name) - result += ' 0,' - - if self.ltype == 'PB_LTYPE_SUBMESSAGE': - result += '\n &%s_fields}' % self.submsgname - elif self.default is None or self.htype == 'PB_HTYPE_CALLBACK': - result += ' 0}' + result = ' PB_FIELD(%3d, ' % self.tag + result += '%-8s, ' % self.pbtype + result += '%s, ' % self.rules + result += '%s, ' % self.allocation + result += '%s, ' % self.struct_name + result += '%s, ' % self.name + result += '%s, ' % (prev_field_name or self.name) + + if self.pbtype == 'MESSAGE': + result += '&%s_fields)' % self.submsgname + elif self.default is None: + result += '0)' + elif self.pbtype in ['BYTES', 'STRING'] and self.allocation != 'STATIC': + result += '0)' # Arbitrary size default values not implemented else: - result += '\n &%s_default}' % (self.struct_name + self.name) + result += '&%s_default)' % (self.struct_name + self.name) return result def largest_field_value(self): '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly. Returns numeric value or a C-expression for assert.''' - if self.ltype == 'PB_LTYPE_SUBMESSAGE': - if self.htype == 'PB_HTYPE_ARRAY': + if self.pbtype == 'MESSAGE': + if self.rules == 'REPEATED' and self.allocation == 'STATIC': return 'pb_membersize(%s, %s[0])' % (self.struct_name, self.name) else: return 'pb_membersize(%s, %s)' % (self.struct_name, self.name) @@ -358,7 +335,7 @@ class Message: prev = None for field in self.ordered_fields: result += field.pb_field_t(prev) - result += ',\n\n' + result += ',\n' prev = field.name result += ' PB_LAST_FIELD\n};' @@ -502,7 +479,7 @@ def generate_header(dependencies, headername, enums, messages): yield msg.fields_declaration() + '\n' if messages: - count_required_fields = lambda m: len([f for f in msg.fields if f.htype == 'PB_HTYPE_REQUIRED']) + count_required_fields = lambda m: len([f for f in msg.fields if f.rules == 'REQUIRED']) largest_msg = max(messages, key = count_required_fields) largest_count = count_required_fields(largest_msg) if largest_count > 64: diff --git a/pb.h b/pb.h index 29aed850..3d4323ae 100644 --- a/pb.h +++ b/pb.h @@ -204,9 +204,84 @@ typedef enum { #define pb_membersize(st, m) (sizeof ((st*)0)->m) #define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0])) #define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2)) -#define pb_delta_end(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) +#define pb_delta_end(st, m1, m2) (offsetof(st, m1) == offsetof(st, m2) \ + ? offsetof(st, m1) \ + : offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) #define PB_LAST_FIELD {0,(pb_type_t) 0,0,0,0,0,0} +/* Required fields are the simplest. They just have delta (padding) from + * previous field end, and the size of the field. Pointer is used for + * submessages and default values. + */ +#define PB_REQUIRED_STATIC(tag, st, m, pm, ltype, ptr) \ + {tag, PB_HTYPE_REQUIRED | ltype, \ + pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr} + +/* Optional fields add the delta to the has_ variable. */ +#define PB_OPTIONAL_STATIC(tag, st, m, pm, ltype, ptr) \ + {tag, PB_HTYPE_OPTIONAL | ltype, \ + pb_delta_end(st, m, pm), \ + pb_delta(st, has_ ## m, m), \ + pb_membersize(st, m), 0, ptr} + +/* Repeated fields have a _count field and also the maximum number of entries. */ +#define PB_REPEATED_STATIC(tag, st, m, pm, ltype, ptr) \ + {tag, PB_HTYPE_ARRAY | ltype, \ + pb_delta_end(st, m, pm), \ + pb_delta(st, m ## _count, m), \ + pb_membersize(st, m[0]), \ + pb_arraysize(st, m), ptr} + +/* Callbacks are much like required fields except with special datatype. */ +#define PB_REQUIRED_CALLBACK(tag, st, m, pm, ltype, ptr) \ + {tag, PB_HTYPE_CALLBACK | ltype, \ + pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr} + +#define PB_OPTIONAL_CALLBACK(tag, st, m, pm, ltype, ptr) \ + {tag, PB_HTYPE_CALLBACK | ltype, \ + pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr} + +#define PB_REPEATED_CALLBACK(tag, st, m, pm, ltype, ptr) \ + {tag, PB_HTYPE_CALLBACK | ltype, \ + pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr} + +/* The mapping from protobuf types to LTYPEs is done using these macros. */ +#define PB_LTYPE_MAP_BOOL PB_LTYPE_VARINT +#define PB_LTYPE_MAP_BYTES PB_LTYPE_BYTES +#define PB_LTYPE_MAP_DOUBLE PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_ENUM PB_LTYPE_VARINT +#define PB_LTYPE_MAP_FIXED32 PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_FIXED64 PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_FLOAT PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_INT32 PB_LTYPE_VARINT +#define PB_LTYPE_MAP_INT64 PB_LTYPE_VARINT +#define PB_LTYPE_MAP_MESSAGE PB_LTYPE_SUBMESSAGE +#define PB_LTYPE_MAP_SFIXED32 PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_SFIXED64 PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_SINT32 PB_LTYPE_SVARINT +#define PB_LTYPE_MAP_SINT64 PB_LTYPE_SVARINT +#define PB_LTYPE_MAP_STRING PB_LTYPE_STRING +#define PB_LTYPE_MAP_UINT32 PB_LTYPE_VARINT +#define PB_LTYPE_MAP_UINT64 PB_LTYPE_VARINT + +/* This is the actual macro used in field descriptions. + * It takes these arguments: + * - Field tag number + * - Field type: BOOL, BYTES, DOUBLE, ENUM, FIXED32, FIXED64, + * FLOAT, INT32, INT64, MESSAGE, SFIXED32, SFIXED64 + * SINT32, SINT64, STRING, UINT32 or UINT64 + * - Field rules: REQUIRED, OPTIONAL or REPEATED + * - Allocation: STATIC or CALLBACK + * - Message name + * - Field name + * - Previous field name (or field name again for first field) + * - Pointer to default value or submsg fields. + */ + +#define PB_FIELD(tag, type, rules, allocation, message, field, prevfield, ptr) \ + PB_ ## rules ## _ ## allocation(tag, message, field, prevfield, \ + PB_LTYPE_MAP_ ## type, ptr) + /* These macros are used for giving out error messages. * They are mostly a debugging aid; the main error information * is the true/false return value from functions. -- cgit 1.2.3-korg From 69085d93877ef75ca28505487d8a1335a4b7fa6c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 20 Feb 2013 21:58:18 +0200 Subject: Rename PB_HTYPE_ARRAY -> PB_HTYPE_REPEATED. This is a more logical name in parallel with PB_HTYPE_REQUIRED and PB_HTYPE_OPTIONAL. Warning: This breaks backwards-compatibility of generated .pb.c files. You will have to regenerate the files and recompile. --- pb.h | 4 +- pb_decode.c | 8 +- pb_encode.c | 4 +- tests/bc_alltypes.pb.c | 361 +++++++++---------------------------------------- tests/bc_alltypes.pb.h | 39 ++++-- 5 files changed, 103 insertions(+), 313 deletions(-) diff --git a/pb.h b/pb.h index 3d4323ae..cd3f465e 100644 --- a/pb.h +++ b/pb.h @@ -97,7 +97,7 @@ typedef uint8_t pb_type_t; /* Read to pre-allocated array * Maximum number of entries is array_size, * actual number is stored at size_offset */ -#define PB_HTYPE_ARRAY 0x20 +#define PB_HTYPE_REPEATED 0x20 /* Works for all required/optional/repeated fields. * data_offset points to pb_callback_t structure. @@ -226,7 +226,7 @@ typedef enum { /* Repeated fields have a _count field and also the maximum number of entries. */ #define PB_REPEATED_STATIC(tag, st, m, pm, ltype, ptr) \ - {tag, PB_HTYPE_ARRAY | ltype, \ + {tag, PB_HTYPE_REPEATED | ltype, \ pb_delta_end(st, m, pm), \ pb_delta(st, m ## _count, m), \ pb_membersize(st, m[0]), \ diff --git a/pb_decode.c b/pb_decode.c index d4741130..351c8ec7 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -303,7 +303,7 @@ static bool pb_field_next(pb_field_iterator_t *iter) bool notwrapped = true; size_t prev_size = iter->current->data_size; - if (PB_HTYPE(iter->current->type) == PB_HTYPE_ARRAY) + if (PB_HTYPE(iter->current->type) == PB_HTYPE_REPEATED) prev_size *= iter->current->array_size; if (PB_HTYPE(iter->current->type) == PB_HTYPE_REQUIRED) @@ -356,7 +356,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t *(bool*)iter->pSize = true; return func(stream, iter->current, iter->pData); - case PB_HTYPE_ARRAY: + case PB_HTYPE_REPEATED: if (wire_type == PB_WT_STRING && PB_LTYPE(iter->current->type) <= PB_LTYPE_LAST_PACKABLE) { @@ -459,7 +459,7 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str { *(bool*)iter.pSize = false; } - else if (PB_HTYPE(iter.current->type) == PB_HTYPE_ARRAY) + else if (PB_HTYPE(iter.current->type) == PB_HTYPE_REPEATED) { *(size_t*)iter.pSize = 0; continue; /* Array is empty, no need to initialize contents */ @@ -715,7 +715,7 @@ bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field /* New array entries need to be initialized, while required and optional * submessages have already been initialized in the top-level pb_decode. */ - if (PB_HTYPE(field->type) == PB_HTYPE_ARRAY) + if (PB_HTYPE(field->type) == PB_HTYPE_REPEATED) status = pb_decode(&substream, submsg_fields, dest); else status = pb_decode_noinit(&substream, submsg_fields, dest); diff --git a/pb_encode.c b/pb_encode.c index bba5dc19..85d65782 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -167,7 +167,7 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons pSize = (const char*)pData + field->size_offset; prev_size = field->data_size; - if (PB_HTYPE(field->type) == PB_HTYPE_ARRAY) + if (PB_HTYPE(field->type) == PB_HTYPE_REPEATED) prev_size *= field->array_size; switch (PB_HTYPE(field->type)) @@ -190,7 +190,7 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons } break; - case PB_HTYPE_ARRAY: + case PB_HTYPE_REPEATED: if (!encode_array(stream, field, pData, *(const size_t*)pSize, func)) return false; break; diff --git a/tests/bc_alltypes.pb.c b/tests/bc_alltypes.pb.c index 322b634e..7d1edf57 100644 --- a/tests/bc_alltypes.pb.c +++ b/tests/bc_alltypes.pb.c @@ -1,7 +1,13 @@ /* Automatically generated nanopb constant definitions */ -#include "bc_alltypes.pb.h" +/* Generated by 0.2.0-dev at Sun Feb 17 00:09:53 2013. */ +/* This is a file generated using nanopb-0.2.0-dev. + * It is used as a part of test suite in order to detect any + * incompatible changes made to the generator in future versions. + */ -const char SubMessage_substuff1_default[17] = "1"; +#include "alltypes.pb.h" + +const char SubMessage_substuff1_default[16] = "1"; const int32_t SubMessage_substuff2_default = 2; const uint32_t SubMessage_substuff3_default = 3; const int32_t AllTypes_opt_int32_default = 4041; @@ -17,310 +23,71 @@ const float AllTypes_opt_float_default = 4050; const uint64_t AllTypes_opt_fixed64_default = 4051; const int64_t AllTypes_opt_sfixed64_default = 4052; const double AllTypes_opt_double_default = 4053; -const char AllTypes_opt_string_default[17] = "4054"; +const char AllTypes_opt_string_default[16] = "4054"; const AllTypes_opt_bytes_t AllTypes_opt_bytes_default = {4, {0x34,0x30,0x35,0x35}}; const MyEnum AllTypes_opt_enum_default = MyEnum_Second; const pb_field_t SubMessage_fields[4] = { - {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING, - offsetof(SubMessage, substuff1), 0, - pb_membersize(SubMessage, substuff1), 0, - &SubMessage_substuff1_default}, - - {2, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, - pb_delta_end(SubMessage, substuff2, substuff1), 0, - pb_membersize(SubMessage, substuff2), 0, - &SubMessage_substuff2_default}, - - {3, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED32, - pb_delta_end(SubMessage, substuff3, substuff2), - pb_delta(SubMessage, has_substuff3, substuff3), - pb_membersize(SubMessage, substuff3), 0, - &SubMessage_substuff3_default}, - + PB_FIELD( 1, STRING , REQUIRED, STATIC, SubMessage, substuff1, substuff1, &SubMessage_substuff1_default), + PB_FIELD( 2, INT32 , REQUIRED, STATIC, SubMessage, substuff2, substuff1, &SubMessage_substuff2_default), + PB_FIELD( 3, FIXED32 , OPTIONAL, STATIC, SubMessage, substuff3, substuff2, &SubMessage_substuff3_default), PB_LAST_FIELD }; const pb_field_t AllTypes_fields[53] = { - {1, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, - offsetof(AllTypes, req_int32), 0, - pb_membersize(AllTypes, req_int32), 0, 0}, - - {2, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, req_int64, req_int32), 0, - pb_membersize(AllTypes, req_int64), 0, 0}, - - {3, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, req_uint32, req_int64), 0, - pb_membersize(AllTypes, req_uint32), 0, 0}, - - {4, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, req_uint64, req_uint32), 0, - pb_membersize(AllTypes, req_uint64), 0, 0}, - - {5, PB_HTYPE_REQUIRED | PB_LTYPE_SVARINT, - pb_delta_end(AllTypes, req_sint32, req_uint64), 0, - pb_membersize(AllTypes, req_sint32), 0, 0}, - - {6, PB_HTYPE_REQUIRED | PB_LTYPE_SVARINT, - pb_delta_end(AllTypes, req_sint64, req_sint32), 0, - pb_membersize(AllTypes, req_sint64), 0, 0}, - - {7, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, req_bool, req_sint64), 0, - pb_membersize(AllTypes, req_bool), 0, 0}, - - {8, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED32, - pb_delta_end(AllTypes, req_fixed32, req_bool), 0, - pb_membersize(AllTypes, req_fixed32), 0, 0}, - - {9, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED32, - pb_delta_end(AllTypes, req_sfixed32, req_fixed32), 0, - pb_membersize(AllTypes, req_sfixed32), 0, 0}, - - {10, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED32, - pb_delta_end(AllTypes, req_float, req_sfixed32), 0, - pb_membersize(AllTypes, req_float), 0, 0}, - - {11, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED64, - pb_delta_end(AllTypes, req_fixed64, req_float), 0, - pb_membersize(AllTypes, req_fixed64), 0, 0}, - - {12, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED64, - pb_delta_end(AllTypes, req_sfixed64, req_fixed64), 0, - pb_membersize(AllTypes, req_sfixed64), 0, 0}, - - {13, PB_HTYPE_REQUIRED | PB_LTYPE_FIXED64, - pb_delta_end(AllTypes, req_double, req_sfixed64), 0, - pb_membersize(AllTypes, req_double), 0, 0}, - - {14, PB_HTYPE_REQUIRED | PB_LTYPE_STRING, - pb_delta_end(AllTypes, req_string, req_double), 0, - pb_membersize(AllTypes, req_string), 0, 0}, - - {15, PB_HTYPE_REQUIRED | PB_LTYPE_BYTES, - pb_delta_end(AllTypes, req_bytes, req_string), 0, - pb_membersize(AllTypes, req_bytes), 0, 0}, - - {16, PB_HTYPE_REQUIRED | PB_LTYPE_SUBMESSAGE, - pb_delta_end(AllTypes, req_submsg, req_bytes), 0, - pb_membersize(AllTypes, req_submsg), 0, - &SubMessage_fields}, - - {17, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, req_enum, req_submsg), 0, - pb_membersize(AllTypes, req_enum), 0, 0}, - - {21, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, rep_int32, req_enum), - pb_delta(AllTypes, rep_int32_count, rep_int32), - pb_membersize(AllTypes, rep_int32[0]), - pb_membersize(AllTypes, rep_int32) / pb_membersize(AllTypes, rep_int32[0]), 0}, - - {22, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, rep_int64, rep_int32), - pb_delta(AllTypes, rep_int64_count, rep_int64), - pb_membersize(AllTypes, rep_int64[0]), - pb_membersize(AllTypes, rep_int64) / pb_membersize(AllTypes, rep_int64[0]), 0}, - - {23, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, rep_uint32, rep_int64), - pb_delta(AllTypes, rep_uint32_count, rep_uint32), - pb_membersize(AllTypes, rep_uint32[0]), - pb_membersize(AllTypes, rep_uint32) / pb_membersize(AllTypes, rep_uint32[0]), 0}, - - {24, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, rep_uint64, rep_uint32), - pb_delta(AllTypes, rep_uint64_count, rep_uint64), - pb_membersize(AllTypes, rep_uint64[0]), - pb_membersize(AllTypes, rep_uint64) / pb_membersize(AllTypes, rep_uint64[0]), 0}, - - {25, PB_HTYPE_ARRAY | PB_LTYPE_SVARINT, - pb_delta_end(AllTypes, rep_sint32, rep_uint64), - pb_delta(AllTypes, rep_sint32_count, rep_sint32), - pb_membersize(AllTypes, rep_sint32[0]), - pb_membersize(AllTypes, rep_sint32) / pb_membersize(AllTypes, rep_sint32[0]), 0}, - - {26, PB_HTYPE_ARRAY | PB_LTYPE_SVARINT, - pb_delta_end(AllTypes, rep_sint64, rep_sint32), - pb_delta(AllTypes, rep_sint64_count, rep_sint64), - pb_membersize(AllTypes, rep_sint64[0]), - pb_membersize(AllTypes, rep_sint64) / pb_membersize(AllTypes, rep_sint64[0]), 0}, - - {27, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, rep_bool, rep_sint64), - pb_delta(AllTypes, rep_bool_count, rep_bool), - pb_membersize(AllTypes, rep_bool[0]), - pb_membersize(AllTypes, rep_bool) / pb_membersize(AllTypes, rep_bool[0]), 0}, - - {28, PB_HTYPE_ARRAY | PB_LTYPE_FIXED32, - pb_delta_end(AllTypes, rep_fixed32, rep_bool), - pb_delta(AllTypes, rep_fixed32_count, rep_fixed32), - pb_membersize(AllTypes, rep_fixed32[0]), - pb_membersize(AllTypes, rep_fixed32) / pb_membersize(AllTypes, rep_fixed32[0]), 0}, - - {29, PB_HTYPE_ARRAY | PB_LTYPE_FIXED32, - pb_delta_end(AllTypes, rep_sfixed32, rep_fixed32), - pb_delta(AllTypes, rep_sfixed32_count, rep_sfixed32), - pb_membersize(AllTypes, rep_sfixed32[0]), - pb_membersize(AllTypes, rep_sfixed32) / pb_membersize(AllTypes, rep_sfixed32[0]), 0}, - - {30, PB_HTYPE_ARRAY | PB_LTYPE_FIXED32, - pb_delta_end(AllTypes, rep_float, rep_sfixed32), - pb_delta(AllTypes, rep_float_count, rep_float), - pb_membersize(AllTypes, rep_float[0]), - pb_membersize(AllTypes, rep_float) / pb_membersize(AllTypes, rep_float[0]), 0}, - - {31, PB_HTYPE_ARRAY | PB_LTYPE_FIXED64, - pb_delta_end(AllTypes, rep_fixed64, rep_float), - pb_delta(AllTypes, rep_fixed64_count, rep_fixed64), - pb_membersize(AllTypes, rep_fixed64[0]), - pb_membersize(AllTypes, rep_fixed64) / pb_membersize(AllTypes, rep_fixed64[0]), 0}, - - {32, PB_HTYPE_ARRAY | PB_LTYPE_FIXED64, - pb_delta_end(AllTypes, rep_sfixed64, rep_fixed64), - pb_delta(AllTypes, rep_sfixed64_count, rep_sfixed64), - pb_membersize(AllTypes, rep_sfixed64[0]), - pb_membersize(AllTypes, rep_sfixed64) / pb_membersize(AllTypes, rep_sfixed64[0]), 0}, - - {33, PB_HTYPE_ARRAY | PB_LTYPE_FIXED64, - pb_delta_end(AllTypes, rep_double, rep_sfixed64), - pb_delta(AllTypes, rep_double_count, rep_double), - pb_membersize(AllTypes, rep_double[0]), - pb_membersize(AllTypes, rep_double) / pb_membersize(AllTypes, rep_double[0]), 0}, - - {34, PB_HTYPE_ARRAY | PB_LTYPE_STRING, - pb_delta_end(AllTypes, rep_string, rep_double), - pb_delta(AllTypes, rep_string_count, rep_string), - pb_membersize(AllTypes, rep_string[0]), - pb_membersize(AllTypes, rep_string) / pb_membersize(AllTypes, rep_string[0]), 0}, - - {35, PB_HTYPE_ARRAY | PB_LTYPE_BYTES, - pb_delta_end(AllTypes, rep_bytes, rep_string), - pb_delta(AllTypes, rep_bytes_count, rep_bytes), - pb_membersize(AllTypes, rep_bytes[0]), - pb_membersize(AllTypes, rep_bytes) / pb_membersize(AllTypes, rep_bytes[0]), 0}, - - {36, PB_HTYPE_ARRAY | PB_LTYPE_SUBMESSAGE, - pb_delta_end(AllTypes, rep_submsg, rep_bytes), - pb_delta(AllTypes, rep_submsg_count, rep_submsg), - pb_membersize(AllTypes, rep_submsg[0]), - pb_membersize(AllTypes, rep_submsg) / pb_membersize(AllTypes, rep_submsg[0]), - &SubMessage_fields}, - - {37, PB_HTYPE_ARRAY | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, rep_enum, rep_submsg), - pb_delta(AllTypes, rep_enum_count, rep_enum), - pb_membersize(AllTypes, rep_enum[0]), - pb_membersize(AllTypes, rep_enum) / pb_membersize(AllTypes, rep_enum[0]), 0}, - - {41, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, opt_int32, rep_enum), - pb_delta(AllTypes, has_opt_int32, opt_int32), - pb_membersize(AllTypes, opt_int32), 0, - &AllTypes_opt_int32_default}, - - {42, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, opt_int64, opt_int32), - pb_delta(AllTypes, has_opt_int64, opt_int64), - pb_membersize(AllTypes, opt_int64), 0, - &AllTypes_opt_int64_default}, - - {43, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, opt_uint32, opt_int64), - pb_delta(AllTypes, has_opt_uint32, opt_uint32), - pb_membersize(AllTypes, opt_uint32), 0, - &AllTypes_opt_uint32_default}, - - {44, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, opt_uint64, opt_uint32), - pb_delta(AllTypes, has_opt_uint64, opt_uint64), - pb_membersize(AllTypes, opt_uint64), 0, - &AllTypes_opt_uint64_default}, - - {45, PB_HTYPE_OPTIONAL | PB_LTYPE_SVARINT, - pb_delta_end(AllTypes, opt_sint32, opt_uint64), - pb_delta(AllTypes, has_opt_sint32, opt_sint32), - pb_membersize(AllTypes, opt_sint32), 0, - &AllTypes_opt_sint32_default}, - - {46, PB_HTYPE_OPTIONAL | PB_LTYPE_SVARINT, - pb_delta_end(AllTypes, opt_sint64, opt_sint32), - pb_delta(AllTypes, has_opt_sint64, opt_sint64), - pb_membersize(AllTypes, opt_sint64), 0, - &AllTypes_opt_sint64_default}, - - {47, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, opt_bool, opt_sint64), - pb_delta(AllTypes, has_opt_bool, opt_bool), - pb_membersize(AllTypes, opt_bool), 0, - &AllTypes_opt_bool_default}, - - {48, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED32, - pb_delta_end(AllTypes, opt_fixed32, opt_bool), - pb_delta(AllTypes, has_opt_fixed32, opt_fixed32), - pb_membersize(AllTypes, opt_fixed32), 0, - &AllTypes_opt_fixed32_default}, - - {49, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED32, - pb_delta_end(AllTypes, opt_sfixed32, opt_fixed32), - pb_delta(AllTypes, has_opt_sfixed32, opt_sfixed32), - pb_membersize(AllTypes, opt_sfixed32), 0, - &AllTypes_opt_sfixed32_default}, - - {50, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED32, - pb_delta_end(AllTypes, opt_float, opt_sfixed32), - pb_delta(AllTypes, has_opt_float, opt_float), - pb_membersize(AllTypes, opt_float), 0, - &AllTypes_opt_float_default}, - - {51, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED64, - pb_delta_end(AllTypes, opt_fixed64, opt_float), - pb_delta(AllTypes, has_opt_fixed64, opt_fixed64), - pb_membersize(AllTypes, opt_fixed64), 0, - &AllTypes_opt_fixed64_default}, - - {52, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED64, - pb_delta_end(AllTypes, opt_sfixed64, opt_fixed64), - pb_delta(AllTypes, has_opt_sfixed64, opt_sfixed64), - pb_membersize(AllTypes, opt_sfixed64), 0, - &AllTypes_opt_sfixed64_default}, - - {53, PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED64, - pb_delta_end(AllTypes, opt_double, opt_sfixed64), - pb_delta(AllTypes, has_opt_double, opt_double), - pb_membersize(AllTypes, opt_double), 0, - &AllTypes_opt_double_default}, - - {54, PB_HTYPE_OPTIONAL | PB_LTYPE_STRING, - pb_delta_end(AllTypes, opt_string, opt_double), - pb_delta(AllTypes, has_opt_string, opt_string), - pb_membersize(AllTypes, opt_string), 0, - &AllTypes_opt_string_default}, - - {55, PB_HTYPE_OPTIONAL | PB_LTYPE_BYTES, - pb_delta_end(AllTypes, opt_bytes, opt_string), - pb_delta(AllTypes, has_opt_bytes, opt_bytes), - pb_membersize(AllTypes, opt_bytes), 0, - &AllTypes_opt_bytes_default}, - - {56, PB_HTYPE_OPTIONAL | PB_LTYPE_SUBMESSAGE, - pb_delta_end(AllTypes, opt_submsg, opt_bytes), - pb_delta(AllTypes, has_opt_submsg, opt_submsg), - pb_membersize(AllTypes, opt_submsg), 0, - &SubMessage_fields}, - - {57, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, opt_enum, opt_submsg), - pb_delta(AllTypes, has_opt_enum, opt_enum), - pb_membersize(AllTypes, opt_enum), 0, - &AllTypes_opt_enum_default}, - - {99, PB_HTYPE_REQUIRED | PB_LTYPE_VARINT, - pb_delta_end(AllTypes, end, opt_enum), 0, - pb_membersize(AllTypes, end), 0, 0}, - + PB_FIELD( 1, INT32 , REQUIRED, STATIC, AllTypes, req_int32, req_int32, 0), + PB_FIELD( 2, INT64 , REQUIRED, STATIC, AllTypes, req_int64, req_int32, 0), + PB_FIELD( 3, UINT32 , REQUIRED, STATIC, AllTypes, req_uint32, req_int64, 0), + PB_FIELD( 4, UINT64 , REQUIRED, STATIC, AllTypes, req_uint64, req_uint32, 0), + PB_FIELD( 5, SINT32 , REQUIRED, STATIC, AllTypes, req_sint32, req_uint64, 0), + PB_FIELD( 6, SINT64 , REQUIRED, STATIC, AllTypes, req_sint64, req_sint32, 0), + PB_FIELD( 7, BOOL , REQUIRED, STATIC, AllTypes, req_bool, req_sint64, 0), + PB_FIELD( 8, FIXED32 , REQUIRED, STATIC, AllTypes, req_fixed32, req_bool, 0), + PB_FIELD( 9, SFIXED32, REQUIRED, STATIC, AllTypes, req_sfixed32, req_fixed32, 0), + PB_FIELD( 10, FLOAT , REQUIRED, STATIC, AllTypes, req_float, req_sfixed32, 0), + PB_FIELD( 11, FIXED64 , REQUIRED, STATIC, AllTypes, req_fixed64, req_float, 0), + PB_FIELD( 12, SFIXED64, REQUIRED, STATIC, AllTypes, req_sfixed64, req_fixed64, 0), + PB_FIELD( 13, DOUBLE , REQUIRED, STATIC, AllTypes, req_double, req_sfixed64, 0), + PB_FIELD( 14, STRING , REQUIRED, STATIC, AllTypes, req_string, req_double, 0), + PB_FIELD( 15, BYTES , REQUIRED, STATIC, AllTypes, req_bytes, req_string, 0), + PB_FIELD( 16, MESSAGE , REQUIRED, STATIC, AllTypes, req_submsg, req_bytes, &SubMessage_fields), + PB_FIELD( 17, ENUM , REQUIRED, STATIC, AllTypes, req_enum, req_submsg, 0), + PB_FIELD( 21, INT32 , REPEATED, STATIC, AllTypes, rep_int32, req_enum, 0), + PB_FIELD( 22, INT64 , REPEATED, STATIC, AllTypes, rep_int64, rep_int32, 0), + PB_FIELD( 23, UINT32 , REPEATED, STATIC, AllTypes, rep_uint32, rep_int64, 0), + PB_FIELD( 24, UINT64 , REPEATED, STATIC, AllTypes, rep_uint64, rep_uint32, 0), + PB_FIELD( 25, SINT32 , REPEATED, STATIC, AllTypes, rep_sint32, rep_uint64, 0), + PB_FIELD( 26, SINT64 , REPEATED, STATIC, AllTypes, rep_sint64, rep_sint32, 0), + PB_FIELD( 27, BOOL , REPEATED, STATIC, AllTypes, rep_bool, rep_sint64, 0), + PB_FIELD( 28, FIXED32 , REPEATED, STATIC, AllTypes, rep_fixed32, rep_bool, 0), + PB_FIELD( 29, SFIXED32, REPEATED, STATIC, AllTypes, rep_sfixed32, rep_fixed32, 0), + PB_FIELD( 30, FLOAT , REPEATED, STATIC, AllTypes, rep_float, rep_sfixed32, 0), + PB_FIELD( 31, FIXED64 , REPEATED, STATIC, AllTypes, rep_fixed64, rep_float, 0), + PB_FIELD( 32, SFIXED64, REPEATED, STATIC, AllTypes, rep_sfixed64, rep_fixed64, 0), + PB_FIELD( 33, DOUBLE , REPEATED, STATIC, AllTypes, rep_double, rep_sfixed64, 0), + PB_FIELD( 34, STRING , REPEATED, STATIC, AllTypes, rep_string, rep_double, 0), + PB_FIELD( 35, BYTES , REPEATED, STATIC, AllTypes, rep_bytes, rep_string, 0), + PB_FIELD( 36, MESSAGE , REPEATED, STATIC, AllTypes, rep_submsg, rep_bytes, &SubMessage_fields), + PB_FIELD( 37, ENUM , REPEATED, STATIC, AllTypes, rep_enum, rep_submsg, 0), + PB_FIELD( 41, INT32 , OPTIONAL, STATIC, AllTypes, opt_int32, rep_enum, &AllTypes_opt_int32_default), + PB_FIELD( 42, INT64 , OPTIONAL, STATIC, AllTypes, opt_int64, opt_int32, &AllTypes_opt_int64_default), + PB_FIELD( 43, UINT32 , OPTIONAL, STATIC, AllTypes, opt_uint32, opt_int64, &AllTypes_opt_uint32_default), + PB_FIELD( 44, UINT64 , OPTIONAL, STATIC, AllTypes, opt_uint64, opt_uint32, &AllTypes_opt_uint64_default), + PB_FIELD( 45, SINT32 , OPTIONAL, STATIC, AllTypes, opt_sint32, opt_uint64, &AllTypes_opt_sint32_default), + PB_FIELD( 46, SINT64 , OPTIONAL, STATIC, AllTypes, opt_sint64, opt_sint32, &AllTypes_opt_sint64_default), + PB_FIELD( 47, BOOL , OPTIONAL, STATIC, AllTypes, opt_bool, opt_sint64, &AllTypes_opt_bool_default), + PB_FIELD( 48, FIXED32 , OPTIONAL, STATIC, AllTypes, opt_fixed32, opt_bool, &AllTypes_opt_fixed32_default), + PB_FIELD( 49, SFIXED32, OPTIONAL, STATIC, AllTypes, opt_sfixed32, opt_fixed32, &AllTypes_opt_sfixed32_default), + PB_FIELD( 50, FLOAT , OPTIONAL, STATIC, AllTypes, opt_float, opt_sfixed32, &AllTypes_opt_float_default), + PB_FIELD( 51, FIXED64 , OPTIONAL, STATIC, AllTypes, opt_fixed64, opt_float, &AllTypes_opt_fixed64_default), + PB_FIELD( 52, SFIXED64, OPTIONAL, STATIC, AllTypes, opt_sfixed64, opt_fixed64, &AllTypes_opt_sfixed64_default), + PB_FIELD( 53, DOUBLE , OPTIONAL, STATIC, AllTypes, opt_double, opt_sfixed64, &AllTypes_opt_double_default), + PB_FIELD( 54, STRING , OPTIONAL, STATIC, AllTypes, opt_string, opt_double, &AllTypes_opt_string_default), + PB_FIELD( 55, BYTES , OPTIONAL, STATIC, AllTypes, opt_bytes, opt_string, &AllTypes_opt_bytes_default), + PB_FIELD( 56, MESSAGE , OPTIONAL, STATIC, AllTypes, opt_submsg, opt_bytes, &SubMessage_fields), + PB_FIELD( 57, ENUM , OPTIONAL, STATIC, AllTypes, opt_enum, opt_submsg, &AllTypes_opt_enum_default), + PB_FIELD( 99, INT32 , REQUIRED, STATIC, AllTypes, end, opt_enum, 0), PB_LAST_FIELD }; diff --git a/tests/bc_alltypes.pb.h b/tests/bc_alltypes.pb.h index fe0b8f76..037b3478 100644 --- a/tests/bc_alltypes.pb.h +++ b/tests/bc_alltypes.pb.h @@ -1,14 +1,19 @@ /* Automatically generated nanopb header */ -/* This is a file generated using nanopb-0.1.1. +/* This is a file generated using nanopb-0.2.0-dev. * It is used as a part of test suite in order to detect any * incompatible changes made to the generator in future versions. */ -#ifndef _PB_BC_ALLTYPES_PB_H_ -#define _PB_BC_ALLTYPES_PB_H_ + +#ifndef _PB_ALLTYPES_PB_H_ +#define _PB_ALLTYPES_PB_H_ #include +#ifdef __cplusplus +extern "C" { +#endif + /* Enum definitions */ -typedef enum { +typedef enum _MyEnum { MyEnum_Zero = 0, MyEnum_First = 1, MyEnum_Second = 2, @@ -16,7 +21,7 @@ typedef enum { } MyEnum; /* Struct definitions */ -typedef struct { +typedef struct _SubMessage { char substuff1[16]; int32_t substuff2; bool has_substuff3; @@ -38,7 +43,7 @@ typedef struct { uint8_t bytes[16]; } AllTypes_opt_bytes_t; -typedef struct { +typedef struct _AllTypes { int32_t req_int32; int64_t req_int64; uint32_t req_uint32; @@ -128,7 +133,7 @@ typedef struct { } AllTypes; /* Default values for struct fields */ -extern const char SubMessage_substuff1_default[17]; +extern const char SubMessage_substuff1_default[16]; extern const int32_t SubMessage_substuff2_default; extern const uint32_t SubMessage_substuff3_default; extern const int32_t AllTypes_opt_int32_default; @@ -144,7 +149,7 @@ extern const float AllTypes_opt_float_default; extern const uint64_t AllTypes_opt_fixed64_default; extern const int64_t AllTypes_opt_sfixed64_default; extern const double AllTypes_opt_double_default; -extern const char AllTypes_opt_string_default[17]; +extern const char AllTypes_opt_string_default[16]; extern const AllTypes_opt_bytes_t AllTypes_opt_bytes_default; extern const MyEnum AllTypes_opt_enum_default; @@ -152,4 +157,22 @@ extern const MyEnum AllTypes_opt_enum_default; extern const pb_field_t SubMessage_fields[4]; extern const pb_field_t AllTypes_fields[53]; +/* Check that field information fits in pb_field_t */ +#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) +STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 256 && pb_membersize(AllTypes, rep_submsg[0]) < 256 && pb_membersize(AllTypes, opt_submsg) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_SubMessage_AllTypes) +#endif + +#if !defined(PB_FIELD_32BIT) +STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 65536 && pb_membersize(AllTypes, rep_submsg[0]) < 65536 && pb_membersize(AllTypes, opt_submsg) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_SubMessage_AllTypes) +#endif + +/* On some platforms (such as AVR), double is really float. + * These are not directly supported by nanopb, but see example_avr_double. + */ +STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES) + +#ifdef __cplusplus +} /* extern "C" */ +#endif + #endif -- cgit 1.2.3-korg From 41f98343c8e4c94180b514902219044510a234b1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 20 Feb 2013 22:55:59 +0200 Subject: Separate PB_HTYPE to PB_ATYPE and PB_HTYPE. Also clean up the logic so that it is easier to implement more allocation types in the future. Update issue 53 Status: FixedInGit --- pb.h | 41 +++++++--------- pb_decode.c | 160 +++++++++++++++++++++++++++++++++++------------------------- pb_encode.c | 97 +++++++++++++++++++++++------------- 3 files changed, 176 insertions(+), 122 deletions(-) diff --git a/pb.h b/pb.h index cd3f465e..dd06c69b 100644 --- a/pb.h +++ b/pb.h @@ -84,29 +84,24 @@ typedef uint8_t pb_type_t; #define PB_LTYPES_COUNT 7 #define PB_LTYPE_MASK 0x0F -/****************** - * Modifier flags * - ******************/ +/************************** + * Field repetition rules * + **************************/ -/* Just the basic, write data at data_offset */ #define PB_HTYPE_REQUIRED 0x00 - -/* Write true at size_offset */ #define PB_HTYPE_OPTIONAL 0x10 - -/* Read to pre-allocated array - * Maximum number of entries is array_size, - * actual number is stored at size_offset */ #define PB_HTYPE_REPEATED 0x20 +#define PB_HTYPE_MASK 0x30 -/* Works for all required/optional/repeated fields. - * data_offset points to pb_callback_t structure. - * LTYPE should be valid or 0 (it is ignored, but - * sometimes used to speculatively index an array). */ -#define PB_HTYPE_CALLBACK 0x30 - -#define PB_HTYPE_MASK 0xF0 +/******************** + * Allocation types * + ********************/ + +#define PB_ATYPE_STATIC 0x00 +#define PB_ATYPE_CALLBACK 0x40 +#define PB_ATYPE_MASK 0xC0 +#define PB_ATYPE(x) ((x) & PB_ATYPE_MASK) #define PB_HTYPE(x) ((x) & PB_HTYPE_MASK) #define PB_LTYPE(x) ((x) & PB_LTYPE_MASK) @@ -214,19 +209,19 @@ typedef enum { * submessages and default values. */ #define PB_REQUIRED_STATIC(tag, st, m, pm, ltype, ptr) \ - {tag, PB_HTYPE_REQUIRED | ltype, \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REQUIRED | ltype, \ pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr} /* Optional fields add the delta to the has_ variable. */ #define PB_OPTIONAL_STATIC(tag, st, m, pm, ltype, ptr) \ - {tag, PB_HTYPE_OPTIONAL | ltype, \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \ pb_delta_end(st, m, pm), \ pb_delta(st, has_ ## m, m), \ pb_membersize(st, m), 0, ptr} /* Repeated fields have a _count field and also the maximum number of entries. */ #define PB_REPEATED_STATIC(tag, st, m, pm, ltype, ptr) \ - {tag, PB_HTYPE_REPEATED | ltype, \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REPEATED | ltype, \ pb_delta_end(st, m, pm), \ pb_delta(st, m ## _count, m), \ pb_membersize(st, m[0]), \ @@ -234,15 +229,15 @@ typedef enum { /* Callbacks are much like required fields except with special datatype. */ #define PB_REQUIRED_CALLBACK(tag, st, m, pm, ltype, ptr) \ - {tag, PB_HTYPE_CALLBACK | ltype, \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REQUIRED | ltype, \ pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr} #define PB_OPTIONAL_CALLBACK(tag, st, m, pm, ltype, ptr) \ - {tag, PB_HTYPE_CALLBACK | ltype, \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr} #define PB_REPEATED_CALLBACK(tag, st, m, pm, ltype, ptr) \ - {tag, PB_HTYPE_CALLBACK | ltype, \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REPEATED | ltype, \ pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr} /* The mapping from protobuf types to LTYPEs is done using these macros. */ diff --git a/pb_decode.c b/pb_decode.c index 351c8ec7..bc6df7b9 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -303,8 +303,11 @@ static bool pb_field_next(pb_field_iterator_t *iter) bool notwrapped = true; size_t prev_size = iter->current->data_size; - if (PB_HTYPE(iter->current->type) == PB_HTYPE_REPEATED) + if (PB_ATYPE(iter->current->type) == PB_ATYPE_STATIC && + PB_HTYPE(iter->current->type) == PB_HTYPE_REPEATED) + { prev_size *= iter->current->array_size; + } if (PB_HTYPE(iter->current->type) == PB_HTYPE_REQUIRED) iter->required_field_index++; @@ -343,11 +346,15 @@ static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag) * Decode a single field * *************************/ -static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) +static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) { - pb_decoder_t func = PB_DECODERS[PB_LTYPE(iter->current->type)]; + pb_type_t type; + pb_decoder_t func; - switch (PB_HTYPE(iter->current->type)) + type = iter->current->type; + func = PB_DECODERS[PB_LTYPE(type)]; + + switch (PB_HTYPE(type)) { case PB_HTYPE_REQUIRED: return func(stream, iter->current, iter->pData); @@ -358,7 +365,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t case PB_HTYPE_REPEATED: if (wire_type == PB_WT_STRING - && PB_LTYPE(iter->current->type) <= PB_LTYPE_LAST_PACKABLE) + && PB_LTYPE(type) <= PB_LTYPE_LAST_PACKABLE) { /* Packed array */ bool status = true; @@ -395,48 +402,63 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t (*size)++; return func(stream, iter->current, pItem); } + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } +} + +static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) +{ + pb_callback_t *pCallback = (pb_callback_t*)iter->pData; + + if (pCallback->funcs.decode == NULL) + return pb_skip_field(stream, wire_type); + + if (wire_type == PB_WT_STRING) + { + pb_istream_t substream; - case PB_HTYPE_CALLBACK: + if (!pb_make_string_substream(stream, &substream)) + return false; + + while (substream.bytes_left) { - pb_callback_t *pCallback = (pb_callback_t*)iter->pData; - - if (pCallback->funcs.decode == NULL) - return pb_skip_field(stream, wire_type); - - if (wire_type == PB_WT_STRING) - { - pb_istream_t substream; - - if (!pb_make_string_substream(stream, &substream)) - return false; - - while (substream.bytes_left) - { - if (!pCallback->funcs.decode(&substream, iter->current, pCallback->arg)) - PB_RETURN_ERROR(stream, "callback failed"); - } - - pb_close_string_substream(stream, &substream); - return true; - } - else - { - /* Copy the single scalar value to stack. - * This is required so that we can limit the stream length, - * which in turn allows to use same callback for packed and - * not-packed fields. */ - pb_istream_t substream; - uint8_t buffer[10]; - size_t size = sizeof(buffer); - - if (!read_raw_value(stream, wire_type, buffer, &size)) - return false; - substream = pb_istream_from_buffer(buffer, size); - - return pCallback->funcs.decode(&substream, iter->current, pCallback->arg); - } + if (!pCallback->funcs.decode(&substream, iter->current, pCallback->arg)) + PB_RETURN_ERROR(stream, "callback failed"); } + pb_close_string_substream(stream, &substream); + return true; + } + else + { + /* Copy the single scalar value to stack. + * This is required so that we can limit the stream length, + * which in turn allows to use same callback for packed and + * not-packed fields. */ + pb_istream_t substream; + uint8_t buffer[10]; + size_t size = sizeof(buffer); + + if (!read_raw_value(stream, wire_type, buffer, &size)) + return false; + substream = pb_istream_from_buffer(buffer, size); + + return pCallback->funcs.decode(&substream, iter->current, pCallback->arg); + } +} + +static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) +{ + switch (PB_ATYPE(iter->current->type)) + { + case PB_ATYPE_STATIC: + return decode_static_field(stream, wire_type, iter); + + case PB_ATYPE_CALLBACK: + return decode_callback_field(stream, wire_type, iter); + default: PB_RETURN_ERROR(stream, "invalid field type"); } @@ -451,37 +473,43 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str /* Initialize size/has fields and apply default values */ do { + pb_type_t type; + type = iter.current->type; + if (iter.current->tag == 0) continue; - /* Initialize the size field for optional/repeated fields to 0. */ - if (PB_HTYPE(iter.current->type) == PB_HTYPE_OPTIONAL) - { - *(bool*)iter.pSize = false; - } - else if (PB_HTYPE(iter.current->type) == PB_HTYPE_REPEATED) + if (PB_ATYPE(type) == PB_ATYPE_STATIC) { - *(size_t*)iter.pSize = 0; - continue; /* Array is empty, no need to initialize contents */ + /* Initialize the size field for optional/repeated fields to 0. */ + if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL) + { + *(bool*)iter.pSize = false; + } + else if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + { + *(size_t*)iter.pSize = 0; + continue; /* Array is empty, no need to initialize contents */ + } + + /* Initialize field contents to default value */ + if (PB_LTYPE(iter.current->type) == PB_LTYPE_SUBMESSAGE) + { + pb_message_set_to_defaults((const pb_field_t *) iter.current->ptr, iter.pData); + } + else if (iter.current->ptr != NULL) + { + memcpy(iter.pData, iter.current->ptr, iter.current->data_size); + } + else + { + memset(iter.pData, 0, iter.current->data_size); + } } - - /* Initialize field contents to default value */ - if (PB_HTYPE(iter.current->type) == PB_HTYPE_CALLBACK) + else if (PB_ATYPE(type) == PB_ATYPE_CALLBACK) { continue; /* Don't overwrite callback */ } - else if (PB_LTYPE(iter.current->type) == PB_LTYPE_SUBMESSAGE) - { - pb_message_set_to_defaults((const pb_field_t *) iter.current->ptr, iter.pData); - } - else if (iter.current->ptr != NULL) - { - memcpy(iter.pData, iter.current->ptr, iter.current->data_size); - } - else - { - memset(iter.pData, 0, iter.current->data_size); - } } while (pb_field_next(&iter)); } diff --git a/pb_encode.c b/pb_encode.c index 85d65782..fbeeacfc 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -153,58 +153,89 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie return true; } +bool checkreturn encode_static_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData) +{ + pb_encoder_t func; + const void *pSize; + + func = PB_ENCODERS[PB_LTYPE(field->type)]; + pSize = (const char*)pData + field->size_offset; + + switch (PB_HTYPE(field->type)) + { + case PB_HTYPE_REQUIRED: + if (!pb_encode_tag_for_field(stream, field)) + return false; + if (!func(stream, field, pData)) + return false; + break; + + case PB_HTYPE_OPTIONAL: + if (*(const bool*)pSize) + { + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!func(stream, field, pData)) + return false; + } + break; + + case PB_HTYPE_REPEATED: + if (!encode_array(stream, field, pData, *(const size_t*)pSize, func)) + return false; + break; + + default: + return false; + } + + return true; +} + +bool checkreturn encode_callback_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData) +{ + const pb_callback_t *callback = (const pb_callback_t*)pData; + if (callback->funcs.encode != NULL) + { + if (!callback->funcs.encode(stream, field, callback->arg)) + return false; + } + return true; +} + bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) { const pb_field_t *field = fields; const void *pData = src_struct; - const void *pSize; size_t prev_size = 0; while (field->tag != 0) { - pb_encoder_t func = PB_ENCODERS[PB_LTYPE(field->type)]; pData = (const char*)pData + prev_size + field->data_offset; - pSize = (const char*)pData + field->size_offset; - prev_size = field->data_size; - if (PB_HTYPE(field->type) == PB_HTYPE_REPEATED) + + /* Special case for static arrays */ + if (PB_ATYPE(field->type) == PB_ATYPE_STATIC && + PB_HTYPE(field->type) == PB_HTYPE_REPEATED) + { prev_size *= field->array_size; + } - switch (PB_HTYPE(field->type)) + switch (PB_ATYPE(field->type)) { - case PB_HTYPE_REQUIRED: - if (!pb_encode_tag_for_field(stream, field)) + case PB_ATYPE_STATIC: + if (!encode_static_field(stream, field, pData)) return false; - if (!func(stream, field, pData)) - return false; - break; - - case PB_HTYPE_OPTIONAL: - if (*(const bool*)pSize) - { - if (!pb_encode_tag_for_field(stream, field)) - return false; - - if (!func(stream, field, pData)) - return false; - } break; - case PB_HTYPE_REPEATED: - if (!encode_array(stream, field, pData, *(const size_t*)pSize, func)) + case PB_ATYPE_CALLBACK: + if (!encode_callback_field(stream, field, pData)) return false; break; - case PB_HTYPE_CALLBACK: - { - const pb_callback_t *callback = (const pb_callback_t*)pData; - if (callback->funcs.encode != NULL) - { - if (!callback->funcs.encode(stream, field, callback->arg)) - return false; - } - break; - } + default: + return false; } field++; -- cgit 1.2.3-korg From 80a2d33fa9a835564b44fe97ba34e713901cf4fb Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 21 Feb 2013 19:35:20 +0200 Subject: Move STATIC_ASSERTs to .pb.c file. This way the .pb.h will remain cleaner and easier to read. --- generator/nanopb_generator.py | 46 ++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 9ecc8f3e..45b2ac68 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -478,6 +478,28 @@ def generate_header(dependencies, headername, enums, messages): for msg in messages: yield msg.fields_declaration() + '\n' + yield '\n#ifdef __cplusplus\n' + yield '} /* extern "C" */\n' + yield '#endif\n' + + # End of header + yield '\n#endif\n' + +def generate_source(headername, enums, messages): + '''Generate content for a source file.''' + + yield '/* Automatically generated nanopb constant definitions */\n' + yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) + yield '#include "%s"\n\n' % headername + + for msg in messages: + yield msg.default_decl(False) + + yield '\n\n' + + for msg in messages: + yield msg.fields_definition() + '\n\n' + if messages: count_required_fields = lambda m: len([f for f in msg.fields if f.rules == 'REQUIRED']) largest_msg = max(messages, key = count_required_fields) @@ -538,31 +560,11 @@ def generate_header(dependencies, headername, enums, messages): yield '\n' yield '/* On some platforms (such as AVR), double is really float.\n' yield ' * These are not directly supported by nanopb, but see example_avr_double.\n' + yield ' * To get rid of this error, remove any double fields from your .proto.\n' yield ' */\n' yield 'STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n' - yield '\n#ifdef __cplusplus\n' - yield '} /* extern "C" */\n' - yield '#endif\n' - - # End of header - yield '\n#endif\n' - -def generate_source(headername, enums, messages): - '''Generate content for a source file.''' - - yield '/* Automatically generated nanopb constant definitions */\n' - yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) - yield '#include "%s"\n\n' % headername - - for msg in messages: - yield msg.default_decl(False) - - yield '\n\n' - - for msg in messages: - yield msg.fields_definition() + '\n\n' - + yield '\n' # --------------------------------------------------------------------------- -- cgit 1.2.3-korg From 57e81ca73b0e0f54e29ca29ec13af551a5335a9b Mon Sep 17 00:00:00 2001 From: Pavel Ilin Date: Thu, 31 Jan 2013 19:01:09 +0100 Subject: Added CMake file for use in projects linking against nanopb. --- cmake/FindNanopb.cmake | 224 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 cmake/FindNanopb.cmake diff --git a/cmake/FindNanopb.cmake b/cmake/FindNanopb.cmake new file mode 100644 index 00000000..9678fb18 --- /dev/null +++ b/cmake/FindNanopb.cmake @@ -0,0 +1,224 @@ +# Locate and configure the nanopb library. +# +# The following varialbes have to be set: +# +# NANOPB_SRC_ROOT_FOLDER - Path to nanopb source folder +# +# The following variables can be set and are optional: +# +# +# PROTOBUF_SRC_ROOT_FOLDER - When compiling with MSVC, if this cache variable is set +# the protobuf-default VS project build locations +# (vsprojects/Debug & vsprojects/Release) will be searched +# for libraries and binaries. +# +# NANOPB_IMPORT_DIRS - List of additional directories to be searched for +# imported .proto files. +# +# NANOPB_GENERATE_CPP_APPEND_PATH - By default -I will be passed to protoc +# for each directory where a proto file is referenced. +# Set to FALSE if you want to disable this behaviour. +# +# Defines the following variables: +# +# NANOPB_FOUND - Found the nanopb library (source&header files, generator tool, protoc compiler tool) +# NANOPB_INCLUDE_DIRS - Include directories for Google Protocol Buffers +# +# The following cache variables are also available to set or use: +# NANOPB_GENERATOR_EXECUTABLE - The nanopb generator +# PROTOBUF_PROTOC_EXECUTABLE - The protoc compiler +# +# ==================================================================== +# +# NANOPB_GENERATE_CPP (public function) +# SRCS = Variable to define with autogenerated +# source files +# HDRS = Variable to define with autogenerated +# header files +# ARGN = proto files +# +# ==================================================================== +# Example: +# +# set(NANOPB_SRC_ROOT_FOLDER "/path/to/nanopb") +# set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${NANOPB_SRC_ROOT_FOLDER}/cmake) +# find_package( Nanopb REQUIRED ) +# include_directories(${NANOPB_INCLUDE_DIRS}) +# +# NANOPB_GENERATE_CPP(PROTO_SRCS PROTO_HDRS foo.proto) +# +# include_directories(${CMAKE_CURRENT_BINARY_DIR}) +# add_executable(bar bar.cc ${PROTO_SRCS} ${PROTO_HDRS}) +# +# ==================================================================== + +#============================================================================= +# Copyright 2009 Kitware, Inc. +# Copyright 2009-2011 Philip Lowman +# Copyright 2008 Esben Mose Hansen, Ange Optimization ApS +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the names of Kitware, Inc., the Insight Software Consortium, +# nor the names of their contributors may be used to endorse or promote +# products derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#============================================================================= +# +# Changes +# 2013.01.31 - Pavlo Ilin - used Modules/FindProtobuf.cmake from cmake 2.8.10 to +# write FindNanopb.cmake +# +#============================================================================= + + +function(NANOPB_GENERATE_CPP SRCS HDRS) + if(NOT ARGN) + return() + endif() + + if(NANOPB_GENERATE_CPP_APPEND_PATH) + # Create an include path for each file specified + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(ABS_PATH ${ABS_FIL} PATH) + + list(FIND _nanobp_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _nanobp_include_path -I ${ABS_PATH}) + endif() + endforeach() + else() + set(_nanobp_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + if(DEFINED NANOPB_IMPORT_DIRS) + foreach(DIR ${NANOPB_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND _nanobp_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _nanobp_include_path -I ${ABS_PATH}) + endif() + endforeach() + endif() + + set(${SRCS}) + set(${HDRS}) + get_filename_component(GENERATOR_PATH ${NANOPB_GENERATOR_EXECUTABLE} PATH) + + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c") + list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") + + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb" + COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + ARGS -I${GENERATOR_PATH} -I${CMAKE_CURRENT_BINARY_DIR} ${_nanobp_include_path} -o${FIL_WE}.pb ${ABS_FIL} + DEPENDS ${ABS_FIL} + COMMENT "Running C++ protocol buffer compiler on ${FIL}" + VERBATIM ) + + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" + COMMAND python + ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb + DEPENDS ${FIL_WE}.pb + COMMENT "Running nanopb generator on ${FIL_WE}.pb" + VERBATIM ) + endforeach() + + set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) + set(${SRCS} ${${SRCS}} ${NANOPB_SRCS} PARENT_SCOPE) + set(${HDRS} ${${HDRS}} ${NANOPB_HDRS} PARENT_SCOPE) + +endfunction() + + + +# +# Main. +# + +# By default have NANOPB_GENERATE_CPP macro pass -I to protoc +# for each directory where a proto file is referenced. +if(NOT DEFINED NANOPB_GENERATE_CPP_APPEND_PATH) + set(NANOPB_GENERATE_CPP_APPEND_PATH TRUE) +endif() + +# Find the include directory +find_path(NANOPB_INCLUDE_DIRS + pb.h + PATHS ${NANOPB_SRC_ROOT_FOLDER} +) +mark_as_advanced(NANOPB_INCLUDE_DIRS) + +# Find nanopb source files +set(NANOPB_SRCS) +set(NANOPB_HDRS) +list(APPEND _nanopb_srcs pb_decode.c pb_encode.c) +list(APPEND _nanopb_hdrs pb_decode.h pb_encode.h pb.h) + +foreach(FIL ${_nanopb_srcs}) + find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_SRC_ROOT_FOLDER} ${NANOPB_INCLUDE_DIRS}) + list(APPEND NANOPB_SRCS "${${FIL}__nano_pb_file}") + mark_as_advanced(${FIL}__nano_pb_file) +endforeach() + +foreach(FIL ${_nanopb_hdrs}) + find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_INCLUDE_DIRS}) + mark_as_advanced(${FIL}__nano_pb_file) + list(APPEND NANOPB_HDRS "${${FIL}__nano_pb_file}") +endforeach() + +# Find the protoc Executable +find_program(PROTOBUF_PROTOC_EXECUTABLE + NAMES protoc + DOC "The Google Protocol Buffers Compiler" + PATHS + ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Release + ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Debug +) +mark_as_advanced(PROTOBUF_PROTOC_EXECUTABLE) + +# Find nanopb generator +find_file(NANOPB_GENERATOR_EXECUTABLE + NAMES nanopb_generator.py + DOC "nanopb generator" + PATHS + ${NANOPB_SRC_ROOT_FOLDER}/generator +) +mark_as_advanced(NANOPB_GENERATOR_EXECUTABLE) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(NANOPB DEFAULT_MSG + NANOPB_INCLUDE_DIRS + NANOPB_SRCS NANOPB_HDRS + NANOPB_GENERATOR_EXECUTABLE + PROTOBUF_PROTOC_EXECUTABLE + ) -- cgit 1.2.3-korg From 0e3053894ff200288fc16056a50ac37b62785471 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 28 Feb 2013 16:42:34 +0200 Subject: Add --extension option to generator. Patch courtesy of Michael Haberler. --- generator/nanopb_generator.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 45b2ac68..c51798de 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -439,7 +439,7 @@ def make_identifier(headername): result += '_' return result -def generate_header(dependencies, headername, enums, messages): +def generate_header(dependencies, headername, enums, messages, options): '''Generate content for a header file. Generates strings, which should be concatenated and stored to file. ''' @@ -454,7 +454,7 @@ def generate_header(dependencies, headername, enums, messages): for dependency in dependencies: noext = os.path.splitext(dependency)[0] - yield '#include "%s.pb.h"\n' % noext + yield '#include "%s.%s.h"\n' % (noext,options.extension) yield '#ifdef __cplusplus\n' yield 'extern "C" {\n' @@ -582,6 +582,8 @@ optparser = OptionParser( "Output will be written to file.pb.h and file.pb.c.") optparser.add_option("-x", dest="exclude", metavar="FILE", action="append", default=[], help="Exclude file from generated #include list.") +optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", default="pb", + help="use extension instead of 'pb' for generated files.") optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Don't print anything except errors.") optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, @@ -638,8 +640,8 @@ def process(filenames, options): enums, messages = parse_file(fdesc.file[0], file_options) noext = os.path.splitext(filename)[0] - headername = noext + '.pb.h' - sourcename = noext + '.pb.c' + headername = noext + '.' + options.extension + '.h' + sourcename = noext + '.' + options.extension + '.c' headerbasename = os.path.basename(headername) if not options.quiet: @@ -651,7 +653,7 @@ def process(filenames, options): dependencies = [d for d in fdesc.file[0].dependency if d not in excludes] header = open(headername, 'w') - for part in generate_header(dependencies, headerbasename, enums, messages): + for part in generate_header(dependencies, headerbasename, enums, messages, options): header.write(part) source = open(sourcename, 'w') -- cgit 1.2.3-korg From f8a143fdfeea69a106e1bc7a774c5f96eba3da7c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 2 Mar 2013 16:27:31 +0200 Subject: Update documentation --- docs/concepts.rst | 16 ++++------------ docs/index.rst | 18 +++++++----------- docs/reference.rst | 36 ++++++++++++++++++++++++------------ 3 files changed, 35 insertions(+), 35 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index b18c5057..052edcc1 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -255,16 +255,8 @@ For example this submessage in the Person.proto file:: generates this field description array for the structure *Person_PhoneNumber*:: const pb_field_t Person_PhoneNumber_fields[3] = { - {1, PB_HTYPE_REQUIRED | PB_LTYPE_STRING, - offsetof(Person_PhoneNumber, number), 0, - pb_membersize(Person_PhoneNumber, number), 0, 0}, - - {2, PB_HTYPE_OPTIONAL | PB_LTYPE_VARINT, - pb_delta(Person_PhoneNumber, type, number), - pb_delta(Person_PhoneNumber, has_type, type), - pb_membersize(Person_PhoneNumber, type), 0, - &Person_PhoneNumber_type_default}, - + PB_FIELD( 1, STRING , REQUIRED, STATIC, Person_PhoneNumber, number, number, 0), + PB_FIELD( 2, ENUM , OPTIONAL, STATIC, Person_PhoneNumber, type, number, &Person_PhoneNumber_type_default), PB_LAST_FIELD }; @@ -276,8 +268,8 @@ Most functions in nanopb return bool: *true* means success, *false* means failur The error messages help in guessing what is the underlying cause of the error. The most common error conditions are: -1) Running out of memory. Because everything is allocated from the stack, nanopb can't detect this itself. Encoding or decoding the same type of a message always takes the same amount of stack space. Therefore, if it works once, it works always. -2) Invalid field description. These are usually stored as constants, so if it works under the debugger, it always does. +1) Running out of memory, i.e. stack overflow. +2) Invalid field descriptors (would usually mean a bug in the generator). 3) IO errors in your own stream callbacks. 4) Errors that happen in your callback functions. 5) Exceeding the max_size or bytes_left of a stream. diff --git a/docs/index.rst b/docs/index.rst index 31d781ea..897f5529 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -36,23 +36,26 @@ Features and limitations **Features** #) Pure C runtime -#) Small code size (2–10 kB depending on processor) -#) Small ram usage (typically 200 bytes) +#) Small code size (2–10 kB depending on processor, plus any message definitions) +#) Small ram usage (typically ~300 bytes, plus any message structs) #) Allows specifying maximum size for strings and arrays, so that they can be allocated statically. #) No malloc needed: everything can be allocated statically or on the stack. #) You can use either encoder or decoder alone to cut the code size in half. +#) Support for most protobuf features, including: all data types, nested submessages, default values, repeated and optional fields, packed arrays. +#) Callback mechanism for handling messages larger than can fit in available RAM. +#) Extensive set of tests. **Limitations** #) User must provide callbacks when decoding arrays or strings without maximum size. Malloc support could be added as a separate module. -#) Some speed has been sacrificed for code size. For example varint calculations are always done in 64 bits. +#) Some speed has been sacrificed for code size. #) Encoding is focused on writing to streams. For memory buffers only it could be made more efficient. #) The deprecated Protocol Buffers feature called "groups" is not supported. #) Fields in the generated structs are ordered by the tag number, instead of the natural ordering in .proto file. #) Unknown fields are not preserved when decoding and re-encoding a message. #) Reflection (runtime introspection) is not supported. E.g. you can't request a field by giving its name in a string. #) Numeric arrays are always encoded as packed, even if not marked as packed in .proto. This causes incompatibility with decoders that do not support packed format. -#) Cyclic references between messages are not supported. They could be supported in callback-mode if there was an option in the generator to set the mode. +#) Cyclic references between messages are supported only in callback mode. Getting started =============== @@ -104,10 +107,3 @@ Debugging and testing ===================== Extensive unittests are included under the *tests* folder. Just type *make* there to run the tests. -This also generates a file called *breakpoints* which includes all lines returning *false* in nanopb. You can use this in gdb by typing *source breakpoints*, after which gdb will break on first nanopb error. - -Wishlist -======== -#) A specialized encoder for encoding to a memory buffer. Should serialize in reverse order to avoid having to determine submessage size beforehand. -#) A cleaner rewrite of the Python-based source generator. -#) Better performance for 16- and 8-bit platforms: use smaller datatypes where possible. diff --git a/docs/reference.rst b/docs/reference.rst index fee1b702..93aae8b0 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -37,22 +37,23 @@ pb_type_t --------- Defines the encoder/decoder behaviour that should be used for a field. :: - typedef enum { ... } pb_type_t; + typedef uint8_t pb_type_t; -The low-order byte of the enumeration values defines the function that can be used for encoding and decoding the field data: +The low-order nibble of the enumeration values defines the function that can be used for encoding and decoding the field data: ==================== ===== ================================================ LTYPE identifier Value Storage format ==================== ===== ================================================ PB_LTYPE_VARINT 0x00 Integer. PB_LTYPE_SVARINT 0x01 Integer, zigzag encoded. -PB_LTYPE_FIXED 0x02 Integer or floating point. -PB_LTYPE_BYTES 0x03 Structure with *size_t* field and byte array. -PB_LTYPE_STRING 0x04 Null-terminated string. -PB_LTYPE_SUBMESSAGE 0x05 Submessage structure. +PB_LTYPE_FIXED32 0x02 32-bit integer or floating point. +PB_LTYPE_FIXED64 0x03 64-bit integer or floating point. +PB_LTYPE_BYTES 0x04 Structure with *size_t* field and byte array. +PB_LTYPE_STRING 0x05 Null-terminated string. +PB_LTYPE_SUBMESSAGE 0x06 Submessage structure. ==================== ===== ================================================ -The high-order byte defines whether the field is required, optional, repeated or callback: +The bits 4-5 define whether the field is required, optional or repeated: ==================== ===== ================================================ HTYPE identifier Value Field handling @@ -60,13 +61,24 @@ HTYPE identifier Value Field handling PB_HTYPE_REQUIRED 0x00 Verify that field exists in decoded message. PB_HTYPE_OPTIONAL 0x10 Use separate *has_* boolean to specify whether the field is present. -PB_HTYPE_ARRAY 0x20 A repeated field with preallocated array. + (Unless it is a callback) +PB_HTYPE_REPEATED 0x20 A repeated field with preallocated array. Separate *_count* for number of items. -PB_HTYPE_CALLBACK 0x30 A field with dynamic storage size, data is - actually a pointer to a structure containing a - callback function. + (Unless it is a callback) ==================== ===== ================================================ +The bits 6-7 define the how the storage for the field is allocated: + +==================== ===== ================================================ +ATYPE identifier Value Allocation method +==================== ===== ================================================ +PB_ATYPE_STATIC 0x00 Statically allocated storage in the structure. +PB_ATYPE_CALLBACK 0x40 A field with dynamic storage size. Struct field + actually contains a pointer to a callback + function. +==================== ===== ================================================ + + pb_field_t ---------- Describes a single structure field with memory position in relation to others. The descriptions are usually autogenerated. :: @@ -83,7 +95,7 @@ Describes a single structure field with memory position in relation to others. T } pb_packed; :tag: Tag number of the field or 0 to terminate a list of fields. -:type: LTYPE and HTYPE of the field. +:type: LTYPE, HTYPE and ATYPE of the field. :data_offset: Offset of field data, relative to the end of the previous field. :size_offset: Offset of *bool* flag for optional fields or *size_t* count for arrays, relative to field data. :data_size: Size of a single data entry, in bytes. For PB_LTYPE_BYTES, the size of the byte array inside the containing structure. For PB_HTYPE_CALLBACK, size of the C data type if known. -- cgit 1.2.3-korg From 86ae2541e6eea3a6ed2ff812a0658ca4fad40557 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 2 Mar 2013 16:32:15 +0200 Subject: Update changelog --- CHANGELOG | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 655acc89..9245d59b 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,16 @@ +nanopb-0.2.0 + NOTE: This release requires you to regenerate all .pb.c + files. Files generated by older versions will not + compile anymore. + + Reformat generated .pb.c files using macros (issue 58) + Rename PB_HTYPE_ARRAY -> PB_HTYPE_REPEATED + Separate PB_HTYPE to PB_ATYPE and PB_HTYPE + Move STATIC_ASSERTs to .pb.c file + Added CMake file (by Pavel Ilin) + Add option to give file extension to generator (by Michael Haberler) + Documentation updates + nanopb-0.1.9 Fixed error message bugs (issues 52, 56) Sanitize #ifndef filename (issue 50) -- cgit 1.2.3-korg From c8e0732e6b855d831888a4f7ffeeb47b9ae0b1c0 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 2 Mar 2013 16:32:54 +0200 Subject: Publishing nanopb-0.2.0 --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index c51798de..6a41f460 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "0.2.0-dev" +nanopb_version = "nanopb-0.2.0" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index dd06c69b..ec9ab26c 100644 --- a/pb.h +++ b/pb.h @@ -6,7 +6,7 @@ * see pb_encode.h or pb_decode.h */ -#define NANOPB_VERSION nanopb-0.2.0-dev +#define NANOPB_VERSION nanopb-0.2.0 #include #include -- cgit 1.2.3-korg From 104710b26c1c4c6aaeb71141dcf1e72e5016b046 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 2 Mar 2013 16:35:17 +0200 Subject: Setting version to 0.2.1-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 6a41f460..e38d3822 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.0" +nanopb_version = "nanopb-0.2.1-dev" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index ec9ab26c..b12debe9 100644 --- a/pb.h +++ b/pb.h @@ -6,7 +6,7 @@ * see pb_encode.h or pb_decode.h */ -#define NANOPB_VERSION nanopb-0.2.0 +#define NANOPB_VERSION nanopb-0.2.1-dev #include #include -- cgit 1.2.3-korg From 64bf72d73d2b1a9bc3be1f7bba117a9664de7c1f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 4 Mar 2013 19:27:42 +0200 Subject: Add generator option to configure #include directives. This suits complex projects, where there are multiple interdependent .proto files in various directories. Patch by Michael Haberler. --- generator/nanopb_generator.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index e38d3822..c3b5fbee 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -450,12 +450,14 @@ def generate_header(dependencies, headername, enums, messages, options): symbol = make_identifier(headername) yield '#ifndef _PB_%s_\n' % symbol yield '#define _PB_%s_\n' % symbol - yield '#include \n\n' + yield options.libformat % ('pb.h') + yield '\n' for dependency in dependencies: noext = os.path.splitext(dependency)[0] - yield '#include "%s.%s.h"\n' % (noext,options.extension) - + yield options.genformat % (noext + '.' + options.extension + '.h') + yield '\n' + yield '#ifdef __cplusplus\n' yield 'extern "C" {\n' yield '#endif\n\n' @@ -490,7 +492,8 @@ def generate_source(headername, enums, messages): yield '/* Automatically generated nanopb constant definitions */\n' yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) - yield '#include "%s"\n\n' % headername + yield options.genformat % (headername) + yield '\n' for msg in messages: yield msg.default_decl(False) @@ -583,7 +586,13 @@ optparser = OptionParser( optparser.add_option("-x", dest="exclude", metavar="FILE", action="append", default=[], help="Exclude file from generated #include list.") optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", default="pb", - help="use extension instead of 'pb' for generated files.") + help="Set extension to use instead of 'pb' for generated files. [default: %default]") +optparser.add_option("-Q", "--generated-include-format", dest="genformat", + metavar="FORMAT", default='#include "%s"\n', + help="Set format string to use for including other .pb.h files. [default: %default]") +optparser.add_option("-L", "--library-include-format", dest="libformat", + metavar="FORMAT", default='#include <%s>\n', + help="Set format string to use for including the nanopb pb.h header. [default: %default]") optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Don't print anything except errors.") optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, -- cgit 1.2.3-korg From 5522e02133be7d9dfd2f350f2774dfc0abc15686 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 6 Mar 2013 18:02:57 +0200 Subject: Add a dummy field if struct would otherwise be empty. Update issue 64 Status: FixedInGit --- generator/nanopb_generator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index c3b5fbee..663745a1 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -300,6 +300,12 @@ class Message: def __str__(self): result = 'typedef struct _%s {\n' % self.name + + if not self.ordered_fields: + # Empty structs are not allowed in C standard. + # Therefore add a dummy field if an empty message occurs. + result += ' uint8_t dummy_field;' + result += '\n'.join([str(f) for f in self.ordered_fields]) result += '\n}' -- cgit 1.2.3-korg From d2e3c1ad930efbee0ab4839124522da94f70ada4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Mar 2013 12:35:07 +0200 Subject: Fix bug with decoding empty message types. Add test for the same. Note: the bug only applies to empty message types. Empty messages of non-empty message types are not affected. Update issue 65 Status: FixedInGit --- pb_decode.c | 2 +- tests/alltypes.proto | 7 +++++++ tests/test_decode3.c | 3 +++ tests/test_encode3.c | 2 ++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pb_decode.c b/pb_decode.c index bc6df7b9..e2e89a0d 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -571,7 +571,7 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ } while (pb_field_next(&iter)); /* Fixup if last field was also required. */ - if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED) + if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.current->tag) req_field_count++; /* Check the whole bytes */ diff --git a/tests/alltypes.proto b/tests/alltypes.proto index b9003a79..ce922c13 100644 --- a/tests/alltypes.proto +++ b/tests/alltypes.proto @@ -6,6 +6,10 @@ message SubMessage { optional fixed32 substuff3 = 3 [default = 3]; } +message EmptyMessage { + +} + enum MyEnum { Zero = 0; First = 1; @@ -34,6 +38,7 @@ message AllTypes { required bytes req_bytes = 15 [(nanopb).max_size = 16]; required SubMessage req_submsg = 16; required MyEnum req_enum = 17; + required EmptyMessage req_emptymsg = 18; repeated int32 rep_int32 = 21 [(nanopb).max_count = 5]; @@ -56,6 +61,7 @@ message AllTypes { repeated bytes rep_bytes = 35 [(nanopb).max_size = 16, (nanopb).max_count = 5]; repeated SubMessage rep_submsg = 36 [(nanopb).max_count = 5]; repeated MyEnum rep_enum = 37 [(nanopb).max_count = 5]; + repeated EmptyMessage rep_emptymsg = 38 [(nanopb).max_count = 5]; optional int32 opt_int32 = 41 [default = 4041]; optional int64 opt_int64 = 42 [default = 4042]; @@ -77,6 +83,7 @@ message AllTypes { optional bytes opt_bytes = 55 [(nanopb).max_size = 16, default = "4055"]; optional SubMessage opt_submsg = 56; optional MyEnum opt_enum = 57 [default = Second]; + optional EmptyMessage opt_emptymsg = 58; // Just to make sure that the size of the fields has been calculated // properly, i.e. otherwise a bug in last field might not be detected. diff --git a/tests/test_decode3.c b/tests/test_decode3.c index cd1b154b..55d025c0 100644 --- a/tests/test_decode3.c +++ b/tests/test_decode3.c @@ -76,6 +76,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3); TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); + TEST(alltypes.rep_emptymsg_count == 5); if (mode == 0) { @@ -120,6 +121,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.opt_submsg.substuff3 == 3); TEST(alltypes.has_opt_enum == false); TEST(alltypes.opt_enum == MyEnum_Second); + TEST(alltypes.has_opt_emptymsg == false); } else { @@ -164,6 +166,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.opt_submsg.substuff3 == 3); TEST(alltypes.has_opt_enum == true); TEST(alltypes.opt_enum == MyEnum_Truth); + TEST(alltypes.has_opt_emptymsg == true); } TEST(alltypes.end == 1099); diff --git a/tests/test_encode3.c b/tests/test_encode3.c index 6db37fdc..4f6859ac 100644 --- a/tests/test_encode3.c +++ b/tests/test_encode3.c @@ -64,6 +64,7 @@ int main(int argc, char **argv) alltypes.rep_submsg[4].substuff3 = 2016; alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; + alltypes.rep_emptymsg_count = 5; if (mode != 0) { @@ -107,6 +108,7 @@ int main(int argc, char **argv) alltypes.opt_submsg.substuff2 = 3056; alltypes.has_opt_enum = true; alltypes.opt_enum = MyEnum_Truth; + alltypes.has_opt_emptymsg = true; } alltypes.end = 1099; -- cgit 1.2.3-korg From a9c88f5570f0c124bf977bdc355f079a2e7f4d57 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Mar 2013 12:43:35 +0200 Subject: Fix error in backwards compatibility testcase --- tests/bc_alltypes.pb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/bc_alltypes.pb.c b/tests/bc_alltypes.pb.c index 7d1edf57..b144b1e3 100644 --- a/tests/bc_alltypes.pb.c +++ b/tests/bc_alltypes.pb.c @@ -5,7 +5,7 @@ * incompatible changes made to the generator in future versions. */ -#include "alltypes.pb.h" +#include "bc_alltypes.pb.h" const char SubMessage_substuff1_default[16] = "1"; const int32_t SubMessage_substuff2_default = 2; -- cgit 1.2.3-korg From 96d40168383519b6b82a7eb850a558aeb62917e9 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Mar 2013 12:51:47 +0200 Subject: Improve the fuzztest. Enable -fstack-protector-all to detect any stack smashing bugs. Also use test_decode3 for maximal vulnerable surface. --- tests/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 7385fe35..181628a2 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -15,8 +15,8 @@ CC_VERSION=$(shell $(CC) -v 2>&1) CFLAGS_CORE= ifneq (,$(findstring gcc,$(CC_VERSION))) CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wlogical-op -Wconversion - CFLAGS+=--coverage - LDFLAGS+=--coverage + CFLAGS+=--coverage -fstack-protector-all + LDFLAGS+=--coverage endif ifneq (,$(findstring clang,$(CC_VERSION))) CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wconversion @@ -127,5 +127,5 @@ test_options: options.pb.h options.expected options.pb.o fi \ done -run_fuzztest: test_decode2 - bash -c 'I=1; while true; do cat /dev/urandom | ./test_decode2 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done' +run_fuzztest: test_decode3 + bash -c 'ulimit -c unlimited; I=1; while true; do cat /dev/urandom | ./test_decode3 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done' -- cgit 1.2.3-korg From 60109c0be1c8b21b38f08b50db36577921eaca7f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Mar 2013 13:09:14 +0200 Subject: Add option to run the tests with mudflap to detect pointer errors. --- tests/Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/Makefile b/tests/Makefile index 181628a2..40fcabe0 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -22,6 +22,14 @@ ifneq (,$(findstring clang,$(CC_VERSION))) CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wconversion endif +# Also use mudflap if it is available +# To enable, run with make -B USE_MUDFLAP=y +USE_MUDFLAP ?= n +ifeq ($(USE_MUDFLAP),y) + CFLAGS += -fmudflap + LDFLAGS += -lmudflap -fmudflap +endif + all: breakpoints $(TESTS) run_unittests clean: -- cgit 1.2.3-korg From e1b8a555f30d77a3c0094cf3678666b38f4b4bd3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Mar 2013 13:12:09 +0200 Subject: Fix additional bug with empty message types. pb_field_next() would access past the fields array. --- pb_decode.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pb_decode.c b/pb_decode.c index e2e89a0d..fba8f64c 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -312,6 +312,9 @@ static bool pb_field_next(pb_field_iterator_t *iter) if (PB_HTYPE(iter->current->type) == PB_HTYPE_REQUIRED) iter->required_field_index++; + if (iter->current->tag == 0) + return false; /* Only happens with empty message types */ + iter->current++; iter->field_index++; if (iter->current->tag == 0) -- cgit 1.2.3-korg From 6f8dbc73eb3e595934751f5b33c7f61a902e946a Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Mar 2013 14:21:21 +0200 Subject: Add simple support for separate options file. Update issue 12 Still needs documentation. --- generator/nanopb.proto | 6 +++ generator/nanopb_generator.py | 114 ++++++++++++++++++++++++++++++------------ 2 files changed, 87 insertions(+), 33 deletions(-) diff --git a/generator/nanopb.proto b/generator/nanopb.proto index 40662522..7b73c7b3 100644 --- a/generator/nanopb.proto +++ b/generator/nanopb.proto @@ -16,6 +16,9 @@ enum FieldType { FT_IGNORE = 3; // Ignore the field completely. } +// This is the inner options message, which basically defines options for +// a field. When it is used in message or file scope, it applies to all +// fields. message NanoPBOptions { // Allocated size for 'bytes' and 'string' fields. optional int32 max_size = 1; @@ -33,6 +36,9 @@ message NanoPBOptions { optional bool packed_struct = 5 [default = false]; } +// Extensions to protoc 'Descriptor' type in order to define options +// inside a .proto file. +// // Protocol Buffers extension number registry // -------------------------------- // Project: Nanopb diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 663745a1..0e898749 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -3,6 +3,7 @@ nanopb_version = "nanopb-0.2.1-dev" try: import google.protobuf.descriptor_pb2 as descriptor + import google.protobuf.text_format as text_format except: print print "*************************************************************" @@ -54,9 +55,7 @@ datatypes = { } class Names: - '''Keeps a set of nested names and formats them to C identifier. - You can subclass this with your own implementation. - ''' + '''Keeps a set of nested names and formats them to C identifier.''' def __init__(self, parts = ()): if isinstance(parts, Names): parts = parts.parts @@ -286,7 +285,7 @@ class Message: self.fields = [] for f in desc.field: - field_options = get_nanopb_suboptions(f, message_options) + field_options = get_nanopb_suboptions(f, message_options, self.name + f.name) if field_options.type != nanopb_pb2.FT_IGNORE: self.fields.append(Field(self.name, f, field_options)) @@ -383,14 +382,14 @@ def parse_file(fdesc, file_options): base_name = Names() for enum in fdesc.enum_type: - enum_options = get_nanopb_suboptions(enum, file_options) + enum_options = get_nanopb_suboptions(enum, file_options, base_name + enum.name) enums.append(Enum(base_name, enum, enum_options)) for names, message in iterate_messages(fdesc, base_name): - message_options = get_nanopb_suboptions(message, file_options) + message_options = get_nanopb_suboptions(message, file_options, names) messages.append(Message(names, message, message_options)) for enum in message.enum_type: - enum_options = get_nanopb_suboptions(enum, message_options) + enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name) enums.append(Enum(names, enum, enum_options)) # Fix field default values where enum short names are used. @@ -575,6 +574,67 @@ def generate_source(headername, enums, messages): yield '\n' +# --------------------------------------------------------------------------- +# Options parsing for the .proto files +# --------------------------------------------------------------------------- + +from fnmatch import fnmatch + +def read_options_file(infile): + '''Parse a separate options file to list: + [(namemask, options), ...] + ''' + results = [] + for line in infile: + line = line.strip() + if not line or line.startswith('//') or line.startswith('#'): + continue + + parts = line.split(None, 1) + opts = nanopb_pb2.NanoPBOptions() + text_format.Merge(parts[1], opts) + results.append((parts[0], opts)) + + return results + +class Globals: + '''Ugly global variables, should find a good way to pass these.''' + verbose_options = False + separate_options = [] + +def get_nanopb_suboptions(subdesc, options, name): + '''Get copy of options, and merge information from subdesc.''' + new_options = nanopb_pb2.NanoPBOptions() + new_options.CopyFrom(options) + + # Handle options defined in a separate file + dotname = '.'.join(name.parts) + for namemask, options in Globals.separate_options: + if fnmatch(dotname, namemask): + new_options.MergeFrom(options) + + # Handle options defined in .proto + if isinstance(subdesc.options, descriptor.FieldOptions): + ext_type = nanopb_pb2.nanopb + elif isinstance(subdesc.options, descriptor.FileOptions): + ext_type = nanopb_pb2.nanopb_fileopt + elif isinstance(subdesc.options, descriptor.MessageOptions): + ext_type = nanopb_pb2.nanopb_msgopt + elif isinstance(subdesc.options, descriptor.EnumOptions): + ext_type = nanopb_pb2.nanopb_enumopt + else: + raise Exception("Unknown options type") + + if subdesc.options.HasExtension(ext_type): + ext = subdesc.options.Extensions[ext_type] + new_options.MergeFrom(ext) + + if Globals.verbose_options: + print "Options for " + dotname + ":" + print text_format.MessageToString(new_options) + + return new_options + # --------------------------------------------------------------------------- # Command line interface @@ -583,7 +643,6 @@ def generate_source(headername, enums, messages): import sys import os.path from optparse import OptionParser -import google.protobuf.text_format as text_format optparser = OptionParser( usage = "Usage: nanopb_generator.py [options] file.pb ...", @@ -593,6 +652,8 @@ optparser.add_option("-x", dest="exclude", metavar="FILE", action="append", defa help="Exclude file from generated #include list.") optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", default="pb", help="Set extension to use instead of 'pb' for generated files. [default: %default]") +optparser.add_option("-f", "--options-file", dest="options_file", metavar="FILE", default="%s.options", + help="Set name of a separate generator options file.") optparser.add_option("-Q", "--generated-include-format", dest="genformat", metavar="FORMAT", default='#include "%s"\n', help="Set format string to use for including other .pb.h files. [default: %default]") @@ -606,28 +667,6 @@ optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", def optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[], help="Set generator option (max_size, max_count etc.).") -def get_nanopb_suboptions(subdesc, options): - '''Get copy of options, and merge information from subdesc.''' - new_options = nanopb_pb2.NanoPBOptions() - new_options.CopyFrom(options) - - if isinstance(subdesc.options, descriptor.FieldOptions): - ext_type = nanopb_pb2.nanopb - elif isinstance(subdesc.options, descriptor.FileOptions): - ext_type = nanopb_pb2.nanopb_fileopt - elif isinstance(subdesc.options, descriptor.MessageOptions): - ext_type = nanopb_pb2.nanopb_msgopt - elif isinstance(subdesc.options, descriptor.EnumOptions): - ext_type = nanopb_pb2.nanopb_enumopt - else: - raise Exception("Unknown options type") - - if subdesc.options.HasExtension(ext_type): - ext = subdesc.options.Extensions[ext_type] - new_options.MergeFrom(ext) - - return new_options - def process(filenames, options): '''Process the files given on the command line.''' @@ -637,6 +676,8 @@ def process(filenames, options): if options.quiet: options.verbose = False + + Globals.verbose_options = options.verbose toplevel_options = nanopb_pb2.NanoPBOptions() for s in options.settings: @@ -646,12 +687,19 @@ def process(filenames, options): data = open(filename, 'rb').read() fdesc = descriptor.FileDescriptorSet.FromString(data) - file_options = get_nanopb_suboptions(fdesc.file[0], toplevel_options) + # Check if any separate options are specified + optfilename = options.options_file % os.path.splitext(filename)[0] if options.verbose: - print "Options for " + filename + ":" - print text_format.MessageToString(file_options) + print 'Reading options from ' + optfilename + + if os.path.isfile(optfilename): + Globals.separate_options = read_options_file(open(optfilename, "rU")) + else: + Globals.separate_options = [] + # Parse the file + file_options = get_nanopb_suboptions(fdesc.file[0], toplevel_options, Names([filename])) enums, messages = parse_file(fdesc.file[0], file_options) noext = os.path.splitext(filename)[0] -- cgit 1.2.3-korg From 5f3bf35e01a0eb5d7c31949bcd32265be056ed5c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Mar 2013 14:23:44 +0200 Subject: Switch alltypes.proto to use the new .options file mechanism. --- tests/alltypes.options | 3 +++ tests/alltypes.proto | 46 +++++++++++++++++++++++----------------------- 2 files changed, 26 insertions(+), 23 deletions(-) create mode 100644 tests/alltypes.options diff --git a/tests/alltypes.options b/tests/alltypes.options new file mode 100644 index 00000000..b31e3cf0 --- /dev/null +++ b/tests/alltypes.options @@ -0,0 +1,3 @@ +* max_size:16 +* max_count:5 + diff --git a/tests/alltypes.proto b/tests/alltypes.proto index ce922c13..cc171f3b 100644 --- a/tests/alltypes.proto +++ b/tests/alltypes.proto @@ -1,7 +1,7 @@ import "nanopb.proto"; message SubMessage { - required string substuff1 = 1 [(nanopb).max_size = 16, default = "1"]; + required string substuff1 = 1 [default = "1"]; required int32 substuff2 = 2 [default = 2]; optional fixed32 substuff3 = 3 [default = 3]; } @@ -34,34 +34,34 @@ message AllTypes { required sfixed64 req_sfixed64= 12; required double req_double = 13; - required string req_string = 14 [(nanopb).max_size = 16]; - required bytes req_bytes = 15 [(nanopb).max_size = 16]; + required string req_string = 14; + required bytes req_bytes = 15; required SubMessage req_submsg = 16; required MyEnum req_enum = 17; required EmptyMessage req_emptymsg = 18; - repeated int32 rep_int32 = 21 [(nanopb).max_count = 5]; - repeated int64 rep_int64 = 22 [(nanopb).max_count = 5]; - repeated uint32 rep_uint32 = 23 [(nanopb).max_count = 5]; - repeated uint64 rep_uint64 = 24 [(nanopb).max_count = 5]; - repeated sint32 rep_sint32 = 25 [(nanopb).max_count = 5]; - repeated sint64 rep_sint64 = 26 [(nanopb).max_count = 5]; - repeated bool rep_bool = 27 [(nanopb).max_count = 5]; + repeated int32 rep_int32 = 21; + repeated int64 rep_int64 = 22; + repeated uint32 rep_uint32 = 23; + repeated uint64 rep_uint64 = 24; + repeated sint32 rep_sint32 = 25; + repeated sint64 rep_sint64 = 26; + repeated bool rep_bool = 27; - repeated fixed32 rep_fixed32 = 28 [(nanopb).max_count = 5]; - repeated sfixed32 rep_sfixed32= 29 [(nanopb).max_count = 5]; - repeated float rep_float = 30 [(nanopb).max_count = 5]; + repeated fixed32 rep_fixed32 = 28; + repeated sfixed32 rep_sfixed32= 29; + repeated float rep_float = 30; - repeated fixed64 rep_fixed64 = 31 [(nanopb).max_count = 5]; - repeated sfixed64 rep_sfixed64= 32 [(nanopb).max_count = 5]; - repeated double rep_double = 33 [(nanopb).max_count = 5]; + repeated fixed64 rep_fixed64 = 31; + repeated sfixed64 rep_sfixed64= 32; + repeated double rep_double = 33; - repeated string rep_string = 34 [(nanopb).max_size = 16, (nanopb).max_count = 5]; - repeated bytes rep_bytes = 35 [(nanopb).max_size = 16, (nanopb).max_count = 5]; - repeated SubMessage rep_submsg = 36 [(nanopb).max_count = 5]; - repeated MyEnum rep_enum = 37 [(nanopb).max_count = 5]; - repeated EmptyMessage rep_emptymsg = 38 [(nanopb).max_count = 5]; + repeated string rep_string = 34; + repeated bytes rep_bytes = 35; + repeated SubMessage rep_submsg = 36; + repeated MyEnum rep_enum = 37; + repeated EmptyMessage rep_emptymsg = 38; optional int32 opt_int32 = 41 [default = 4041]; optional int64 opt_int64 = 42 [default = 4042]; @@ -79,8 +79,8 @@ message AllTypes { optional sfixed64 opt_sfixed64= 52 [default = 4052]; optional double opt_double = 53 [default = 4053]; - optional string opt_string = 54 [(nanopb).max_size = 16, default = "4054"]; - optional bytes opt_bytes = 55 [(nanopb).max_size = 16, default = "4055"]; + optional string opt_string = 54 [default = "4054"]; + optional bytes opt_bytes = 55 [default = "4055"]; optional SubMessage opt_submsg = 56; optional MyEnum opt_enum = 57 [default = Second]; optional EmptyMessage opt_emptymsg = 58; -- cgit 1.2.3-korg From 9b6641ac643af4f301e05421f2b228084dcc8693 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Mar 2013 14:45:41 +0200 Subject: alltypes.proto no longer needs to include nanopb.proto --- tests/alltypes.proto | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/alltypes.proto b/tests/alltypes.proto index cc171f3b..6ccf57d7 100644 --- a/tests/alltypes.proto +++ b/tests/alltypes.proto @@ -1,5 +1,3 @@ -import "nanopb.proto"; - message SubMessage { required string substuff1 = 1 [default = "1"]; required int32 substuff2 = 2 [default = 2]; -- cgit 1.2.3-korg From 03526471189711f6656dfa074fc6a8fd7f3d340b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Mar 2013 14:49:15 +0200 Subject: Implement error message support for the encoder side. Update issue 7 Status: FixedInGit --- pb_encode.c | 37 +++++++++++++++++++++++-------------- pb_encode.h | 11 +++++++++++ tests/test_encode1.c | 1 + tests/test_encode2.c | 5 +++++ tests/test_encode3.c | 2 +- 5 files changed, 41 insertions(+), 15 deletions(-) diff --git a/pb_encode.c b/pb_encode.c index fbeeacfc..fd4b6ba5 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -56,6 +56,9 @@ pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize) stream.state = buf; stream.max_size = bufsize; stream.bytes_written = 0; +#ifndef PB_NO_ERRMSG + stream.errmsg = NULL; +#endif return stream; } @@ -64,14 +67,14 @@ bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count if (stream->callback != NULL) { if (stream->bytes_written + count > stream->max_size) - return false; + PB_RETURN_ERROR(stream, "stream full"); #ifdef PB_BUFFER_ONLY if (!buf_write(stream, buf, count)) - return false; + PB_RETURN_ERROR(stream, "io error"); #else if (!stream->callback(stream, buf, count)) - return false; + PB_RETURN_ERROR(stream, "io error"); #endif } @@ -111,7 +114,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie } else { - pb_ostream_t sizestream = {0,0,0,0}; + pb_ostream_t sizestream = PB_OSTREAM_SIZING; p = pData; for (i = 0; i < count; i++) { @@ -187,7 +190,7 @@ bool checkreturn encode_static_field(pb_ostream_t *stream, const pb_field_t *fie break; default: - return false; + PB_RETURN_ERROR(stream, "invalid field type"); } return true; @@ -199,7 +202,7 @@ bool checkreturn encode_callback_field(pb_ostream_t *stream, const pb_field_t *f if (callback->funcs.encode != NULL) { if (!callback->funcs.encode(stream, field, callback->arg)) - return false; + PB_RETURN_ERROR(stream, "callback error"); } return true; } @@ -235,7 +238,7 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons break; default: - return false; + PB_RETURN_ERROR(stream, "invalid field type"); } field++; @@ -340,7 +343,7 @@ bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t break; default: - return false; + PB_RETURN_ERROR(stream, "invalid field type"); } return pb_encode_tag(stream, wiretype, field->tag); @@ -357,7 +360,7 @@ bool checkreturn pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, s bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) { /* First calculate the message size using a non-writing substream. */ - pb_ostream_t substream = {0,0,0,0}; + pb_ostream_t substream = PB_OSTREAM_SIZING; size_t size; bool status; @@ -373,7 +376,7 @@ bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fie return pb_write(stream, NULL, size); /* Just sizing */ if (stream->bytes_written + size > stream->max_size) - return false; + PB_RETURN_ERROR(stream, "stream full"); /* Use a substream to verify that a callback doesn't write more than * what it did the first time. */ @@ -381,14 +384,20 @@ bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fie substream.state = stream->state; substream.max_size = size; substream.bytes_written = 0; +#ifndef PB_NO_ERRMSG + substream.errmsg = NULL; +#endif status = pb_encode(&substream, fields, src_struct); stream->bytes_written += substream.bytes_written; stream->state = substream.state; +#ifndef PB_NO_ERRMSG + stream->errmsg = substream.errmsg; +#endif if (substream.bytes_written != size) - return false; + PB_RETURN_ERROR(stream, "submsg size changed"); return status; } @@ -405,7 +414,7 @@ bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, co case 2: value = *(const uint16_t*)src; break; case 4: value = *(const uint32_t*)src; break; case 8: value = *(const uint64_t*)src; break; - default: return false; + default: PB_RETURN_ERROR(stream, "invalid data_size"); } return pb_encode_varint(stream, value); @@ -419,7 +428,7 @@ bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, c { case 4: value = *(const int32_t*)src; break; case 8: value = *(const int64_t*)src; break; - default: return false; + default: PB_RETURN_ERROR(stream, "invalid data_size"); } return pb_encode_svarint(stream, value); @@ -453,7 +462,7 @@ bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, co bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src) { if (field->ptr == NULL) - return false; + PB_RETURN_ERROR(stream, "invalid field descriptor"); return pb_encode_submessage(stream, (const pb_field_t*)field->ptr, src); } diff --git a/pb_encode.h b/pb_encode.h index 69b88e86..bd6132b6 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -46,11 +46,22 @@ struct _pb_ostream_t void *state; /* Free field for use by callback implementation */ size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ size_t bytes_written; + +#ifndef PB_NO_ERRMSG + const char *errmsg; +#endif }; pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize); bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); +/* Stream type for use in computing message sizes */ +#ifndef PB_NO_ERRMSG +#define PB_OSTREAM_SIZING {0,0,0,0,0} +#else +#define PB_OSTREAM_SIZING {0,0,0,0} +#endif + /* Encode struct to given output stream. * Returns true on success, false on any failure. * The actual struct pointed to by src_struct must match the description in fields. diff --git a/tests/test_encode1.c b/tests/test_encode1.c index 2e978296..742c99f4 100644 --- a/tests/test_encode1.c +++ b/tests/test_encode1.c @@ -27,6 +27,7 @@ int main() } else { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); return 1; /* Failure */ } } diff --git a/tests/test_encode2.c b/tests/test_encode2.c index b1105ce6..fd25c6cb 100644 --- a/tests/test_encode2.c +++ b/tests/test_encode2.c @@ -26,7 +26,12 @@ int main() /* Now encode it and check if we succeeded. */ if (pb_encode(&stream, Person_fields, &person)) + { return 0; /* Success */ + } else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); return 1; /* Failure */ + } } diff --git a/tests/test_encode3.c b/tests/test_encode3.c index 4f6859ac..982ad3c8 100644 --- a/tests/test_encode3.c +++ b/tests/test_encode3.c @@ -124,7 +124,7 @@ int main(int argc, char **argv) } else { - fprintf(stderr, "Encoding failed!\n"); + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); return 1; /* Failure */ } } -- cgit 1.2.3-korg From d580b225e859ef0462e1d96553003eff2793b796 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Mar 2013 14:52:38 +0200 Subject: Rename pb_field_iterator_t field 'current' to 'pos'. This avoids a name clash when compiling as Linux kernel module. Update issue 60 Status: FixedInGit --- pb_decode.c | 76 ++++++++++++++++++++++++++++++------------------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index fba8f64c..91b68ea3 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -280,7 +280,7 @@ void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) /* Iterator for pb_field_t list */ typedef struct { const pb_field_t *start; /* Start of the pb_field_t array */ - const pb_field_t *current; /* Current position of the iterator */ + const pb_field_t *pos; /* Current position of the iterator */ unsigned field_index; /* Zero-based index of the field. */ unsigned required_field_index; /* Zero-based index that counts only the required fields */ void *dest_struct; /* Pointer to the destination structure to decode to */ @@ -290,36 +290,36 @@ typedef struct { static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct) { - iter->start = iter->current = fields; + iter->start = iter->pos = fields; iter->field_index = 0; iter->required_field_index = 0; - iter->pData = (char*)dest_struct + iter->current->data_offset; - iter->pSize = (char*)iter->pData + iter->current->size_offset; + iter->pData = (char*)dest_struct + iter->pos->data_offset; + iter->pSize = (char*)iter->pData + iter->pos->size_offset; iter->dest_struct = dest_struct; } static bool pb_field_next(pb_field_iterator_t *iter) { bool notwrapped = true; - size_t prev_size = iter->current->data_size; + size_t prev_size = iter->pos->data_size; - if (PB_ATYPE(iter->current->type) == PB_ATYPE_STATIC && - PB_HTYPE(iter->current->type) == PB_HTYPE_REPEATED) + if (PB_ATYPE(iter->pos->type) == PB_ATYPE_STATIC && + PB_HTYPE(iter->pos->type) == PB_HTYPE_REPEATED) { - prev_size *= iter->current->array_size; + prev_size *= iter->pos->array_size; } - if (PB_HTYPE(iter->current->type) == PB_HTYPE_REQUIRED) + if (PB_HTYPE(iter->pos->type) == PB_HTYPE_REQUIRED) iter->required_field_index++; - if (iter->current->tag == 0) + if (iter->pos->tag == 0) return false; /* Only happens with empty message types */ - iter->current++; + iter->pos++; iter->field_index++; - if (iter->current->tag == 0) + if (iter->pos->tag == 0) { - iter->current = iter->start; + iter->pos = iter->start; iter->field_index = 0; iter->required_field_index = 0; iter->pData = iter->dest_struct; @@ -327,8 +327,8 @@ static bool pb_field_next(pb_field_iterator_t *iter) notwrapped = false; } - iter->pData = (char*)iter->pData + prev_size + iter->current->data_offset; - iter->pSize = (char*)iter->pData + iter->current->size_offset; + iter->pData = (char*)iter->pData + prev_size + iter->pos->data_offset; + iter->pSize = (char*)iter->pData + iter->pos->size_offset; return notwrapped; } @@ -337,7 +337,7 @@ static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag) unsigned start = iter->field_index; do { - if (iter->current->tag == tag) + if (iter->pos->tag == tag) return true; pb_field_next(iter); } while (iter->field_index != start); @@ -354,17 +354,17 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t pb_type_t type; pb_decoder_t func; - type = iter->current->type; + type = iter->pos->type; func = PB_DECODERS[PB_LTYPE(type)]; switch (PB_HTYPE(type)) { case PB_HTYPE_REQUIRED: - return func(stream, iter->current, iter->pData); + return func(stream, iter->pos, iter->pData); case PB_HTYPE_OPTIONAL: *(bool*)iter->pSize = true; - return func(stream, iter->current, iter->pData); + return func(stream, iter->pos, iter->pData); case PB_HTYPE_REPEATED: if (wire_type == PB_WT_STRING @@ -377,10 +377,10 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t if (!pb_make_string_substream(stream, &substream)) return false; - while (substream.bytes_left && *size < iter->current->array_size) + while (substream.bytes_left && *size < iter->pos->array_size) { - void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size); - if (!func(&substream, iter->current, pItem)) + void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); + if (!func(&substream, iter->pos, pItem)) { status = false; break; @@ -398,12 +398,12 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t { /* Repeated field */ size_t *size = (size_t*)iter->pSize; - void *pItem = (uint8_t*)iter->pData + iter->current->data_size * (*size); - if (*size >= iter->current->array_size) + void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); + if (*size >= iter->pos->array_size) PB_RETURN_ERROR(stream, "array overflow"); (*size)++; - return func(stream, iter->current, pItem); + return func(stream, iter->pos, pItem); } default: @@ -427,7 +427,7 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type while (substream.bytes_left) { - if (!pCallback->funcs.decode(&substream, iter->current, pCallback->arg)) + if (!pCallback->funcs.decode(&substream, iter->pos, pCallback->arg)) PB_RETURN_ERROR(stream, "callback failed"); } @@ -448,13 +448,13 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type return false; substream = pb_istream_from_buffer(buffer, size); - return pCallback->funcs.decode(&substream, iter->current, pCallback->arg); + return pCallback->funcs.decode(&substream, iter->pos, pCallback->arg); } } static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) { - switch (PB_ATYPE(iter->current->type)) + switch (PB_ATYPE(iter->pos->type)) { case PB_ATYPE_STATIC: return decode_static_field(stream, wire_type, iter); @@ -477,9 +477,9 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str do { pb_type_t type; - type = iter.current->type; + type = iter.pos->type; - if (iter.current->tag == 0) + if (iter.pos->tag == 0) continue; if (PB_ATYPE(type) == PB_ATYPE_STATIC) @@ -496,17 +496,17 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str } /* Initialize field contents to default value */ - if (PB_LTYPE(iter.current->type) == PB_LTYPE_SUBMESSAGE) + if (PB_LTYPE(iter.pos->type) == PB_LTYPE_SUBMESSAGE) { - pb_message_set_to_defaults((const pb_field_t *) iter.current->ptr, iter.pData); + pb_message_set_to_defaults((const pb_field_t *) iter.pos->ptr, iter.pData); } - else if (iter.current->ptr != NULL) + else if (iter.pos->ptr != NULL) { - memcpy(iter.pData, iter.current->ptr, iter.current->data_size); + memcpy(iter.pData, iter.pos->ptr, iter.pos->data_size); } else { - memset(iter.pData, 0, iter.current->data_size); + memset(iter.pData, 0, iter.pos->data_size); } } else if (PB_ATYPE(type) == PB_ATYPE_CALLBACK) @@ -549,7 +549,7 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ continue; } - if (PB_HTYPE(iter.current->type) == PB_HTYPE_REQUIRED + if (PB_HTYPE(iter.pos->type) == PB_HTYPE_REQUIRED && iter.required_field_index < PB_MAX_REQUIRED_FIELDS) { fields_seen[iter.required_field_index >> 3] |= (uint8_t)(1 << (iter.required_field_index & 7)); @@ -570,11 +570,11 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ unsigned i; do { req_field_count = iter.required_field_index; - last_type = iter.current->type; + last_type = iter.pos->type; } while (pb_field_next(&iter)); /* Fixup if last field was also required. */ - if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.current->tag) + if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.pos->tag) req_field_count++; /* Check the whole bytes */ -- cgit 1.2.3-korg From 03e53930723dce5793678365f350e94c2bc358dc Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Mar 2013 14:56:34 +0200 Subject: Add PB_SYSTEM_HEADER compile time option. This allows replacing the C99 standard include file names with a single system-specific file. It should provide all the necessary system functions (typedefs, memset, memcpy, strlen). Update issue 62 Status: FixedInGit --- pb.h | 5 +++++ pb_decode.c | 1 - pb_encode.c | 1 - 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pb.h b/pb.h index b12debe9..138c6bb8 100644 --- a/pb.h +++ b/pb.h @@ -8,9 +8,14 @@ #define NANOPB_VERSION nanopb-0.2.1-dev +#ifdef PB_SYSTEM_HEADER +#include PB_SYSTEM_HEADER +#else #include #include #include +#include +#endif #ifdef __GNUC__ /* This just reduces memory requirements, but is not required. */ diff --git a/pb_decode.c b/pb_decode.c index 91b68ea3..6e81b40b 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -14,7 +14,6 @@ #define NANOPB_INTERNALS #include "pb.h" #include "pb_decode.h" -#include typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn; diff --git a/pb_encode.c b/pb_encode.c index fd4b6ba5..7acee360 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -6,7 +6,6 @@ #define NANOPB_INTERNALS #include "pb.h" #include "pb_encode.h" -#include /* The warn_unused_result attribute appeared first in gcc-3.4.0 */ #if !defined(__GNUC__) || ( __GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4) -- cgit 1.2.3-korg From 6468f23d231343f1e08007621b8f3d566f0ec3ac Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Mar 2013 23:03:09 +0200 Subject: Make the generator options accept a file name in addition to format string. --- generator/nanopb_generator.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 0e898749..ddfc45b5 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -455,7 +455,11 @@ def generate_header(dependencies, headername, enums, messages, options): symbol = make_identifier(headername) yield '#ifndef _PB_%s_\n' % symbol yield '#define _PB_%s_\n' % symbol - yield options.libformat % ('pb.h') + try: + yield options.libformat % ('pb.h') + except TypeError: + # no %s specified - use whatever was passed in as options.libformat + yield options.libformat yield '\n' for dependency in dependencies: @@ -688,7 +692,11 @@ def process(filenames, options): fdesc = descriptor.FileDescriptorSet.FromString(data) # Check if any separate options are specified - optfilename = options.options_file % os.path.splitext(filename)[0] + try: + optfilename = options.options_file % os.path.splitext(filename)[0] + except TypeError: + # No %s specified, use the filename as-is + optfilename = options.options_file if options.verbose: print 'Reading options from ' + optfilename -- cgit 1.2.3-korg From 1396dce2aeeaea9f15ca5f495718bc85aeda8bd8 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 13 Mar 2013 15:22:00 +0200 Subject: Add support for packed structures on IAR and MSVC. Update issue 66 Status: FixedInGit --- generator/nanopb_generator.py | 5 +++++ pb.h | 28 ++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index ddfc45b5..5ef4ab7a 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -312,6 +312,11 @@ class Message: result += ' pb_packed' result += ' %s;' % self.name + + if self.packed: + result = 'PB_PACKED_STRUCT_START\n' + result + result += '\nPB_PACKED_STRUCT_END' + return result def types(self): diff --git a/pb.h b/pb.h index 138c6bb8..bf6b35e7 100644 --- a/pb.h +++ b/pb.h @@ -17,11 +17,29 @@ #include #endif -#ifdef __GNUC__ -/* This just reduces memory requirements, but is not required. */ -#define pb_packed __attribute__((packed)) +/* Macro for defining packed structures (compiler dependent). + * This just reduces memory requirements, but is not required. + */ +#if defined(__GNUC__) || defined(__clang__) + /* For GCC and clang */ +# define PB_PACKED_STRUCT_START +# define PB_PACKED_STRUCT_END +# define pb_packed __attribute__((packed)) +#elif defined(__ICCARM__) + /* For IAR ARM compiler */ +# define PB_PACKED_STRUCT_START _Pragma("pack(push, 1)") +# define PB_PACKED_STRUCT_END _Pragma("pack(pop)") +# define pb_packed +#elif defined(_MSC_VER) + /* For Microsoft Visual C++ */ +# define PB_PACKED_STRUCT_START __pragma(pack(push, 1)) +# define PB_PACKED_STRUCT_END __pragma(pack(pop)) +# define pb_packed #else -#define pb_packed + /* Unknown compiler */ +# define PB_PACKED_STRUCT_START +# define PB_PACKED_STRUCT_END +# define pb_packed #endif /* Handly macro for suppressing unreferenced-parameter compiler warnings. */ @@ -118,6 +136,7 @@ typedef uint8_t pb_type_t; * structures. Fix that by defining PB_FIELD_16BIT or * PB_FIELD_32BIT. */ +PB_PACKED_STRUCT_START typedef struct _pb_field_t pb_field_t; struct _pb_field_t { @@ -149,6 +168,7 @@ struct _pb_field_t { * If null, then field will zeroed. */ const void *ptr; } pb_packed; +PB_PACKED_STRUCT_END /* This structure is used for 'bytes' arrays. * It has the number of bytes in the beginning, and after that an array. -- cgit 1.2.3-korg From 6f3740f74ed48daf51908676b203f1889455c17d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 13 Mar 2013 15:34:12 +0200 Subject: Fix warning on clang. Update issue 67 Status: FixedInGit --- pb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pb.h b/pb.h index bf6b35e7..b9c3f759 100644 --- a/pb.h +++ b/pb.h @@ -224,7 +224,7 @@ typedef enum { #define pb_membersize(st, m) (sizeof ((st*)0)->m) #define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0])) #define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2)) -#define pb_delta_end(st, m1, m2) (offsetof(st, m1) == offsetof(st, m2) \ +#define pb_delta_end(st, m1, m2) (int)(offsetof(st, m1) == offsetof(st, m2) \ ? offsetof(st, m1) \ : offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) #define PB_LAST_FIELD {0,(pb_type_t) 0,0,0,0,0,0} -- cgit 1.2.3-korg From 214b0eae8aa011fa8b3e8a3dcc784f8d423aeffb Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 2 Apr 2013 19:55:21 +0300 Subject: Change the callback function to use void**. NOTE: This change breaks backwards-compatibility by default. If you have old callback functions, you can define PB_OLD_CALLBACK_STYLE to retain the old behaviour. If you want to convert your old callbacks to new signature, you need to do the following: 1) Change decode callback argument to void **arg and encode callback argument to void * const *arg. 2) Change any reference to arg into *arg. The rationale for making the new behaviour the default is that it simplifies the common case of "allocate some memory in decode callback". Update issue 69 Status: FixedInGit --- docs/concepts.rst | 12 +++++++----- docs/reference.rst | 10 +++++++--- example/client.c | 2 +- example/server.c | 4 ++-- pb.h | 9 +++++++++ pb_decode.c | 10 ++++++++-- pb_encode.c | 9 ++++++++- tests/decode_unittests.c | 4 ++-- tests/encode_unittests.c | 4 ++-- tests/test_decode_callbacks.c | 16 ++++++++-------- tests/test_encode_callbacks.c | 8 ++++---- 11 files changed, 58 insertions(+), 30 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 052edcc1..4bc0dee8 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -181,7 +181,9 @@ Field callbacks =============== When a field has dynamic length, nanopb cannot statically allocate storage for it. Instead, it allows you to handle the field in whatever way you want, using a callback function. -The `pb_callback_t`_ structure contains a function pointer and a *void* pointer you can use for passing data to the callback. If the function pointer is NULL, the field will be skipped. The actual behavior of the callback function is different in encoding and decoding modes. +The `pb_callback_t`_ structure contains a function pointer and a *void* pointer called *arg* you can use for passing data to the callback. If the function pointer is NULL, the field will be skipped. A pointer to the *arg* is passed to the function, so that it can modify it and retrieve the value. + +The actual behavior of the callback function is different in encoding and decoding modes. In encoding mode, the callback is called once and should write out everything, including field tags. In decoding mode, the callback is called repeatedly for every data item. .. _`pb_callback_t`: reference.html#pb-callback-t @@ -189,7 +191,7 @@ Encoding callbacks ------------------ :: - bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, const void *arg); + bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, void * const *arg); When encoding, the callback should write out complete fields, including the wire type and field number tag. It can write as many or as few fields as it likes. For example, if you want to write out an array as *repeated* field, you should do it all in a single call. @@ -203,7 +205,7 @@ If the callback is used in a submessage, it will be called multiple times during This callback writes out a dynamically sized string:: - bool write_string(pb_ostream_t *stream, const pb_field_t *field, const void *arg) + bool write_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) { char *str = get_string_from_somewhere(); if (!pb_encode_tag_for_field(stream, field)) @@ -216,7 +218,7 @@ Decoding callbacks ------------------ :: - bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void *arg); + bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void **arg); When decoding, the callback receives a length-limited substring that reads the contents of a single field. The field tag has already been read. For *string* and *bytes*, the length value has already been parsed, and is available at *stream->bytes_left*. @@ -226,7 +228,7 @@ The callback will be called multiple times for repeated fields. For packed field This callback reads multiple integers and prints them:: - bool read_ints(pb_istream_t *stream, const pb_field_t *field, void *arg) + bool read_ints(pb_istream_t *stream, const pb_field_t *field, void **arg) { while (stream->bytes_left) { diff --git a/docs/reference.rst b/docs/reference.rst index 93aae8b0..0db8e43e 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -24,6 +24,8 @@ PB_NO_ERRMSG Disables the support for error messages; only err Decreases the code size by a few hundred bytes. PB_BUFFER_ONLY Disables the support for custom streams. Only supports encoding to memory buffers. Speeds up execution and decreases code size slightly. +PB_OLD_CALLBACK_STYLE Use the old function signature (void\* instead of void\*\*) for callback fields. This was the + default until nanopb-0.2.1. ============================ ================================================================================================ The PB_MAX_REQUIRED_FIELDS, PB_FIELD_16BIT and PB_FIELD_32BIT settings allow raising some datatype limits to suit larger messages. @@ -122,14 +124,16 @@ Part of a message structure, for fields with type PB_HTYPE_CALLBACK:: typedef struct _pb_callback_t pb_callback_t; struct _pb_callback_t { union { - bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void *arg); - bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, const void *arg); + bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void **arg); + bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, void * const *arg); } funcs; void *arg; }; -The *arg* is passed to the callback when calling. It can be used to store any information that the callback might need. +A pointer to the *arg* is passed to the callback when calling. It can be used to store any information that the callback might need. + +Previously the function received just the value of *arg* instead of a pointer to it. This old behaviour can be enabled by defining *PB_OLD_CALLBACK_STYLE*. When calling `pb_encode`_, *funcs.encode* is used, and similarly when calling `pb_decode`_, *funcs.decode* is used. The function pointers are stored in the same memory location but are of incompatible types. You can set the function pointer to NULL to skip the field. diff --git a/example/client.c b/example/client.c index edc83948..e6e9a2e0 100644 --- a/example/client.c +++ b/example/client.c @@ -23,7 +23,7 @@ #include "fileproto.pb.h" #include "common.h" -bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void *arg) +bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void **arg) { FileInfo fileinfo; diff --git a/example/server.c b/example/server.c index 346e9fb9..9a9c2644 100644 --- a/example/server.c +++ b/example/server.c @@ -23,9 +23,9 @@ #include "fileproto.pb.h" #include "common.h" -bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) { - DIR *dir = (DIR*) arg; + DIR *dir = (DIR*) *arg; struct dirent *file; FileInfo fileinfo; diff --git a/pb.h b/pb.h index b9c3f759..61649e93 100644 --- a/pb.h +++ b/pb.h @@ -203,10 +203,19 @@ typedef struct _pb_istream_t pb_istream_t; typedef struct _pb_ostream_t pb_ostream_t; typedef struct _pb_callback_t pb_callback_t; struct _pb_callback_t { +#ifdef PB_OLD_CALLBACK_STYLE + /* Deprecated since nanopb-0.2.1 */ union { bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void *arg); bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, const void *arg); } funcs; +#else + /* New function signature, which allows modifying arg contents in callback. */ + union { + bool (*decode)(pb_istream_t *stream, const pb_field_t *field, void **arg); + bool (*encode)(pb_ostream_t *stream, const pb_field_t *field, void * const *arg); + } funcs; +#endif /* Free arg for use by callback */ void *arg; diff --git a/pb_decode.c b/pb_decode.c index 6e81b40b..e727f334 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -414,6 +414,12 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type { pb_callback_t *pCallback = (pb_callback_t*)iter->pData; +#ifdef PB_OLD_CALLBACK_STYLE + void *arg = pCallback->arg; +#else + void **arg = &(pCallback->arg); +#endif + if (pCallback->funcs.decode == NULL) return pb_skip_field(stream, wire_type); @@ -426,7 +432,7 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type while (substream.bytes_left) { - if (!pCallback->funcs.decode(&substream, iter->pos, pCallback->arg)) + if (!pCallback->funcs.decode(&substream, iter->pos, arg)) PB_RETURN_ERROR(stream, "callback failed"); } @@ -447,7 +453,7 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type return false; substream = pb_istream_from_buffer(buffer, size); - return pCallback->funcs.decode(&substream, iter->pos, pCallback->arg); + return pCallback->funcs.decode(&substream, iter->pos, arg); } } diff --git a/pb_encode.c b/pb_encode.c index 7acee360..48a3c950 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -198,9 +198,16 @@ bool checkreturn encode_static_field(pb_ostream_t *stream, const pb_field_t *fie bool checkreturn encode_callback_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData) { const pb_callback_t *callback = (const pb_callback_t*)pData; + +#ifdef PB_OLD_CALLBACK_STYLE + const void *arg = callback->arg; +#else + void * const *arg = &(callback->arg); +#endif + if (callback->funcs.encode != NULL) { - if (!callback->funcs.encode(stream, field, callback->arg)) + if (!callback->funcs.encode(stream, field, arg)) PB_RETURN_ERROR(stream, "callback error"); } return true; diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c index 039c9fa4..1e74c34d 100644 --- a/tests/decode_unittests.c +++ b/tests/decode_unittests.c @@ -19,11 +19,11 @@ bool stream_callback(pb_istream_t *stream, uint8_t *buf, size_t count) } /* Verifies that the stream passed to callback matches the byte array pointed to by arg. */ -bool callback_check(pb_istream_t *stream, const pb_field_t *field, void *arg) +bool callback_check(pb_istream_t *stream, const pb_field_t *field, void **arg) { int i; uint8_t byte; - pb_bytes_array_t *ref = (pb_bytes_array_t*) arg; + pb_bytes_array_t *ref = (pb_bytes_array_t*) *arg; for (i = 0; i < ref->size; i++) { diff --git a/tests/encode_unittests.c b/tests/encode_unittests.c index 9cdbc66e..3078998e 100644 --- a/tests/encode_unittests.c +++ b/tests/encode_unittests.c @@ -17,7 +17,7 @@ bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) return true; } -bool fieldcallback(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +bool fieldcallback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) { int value = 0x55; if (!pb_encode_tag_for_field(stream, field)) @@ -25,7 +25,7 @@ bool fieldcallback(pb_ostream_t *stream, const pb_field_t *field, const void *ar return pb_encode_varint(stream, value); } -bool crazyfieldcallback(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +bool crazyfieldcallback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) { /* This callback writes different amount of data the second time. */ uint32_t *state = (uint32_t*)arg; diff --git a/tests/test_decode_callbacks.c b/tests/test_decode_callbacks.c index 95824d1a..7ce4ec0b 100644 --- a/tests/test_decode_callbacks.c +++ b/tests/test_decode_callbacks.c @@ -6,7 +6,7 @@ #include #include "callbacks.pb.h" -bool print_string(pb_istream_t *stream, const pb_field_t *field, void *arg) +bool print_string(pb_istream_t *stream, const pb_field_t *field, void **arg) { uint8_t buffer[1024] = {0}; @@ -20,37 +20,37 @@ bool print_string(pb_istream_t *stream, const pb_field_t *field, void *arg) /* Print the string, in format comparable with protoc --decode. * Format comes from the arg defined in main(). */ - printf((char*)arg, buffer); + printf((char*)*arg, buffer); return true; } -bool print_int32(pb_istream_t *stream, const pb_field_t *field, void *arg) +bool print_int32(pb_istream_t *stream, const pb_field_t *field, void **arg) { uint64_t value; if (!pb_decode_varint(stream, &value)) return false; - printf((char*)arg, (long)value); + printf((char*)*arg, (long)value); return true; } -bool print_fixed32(pb_istream_t *stream, const pb_field_t *field, void *arg) +bool print_fixed32(pb_istream_t *stream, const pb_field_t *field, void **arg) { uint32_t value; if (!pb_decode_fixed32(stream, &value)) return false; - printf((char*)arg, (long)value); + printf((char*)*arg, (long)value); return true; } -bool print_fixed64(pb_istream_t *stream, const pb_field_t *field, void *arg) +bool print_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg) { uint64_t value; if (!pb_decode_fixed64(stream, &value)) return false; - printf((char*)arg, (long long)value); + printf((char*)*arg, (long long)value); return true; } diff --git a/tests/test_encode_callbacks.c b/tests/test_encode_callbacks.c index 7fa017f9..afab48e8 100644 --- a/tests/test_encode_callbacks.c +++ b/tests/test_encode_callbacks.c @@ -5,7 +5,7 @@ #include #include "callbacks.pb.h" -bool encode_string(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +bool encode_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) { char *str = "Hello world!"; @@ -15,7 +15,7 @@ bool encode_string(pb_ostream_t *stream, const pb_field_t *field, const void *ar return pb_encode_string(stream, (uint8_t*)str, strlen(str)); } -bool encode_int32(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +bool encode_int32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) { if (!pb_encode_tag_for_field(stream, field)) return false; @@ -23,7 +23,7 @@ bool encode_int32(pb_ostream_t *stream, const pb_field_t *field, const void *arg return pb_encode_varint(stream, 42); } -bool encode_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +bool encode_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) { if (!pb_encode_tag_for_field(stream, field)) return false; @@ -32,7 +32,7 @@ bool encode_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *a return pb_encode_fixed32(stream, &value); } -bool encode_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +bool encode_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) { if (!pb_encode_tag_for_field(stream, field)) return false; -- cgit 1.2.3-korg From 384e686fe6d7b7088cdbc4b44bc988eba3a21945 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 2 Apr 2013 20:01:31 +0300 Subject: Do not generate has_ fields for callback fields. The arg field can be used to store the field presence from inside the callback. Furthermore, having the has_ field for encoding callbacks would be more annoying than useful. Update issue 70 Status: FixedInGit --- generator/nanopb_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 5ef4ab7a..5366f1bd 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -186,7 +186,7 @@ class Field: return cmp(self.tag, other.tag) def __str__(self): - if self.rules == 'OPTIONAL': + if self.rules == 'OPTIONAL' and self.allocation == 'STATIC': result = ' bool has_' + self.name + ';\n' elif self.rules == 'REPEATED' and self.allocation == 'STATIC': result = ' size_t ' + self.name + '_count;\n' -- cgit 1.2.3-korg From 710465a8e060d5ef7848a6f348f043435b1e39e0 Mon Sep 17 00:00:00 2001 From: dch Date: Sun, 7 Apr 2013 15:28:05 +0100 Subject: __pragma keyword is only supported by recent Microsoft compilers --- pb.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pb.h b/pb.h index 61649e93..4ce58acf 100644 --- a/pb.h +++ b/pb.h @@ -30,7 +30,7 @@ # define PB_PACKED_STRUCT_START _Pragma("pack(push, 1)") # define PB_PACKED_STRUCT_END _Pragma("pack(pop)") # define pb_packed -#elif defined(_MSC_VER) +#elif defined(_MSC_VER) && (_MSC_VER >= 1500) /* For Microsoft Visual C++ */ # define PB_PACKED_STRUCT_START __pragma(pack(push, 1)) # define PB_PACKED_STRUCT_END __pragma(pack(pop)) -- cgit 1.2.3-korg From a968233777e0781cfd3dd7b9566b631dc576fed1 Mon Sep 17 00:00:00 2001 From: dch Date: Sun, 7 Apr 2013 15:28:51 +0100 Subject: No need to include stdbool.h separately --- pb_decode.h | 1 - pb_encode.h | 1 - 2 files changed, 2 deletions(-) diff --git a/pb_decode.h b/pb_decode.h index 35de0a22..e5d98c55 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -9,7 +9,6 @@ * These are usually generated from .proto-files with a script. */ -#include #include "pb.h" #ifdef __cplusplus diff --git a/pb_encode.h b/pb_encode.h index bd6132b6..773b7178 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -6,7 +6,6 @@ * and their field descriptions (just like with pb_decode). */ -#include #include "pb.h" #ifdef __cplusplus -- cgit 1.2.3-korg From 6a022985845a0a50b32b7c1fb22f9aee1f675825 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 8 Apr 2013 11:00:28 +0300 Subject: Avoid maybe-uninitialized warning Patch from dch. --- pb_decode.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index e727f334..7eda89a4 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -664,7 +664,8 @@ bool pb_decode_fixed64(pb_istream_t *stream, void *dest) bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint64_t value; - bool status = pb_decode_varint(stream, &value); + if (!pb_decode_varint(stream, &value)) + return false; switch (field->data_size) { @@ -675,13 +676,14 @@ bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, vo default: PB_RETURN_ERROR(stream, "invalid data_size"); } - return status; + return true; } bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) { int64_t value; - bool status = pb_decode_svarint(stream, &value); + if (!pb_decode_svarint(stream, &value)) + return false; switch (field->data_size) { @@ -690,7 +692,7 @@ bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, v default: PB_RETURN_ERROR(stream, "invalid data_size"); } - return status; + return true; } bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest) -- cgit 1.2.3-korg From 9939910833a9289b5913eff29f951195e7bc61c6 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 14 Apr 2013 09:26:42 +0300 Subject: Fix bug with empty strings in repeated string callbacks. Fix suggested by Henrik Carlgren. Added also unit test for the bug. Update issue 73 Status: FixedInGit --- pb_decode.c | 4 ++-- tests/callbacks.proto | 1 + tests/test_decode_callbacks.c | 2 ++ tests/test_encode_callbacks.c | 18 ++++++++++++++++++ 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 7eda89a4..d0e18ccb 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -430,11 +430,11 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type if (!pb_make_string_substream(stream, &substream)) return false; - while (substream.bytes_left) + do { if (!pCallback->funcs.decode(&substream, iter->pos, arg)) PB_RETURN_ERROR(stream, "callback failed"); - } + } while (substream.bytes_left); pb_close_string_substream(stream, &substream); return true; diff --git a/tests/callbacks.proto b/tests/callbacks.proto index 8beeaabf..ccd1edd8 100644 --- a/tests/callbacks.proto +++ b/tests/callbacks.proto @@ -11,5 +11,6 @@ message TestMessage { repeated fixed32 fixed32value = 3; repeated fixed64 fixed64value = 4; optional SubMessage submsg = 5; + repeated string repeatedstring = 6; } diff --git a/tests/test_decode_callbacks.c b/tests/test_decode_callbacks.c index 7ce4ec0b..b5056923 100644 --- a/tests/test_decode_callbacks.c +++ b/tests/test_decode_callbacks.c @@ -83,6 +83,8 @@ int main() testmessage.fixed32value.arg = "fixed32value: %ld\n"; testmessage.fixed64value.funcs.decode = &print_fixed64; testmessage.fixed64value.arg = "fixed64value: %lld\n"; + testmessage.repeatedstring.funcs.decode = &print_string; + testmessage.repeatedstring.arg = "repeatedstring: \"%s\"\n"; if (!pb_decode(&stream, TestMessage_fields, &testmessage)) return 1; diff --git a/tests/test_encode_callbacks.c b/tests/test_encode_callbacks.c index afab48e8..3bb6a45e 100644 --- a/tests/test_encode_callbacks.c +++ b/tests/test_encode_callbacks.c @@ -41,6 +41,22 @@ bool encode_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const return pb_encode_fixed64(stream, &value); } +bool encode_repeatedstring(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + char *str[4] = {"Hello world!", "", "Test", "Test2"}; + int i; + + for (i = 0; i < 4; i++) + { + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!pb_encode_string(stream, (uint8_t*)str[i], strlen(str[i]))) + return false; + } + return true; +} + int main() { uint8_t buffer[1024]; @@ -57,6 +73,8 @@ int main() testmessage.submsg.int32value.funcs.encode = &encode_int32; testmessage.submsg.fixed32value.funcs.encode = &encode_fixed32; testmessage.submsg.fixed64value.funcs.encode = &encode_fixed64; + + testmessage.repeatedstring.funcs.encode = &encode_repeatedstring; if (!pb_encode(&stream, TestMessage_fields, &testmessage)) return 1; -- cgit 1.2.3-korg From d2063ff0b63b3b4f2f2081e61874a3c2487c4994 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 14 Apr 2013 09:46:39 +0300 Subject: Handle unterminated strings when encoding. If the null terminator is not present, string will be limited to the data size of the field. If you are still using the pb_enc_string (deprecated since 0.1.3) from callbacks, now would be an excellent time to stop. The pb_field_t for the callback will not contain proper data_size. Use pb_encode_string() instead. Update issue 68 Status: FixedInGit --- pb_encode.c | 12 ++++++++++-- tests/encode_unittests.c | 8 +++++--- tests/unittestproto.proto | 4 ++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/pb_encode.c b/pb_encode.c index 48a3c950..0e048ac6 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -461,8 +461,16 @@ bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, con bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - UNUSED(field); - return pb_encode_string(stream, (const uint8_t*)src, strlen((const char*)src)); + /* strnlen() is not always available, so just use a for-loop */ + size_t size = 0; + const char *p = (const char*)src; + while (size < field->data_size && *p != '\0') + { + size++; + p++; + } + + return pb_encode_string(stream, (const uint8_t*)src, size); } bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src) diff --git a/tests/encode_unittests.c b/tests/encode_unittests.c index 3078998e..6a8f5e9e 100644 --- a/tests/encode_unittests.c +++ b/tests/encode_unittests.c @@ -180,12 +180,14 @@ int main() { uint8_t buffer[30]; pb_ostream_t s; - char value[] = "xyzzy"; + char value[30] = "xyzzy"; COMMENT("Test pb_enc_string") - TEST(WRITES(pb_enc_string(&s, NULL, &value), "\x05xyzzy")) + TEST(WRITES(pb_enc_string(&s, &StringMessage_fields[0], &value), "\x05xyzzy")) value[0] = '\0'; - TEST(WRITES(pb_enc_string(&s, NULL, &value), "\x00")) + TEST(WRITES(pb_enc_string(&s, &StringMessage_fields[0], &value), "\x00")) + memset(value, 'x', 30); + TEST(WRITES(pb_enc_string(&s, &StringMessage_fields[0], &value), "\x0Axxxxxxxxxx")) } { diff --git a/tests/unittestproto.proto b/tests/unittestproto.proto index c8a39dd6..7024942e 100644 --- a/tests/unittestproto.proto +++ b/tests/unittestproto.proto @@ -8,6 +8,10 @@ message FloatArray { repeated float data = 1 [(nanopb).max_count = 10]; } +message StringMessage { + required string data = 1 [(nanopb).max_size = 10]; +} + message CallbackArray { // We cheat a bit and use this message for testing other types, too. // Nanopb does not care about the actual defined data type for callback -- cgit 1.2.3-korg From 26b52b79ad339c79aa1f0bbe09fcd1fce872b9b9 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 14 Apr 2013 10:04:46 +0300 Subject: Update changelog --- CHANGELOG | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 9245d59b..a58be47a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,20 @@ +nanopb-0.2.1 + NOTE: The default callback function signature has changed. + If you don't want to update your code, define PB_OLD_CALLBACK_STYLE. + + Change the callback function to use void** (issue 69) + Add support for defining the nanopb options in a separate file (issue 12) + Add support for packed structs in IAR and MSVC (in addition to GCC) (issue 66) + Implement error message support for the encoder side (issue 7) + Handle unterminated strings when encoding (issue 68) + Fix bug with empty strings in repeated string callbacks (issue 73) + Fix regression in 0.2.0 with optional callback fields (issue 70) + Fix bugs with empty message types (issues 64, 65) + Fix some compiler warnings on clang (issue 67) + Some portability improvements (issues 60, 62) + Various new generator options + Improved tests + nanopb-0.2.0 NOTE: This release requires you to regenerate all .pb.c files. Files generated by older versions will not -- cgit 1.2.3-korg From ef741ea530e4086344ed0ad0fc90f21c224b9503 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 14 Apr 2013 10:06:47 +0300 Subject: Publishing nanopb-0.2.1 --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 5366f1bd..c772c8b5 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.1-dev" +nanopb_version = "nanopb-0.2.1" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index 4ce58acf..bf07ceb1 100644 --- a/pb.h +++ b/pb.h @@ -6,7 +6,7 @@ * see pb_encode.h or pb_decode.h */ -#define NANOPB_VERSION nanopb-0.2.1-dev +#define NANOPB_VERSION nanopb-0.2.1 #ifdef PB_SYSTEM_HEADER #include PB_SYSTEM_HEADER -- cgit 1.2.3-korg From bfd7cb36b90dfa22d37c918d8cdcdce60bc8af85 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 14 Apr 2013 10:07:46 +0300 Subject: Setting version to 0.2.2-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index c772c8b5..3529a346 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.1" +nanopb_version = "nanopb-0.2.2-dev" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index bf07ceb1..c996bf99 100644 --- a/pb.h +++ b/pb.h @@ -6,7 +6,7 @@ * see pb_encode.h or pb_decode.h */ -#define NANOPB_VERSION nanopb-0.2.1 +#define NANOPB_VERSION nanopb-0.2.2-dev #ifdef PB_SYSTEM_HEADER #include PB_SYSTEM_HEADER -- cgit 1.2.3-korg From a1cec46b3e824292b4ae22fa1f5c4faf9846196c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 3 Jun 2013 22:46:19 +0300 Subject: Include the field tags in the generated .pb.h file. Patch from Michael Haberler. --- generator/nanopb_generator.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 3529a346..bd365dfd 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -235,6 +235,11 @@ class Field: else: return 'const %s %s_default%s = %s;' % (ctype, self.struct_name + self.name, array_decl, default) + def tags(self): + '''Return the #define for the tag number of this field.''' + identifier = '%s_%s_tag' % (self.struct_name, self.name) + return '#define %-40s %d\n' % (identifier, self.tag) + def pb_field_t(self, prev_field_name): '''Return the pb_field_t initializer to use in the constant array. prev_field_name is the name of the previous field or None. @@ -490,6 +495,12 @@ def generate_header(dependencies, headername, enums, messages, options): yield msg.default_decl(True) yield '\n' + yield '/* Field tags (for use in manual encoding/decoding) */\n' + for msg in sort_dependencies(messages): + for field in msg.fields: + yield field.tags() + yield '\n' + yield '/* Struct field encoding specification for nanopb */\n' for msg in messages: yield msg.fields_declaration() + '\n' -- cgit 1.2.3-korg From 68dd0171bc51e871a522d21a40c35c08de0fb573 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 6 Jul 2013 13:01:21 +0300 Subject: Switch the example project to use the new .options file --- example/fileproto.options | 13 +++++++++++++ example/fileproto.proto | 14 +++----------- 2 files changed, 16 insertions(+), 11 deletions(-) create mode 100644 example/fileproto.options diff --git a/example/fileproto.options b/example/fileproto.options new file mode 100644 index 00000000..29a2ab0e --- /dev/null +++ b/example/fileproto.options @@ -0,0 +1,13 @@ +# This file defines the nanopb-specific options for the messages defined +# in fileproto.proto. +# +# If you come from high-level programming background, the hardcoded +# maximum lengths may disgust you. However, if your microcontroller only +# has a few kB of ram to begin with, setting reasonable limits for +# filenames is ok. +# +# On the other hand, using the callback interface, it is not necessary +# to set a limit on the number of files in the response. + +ListFilesRequest.path max_size:128 +FileInfo.name max_size:128 diff --git a/example/fileproto.proto b/example/fileproto.proto index e2786b15..3e70c492 100644 --- a/example/fileproto.proto +++ b/example/fileproto.proto @@ -1,22 +1,14 @@ -import "nanopb.proto"; - // This defines protocol for a simple server that lists files. // -// If you come from high-level programming background, the hardcoded -// maximum lengths may disgust you. However, if your microcontroller only -// has a few kB of ram to begin with, setting reasonable limits for -// filenames is ok. -// -// On the other hand, using the callback interface, it is not necessary -// to set a limit on the number of files in the response. +// See also the nanopb-specific options in fileproto.options. message ListFilesRequest { - optional string path = 1 [default = "/", (nanopb).max_size = 128]; + optional string path = 1 [default = "/"]; } message FileInfo { required uint64 inode = 1; - required string name = 2 [(nanopb).max_size = 128]; + required string name = 2; } message ListFilesResponse { -- cgit 1.2.3-korg From 6e9e5329278b04a8e76d63f06fed2f3bfa80e2f8 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 6 Jul 2013 13:49:47 +0300 Subject: Document the .options file usage. Also add note about the 'packed' message option being incompatible with CPUs that do not support unaligned access. Update issue 12 Status: FixedInGit Update issue 77 Status: FixedInGit --- docs/concepts.rst | 47 ++++++------ docs/reference.rst | 192 +++++++++++++++++++++++++++++++++++++++++++------ generator/nanopb.proto | 2 + 3 files changed, 193 insertions(+), 48 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 4bc0dee8..2ae76521 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -10,47 +10,40 @@ The things outlined here are the underlying concepts of the nanopb design. Proto files =========== -All Protocol Buffers implementations use .proto files to describe the message format. -The point of these files is to be a portable interface description language. +All Protocol Buffers implementations use .proto files to describe the message +format. The point of these files is to be a portable interface description +language. Compiling .proto files for nanopb --------------------------------- -Nanopb uses the Google's protoc compiler to parse the .proto file, and then a python script to generate the C header and source code from it:: +Nanopb uses the Google's protoc compiler to parse the .proto file, and then a +python script to generate the C header and source code from it:: user@host:~$ protoc -omessage.pb message.proto user@host:~$ python ../generator/nanopb_generator.py message.pb Writing to message.h and message.c user@host:~$ -Compiling .proto files with nanopb options ------------------------------------------- -Nanopb defines two extensions for message fields described in .proto files: *max_size* and *max_count*. -These are the maximum size of a string and maximum count of items in an array:: +Modifying generator behaviour +----------------------------- +Using generator options, you can set maximum sizes for fields in order to +allocate them statically. The preferred way to do this is to create an .options +file with the same name as your .proto file:: - required string name = 1 [(nanopb).max_size = 40]; - repeated PhoneNumber phone = 4 [(nanopb).max_count = 5]; + # Foo.proto + message Foo { + required string name = 1; + } -To use these extensions, you need to place an import statement in the beginning of the file:: - - import "nanopb.proto"; - -This file, in turn, requires the file *google/protobuf/descriptor.proto*. This is usually installed under */usr/include*. Therefore, to compile a .proto file which uses options, use a protoc command similar to:: - - protoc -I/usr/include -Inanopb/generator -I. -omessage.pb message.proto - -The options can be defined in file, message and field scopes:: - - option (nanopb_fileopt).max_size = 20; // File scope - message Message - { - option (nanopb_msgopt).max_size = 30; // Message scope - required string fieldsize = 1 [(nanopb).max_size = 40]; // Field scope - } +:: -It is also possible to give the options on command line, but then they will affect the whole file. For example:: + # Foo.options + Foo.name max_size:16 - user@host:~$ python ../generator/nanopb_generator.py -s 'max_size: 20' message.pb +For more information on this, see the `Proto file options`_ section in the +reference manual. +.. _`Proto file options`: reference.html#proto-file-options Streams ======= diff --git a/docs/reference.rst b/docs/reference.rst index 0db8e43e..6094e13c 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -6,31 +6,160 @@ Nanopb: API reference .. contents :: + + + Compilation options =================== -The following options can be specified using -D switch given to the C compiler: - -============================ ================================================================================================ -__BIG_ENDIAN__ Set this if your platform stores integers and floats in big-endian format. - Mixed-endian systems (different layout for ints and floats) are currently not supported. -NANOPB_INTERNALS Set this to expose the field encoder functions that are hidden since nanopb-0.1.3. -PB_MAX_REQUIRED_FIELDS Maximum number of required fields to check for presence. Default value is 64. Increases stack - usage 1 byte per every 8 fields. Compiler warning will tell if you need this. -PB_FIELD_16BIT Add support for tag numbers > 255 and fields larger than 255 bytes or 255 array entries. - Increases code size 3 bytes per each field. Compiler error will tell if you need this. -PB_FIELD_32BIT Add support for tag numbers > 65535 and fields larger than 65535 bytes or 65535 array entries. - Increases code size 9 bytes per each field. Compiler error will tell if you need this. -PB_NO_ERRMSG Disables the support for error messages; only error information is the true/false return value. - Decreases the code size by a few hundred bytes. -PB_BUFFER_ONLY Disables the support for custom streams. Only supports encoding to memory buffers. - Speeds up execution and decreases code size slightly. -PB_OLD_CALLBACK_STYLE Use the old function signature (void\* instead of void\*\*) for callback fields. This was the +The following options can be specified using -D switch given to the C compiler +when compiling the nanopb library and applications using it. You must have the +same settings for the nanopb library and all code that includes pb.h. + +============================ ================================================ +__BIG_ENDIAN__ Set this if your platform stores integers and + floats in big-endian format. Mixed-endian + systems (different layout for ints and floats) + are currently not supported. +NANOPB_INTERNALS Set this to expose the field encoder functions + that are hidden since nanopb-0.1.3. +PB_MAX_REQUIRED_FIELDS Maximum number of required fields to check for + presence. Default value is 64. Increases stack + usage 1 byte per every 8 fields. Compiler + warning will tell if you need this. +PB_FIELD_16BIT Add support for tag numbers > 255 and fields + larger than 255 bytes or 255 array entries. + Increases code size 3 bytes per each field. + Compiler error will tell if you need this. +PB_FIELD_32BIT Add support for tag numbers > 65535 and fields + larger than 65535 bytes or 65535 array entries. + Increases code size 9 bytes per each field. + Compiler error will tell if you need this. +PB_NO_ERRMSG Disables the support for error messages; only + error information is the true/false return + value. Decreases the code size by a few hundred + bytes. +PB_BUFFER_ONLY Disables the support for custom streams. Only + supports encoding and decoding with memory + buffers. Speeds up execution and decreases code + size slightly. +PB_OLD_CALLBACK_STYLE Use the old function signature (void\* instead + of void\*\*) for callback fields. This was the default until nanopb-0.2.1. -============================ ================================================================================================ +============================ ================================================ + +The PB_MAX_REQUIRED_FIELDS, PB_FIELD_16BIT and PB_FIELD_32BIT settings allow +raising some datatype limits to suit larger messages. Their need is recognized +automatically by C-preprocessor #if-directives in the generated .pb.h files. +The default setting is to use the smallest datatypes (least resources used). + + + + +Proto file options +================== +The generator behaviour can be adjusted using these options, defined in the +'nanopb.proto' file in the generator folder: + +============================ ================================================ +max_size Allocated size for 'bytes' and 'string' fields. +max_count Allocated number of entries in arrays + ('repeated' fields). +type Type of the generated field. Default value + is FT_DEFAULT, which selects automatically. + You can use FT_CALLBACK, FT_STATIC or FT_IGNORE + to force a callback field, a static field or + to completely ignore the field. +long_names Prefix the enum name to the enum value in + definitions, i.e. 'EnumName_EnumValue'. Enabled + by default. +packed_struct Make the generated structures packed. + NOTE: This cannot be used on CPUs that break + on unaligned accesses to variables. +============================ ================================================ + +These options can be defined for the .proto files before they are converted +using the nanopb-generatory.py. There are three ways to define the options: + +1. Using a separate .options file. + This is the preferred way as of nanopb-0.2.1, because it has the best + compatibility with other protobuf libraries. +2. Defining the options on the command line of nanopb_generator.py. + This only makes sense for settings that apply to a whole file. +3. Defining the options in the .proto file using the nanopb extensions. + This is the way used in nanopb-0.1, and will remain supported in the + future. It however sometimes causes trouble when using the .proto file + with other protobuf libraries. + +The effect of the options is the same no matter how they are given. The most +common purpose is to define maximum size for string fields in order to +statically allocate them. + +Defining the options in a .options file +--------------------------------------- +The preferred way to define options is to have a separate file +'myproto.options' in the same directory as the 'myproto.proto'. The +generator will automatically search for this file and read the options from +it. The file format is as follows: + +* Lines starting with '#' or '//' are regarded as comments. +* Blank lines are ignored. +* All other lines should start with a field name pattern, followed by one or + more options. For example: *"MyMessage.myfield max_size:5 max_count:10"*. +* The field name pattern is matched against a string of form 'Message.field'. + For nested messages, the string is 'Message.SubMessage.field'. +* The field name pattern may use the notation recognized by Python fnmatch(): + - \* matches any part of string, like 'Message.\*' for all fields + - \? matches any single character + - [seq] matches any of characters 's', 'e' and 'q' + - [!seq] matches any other character +* The options are written as 'option_name:option_value' and several options + can be defined on same line, separated by whitespace. +* Options defined later in the file override the ones specified earlier, so + it makes sense to define wildcard options first in the file and more specific + ones later. + +If preferred, the name of the options file can be set using the command line +switch '-f' to nanopb_generator.py. + +Defining the options on command line +------------------------------------ +The nanopb_generator.py has a simple command line option '-s OPTION:VALUE'. +The setting applies to the whole file that is being processed. + +Defining the options in the .proto file +--------------------------------------- +The .proto file format allows defining custom options for the fields. +The nanopb library comes with 'nanopb.proto' which does exactly that, allowing +you do define the options directly in the .proto file: + + import "nanopb.proto"; + required string name = 1 [(nanopb).max_size = 40]; + repeated PhoneNumber phone = 4 [(nanopb).max_count = 5]; + +A small complication is that you have to set the include path of protoc so that +nanopb.proto can be found. This file, in turn, requires the file +*google/protobuf/descriptor.proto*. This is usually installed under +*/usr/include*. Therefore, to compile a .proto file which uses options, use a +protoc command similar to:: + + protoc -I/usr/include -Inanopb/generator -I. -omessage.pb message.proto + +The options can be defined in file, message and field scopes:: + + option (nanopb_fileopt).max_size = 20; // File scope + message Message + { + option (nanopb_msgopt).max_size = 30; // Message scope + required string fieldsize = 1 [(nanopb).max_size = 40]; // Field scope + } + + + + + + + -The PB_MAX_REQUIRED_FIELDS, PB_FIELD_16BIT and PB_FIELD_32BIT settings allow raising some datatype limits to suit larger messages. -Their need is recognized automatically by C-preprocessor #if-directives in the generated .pb.h files. The default setting is to use -the smallest datatypes (least resources used). pb.h ==== @@ -148,6 +277,16 @@ Protocol Buffers wire types. These are used with `pb_encode_tag`_. :: PB_WT_32BIT = 5 } pb_wire_type_t; + + + + + + + + + + pb_encode.h =========== @@ -297,6 +436,17 @@ In Protocol Buffers format, the submessage size must be written before the subme If the submessage contains callback fields, the callback function might misbehave and write out a different amount of data on the second call. This situation is recognized and *false* is returned, but garbage will be written to the output before the problem is detected. + + + + + + + + + + + pb_decode.h =========== diff --git a/generator/nanopb.proto b/generator/nanopb.proto index 7b73c7b3..fe564b5c 100644 --- a/generator/nanopb.proto +++ b/generator/nanopb.proto @@ -33,6 +33,8 @@ message NanoPBOptions { optional bool long_names = 4 [default = true]; // Add 'packed' attribute to generated structs. + // Note: this cannot be used on CPUs that break on unaligned + // accesses to variables. optional bool packed_struct = 5 [default = false]; } -- cgit 1.2.3-korg From 4b705bf64b9a7b6bf0f61da3c84234847a7e8404 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 6 Jul 2013 15:25:42 +0300 Subject: Add error message macros to API reference. --- docs/reference.rst | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/docs/reference.rst b/docs/reference.rst index 6094e13c..6cd3c646 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -277,13 +277,38 @@ Protocol Buffers wire types. These are used with `pb_encode_tag`_. :: PB_WT_32BIT = 5 } pb_wire_type_t; +PB_GET_ERROR +------------ +Get the current error message from a stream, or a placeholder string if +there is no error message:: + #define PB_GET_ERROR(stream) (string expression) +This should be used for printing errors, for example:: + if (!pb_decode(...)) + { + printf("Decode failed: %s\n", PB_GET_ERROR(stream)); + } + +The macro only returns pointers to constant strings (in code memory), +so that there is no need to release the returned pointer. +PB_RETURN_ERROR +--------------- +Set the error message and return false:: + #define PB_RETURN_ERROR(stream,msg) (sets error and returns false) +This should be used to handle error conditions inside nanopb functions +and user callback functions:: + + if (error_condition) + { + PB_RETURN_ERROR(stream, "something went wrong"); + } +The *msg* parameter must be a constant string. -- cgit 1.2.3-korg From 0ed3158560fe4b96d1d0df7f6b655409917f5989 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 6 Jul 2013 15:27:31 +0300 Subject: Add section in pb.h for changing compilation settings. Update issue 76 Status: FixedInGit --- docs/reference.rst | 10 +++++--- pb.h | 73 ++++++++++++++++++++++++++++++++++++++++++------------ 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 6cd3c646..33884885 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -11,9 +11,13 @@ Nanopb: API reference Compilation options =================== -The following options can be specified using -D switch given to the C compiler -when compiling the nanopb library and applications using it. You must have the -same settings for the nanopb library and all code that includes pb.h. +The following options can be specified in one of two ways: + +1. Using the -D switch on the C compiler command line. +2. By #defining them at the top of pb.h. + +You must have the same settings for the nanopb library and all code that +includes pb.h. ============================ ================================================ __BIG_ENDIAN__ Set this if your platform stores integers and diff --git a/pb.h b/pb.h index c996bf99..d5047ac5 100644 --- a/pb.h +++ b/pb.h @@ -1,13 +1,61 @@ +/* Common parts of the nanopb library. Most of these are quite low-level + * stuff. For the high-level interface, see pb_encode.h and pb_decode.h. + */ + #ifndef _PB_H_ #define _PB_H_ -/* pb.h: Common parts for nanopb library. - * Most of these are quite low-level stuff. For the high-level interface, - * see pb_encode.h or pb_decode.h - */ +/***************************************************************** + * Nanopb compilation time options. You can change these here by * + * uncommenting the lines, or on the compiler command line. * + *****************************************************************/ + +/* Define this if your CPU architecture is big endian, i.e. it + * stores the most-significant byte first. */ +/* #define __BIG_ENDIAN__ 1 */ + +/* Increase the number of required fields that are tracked. + * A compiler warning will tell if you need this. */ +/* #define PB_MAX_REQUIRED_FIELDS 256 */ + +/* Add support for tag numbers > 255 and fields larger than 255 bytes. */ +/* #define PB_FIELD_16BIT 1 */ + +/* Add support for tag numbers > 65536 and fields larger than 65536 bytes. */ +/* #define PB_FIELD_32BIT 1 */ + +/* Disable support for error messages in order to save some code space. */ +/* #define PB_NO_ERRMSG 1 */ +/* Disable support for custom streams (support only memory buffers). */ +/* #define PB_BUFFER_ONLY 1 */ + +/* Switch back to the old-style callback function signature. + * This was the default until nanopb-0.2.1. */ +/* #define PB_OLD_CALLBACK_STYLE */ + + +/****************************************************************** + * You usually don't need to change anything below this line. * + * Feel free to look around and use the defined macros, though. * + ******************************************************************/ + + +/* Version of the nanopb library. Just in case you want to check it in + * your own program. */ #define NANOPB_VERSION nanopb-0.2.2-dev +/* Include all the system headers needed by nanopb. You will need the + * definitions of the following: + * - strlen, memcpy, memset functions + * - [u]int8_t, [u]int16_t, [u]int32_t, [u]int64_t + * - size_t + * - bool + * + * If you don't have the standard header files, you can instead provide + * a custom header that defines or includes all this. In that case, + * define PB_SYSTEM_HEADER to the path of this file. + */ #ifdef PB_SYSTEM_HEADER #include PB_SYSTEM_HEADER #else @@ -42,7 +90,7 @@ # define pb_packed #endif -/* Handly macro for suppressing unreferenced-parameter compiler warnings. */ +/* Handly macro for suppressing unreferenced-parameter compiler warnings. */ #ifndef UNUSED #define UNUSED(x) (void)(x) #endif @@ -56,8 +104,7 @@ #define STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) static_assertion_##MSG##LINE##COUNTER #endif -/* Number of required fields to keep track of - * (change here or on compiler command line). */ +/* Number of required fields to keep track of. */ #ifndef PB_MAX_REQUIRED_FIELDS #define PB_MAX_REQUIRED_FIELDS 64 #endif @@ -78,9 +125,7 @@ typedef uint8_t pb_type_t; -/************************ - * Field contents types * - ************************/ +/**** Field data types ****/ /* Numeric types */ #define PB_LTYPE_VARINT 0x00 /* int32, uint32, int64, uint64, bool, enum */ @@ -107,18 +152,14 @@ typedef uint8_t pb_type_t; #define PB_LTYPES_COUNT 7 #define PB_LTYPE_MASK 0x0F -/************************** - * Field repetition rules * - **************************/ +/**** Field repetition rules ****/ #define PB_HTYPE_REQUIRED 0x00 #define PB_HTYPE_OPTIONAL 0x10 #define PB_HTYPE_REPEATED 0x20 #define PB_HTYPE_MASK 0x30 -/******************** - * Allocation types * - ********************/ +/**** Field allocation types ****/ #define PB_ATYPE_STATIC 0x00 #define PB_ATYPE_CALLBACK 0x40 -- cgit 1.2.3-korg From 314460c2a518bdfc4e5dfb87992d907461a7c0e6 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 6 Jul 2013 15:55:15 +0300 Subject: Clean up the comments in pb_encode.h and pb_decode.h --- pb_decode.h | 106 +++++++++++++++++++++++++++++++------------------ pb_encode.h | 130 ++++++++++++++++++++++++++++++++++++++---------------------- 2 files changed, 150 insertions(+), 86 deletions(-) diff --git a/pb_decode.h b/pb_decode.h index e5d98c55..85efa212 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -1,33 +1,77 @@ -#ifndef _PB_DECODE_H_ -#define _PB_DECODE_H_ - /* pb_decode.h: Functions to decode protocol buffers. Depends on pb_decode.c. - * The main function is pb_decode. You will also need to create an input - * stream, which is easiest to do with pb_istream_from_buffer(). - * - * You also need structures and their corresponding pb_field_t descriptions. - * These are usually generated from .proto-files with a script. + * The main function is pb_decode. You also need an input stream, and the + * field descriptions created by nanopb_generator.py. */ +#ifndef _PB_DECODE_H_ +#define _PB_DECODE_H_ + #include "pb.h" #ifdef __cplusplus extern "C" { #endif -/* Lightweight input stream. - * You can provide a callback function for reading or use - * pb_istream_from_buffer. +/*************************** + * Main decoding functions * + ***************************/ + +/* Decode a single protocol buffers message from input stream into a C structure. + * Returns true on success, false on any failure. + * The actual struct pointed to by dest must match the description in fields. + * Callback fields of the destination structure must be initialized by caller. + * All other fields will be initialized by this function. + * + * Example usage: + * MyMessage msg = {}; + * uint8_t buffer[64]; + * pb_istream_t stream; + * + * // ... read some data into buffer ... + * + * stream = pb_istream_from_buffer(buffer, count); + * pb_decode(&stream, MyMessage_fields, &msg); + */ +bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +/* Same as pb_decode, except does not initialize the destination structure + * to default values. This is slightly faster if you need no default values + * and just do memset(struct, 0, sizeof(struct)) yourself. + * + * This can also be used for 'merging' two messages, i.e. update only the + * fields that exist in the new message. + */ +bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + + +/************************************** + * Functions for manipulating streams * + **************************************/ + +/* Create an input stream for reading from a memory buffer. + * + * Alternatively, you can use a custom stream that reads directly from e.g. + * a file or a network socket. + */ +pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); + +/* Function to read from a pb_istream_t. You can use this if you need to + * read some custom header data, or to read data in field callbacks. + */ +bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); + +/* Structure for defining custom input streams. You will need to provide + * a callback function to read the bytes from your storage, which can be + * for example a file or a network socket. * - * Rules for callback: + * The callback must conform to these rules: + * * 1) Return false on IO errors. This will cause decoding to abort. - * * 2) You can use state to store your own data (e.g. buffer pointer), - * and rely on pb_read to verify that no-body reads past bytes_left. - * + * and rely on pb_read to verify that no-body reads past bytes_left. * 3) Your callback may be used with substreams, in which case bytes_left - * is different than from the main stream. Don't use bytes_left to compute - * any pointers. + * is different than from the main stream. Don't use bytes_left to compute + * any pointers. */ struct _pb_istream_t { @@ -49,24 +93,10 @@ struct _pb_istream_t #endif }; -pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); -bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); - -/* Decode from stream to destination struct. - * Returns true on success, false on any failure. - * The actual struct pointed to by dest must match the description in fields. - */ -bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); - -/* Same as pb_decode, except does not initialize the destination structure - * to default values. This is slightly faster if you need no default values - * and just do memset(struct, 0, sizeof(struct)) yourself. - */ -bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); -/* --- Helper functions --- - * You may want to use these from your caller or callbacks. - */ +/************************************************ + * Helper functions for writing field callbacks * + ************************************************/ /* Decode the tag for the next field in the stream. Gives the wire type and * field tag. At end of the message, returns false and sets eof to true. */ @@ -95,10 +125,10 @@ bool pb_decode_fixed64(pb_istream_t *stream, void *dest); bool pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream); void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream); -/* --- Internal functions --- - * These functions are not terribly useful for the average library user, but - * are exported to make the unit testing and extending nanopb easier. - */ + +/******************************* + * Internal / legacy functions * + *******************************/ #ifdef NANOPB_INTERNALS bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest); diff --git a/pb_encode.h b/pb_encode.h index 773b7178..d9e0336a 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -1,33 +1,82 @@ -#ifndef _PB_ENCODE_H_ -#define _PB_ENCODE_H_ - /* pb_encode.h: Functions to encode protocol buffers. Depends on pb_encode.c. - * The main function is pb_encode. You also need an output stream, structures - * and their field descriptions (just like with pb_decode). + * The main function is pb_encode. You also need an output stream, and the + * field descriptions created by nanopb_generator.py. */ +#ifndef _PB_ENCODE_H_ +#define _PB_ENCODE_H_ + #include "pb.h" #ifdef __cplusplus extern "C" { #endif -/* Lightweight output stream. - * You can provide callback for writing or use pb_ostream_from_buffer. +/*************************** + * Main encoding functions * + ***************************/ + +/* Encode a single protocol buffers message from C structure into a stream. + * Returns true on success, false on any failure. + * The actual struct pointed to by src_struct must match the description in fields. + * All required fields in the struct are assumed to have been filled in. + * + * Example usage: + * MyMessage msg = {}; + * uint8_t buffer[64]; + * pb_ostream_t stream; + * + * msg.field1 = 42; + * stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + * pb_encode(&stream, MyMessage_fields, &msg); + */ +bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); + + +/************************************** + * Functions for manipulating streams * + **************************************/ + +/* Create an output stream for writing into a memory buffer. + * The number of bytes written can be found in stream.bytes_written after + * encoding the message. + * + * Alternatively, you can use a custom stream that writes directly to e.g. + * a file or a network socket. + */ +pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize); + +/* Pseudo-stream for measuring the size of a message without actually storing + * the encoded data. * - * Alternatively, callback can be NULL in which case the stream will just - * count the number of bytes that would have been written. In this case - * max_size is not checked. + * Example usage: + * MyMessage msg = {}; + * pb_ostream_t stream = PB_OSTREAM_SIZING; + * pb_encode(&stream, MyMessage_fields, &msg); + * printf("Message size is %d\n", stream.bytes_written); + */ +#ifndef PB_NO_ERRMSG +#define PB_OSTREAM_SIZING {0,0,0,0,0} +#else +#define PB_OSTREAM_SIZING {0,0,0,0} +#endif + +/* Function to write into a pb_ostream_t stream. You can use this if you need + * to append or prepend some custom headers to the message. + */ +bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); + +/* Structure for defining custom output streams. You will need to provide + * a callback function to write the bytes to your storage, which can be + * for example a file or a network socket. + * + * The callback must conform to these rules: * - * Rules for callback: * 1) Return false on IO errors. This will cause encoding to abort. - * * 2) You can use state to store your own data (e.g. buffer pointer). - * * 3) pb_write will update bytes_written after your callback runs. - * - * 4) Substreams will modify max_size and bytes_written. Don't use them to - * calculate any pointers. + * 4) Substreams will modify max_size and bytes_written. Don't use them + * to calculate any pointers. */ struct _pb_ostream_t { @@ -42,42 +91,26 @@ struct _pb_ostream_t #else bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); #endif - void *state; /* Free field for use by callback implementation */ - size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ - size_t bytes_written; + void *state; /* Free field for use by callback implementation. */ + size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ + size_t bytes_written; /* Number of bytes written so far. */ #ifndef PB_NO_ERRMSG const char *errmsg; #endif }; -pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize); -bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); - -/* Stream type for use in computing message sizes */ -#ifndef PB_NO_ERRMSG -#define PB_OSTREAM_SIZING {0,0,0,0,0} -#else -#define PB_OSTREAM_SIZING {0,0,0,0} -#endif - -/* Encode struct to given output stream. - * Returns true on success, false on any failure. - * The actual struct pointed to by src_struct must match the description in fields. - * All required fields in the struct are assumed to have been filled in. - */ -bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); -/* --- Helper functions --- - * You may want to use these from your caller or callbacks. - */ +/************************************************ + * Helper functions for writing field callbacks * + ************************************************/ -/* Encode field header based on LTYPE and field number defined in the field structure. - * Call this from the callback before writing out field contents. */ +/* Encode field header based on type and field number defined in the field + * structure. Call this from the callback before writing out field contents. */ bool pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t *field); -/* Encode field header by manually specifing wire type. You need to use this if - * you want to write out packed arrays from a callback field. */ +/* Encode field header by manually specifing wire type. You need to use this + * if you want to write out packed arrays from a callback field. */ bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, uint32_t field_number); /* Encode an integer in the varint format. @@ -100,15 +133,16 @@ bool pb_encode_fixed32(pb_ostream_t *stream, const void *value); bool pb_encode_fixed64(pb_ostream_t *stream, const void *value); /* Encode a submessage field. - * You need to pass the pb_field_t array and pointer to struct, just like with pb_encode(). - * This internally encodes the submessage twice, first to calculate message size and then to actually write it out. + * You need to pass the pb_field_t array and pointer to struct, just like + * with pb_encode(). This internally encodes the submessage twice, first to + * calculate message size and then to actually write it out. */ bool pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); -/* --- Internal functions --- - * These functions are not terribly useful for the average library user, but - * are exported to make the unit testing and extending nanopb easier. - */ + +/******************************* + * Internal / legacy functions * + *******************************/ #ifdef NANOPB_INTERNALS bool pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src); -- cgit 1.2.3-korg From bb985e99274b537ee662d630b02664a3825d8829 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 6 Jul 2013 16:16:00 +0300 Subject: Add pb_decode_delimited and pb_encode_delimited wrapper functions. Update issue 74 Status: FixedInGit --- pb_decode.c | 13 +++++++++++++ pb_decode.h | 6 ++++++ pb_encode.c | 5 +++++ pb_encode.h | 4 ++++ tests/decode_unittests.c | 10 ++++++++++ tests/encode_unittests.c | 10 ++++++++++ 6 files changed, 48 insertions(+) diff --git a/pb_decode.c b/pb_decode.c index d0e18ccb..c5336981 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -603,6 +603,19 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void return pb_decode_noinit(stream, fields, dest_struct); } +bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + pb_istream_t substream; + bool status; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + status = pb_decode(&substream, fields, dest_struct); + pb_close_string_substream(stream, &substream); + return status; +} + /* Field decoders */ bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest) diff --git a/pb_decode.h b/pb_decode.h index 85efa212..3da3f763 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -43,6 +43,12 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc */ bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); +/* Same as pb_decode, except expects the stream to start with the message size + * encoded as varint. Corresponds to parseDelimitedFrom() in Google's + * protobuf API. + */ +bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + /************************************** * Functions for manipulating streams * diff --git a/pb_encode.c b/pb_encode.c index 0e048ac6..bedfed46 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -253,6 +253,11 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons return true; } +bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) +{ + return pb_encode_submessage(stream, fields, src_struct); +} + /* Helper functions */ bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value) { diff --git a/pb_encode.h b/pb_encode.h index d9e0336a..04bdabee 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -32,6 +32,10 @@ extern "C" { */ bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); +/* Same as pb_encode, but prepends the length of the message as a varint. + * Corresponds to writeDelimitedTo() in Google's protobuf API. + */ +bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); /************************************** * Functions for manipulating streams * diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c index 1e74c34d..6ad05f00 100644 --- a/tests/decode_unittests.c +++ b/tests/decode_unittests.c @@ -289,6 +289,16 @@ int main() TEST((s = S("\x08"), !pb_decode(&s, IntegerArray_fields, &dest))) } + { + pb_istream_t s; + IntegerContainer dest = {}; + + COMMENT("Testing pb_decode_delimited") + TEST((s = S("\x09\x0A\x07\x0A\x05\x01\x02\x03\x04\x05"), + pb_decode_delimited(&s, IntegerContainer_fields, &dest)) && + dest.submsg.data_count == 5) + } + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); diff --git a/tests/encode_unittests.c b/tests/encode_unittests.c index 6a8f5e9e..c3634ac8 100644 --- a/tests/encode_unittests.c +++ b/tests/encode_unittests.c @@ -244,6 +244,16 @@ int main() "\x0A\x07\x0A\x05\x01\x02\x03\x04\x05")) } + { + uint8_t buffer[20]; + pb_ostream_t s; + IntegerContainer msg = {{5, {1,2,3,4,5}}}; + + COMMENT("Test pb_encode_delimited.") + TEST(WRITES(pb_encode_delimited(&s, IntegerContainer_fields, &msg), + "\x09\x0A\x07\x0A\x05\x01\x02\x03\x04\x05")) + } + { uint8_t buffer[10]; pb_ostream_t s; -- cgit 1.2.3-korg From a01856f4ab7f51f129dda7387acca8a63d039105 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 16 Jul 2013 11:07:34 +0300 Subject: Add test case for extra fields in AllTypes --- tests/Makefile | 1 + tests/alltypes_with_extra_fields.pb | Bin 0 -> 523 bytes 2 files changed, 1 insertion(+) create mode 100644 tests/alltypes_with_extra_fields.pb diff --git a/tests/Makefile b/tests/Makefile index 40fcabe0..16f62e5c 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -123,6 +123,7 @@ run_unittests: $(TESTS) ./test_encode3 1 | ./test_decode3 1 ./test_encode3 1 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null ./test_encode3_buf 1 | ./test_decode3_buf 1 + ./test_decode3 < alltypes_with_extra_fields.pb ./bc_encode | ./bc_decode ./test_missing_fields diff --git a/tests/alltypes_with_extra_fields.pb b/tests/alltypes_with_extra_fields.pb new file mode 100644 index 00000000..f9f53941 Binary files /dev/null and b/tests/alltypes_with_extra_fields.pb differ -- cgit 1.2.3-korg From 3c10e6fa71cbee85c540c0702a5a95333cd3be32 Mon Sep 17 00:00:00 2001 From: Kent Ryhorchuk Date: Mon, 15 Jul 2013 18:04:47 -0700 Subject: Check for empty message type before incrementing required_field_index. If you have a message that defined as empty, but attempt to decode a message that has one or more unknown fields then pb_decode fails. The method used to count the number of required fields counts 1 required field because the default type of PB_LAST_FIELD is PB_HTYPE_REQUIRED. --- pb_decode.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index c5336981..a079556e 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -308,12 +308,12 @@ static bool pb_field_next(pb_field_iterator_t *iter) prev_size *= iter->pos->array_size; } - if (PB_HTYPE(iter->pos->type) == PB_HTYPE_REQUIRED) - iter->required_field_index++; - if (iter->pos->tag == 0) return false; /* Only happens with empty message types */ + if (PB_HTYPE(iter->pos->type) == PB_HTYPE_REQUIRED) + iter->required_field_index++; + iter->pos++; iter->field_index++; if (iter->pos->tag == 0) -- cgit 1.2.3-korg From f064c2c48a43b0c1b011d52f53d519d809b8cee7 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 16 Jul 2013 11:31:38 +0300 Subject: Fix formatting in documentation --- docs/reference.rst | 64 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 33884885..42f4864f 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -65,16 +65,16 @@ The generator behaviour can be adjusted using these options, defined in the 'nanopb.proto' file in the generator folder: ============================ ================================================ -max_size Allocated size for 'bytes' and 'string' fields. +max_size Allocated size for *bytes* and *string* fields. max_count Allocated number of entries in arrays - ('repeated' fields). + (*repeated* fields). type Type of the generated field. Default value - is FT_DEFAULT, which selects automatically. - You can use FT_CALLBACK, FT_STATIC or FT_IGNORE - to force a callback field, a static field or - to completely ignore the field. + is *FT_DEFAULT*, which selects automatically. + You can use *FT_CALLBACK*, *FT_STATIC* or + *FT_IGNORE* to force a callback field, a static + field or to completely ignore the field. long_names Prefix the enum name to the enum value in - definitions, i.e. 'EnumName_EnumValue'. Enabled + definitions, i.e. *EnumName_EnumValue*. Enabled by default. packed_struct Make the generated structures packed. NOTE: This cannot be used on CPUs that break @@ -101,44 +101,62 @@ statically allocate them. Defining the options in a .options file --------------------------------------- The preferred way to define options is to have a separate file -'myproto.options' in the same directory as the 'myproto.proto'. The -generator will automatically search for this file and read the options from -it. The file format is as follows: +'myproto.options' in the same directory as the 'myproto.proto'. :: + + # myproto.proto + message MyMessage { + required string name = 1; + repeated int32 ids = 4; + } + +:: + + # myproto.options + MyMessage.name max_size:40 + MyMessage.ids max_count:5 + +The generator will automatically search for this file and read the +options from it. The file format is as follows: * Lines starting with '#' or '//' are regarded as comments. * Blank lines are ignored. * All other lines should start with a field name pattern, followed by one or more options. For example: *"MyMessage.myfield max_size:5 max_count:10"*. -* The field name pattern is matched against a string of form 'Message.field'. - For nested messages, the string is 'Message.SubMessage.field'. +* The field name pattern is matched against a string of form *'Message.field'*. + For nested messages, the string is *'Message.SubMessage.field'*. * The field name pattern may use the notation recognized by Python fnmatch(): - - \* matches any part of string, like 'Message.\*' for all fields - - \? matches any single character - - [seq] matches any of characters 's', 'e' and 'q' - - [!seq] matches any other character -* The options are written as 'option_name:option_value' and several options + + - *\** matches any part of string, like 'Message.\*' for all fields + - *\?* matches any single character + - *[seq]* matches any of characters 's', 'e' and 'q' + - *[!seq]* matches any other character + +* The options are written as *'option_name:option_value'* and several options can be defined on same line, separated by whitespace. * Options defined later in the file override the ones specified earlier, so it makes sense to define wildcard options first in the file and more specific ones later. If preferred, the name of the options file can be set using the command line -switch '-f' to nanopb_generator.py. +switch *-f* to nanopb_generator.py. Defining the options on command line ------------------------------------ -The nanopb_generator.py has a simple command line option '-s OPTION:VALUE'. +The nanopb_generator.py has a simple command line option *-s OPTION:VALUE*. The setting applies to the whole file that is being processed. Defining the options in the .proto file --------------------------------------- The .proto file format allows defining custom options for the fields. -The nanopb library comes with 'nanopb.proto' which does exactly that, allowing -you do define the options directly in the .proto file: +The nanopb library comes with *nanopb.proto* which does exactly that, allowing +you do define the options directly in the .proto file:: import "nanopb.proto"; - required string name = 1 [(nanopb).max_size = 40]; - repeated PhoneNumber phone = 4 [(nanopb).max_count = 5]; + + message MyMessage { + required string name = 1 [(nanopb).max_size = 40]; + repeated int32 ids = 4 [(nanopb).max_count = 5]; + } A small complication is that you have to set the include path of protoc so that nanopb.proto can be found. This file, in turn, requires the file -- cgit 1.2.3-korg From 7c5e184c261bd5f5652993232f2125a6802004ab Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 17 Jul 2013 00:06:54 +0300 Subject: Implement generator support for extension fields (no encoder/decoder support yet) --- generator/nanopb_generator.py | 97 ++++++++++++++++++++++++++++++++++++------ pb.h | 55 +++++++++++++++++++++++- tests/Makefile | 5 ++- tests/alltypes.proto | 3 ++ tests/extensions.proto | 5 +++ tests/test_encode_extensions.c | 34 +++++++++++++++ 6 files changed, 183 insertions(+), 16 deletions(-) create mode 100644 tests/extensions.proto create mode 100644 tests/test_encode_extensions.c diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index bd365dfd..61f4d7b3 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -275,8 +275,53 @@ class Field: return max(self.tag, self.max_size, self.max_count) +class ExtensionRange(Field): + def __init__(self, struct_name, desc, field_options): + '''desc is ExtensionRange''' + self.tag = desc.start + self.struct_name = struct_name + self.name = 'extensions' + self.pbtype = 'EXTENSION' + self.rules = 'OPTIONAL' + self.allocation = 'CALLBACK' + self.ctype = 'pb_extension_t' + self.array_decl = '' + self.default = None + self.max_size = 0 + self.max_count = 0 + + def __str__(self): + return ' pb_extension_t *extensions;' + + def types(self): + return None + + def tags(self): + return '' - +class ExtensionField(Field): + def __init__(self, struct_name, desc, field_options): + self.fullname = struct_name + desc.name + self.extendee_name = names_from_type_name(desc.extendee) + Field.__init__(self, self.fullname + 'struct', desc, field_options) + + def extension_decl(self): + '''Declaration of the extension type in the .pb.h file''' + return 'extern const pb_extension_type_t %s;' % self.fullname + + def extension_def(self): + '''Definition of the extension type in the .pb.c file''' + result = 'typedef struct {\n' + result += str(self) + result += '} %s;\n' % self.struct_name + result += ('static const pb_field_t %s_field = %s;\n' % + (self.fullname, self.pb_field_t(None))) + result += 'const pb_extension_type_t %s = {\n' % self.fullname + result += ' NULL,\n' + result += ' NULL,\n' + result += ' &%s_field\n' % self.fullname + result += '};\n' + return result # --------------------------------------------------------------------------- @@ -294,6 +339,11 @@ class Message: if field_options.type != nanopb_pb2.FT_IGNORE: self.fields.append(Field(self.name, f, field_options)) + if len(desc.extension_range) > 0: + field_options = get_nanopb_suboptions(desc, message_options, self.name + 'extensions') + if field_options.type != nanopb_pb2.FT_IGNORE: + self.fields.append(ExtensionRange(self.name, desc.extension_range[0], field_options)) + self.packed = message_options.packed_struct self.ordered_fields = self.fields[:] self.ordered_fields.sort() @@ -358,9 +408,6 @@ class Message: - - - # --------------------------------------------------------------------------- # Processing of entire .proto files # --------------------------------------------------------------------------- @@ -380,11 +427,23 @@ def iterate_messages(desc, names = Names()): for x in iterate_messages(submsg, sub_names): yield x +def iterate_extensions(desc, names = Names()): + '''Recursively find all extensions. + For each, yield name, FieldDescriptorProto. + ''' + for extension in desc.extension: + yield names, extension + + for subname, subdesc in iterate_messages(desc, names): + for extension in subdesc.extension: + yield subname, extension + def parse_file(fdesc, file_options): - '''Takes a FileDescriptorProto and returns tuple (enum, messages).''' + '''Takes a FileDescriptorProto and returns tuple (enums, messages, extensions).''' enums = [] messages = [] + extensions = [] if fdesc.package: base_name = Names(fdesc.package.split('.')) @@ -402,6 +461,10 @@ def parse_file(fdesc, file_options): enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name) enums.append(Enum(names, enum, enum_options)) + for names, extension in iterate_extensions(fdesc, base_name): + field_options = get_nanopb_suboptions(extension, file_options, names) + extensions.append(ExtensionField(names, extension, field_options)) + # Fix field default values where enum short names are used. for enum in enums: if not enum.options.long_names: @@ -411,7 +474,7 @@ def parse_file(fdesc, file_options): idx = enum.value_longnames.index(field.default) field.default = enum.values[idx][0] - return enums, messages + return enums, messages, extensions def toposort2(data): '''Topological sort. @@ -454,7 +517,7 @@ def make_identifier(headername): result += '_' return result -def generate_header(dependencies, headername, enums, messages, options): +def generate_header(dependencies, headername, enums, messages, extensions, options): '''Generate content for a header file. Generates strings, which should be concatenated and stored to file. ''' @@ -489,6 +552,12 @@ def generate_header(dependencies, headername, enums, messages, options): for msg in sort_dependencies(messages): yield msg.types() yield str(msg) + '\n\n' + + if extensions: + yield '/* Extensions */\n' + for extension in extensions: + yield extension.extension_decl() + yield '\n' yield '/* Default values for struct fields */\n' for msg in messages: @@ -512,7 +581,7 @@ def generate_header(dependencies, headername, enums, messages, options): # End of header yield '\n#endif\n' -def generate_source(headername, enums, messages): +def generate_source(headername, enums, messages, extensions): '''Generate content for a source file.''' yield '/* Automatically generated nanopb constant definitions */\n' @@ -527,7 +596,11 @@ def generate_source(headername, enums, messages): for msg in messages: yield msg.fields_definition() + '\n\n' + + for ext in extensions: + yield ext.extension_def() + '\n' + # Add checks for numeric limits if messages: count_required_fields = lambda m: len([f for f in msg.fields if f.rules == 'REQUIRED']) largest_msg = max(messages, key = count_required_fields) @@ -539,7 +612,6 @@ def generate_source(headername, enums, messages): yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count yield '#endif\n' - # Add checks for numeric limits worst = 0 worst_field = '' checks = [] @@ -724,7 +796,7 @@ def process(filenames, options): # Parse the file file_options = get_nanopb_suboptions(fdesc.file[0], toplevel_options, Names([filename])) - enums, messages = parse_file(fdesc.file[0], file_options) + enums, messages, extensions = parse_file(fdesc.file[0], file_options) noext = os.path.splitext(filename)[0] headername = noext + '.' + options.extension + '.h' @@ -740,11 +812,12 @@ def process(filenames, options): dependencies = [d for d in fdesc.file[0].dependency if d not in excludes] header = open(headername, 'w') - for part in generate_header(dependencies, headerbasename, enums, messages, options): + for part in generate_header(dependencies, headerbasename, enums, + messages, extensions, options): header.write(part) source = open(sourcename, 'w') - for part in generate_source(headerbasename, enums, messages): + for part in generate_source(headerbasename, enums, messages, extensions): source.write(part) return True diff --git a/pb.h b/pb.h index d5047ac5..15c346d8 100644 --- a/pb.h +++ b/pb.h @@ -148,8 +148,12 @@ typedef uint8_t pb_type_t; * submsg_fields is pointer to field descriptions */ #define PB_LTYPE_SUBMESSAGE 0x06 +/* Extension pseudo-field + * The field contains a pointer to pb_extension_t */ +#define PB_LTYPE_EXTENSION 0x07 + /* Number of declared LTYPES */ -#define PB_LTYPES_COUNT 7 +#define PB_LTYPES_COUNT 8 #define PB_LTYPE_MASK 0x0F /**** Field repetition rules ****/ @@ -270,6 +274,51 @@ typedef enum { PB_WT_32BIT = 5 } pb_wire_type_t; +/* Structure for defining the handling of unknown/extension fields. + * Usually the pb_extension_type_t structure is automatically generated, + * while the pb_extension_t structure is created by the user. However, + * if you want to catch all unknown fields, you can also create a custom + * pb_extension_type_t with your own callback. + */ +typedef struct _pb_extension_type_t pb_extension_type_t; +typedef struct _pb_extension_t pb_extension_t; +struct _pb_extension_type_t { + /* Called for each unknown field in the message. + * If you handle the field, read off all of its data and return true. + * If you do not handle the field, do not read anything and return true. + * If you run into an error, return false. + * Set to NULL for default handler. + */ + bool (*decode)(pb_istream_t *stream, pb_extension_t *extension, + uint32_t tag, pb_wire_type_t wire_type); + + /* Called once after all regular fields have been encoded. + * If you have something to write, do so and return true. + * If you do not have anything to write, just return true. + * If you run into an error, return false. + * Set to NULL for default handler. + */ + bool (*encode)(pb_ostream_t *stream, pb_extension_t *extension); + + /* Free field for use by the callback. */ + const void *arg; +}; + +struct _pb_extension_t { + /* Type describing the extension field. Usually you'll initialize + * this to a pointer to the automatically generated structure. */ + const pb_extension_type_t *type; + + /* Destination for the decoded data. This must match the datatype + * of the extension field. */ + void *dest; + + /* Pointer to the next extension handler, or NULL. + * If this extension does not match a field, the next handler is + * automatically called. */ + pb_extension_t *next; +}; + /* These macros are used to declare pb_field_t's in the constant array. */ #define pb_membersize(st, m) (sizeof ((st*)0)->m) #define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0])) @@ -333,13 +382,14 @@ typedef enum { #define PB_LTYPE_MAP_STRING PB_LTYPE_STRING #define PB_LTYPE_MAP_UINT32 PB_LTYPE_VARINT #define PB_LTYPE_MAP_UINT64 PB_LTYPE_VARINT +#define PB_LTYPE_MAP_EXTENSION PB_LTYPE_EXTENSION /* This is the actual macro used in field descriptions. * It takes these arguments: * - Field tag number * - Field type: BOOL, BYTES, DOUBLE, ENUM, FIXED32, FIXED64, * FLOAT, INT32, INT64, MESSAGE, SFIXED32, SFIXED64 - * SINT32, SINT64, STRING, UINT32 or UINT64 + * SINT32, SINT64, STRING, UINT32, UINT64 or EXTENSION * - Field rules: REQUIRED, OPTIONAL or REPEATED * - Allocation: STATIC or CALLBACK * - Message name @@ -352,6 +402,7 @@ typedef enum { PB_ ## rules ## _ ## allocation(tag, message, field, prevfield, \ PB_LTYPE_MAP_ ## type, ptr) + /* These macros are used for giving out error messages. * They are mostly a debugging aid; the main error information * is the true/false return value from functions. diff --git a/tests/Makefile b/tests/Makefile index 16f62e5c..dc564c28 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -8,7 +8,7 @@ TESTS= decode_unittests encode_unittests \ test_decode_callbacks test_encode_callbacks \ test_missing_fields test_no_messages test_funny_name \ test_multiple_files test_cxxcompile test_options \ - bc_encode bc_decode + bc_encode bc_decode test_encode_extensions # More strict checks for the core part of nanopb CC_VERSION=$(shell $(CC) -v 2>&1) @@ -84,6 +84,7 @@ test_no_messages: no_messages.pb.h no_messages.pb.c no_messages.pb.o test_funny_name: funny-proto+name.pb.h funny-proto+name.pb.o bc_encode: bc_alltypes.pb.o pb_encode.o bc_encode.o bc_decode: bc_alltypes.pb.o pb_decode.o bc_decode.o +test_encode_extensions: test_encode_extensions.c pb_encode.o alltypes.pb.o extensions.pb.o %.pb: %.proto protoc -I. -I../generator -I/usr/include -o$@ $< @@ -125,7 +126,7 @@ run_unittests: $(TESTS) ./test_encode3_buf 1 | ./test_decode3_buf 1 ./test_decode3 < alltypes_with_extra_fields.pb ./bc_encode | ./bc_decode - + ./test_missing_fields test_options: options.pb.h options.expected options.pb.o diff --git a/tests/alltypes.proto b/tests/alltypes.proto index 6ccf57d7..a2cf8bbc 100644 --- a/tests/alltypes.proto +++ b/tests/alltypes.proto @@ -86,5 +86,8 @@ message AllTypes { // Just to make sure that the size of the fields has been calculated // properly, i.e. otherwise a bug in last field might not be detected. required int32 end = 99; + + + extensions 200 to 255; } diff --git a/tests/extensions.proto b/tests/extensions.proto new file mode 100644 index 00000000..12c7c9a5 --- /dev/null +++ b/tests/extensions.proto @@ -0,0 +1,5 @@ +import 'alltypes.proto'; + +extend AllTypes { + optional int32 AllTypes_extensionfield1 = 255; +} diff --git a/tests/test_encode_extensions.c b/tests/test_encode_extensions.c new file mode 100644 index 00000000..a3500d7d --- /dev/null +++ b/tests/test_encode_extensions.c @@ -0,0 +1,34 @@ +/* Tests extension fields. + */ + +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "extensions.pb.h" + +int main(int argc, char **argv) +{ + AllTypes alltypes = {0}; + int32_t extensionfield1 = 12345; + pb_extension_t ext1 = {&AllTypes_extensionfield1, &extensionfield1, NULL}; + + alltypes.extensions = &ext1; + + uint8_t buffer[1024]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } +} + -- cgit 1.2.3-korg From ebddda9b5cdda5d082f07bbb6beb010f48c8f9b3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 17 Jul 2013 19:23:19 +0300 Subject: Implement extension support for the encoder --- pb.h | 2 +- pb_encode.c | 94 +++++++++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 74 insertions(+), 22 deletions(-) diff --git a/pb.h b/pb.h index 15c346d8..da38d7b9 100644 --- a/pb.h +++ b/pb.h @@ -298,7 +298,7 @@ struct _pb_extension_type_t { * If you run into an error, return false. * Set to NULL for default handler. */ - bool (*encode)(pb_ostream_t *stream, pb_extension_t *extension); + bool (*encode)(pb_ostream_t *stream, const pb_extension_t *extension); /* Free field for use by the callback. */ const void *arg; diff --git a/pb_encode.c b/pb_encode.c index bedfed46..f3c62a16 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -83,10 +83,7 @@ bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count /* Main encoding stuff */ -/* Callbacks don't need this function because they usually know the data type - * without examining the field structure. - * Therefore it is static for now. - */ +/* Encode a static array. Handles the size calculations and possible packing. */ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, const void *pData, size_t count, pb_encoder_t func) { @@ -97,6 +94,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie if (count == 0) return true; + /* We always pack arrays if the datatype allows it. */ if (PB_LTYPE(field->type) <= PB_LTYPE_LAST_PACKABLE) { if (!pb_encode_tag(stream, PB_WT_STRING, field->tag)) @@ -112,7 +110,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie size = 8 * count; } else - { + { pb_ostream_t sizestream = PB_OSTREAM_SIZING; p = pData; for (i = 0; i < count; i++) @@ -155,7 +153,10 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie return true; } -bool checkreturn encode_static_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData) +/* Encode a field with static allocation, i.e. one whose data is stored + * in the structure itself. */ +static bool checkreturn encode_static_field(pb_ostream_t *stream, + const pb_field_t *field, const void *pData) { pb_encoder_t func; const void *pSize; @@ -195,7 +196,10 @@ bool checkreturn encode_static_field(pb_ostream_t *stream, const pb_field_t *fie return true; } -bool checkreturn encode_callback_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData) +/* Encode a field with callback semantics. This means that a user function is + * called to provide and encode the actual data. */ +static bool checkreturn encode_callback_field(pb_ostream_t *stream, + const pb_field_t *field, const void *pData) { const pb_callback_t *callback = (const pb_callback_t*)pData; @@ -213,6 +217,57 @@ bool checkreturn encode_callback_field(pb_ostream_t *stream, const pb_field_t *f return true; } +/* Encode a single field of any callback or static type. */ +static bool checkreturn encode_field(pb_ostream_t *stream, + const pb_field_t *field, const void *pData) +{ + switch (PB_ATYPE(field->type)) + { + case PB_ATYPE_STATIC: + return encode_static_field(stream, field, pData); + + case PB_ATYPE_CALLBACK: + return encode_callback_field(stream, field, pData); + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } +} + +/* Default handler for extension fields. Expects to have a pb_field_t + * pointer in the extension->type->arg field. */ +static bool checkreturn default_extension_handler(pb_ostream_t *stream, + const pb_extension_t *extension) +{ + const pb_field_t *field = (const pb_field_t*)extension->type->arg; + return encode_field(stream, field, extension->dest); +} + +/* Walk through all the registered extensions and give them a chance + * to encode themselves. */ +static bool checkreturn encode_extension_field(pb_ostream_t *stream, + const pb_field_t *field, const void *pData) +{ + const pb_extension_t *extension = *(const pb_extension_t* const *)pData; + UNUSED(field); + + while (extension) + { + bool status; + if (extension->type->encode) + status = extension->type->encode(stream, extension); + else + status = default_extension_handler(stream, extension); + + if (!status) + return false; + + extension = extension->next; + } + + return true; +} + bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) { const pb_field_t *field = fields; @@ -230,21 +285,18 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons { prev_size *= field->array_size; } - - switch (PB_ATYPE(field->type)) + + if (PB_LTYPE(field->type) == PB_LTYPE_EXTENSION) + { + /* Special case for the extension field placeholder */ + if (!encode_extension_field(stream, field, pData)) + return false; + } + else { - case PB_ATYPE_STATIC: - if (!encode_static_field(stream, field, pData)) - return false; - break; - - case PB_ATYPE_CALLBACK: - if (!encode_callback_field(stream, field, pData)) - return false; - break; - - default: - PB_RETURN_ERROR(stream, "invalid field type"); + /* Regular field */ + if (!encode_field(stream, field, pData)) + return false; } field++; -- cgit 1.2.3-korg From 0343e2bcfe97748e8b1b93ecacb2e72fc0be7453 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 17 Jul 2013 19:29:06 +0300 Subject: Add test case for extensions decoding --- tests/Makefile | 4 +++- tests/test_decode_extensions.c | 35 +++++++++++++++++++++++++++++++++++ tests/test_encode_extensions.c | 2 +- 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 tests/test_decode_extensions.c diff --git a/tests/Makefile b/tests/Makefile index dc564c28..9696b794 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -8,7 +8,7 @@ TESTS= decode_unittests encode_unittests \ test_decode_callbacks test_encode_callbacks \ test_missing_fields test_no_messages test_funny_name \ test_multiple_files test_cxxcompile test_options \ - bc_encode bc_decode test_encode_extensions + bc_encode bc_decode test_encode_extensions test_decode_extensions # More strict checks for the core part of nanopb CC_VERSION=$(shell $(CC) -v 2>&1) @@ -85,6 +85,7 @@ test_funny_name: funny-proto+name.pb.h funny-proto+name.pb.o bc_encode: bc_alltypes.pb.o pb_encode.o bc_encode.o bc_decode: bc_alltypes.pb.o pb_decode.o bc_decode.o test_encode_extensions: test_encode_extensions.c pb_encode.o alltypes.pb.o extensions.pb.o +test_decode_extensions: test_decode_extensions.c pb_decode.o alltypes.pb.o extensions.pb.o %.pb: %.proto protoc -I. -I../generator -I/usr/include -o$@ $< @@ -126,6 +127,7 @@ run_unittests: $(TESTS) ./test_encode3_buf 1 | ./test_decode3_buf 1 ./test_decode3 < alltypes_with_extra_fields.pb ./bc_encode | ./bc_decode + ./test_encode_extensions | ./test_decode_extensions ./test_missing_fields diff --git a/tests/test_decode_extensions.c b/tests/test_decode_extensions.c new file mode 100644 index 00000000..b384c337 --- /dev/null +++ b/tests/test_decode_extensions.c @@ -0,0 +1,35 @@ +/* Test decoding of extension fields. */ + +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "extensions.pb.h" + +int main(int argc, char **argv) +{ + uint8_t buffer[1024]; + size_t count = fread(buffer, 1, sizeof(buffer), stdin); + pb_istream_t stream = pb_istream_from_buffer(buffer, count); + + AllTypes alltypes = {}; + int32_t extensionfield1; + pb_extension_t ext1 = {&AllTypes_extensionfield1, &extensionfield1, NULL}; + alltypes.extensions = &ext1; + + if (!pb_decode(&stream, AllTypes_fields, &alltypes)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + + if (extensionfield1 != 12345) + { + printf("Wrong value for extension field: %d\n", extensionfield1); + return 2; + } + + return 0; +} + diff --git a/tests/test_encode_extensions.c b/tests/test_encode_extensions.c index a3500d7d..c889dec6 100644 --- a/tests/test_encode_extensions.c +++ b/tests/test_encode_extensions.c @@ -10,7 +10,7 @@ int main(int argc, char **argv) { - AllTypes alltypes = {0}; + AllTypes alltypes = {}; int32_t extensionfield1 = 12345; pb_extension_t ext1 = {&AllTypes_extensionfield1, &extensionfield1, NULL}; -- cgit 1.2.3-korg From 64947cb382271e2ab17dbf40ab634846d7d30ad9 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 17 Jul 2013 20:21:51 +0300 Subject: Extension support implemented for decoder. Testing is still needed. Also only 'optional' extension fields are supported now, 'repeated' fields are not yet supported. --- generator/nanopb_generator.py | 17 ++++++-- pb_decode.c | 96 ++++++++++++++++++++++++++++++++++++++++++- pb_encode.c | 3 +- 3 files changed, 109 insertions(+), 7 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 61f4d7b3..3bac9a92 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -276,9 +276,13 @@ class Field: class ExtensionRange(Field): - def __init__(self, struct_name, desc, field_options): - '''desc is ExtensionRange''' - self.tag = desc.start + def __init__(self, struct_name, range_start, field_options): + '''Implements a special pb_extension_t* field in an extensible message + structure. The range_start signifies the index at which the extensions + start. Not necessarily all tags above this are extensions, it is merely + a speed optimization. + ''' + self.tag = range_start self.struct_name = struct_name self.name = 'extensions' self.pbtype = 'EXTENSION' @@ -304,6 +308,10 @@ class ExtensionField(Field): self.fullname = struct_name + desc.name self.extendee_name = names_from_type_name(desc.extendee) Field.__init__(self, self.fullname + 'struct', desc, field_options) + + if self.rules != 'OPTIONAL': + raise NotImplementedError("Only 'optional' is supported for extension fields. " + + "(%s.rules == %s)" % (self.fullname, self.rules)) def extension_decl(self): '''Declaration of the extension type in the .pb.h file''' @@ -341,8 +349,9 @@ class Message: if len(desc.extension_range) > 0: field_options = get_nanopb_suboptions(desc, message_options, self.name + 'extensions') + range_start = min([r.start for r in desc.extension_range]) if field_options.type != nanopb_pb2.FT_IGNORE: - self.fields.append(ExtensionRange(self.name, desc.extension_range[0], field_options)) + self.fields.append(ExtensionRange(self.name, range_start, field_options)) self.packed = message_options.packed_struct self.ordered_fields = self.fields[:] diff --git a/pb_decode.c b/pb_decode.c index a079556e..e3be4127 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -28,7 +28,8 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { &pb_dec_bytes, &pb_dec_string, - &pb_dec_submessage + &pb_dec_submessage, + NULL /* extensions */ }; /************** @@ -336,8 +337,11 @@ static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag) unsigned start = iter->field_index; do { - if (iter->pos->tag == tag) + if (iter->pos->tag == tag && + PB_LTYPE(iter->pos->type) != PB_LTYPE_EXTENSION) + { return true; + } pb_field_next(iter); } while (iter->field_index != start); @@ -472,6 +476,70 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t } } +/* Default handler for extension fields. Expects a pb_field_t structure + * in extension->type->arg. */ +static bool checkreturn default_extension_handler(pb_istream_t *stream, + pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type) +{ + const pb_field_t *field = (const pb_field_t*)extension->type->arg; + pb_field_iterator_t iter; + bool dummy; + + if (field->tag != tag) + return true; + + iter.start = field; + iter.pos = field; + iter.field_index = 0; + iter.required_field_index = 0; + iter.dest_struct = extension->dest; + iter.pData = extension->dest; + iter.pSize = &dummy; + + return decode_field(stream, wire_type, &iter); +} + +/* Try to decode an unknown field as an extension field. Tries each extension + * decoder in turn, until one of them handles the field or loop ends. */ +static bool checkreturn decode_extension(pb_istream_t *stream, + uint32_t tag, pb_wire_type_t wire_type, pb_field_iterator_t *iter) +{ + pb_extension_t *extension = *(pb_extension_t* const *)iter->pData; + size_t pos = stream->bytes_left; + + while (extension && pos == stream->bytes_left) + { + bool status; + if (extension->type->decode) + status = extension->type->decode(stream, extension, tag, wire_type); + else + status = default_extension_handler(stream, extension, tag, wire_type); + + if (!status) + return false; + + extension = extension->next; + } + + return true; +} + +/* Step through the iterator until an extension field is found or until all + * entries have been checked. There can be only one extension field per + * message. Returns false if no extension field is found. */ +static bool checkreturn find_extension_field(pb_field_iterator_t *iter) +{ + unsigned start = iter->field_index; + + do { + if (PB_LTYPE(iter->pos->type) == PB_LTYPE_EXTENSION) + return true; + pb_field_next(iter); + } while (iter->field_index != start); + + return false; +} + /* Initialize message fields to default values, recursively */ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct) { @@ -528,6 +596,7 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {0}; /* Used to check for required fields */ + uint32_t extension_range_start = 0; pb_field_iterator_t iter; pb_field_init(&iter, fields, dest_struct); @@ -548,6 +617,29 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ if (!pb_field_find(&iter, tag)) { + /* No match found, check if it matches an extension. */ + if (tag >= extension_range_start) + { + if (!find_extension_field(&iter)) + extension_range_start = (uint32_t)-1; + else + extension_range_start = iter.pos->tag; + + if (tag >= extension_range_start) + { + size_t pos = stream->bytes_left; + + if (!decode_extension(stream, tag, wire_type, &iter)) + return false; + + if (pos != stream->bytes_left) + { + /* The field was handled */ + continue; + } + } + } + /* No match found, skip data */ if (!pb_skip_field(stream, wire_type)) return false; diff --git a/pb_encode.c b/pb_encode.c index f3c62a16..58d76a7a 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -28,7 +28,8 @@ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { &pb_enc_bytes, &pb_enc_string, - &pb_enc_submessage + &pb_enc_submessage, + NULL /* extensions */ }; /* pb_ostream_t implementation */ -- cgit 1.2.3-korg From 1f13e8cd2c5bb091677f35e53ae5757774e7d1ba Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 22 Jul 2013 18:59:15 +0300 Subject: Fix bugs in extension support when multiple extension fields are present. --- generator/nanopb_generator.py | 9 ++++++--- pb.h | 11 +++++++++++ pb_encode.c | 7 ++++++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 3bac9a92..f39614b3 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -313,16 +313,19 @@ class ExtensionField(Field): raise NotImplementedError("Only 'optional' is supported for extension fields. " + "(%s.rules == %s)" % (self.fullname, self.rules)) + self.rules = 'OPTEXT' + def extension_decl(self): '''Declaration of the extension type in the .pb.h file''' - return 'extern const pb_extension_type_t %s;' % self.fullname + return 'extern const pb_extension_type_t %s;\n' % self.fullname def extension_def(self): '''Definition of the extension type in the .pb.c file''' + result = 'typedef struct {\n' result += str(self) - result += '} %s;\n' % self.struct_name - result += ('static const pb_field_t %s_field = %s;\n' % + result += '\n} %s;\n\n' % self.struct_name + result += ('static const pb_field_t %s_field = \n %s;\n\n' % (self.fullname, self.pb_field_t(None))) result += 'const pb_extension_type_t %s = {\n' % self.fullname result += ' NULL,\n' diff --git a/pb.h b/pb.h index da38d7b9..bb0a584d 100644 --- a/pb.h +++ b/pb.h @@ -364,6 +364,17 @@ struct _pb_extension_t { {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REPEATED | ltype, \ pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr} +/* Optional extensions don't have the has_ field, as that would be redundant. */ +#define PB_OPTEXT_STATIC(tag, st, m, pm, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \ + 0, \ + 0, \ + pb_membersize(st, m), 0, ptr} + +#define PB_OPTEXT_CALLBACK(tag, st, m, pm, ltype, ptr) \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ + 0, 0, pb_membersize(st, m), 0, ptr} + /* The mapping from protobuf types to LTYPEs is done using these macros. */ #define PB_LTYPE_MAP_BOOL PB_LTYPE_VARINT #define PB_LTYPE_MAP_BYTES PB_LTYPE_BYTES diff --git a/pb_encode.c b/pb_encode.c index 58d76a7a..d6ba7e34 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -161,9 +161,14 @@ static bool checkreturn encode_static_field(pb_ostream_t *stream, { pb_encoder_t func; const void *pSize; + bool dummy = true; func = PB_ENCODERS[PB_LTYPE(field->type)]; - pSize = (const char*)pData + field->size_offset; + + if (field->size_offset) + pSize = (const char*)pData + field->size_offset; + else + pSize = &dummy; switch (PB_HTYPE(field->type)) { -- cgit 1.2.3-korg From ba40cacd3c4a60ccc9108d6acf7e813b1757dfd1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 22 Jul 2013 19:00:04 +0300 Subject: Finish the testcase for extensions --- tests/extensions.options | 1 + tests/extensions.proto | 10 ++++++++++ tests/test_decode_extensions.c | 18 +++++++++++++----- tests/test_encode_extensions.c | 6 +++++- 4 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 tests/extensions.options diff --git a/tests/extensions.options b/tests/extensions.options new file mode 100644 index 00000000..a5cd61dd --- /dev/null +++ b/tests/extensions.options @@ -0,0 +1 @@ +* max_size:16 diff --git a/tests/extensions.proto b/tests/extensions.proto index 12c7c9a5..cbffdce6 100644 --- a/tests/extensions.proto +++ b/tests/extensions.proto @@ -3,3 +3,13 @@ import 'alltypes.proto'; extend AllTypes { optional int32 AllTypes_extensionfield1 = 255; } + +message ExtensionMessage { + extend AllTypes { + optional ExtensionMessage AllTypes_extensionfield2 = 254; + } + + required string test1 = 1; + required int32 test2 = 2; +} + diff --git a/tests/test_decode_extensions.c b/tests/test_decode_extensions.c index b384c337..ef6a0228 100644 --- a/tests/test_decode_extensions.c +++ b/tests/test_decode_extensions.c @@ -7,6 +7,11 @@ #include "alltypes.pb.h" #include "extensions.pb.h" +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed.\n"); \ + return 2; \ + } + int main(int argc, char **argv) { uint8_t buffer[1024]; @@ -14,9 +19,14 @@ int main(int argc, char **argv) pb_istream_t stream = pb_istream_from_buffer(buffer, count); AllTypes alltypes = {}; + int32_t extensionfield1; pb_extension_t ext1 = {&AllTypes_extensionfield1, &extensionfield1, NULL}; alltypes.extensions = &ext1; + + ExtensionMessage extensionfield2 = {}; + pb_extension_t ext2 = {&ExtensionMessage_AllTypes_extensionfield2, &extensionfield2, NULL}; + ext1.next = &ext2; if (!pb_decode(&stream, AllTypes_fields, &alltypes)) { @@ -24,11 +34,9 @@ int main(int argc, char **argv) return 1; } - if (extensionfield1 != 12345) - { - printf("Wrong value for extension field: %d\n", extensionfield1); - return 2; - } + TEST(extensionfield1 == 12345) + TEST(strcmp(extensionfield2.test1, "test") == 0) + TEST(extensionfield2.test2 == 54321) return 0; } diff --git a/tests/test_encode_extensions.c b/tests/test_encode_extensions.c index c889dec6..8857f148 100644 --- a/tests/test_encode_extensions.c +++ b/tests/test_encode_extensions.c @@ -11,10 +11,14 @@ int main(int argc, char **argv) { AllTypes alltypes = {}; + int32_t extensionfield1 = 12345; pb_extension_t ext1 = {&AllTypes_extensionfield1, &extensionfield1, NULL}; - alltypes.extensions = &ext1; + + ExtensionMessage extensionfield2 = {"test", 54321}; + pb_extension_t ext2 = {&ExtensionMessage_AllTypes_extensionfield2, &extensionfield2, NULL}; + ext1.next = &ext2; uint8_t buffer[1024]; pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); -- cgit 1.2.3-korg From b663909fb6e86f0ae0f450523e72fb7fbfb719ab Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 8 Aug 2013 20:05:30 +0300 Subject: Document PB_SYSTEM_HEADER --- docs/index.rst | 2 ++ docs/reference.rst | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 897f5529..cb7a201f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -103,6 +103,8 @@ Nanopb should compile with most ansi-C compatible compilers. It however requires If these header files do not come with your compiler, you should be able to find suitable replacements online. Mostly the requirements are very simple, just a few basic functions and typedefs. +Alternatively, you can define *PB_SYSTEM_HEADER*, which should be the name of a single header file including all the necessary definitions. + Debugging and testing ===================== Extensive unittests are included under the *tests* folder. Just type *make* there to run the tests. diff --git a/docs/reference.rst b/docs/reference.rst index 42f4864f..6c38f6b4 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -49,6 +49,11 @@ PB_BUFFER_ONLY Disables the support for custom streams. Only PB_OLD_CALLBACK_STYLE Use the old function signature (void\* instead of void\*\*) for callback fields. This was the default until nanopb-0.2.1. +PB_SYSTEM_HEADER Replace the standard header files with a single + header file. It should define all the required + functions and typedefs listed on the + `overview page`_. Value must include quotes, + for example *#define PB_SYSTEM_HEADER "foo.h"*. ============================ ================================================ The PB_MAX_REQUIRED_FIELDS, PB_FIELD_16BIT and PB_FIELD_32BIT settings allow @@ -56,7 +61,7 @@ raising some datatype limits to suit larger messages. Their need is recognized automatically by C-preprocessor #if-directives in the generated .pb.h files. The default setting is to use the smallest datatypes (least resources used). - +.. _`overview page`: index.html#compiler-requirements Proto file options -- cgit 1.2.3-korg From f15093e8bde18bb9fc6f56a7f6fff727eef74e6c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 8 Aug 2013 20:37:59 +0300 Subject: Document field extensions support Update issue 17 Status: FixedInGit --- docs/concepts.rst | 44 ++++++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 2 +- docs/reference.rst | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index 2ae76521..0df5ad66 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -256,6 +256,50 @@ generates this field description array for the structure *Person_PhoneNumber*:: }; +Extension fields +================ +Protocol Buffers supports a concept of `extension fields`_, which are +additional fields to a message, but defined outside the actual message. +The definition can even be in a completely separate .proto file. + +The base message is declared as extensible by keyword *extensions* in +the .proto file:: + + message MyMessage { + .. fields .. + extensions 100 to 199; + } + +For each extensible message, *nanopb_generator.py* declares an additional +callback field called *extensions*. The field and associated datatype +*pb_extension_t* forms a linked list of handlers. When an unknown field is +encountered, the decoder calls each handler in turn until either one of them +handles the field, or the list is exhausted. + +The actual extensions are declared using the *extend* keyword in the .proto, +and are in the global namespace:: + + extend MyMessage { + optional int32 myextension = 100; + } + +For each extension, *nanopb_generator.py* creates a constant of type +*pb_extension_type_t*. To link together the base message and the extension, +you have to: + +1. Allocate storage for your field, matching the datatype in the .proto. + For example, for a *int32* field, you need a *int32_t* variable to store + the value. +2. Create a *pb_extension_t* constant, with pointers to your variable and + to the generated *pb_extension_type_t*. +3. Set the *message.extensions* pointer to point to the *pb_extension_t*. + +An example of this is available in *tests/test_encode_extensions.c* and +*tests/test_decode_extensions.c*. + +.. _`extension fields`: https://developers.google.com/protocol-buffers/docs/proto#extensions + + Return values and error handling ================================ diff --git a/docs/index.rst b/docs/index.rst index cb7a201f..e56ff4cc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -41,7 +41,7 @@ Features and limitations #) Allows specifying maximum size for strings and arrays, so that they can be allocated statically. #) No malloc needed: everything can be allocated statically or on the stack. #) You can use either encoder or decoder alone to cut the code size in half. -#) Support for most protobuf features, including: all data types, nested submessages, default values, repeated and optional fields, packed arrays. +#) Support for most protobuf features, including: all data types, nested submessages, default values, repeated and optional fields, packed arrays, extension fields. #) Callback mechanism for handling messages larger than can fit in available RAM. #) Extensive set of tests. diff --git a/docs/reference.rst b/docs/reference.rst index 6c38f6b4..51556d35 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -304,6 +304,41 @@ Protocol Buffers wire types. These are used with `pb_encode_tag`_. :: PB_WT_32BIT = 5 } pb_wire_type_t; +pb_extension_type_t +------------------- +Defines the handler functions and auxiliary data for a field that extends +another message. Usually autogenerated by *nanopb_generator.py*:: + + typedef struct { + bool (*decode)(pb_istream_t *stream, pb_extension_t *extension, + uint32_t tag, pb_wire_type_t wire_type); + bool (*encode)(pb_ostream_t *stream, const pb_extension_t *extension); + const void *arg; + } pb_extension_type_t; + +In the normal case, the function pointers are *NULL* and the decoder and +encoder use their internal implementations. The internal implementations +assume that *arg* points to a *pb_field_t* that describes the field in question. + +To implement custom processing of unknown fields, you can provide pointers +to your own functions. Their functionality is mostly the same as for normal +callback fields, except that they get called for any unknown field when decoding. + +pb_extension_t +-------------- +Ties together the extension field type and the storage for the field value:: + + typedef struct { + const pb_extension_type_t *type; + void *dest; + pb_extension_t *next; + } pb_extension_t; + +:type: Pointer to the structure that defines the callback functions. +:dest: Pointer to the variable that stores the field value + (as used by the default extension callback functions.) +:next: Pointer to the next extension handler, or *NULL*. + PB_GET_ERROR ------------ Get the current error message from a stream, or a placeholder string if -- cgit 1.2.3-korg From dd726985086f85734fc674a6aaa227cd9c23d1ef Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 8 Aug 2013 20:45:30 +0300 Subject: Update changelog --- CHANGELOG | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index a58be47a..6dfefec8 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,12 @@ +nanopb-0.2.2 + Add support for extension fields (issue 17) + Fix unknown fields in empty message (issue 78) + Include the field tags in the generated .pb.h file. + Add pb_decode_delimited and pb_encode_delimited wrapper functions (issue 74) + Add a section in top of pb.h for changing compilation settings (issue 76) + Documentation improvements (issues 12, 77 and others) + Improved tests + nanopb-0.2.1 NOTE: The default callback function signature has changed. If you don't want to update your code, define PB_OLD_CALLBACK_STYLE. -- cgit 1.2.3-korg From 81cf30034b420cc6fbd769b172a13f7e57796b7b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 18 Aug 2013 22:11:38 +0300 Subject: Publishing nanopb-0.2.2 --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index f39614b3..429742e0 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.2-dev" +nanopb_version = "nanopb-0.2.2" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index bb0a584d..91740032 100644 --- a/pb.h +++ b/pb.h @@ -43,7 +43,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.2-dev +#define NANOPB_VERSION nanopb-0.2.2 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 4aef194a99705805153471c371e16a3633a4cc4e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 18 Aug 2013 22:12:31 +0300 Subject: Setting version to 0.2.3-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 429742e0..e3cddb61 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,5 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.2" +nanopb_version = "nanopb-0.2.3-dev" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index 91740032..e3e68ce5 100644 --- a/pb.h +++ b/pb.h @@ -43,7 +43,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.2 +#define NANOPB_VERSION nanopb-0.2.3-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From d7f3a74388b4825d2c980c53d0a740ddfd0e4770 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 8 Sep 2013 11:05:20 +0300 Subject: Handle unsupported extension field types more gracefully. Previously the generator would stop with NotImplementedException as soon as a required or repeated extension field is found. New behaviour is to just ignore the unsupported field and note that in a comment in the generated file. Furthermore, allow skipping of extension fields using the generator option (nanopb).type = FT_IGNORE. Update issue 83 Status: FixedInGit --- generator/nanopb_generator.py | 19 ++++++++++++++----- tests/extensions.proto | 2 ++ 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index e3cddb61..55e5ab65 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -310,18 +310,26 @@ class ExtensionField(Field): Field.__init__(self, self.fullname + 'struct', desc, field_options) if self.rules != 'OPTIONAL': - raise NotImplementedError("Only 'optional' is supported for extension fields. " - + "(%s.rules == %s)" % (self.fullname, self.rules)) - - self.rules = 'OPTEXT' + self.skip = True + else: + self.skip = False + self.rules = 'OPTEXT' def extension_decl(self): '''Declaration of the extension type in the .pb.h file''' + if self.skip: + msg = '/* Extension field %s was skipped because only "optional"\n' % self.fullname + msg +=' type of extension fields is currently supported. */\n' + return msg + return 'extern const pb_extension_type_t %s;\n' % self.fullname def extension_def(self): '''Definition of the extension type in the .pb.c file''' + if self.skip: + return '' + result = 'typedef struct {\n' result += str(self) result += '\n} %s;\n\n' % self.struct_name @@ -475,7 +483,8 @@ def parse_file(fdesc, file_options): for names, extension in iterate_extensions(fdesc, base_name): field_options = get_nanopb_suboptions(extension, file_options, names) - extensions.append(ExtensionField(names, extension, field_options)) + if field_options.type != nanopb_pb2.FT_IGNORE: + extensions.append(ExtensionField(names, extension, field_options)) # Fix field default values where enum short names are used. for enum in enums: diff --git a/tests/extensions.proto b/tests/extensions.proto index cbffdce6..d85e8193 100644 --- a/tests/extensions.proto +++ b/tests/extensions.proto @@ -7,6 +7,8 @@ extend AllTypes { message ExtensionMessage { extend AllTypes { optional ExtensionMessage AllTypes_extensionfield2 = 254; + required ExtensionMessage AllTypes_extensionfield3 = 253; + repeated ExtensionMessage AllTypes_extensionfield4 = 252; } required string test1 = 1; -- cgit 1.2.3-korg From 262c62676cf740ec3ce14a22bde47b7968fec8f0 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 8 Sep 2013 17:52:03 +0300 Subject: Start moving the tests into subfolders. Transition to SCons for build system for the tests. Only a few tests updated so far. Have to include all the rest before merging to mainline. Update issue 63 Status: Started --- .gitignore | 20 +- tests/Makefile | 143 +---------- tests/SConstruct | 77 ++++++ tests/alltypes.options | 3 - tests/alltypes.proto | 93 ------- tests/alltypes/alltypes.options | 3 + tests/alltypes/alltypes.proto | 93 +++++++ tests/alltypes/decode_alltypes.c | 197 +++++++++++++++ tests/alltypes/encode_alltypes.c | 130 ++++++++++ tests/alltypes_with_extra_fields.pb | Bin 523 -> 0 bytes tests/backwards_compatibility/alltypes_legacy.c | 93 +++++++ tests/backwards_compatibility/alltypes_legacy.h | 178 +++++++++++++ tests/backwards_compatibility/decode_legacy.c | 197 +++++++++++++++ tests/backwards_compatibility/encode_legacy.c | 131 ++++++++++ tests/basic_buffer/SConscript | 12 + tests/basic_buffer/decode_buffer.c | 83 ++++++ tests/basic_buffer/encode_buffer.c | 33 +++ tests/basic_stream/decode_stream.c | 83 ++++++ tests/basic_stream/encode_stream.c | 37 +++ tests/bc_alltypes.pb.c | 93 ------- tests/bc_alltypes.pb.h | 178 ------------- tests/bc_decode.c | 197 --------------- tests/bc_encode.c | 131 ---------- tests/callbacks.proto | 16 -- tests/callbacks/callbacks.proto | 16 ++ tests/callbacks/decode_callbacks.c | 93 +++++++ tests/callbacks/encode_callbacks.c | 86 +++++++ tests/callbacks2.proto | 9 - tests/common/SConscript | 14 ++ tests/common/person.proto | 20 ++ tests/common/unittestproto.proto | 32 +++ tests/common/unittests.h | 14 ++ tests/decode_unittests.c | 306 ----------------------- tests/decode_unittests/SConscript | 4 + tests/decode_unittests/decode_unittests.c | 306 +++++++++++++++++++++++ tests/encode_unittests.c | 287 --------------------- tests/encode_unittests/SConscript | 5 + tests/encode_unittests/encode_unittests.c | 287 +++++++++++++++++++++ tests/extensions.options | 1 - tests/extensions.proto | 17 -- tests/extensions/decode_extensions.c | 43 ++++ tests/extensions/encode_extensions.c | 38 +++ tests/extensions/extensions.options | 1 + tests/extensions/extensions.proto | 17 ++ tests/extra_fields/alltypes_with_extra_fields.pb | Bin 0 -> 523 bytes tests/extra_fields/person_with_extra_field.pb | Bin 0 -> 90 bytes tests/funny-proto+name.proto | 0 tests/missing_fields.proto | 138 ---------- tests/missing_fields/missing_fields.c | 49 ++++ tests/missing_fields/missing_fields.proto | 138 ++++++++++ tests/multiple_files/callbacks.proto | 16 ++ tests/multiple_files/callbacks2.proto | 9 + tests/multiple_files/test_multiple_files.c | 13 + tests/no_messages.proto | 7 - tests/no_messages/no_messages.proto | 7 + tests/options.expected | 7 - tests/options.proto | 73 ------ tests/options/options.expected | 7 + tests/options/options.proto | 73 ++++++ tests/person.proto | 20 -- tests/person_with_extra_field.pb | Bin 90 -> 0 bytes tests/special_characters/funny-proto+name.proto | 0 tests/test_decode1.c | 83 ------ tests/test_decode2.c | 83 ------ tests/test_decode3.c | 197 --------------- tests/test_decode_callbacks.c | 93 ------- tests/test_decode_extensions.c | 43 ---- tests/test_encode1.c | 33 --- tests/test_encode2.c | 37 --- tests/test_encode3.c | 130 ---------- tests/test_encode_callbacks.c | 86 ------- tests/test_encode_extensions.c | 38 --- tests/test_missing_fields.c | 49 ---- tests/test_multiple_files.c | 13 - tests/testperson.pb | 3 - tests/unittestproto.proto | 32 --- tests/unittests.h | 14 -- 77 files changed, 2639 insertions(+), 2669 deletions(-) create mode 100644 tests/SConstruct delete mode 100644 tests/alltypes.options delete mode 100644 tests/alltypes.proto create mode 100644 tests/alltypes/alltypes.options create mode 100644 tests/alltypes/alltypes.proto create mode 100644 tests/alltypes/decode_alltypes.c create mode 100644 tests/alltypes/encode_alltypes.c delete mode 100644 tests/alltypes_with_extra_fields.pb create mode 100644 tests/backwards_compatibility/alltypes_legacy.c create mode 100644 tests/backwards_compatibility/alltypes_legacy.h create mode 100644 tests/backwards_compatibility/decode_legacy.c create mode 100644 tests/backwards_compatibility/encode_legacy.c create mode 100644 tests/basic_buffer/SConscript create mode 100644 tests/basic_buffer/decode_buffer.c create mode 100644 tests/basic_buffer/encode_buffer.c create mode 100644 tests/basic_stream/decode_stream.c create mode 100644 tests/basic_stream/encode_stream.c delete mode 100644 tests/bc_alltypes.pb.c delete mode 100644 tests/bc_alltypes.pb.h delete mode 100644 tests/bc_decode.c delete mode 100644 tests/bc_encode.c delete mode 100644 tests/callbacks.proto create mode 100644 tests/callbacks/callbacks.proto create mode 100644 tests/callbacks/decode_callbacks.c create mode 100644 tests/callbacks/encode_callbacks.c delete mode 100644 tests/callbacks2.proto create mode 100644 tests/common/SConscript create mode 100644 tests/common/person.proto create mode 100644 tests/common/unittestproto.proto create mode 100644 tests/common/unittests.h delete mode 100644 tests/decode_unittests.c create mode 100644 tests/decode_unittests/SConscript create mode 100644 tests/decode_unittests/decode_unittests.c delete mode 100644 tests/encode_unittests.c create mode 100644 tests/encode_unittests/SConscript create mode 100644 tests/encode_unittests/encode_unittests.c delete mode 100644 tests/extensions.options delete mode 100644 tests/extensions.proto create mode 100644 tests/extensions/decode_extensions.c create mode 100644 tests/extensions/encode_extensions.c create mode 100644 tests/extensions/extensions.options create mode 100644 tests/extensions/extensions.proto create mode 100644 tests/extra_fields/alltypes_with_extra_fields.pb create mode 100644 tests/extra_fields/person_with_extra_field.pb delete mode 100644 tests/funny-proto+name.proto delete mode 100644 tests/missing_fields.proto create mode 100644 tests/missing_fields/missing_fields.c create mode 100644 tests/missing_fields/missing_fields.proto create mode 100644 tests/multiple_files/callbacks.proto create mode 100644 tests/multiple_files/callbacks2.proto create mode 100644 tests/multiple_files/test_multiple_files.c delete mode 100644 tests/no_messages.proto create mode 100644 tests/no_messages/no_messages.proto delete mode 100644 tests/options.expected delete mode 100644 tests/options.proto create mode 100644 tests/options/options.expected create mode 100644 tests/options/options.proto delete mode 100644 tests/person.proto delete mode 100644 tests/person_with_extra_field.pb create mode 100644 tests/special_characters/funny-proto+name.proto delete mode 100644 tests/test_decode1.c delete mode 100644 tests/test_decode2.c delete mode 100644 tests/test_decode3.c delete mode 100644 tests/test_decode_callbacks.c delete mode 100644 tests/test_decode_extensions.c delete mode 100644 tests/test_encode1.c delete mode 100644 tests/test_encode2.c delete mode 100644 tests/test_encode3.c delete mode 100644 tests/test_encode_callbacks.c delete mode 100644 tests/test_encode_extensions.c delete mode 100644 tests/test_missing_fields.c delete mode 100644 tests/test_multiple_files.c delete mode 100644 tests/testperson.pb delete mode 100644 tests/unittestproto.proto delete mode 100644 tests/unittests.h diff --git a/.gitignore b/.gitignore index dec8b00e..b83afeff 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *.pb *~ *.tar.gz +.sconsign.dblite julkaisu.txt docs/*.html docs/generator_flow.png @@ -18,22 +19,3 @@ example_avr_double/test_conversions example_unions/decode example_unions/encode generator/nanopb_pb2.pyc -tests/decode_unittests -tests/encode_unittests -tests/test_compiles -tests/test_decode1 -tests/test_decode2 -tests/test_decode3 -tests/test_decode3_buf -tests/test_decode_callbacks -tests/test_encode1 -tests/test_encode2 -tests/test_encode3 -tests/test_encode3_buf -tests/test_encode_callbacks -tests/test_missing_fields -tests/test_multiple_files -tests/bc_decode -tests/bc_encode -tests/breakpoints - diff --git a/tests/Makefile b/tests/Makefile index 9696b794..fb37e630 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,143 +1,6 @@ -CFLAGS=-ansi -Wall -Werror -I .. -g -O0 -DEPS=../pb_decode.h ../pb_encode.h ../pb.h person.pb.h \ - callbacks2.pb.h callbacks.pb.h unittests.h unittestproto.pb.h \ - alltypes.pb.h missing_fields.pb.h -TESTS= decode_unittests encode_unittests \ - test_decode1 test_decode2 test_decode3 test_decode3_buf \ - test_encode1 test_encode2 test_encode3 test_encode3_buf \ - test_decode_callbacks test_encode_callbacks \ - test_missing_fields test_no_messages test_funny_name \ - test_multiple_files test_cxxcompile test_options \ - bc_encode bc_decode test_encode_extensions test_decode_extensions - -# More strict checks for the core part of nanopb -CC_VERSION=$(shell $(CC) -v 2>&1) -CFLAGS_CORE= -ifneq (,$(findstring gcc,$(CC_VERSION))) - CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wlogical-op -Wconversion - CFLAGS+=--coverage -fstack-protector-all - LDFLAGS+=--coverage -endif -ifneq (,$(findstring clang,$(CC_VERSION))) - CFLAGS_CORE=-pedantic -Wextra -Wcast-qual -Wconversion -endif - -# Also use mudflap if it is available -# To enable, run with make -B USE_MUDFLAP=y -USE_MUDFLAP ?= n -ifeq ($(USE_MUDFLAP),y) - CFLAGS += -fmudflap - LDFLAGS += -lmudflap -fmudflap -endif - -all: breakpoints $(TESTS) run_unittests +all: + scons clean: - rm -f $(TESTS) person.pb* alltypes.pb* *.o *.gcda *.gcno *.pb.h *.pb.c - -%.pb.o: %.pb.c %.pb.h - $(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< - -%.o: %.c -%.o: %.c $(DEPS) - $(CC) $(CFLAGS) -c -o $@ $< - -pb_encode.o: ../pb_encode.c $(DEPS) - $(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< -pb_decode.o: ../pb_decode.c $(DEPS) - $(CC) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< - -# Test for compilability with c++ compiler - -pb_encode.cxx.o: ../pb_encode.c $(DEPS) - $(CXX) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< -pb_decode.cxx.o: ../pb_decode.c $(DEPS) - $(CXX) $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< - -# Test for PB_BUF_ONLY compilation option - -pb_encode.buf.o: ../pb_encode.c $(DEPS) - $(CC) -DPB_BUFFER_ONLY $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< -pb_decode.buf.o: ../pb_decode.c $(DEPS) - $(CC) -DPB_BUFFER_ONLY $(CFLAGS) $(CFLAGS_CORE) -c -o $@ $< -%.buf.o: %.c $(DEPS) - $(CC) -DPB_BUFFER_ONLY $(CFLAGS) -c -o $@ $< -test_encode3_buf: test_encode3.buf.o pb_encode.buf.o alltypes.pb.o - $(CC) $(LDFLAGS) $^ -o $@ -test_decode3_buf: test_decode3.buf.o pb_decode.buf.o alltypes.pb.o - $(CC) $(LDFLAGS) $^ -o $@ - -test_cxxcompile: pb_encode.cxx.o pb_decode.cxx.o -test_decode1: test_decode1.o pb_decode.o person.pb.o -test_decode2: test_decode2.o pb_decode.o person.pb.o -test_decode3: test_decode3.o pb_decode.o alltypes.pb.o -test_encode1: test_encode1.o pb_encode.o person.pb.o -test_encode2: test_encode2.o pb_encode.o person.pb.o -test_encode3: test_encode3.o pb_encode.o alltypes.pb.o -test_multiple_files: test_multiple_files.o pb_encode.o callbacks2.pb.o callbacks.pb.o -test_decode_callbacks: test_decode_callbacks.o pb_decode.o callbacks.pb.o -test_encode_callbacks: test_encode_callbacks.o pb_encode.o callbacks.pb.o -test_missing_fields: test_missing_fields.o pb_encode.o pb_decode.o missing_fields.pb.o -decode_unittests: decode_unittests.o pb_decode.o unittestproto.pb.o -encode_unittests: encode_unittests.o pb_encode.o unittestproto.pb.o -test_no_messages: no_messages.pb.h no_messages.pb.c no_messages.pb.o -test_funny_name: funny-proto+name.pb.h funny-proto+name.pb.o -bc_encode: bc_alltypes.pb.o pb_encode.o bc_encode.o -bc_decode: bc_alltypes.pb.o pb_decode.o bc_decode.o -test_encode_extensions: test_encode_extensions.c pb_encode.o alltypes.pb.o extensions.pb.o -test_decode_extensions: test_decode_extensions.c pb_decode.o alltypes.pb.o extensions.pb.o - -%.pb: %.proto - protoc -I. -I../generator -I/usr/include -o$@ $< - -%.pb.c %.pb.h: %.pb ../generator/nanopb_generator.py - python ../generator/nanopb_generator.py $< - -breakpoints: ../*.c *.c - grep -n 'return false;' $^ | cut -d: -f-2 | xargs -n 1 echo b > $@ - -coverage: run_unittests - gcov pb_encode.gcda - gcov pb_decode.gcda - -run_unittests: $(TESTS) - rm -f *.gcda - - ./decode_unittests > /dev/null - ./encode_unittests > /dev/null - - [ "`./test_encode1 | ./test_decode1`" = \ - "`./test_encode1 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] - - [ "`./test_encode2 | ./test_decode1`" = \ - "`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] - - [ "`./test_encode2 | ./test_decode2`" = \ - "`./test_encode2 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] - - [ "`./test_decode2 < person_with_extra_field.pb`" = \ - "`./test_encode2 | ./test_decode2`" ] - - [ "`./test_encode_callbacks | ./test_decode_callbacks`" = \ - "`./test_encode_callbacks | protoc --decode=TestMessage callbacks.proto`" ] - - ./test_encode3 | ./test_decode3 - ./test_encode3 1 | ./test_decode3 1 - ./test_encode3 1 | protoc --decode=AllTypes -I. -I../generator -I/usr/include alltypes.proto >/dev/null - ./test_encode3_buf 1 | ./test_decode3_buf 1 - ./test_decode3 < alltypes_with_extra_fields.pb - ./bc_encode | ./bc_decode - ./test_encode_extensions | ./test_decode_extensions - - ./test_missing_fields - -test_options: options.pb.h options.expected options.pb.o - cat options.expected | while read -r p; do \ - if ! grep -q "$$p" $<; then \ - echo Expected: "$$p"; \ - exit 1; \ - fi \ - done + scons -c -run_fuzztest: test_decode3 - bash -c 'ulimit -c unlimited; I=1; while true; do cat /dev/urandom | ./test_decode3 > /dev/null; I=$$(($$I+1)); echo -en "\r$$I"; done' diff --git a/tests/SConstruct b/tests/SConstruct new file mode 100644 index 00000000..0ec1e54e --- /dev/null +++ b/tests/SConstruct @@ -0,0 +1,77 @@ +env = DefaultEnvironment() + +env.Append(CPPPATH = ["#../", "#common"]) + +# Build command for building .pb from .proto using protoc +def proto_actions(source, target, env, for_signature): + dirs = ' '.join(['-I' + env.GetBuildPath(d) for d in env['PROTOCPATH']]) + return '$PROTOC $PROTOCFLAGS %s -o%s %s' % (dirs, target[0], source[0]) + +proto_file_builder = Builder(generator = proto_actions, + suffix = '.pb', + src_suffix = '.proto') +env.Append(BUILDERS = {'Proto': proto_file_builder}) +env.SetDefault(PROTOC = 'protoc') + +# Define the include path to find nanopb.proto +env.Append(PROTOCPATH = ['#../generator', '/usr/include', '.']) + +# Build command for running nanopb generator +import os.path +def nanopb_targets(target, source, env): + basename = os.path.splitext(str(source[0]))[0] + target.append(basename + '.pb.h') + return target, source + +nanopb_file_builder = Builder(action = '$NANOPB_GENERATOR $NANOPB_FLAGS $SOURCE', + suffix = '.pb.c', + src_suffix = '.pb', + emitter = nanopb_targets) +env.Append(BUILDERS = {'Nanopb': nanopb_file_builder}) +env.SetDefault(NANOPB_GENERATOR = 'python ' + env.GetBuildPath("#../generator/nanopb_generator.py")) + +# Combined method to run both protoc and nanopb generator +def run_protoc_and_nanopb(env, source): + b1 = env.Proto(source) + b2 = env.Nanopb(source) + return b1 + b2 +env.AddMethod(run_protoc_and_nanopb, "NanopbProto") + +# Build command that runs a test program and saves the output +def run_test_actions(source, target, env, for_signature): + cmd = str(source[0]) # Name of binary + if len(source) > 1: + cmd += ' <' + str(source[1]) # Input file + cmd += ' >' + str(target[0]) + return cmd + +run_test_builder = Builder(generator = run_test_actions, + suffix = '.output') +env.Append(BUILDERS = {'RunTest': run_test_builder}) + +# Build command that decodes a message using protoc +def decode_actions(source, target, env, for_signature): + dirs = ' '.join(['-I' + env.GetBuildPath(d) for d in env['PROTOCPATH']]) + return '$PROTOC $PROTOCFLAGS %s --decode=%s %s <%s >%s' % (dirs, env['MESSAGE'], source[1], source[0], target[0]) + +decode_builder = Builder(generator = decode_actions, + suffix = '.decoded') +env.Append(BUILDERS = {'Decode': decode_builder}) + +# Build command that asserts that two files be equal +def compare_files(target, source, env): + data1 = open(str(source[0]), 'rb').read() + data2 = open(str(source[1]), 'rb').read() + if data1 == data2: +# open(str(target[0]), 'w').write('OK') + return 0 + else: + return "Test failed: %s and %s differ!" % (source[0], source[1]) + +compare_builder = Builder(action = compare_files, + suffix = '.equal') +env.Append(BUILDERS = {'Compare': compare_builder}) + +# Now include the SConscript files from all subdirectories +SConscript(Glob('*/SConscript'), exports = 'env') + diff --git a/tests/alltypes.options b/tests/alltypes.options deleted file mode 100644 index b31e3cf0..00000000 --- a/tests/alltypes.options +++ /dev/null @@ -1,3 +0,0 @@ -* max_size:16 -* max_count:5 - diff --git a/tests/alltypes.proto b/tests/alltypes.proto deleted file mode 100644 index a2cf8bbc..00000000 --- a/tests/alltypes.proto +++ /dev/null @@ -1,93 +0,0 @@ -message SubMessage { - required string substuff1 = 1 [default = "1"]; - required int32 substuff2 = 2 [default = 2]; - optional fixed32 substuff3 = 3 [default = 3]; -} - -message EmptyMessage { - -} - -enum MyEnum { - Zero = 0; - First = 1; - Second = 2; - Truth = 42; -} - -message AllTypes { - required int32 req_int32 = 1; - required int64 req_int64 = 2; - required uint32 req_uint32 = 3; - required uint64 req_uint64 = 4; - required sint32 req_sint32 = 5; - required sint64 req_sint64 = 6; - required bool req_bool = 7; - - required fixed32 req_fixed32 = 8; - required sfixed32 req_sfixed32= 9; - required float req_float = 10; - - required fixed64 req_fixed64 = 11; - required sfixed64 req_sfixed64= 12; - required double req_double = 13; - - required string req_string = 14; - required bytes req_bytes = 15; - required SubMessage req_submsg = 16; - required MyEnum req_enum = 17; - required EmptyMessage req_emptymsg = 18; - - - repeated int32 rep_int32 = 21; - repeated int64 rep_int64 = 22; - repeated uint32 rep_uint32 = 23; - repeated uint64 rep_uint64 = 24; - repeated sint32 rep_sint32 = 25; - repeated sint64 rep_sint64 = 26; - repeated bool rep_bool = 27; - - repeated fixed32 rep_fixed32 = 28; - repeated sfixed32 rep_sfixed32= 29; - repeated float rep_float = 30; - - repeated fixed64 rep_fixed64 = 31; - repeated sfixed64 rep_sfixed64= 32; - repeated double rep_double = 33; - - repeated string rep_string = 34; - repeated bytes rep_bytes = 35; - repeated SubMessage rep_submsg = 36; - repeated MyEnum rep_enum = 37; - repeated EmptyMessage rep_emptymsg = 38; - - optional int32 opt_int32 = 41 [default = 4041]; - optional int64 opt_int64 = 42 [default = 4042]; - optional uint32 opt_uint32 = 43 [default = 4043]; - optional uint64 opt_uint64 = 44 [default = 4044]; - optional sint32 opt_sint32 = 45 [default = 4045]; - optional sint64 opt_sint64 = 46 [default = 4046]; - optional bool opt_bool = 47 [default = false]; - - optional fixed32 opt_fixed32 = 48 [default = 4048]; - optional sfixed32 opt_sfixed32= 49 [default = 4049]; - optional float opt_float = 50 [default = 4050]; - - optional fixed64 opt_fixed64 = 51 [default = 4051]; - optional sfixed64 opt_sfixed64= 52 [default = 4052]; - optional double opt_double = 53 [default = 4053]; - - optional string opt_string = 54 [default = "4054"]; - optional bytes opt_bytes = 55 [default = "4055"]; - optional SubMessage opt_submsg = 56; - optional MyEnum opt_enum = 57 [default = Second]; - optional EmptyMessage opt_emptymsg = 58; - - // Just to make sure that the size of the fields has been calculated - // properly, i.e. otherwise a bug in last field might not be detected. - required int32 end = 99; - - - extensions 200 to 255; -} - diff --git a/tests/alltypes/alltypes.options b/tests/alltypes/alltypes.options new file mode 100644 index 00000000..b31e3cf0 --- /dev/null +++ b/tests/alltypes/alltypes.options @@ -0,0 +1,3 @@ +* max_size:16 +* max_count:5 + diff --git a/tests/alltypes/alltypes.proto b/tests/alltypes/alltypes.proto new file mode 100644 index 00000000..a2cf8bbc --- /dev/null +++ b/tests/alltypes/alltypes.proto @@ -0,0 +1,93 @@ +message SubMessage { + required string substuff1 = 1 [default = "1"]; + required int32 substuff2 = 2 [default = 2]; + optional fixed32 substuff3 = 3 [default = 3]; +} + +message EmptyMessage { + +} + +enum MyEnum { + Zero = 0; + First = 1; + Second = 2; + Truth = 42; +} + +message AllTypes { + required int32 req_int32 = 1; + required int64 req_int64 = 2; + required uint32 req_uint32 = 3; + required uint64 req_uint64 = 4; + required sint32 req_sint32 = 5; + required sint64 req_sint64 = 6; + required bool req_bool = 7; + + required fixed32 req_fixed32 = 8; + required sfixed32 req_sfixed32= 9; + required float req_float = 10; + + required fixed64 req_fixed64 = 11; + required sfixed64 req_sfixed64= 12; + required double req_double = 13; + + required string req_string = 14; + required bytes req_bytes = 15; + required SubMessage req_submsg = 16; + required MyEnum req_enum = 17; + required EmptyMessage req_emptymsg = 18; + + + repeated int32 rep_int32 = 21; + repeated int64 rep_int64 = 22; + repeated uint32 rep_uint32 = 23; + repeated uint64 rep_uint64 = 24; + repeated sint32 rep_sint32 = 25; + repeated sint64 rep_sint64 = 26; + repeated bool rep_bool = 27; + + repeated fixed32 rep_fixed32 = 28; + repeated sfixed32 rep_sfixed32= 29; + repeated float rep_float = 30; + + repeated fixed64 rep_fixed64 = 31; + repeated sfixed64 rep_sfixed64= 32; + repeated double rep_double = 33; + + repeated string rep_string = 34; + repeated bytes rep_bytes = 35; + repeated SubMessage rep_submsg = 36; + repeated MyEnum rep_enum = 37; + repeated EmptyMessage rep_emptymsg = 38; + + optional int32 opt_int32 = 41 [default = 4041]; + optional int64 opt_int64 = 42 [default = 4042]; + optional uint32 opt_uint32 = 43 [default = 4043]; + optional uint64 opt_uint64 = 44 [default = 4044]; + optional sint32 opt_sint32 = 45 [default = 4045]; + optional sint64 opt_sint64 = 46 [default = 4046]; + optional bool opt_bool = 47 [default = false]; + + optional fixed32 opt_fixed32 = 48 [default = 4048]; + optional sfixed32 opt_sfixed32= 49 [default = 4049]; + optional float opt_float = 50 [default = 4050]; + + optional fixed64 opt_fixed64 = 51 [default = 4051]; + optional sfixed64 opt_sfixed64= 52 [default = 4052]; + optional double opt_double = 53 [default = 4053]; + + optional string opt_string = 54 [default = "4054"]; + optional bytes opt_bytes = 55 [default = "4055"]; + optional SubMessage opt_submsg = 56; + optional MyEnum opt_enum = 57 [default = Second]; + optional EmptyMessage opt_emptymsg = 58; + + // Just to make sure that the size of the fields has been calculated + // properly, i.e. otherwise a bug in last field might not be detected. + required int32 end = 99; + + + extensions 200 to 255; +} + diff --git a/tests/alltypes/decode_alltypes.c b/tests/alltypes/decode_alltypes.c new file mode 100644 index 00000000..55d025c0 --- /dev/null +++ b/tests/alltypes/decode_alltypes.c @@ -0,0 +1,197 @@ +/* Tests the decoding of all types. + * This is the counterpart of test_encode3. + * Run e.g. ./test_encode3 | ./test_decode3 + */ + +#include +#include +#include +#include +#include "alltypes.pb.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed.\n"); \ + return false; \ + } + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_alltypes(pb_istream_t *stream, int mode) +{ + AllTypes alltypes; + + /* Fill with garbage to better detect initialization errors */ + memset(&alltypes, 0xAA, sizeof(alltypes)); + + if (!pb_decode(stream, AllTypes_fields, &alltypes)) + return false; + + TEST(alltypes.req_int32 == -1001); + TEST(alltypes.req_int64 == -1002); + TEST(alltypes.req_uint32 == 1003); + TEST(alltypes.req_uint64 == 1004); + TEST(alltypes.req_sint32 == -1005); + TEST(alltypes.req_sint64 == -1006); + TEST(alltypes.req_bool == true); + + TEST(alltypes.req_fixed32 == 1008); + TEST(alltypes.req_sfixed32 == -1009); + TEST(alltypes.req_float == 1010.0f); + + TEST(alltypes.req_fixed64 == 1011); + TEST(alltypes.req_sfixed64 == -1012); + TEST(alltypes.req_double == 1013.0f); + + TEST(strcmp(alltypes.req_string, "1014") == 0); + TEST(alltypes.req_bytes.size == 4); + TEST(memcmp(alltypes.req_bytes.bytes, "1015", 4) == 0); + TEST(strcmp(alltypes.req_submsg.substuff1, "1016") == 0); + TEST(alltypes.req_submsg.substuff2 == 1016); + TEST(alltypes.req_submsg.substuff3 == 3); + TEST(alltypes.req_enum == MyEnum_Truth); + + TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); + TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); + TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); + TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); + TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); + TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0); + TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); + + TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); + TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); + TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); + + TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); + TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0); + TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); + + TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); + TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); + TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); + + TEST(alltypes.rep_submsg_count == 5); + TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); + TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); + TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3); + + TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); + TEST(alltypes.rep_emptymsg_count == 5); + + if (mode == 0) + { + /* Expect default values */ + TEST(alltypes.has_opt_int32 == false); + TEST(alltypes.opt_int32 == 4041); + TEST(alltypes.has_opt_int64 == false); + TEST(alltypes.opt_int64 == 4042); + TEST(alltypes.has_opt_uint32 == false); + TEST(alltypes.opt_uint32 == 4043); + TEST(alltypes.has_opt_uint64 == false); + TEST(alltypes.opt_uint64 == 4044); + TEST(alltypes.has_opt_sint32 == false); + TEST(alltypes.opt_sint32 == 4045); + TEST(alltypes.has_opt_sint64 == false); + TEST(alltypes.opt_sint64 == 4046); + TEST(alltypes.has_opt_bool == false); + TEST(alltypes.opt_bool == false); + + TEST(alltypes.has_opt_fixed32 == false); + TEST(alltypes.opt_fixed32 == 4048); + TEST(alltypes.has_opt_sfixed32 == false); + TEST(alltypes.opt_sfixed32 == 4049); + TEST(alltypes.has_opt_float == false); + TEST(alltypes.opt_float == 4050.0f); + + TEST(alltypes.has_opt_fixed64 == false); + TEST(alltypes.opt_fixed64 == 4051); + TEST(alltypes.has_opt_sfixed64 == false); + TEST(alltypes.opt_sfixed64 == 4052); + TEST(alltypes.has_opt_double == false); + TEST(alltypes.opt_double == 4053.0); + + TEST(alltypes.has_opt_string == false); + TEST(strcmp(alltypes.opt_string, "4054") == 0); + TEST(alltypes.has_opt_bytes == false); + TEST(alltypes.opt_bytes.size == 4); + TEST(memcmp(alltypes.opt_bytes.bytes, "4055", 4) == 0); + TEST(alltypes.has_opt_submsg == false); + TEST(strcmp(alltypes.opt_submsg.substuff1, "1") == 0); + TEST(alltypes.opt_submsg.substuff2 == 2); + TEST(alltypes.opt_submsg.substuff3 == 3); + TEST(alltypes.has_opt_enum == false); + TEST(alltypes.opt_enum == MyEnum_Second); + TEST(alltypes.has_opt_emptymsg == false); + } + else + { + /* Expect filled-in values */ + TEST(alltypes.has_opt_int32 == true); + TEST(alltypes.opt_int32 == 3041); + TEST(alltypes.has_opt_int64 == true); + TEST(alltypes.opt_int64 == 3042); + TEST(alltypes.has_opt_uint32 == true); + TEST(alltypes.opt_uint32 == 3043); + TEST(alltypes.has_opt_uint64 == true); + TEST(alltypes.opt_uint64 == 3044); + TEST(alltypes.has_opt_sint32 == true); + TEST(alltypes.opt_sint32 == 3045); + TEST(alltypes.has_opt_sint64 == true); + TEST(alltypes.opt_sint64 == 3046); + TEST(alltypes.has_opt_bool == true); + TEST(alltypes.opt_bool == true); + + TEST(alltypes.has_opt_fixed32 == true); + TEST(alltypes.opt_fixed32 == 3048); + TEST(alltypes.has_opt_sfixed32 == true); + TEST(alltypes.opt_sfixed32 == 3049); + TEST(alltypes.has_opt_float == true); + TEST(alltypes.opt_float == 3050.0f); + + TEST(alltypes.has_opt_fixed64 == true); + TEST(alltypes.opt_fixed64 == 3051); + TEST(alltypes.has_opt_sfixed64 == true); + TEST(alltypes.opt_sfixed64 == 3052); + TEST(alltypes.has_opt_double == true); + TEST(alltypes.opt_double == 3053.0); + + TEST(alltypes.has_opt_string == true); + TEST(strcmp(alltypes.opt_string, "3054") == 0); + TEST(alltypes.has_opt_bytes == true); + TEST(alltypes.opt_bytes.size == 4); + TEST(memcmp(alltypes.opt_bytes.bytes, "3055", 4) == 0); + TEST(alltypes.has_opt_submsg == true); + TEST(strcmp(alltypes.opt_submsg.substuff1, "3056") == 0); + TEST(alltypes.opt_submsg.substuff2 == 3056); + TEST(alltypes.opt_submsg.substuff3 == 3); + TEST(alltypes.has_opt_enum == true); + TEST(alltypes.opt_enum == MyEnum_Truth); + TEST(alltypes.has_opt_emptymsg == true); + } + + TEST(alltypes.end == 1099); + + return true; +} + +int main(int argc, char **argv) +{ + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Read the data into buffer */ + uint8_t buffer[1024]; + size_t count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + pb_istream_t stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_alltypes(&stream, mode)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} diff --git a/tests/alltypes/encode_alltypes.c b/tests/alltypes/encode_alltypes.c new file mode 100644 index 00000000..982ad3c8 --- /dev/null +++ b/tests/alltypes/encode_alltypes.c @@ -0,0 +1,130 @@ +/* Attempts to test all the datatypes supported by ProtoBuf. + */ + +#include +#include +#include +#include +#include "alltypes.pb.h" + +int main(int argc, char **argv) +{ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Initialize the structure with constants */ + AllTypes alltypes = {0}; + + alltypes.req_int32 = -1001; + alltypes.req_int64 = -1002; + alltypes.req_uint32 = 1003; + alltypes.req_uint64 = 1004; + alltypes.req_sint32 = -1005; + alltypes.req_sint64 = -1006; + alltypes.req_bool = true; + + alltypes.req_fixed32 = 1008; + alltypes.req_sfixed32 = -1009; + alltypes.req_float = 1010.0f; + + alltypes.req_fixed64 = 1011; + alltypes.req_sfixed64 = -1012; + alltypes.req_double = 1013.0; + + strcpy(alltypes.req_string, "1014"); + alltypes.req_bytes.size = 4; + memcpy(alltypes.req_bytes.bytes, "1015", 4); + strcpy(alltypes.req_submsg.substuff1, "1016"); + alltypes.req_submsg.substuff2 = 1016; + alltypes.req_enum = MyEnum_Truth; + + alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; + alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002; + alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; + alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004; + alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005; + alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006; + alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; + + alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; + alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009; + alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; + + alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011; + alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012; + alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0; + + alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); + alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4; + memcpy(alltypes.rep_bytes[4].bytes, "2015", 4); + + alltypes.rep_submsg_count = 5; + strcpy(alltypes.rep_submsg[4].substuff1, "2016"); + alltypes.rep_submsg[4].substuff2 = 2016; + alltypes.rep_submsg[4].has_substuff3 = true; + alltypes.rep_submsg[4].substuff3 = 2016; + + alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; + alltypes.rep_emptymsg_count = 5; + + if (mode != 0) + { + /* Fill in values for optional fields */ + alltypes.has_opt_int32 = true; + alltypes.opt_int32 = 3041; + alltypes.has_opt_int64 = true; + alltypes.opt_int64 = 3042; + alltypes.has_opt_uint32 = true; + alltypes.opt_uint32 = 3043; + alltypes.has_opt_uint64 = true; + alltypes.opt_uint64 = 3044; + alltypes.has_opt_sint32 = true; + alltypes.opt_sint32 = 3045; + alltypes.has_opt_sint64 = true; + alltypes.opt_sint64 = 3046; + alltypes.has_opt_bool = true; + alltypes.opt_bool = true; + + alltypes.has_opt_fixed32 = true; + alltypes.opt_fixed32 = 3048; + alltypes.has_opt_sfixed32 = true; + alltypes.opt_sfixed32 = 3049; + alltypes.has_opt_float = true; + alltypes.opt_float = 3050.0f; + + alltypes.has_opt_fixed64 = true; + alltypes.opt_fixed64 = 3051; + alltypes.has_opt_sfixed64 = true; + alltypes.opt_sfixed64 = 3052; + alltypes.has_opt_double = true; + alltypes.opt_double = 3053.0; + + alltypes.has_opt_string = true; + strcpy(alltypes.opt_string, "3054"); + alltypes.has_opt_bytes = true; + alltypes.opt_bytes.size = 4; + memcpy(alltypes.opt_bytes.bytes, "3055", 4); + alltypes.has_opt_submsg = true; + strcpy(alltypes.opt_submsg.substuff1, "3056"); + alltypes.opt_submsg.substuff2 = 3056; + alltypes.has_opt_enum = true; + alltypes.opt_enum = MyEnum_Truth; + alltypes.has_opt_emptymsg = true; + } + + alltypes.end = 1099; + + uint8_t buffer[1024]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } +} diff --git a/tests/alltypes_with_extra_fields.pb b/tests/alltypes_with_extra_fields.pb deleted file mode 100644 index f9f53941..00000000 Binary files a/tests/alltypes_with_extra_fields.pb and /dev/null differ diff --git a/tests/backwards_compatibility/alltypes_legacy.c b/tests/backwards_compatibility/alltypes_legacy.c new file mode 100644 index 00000000..b144b1e3 --- /dev/null +++ b/tests/backwards_compatibility/alltypes_legacy.c @@ -0,0 +1,93 @@ +/* Automatically generated nanopb constant definitions */ +/* Generated by 0.2.0-dev at Sun Feb 17 00:09:53 2013. */ +/* This is a file generated using nanopb-0.2.0-dev. + * It is used as a part of test suite in order to detect any + * incompatible changes made to the generator in future versions. + */ + +#include "bc_alltypes.pb.h" + +const char SubMessage_substuff1_default[16] = "1"; +const int32_t SubMessage_substuff2_default = 2; +const uint32_t SubMessage_substuff3_default = 3; +const int32_t AllTypes_opt_int32_default = 4041; +const int64_t AllTypes_opt_int64_default = 4042; +const uint32_t AllTypes_opt_uint32_default = 4043; +const uint64_t AllTypes_opt_uint64_default = 4044; +const int32_t AllTypes_opt_sint32_default = 4045; +const int64_t AllTypes_opt_sint64_default = 4046; +const bool AllTypes_opt_bool_default = false; +const uint32_t AllTypes_opt_fixed32_default = 4048; +const int32_t AllTypes_opt_sfixed32_default = 4049; +const float AllTypes_opt_float_default = 4050; +const uint64_t AllTypes_opt_fixed64_default = 4051; +const int64_t AllTypes_opt_sfixed64_default = 4052; +const double AllTypes_opt_double_default = 4053; +const char AllTypes_opt_string_default[16] = "4054"; +const AllTypes_opt_bytes_t AllTypes_opt_bytes_default = {4, {0x34,0x30,0x35,0x35}}; +const MyEnum AllTypes_opt_enum_default = MyEnum_Second; + + +const pb_field_t SubMessage_fields[4] = { + PB_FIELD( 1, STRING , REQUIRED, STATIC, SubMessage, substuff1, substuff1, &SubMessage_substuff1_default), + PB_FIELD( 2, INT32 , REQUIRED, STATIC, SubMessage, substuff2, substuff1, &SubMessage_substuff2_default), + PB_FIELD( 3, FIXED32 , OPTIONAL, STATIC, SubMessage, substuff3, substuff2, &SubMessage_substuff3_default), + PB_LAST_FIELD +}; + +const pb_field_t AllTypes_fields[53] = { + PB_FIELD( 1, INT32 , REQUIRED, STATIC, AllTypes, req_int32, req_int32, 0), + PB_FIELD( 2, INT64 , REQUIRED, STATIC, AllTypes, req_int64, req_int32, 0), + PB_FIELD( 3, UINT32 , REQUIRED, STATIC, AllTypes, req_uint32, req_int64, 0), + PB_FIELD( 4, UINT64 , REQUIRED, STATIC, AllTypes, req_uint64, req_uint32, 0), + PB_FIELD( 5, SINT32 , REQUIRED, STATIC, AllTypes, req_sint32, req_uint64, 0), + PB_FIELD( 6, SINT64 , REQUIRED, STATIC, AllTypes, req_sint64, req_sint32, 0), + PB_FIELD( 7, BOOL , REQUIRED, STATIC, AllTypes, req_bool, req_sint64, 0), + PB_FIELD( 8, FIXED32 , REQUIRED, STATIC, AllTypes, req_fixed32, req_bool, 0), + PB_FIELD( 9, SFIXED32, REQUIRED, STATIC, AllTypes, req_sfixed32, req_fixed32, 0), + PB_FIELD( 10, FLOAT , REQUIRED, STATIC, AllTypes, req_float, req_sfixed32, 0), + PB_FIELD( 11, FIXED64 , REQUIRED, STATIC, AllTypes, req_fixed64, req_float, 0), + PB_FIELD( 12, SFIXED64, REQUIRED, STATIC, AllTypes, req_sfixed64, req_fixed64, 0), + PB_FIELD( 13, DOUBLE , REQUIRED, STATIC, AllTypes, req_double, req_sfixed64, 0), + PB_FIELD( 14, STRING , REQUIRED, STATIC, AllTypes, req_string, req_double, 0), + PB_FIELD( 15, BYTES , REQUIRED, STATIC, AllTypes, req_bytes, req_string, 0), + PB_FIELD( 16, MESSAGE , REQUIRED, STATIC, AllTypes, req_submsg, req_bytes, &SubMessage_fields), + PB_FIELD( 17, ENUM , REQUIRED, STATIC, AllTypes, req_enum, req_submsg, 0), + PB_FIELD( 21, INT32 , REPEATED, STATIC, AllTypes, rep_int32, req_enum, 0), + PB_FIELD( 22, INT64 , REPEATED, STATIC, AllTypes, rep_int64, rep_int32, 0), + PB_FIELD( 23, UINT32 , REPEATED, STATIC, AllTypes, rep_uint32, rep_int64, 0), + PB_FIELD( 24, UINT64 , REPEATED, STATIC, AllTypes, rep_uint64, rep_uint32, 0), + PB_FIELD( 25, SINT32 , REPEATED, STATIC, AllTypes, rep_sint32, rep_uint64, 0), + PB_FIELD( 26, SINT64 , REPEATED, STATIC, AllTypes, rep_sint64, rep_sint32, 0), + PB_FIELD( 27, BOOL , REPEATED, STATIC, AllTypes, rep_bool, rep_sint64, 0), + PB_FIELD( 28, FIXED32 , REPEATED, STATIC, AllTypes, rep_fixed32, rep_bool, 0), + PB_FIELD( 29, SFIXED32, REPEATED, STATIC, AllTypes, rep_sfixed32, rep_fixed32, 0), + PB_FIELD( 30, FLOAT , REPEATED, STATIC, AllTypes, rep_float, rep_sfixed32, 0), + PB_FIELD( 31, FIXED64 , REPEATED, STATIC, AllTypes, rep_fixed64, rep_float, 0), + PB_FIELD( 32, SFIXED64, REPEATED, STATIC, AllTypes, rep_sfixed64, rep_fixed64, 0), + PB_FIELD( 33, DOUBLE , REPEATED, STATIC, AllTypes, rep_double, rep_sfixed64, 0), + PB_FIELD( 34, STRING , REPEATED, STATIC, AllTypes, rep_string, rep_double, 0), + PB_FIELD( 35, BYTES , REPEATED, STATIC, AllTypes, rep_bytes, rep_string, 0), + PB_FIELD( 36, MESSAGE , REPEATED, STATIC, AllTypes, rep_submsg, rep_bytes, &SubMessage_fields), + PB_FIELD( 37, ENUM , REPEATED, STATIC, AllTypes, rep_enum, rep_submsg, 0), + PB_FIELD( 41, INT32 , OPTIONAL, STATIC, AllTypes, opt_int32, rep_enum, &AllTypes_opt_int32_default), + PB_FIELD( 42, INT64 , OPTIONAL, STATIC, AllTypes, opt_int64, opt_int32, &AllTypes_opt_int64_default), + PB_FIELD( 43, UINT32 , OPTIONAL, STATIC, AllTypes, opt_uint32, opt_int64, &AllTypes_opt_uint32_default), + PB_FIELD( 44, UINT64 , OPTIONAL, STATIC, AllTypes, opt_uint64, opt_uint32, &AllTypes_opt_uint64_default), + PB_FIELD( 45, SINT32 , OPTIONAL, STATIC, AllTypes, opt_sint32, opt_uint64, &AllTypes_opt_sint32_default), + PB_FIELD( 46, SINT64 , OPTIONAL, STATIC, AllTypes, opt_sint64, opt_sint32, &AllTypes_opt_sint64_default), + PB_FIELD( 47, BOOL , OPTIONAL, STATIC, AllTypes, opt_bool, opt_sint64, &AllTypes_opt_bool_default), + PB_FIELD( 48, FIXED32 , OPTIONAL, STATIC, AllTypes, opt_fixed32, opt_bool, &AllTypes_opt_fixed32_default), + PB_FIELD( 49, SFIXED32, OPTIONAL, STATIC, AllTypes, opt_sfixed32, opt_fixed32, &AllTypes_opt_sfixed32_default), + PB_FIELD( 50, FLOAT , OPTIONAL, STATIC, AllTypes, opt_float, opt_sfixed32, &AllTypes_opt_float_default), + PB_FIELD( 51, FIXED64 , OPTIONAL, STATIC, AllTypes, opt_fixed64, opt_float, &AllTypes_opt_fixed64_default), + PB_FIELD( 52, SFIXED64, OPTIONAL, STATIC, AllTypes, opt_sfixed64, opt_fixed64, &AllTypes_opt_sfixed64_default), + PB_FIELD( 53, DOUBLE , OPTIONAL, STATIC, AllTypes, opt_double, opt_sfixed64, &AllTypes_opt_double_default), + PB_FIELD( 54, STRING , OPTIONAL, STATIC, AllTypes, opt_string, opt_double, &AllTypes_opt_string_default), + PB_FIELD( 55, BYTES , OPTIONAL, STATIC, AllTypes, opt_bytes, opt_string, &AllTypes_opt_bytes_default), + PB_FIELD( 56, MESSAGE , OPTIONAL, STATIC, AllTypes, opt_submsg, opt_bytes, &SubMessage_fields), + PB_FIELD( 57, ENUM , OPTIONAL, STATIC, AllTypes, opt_enum, opt_submsg, &AllTypes_opt_enum_default), + PB_FIELD( 99, INT32 , REQUIRED, STATIC, AllTypes, end, opt_enum, 0), + PB_LAST_FIELD +}; + diff --git a/tests/backwards_compatibility/alltypes_legacy.h b/tests/backwards_compatibility/alltypes_legacy.h new file mode 100644 index 00000000..037b3478 --- /dev/null +++ b/tests/backwards_compatibility/alltypes_legacy.h @@ -0,0 +1,178 @@ +/* Automatically generated nanopb header */ +/* This is a file generated using nanopb-0.2.0-dev. + * It is used as a part of test suite in order to detect any + * incompatible changes made to the generator in future versions. + */ + +#ifndef _PB_ALLTYPES_PB_H_ +#define _PB_ALLTYPES_PB_H_ +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/* Enum definitions */ +typedef enum _MyEnum { + MyEnum_Zero = 0, + MyEnum_First = 1, + MyEnum_Second = 2, + MyEnum_Truth = 42 +} MyEnum; + +/* Struct definitions */ +typedef struct _SubMessage { + char substuff1[16]; + int32_t substuff2; + bool has_substuff3; + uint32_t substuff3; +} SubMessage; + +typedef struct { + size_t size; + uint8_t bytes[16]; +} AllTypes_req_bytes_t; + +typedef struct { + size_t size; + uint8_t bytes[16]; +} AllTypes_rep_bytes_t; + +typedef struct { + size_t size; + uint8_t bytes[16]; +} AllTypes_opt_bytes_t; + +typedef struct _AllTypes { + int32_t req_int32; + int64_t req_int64; + uint32_t req_uint32; + uint64_t req_uint64; + int32_t req_sint32; + int64_t req_sint64; + bool req_bool; + uint32_t req_fixed32; + int32_t req_sfixed32; + float req_float; + uint64_t req_fixed64; + int64_t req_sfixed64; + double req_double; + char req_string[16]; + AllTypes_req_bytes_t req_bytes; + SubMessage req_submsg; + MyEnum req_enum; + size_t rep_int32_count; + int32_t rep_int32[5]; + size_t rep_int64_count; + int64_t rep_int64[5]; + size_t rep_uint32_count; + uint32_t rep_uint32[5]; + size_t rep_uint64_count; + uint64_t rep_uint64[5]; + size_t rep_sint32_count; + int32_t rep_sint32[5]; + size_t rep_sint64_count; + int64_t rep_sint64[5]; + size_t rep_bool_count; + bool rep_bool[5]; + size_t rep_fixed32_count; + uint32_t rep_fixed32[5]; + size_t rep_sfixed32_count; + int32_t rep_sfixed32[5]; + size_t rep_float_count; + float rep_float[5]; + size_t rep_fixed64_count; + uint64_t rep_fixed64[5]; + size_t rep_sfixed64_count; + int64_t rep_sfixed64[5]; + size_t rep_double_count; + double rep_double[5]; + size_t rep_string_count; + char rep_string[5][16]; + size_t rep_bytes_count; + AllTypes_rep_bytes_t rep_bytes[5]; + size_t rep_submsg_count; + SubMessage rep_submsg[5]; + size_t rep_enum_count; + MyEnum rep_enum[5]; + bool has_opt_int32; + int32_t opt_int32; + bool has_opt_int64; + int64_t opt_int64; + bool has_opt_uint32; + uint32_t opt_uint32; + bool has_opt_uint64; + uint64_t opt_uint64; + bool has_opt_sint32; + int32_t opt_sint32; + bool has_opt_sint64; + int64_t opt_sint64; + bool has_opt_bool; + bool opt_bool; + bool has_opt_fixed32; + uint32_t opt_fixed32; + bool has_opt_sfixed32; + int32_t opt_sfixed32; + bool has_opt_float; + float opt_float; + bool has_opt_fixed64; + uint64_t opt_fixed64; + bool has_opt_sfixed64; + int64_t opt_sfixed64; + bool has_opt_double; + double opt_double; + bool has_opt_string; + char opt_string[16]; + bool has_opt_bytes; + AllTypes_opt_bytes_t opt_bytes; + bool has_opt_submsg; + SubMessage opt_submsg; + bool has_opt_enum; + MyEnum opt_enum; + int32_t end; +} AllTypes; + +/* Default values for struct fields */ +extern const char SubMessage_substuff1_default[16]; +extern const int32_t SubMessage_substuff2_default; +extern const uint32_t SubMessage_substuff3_default; +extern const int32_t AllTypes_opt_int32_default; +extern const int64_t AllTypes_opt_int64_default; +extern const uint32_t AllTypes_opt_uint32_default; +extern const uint64_t AllTypes_opt_uint64_default; +extern const int32_t AllTypes_opt_sint32_default; +extern const int64_t AllTypes_opt_sint64_default; +extern const bool AllTypes_opt_bool_default; +extern const uint32_t AllTypes_opt_fixed32_default; +extern const int32_t AllTypes_opt_sfixed32_default; +extern const float AllTypes_opt_float_default; +extern const uint64_t AllTypes_opt_fixed64_default; +extern const int64_t AllTypes_opt_sfixed64_default; +extern const double AllTypes_opt_double_default; +extern const char AllTypes_opt_string_default[16]; +extern const AllTypes_opt_bytes_t AllTypes_opt_bytes_default; +extern const MyEnum AllTypes_opt_enum_default; + +/* Struct field encoding specification for nanopb */ +extern const pb_field_t SubMessage_fields[4]; +extern const pb_field_t AllTypes_fields[53]; + +/* Check that field information fits in pb_field_t */ +#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) +STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 256 && pb_membersize(AllTypes, rep_submsg[0]) < 256 && pb_membersize(AllTypes, opt_submsg) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_SubMessage_AllTypes) +#endif + +#if !defined(PB_FIELD_32BIT) +STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 65536 && pb_membersize(AllTypes, rep_submsg[0]) < 65536 && pb_membersize(AllTypes, opt_submsg) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_SubMessage_AllTypes) +#endif + +/* On some platforms (such as AVR), double is really float. + * These are not directly supported by nanopb, but see example_avr_double. + */ +STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES) + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/tests/backwards_compatibility/decode_legacy.c b/tests/backwards_compatibility/decode_legacy.c new file mode 100644 index 00000000..b74172fd --- /dev/null +++ b/tests/backwards_compatibility/decode_legacy.c @@ -0,0 +1,197 @@ +/* Tests the decoding of all types. + * This is a backwards-compatibility test, using bc_alltypes.pb.h. + * It is similar to test_decode3, but duplicated in order to allow + * test_decode3 to test any new features introduced later. + * + * Run e.g. ./bc_encode | ./bc_decode + */ + +#include +#include +#include +#include +#include "bc_alltypes.pb.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed.\n"); \ + return false; \ + } + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_alltypes(pb_istream_t *stream, int mode) +{ + AllTypes alltypes; + + /* Fill with garbage to better detect initialization errors */ + memset(&alltypes, 0xAA, sizeof(alltypes)); + + if (!pb_decode(stream, AllTypes_fields, &alltypes)) + return false; + + TEST(alltypes.req_int32 == -1001); + TEST(alltypes.req_int64 == -1002); + TEST(alltypes.req_uint32 == 1003); + TEST(alltypes.req_uint64 == 1004); + TEST(alltypes.req_sint32 == -1005); + TEST(alltypes.req_sint64 == -1006); + TEST(alltypes.req_bool == true); + + TEST(alltypes.req_fixed32 == 1008); + TEST(alltypes.req_sfixed32 == -1009); + TEST(alltypes.req_float == 1010.0f); + + TEST(alltypes.req_fixed64 == 1011); + TEST(alltypes.req_sfixed64 == -1012); + TEST(alltypes.req_double == 1013.0f); + + TEST(strcmp(alltypes.req_string, "1014") == 0); + TEST(alltypes.req_bytes.size == 4); + TEST(memcmp(alltypes.req_bytes.bytes, "1015", 4) == 0); + TEST(strcmp(alltypes.req_submsg.substuff1, "1016") == 0); + TEST(alltypes.req_submsg.substuff2 == 1016); + TEST(alltypes.req_submsg.substuff3 == 3); + TEST(alltypes.req_enum == MyEnum_Truth); + + TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); + TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); + TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); + TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); + TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); + TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0); + TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); + + TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); + TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); + TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); + + TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); + TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0); + TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); + + TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); + TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); + TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); + + TEST(alltypes.rep_submsg_count == 5); + TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); + TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); + TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3); + + TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); + + if (mode == 0) + { + /* Expect default values */ + TEST(alltypes.has_opt_int32 == false); + TEST(alltypes.opt_int32 == 4041); + TEST(alltypes.has_opt_int64 == false); + TEST(alltypes.opt_int64 == 4042); + TEST(alltypes.has_opt_uint32 == false); + TEST(alltypes.opt_uint32 == 4043); + TEST(alltypes.has_opt_uint64 == false); + TEST(alltypes.opt_uint64 == 4044); + TEST(alltypes.has_opt_sint32 == false); + TEST(alltypes.opt_sint32 == 4045); + TEST(alltypes.has_opt_sint64 == false); + TEST(alltypes.opt_sint64 == 4046); + TEST(alltypes.has_opt_bool == false); + TEST(alltypes.opt_bool == false); + + TEST(alltypes.has_opt_fixed32 == false); + TEST(alltypes.opt_fixed32 == 4048); + TEST(alltypes.has_opt_sfixed32 == false); + TEST(alltypes.opt_sfixed32 == 4049); + TEST(alltypes.has_opt_float == false); + TEST(alltypes.opt_float == 4050.0f); + + TEST(alltypes.has_opt_fixed64 == false); + TEST(alltypes.opt_fixed64 == 4051); + TEST(alltypes.has_opt_sfixed64 == false); + TEST(alltypes.opt_sfixed64 == 4052); + TEST(alltypes.has_opt_double == false); + TEST(alltypes.opt_double == 4053.0); + + TEST(alltypes.has_opt_string == false); + TEST(strcmp(alltypes.opt_string, "4054") == 0); + TEST(alltypes.has_opt_bytes == false); + TEST(alltypes.opt_bytes.size == 4); + TEST(memcmp(alltypes.opt_bytes.bytes, "4055", 4) == 0); + TEST(alltypes.has_opt_submsg == false); + TEST(strcmp(alltypes.opt_submsg.substuff1, "1") == 0); + TEST(alltypes.opt_submsg.substuff2 == 2); + TEST(alltypes.opt_submsg.substuff3 == 3); + TEST(alltypes.has_opt_enum == false); + TEST(alltypes.opt_enum == MyEnum_Second); + } + else + { + /* Expect filled-in values */ + TEST(alltypes.has_opt_int32 == true); + TEST(alltypes.opt_int32 == 3041); + TEST(alltypes.has_opt_int64 == true); + TEST(alltypes.opt_int64 == 3042); + TEST(alltypes.has_opt_uint32 == true); + TEST(alltypes.opt_uint32 == 3043); + TEST(alltypes.has_opt_uint64 == true); + TEST(alltypes.opt_uint64 == 3044); + TEST(alltypes.has_opt_sint32 == true); + TEST(alltypes.opt_sint32 == 3045); + TEST(alltypes.has_opt_sint64 == true); + TEST(alltypes.opt_sint64 == 3046); + TEST(alltypes.has_opt_bool == true); + TEST(alltypes.opt_bool == true); + + TEST(alltypes.has_opt_fixed32 == true); + TEST(alltypes.opt_fixed32 == 3048); + TEST(alltypes.has_opt_sfixed32 == true); + TEST(alltypes.opt_sfixed32 == 3049); + TEST(alltypes.has_opt_float == true); + TEST(alltypes.opt_float == 3050.0f); + + TEST(alltypes.has_opt_fixed64 == true); + TEST(alltypes.opt_fixed64 == 3051); + TEST(alltypes.has_opt_sfixed64 == true); + TEST(alltypes.opt_sfixed64 == 3052); + TEST(alltypes.has_opt_double == true); + TEST(alltypes.opt_double == 3053.0); + + TEST(alltypes.has_opt_string == true); + TEST(strcmp(alltypes.opt_string, "3054") == 0); + TEST(alltypes.has_opt_bytes == true); + TEST(alltypes.opt_bytes.size == 4); + TEST(memcmp(alltypes.opt_bytes.bytes, "3055", 4) == 0); + TEST(alltypes.has_opt_submsg == true); + TEST(strcmp(alltypes.opt_submsg.substuff1, "3056") == 0); + TEST(alltypes.opt_submsg.substuff2 == 3056); + TEST(alltypes.opt_submsg.substuff3 == 3); + TEST(alltypes.has_opt_enum == true); + TEST(alltypes.opt_enum == MyEnum_Truth); + } + + TEST(alltypes.end == 1099); + + return true; +} + +int main(int argc, char **argv) +{ + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Read the data into buffer */ + uint8_t buffer[1024]; + size_t count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + pb_istream_t stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_alltypes(&stream, mode)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} diff --git a/tests/backwards_compatibility/encode_legacy.c b/tests/backwards_compatibility/encode_legacy.c new file mode 100644 index 00000000..e84f0908 --- /dev/null +++ b/tests/backwards_compatibility/encode_legacy.c @@ -0,0 +1,131 @@ +/* Attempts to test all the datatypes supported by ProtoBuf. + * This is a backwards-compatibility test, using bc_alltypes.pb.h. + * It is similar to test_encode3, but duplicated in order to allow + * test_encode3 to test any new features introduced later. + */ + +#include +#include +#include +#include +#include "bc_alltypes.pb.h" + +int main(int argc, char **argv) +{ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Initialize the structure with constants */ + AllTypes alltypes = {0}; + + alltypes.req_int32 = -1001; + alltypes.req_int64 = -1002; + alltypes.req_uint32 = 1003; + alltypes.req_uint64 = 1004; + alltypes.req_sint32 = -1005; + alltypes.req_sint64 = -1006; + alltypes.req_bool = true; + + alltypes.req_fixed32 = 1008; + alltypes.req_sfixed32 = -1009; + alltypes.req_float = 1010.0f; + + alltypes.req_fixed64 = 1011; + alltypes.req_sfixed64 = -1012; + alltypes.req_double = 1013.0; + + strcpy(alltypes.req_string, "1014"); + alltypes.req_bytes.size = 4; + memcpy(alltypes.req_bytes.bytes, "1015", 4); + strcpy(alltypes.req_submsg.substuff1, "1016"); + alltypes.req_submsg.substuff2 = 1016; + alltypes.req_enum = MyEnum_Truth; + + alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; + alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002; + alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; + alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004; + alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005; + alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006; + alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; + + alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; + alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009; + alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; + + alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011; + alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012; + alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0; + + alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); + alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4; + memcpy(alltypes.rep_bytes[4].bytes, "2015", 4); + + alltypes.rep_submsg_count = 5; + strcpy(alltypes.rep_submsg[4].substuff1, "2016"); + alltypes.rep_submsg[4].substuff2 = 2016; + alltypes.rep_submsg[4].has_substuff3 = true; + alltypes.rep_submsg[4].substuff3 = 2016; + + alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; + + if (mode != 0) + { + /* Fill in values for optional fields */ + alltypes.has_opt_int32 = true; + alltypes.opt_int32 = 3041; + alltypes.has_opt_int64 = true; + alltypes.opt_int64 = 3042; + alltypes.has_opt_uint32 = true; + alltypes.opt_uint32 = 3043; + alltypes.has_opt_uint64 = true; + alltypes.opt_uint64 = 3044; + alltypes.has_opt_sint32 = true; + alltypes.opt_sint32 = 3045; + alltypes.has_opt_sint64 = true; + alltypes.opt_sint64 = 3046; + alltypes.has_opt_bool = true; + alltypes.opt_bool = true; + + alltypes.has_opt_fixed32 = true; + alltypes.opt_fixed32 = 3048; + alltypes.has_opt_sfixed32 = true; + alltypes.opt_sfixed32 = 3049; + alltypes.has_opt_float = true; + alltypes.opt_float = 3050.0f; + + alltypes.has_opt_fixed64 = true; + alltypes.opt_fixed64 = 3051; + alltypes.has_opt_sfixed64 = true; + alltypes.opt_sfixed64 = 3052; + alltypes.has_opt_double = true; + alltypes.opt_double = 3053.0; + + alltypes.has_opt_string = true; + strcpy(alltypes.opt_string, "3054"); + alltypes.has_opt_bytes = true; + alltypes.opt_bytes.size = 4; + memcpy(alltypes.opt_bytes.bytes, "3055", 4); + alltypes.has_opt_submsg = true; + strcpy(alltypes.opt_submsg.substuff1, "3056"); + alltypes.opt_submsg.substuff2 = 3056; + alltypes.has_opt_enum = true; + alltypes.opt_enum = MyEnum_Truth; + } + + alltypes.end = 1099; + + uint8_t buffer[1024]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed!\n"); + return 1; /* Failure */ + } +} diff --git a/tests/basic_buffer/SConscript b/tests/basic_buffer/SConscript new file mode 100644 index 00000000..5b85e132 --- /dev/null +++ b/tests/basic_buffer/SConscript @@ -0,0 +1,12 @@ +# Build and run a basic round-trip test using memory buffer encoding. + +Import("env") + +env.Program(["encode_buffer.c", "#common/person.pb.c", "#common/pb_encode.o"]) +env.Program(["decode_buffer.c", "#common/person.pb.c", "#common/pb_decode.o"]) + +env.RunTest("encode_buffer") +env.RunTest(["decode_buffer", "encode_buffer.output"]) +env.Decode(["encode_buffer.output", "#common/person.proto"], MESSAGE = "Person") +env.Compare(["decode_buffer.output", "encode_buffer.decoded"]) + diff --git a/tests/basic_buffer/decode_buffer.c b/tests/basic_buffer/decode_buffer.c new file mode 100644 index 00000000..56bbd8f8 --- /dev/null +++ b/tests/basic_buffer/decode_buffer.c @@ -0,0 +1,83 @@ +/* A very simple decoding test case, using person.proto. + * Produces output compatible with protoc --decode. + * Reads the encoded data from stdin and prints the values + * to stdout as text. + * + * Run e.g. ./test_encode1 | ./test_decode1 + */ + +#include +#include +#include "person.pb.h" + +/* This function is called once from main(), it handles + the decoding and printing. */ +bool print_person(pb_istream_t *stream) +{ + int i; + Person person; + + if (!pb_decode(stream, Person_fields, &person)) + return false; + + /* Now the decoding is done, rest is just to print stuff out. */ + + printf("name: \"%s\"\n", person.name); + printf("id: %ld\n", (long)person.id); + + if (person.has_email) + printf("email: \"%s\"\n", person.email); + + for (i = 0; i < person.phone_count; i++) + { + Person_PhoneNumber *phone = &person.phone[i]; + printf("phone {\n"); + printf(" number: \"%s\"\n", phone->number); + + if (phone->has_type) + { + switch (phone->type) + { + case Person_PhoneType_WORK: + printf(" type: WORK\n"); + break; + + case Person_PhoneType_HOME: + printf(" type: HOME\n"); + break; + + case Person_PhoneType_MOBILE: + printf(" type: MOBILE\n"); + break; + } + } + printf("}\n"); + } + + return true; +} + +int main() +{ + /* Read the data into buffer */ + uint8_t buffer[512]; + size_t count = fread(buffer, 1, sizeof(buffer), stdin); + + if (!feof(stdin)) + { + printf("Message does not fit in buffer\n"); + return 1; + } + + /* Construct a pb_istream_t for reading from the buffer */ + pb_istream_t stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!print_person(&stream)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} diff --git a/tests/basic_buffer/encode_buffer.c b/tests/basic_buffer/encode_buffer.c new file mode 100644 index 00000000..742c99f4 --- /dev/null +++ b/tests/basic_buffer/encode_buffer.c @@ -0,0 +1,33 @@ +/* A very simple encoding test case using person.proto. + * Just puts constant data in the fields and encodes into + * buffer, which is then written to stdout. + */ + +#include +#include +#include "person.pb.h" + +int main() +{ + /* Initialize the structure with constants */ + Person person = {"Test Person 99", 99, true, "test@person.com", + 3, {{"555-12345678", true, Person_PhoneType_MOBILE}, + {"99-2342", false, 0}, + {"1234-5678", true, Person_PhoneType_WORK}, + }}; + + uint8_t buffer[512]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, Person_fields, &person)) + { + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } +} diff --git a/tests/basic_stream/decode_stream.c b/tests/basic_stream/decode_stream.c new file mode 100644 index 00000000..2142977e --- /dev/null +++ b/tests/basic_stream/decode_stream.c @@ -0,0 +1,83 @@ +/* Same as test_decode1 but reads from stdin directly. + */ + +#include +#include +#include "person.pb.h" + +/* This function is called once from main(), it handles + the decoding and printing. + Ugly copy-paste from test_decode1.c. */ +bool print_person(pb_istream_t *stream) +{ + int i; + Person person; + + if (!pb_decode(stream, Person_fields, &person)) + return false; + + /* Now the decoding is done, rest is just to print stuff out. */ + + printf("name: \"%s\"\n", person.name); + printf("id: %ld\n", (long)person.id); + + if (person.has_email) + printf("email: \"%s\"\n", person.email); + + for (i = 0; i < person.phone_count; i++) + { + Person_PhoneNumber *phone = &person.phone[i]; + printf("phone {\n"); + printf(" number: \"%s\"\n", phone->number); + + if (phone->has_type) + { + switch (phone->type) + { + case Person_PhoneType_WORK: + printf(" type: WORK\n"); + break; + + case Person_PhoneType_HOME: + printf(" type: HOME\n"); + break; + + case Person_PhoneType_MOBILE: + printf(" type: MOBILE\n"); + break; + } + } + printf("}\n"); + } + + return true; +} + +/* This binds the pb_istream_t to stdin */ +bool callback(pb_istream_t *stream, uint8_t *buf, size_t count) +{ + FILE *file = (FILE*)stream->state; + bool status; + + status = (fread(buf, 1, count, file) == count); + + if (feof(file)) + stream->bytes_left = 0; + + return status; +} + +int main() +{ + /* Maximum size is specified to prevent infinite length messages from + * hanging this in the fuzz test. + */ + pb_istream_t stream = {&callback, stdin, 10000}; + if (!print_person(&stream)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} diff --git a/tests/basic_stream/encode_stream.c b/tests/basic_stream/encode_stream.c new file mode 100644 index 00000000..fd25c6cb --- /dev/null +++ b/tests/basic_stream/encode_stream.c @@ -0,0 +1,37 @@ +/* Same as test_encode1.c, except writes directly to stdout. + */ + +#include +#include +#include "person.pb.h" + +/* This binds the pb_ostream_t into the stdout stream */ +bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) +{ + FILE *file = (FILE*) stream->state; + return fwrite(buf, 1, count, file) == count; +} + +int main() +{ + /* Initialize the structure with constants */ + Person person = {"Test Person 99", 99, true, "test@person.com", + 3, {{"555-12345678", true, Person_PhoneType_MOBILE}, + {"99-2342", false, 0}, + {"1234-5678", true, Person_PhoneType_WORK}, + }}; + + /* Prepare the stream, output goes directly to stdout */ + pb_ostream_t stream = {&streamcallback, stdout, SIZE_MAX, 0}; + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, Person_fields, &person)) + { + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } +} diff --git a/tests/bc_alltypes.pb.c b/tests/bc_alltypes.pb.c deleted file mode 100644 index b144b1e3..00000000 --- a/tests/bc_alltypes.pb.c +++ /dev/null @@ -1,93 +0,0 @@ -/* Automatically generated nanopb constant definitions */ -/* Generated by 0.2.0-dev at Sun Feb 17 00:09:53 2013. */ -/* This is a file generated using nanopb-0.2.0-dev. - * It is used as a part of test suite in order to detect any - * incompatible changes made to the generator in future versions. - */ - -#include "bc_alltypes.pb.h" - -const char SubMessage_substuff1_default[16] = "1"; -const int32_t SubMessage_substuff2_default = 2; -const uint32_t SubMessage_substuff3_default = 3; -const int32_t AllTypes_opt_int32_default = 4041; -const int64_t AllTypes_opt_int64_default = 4042; -const uint32_t AllTypes_opt_uint32_default = 4043; -const uint64_t AllTypes_opt_uint64_default = 4044; -const int32_t AllTypes_opt_sint32_default = 4045; -const int64_t AllTypes_opt_sint64_default = 4046; -const bool AllTypes_opt_bool_default = false; -const uint32_t AllTypes_opt_fixed32_default = 4048; -const int32_t AllTypes_opt_sfixed32_default = 4049; -const float AllTypes_opt_float_default = 4050; -const uint64_t AllTypes_opt_fixed64_default = 4051; -const int64_t AllTypes_opt_sfixed64_default = 4052; -const double AllTypes_opt_double_default = 4053; -const char AllTypes_opt_string_default[16] = "4054"; -const AllTypes_opt_bytes_t AllTypes_opt_bytes_default = {4, {0x34,0x30,0x35,0x35}}; -const MyEnum AllTypes_opt_enum_default = MyEnum_Second; - - -const pb_field_t SubMessage_fields[4] = { - PB_FIELD( 1, STRING , REQUIRED, STATIC, SubMessage, substuff1, substuff1, &SubMessage_substuff1_default), - PB_FIELD( 2, INT32 , REQUIRED, STATIC, SubMessage, substuff2, substuff1, &SubMessage_substuff2_default), - PB_FIELD( 3, FIXED32 , OPTIONAL, STATIC, SubMessage, substuff3, substuff2, &SubMessage_substuff3_default), - PB_LAST_FIELD -}; - -const pb_field_t AllTypes_fields[53] = { - PB_FIELD( 1, INT32 , REQUIRED, STATIC, AllTypes, req_int32, req_int32, 0), - PB_FIELD( 2, INT64 , REQUIRED, STATIC, AllTypes, req_int64, req_int32, 0), - PB_FIELD( 3, UINT32 , REQUIRED, STATIC, AllTypes, req_uint32, req_int64, 0), - PB_FIELD( 4, UINT64 , REQUIRED, STATIC, AllTypes, req_uint64, req_uint32, 0), - PB_FIELD( 5, SINT32 , REQUIRED, STATIC, AllTypes, req_sint32, req_uint64, 0), - PB_FIELD( 6, SINT64 , REQUIRED, STATIC, AllTypes, req_sint64, req_sint32, 0), - PB_FIELD( 7, BOOL , REQUIRED, STATIC, AllTypes, req_bool, req_sint64, 0), - PB_FIELD( 8, FIXED32 , REQUIRED, STATIC, AllTypes, req_fixed32, req_bool, 0), - PB_FIELD( 9, SFIXED32, REQUIRED, STATIC, AllTypes, req_sfixed32, req_fixed32, 0), - PB_FIELD( 10, FLOAT , REQUIRED, STATIC, AllTypes, req_float, req_sfixed32, 0), - PB_FIELD( 11, FIXED64 , REQUIRED, STATIC, AllTypes, req_fixed64, req_float, 0), - PB_FIELD( 12, SFIXED64, REQUIRED, STATIC, AllTypes, req_sfixed64, req_fixed64, 0), - PB_FIELD( 13, DOUBLE , REQUIRED, STATIC, AllTypes, req_double, req_sfixed64, 0), - PB_FIELD( 14, STRING , REQUIRED, STATIC, AllTypes, req_string, req_double, 0), - PB_FIELD( 15, BYTES , REQUIRED, STATIC, AllTypes, req_bytes, req_string, 0), - PB_FIELD( 16, MESSAGE , REQUIRED, STATIC, AllTypes, req_submsg, req_bytes, &SubMessage_fields), - PB_FIELD( 17, ENUM , REQUIRED, STATIC, AllTypes, req_enum, req_submsg, 0), - PB_FIELD( 21, INT32 , REPEATED, STATIC, AllTypes, rep_int32, req_enum, 0), - PB_FIELD( 22, INT64 , REPEATED, STATIC, AllTypes, rep_int64, rep_int32, 0), - PB_FIELD( 23, UINT32 , REPEATED, STATIC, AllTypes, rep_uint32, rep_int64, 0), - PB_FIELD( 24, UINT64 , REPEATED, STATIC, AllTypes, rep_uint64, rep_uint32, 0), - PB_FIELD( 25, SINT32 , REPEATED, STATIC, AllTypes, rep_sint32, rep_uint64, 0), - PB_FIELD( 26, SINT64 , REPEATED, STATIC, AllTypes, rep_sint64, rep_sint32, 0), - PB_FIELD( 27, BOOL , REPEATED, STATIC, AllTypes, rep_bool, rep_sint64, 0), - PB_FIELD( 28, FIXED32 , REPEATED, STATIC, AllTypes, rep_fixed32, rep_bool, 0), - PB_FIELD( 29, SFIXED32, REPEATED, STATIC, AllTypes, rep_sfixed32, rep_fixed32, 0), - PB_FIELD( 30, FLOAT , REPEATED, STATIC, AllTypes, rep_float, rep_sfixed32, 0), - PB_FIELD( 31, FIXED64 , REPEATED, STATIC, AllTypes, rep_fixed64, rep_float, 0), - PB_FIELD( 32, SFIXED64, REPEATED, STATIC, AllTypes, rep_sfixed64, rep_fixed64, 0), - PB_FIELD( 33, DOUBLE , REPEATED, STATIC, AllTypes, rep_double, rep_sfixed64, 0), - PB_FIELD( 34, STRING , REPEATED, STATIC, AllTypes, rep_string, rep_double, 0), - PB_FIELD( 35, BYTES , REPEATED, STATIC, AllTypes, rep_bytes, rep_string, 0), - PB_FIELD( 36, MESSAGE , REPEATED, STATIC, AllTypes, rep_submsg, rep_bytes, &SubMessage_fields), - PB_FIELD( 37, ENUM , REPEATED, STATIC, AllTypes, rep_enum, rep_submsg, 0), - PB_FIELD( 41, INT32 , OPTIONAL, STATIC, AllTypes, opt_int32, rep_enum, &AllTypes_opt_int32_default), - PB_FIELD( 42, INT64 , OPTIONAL, STATIC, AllTypes, opt_int64, opt_int32, &AllTypes_opt_int64_default), - PB_FIELD( 43, UINT32 , OPTIONAL, STATIC, AllTypes, opt_uint32, opt_int64, &AllTypes_opt_uint32_default), - PB_FIELD( 44, UINT64 , OPTIONAL, STATIC, AllTypes, opt_uint64, opt_uint32, &AllTypes_opt_uint64_default), - PB_FIELD( 45, SINT32 , OPTIONAL, STATIC, AllTypes, opt_sint32, opt_uint64, &AllTypes_opt_sint32_default), - PB_FIELD( 46, SINT64 , OPTIONAL, STATIC, AllTypes, opt_sint64, opt_sint32, &AllTypes_opt_sint64_default), - PB_FIELD( 47, BOOL , OPTIONAL, STATIC, AllTypes, opt_bool, opt_sint64, &AllTypes_opt_bool_default), - PB_FIELD( 48, FIXED32 , OPTIONAL, STATIC, AllTypes, opt_fixed32, opt_bool, &AllTypes_opt_fixed32_default), - PB_FIELD( 49, SFIXED32, OPTIONAL, STATIC, AllTypes, opt_sfixed32, opt_fixed32, &AllTypes_opt_sfixed32_default), - PB_FIELD( 50, FLOAT , OPTIONAL, STATIC, AllTypes, opt_float, opt_sfixed32, &AllTypes_opt_float_default), - PB_FIELD( 51, FIXED64 , OPTIONAL, STATIC, AllTypes, opt_fixed64, opt_float, &AllTypes_opt_fixed64_default), - PB_FIELD( 52, SFIXED64, OPTIONAL, STATIC, AllTypes, opt_sfixed64, opt_fixed64, &AllTypes_opt_sfixed64_default), - PB_FIELD( 53, DOUBLE , OPTIONAL, STATIC, AllTypes, opt_double, opt_sfixed64, &AllTypes_opt_double_default), - PB_FIELD( 54, STRING , OPTIONAL, STATIC, AllTypes, opt_string, opt_double, &AllTypes_opt_string_default), - PB_FIELD( 55, BYTES , OPTIONAL, STATIC, AllTypes, opt_bytes, opt_string, &AllTypes_opt_bytes_default), - PB_FIELD( 56, MESSAGE , OPTIONAL, STATIC, AllTypes, opt_submsg, opt_bytes, &SubMessage_fields), - PB_FIELD( 57, ENUM , OPTIONAL, STATIC, AllTypes, opt_enum, opt_submsg, &AllTypes_opt_enum_default), - PB_FIELD( 99, INT32 , REQUIRED, STATIC, AllTypes, end, opt_enum, 0), - PB_LAST_FIELD -}; - diff --git a/tests/bc_alltypes.pb.h b/tests/bc_alltypes.pb.h deleted file mode 100644 index 037b3478..00000000 --- a/tests/bc_alltypes.pb.h +++ /dev/null @@ -1,178 +0,0 @@ -/* Automatically generated nanopb header */ -/* This is a file generated using nanopb-0.2.0-dev. - * It is used as a part of test suite in order to detect any - * incompatible changes made to the generator in future versions. - */ - -#ifndef _PB_ALLTYPES_PB_H_ -#define _PB_ALLTYPES_PB_H_ -#include - -#ifdef __cplusplus -extern "C" { -#endif - -/* Enum definitions */ -typedef enum _MyEnum { - MyEnum_Zero = 0, - MyEnum_First = 1, - MyEnum_Second = 2, - MyEnum_Truth = 42 -} MyEnum; - -/* Struct definitions */ -typedef struct _SubMessage { - char substuff1[16]; - int32_t substuff2; - bool has_substuff3; - uint32_t substuff3; -} SubMessage; - -typedef struct { - size_t size; - uint8_t bytes[16]; -} AllTypes_req_bytes_t; - -typedef struct { - size_t size; - uint8_t bytes[16]; -} AllTypes_rep_bytes_t; - -typedef struct { - size_t size; - uint8_t bytes[16]; -} AllTypes_opt_bytes_t; - -typedef struct _AllTypes { - int32_t req_int32; - int64_t req_int64; - uint32_t req_uint32; - uint64_t req_uint64; - int32_t req_sint32; - int64_t req_sint64; - bool req_bool; - uint32_t req_fixed32; - int32_t req_sfixed32; - float req_float; - uint64_t req_fixed64; - int64_t req_sfixed64; - double req_double; - char req_string[16]; - AllTypes_req_bytes_t req_bytes; - SubMessage req_submsg; - MyEnum req_enum; - size_t rep_int32_count; - int32_t rep_int32[5]; - size_t rep_int64_count; - int64_t rep_int64[5]; - size_t rep_uint32_count; - uint32_t rep_uint32[5]; - size_t rep_uint64_count; - uint64_t rep_uint64[5]; - size_t rep_sint32_count; - int32_t rep_sint32[5]; - size_t rep_sint64_count; - int64_t rep_sint64[5]; - size_t rep_bool_count; - bool rep_bool[5]; - size_t rep_fixed32_count; - uint32_t rep_fixed32[5]; - size_t rep_sfixed32_count; - int32_t rep_sfixed32[5]; - size_t rep_float_count; - float rep_float[5]; - size_t rep_fixed64_count; - uint64_t rep_fixed64[5]; - size_t rep_sfixed64_count; - int64_t rep_sfixed64[5]; - size_t rep_double_count; - double rep_double[5]; - size_t rep_string_count; - char rep_string[5][16]; - size_t rep_bytes_count; - AllTypes_rep_bytes_t rep_bytes[5]; - size_t rep_submsg_count; - SubMessage rep_submsg[5]; - size_t rep_enum_count; - MyEnum rep_enum[5]; - bool has_opt_int32; - int32_t opt_int32; - bool has_opt_int64; - int64_t opt_int64; - bool has_opt_uint32; - uint32_t opt_uint32; - bool has_opt_uint64; - uint64_t opt_uint64; - bool has_opt_sint32; - int32_t opt_sint32; - bool has_opt_sint64; - int64_t opt_sint64; - bool has_opt_bool; - bool opt_bool; - bool has_opt_fixed32; - uint32_t opt_fixed32; - bool has_opt_sfixed32; - int32_t opt_sfixed32; - bool has_opt_float; - float opt_float; - bool has_opt_fixed64; - uint64_t opt_fixed64; - bool has_opt_sfixed64; - int64_t opt_sfixed64; - bool has_opt_double; - double opt_double; - bool has_opt_string; - char opt_string[16]; - bool has_opt_bytes; - AllTypes_opt_bytes_t opt_bytes; - bool has_opt_submsg; - SubMessage opt_submsg; - bool has_opt_enum; - MyEnum opt_enum; - int32_t end; -} AllTypes; - -/* Default values for struct fields */ -extern const char SubMessage_substuff1_default[16]; -extern const int32_t SubMessage_substuff2_default; -extern const uint32_t SubMessage_substuff3_default; -extern const int32_t AllTypes_opt_int32_default; -extern const int64_t AllTypes_opt_int64_default; -extern const uint32_t AllTypes_opt_uint32_default; -extern const uint64_t AllTypes_opt_uint64_default; -extern const int32_t AllTypes_opt_sint32_default; -extern const int64_t AllTypes_opt_sint64_default; -extern const bool AllTypes_opt_bool_default; -extern const uint32_t AllTypes_opt_fixed32_default; -extern const int32_t AllTypes_opt_sfixed32_default; -extern const float AllTypes_opt_float_default; -extern const uint64_t AllTypes_opt_fixed64_default; -extern const int64_t AllTypes_opt_sfixed64_default; -extern const double AllTypes_opt_double_default; -extern const char AllTypes_opt_string_default[16]; -extern const AllTypes_opt_bytes_t AllTypes_opt_bytes_default; -extern const MyEnum AllTypes_opt_enum_default; - -/* Struct field encoding specification for nanopb */ -extern const pb_field_t SubMessage_fields[4]; -extern const pb_field_t AllTypes_fields[53]; - -/* Check that field information fits in pb_field_t */ -#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) -STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 256 && pb_membersize(AllTypes, rep_submsg[0]) < 256 && pb_membersize(AllTypes, opt_submsg) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_SubMessage_AllTypes) -#endif - -#if !defined(PB_FIELD_32BIT) -STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 65536 && pb_membersize(AllTypes, rep_submsg[0]) < 65536 && pb_membersize(AllTypes, opt_submsg) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_SubMessage_AllTypes) -#endif - -/* On some platforms (such as AVR), double is really float. - * These are not directly supported by nanopb, but see example_avr_double. - */ -STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES) - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif diff --git a/tests/bc_decode.c b/tests/bc_decode.c deleted file mode 100644 index b74172fd..00000000 --- a/tests/bc_decode.c +++ /dev/null @@ -1,197 +0,0 @@ -/* Tests the decoding of all types. - * This is a backwards-compatibility test, using bc_alltypes.pb.h. - * It is similar to test_decode3, but duplicated in order to allow - * test_decode3 to test any new features introduced later. - * - * Run e.g. ./bc_encode | ./bc_decode - */ - -#include -#include -#include -#include -#include "bc_alltypes.pb.h" - -#define TEST(x) if (!(x)) { \ - printf("Test " #x " failed.\n"); \ - return false; \ - } - -/* This function is called once from main(), it handles - the decoding and checks the fields. */ -bool check_alltypes(pb_istream_t *stream, int mode) -{ - AllTypes alltypes; - - /* Fill with garbage to better detect initialization errors */ - memset(&alltypes, 0xAA, sizeof(alltypes)); - - if (!pb_decode(stream, AllTypes_fields, &alltypes)) - return false; - - TEST(alltypes.req_int32 == -1001); - TEST(alltypes.req_int64 == -1002); - TEST(alltypes.req_uint32 == 1003); - TEST(alltypes.req_uint64 == 1004); - TEST(alltypes.req_sint32 == -1005); - TEST(alltypes.req_sint64 == -1006); - TEST(alltypes.req_bool == true); - - TEST(alltypes.req_fixed32 == 1008); - TEST(alltypes.req_sfixed32 == -1009); - TEST(alltypes.req_float == 1010.0f); - - TEST(alltypes.req_fixed64 == 1011); - TEST(alltypes.req_sfixed64 == -1012); - TEST(alltypes.req_double == 1013.0f); - - TEST(strcmp(alltypes.req_string, "1014") == 0); - TEST(alltypes.req_bytes.size == 4); - TEST(memcmp(alltypes.req_bytes.bytes, "1015", 4) == 0); - TEST(strcmp(alltypes.req_submsg.substuff1, "1016") == 0); - TEST(alltypes.req_submsg.substuff2 == 1016); - TEST(alltypes.req_submsg.substuff3 == 3); - TEST(alltypes.req_enum == MyEnum_Truth); - - TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); - TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); - TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); - TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); - TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); - TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0); - TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); - - TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); - TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); - TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); - - TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); - TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0); - TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); - - TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); - TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); - TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); - - TEST(alltypes.rep_submsg_count == 5); - TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); - TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); - TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3); - - TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); - - if (mode == 0) - { - /* Expect default values */ - TEST(alltypes.has_opt_int32 == false); - TEST(alltypes.opt_int32 == 4041); - TEST(alltypes.has_opt_int64 == false); - TEST(alltypes.opt_int64 == 4042); - TEST(alltypes.has_opt_uint32 == false); - TEST(alltypes.opt_uint32 == 4043); - TEST(alltypes.has_opt_uint64 == false); - TEST(alltypes.opt_uint64 == 4044); - TEST(alltypes.has_opt_sint32 == false); - TEST(alltypes.opt_sint32 == 4045); - TEST(alltypes.has_opt_sint64 == false); - TEST(alltypes.opt_sint64 == 4046); - TEST(alltypes.has_opt_bool == false); - TEST(alltypes.opt_bool == false); - - TEST(alltypes.has_opt_fixed32 == false); - TEST(alltypes.opt_fixed32 == 4048); - TEST(alltypes.has_opt_sfixed32 == false); - TEST(alltypes.opt_sfixed32 == 4049); - TEST(alltypes.has_opt_float == false); - TEST(alltypes.opt_float == 4050.0f); - - TEST(alltypes.has_opt_fixed64 == false); - TEST(alltypes.opt_fixed64 == 4051); - TEST(alltypes.has_opt_sfixed64 == false); - TEST(alltypes.opt_sfixed64 == 4052); - TEST(alltypes.has_opt_double == false); - TEST(alltypes.opt_double == 4053.0); - - TEST(alltypes.has_opt_string == false); - TEST(strcmp(alltypes.opt_string, "4054") == 0); - TEST(alltypes.has_opt_bytes == false); - TEST(alltypes.opt_bytes.size == 4); - TEST(memcmp(alltypes.opt_bytes.bytes, "4055", 4) == 0); - TEST(alltypes.has_opt_submsg == false); - TEST(strcmp(alltypes.opt_submsg.substuff1, "1") == 0); - TEST(alltypes.opt_submsg.substuff2 == 2); - TEST(alltypes.opt_submsg.substuff3 == 3); - TEST(alltypes.has_opt_enum == false); - TEST(alltypes.opt_enum == MyEnum_Second); - } - else - { - /* Expect filled-in values */ - TEST(alltypes.has_opt_int32 == true); - TEST(alltypes.opt_int32 == 3041); - TEST(alltypes.has_opt_int64 == true); - TEST(alltypes.opt_int64 == 3042); - TEST(alltypes.has_opt_uint32 == true); - TEST(alltypes.opt_uint32 == 3043); - TEST(alltypes.has_opt_uint64 == true); - TEST(alltypes.opt_uint64 == 3044); - TEST(alltypes.has_opt_sint32 == true); - TEST(alltypes.opt_sint32 == 3045); - TEST(alltypes.has_opt_sint64 == true); - TEST(alltypes.opt_sint64 == 3046); - TEST(alltypes.has_opt_bool == true); - TEST(alltypes.opt_bool == true); - - TEST(alltypes.has_opt_fixed32 == true); - TEST(alltypes.opt_fixed32 == 3048); - TEST(alltypes.has_opt_sfixed32 == true); - TEST(alltypes.opt_sfixed32 == 3049); - TEST(alltypes.has_opt_float == true); - TEST(alltypes.opt_float == 3050.0f); - - TEST(alltypes.has_opt_fixed64 == true); - TEST(alltypes.opt_fixed64 == 3051); - TEST(alltypes.has_opt_sfixed64 == true); - TEST(alltypes.opt_sfixed64 == 3052); - TEST(alltypes.has_opt_double == true); - TEST(alltypes.opt_double == 3053.0); - - TEST(alltypes.has_opt_string == true); - TEST(strcmp(alltypes.opt_string, "3054") == 0); - TEST(alltypes.has_opt_bytes == true); - TEST(alltypes.opt_bytes.size == 4); - TEST(memcmp(alltypes.opt_bytes.bytes, "3055", 4) == 0); - TEST(alltypes.has_opt_submsg == true); - TEST(strcmp(alltypes.opt_submsg.substuff1, "3056") == 0); - TEST(alltypes.opt_submsg.substuff2 == 3056); - TEST(alltypes.opt_submsg.substuff3 == 3); - TEST(alltypes.has_opt_enum == true); - TEST(alltypes.opt_enum == MyEnum_Truth); - } - - TEST(alltypes.end == 1099); - - return true; -} - -int main(int argc, char **argv) -{ - /* Whether to expect the optional values or the default values. */ - int mode = (argc > 1) ? atoi(argv[1]) : 0; - - /* Read the data into buffer */ - uint8_t buffer[1024]; - size_t count = fread(buffer, 1, sizeof(buffer), stdin); - - /* Construct a pb_istream_t for reading from the buffer */ - pb_istream_t stream = pb_istream_from_buffer(buffer, count); - - /* Decode and print out the stuff */ - if (!check_alltypes(&stream, mode)) - { - printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); - return 1; - } else { - return 0; - } -} diff --git a/tests/bc_encode.c b/tests/bc_encode.c deleted file mode 100644 index e84f0908..00000000 --- a/tests/bc_encode.c +++ /dev/null @@ -1,131 +0,0 @@ -/* Attempts to test all the datatypes supported by ProtoBuf. - * This is a backwards-compatibility test, using bc_alltypes.pb.h. - * It is similar to test_encode3, but duplicated in order to allow - * test_encode3 to test any new features introduced later. - */ - -#include -#include -#include -#include -#include "bc_alltypes.pb.h" - -int main(int argc, char **argv) -{ - int mode = (argc > 1) ? atoi(argv[1]) : 0; - - /* Initialize the structure with constants */ - AllTypes alltypes = {0}; - - alltypes.req_int32 = -1001; - alltypes.req_int64 = -1002; - alltypes.req_uint32 = 1003; - alltypes.req_uint64 = 1004; - alltypes.req_sint32 = -1005; - alltypes.req_sint64 = -1006; - alltypes.req_bool = true; - - alltypes.req_fixed32 = 1008; - alltypes.req_sfixed32 = -1009; - alltypes.req_float = 1010.0f; - - alltypes.req_fixed64 = 1011; - alltypes.req_sfixed64 = -1012; - alltypes.req_double = 1013.0; - - strcpy(alltypes.req_string, "1014"); - alltypes.req_bytes.size = 4; - memcpy(alltypes.req_bytes.bytes, "1015", 4); - strcpy(alltypes.req_submsg.substuff1, "1016"); - alltypes.req_submsg.substuff2 = 1016; - alltypes.req_enum = MyEnum_Truth; - - alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; - alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002; - alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; - alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004; - alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005; - alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006; - alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; - - alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; - alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009; - alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; - - alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011; - alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012; - alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0; - - alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); - alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4; - memcpy(alltypes.rep_bytes[4].bytes, "2015", 4); - - alltypes.rep_submsg_count = 5; - strcpy(alltypes.rep_submsg[4].substuff1, "2016"); - alltypes.rep_submsg[4].substuff2 = 2016; - alltypes.rep_submsg[4].has_substuff3 = true; - alltypes.rep_submsg[4].substuff3 = 2016; - - alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; - - if (mode != 0) - { - /* Fill in values for optional fields */ - alltypes.has_opt_int32 = true; - alltypes.opt_int32 = 3041; - alltypes.has_opt_int64 = true; - alltypes.opt_int64 = 3042; - alltypes.has_opt_uint32 = true; - alltypes.opt_uint32 = 3043; - alltypes.has_opt_uint64 = true; - alltypes.opt_uint64 = 3044; - alltypes.has_opt_sint32 = true; - alltypes.opt_sint32 = 3045; - alltypes.has_opt_sint64 = true; - alltypes.opt_sint64 = 3046; - alltypes.has_opt_bool = true; - alltypes.opt_bool = true; - - alltypes.has_opt_fixed32 = true; - alltypes.opt_fixed32 = 3048; - alltypes.has_opt_sfixed32 = true; - alltypes.opt_sfixed32 = 3049; - alltypes.has_opt_float = true; - alltypes.opt_float = 3050.0f; - - alltypes.has_opt_fixed64 = true; - alltypes.opt_fixed64 = 3051; - alltypes.has_opt_sfixed64 = true; - alltypes.opt_sfixed64 = 3052; - alltypes.has_opt_double = true; - alltypes.opt_double = 3053.0; - - alltypes.has_opt_string = true; - strcpy(alltypes.opt_string, "3054"); - alltypes.has_opt_bytes = true; - alltypes.opt_bytes.size = 4; - memcpy(alltypes.opt_bytes.bytes, "3055", 4); - alltypes.has_opt_submsg = true; - strcpy(alltypes.opt_submsg.substuff1, "3056"); - alltypes.opt_submsg.substuff2 = 3056; - alltypes.has_opt_enum = true; - alltypes.opt_enum = MyEnum_Truth; - } - - alltypes.end = 1099; - - uint8_t buffer[1024]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); - - /* Now encode it and check if we succeeded. */ - if (pb_encode(&stream, AllTypes_fields, &alltypes)) - { - fwrite(buffer, 1, stream.bytes_written, stdout); - return 0; /* Success */ - } - else - { - fprintf(stderr, "Encoding failed!\n"); - return 1; /* Failure */ - } -} diff --git a/tests/callbacks.proto b/tests/callbacks.proto deleted file mode 100644 index ccd1edd8..00000000 --- a/tests/callbacks.proto +++ /dev/null @@ -1,16 +0,0 @@ -message SubMessage { - optional string stringvalue = 1; - repeated int32 int32value = 2; - repeated fixed32 fixed32value = 3; - repeated fixed64 fixed64value = 4; -} - -message TestMessage { - optional string stringvalue = 1; - repeated int32 int32value = 2; - repeated fixed32 fixed32value = 3; - repeated fixed64 fixed64value = 4; - optional SubMessage submsg = 5; - repeated string repeatedstring = 6; -} - diff --git a/tests/callbacks/callbacks.proto b/tests/callbacks/callbacks.proto new file mode 100644 index 00000000..ccd1edd8 --- /dev/null +++ b/tests/callbacks/callbacks.proto @@ -0,0 +1,16 @@ +message SubMessage { + optional string stringvalue = 1; + repeated int32 int32value = 2; + repeated fixed32 fixed32value = 3; + repeated fixed64 fixed64value = 4; +} + +message TestMessage { + optional string stringvalue = 1; + repeated int32 int32value = 2; + repeated fixed32 fixed32value = 3; + repeated fixed64 fixed64value = 4; + optional SubMessage submsg = 5; + repeated string repeatedstring = 6; +} + diff --git a/tests/callbacks/decode_callbacks.c b/tests/callbacks/decode_callbacks.c new file mode 100644 index 00000000..b5056923 --- /dev/null +++ b/tests/callbacks/decode_callbacks.c @@ -0,0 +1,93 @@ +/* Decoding testcase for callback fields. + * Run e.g. ./test_encode_callbacks | ./test_decode_callbacks + */ + +#include +#include +#include "callbacks.pb.h" + +bool print_string(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint8_t buffer[1024] = {0}; + + /* We could read block-by-block to avoid the large buffer... */ + if (stream->bytes_left > sizeof(buffer) - 1) + return false; + + if (!pb_read(stream, buffer, stream->bytes_left)) + return false; + + /* Print the string, in format comparable with protoc --decode. + * Format comes from the arg defined in main(). + */ + printf((char*)*arg, buffer); + return true; +} + +bool print_int32(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint64_t value; + if (!pb_decode_varint(stream, &value)) + return false; + + printf((char*)*arg, (long)value); + return true; +} + +bool print_fixed32(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint32_t value; + if (!pb_decode_fixed32(stream, &value)) + return false; + + printf((char*)*arg, (long)value); + return true; +} + +bool print_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint64_t value; + if (!pb_decode_fixed64(stream, &value)) + return false; + + printf((char*)*arg, (long long)value); + return true; +} + +int main() +{ + uint8_t buffer[1024]; + size_t length = fread(buffer, 1, 1024, stdin); + pb_istream_t stream = pb_istream_from_buffer(buffer, length); + + /* Note: empty initializer list initializes the struct with all-0. + * This is recommended so that unused callbacks are set to NULL instead + * of crashing at runtime. + */ + TestMessage testmessage = {}; + + testmessage.submsg.stringvalue.funcs.decode = &print_string; + testmessage.submsg.stringvalue.arg = "submsg {\n stringvalue: \"%s\"\n"; + testmessage.submsg.int32value.funcs.decode = &print_int32; + testmessage.submsg.int32value.arg = " int32value: %ld\n"; + testmessage.submsg.fixed32value.funcs.decode = &print_fixed32; + testmessage.submsg.fixed32value.arg = " fixed32value: %ld\n"; + testmessage.submsg.fixed64value.funcs.decode = &print_fixed64; + testmessage.submsg.fixed64value.arg = " fixed64value: %lld\n}\n"; + + testmessage.stringvalue.funcs.decode = &print_string; + testmessage.stringvalue.arg = "stringvalue: \"%s\"\n"; + testmessage.int32value.funcs.decode = &print_int32; + testmessage.int32value.arg = "int32value: %ld\n"; + testmessage.fixed32value.funcs.decode = &print_fixed32; + testmessage.fixed32value.arg = "fixed32value: %ld\n"; + testmessage.fixed64value.funcs.decode = &print_fixed64; + testmessage.fixed64value.arg = "fixed64value: %lld\n"; + testmessage.repeatedstring.funcs.decode = &print_string; + testmessage.repeatedstring.arg = "repeatedstring: \"%s\"\n"; + + if (!pb_decode(&stream, TestMessage_fields, &testmessage)) + return 1; + + return 0; +} diff --git a/tests/callbacks/encode_callbacks.c b/tests/callbacks/encode_callbacks.c new file mode 100644 index 00000000..3bb6a45e --- /dev/null +++ b/tests/callbacks/encode_callbacks.c @@ -0,0 +1,86 @@ +/* Encoding testcase for callback fields */ + +#include +#include +#include +#include "callbacks.pb.h" + +bool encode_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + char *str = "Hello world!"; + + if (!pb_encode_tag_for_field(stream, field)) + return false; + + return pb_encode_string(stream, (uint8_t*)str, strlen(str)); +} + +bool encode_int32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + if (!pb_encode_tag_for_field(stream, field)) + return false; + + return pb_encode_varint(stream, 42); +} + +bool encode_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + if (!pb_encode_tag_for_field(stream, field)) + return false; + + uint32_t value = 42; + return pb_encode_fixed32(stream, &value); +} + +bool encode_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + if (!pb_encode_tag_for_field(stream, field)) + return false; + + uint64_t value = 42; + return pb_encode_fixed64(stream, &value); +} + +bool encode_repeatedstring(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + char *str[4] = {"Hello world!", "", "Test", "Test2"}; + int i; + + for (i = 0; i < 4; i++) + { + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!pb_encode_string(stream, (uint8_t*)str[i], strlen(str[i]))) + return false; + } + return true; +} + +int main() +{ + uint8_t buffer[1024]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, 1024); + TestMessage testmessage = {}; + + testmessage.stringvalue.funcs.encode = &encode_string; + testmessage.int32value.funcs.encode = &encode_int32; + testmessage.fixed32value.funcs.encode = &encode_fixed32; + testmessage.fixed64value.funcs.encode = &encode_fixed64; + + testmessage.has_submsg = true; + testmessage.submsg.stringvalue.funcs.encode = &encode_string; + testmessage.submsg.int32value.funcs.encode = &encode_int32; + testmessage.submsg.fixed32value.funcs.encode = &encode_fixed32; + testmessage.submsg.fixed64value.funcs.encode = &encode_fixed64; + + testmessage.repeatedstring.funcs.encode = &encode_repeatedstring; + + if (!pb_encode(&stream, TestMessage_fields, &testmessage)) + return 1; + + if (fwrite(buffer, stream.bytes_written, 1, stdout) != 1) + return 2; + + return 0; +} diff --git a/tests/callbacks2.proto b/tests/callbacks2.proto deleted file mode 100644 index 9a55e15a..00000000 --- a/tests/callbacks2.proto +++ /dev/null @@ -1,9 +0,0 @@ -// Test if including generated header file for this file + implicit include of -// callbacks.pb.h still compiles. Used with test_compiles.c. -import "callbacks.proto"; - -message Callback2Message { - required TestMessage tstmsg = 1; - required SubMessage submsg = 2; -} - diff --git a/tests/common/SConscript b/tests/common/SConscript new file mode 100644 index 00000000..ef3cdcaa --- /dev/null +++ b/tests/common/SConscript @@ -0,0 +1,14 @@ +# Build the common files needed by multiple test cases + +Import('env') + +# Protocol definitions for the encode/decode_unittests +env.NanopbProto("unittestproto") + +# Protocol definitions for basic_buffer/stream tests +env.NanopbProto("person") + +# Binaries of the pb_decode.c and pb_encode.c +env.Object("pb_decode.o", "#../pb_decode.c") +env.Object("pb_encode.o", "#../pb_encode.c") + diff --git a/tests/common/person.proto b/tests/common/person.proto new file mode 100644 index 00000000..dafcf934 --- /dev/null +++ b/tests/common/person.proto @@ -0,0 +1,20 @@ +import "nanopb.proto"; + +message Person { + required string name = 1 [(nanopb).max_size = 40]; + required int32 id = 2; + optional string email = 3 [(nanopb).max_size = 40]; + + enum PhoneType { + MOBILE = 0; + HOME = 1; + WORK = 2; + } + + message PhoneNumber { + required string number = 1 [(nanopb).max_size = 40]; + optional PhoneType type = 2 [default = HOME]; + } + + repeated PhoneNumber phone = 4 [(nanopb).max_count = 5]; +} diff --git a/tests/common/unittestproto.proto b/tests/common/unittestproto.proto new file mode 100644 index 00000000..7024942e --- /dev/null +++ b/tests/common/unittestproto.proto @@ -0,0 +1,32 @@ +import 'nanopb.proto'; + +message IntegerArray { + repeated int32 data = 1 [(nanopb).max_count = 10]; +} + +message FloatArray { + repeated float data = 1 [(nanopb).max_count = 10]; +} + +message StringMessage { + required string data = 1 [(nanopb).max_size = 10]; +} + +message CallbackArray { + // We cheat a bit and use this message for testing other types, too. + // Nanopb does not care about the actual defined data type for callback + // fields. + repeated int32 data = 1; +} + +message IntegerContainer { + required IntegerArray submsg = 1; +} + +message CallbackContainer { + required CallbackArray submsg = 1; +} + +message CallbackContainerContainer { + required CallbackContainer submsg = 1; +} diff --git a/tests/common/unittests.h b/tests/common/unittests.h new file mode 100644 index 00000000..c2b470ad --- /dev/null +++ b/tests/common/unittests.h @@ -0,0 +1,14 @@ +#include + +#define COMMENT(x) printf("\n----" x "----\n"); +#define STR(x) #x +#define STR2(x) STR(x) +#define TEST(x) \ + if (!(x)) { \ + fprintf(stderr, "\033[31;1mFAILED:\033[22;39m " __FILE__ ":" STR2(__LINE__) " " #x "\n"); \ + status = 1; \ + } else { \ + printf("\033[32;1mOK:\033[22;39m " #x "\n"); \ + } + + diff --git a/tests/decode_unittests.c b/tests/decode_unittests.c deleted file mode 100644 index 6ad05f00..00000000 --- a/tests/decode_unittests.c +++ /dev/null @@ -1,306 +0,0 @@ -#define NANOPB_INTERNALS - -#include -#include -#include "pb_decode.h" -#include "unittests.h" -#include "unittestproto.pb.h" - -#define S(x) pb_istream_from_buffer((uint8_t*)x, sizeof(x) - 1) - -bool stream_callback(pb_istream_t *stream, uint8_t *buf, size_t count) -{ - if (stream->state != NULL) - return false; /* Simulate error */ - - if (buf != NULL) - memset(buf, 'x', count); - return true; -} - -/* Verifies that the stream passed to callback matches the byte array pointed to by arg. */ -bool callback_check(pb_istream_t *stream, const pb_field_t *field, void **arg) -{ - int i; - uint8_t byte; - pb_bytes_array_t *ref = (pb_bytes_array_t*) *arg; - - for (i = 0; i < ref->size; i++) - { - if (!pb_read(stream, &byte, 1)) - return false; - - if (byte != ref->bytes[i]) - return false; - } - - return true; -} - -int main() -{ - int status = 0; - - { - uint8_t buffer1[] = "foobartest1234"; - uint8_t buffer2[sizeof(buffer1)]; - pb_istream_t stream = pb_istream_from_buffer(buffer1, sizeof(buffer1)); - - COMMENT("Test pb_read and pb_istream_t"); - TEST(pb_read(&stream, buffer2, 6)) - TEST(memcmp(buffer2, "foobar", 6) == 0) - TEST(stream.bytes_left == sizeof(buffer1) - 6) - TEST(pb_read(&stream, buffer2 + 6, stream.bytes_left)) - TEST(memcmp(buffer1, buffer2, sizeof(buffer1)) == 0) - TEST(stream.bytes_left == 0) - TEST(!pb_read(&stream, buffer2, 1)) - } - - { - uint8_t buffer[20]; - pb_istream_t stream = {&stream_callback, NULL, 20}; - - COMMENT("Test pb_read with custom callback"); - TEST(pb_read(&stream, buffer, 5)) - TEST(memcmp(buffer, "xxxxx", 5) == 0) - TEST(!pb_read(&stream, buffer, 50)) - stream.state = (void*)1; /* Simulated error return from callback */ - TEST(!pb_read(&stream, buffer, 5)) - stream.state = NULL; - TEST(pb_read(&stream, buffer, 15)) - } - - { - pb_istream_t s; - uint64_t u; - int64_t i; - - COMMENT("Test pb_decode_varint"); - TEST((s = S("\x00"), pb_decode_varint(&s, &u) && u == 0)); - TEST((s = S("\x01"), pb_decode_varint(&s, &u) && u == 1)); - TEST((s = S("\xAC\x02"), pb_decode_varint(&s, &u) && u == 300)); - TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint(&s, &u) && u == UINT32_MAX)); - TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint(&s, (uint64_t*)&i) && i == UINT32_MAX)); - TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), - pb_decode_varint(&s, (uint64_t*)&i) && i == -1)); - TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), - pb_decode_varint(&s, &u) && u == UINT64_MAX)); - } - - { - pb_istream_t s; - COMMENT("Test pb_skip_varint"); - TEST((s = S("\x00""foobar"), pb_skip_varint(&s) && s.bytes_left == 6)) - TEST((s = S("\xAC\x02""foobar"), pb_skip_varint(&s) && s.bytes_left == 6)) - TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01""foobar"), - pb_skip_varint(&s) && s.bytes_left == 6)) - TEST((s = S("\xFF"), !pb_skip_varint(&s))) - } - - { - pb_istream_t s; - COMMENT("Test pb_skip_string") - TEST((s = S("\x00""foobar"), pb_skip_string(&s) && s.bytes_left == 6)) - TEST((s = S("\x04""testfoobar"), pb_skip_string(&s) && s.bytes_left == 6)) - TEST((s = S("\x04"), !pb_skip_string(&s))) - TEST((s = S("\xFF"), !pb_skip_string(&s))) - } - - { - pb_istream_t s = S("\x01\xFF\xFF\x03"); - pb_field_t f = {1, PB_LTYPE_VARINT, 0, 0, 4, 0, 0}; - uint32_t d; - COMMENT("Test pb_dec_varint using uint32_t") - TEST(pb_dec_varint(&s, &f, &d) && d == 1) - - /* Verify that no more than data_size is written. */ - d = 0; - f.data_size = 1; - TEST(pb_dec_varint(&s, &f, &d) && (d == 0xFF || d == 0xFF000000)) - } - - { - pb_istream_t s; - pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 4, 0, 0}; - int32_t d; - - COMMENT("Test pb_dec_svarint using int32_t") - TEST((s = S("\x01"), pb_dec_svarint(&s, &f, &d) && d == -1)) - TEST((s = S("\x02"), pb_dec_svarint(&s, &f, &d) && d == 1)) - TEST((s = S("\xfe\xff\xff\xff\x0f"), pb_dec_svarint(&s, &f, &d) && d == INT32_MAX)) - TEST((s = S("\xff\xff\xff\xff\x0f"), pb_dec_svarint(&s, &f, &d) && d == INT32_MIN)) - } - - { - pb_istream_t s; - pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 8, 0, 0}; - uint64_t d; - - COMMENT("Test pb_dec_svarint using uint64_t") - TEST((s = S("\x01"), pb_dec_svarint(&s, &f, &d) && d == -1)) - TEST((s = S("\x02"), pb_dec_svarint(&s, &f, &d) && d == 1)) - TEST((s = S("\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_dec_svarint(&s, &f, &d) && d == INT64_MAX)) - TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_dec_svarint(&s, &f, &d) && d == INT64_MIN)) - } - - { - pb_istream_t s; - pb_field_t f = {1, PB_LTYPE_FIXED32, 0, 0, 4, 0, 0}; - float d; - - COMMENT("Test pb_dec_fixed32 using float (failures here may be caused by imperfect rounding)") - TEST((s = S("\x00\x00\x00\x00"), pb_dec_fixed32(&s, &f, &d) && d == 0.0f)) - TEST((s = S("\x00\x00\xc6\x42"), pb_dec_fixed32(&s, &f, &d) && d == 99.0f)) - TEST((s = S("\x4e\x61\x3c\xcb"), pb_dec_fixed32(&s, &f, &d) && d == -12345678.0f)) - TEST((s = S("\x00"), !pb_dec_fixed32(&s, &f, &d) && d == -12345678.0f)) - } - - { - pb_istream_t s; - pb_field_t f = {1, PB_LTYPE_FIXED64, 0, 0, 8, 0, 0}; - double d; - - COMMENT("Test pb_dec_fixed64 using double (failures here may be caused by imperfect rounding)") - TEST((s = S("\x00\x00\x00\x00\x00\x00\x00\x00"), pb_dec_fixed64(&s, &f, &d) && d == 0.0)) - TEST((s = S("\x00\x00\x00\x00\x00\xc0\x58\x40"), pb_dec_fixed64(&s, &f, &d) && d == 99.0)) - TEST((s = S("\x00\x00\x00\xc0\x29\x8c\x67\xc1"), pb_dec_fixed64(&s, &f, &d) && d == -12345678.0f)) - } - - { - pb_istream_t s; - struct { size_t size; uint8_t bytes[5]; } d; - pb_field_t f = {1, PB_LTYPE_BYTES, 0, 0, sizeof(d), 0, 0}; - - COMMENT("Test pb_dec_bytes") - TEST((s = S("\x00"), pb_dec_bytes(&s, &f, &d) && d.size == 0)) - TEST((s = S("\x01\xFF"), pb_dec_bytes(&s, &f, &d) && d.size == 1 && d.bytes[0] == 0xFF)) - TEST((s = S("\x05xxxxx"), pb_dec_bytes(&s, &f, &d) && d.size == 5)) - TEST((s = S("\x05xxxx"), !pb_dec_bytes(&s, &f, &d))) - - /* Note: the size limit on bytes-fields is not strictly obeyed, as - * the compiler may add some padding to the struct. Using this padding - * is not a very good thing to do, but it is difficult to avoid when - * we use only a single uint8_t to store the size of the field. - * Therefore this tests against a 10-byte string, while otherwise even - * 6 bytes should error out. - */ - TEST((s = S("\x10xxxxxxxxxx"), !pb_dec_bytes(&s, &f, &d))) - } - - { - pb_istream_t s; - pb_field_t f = {1, PB_LTYPE_STRING, 0, 0, 5, 0, 0}; - char d[5]; - - COMMENT("Test pb_dec_string") - TEST((s = S("\x00"), pb_dec_string(&s, &f, &d) && d[0] == '\0')) - TEST((s = S("\x04xyzz"), pb_dec_string(&s, &f, &d) && strcmp(d, "xyzz") == 0)) - TEST((s = S("\x05xyzzy"), !pb_dec_string(&s, &f, &d))) - } - - { - pb_istream_t s; - IntegerArray dest; - - COMMENT("Testing pb_decode with repeated int32 field") - TEST((s = S(""), pb_decode(&s, IntegerArray_fields, &dest) && dest.data_count == 0)) - TEST((s = S("\x08\x01\x08\x02"), pb_decode(&s, IntegerArray_fields, &dest) - && dest.data_count == 2 && dest.data[0] == 1 && dest.data[1] == 2)) - s = S("\x08\x01\x08\x02\x08\x03\x08\x04\x08\x05\x08\x06\x08\x07\x08\x08\x08\x09\x08\x0A"); - TEST(pb_decode(&s, IntegerArray_fields, &dest) && dest.data_count == 10 && dest.data[9] == 10) - s = S("\x08\x01\x08\x02\x08\x03\x08\x04\x08\x05\x08\x06\x08\x07\x08\x08\x08\x09\x08\x0A\x08\x0B"); - TEST(!pb_decode(&s, IntegerArray_fields, &dest)) - } - - { - pb_istream_t s; - IntegerArray dest; - - COMMENT("Testing pb_decode with packed int32 field") - TEST((s = S("\x0A\x00"), pb_decode(&s, IntegerArray_fields, &dest) - && dest.data_count == 0)) - TEST((s = S("\x0A\x01\x01"), pb_decode(&s, IntegerArray_fields, &dest) - && dest.data_count == 1 && dest.data[0] == 1)) - TEST((s = S("\x0A\x0A\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A"), pb_decode(&s, IntegerArray_fields, &dest) - && dest.data_count == 10 && dest.data[0] == 1 && dest.data[9] == 10)) - TEST((s = S("\x0A\x0B\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B"), !pb_decode(&s, IntegerArray_fields, &dest))) - - /* Test invalid wire data */ - TEST((s = S("\x0A\xFF"), !pb_decode(&s, IntegerArray_fields, &dest))) - TEST((s = S("\x0A\x01"), !pb_decode(&s, IntegerArray_fields, &dest))) - } - - { - pb_istream_t s; - IntegerArray dest; - - COMMENT("Testing pb_decode with unknown fields") - TEST((s = S("\x18\x0F\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest) - && dest.data_count == 1 && dest.data[0] == 1)) - TEST((s = S("\x19\x00\x00\x00\x00\x00\x00\x00\x00\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest) - && dest.data_count == 1 && dest.data[0] == 1)) - TEST((s = S("\x1A\x00\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest) - && dest.data_count == 1 && dest.data[0] == 1)) - TEST((s = S("\x1B\x08\x01"), !pb_decode(&s, IntegerArray_fields, &dest))) - TEST((s = S("\x1D\x00\x00\x00\x00\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest) - && dest.data_count == 1 && dest.data[0] == 1)) - } - - { - pb_istream_t s; - CallbackArray dest; - struct { size_t size; uint8_t bytes[10]; } ref; - dest.data.funcs.decode = &callback_check; - dest.data.arg = &ref; - - COMMENT("Testing pb_decode with callbacks") - /* Single varint */ - ref.size = 1; ref.bytes[0] = 0x55; - TEST((s = S("\x08\x55"), pb_decode(&s, CallbackArray_fields, &dest))) - /* Packed varint */ - ref.size = 3; ref.bytes[0] = ref.bytes[1] = ref.bytes[2] = 0x55; - TEST((s = S("\x0A\x03\x55\x55\x55"), pb_decode(&s, CallbackArray_fields, &dest))) - /* Packed varint with loop */ - ref.size = 1; ref.bytes[0] = 0x55; - TEST((s = S("\x0A\x03\x55\x55\x55"), pb_decode(&s, CallbackArray_fields, &dest))) - /* Single fixed32 */ - ref.size = 4; ref.bytes[0] = ref.bytes[1] = ref.bytes[2] = ref.bytes[3] = 0xAA; - TEST((s = S("\x0D\xAA\xAA\xAA\xAA"), pb_decode(&s, CallbackArray_fields, &dest))) - /* Single fixed64 */ - ref.size = 8; memset(ref.bytes, 0xAA, 8); - TEST((s = S("\x09\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"), pb_decode(&s, CallbackArray_fields, &dest))) - /* Unsupported field type */ - TEST((s = S("\x0B\x00"), !pb_decode(&s, CallbackArray_fields, &dest))) - - /* Just make sure that our test function works */ - ref.size = 1; ref.bytes[0] = 0x56; - TEST((s = S("\x08\x55"), !pb_decode(&s, CallbackArray_fields, &dest))) - } - - { - pb_istream_t s; - IntegerArray dest; - - COMMENT("Testing pb_decode message termination") - TEST((s = S(""), pb_decode(&s, IntegerArray_fields, &dest))) - TEST((s = S("\x00"), pb_decode(&s, IntegerArray_fields, &dest))) - TEST((s = S("\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest))) - TEST((s = S("\x08\x01\x00"), pb_decode(&s, IntegerArray_fields, &dest))) - TEST((s = S("\x08"), !pb_decode(&s, IntegerArray_fields, &dest))) - } - - { - pb_istream_t s; - IntegerContainer dest = {}; - - COMMENT("Testing pb_decode_delimited") - TEST((s = S("\x09\x0A\x07\x0A\x05\x01\x02\x03\x04\x05"), - pb_decode_delimited(&s, IntegerContainer_fields, &dest)) && - dest.submsg.data_count == 5) - } - - if (status != 0) - fprintf(stdout, "\n\nSome tests FAILED!\n"); - - return status; -} diff --git a/tests/decode_unittests/SConscript b/tests/decode_unittests/SConscript new file mode 100644 index 00000000..860d7738 --- /dev/null +++ b/tests/decode_unittests/SConscript @@ -0,0 +1,4 @@ +Import('env') +env.Program(["decode_unittests.c", "#common/unittestproto.pb.c", "#common/pb_decode.o"]) +env.RunTest('decode_unittests') + diff --git a/tests/decode_unittests/decode_unittests.c b/tests/decode_unittests/decode_unittests.c new file mode 100644 index 00000000..6ad05f00 --- /dev/null +++ b/tests/decode_unittests/decode_unittests.c @@ -0,0 +1,306 @@ +#define NANOPB_INTERNALS + +#include +#include +#include "pb_decode.h" +#include "unittests.h" +#include "unittestproto.pb.h" + +#define S(x) pb_istream_from_buffer((uint8_t*)x, sizeof(x) - 1) + +bool stream_callback(pb_istream_t *stream, uint8_t *buf, size_t count) +{ + if (stream->state != NULL) + return false; /* Simulate error */ + + if (buf != NULL) + memset(buf, 'x', count); + return true; +} + +/* Verifies that the stream passed to callback matches the byte array pointed to by arg. */ +bool callback_check(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + int i; + uint8_t byte; + pb_bytes_array_t *ref = (pb_bytes_array_t*) *arg; + + for (i = 0; i < ref->size; i++) + { + if (!pb_read(stream, &byte, 1)) + return false; + + if (byte != ref->bytes[i]) + return false; + } + + return true; +} + +int main() +{ + int status = 0; + + { + uint8_t buffer1[] = "foobartest1234"; + uint8_t buffer2[sizeof(buffer1)]; + pb_istream_t stream = pb_istream_from_buffer(buffer1, sizeof(buffer1)); + + COMMENT("Test pb_read and pb_istream_t"); + TEST(pb_read(&stream, buffer2, 6)) + TEST(memcmp(buffer2, "foobar", 6) == 0) + TEST(stream.bytes_left == sizeof(buffer1) - 6) + TEST(pb_read(&stream, buffer2 + 6, stream.bytes_left)) + TEST(memcmp(buffer1, buffer2, sizeof(buffer1)) == 0) + TEST(stream.bytes_left == 0) + TEST(!pb_read(&stream, buffer2, 1)) + } + + { + uint8_t buffer[20]; + pb_istream_t stream = {&stream_callback, NULL, 20}; + + COMMENT("Test pb_read with custom callback"); + TEST(pb_read(&stream, buffer, 5)) + TEST(memcmp(buffer, "xxxxx", 5) == 0) + TEST(!pb_read(&stream, buffer, 50)) + stream.state = (void*)1; /* Simulated error return from callback */ + TEST(!pb_read(&stream, buffer, 5)) + stream.state = NULL; + TEST(pb_read(&stream, buffer, 15)) + } + + { + pb_istream_t s; + uint64_t u; + int64_t i; + + COMMENT("Test pb_decode_varint"); + TEST((s = S("\x00"), pb_decode_varint(&s, &u) && u == 0)); + TEST((s = S("\x01"), pb_decode_varint(&s, &u) && u == 1)); + TEST((s = S("\xAC\x02"), pb_decode_varint(&s, &u) && u == 300)); + TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint(&s, &u) && u == UINT32_MAX)); + TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint(&s, (uint64_t*)&i) && i == UINT32_MAX)); + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), + pb_decode_varint(&s, (uint64_t*)&i) && i == -1)); + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), + pb_decode_varint(&s, &u) && u == UINT64_MAX)); + } + + { + pb_istream_t s; + COMMENT("Test pb_skip_varint"); + TEST((s = S("\x00""foobar"), pb_skip_varint(&s) && s.bytes_left == 6)) + TEST((s = S("\xAC\x02""foobar"), pb_skip_varint(&s) && s.bytes_left == 6)) + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01""foobar"), + pb_skip_varint(&s) && s.bytes_left == 6)) + TEST((s = S("\xFF"), !pb_skip_varint(&s))) + } + + { + pb_istream_t s; + COMMENT("Test pb_skip_string") + TEST((s = S("\x00""foobar"), pb_skip_string(&s) && s.bytes_left == 6)) + TEST((s = S("\x04""testfoobar"), pb_skip_string(&s) && s.bytes_left == 6)) + TEST((s = S("\x04"), !pb_skip_string(&s))) + TEST((s = S("\xFF"), !pb_skip_string(&s))) + } + + { + pb_istream_t s = S("\x01\xFF\xFF\x03"); + pb_field_t f = {1, PB_LTYPE_VARINT, 0, 0, 4, 0, 0}; + uint32_t d; + COMMENT("Test pb_dec_varint using uint32_t") + TEST(pb_dec_varint(&s, &f, &d) && d == 1) + + /* Verify that no more than data_size is written. */ + d = 0; + f.data_size = 1; + TEST(pb_dec_varint(&s, &f, &d) && (d == 0xFF || d == 0xFF000000)) + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 4, 0, 0}; + int32_t d; + + COMMENT("Test pb_dec_svarint using int32_t") + TEST((s = S("\x01"), pb_dec_svarint(&s, &f, &d) && d == -1)) + TEST((s = S("\x02"), pb_dec_svarint(&s, &f, &d) && d == 1)) + TEST((s = S("\xfe\xff\xff\xff\x0f"), pb_dec_svarint(&s, &f, &d) && d == INT32_MAX)) + TEST((s = S("\xff\xff\xff\xff\x0f"), pb_dec_svarint(&s, &f, &d) && d == INT32_MIN)) + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 8, 0, 0}; + uint64_t d; + + COMMENT("Test pb_dec_svarint using uint64_t") + TEST((s = S("\x01"), pb_dec_svarint(&s, &f, &d) && d == -1)) + TEST((s = S("\x02"), pb_dec_svarint(&s, &f, &d) && d == 1)) + TEST((s = S("\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_dec_svarint(&s, &f, &d) && d == INT64_MAX)) + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_dec_svarint(&s, &f, &d) && d == INT64_MIN)) + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_FIXED32, 0, 0, 4, 0, 0}; + float d; + + COMMENT("Test pb_dec_fixed32 using float (failures here may be caused by imperfect rounding)") + TEST((s = S("\x00\x00\x00\x00"), pb_dec_fixed32(&s, &f, &d) && d == 0.0f)) + TEST((s = S("\x00\x00\xc6\x42"), pb_dec_fixed32(&s, &f, &d) && d == 99.0f)) + TEST((s = S("\x4e\x61\x3c\xcb"), pb_dec_fixed32(&s, &f, &d) && d == -12345678.0f)) + TEST((s = S("\x00"), !pb_dec_fixed32(&s, &f, &d) && d == -12345678.0f)) + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_FIXED64, 0, 0, 8, 0, 0}; + double d; + + COMMENT("Test pb_dec_fixed64 using double (failures here may be caused by imperfect rounding)") + TEST((s = S("\x00\x00\x00\x00\x00\x00\x00\x00"), pb_dec_fixed64(&s, &f, &d) && d == 0.0)) + TEST((s = S("\x00\x00\x00\x00\x00\xc0\x58\x40"), pb_dec_fixed64(&s, &f, &d) && d == 99.0)) + TEST((s = S("\x00\x00\x00\xc0\x29\x8c\x67\xc1"), pb_dec_fixed64(&s, &f, &d) && d == -12345678.0f)) + } + + { + pb_istream_t s; + struct { size_t size; uint8_t bytes[5]; } d; + pb_field_t f = {1, PB_LTYPE_BYTES, 0, 0, sizeof(d), 0, 0}; + + COMMENT("Test pb_dec_bytes") + TEST((s = S("\x00"), pb_dec_bytes(&s, &f, &d) && d.size == 0)) + TEST((s = S("\x01\xFF"), pb_dec_bytes(&s, &f, &d) && d.size == 1 && d.bytes[0] == 0xFF)) + TEST((s = S("\x05xxxxx"), pb_dec_bytes(&s, &f, &d) && d.size == 5)) + TEST((s = S("\x05xxxx"), !pb_dec_bytes(&s, &f, &d))) + + /* Note: the size limit on bytes-fields is not strictly obeyed, as + * the compiler may add some padding to the struct. Using this padding + * is not a very good thing to do, but it is difficult to avoid when + * we use only a single uint8_t to store the size of the field. + * Therefore this tests against a 10-byte string, while otherwise even + * 6 bytes should error out. + */ + TEST((s = S("\x10xxxxxxxxxx"), !pb_dec_bytes(&s, &f, &d))) + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_STRING, 0, 0, 5, 0, 0}; + char d[5]; + + COMMENT("Test pb_dec_string") + TEST((s = S("\x00"), pb_dec_string(&s, &f, &d) && d[0] == '\0')) + TEST((s = S("\x04xyzz"), pb_dec_string(&s, &f, &d) && strcmp(d, "xyzz") == 0)) + TEST((s = S("\x05xyzzy"), !pb_dec_string(&s, &f, &d))) + } + + { + pb_istream_t s; + IntegerArray dest; + + COMMENT("Testing pb_decode with repeated int32 field") + TEST((s = S(""), pb_decode(&s, IntegerArray_fields, &dest) && dest.data_count == 0)) + TEST((s = S("\x08\x01\x08\x02"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 2 && dest.data[0] == 1 && dest.data[1] == 2)) + s = S("\x08\x01\x08\x02\x08\x03\x08\x04\x08\x05\x08\x06\x08\x07\x08\x08\x08\x09\x08\x0A"); + TEST(pb_decode(&s, IntegerArray_fields, &dest) && dest.data_count == 10 && dest.data[9] == 10) + s = S("\x08\x01\x08\x02\x08\x03\x08\x04\x08\x05\x08\x06\x08\x07\x08\x08\x08\x09\x08\x0A\x08\x0B"); + TEST(!pb_decode(&s, IntegerArray_fields, &dest)) + } + + { + pb_istream_t s; + IntegerArray dest; + + COMMENT("Testing pb_decode with packed int32 field") + TEST((s = S("\x0A\x00"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 0)) + TEST((s = S("\x0A\x01\x01"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 1 && dest.data[0] == 1)) + TEST((s = S("\x0A\x0A\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 10 && dest.data[0] == 1 && dest.data[9] == 10)) + TEST((s = S("\x0A\x0B\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B"), !pb_decode(&s, IntegerArray_fields, &dest))) + + /* Test invalid wire data */ + TEST((s = S("\x0A\xFF"), !pb_decode(&s, IntegerArray_fields, &dest))) + TEST((s = S("\x0A\x01"), !pb_decode(&s, IntegerArray_fields, &dest))) + } + + { + pb_istream_t s; + IntegerArray dest; + + COMMENT("Testing pb_decode with unknown fields") + TEST((s = S("\x18\x0F\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 1 && dest.data[0] == 1)) + TEST((s = S("\x19\x00\x00\x00\x00\x00\x00\x00\x00\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 1 && dest.data[0] == 1)) + TEST((s = S("\x1A\x00\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 1 && dest.data[0] == 1)) + TEST((s = S("\x1B\x08\x01"), !pb_decode(&s, IntegerArray_fields, &dest))) + TEST((s = S("\x1D\x00\x00\x00\x00\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest) + && dest.data_count == 1 && dest.data[0] == 1)) + } + + { + pb_istream_t s; + CallbackArray dest; + struct { size_t size; uint8_t bytes[10]; } ref; + dest.data.funcs.decode = &callback_check; + dest.data.arg = &ref; + + COMMENT("Testing pb_decode with callbacks") + /* Single varint */ + ref.size = 1; ref.bytes[0] = 0x55; + TEST((s = S("\x08\x55"), pb_decode(&s, CallbackArray_fields, &dest))) + /* Packed varint */ + ref.size = 3; ref.bytes[0] = ref.bytes[1] = ref.bytes[2] = 0x55; + TEST((s = S("\x0A\x03\x55\x55\x55"), pb_decode(&s, CallbackArray_fields, &dest))) + /* Packed varint with loop */ + ref.size = 1; ref.bytes[0] = 0x55; + TEST((s = S("\x0A\x03\x55\x55\x55"), pb_decode(&s, CallbackArray_fields, &dest))) + /* Single fixed32 */ + ref.size = 4; ref.bytes[0] = ref.bytes[1] = ref.bytes[2] = ref.bytes[3] = 0xAA; + TEST((s = S("\x0D\xAA\xAA\xAA\xAA"), pb_decode(&s, CallbackArray_fields, &dest))) + /* Single fixed64 */ + ref.size = 8; memset(ref.bytes, 0xAA, 8); + TEST((s = S("\x09\xAA\xAA\xAA\xAA\xAA\xAA\xAA\xAA"), pb_decode(&s, CallbackArray_fields, &dest))) + /* Unsupported field type */ + TEST((s = S("\x0B\x00"), !pb_decode(&s, CallbackArray_fields, &dest))) + + /* Just make sure that our test function works */ + ref.size = 1; ref.bytes[0] = 0x56; + TEST((s = S("\x08\x55"), !pb_decode(&s, CallbackArray_fields, &dest))) + } + + { + pb_istream_t s; + IntegerArray dest; + + COMMENT("Testing pb_decode message termination") + TEST((s = S(""), pb_decode(&s, IntegerArray_fields, &dest))) + TEST((s = S("\x00"), pb_decode(&s, IntegerArray_fields, &dest))) + TEST((s = S("\x08\x01"), pb_decode(&s, IntegerArray_fields, &dest))) + TEST((s = S("\x08\x01\x00"), pb_decode(&s, IntegerArray_fields, &dest))) + TEST((s = S("\x08"), !pb_decode(&s, IntegerArray_fields, &dest))) + } + + { + pb_istream_t s; + IntegerContainer dest = {}; + + COMMENT("Testing pb_decode_delimited") + TEST((s = S("\x09\x0A\x07\x0A\x05\x01\x02\x03\x04\x05"), + pb_decode_delimited(&s, IntegerContainer_fields, &dest)) && + dest.submsg.data_count == 5) + } + + if (status != 0) + fprintf(stdout, "\n\nSome tests FAILED!\n"); + + return status; +} diff --git a/tests/encode_unittests.c b/tests/encode_unittests.c deleted file mode 100644 index c3634ac8..00000000 --- a/tests/encode_unittests.c +++ /dev/null @@ -1,287 +0,0 @@ -#define NANOPB_INTERNALS - -#include -#include -#include "pb_encode.h" -#include "unittests.h" -#include "unittestproto.pb.h" - -bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) -{ - /* Allow only 'x' to be written */ - while (count--) - { - if (*buf++ != 'x') - return false; - } - return true; -} - -bool fieldcallback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) -{ - int value = 0x55; - if (!pb_encode_tag_for_field(stream, field)) - return false; - return pb_encode_varint(stream, value); -} - -bool crazyfieldcallback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) -{ - /* This callback writes different amount of data the second time. */ - uint32_t *state = (uint32_t*)arg; - *state <<= 8; - if (!pb_encode_tag_for_field(stream, field)) - return false; - return pb_encode_varint(stream, *state); -} - -/* Check that expression x writes data y. - * Y is a string, which may contain null bytes. Null terminator is ignored. - */ -#define WRITES(x, y) \ -memset(buffer, 0xAA, sizeof(buffer)), \ -s = pb_ostream_from_buffer(buffer, sizeof(buffer)), \ -(x) && \ -memcmp(buffer, y, sizeof(y) - 1) == 0 && \ -buffer[sizeof(y) - 1] == 0xAA - -int main() -{ - int status = 0; - - { - uint8_t buffer1[] = "foobartest1234"; - uint8_t buffer2[sizeof(buffer1)]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer2, sizeof(buffer1)); - - COMMENT("Test pb_write and pb_ostream_t"); - TEST(pb_write(&stream, buffer1, sizeof(buffer1))); - TEST(memcmp(buffer1, buffer2, sizeof(buffer1)) == 0); - TEST(!pb_write(&stream, buffer1, 1)); - TEST(stream.bytes_written == sizeof(buffer1)); - } - - { - uint8_t buffer1[] = "xxxxxxx"; - pb_ostream_t stream = {&streamcallback, 0, SIZE_MAX, 0}; - - COMMENT("Test pb_write with custom callback"); - TEST(pb_write(&stream, buffer1, 5)); - buffer1[0] = 'a'; - TEST(!pb_write(&stream, buffer1, 5)); - } - - { - uint8_t buffer[30]; - pb_ostream_t s; - - COMMENT("Test pb_encode_varint") - TEST(WRITES(pb_encode_varint(&s, 0), "\0")); - TEST(WRITES(pb_encode_varint(&s, 1), "\1")); - TEST(WRITES(pb_encode_varint(&s, 0x7F), "\x7F")); - TEST(WRITES(pb_encode_varint(&s, 0x80), "\x80\x01")); - TEST(WRITES(pb_encode_varint(&s, UINT32_MAX), "\xFF\xFF\xFF\xFF\x0F")); - TEST(WRITES(pb_encode_varint(&s, UINT64_MAX), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01")); - } - - { - uint8_t buffer[30]; - pb_ostream_t s; - - COMMENT("Test pb_encode_tag") - TEST(WRITES(pb_encode_tag(&s, PB_WT_STRING, 5), "\x2A")); - TEST(WRITES(pb_encode_tag(&s, PB_WT_VARINT, 99), "\x98\x06")); - } - - { - uint8_t buffer[30]; - pb_ostream_t s; - pb_field_t field = {10, PB_LTYPE_SVARINT}; - - COMMENT("Test pb_encode_tag_for_field") - TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x50")); - - field.type = PB_LTYPE_FIXED64; - TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x51")); - - field.type = PB_LTYPE_STRING; - TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x52")); - - field.type = PB_LTYPE_FIXED32; - TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x55")); - } - - { - uint8_t buffer[30]; - pb_ostream_t s; - - COMMENT("Test pb_encode_string") - TEST(WRITES(pb_encode_string(&s, (const uint8_t*)"abcd", 4), "\x04""abcd")); - TEST(WRITES(pb_encode_string(&s, (const uint8_t*)"abcd\x00", 5), "\x05""abcd\x00")); - TEST(WRITES(pb_encode_string(&s, (const uint8_t*)"", 0), "\x00")); - } - - { - uint8_t buffer[30]; - pb_ostream_t s; - uint8_t value = 1; - int32_t max = INT32_MAX; - int32_t min = INT32_MIN; - int64_t lmax = INT64_MAX; - int64_t lmin = INT64_MIN; - pb_field_t field = {1, PB_LTYPE_VARINT, 0, 0, sizeof(value)}; - - COMMENT("Test pb_enc_varint and pb_enc_svarint") - TEST(WRITES(pb_enc_varint(&s, &field, &value), "\x01")); - - field.data_size = sizeof(max); - TEST(WRITES(pb_enc_svarint(&s, &field, &max), "\xfe\xff\xff\xff\x0f")); - TEST(WRITES(pb_enc_svarint(&s, &field, &min), "\xff\xff\xff\xff\x0f")); - - field.data_size = sizeof(lmax); - TEST(WRITES(pb_enc_svarint(&s, &field, &lmax), "\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01")); - TEST(WRITES(pb_enc_svarint(&s, &field, &lmin), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01")); - } - - { - uint8_t buffer[30]; - pb_ostream_t s; - float fvalue; - double dvalue; - - COMMENT("Test pb_enc_fixed32 using float") - fvalue = 0.0f; - TEST(WRITES(pb_enc_fixed32(&s, NULL, &fvalue), "\x00\x00\x00\x00")) - fvalue = 99.0f; - TEST(WRITES(pb_enc_fixed32(&s, NULL, &fvalue), "\x00\x00\xc6\x42")) - fvalue = -12345678.0f; - TEST(WRITES(pb_enc_fixed32(&s, NULL, &fvalue), "\x4e\x61\x3c\xcb")) - - COMMENT("Test pb_enc_fixed64 using double") - dvalue = 0.0; - TEST(WRITES(pb_enc_fixed64(&s, NULL, &dvalue), "\x00\x00\x00\x00\x00\x00\x00\x00")) - dvalue = 99.0; - TEST(WRITES(pb_enc_fixed64(&s, NULL, &dvalue), "\x00\x00\x00\x00\x00\xc0\x58\x40")) - dvalue = -12345678.0; - TEST(WRITES(pb_enc_fixed64(&s, NULL, &dvalue), "\x00\x00\x00\xc0\x29\x8c\x67\xc1")) - } - - { - uint8_t buffer[30]; - pb_ostream_t s; - struct { size_t size; uint8_t bytes[5]; } value = {5, {'x', 'y', 'z', 'z', 'y'}}; - - COMMENT("Test pb_enc_bytes") - TEST(WRITES(pb_enc_bytes(&s, NULL, &value), "\x05xyzzy")) - value.size = 0; - TEST(WRITES(pb_enc_bytes(&s, NULL, &value), "\x00")) - } - - { - uint8_t buffer[30]; - pb_ostream_t s; - char value[30] = "xyzzy"; - - COMMENT("Test pb_enc_string") - TEST(WRITES(pb_enc_string(&s, &StringMessage_fields[0], &value), "\x05xyzzy")) - value[0] = '\0'; - TEST(WRITES(pb_enc_string(&s, &StringMessage_fields[0], &value), "\x00")) - memset(value, 'x', 30); - TEST(WRITES(pb_enc_string(&s, &StringMessage_fields[0], &value), "\x0Axxxxxxxxxx")) - } - - { - uint8_t buffer[10]; - pb_ostream_t s; - IntegerArray msg = {5, {1, 2, 3, 4, 5}}; - - COMMENT("Test pb_encode with int32 array") - - TEST(WRITES(pb_encode(&s, IntegerArray_fields, &msg), "\x0A\x05\x01\x02\x03\x04\x05")) - - msg.data_count = 0; - TEST(WRITES(pb_encode(&s, IntegerArray_fields, &msg), "")) - - msg.data_count = 10; - TEST(!pb_encode(&s, IntegerArray_fields, &msg)) - } - - { - uint8_t buffer[10]; - pb_ostream_t s; - FloatArray msg = {1, {99.0f}}; - - COMMENT("Test pb_encode with float array") - - TEST(WRITES(pb_encode(&s, FloatArray_fields, &msg), - "\x0A\x04\x00\x00\xc6\x42")) - - msg.data_count = 0; - TEST(WRITES(pb_encode(&s, FloatArray_fields, &msg), "")) - - msg.data_count = 3; - TEST(!pb_encode(&s, FloatArray_fields, &msg)) - } - - { - uint8_t buffer[10]; - pb_ostream_t s; - CallbackArray msg; - - msg.data.funcs.encode = &fieldcallback; - - COMMENT("Test pb_encode with callback field.") - TEST(WRITES(pb_encode(&s, CallbackArray_fields, &msg), "\x08\x55")) - } - - { - uint8_t buffer[10]; - pb_ostream_t s; - IntegerContainer msg = {{5, {1,2,3,4,5}}}; - - COMMENT("Test pb_encode with packed array in a submessage.") - TEST(WRITES(pb_encode(&s, IntegerContainer_fields, &msg), - "\x0A\x07\x0A\x05\x01\x02\x03\x04\x05")) - } - - { - uint8_t buffer[20]; - pb_ostream_t s; - IntegerContainer msg = {{5, {1,2,3,4,5}}}; - - COMMENT("Test pb_encode_delimited.") - TEST(WRITES(pb_encode_delimited(&s, IntegerContainer_fields, &msg), - "\x09\x0A\x07\x0A\x05\x01\x02\x03\x04\x05")) - } - - { - uint8_t buffer[10]; - pb_ostream_t s; - CallbackContainer msg; - CallbackContainerContainer msg2; - uint32_t state = 1; - - msg.submsg.data.funcs.encode = &fieldcallback; - msg2.submsg.submsg.data.funcs.encode = &fieldcallback; - - COMMENT("Test pb_encode with callback field in a submessage.") - TEST(WRITES(pb_encode(&s, CallbackContainer_fields, &msg), "\x0A\x02\x08\x55")) - TEST(WRITES(pb_encode(&s, CallbackContainerContainer_fields, &msg2), - "\x0A\x04\x0A\x02\x08\x55")) - - /* Misbehaving callback: varying output between calls */ - msg.submsg.data.funcs.encode = &crazyfieldcallback; - msg.submsg.data.arg = &state; - msg2.submsg.submsg.data.funcs.encode = &crazyfieldcallback; - msg2.submsg.submsg.data.arg = &state; - - TEST(!pb_encode(&s, CallbackContainer_fields, &msg)) - state = 1; - TEST(!pb_encode(&s, CallbackContainerContainer_fields, &msg2)) - } - - if (status != 0) - fprintf(stdout, "\n\nSome tests FAILED!\n"); - - return status; -} diff --git a/tests/encode_unittests/SConscript b/tests/encode_unittests/SConscript new file mode 100644 index 00000000..0864a910 --- /dev/null +++ b/tests/encode_unittests/SConscript @@ -0,0 +1,5 @@ +# Build and run the stand-alone unit tests for the nanopb encoder part. + +Import('env') +env.Program(["encode_unittests.c", "#common/unittestproto.pb.c", "#common/pb_encode.o"]) +env.RunTest('encode_unittests') diff --git a/tests/encode_unittests/encode_unittests.c b/tests/encode_unittests/encode_unittests.c new file mode 100644 index 00000000..c3634ac8 --- /dev/null +++ b/tests/encode_unittests/encode_unittests.c @@ -0,0 +1,287 @@ +#define NANOPB_INTERNALS + +#include +#include +#include "pb_encode.h" +#include "unittests.h" +#include "unittestproto.pb.h" + +bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) +{ + /* Allow only 'x' to be written */ + while (count--) + { + if (*buf++ != 'x') + return false; + } + return true; +} + +bool fieldcallback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + int value = 0x55; + if (!pb_encode_tag_for_field(stream, field)) + return false; + return pb_encode_varint(stream, value); +} + +bool crazyfieldcallback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + /* This callback writes different amount of data the second time. */ + uint32_t *state = (uint32_t*)arg; + *state <<= 8; + if (!pb_encode_tag_for_field(stream, field)) + return false; + return pb_encode_varint(stream, *state); +} + +/* Check that expression x writes data y. + * Y is a string, which may contain null bytes. Null terminator is ignored. + */ +#define WRITES(x, y) \ +memset(buffer, 0xAA, sizeof(buffer)), \ +s = pb_ostream_from_buffer(buffer, sizeof(buffer)), \ +(x) && \ +memcmp(buffer, y, sizeof(y) - 1) == 0 && \ +buffer[sizeof(y) - 1] == 0xAA + +int main() +{ + int status = 0; + + { + uint8_t buffer1[] = "foobartest1234"; + uint8_t buffer2[sizeof(buffer1)]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer2, sizeof(buffer1)); + + COMMENT("Test pb_write and pb_ostream_t"); + TEST(pb_write(&stream, buffer1, sizeof(buffer1))); + TEST(memcmp(buffer1, buffer2, sizeof(buffer1)) == 0); + TEST(!pb_write(&stream, buffer1, 1)); + TEST(stream.bytes_written == sizeof(buffer1)); + } + + { + uint8_t buffer1[] = "xxxxxxx"; + pb_ostream_t stream = {&streamcallback, 0, SIZE_MAX, 0}; + + COMMENT("Test pb_write with custom callback"); + TEST(pb_write(&stream, buffer1, 5)); + buffer1[0] = 'a'; + TEST(!pb_write(&stream, buffer1, 5)); + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + + COMMENT("Test pb_encode_varint") + TEST(WRITES(pb_encode_varint(&s, 0), "\0")); + TEST(WRITES(pb_encode_varint(&s, 1), "\1")); + TEST(WRITES(pb_encode_varint(&s, 0x7F), "\x7F")); + TEST(WRITES(pb_encode_varint(&s, 0x80), "\x80\x01")); + TEST(WRITES(pb_encode_varint(&s, UINT32_MAX), "\xFF\xFF\xFF\xFF\x0F")); + TEST(WRITES(pb_encode_varint(&s, UINT64_MAX), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01")); + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + + COMMENT("Test pb_encode_tag") + TEST(WRITES(pb_encode_tag(&s, PB_WT_STRING, 5), "\x2A")); + TEST(WRITES(pb_encode_tag(&s, PB_WT_VARINT, 99), "\x98\x06")); + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + pb_field_t field = {10, PB_LTYPE_SVARINT}; + + COMMENT("Test pb_encode_tag_for_field") + TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x50")); + + field.type = PB_LTYPE_FIXED64; + TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x51")); + + field.type = PB_LTYPE_STRING; + TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x52")); + + field.type = PB_LTYPE_FIXED32; + TEST(WRITES(pb_encode_tag_for_field(&s, &field), "\x55")); + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + + COMMENT("Test pb_encode_string") + TEST(WRITES(pb_encode_string(&s, (const uint8_t*)"abcd", 4), "\x04""abcd")); + TEST(WRITES(pb_encode_string(&s, (const uint8_t*)"abcd\x00", 5), "\x05""abcd\x00")); + TEST(WRITES(pb_encode_string(&s, (const uint8_t*)"", 0), "\x00")); + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + uint8_t value = 1; + int32_t max = INT32_MAX; + int32_t min = INT32_MIN; + int64_t lmax = INT64_MAX; + int64_t lmin = INT64_MIN; + pb_field_t field = {1, PB_LTYPE_VARINT, 0, 0, sizeof(value)}; + + COMMENT("Test pb_enc_varint and pb_enc_svarint") + TEST(WRITES(pb_enc_varint(&s, &field, &value), "\x01")); + + field.data_size = sizeof(max); + TEST(WRITES(pb_enc_svarint(&s, &field, &max), "\xfe\xff\xff\xff\x0f")); + TEST(WRITES(pb_enc_svarint(&s, &field, &min), "\xff\xff\xff\xff\x0f")); + + field.data_size = sizeof(lmax); + TEST(WRITES(pb_enc_svarint(&s, &field, &lmax), "\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01")); + TEST(WRITES(pb_enc_svarint(&s, &field, &lmin), "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01")); + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + float fvalue; + double dvalue; + + COMMENT("Test pb_enc_fixed32 using float") + fvalue = 0.0f; + TEST(WRITES(pb_enc_fixed32(&s, NULL, &fvalue), "\x00\x00\x00\x00")) + fvalue = 99.0f; + TEST(WRITES(pb_enc_fixed32(&s, NULL, &fvalue), "\x00\x00\xc6\x42")) + fvalue = -12345678.0f; + TEST(WRITES(pb_enc_fixed32(&s, NULL, &fvalue), "\x4e\x61\x3c\xcb")) + + COMMENT("Test pb_enc_fixed64 using double") + dvalue = 0.0; + TEST(WRITES(pb_enc_fixed64(&s, NULL, &dvalue), "\x00\x00\x00\x00\x00\x00\x00\x00")) + dvalue = 99.0; + TEST(WRITES(pb_enc_fixed64(&s, NULL, &dvalue), "\x00\x00\x00\x00\x00\xc0\x58\x40")) + dvalue = -12345678.0; + TEST(WRITES(pb_enc_fixed64(&s, NULL, &dvalue), "\x00\x00\x00\xc0\x29\x8c\x67\xc1")) + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + struct { size_t size; uint8_t bytes[5]; } value = {5, {'x', 'y', 'z', 'z', 'y'}}; + + COMMENT("Test pb_enc_bytes") + TEST(WRITES(pb_enc_bytes(&s, NULL, &value), "\x05xyzzy")) + value.size = 0; + TEST(WRITES(pb_enc_bytes(&s, NULL, &value), "\x00")) + } + + { + uint8_t buffer[30]; + pb_ostream_t s; + char value[30] = "xyzzy"; + + COMMENT("Test pb_enc_string") + TEST(WRITES(pb_enc_string(&s, &StringMessage_fields[0], &value), "\x05xyzzy")) + value[0] = '\0'; + TEST(WRITES(pb_enc_string(&s, &StringMessage_fields[0], &value), "\x00")) + memset(value, 'x', 30); + TEST(WRITES(pb_enc_string(&s, &StringMessage_fields[0], &value), "\x0Axxxxxxxxxx")) + } + + { + uint8_t buffer[10]; + pb_ostream_t s; + IntegerArray msg = {5, {1, 2, 3, 4, 5}}; + + COMMENT("Test pb_encode with int32 array") + + TEST(WRITES(pb_encode(&s, IntegerArray_fields, &msg), "\x0A\x05\x01\x02\x03\x04\x05")) + + msg.data_count = 0; + TEST(WRITES(pb_encode(&s, IntegerArray_fields, &msg), "")) + + msg.data_count = 10; + TEST(!pb_encode(&s, IntegerArray_fields, &msg)) + } + + { + uint8_t buffer[10]; + pb_ostream_t s; + FloatArray msg = {1, {99.0f}}; + + COMMENT("Test pb_encode with float array") + + TEST(WRITES(pb_encode(&s, FloatArray_fields, &msg), + "\x0A\x04\x00\x00\xc6\x42")) + + msg.data_count = 0; + TEST(WRITES(pb_encode(&s, FloatArray_fields, &msg), "")) + + msg.data_count = 3; + TEST(!pb_encode(&s, FloatArray_fields, &msg)) + } + + { + uint8_t buffer[10]; + pb_ostream_t s; + CallbackArray msg; + + msg.data.funcs.encode = &fieldcallback; + + COMMENT("Test pb_encode with callback field.") + TEST(WRITES(pb_encode(&s, CallbackArray_fields, &msg), "\x08\x55")) + } + + { + uint8_t buffer[10]; + pb_ostream_t s; + IntegerContainer msg = {{5, {1,2,3,4,5}}}; + + COMMENT("Test pb_encode with packed array in a submessage.") + TEST(WRITES(pb_encode(&s, IntegerContainer_fields, &msg), + "\x0A\x07\x0A\x05\x01\x02\x03\x04\x05")) + } + + { + uint8_t buffer[20]; + pb_ostream_t s; + IntegerContainer msg = {{5, {1,2,3,4,5}}}; + + COMMENT("Test pb_encode_delimited.") + TEST(WRITES(pb_encode_delimited(&s, IntegerContainer_fields, &msg), + "\x09\x0A\x07\x0A\x05\x01\x02\x03\x04\x05")) + } + + { + uint8_t buffer[10]; + pb_ostream_t s; + CallbackContainer msg; + CallbackContainerContainer msg2; + uint32_t state = 1; + + msg.submsg.data.funcs.encode = &fieldcallback; + msg2.submsg.submsg.data.funcs.encode = &fieldcallback; + + COMMENT("Test pb_encode with callback field in a submessage.") + TEST(WRITES(pb_encode(&s, CallbackContainer_fields, &msg), "\x0A\x02\x08\x55")) + TEST(WRITES(pb_encode(&s, CallbackContainerContainer_fields, &msg2), + "\x0A\x04\x0A\x02\x08\x55")) + + /* Misbehaving callback: varying output between calls */ + msg.submsg.data.funcs.encode = &crazyfieldcallback; + msg.submsg.data.arg = &state; + msg2.submsg.submsg.data.funcs.encode = &crazyfieldcallback; + msg2.submsg.submsg.data.arg = &state; + + TEST(!pb_encode(&s, CallbackContainer_fields, &msg)) + state = 1; + TEST(!pb_encode(&s, CallbackContainerContainer_fields, &msg2)) + } + + if (status != 0) + fprintf(stdout, "\n\nSome tests FAILED!\n"); + + return status; +} diff --git a/tests/extensions.options b/tests/extensions.options deleted file mode 100644 index a5cd61dd..00000000 --- a/tests/extensions.options +++ /dev/null @@ -1 +0,0 @@ -* max_size:16 diff --git a/tests/extensions.proto b/tests/extensions.proto deleted file mode 100644 index d85e8193..00000000 --- a/tests/extensions.proto +++ /dev/null @@ -1,17 +0,0 @@ -import 'alltypes.proto'; - -extend AllTypes { - optional int32 AllTypes_extensionfield1 = 255; -} - -message ExtensionMessage { - extend AllTypes { - optional ExtensionMessage AllTypes_extensionfield2 = 254; - required ExtensionMessage AllTypes_extensionfield3 = 253; - repeated ExtensionMessage AllTypes_extensionfield4 = 252; - } - - required string test1 = 1; - required int32 test2 = 2; -} - diff --git a/tests/extensions/decode_extensions.c b/tests/extensions/decode_extensions.c new file mode 100644 index 00000000..ef6a0228 --- /dev/null +++ b/tests/extensions/decode_extensions.c @@ -0,0 +1,43 @@ +/* Test decoding of extension fields. */ + +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "extensions.pb.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed.\n"); \ + return 2; \ + } + +int main(int argc, char **argv) +{ + uint8_t buffer[1024]; + size_t count = fread(buffer, 1, sizeof(buffer), stdin); + pb_istream_t stream = pb_istream_from_buffer(buffer, count); + + AllTypes alltypes = {}; + + int32_t extensionfield1; + pb_extension_t ext1 = {&AllTypes_extensionfield1, &extensionfield1, NULL}; + alltypes.extensions = &ext1; + + ExtensionMessage extensionfield2 = {}; + pb_extension_t ext2 = {&ExtensionMessage_AllTypes_extensionfield2, &extensionfield2, NULL}; + ext1.next = &ext2; + + if (!pb_decode(&stream, AllTypes_fields, &alltypes)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + + TEST(extensionfield1 == 12345) + TEST(strcmp(extensionfield2.test1, "test") == 0) + TEST(extensionfield2.test2 == 54321) + + return 0; +} + diff --git a/tests/extensions/encode_extensions.c b/tests/extensions/encode_extensions.c new file mode 100644 index 00000000..8857f148 --- /dev/null +++ b/tests/extensions/encode_extensions.c @@ -0,0 +1,38 @@ +/* Tests extension fields. + */ + +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "extensions.pb.h" + +int main(int argc, char **argv) +{ + AllTypes alltypes = {}; + + int32_t extensionfield1 = 12345; + pb_extension_t ext1 = {&AllTypes_extensionfield1, &extensionfield1, NULL}; + alltypes.extensions = &ext1; + + ExtensionMessage extensionfield2 = {"test", 54321}; + pb_extension_t ext2 = {&ExtensionMessage_AllTypes_extensionfield2, &extensionfield2, NULL}; + ext1.next = &ext2; + + uint8_t buffer[1024]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } +} + diff --git a/tests/extensions/extensions.options b/tests/extensions/extensions.options new file mode 100644 index 00000000..a5cd61dd --- /dev/null +++ b/tests/extensions/extensions.options @@ -0,0 +1 @@ +* max_size:16 diff --git a/tests/extensions/extensions.proto b/tests/extensions/extensions.proto new file mode 100644 index 00000000..d85e8193 --- /dev/null +++ b/tests/extensions/extensions.proto @@ -0,0 +1,17 @@ +import 'alltypes.proto'; + +extend AllTypes { + optional int32 AllTypes_extensionfield1 = 255; +} + +message ExtensionMessage { + extend AllTypes { + optional ExtensionMessage AllTypes_extensionfield2 = 254; + required ExtensionMessage AllTypes_extensionfield3 = 253; + repeated ExtensionMessage AllTypes_extensionfield4 = 252; + } + + required string test1 = 1; + required int32 test2 = 2; +} + diff --git a/tests/extra_fields/alltypes_with_extra_fields.pb b/tests/extra_fields/alltypes_with_extra_fields.pb new file mode 100644 index 00000000..f9f53941 Binary files /dev/null and b/tests/extra_fields/alltypes_with_extra_fields.pb differ diff --git a/tests/extra_fields/person_with_extra_field.pb b/tests/extra_fields/person_with_extra_field.pb new file mode 100644 index 00000000..ced3057c Binary files /dev/null and b/tests/extra_fields/person_with_extra_field.pb differ diff --git a/tests/funny-proto+name.proto b/tests/funny-proto+name.proto deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/missing_fields.proto b/tests/missing_fields.proto deleted file mode 100644 index cbb23ba1..00000000 --- a/tests/missing_fields.proto +++ /dev/null @@ -1,138 +0,0 @@ -/* Test for one missing field among many */ - -message AllFields -{ - required int32 field1 = 1; - required int32 field2 = 2; - required int32 field3 = 3; - required int32 field4 = 4; - required int32 field5 = 5; - required int32 field6 = 6; - required int32 field7 = 7; - required int32 field8 = 8; - required int32 field9 = 9; - required int32 field10 = 10; - required int32 field11 = 11; - required int32 field12 = 12; - required int32 field13 = 13; - required int32 field14 = 14; - required int32 field15 = 15; - required int32 field16 = 16; - required int32 field17 = 17; - required int32 field18 = 18; - required int32 field19 = 19; - required int32 field20 = 20; - required int32 field21 = 21; - required int32 field22 = 22; - required int32 field23 = 23; - required int32 field24 = 24; - required int32 field25 = 25; - required int32 field26 = 26; - required int32 field27 = 27; - required int32 field28 = 28; - required int32 field29 = 29; - required int32 field30 = 30; - required int32 field31 = 31; - required int32 field32 = 32; - required int32 field33 = 33; - required int32 field34 = 34; - required int32 field35 = 35; - required int32 field36 = 36; - required int32 field37 = 37; - required int32 field38 = 38; - required int32 field39 = 39; - required int32 field40 = 40; - required int32 field41 = 41; - required int32 field42 = 42; - required int32 field43 = 43; - required int32 field44 = 44; - required int32 field45 = 45; - required int32 field46 = 46; - required int32 field47 = 47; - required int32 field48 = 48; - required int32 field49 = 49; - required int32 field50 = 50; - required int32 field51 = 51; - required int32 field52 = 52; - required int32 field53 = 53; - required int32 field54 = 54; - required int32 field55 = 55; - required int32 field56 = 56; - required int32 field57 = 57; - required int32 field58 = 58; - required int32 field59 = 59; - required int32 field60 = 60; - required int32 field61 = 61; - required int32 field62 = 62; - required int32 field63 = 63; - required int32 field64 = 64; -} - -message MissingField -{ - required int32 field1 = 1; - required int32 field2 = 2; - required int32 field3 = 3; - required int32 field4 = 4; - required int32 field5 = 5; - required int32 field6 = 6; - required int32 field7 = 7; - required int32 field8 = 8; - required int32 field9 = 9; - required int32 field10 = 10; - required int32 field11 = 11; - required int32 field12 = 12; - required int32 field13 = 13; - required int32 field14 = 14; - required int32 field15 = 15; - required int32 field16 = 16; - required int32 field17 = 17; - required int32 field18 = 18; - required int32 field19 = 19; - required int32 field20 = 20; - required int32 field21 = 21; - required int32 field22 = 22; - required int32 field23 = 23; - required int32 field24 = 24; - required int32 field25 = 25; - required int32 field26 = 26; - required int32 field27 = 27; - required int32 field28 = 28; - required int32 field29 = 29; - required int32 field30 = 30; - required int32 field31 = 31; - required int32 field32 = 32; - required int32 field33 = 33; - required int32 field34 = 34; - required int32 field35 = 35; - required int32 field36 = 36; - required int32 field37 = 37; - required int32 field38 = 38; - required int32 field39 = 39; - required int32 field40 = 40; - required int32 field41 = 41; - required int32 field42 = 42; - required int32 field43 = 43; - required int32 field44 = 44; - required int32 field45 = 45; - required int32 field46 = 46; - required int32 field47 = 47; - required int32 field48 = 48; - required int32 field49 = 49; - required int32 field50 = 50; - required int32 field51 = 51; - required int32 field52 = 52; - required int32 field53 = 53; - required int32 field54 = 54; - required int32 field55 = 55; - required int32 field56 = 56; - required int32 field57 = 57; - required int32 field58 = 58; - required int32 field59 = 59; - required int32 field60 = 60; - required int32 field61 = 61; - required int32 field62 = 62; -/* required int32 field63 = 63; */ - required int32 field64 = 64; -} - diff --git a/tests/missing_fields/missing_fields.c b/tests/missing_fields/missing_fields.c new file mode 100644 index 00000000..27741847 --- /dev/null +++ b/tests/missing_fields/missing_fields.c @@ -0,0 +1,49 @@ +/* Checks that missing required fields are detected properly */ + +#include +#include +#include +#include "missing_fields.pb.h" + +int main() +{ + uint8_t buffer[512] = {}; + + /* Create a message with one missing field */ + { + MissingField msg = {}; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + if (!pb_encode(&stream, MissingField_fields, &msg)) + { + printf("Encode failed.\n"); + return 1; + } + } + + /* Test that it decodes properly if we don't require that field */ + { + MissingField msg = {}; + pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer)); + + if (!pb_decode(&stream, MissingField_fields, &msg)) + { + printf("Decode failed: %s\n", PB_GET_ERROR(&stream)); + return 2; + } + } + + /* Test that it does *not* decode properly if we require the field */ + { + AllFields msg = {}; + pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer)); + + if (pb_decode(&stream, AllFields_fields, &msg)) + { + printf("Decode didn't detect missing field.\n"); + return 3; + } + } + + return 0; /* All ok */ +} + diff --git a/tests/missing_fields/missing_fields.proto b/tests/missing_fields/missing_fields.proto new file mode 100644 index 00000000..cbb23ba1 --- /dev/null +++ b/tests/missing_fields/missing_fields.proto @@ -0,0 +1,138 @@ +/* Test for one missing field among many */ + +message AllFields +{ + required int32 field1 = 1; + required int32 field2 = 2; + required int32 field3 = 3; + required int32 field4 = 4; + required int32 field5 = 5; + required int32 field6 = 6; + required int32 field7 = 7; + required int32 field8 = 8; + required int32 field9 = 9; + required int32 field10 = 10; + required int32 field11 = 11; + required int32 field12 = 12; + required int32 field13 = 13; + required int32 field14 = 14; + required int32 field15 = 15; + required int32 field16 = 16; + required int32 field17 = 17; + required int32 field18 = 18; + required int32 field19 = 19; + required int32 field20 = 20; + required int32 field21 = 21; + required int32 field22 = 22; + required int32 field23 = 23; + required int32 field24 = 24; + required int32 field25 = 25; + required int32 field26 = 26; + required int32 field27 = 27; + required int32 field28 = 28; + required int32 field29 = 29; + required int32 field30 = 30; + required int32 field31 = 31; + required int32 field32 = 32; + required int32 field33 = 33; + required int32 field34 = 34; + required int32 field35 = 35; + required int32 field36 = 36; + required int32 field37 = 37; + required int32 field38 = 38; + required int32 field39 = 39; + required int32 field40 = 40; + required int32 field41 = 41; + required int32 field42 = 42; + required int32 field43 = 43; + required int32 field44 = 44; + required int32 field45 = 45; + required int32 field46 = 46; + required int32 field47 = 47; + required int32 field48 = 48; + required int32 field49 = 49; + required int32 field50 = 50; + required int32 field51 = 51; + required int32 field52 = 52; + required int32 field53 = 53; + required int32 field54 = 54; + required int32 field55 = 55; + required int32 field56 = 56; + required int32 field57 = 57; + required int32 field58 = 58; + required int32 field59 = 59; + required int32 field60 = 60; + required int32 field61 = 61; + required int32 field62 = 62; + required int32 field63 = 63; + required int32 field64 = 64; +} + +message MissingField +{ + required int32 field1 = 1; + required int32 field2 = 2; + required int32 field3 = 3; + required int32 field4 = 4; + required int32 field5 = 5; + required int32 field6 = 6; + required int32 field7 = 7; + required int32 field8 = 8; + required int32 field9 = 9; + required int32 field10 = 10; + required int32 field11 = 11; + required int32 field12 = 12; + required int32 field13 = 13; + required int32 field14 = 14; + required int32 field15 = 15; + required int32 field16 = 16; + required int32 field17 = 17; + required int32 field18 = 18; + required int32 field19 = 19; + required int32 field20 = 20; + required int32 field21 = 21; + required int32 field22 = 22; + required int32 field23 = 23; + required int32 field24 = 24; + required int32 field25 = 25; + required int32 field26 = 26; + required int32 field27 = 27; + required int32 field28 = 28; + required int32 field29 = 29; + required int32 field30 = 30; + required int32 field31 = 31; + required int32 field32 = 32; + required int32 field33 = 33; + required int32 field34 = 34; + required int32 field35 = 35; + required int32 field36 = 36; + required int32 field37 = 37; + required int32 field38 = 38; + required int32 field39 = 39; + required int32 field40 = 40; + required int32 field41 = 41; + required int32 field42 = 42; + required int32 field43 = 43; + required int32 field44 = 44; + required int32 field45 = 45; + required int32 field46 = 46; + required int32 field47 = 47; + required int32 field48 = 48; + required int32 field49 = 49; + required int32 field50 = 50; + required int32 field51 = 51; + required int32 field52 = 52; + required int32 field53 = 53; + required int32 field54 = 54; + required int32 field55 = 55; + required int32 field56 = 56; + required int32 field57 = 57; + required int32 field58 = 58; + required int32 field59 = 59; + required int32 field60 = 60; + required int32 field61 = 61; + required int32 field62 = 62; +/* required int32 field63 = 63; */ + required int32 field64 = 64; +} + diff --git a/tests/multiple_files/callbacks.proto b/tests/multiple_files/callbacks.proto new file mode 100644 index 00000000..ccd1edd8 --- /dev/null +++ b/tests/multiple_files/callbacks.proto @@ -0,0 +1,16 @@ +message SubMessage { + optional string stringvalue = 1; + repeated int32 int32value = 2; + repeated fixed32 fixed32value = 3; + repeated fixed64 fixed64value = 4; +} + +message TestMessage { + optional string stringvalue = 1; + repeated int32 int32value = 2; + repeated fixed32 fixed32value = 3; + repeated fixed64 fixed64value = 4; + optional SubMessage submsg = 5; + repeated string repeatedstring = 6; +} + diff --git a/tests/multiple_files/callbacks2.proto b/tests/multiple_files/callbacks2.proto new file mode 100644 index 00000000..9a55e15a --- /dev/null +++ b/tests/multiple_files/callbacks2.proto @@ -0,0 +1,9 @@ +// Test if including generated header file for this file + implicit include of +// callbacks.pb.h still compiles. Used with test_compiles.c. +import "callbacks.proto"; + +message Callback2Message { + required TestMessage tstmsg = 1; + required SubMessage submsg = 2; +} + diff --git a/tests/multiple_files/test_multiple_files.c b/tests/multiple_files/test_multiple_files.c new file mode 100644 index 00000000..cb4e16d3 --- /dev/null +++ b/tests/multiple_files/test_multiple_files.c @@ -0,0 +1,13 @@ +/* + * Tests if still compile if typedefs are redfefined in STATIC_ASSERTS when + * proto file includes another poto file + */ + +#include +#include +#include "callbacks2.pb.h" + +int main() +{ + return 0; +} diff --git a/tests/no_messages.proto b/tests/no_messages.proto deleted file mode 100644 index 279216b0..00000000 --- a/tests/no_messages.proto +++ /dev/null @@ -1,7 +0,0 @@ -/* Test that a file without any messages works. */ - -enum Test { - First = 1; -} - - diff --git a/tests/no_messages/no_messages.proto b/tests/no_messages/no_messages.proto new file mode 100644 index 00000000..279216b0 --- /dev/null +++ b/tests/no_messages/no_messages.proto @@ -0,0 +1,7 @@ +/* Test that a file without any messages works. */ + +enum Test { + First = 1; +} + + diff --git a/tests/options.expected b/tests/options.expected deleted file mode 100644 index e6179a27..00000000 --- a/tests/options.expected +++ /dev/null @@ -1,7 +0,0 @@ -char filesize\[20\]; -char msgsize\[30\]; -char fieldsize\[40\]; -pb_callback_t int32_callback; -\sEnumValue1 = 1 -Message5_EnumValue1 -} pb_packed my_packed_struct; diff --git a/tests/options.proto b/tests/options.proto deleted file mode 100644 index b5badcfd..00000000 --- a/tests/options.proto +++ /dev/null @@ -1,73 +0,0 @@ -/* Test nanopb option parsing. - * options.expected lists the patterns that are searched for in the output. - */ - -import "nanopb.proto"; - -// File level options -option (nanopb_fileopt).max_size = 20; - -message Message1 -{ - required string filesize = 1; -} - -// Message level options -message Message2 -{ - option (nanopb_msgopt).max_size = 30; - required string msgsize = 1; -} - -// Field level options -message Message3 -{ - required string fieldsize = 1 [(nanopb).max_size = 40]; -} - -// Forced callback field -message Message4 -{ - required int32 int32_callback = 1 [(nanopb).type = FT_CALLBACK]; -} - -// Short enum names -enum Enum1 -{ - option (nanopb_enumopt).long_names = false; - EnumValue1 = 1; - EnumValue2 = 2; -} - -message EnumTest -{ - required Enum1 field = 1 [default = EnumValue2]; -} - -// Short enum names inside message -message Message5 -{ - enum Enum2 - { - option (nanopb_enumopt).long_names = false; - EnumValue1 = 1; - } - required Enum2 field = 1 [default = EnumValue1]; -} - -// Packed structure -message my_packed_struct -{ - option (nanopb_msgopt).packed_struct = true; - optional int32 myfield = 1; -} - -// Message with ignored field -// Note: doesn't really test if the field is missing in the output, -// but atleast tests that the output compiles. -message Message6 -{ - required int32 field1 = 1; - optional int32 field2 = 2 [(nanopb).type = FT_IGNORE]; -} - diff --git a/tests/options/options.expected b/tests/options/options.expected new file mode 100644 index 00000000..e6179a27 --- /dev/null +++ b/tests/options/options.expected @@ -0,0 +1,7 @@ +char filesize\[20\]; +char msgsize\[30\]; +char fieldsize\[40\]; +pb_callback_t int32_callback; +\sEnumValue1 = 1 +Message5_EnumValue1 +} pb_packed my_packed_struct; diff --git a/tests/options/options.proto b/tests/options/options.proto new file mode 100644 index 00000000..b5badcfd --- /dev/null +++ b/tests/options/options.proto @@ -0,0 +1,73 @@ +/* Test nanopb option parsing. + * options.expected lists the patterns that are searched for in the output. + */ + +import "nanopb.proto"; + +// File level options +option (nanopb_fileopt).max_size = 20; + +message Message1 +{ + required string filesize = 1; +} + +// Message level options +message Message2 +{ + option (nanopb_msgopt).max_size = 30; + required string msgsize = 1; +} + +// Field level options +message Message3 +{ + required string fieldsize = 1 [(nanopb).max_size = 40]; +} + +// Forced callback field +message Message4 +{ + required int32 int32_callback = 1 [(nanopb).type = FT_CALLBACK]; +} + +// Short enum names +enum Enum1 +{ + option (nanopb_enumopt).long_names = false; + EnumValue1 = 1; + EnumValue2 = 2; +} + +message EnumTest +{ + required Enum1 field = 1 [default = EnumValue2]; +} + +// Short enum names inside message +message Message5 +{ + enum Enum2 + { + option (nanopb_enumopt).long_names = false; + EnumValue1 = 1; + } + required Enum2 field = 1 [default = EnumValue1]; +} + +// Packed structure +message my_packed_struct +{ + option (nanopb_msgopt).packed_struct = true; + optional int32 myfield = 1; +} + +// Message with ignored field +// Note: doesn't really test if the field is missing in the output, +// but atleast tests that the output compiles. +message Message6 +{ + required int32 field1 = 1; + optional int32 field2 = 2 [(nanopb).type = FT_IGNORE]; +} + diff --git a/tests/person.proto b/tests/person.proto deleted file mode 100644 index dafcf934..00000000 --- a/tests/person.proto +++ /dev/null @@ -1,20 +0,0 @@ -import "nanopb.proto"; - -message Person { - required string name = 1 [(nanopb).max_size = 40]; - required int32 id = 2; - optional string email = 3 [(nanopb).max_size = 40]; - - enum PhoneType { - MOBILE = 0; - HOME = 1; - WORK = 2; - } - - message PhoneNumber { - required string number = 1 [(nanopb).max_size = 40]; - optional PhoneType type = 2 [default = HOME]; - } - - repeated PhoneNumber phone = 4 [(nanopb).max_count = 5]; -} diff --git a/tests/person_with_extra_field.pb b/tests/person_with_extra_field.pb deleted file mode 100644 index ced3057c..00000000 Binary files a/tests/person_with_extra_field.pb and /dev/null differ diff --git a/tests/special_characters/funny-proto+name.proto b/tests/special_characters/funny-proto+name.proto new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_decode1.c b/tests/test_decode1.c deleted file mode 100644 index 56bbd8f8..00000000 --- a/tests/test_decode1.c +++ /dev/null @@ -1,83 +0,0 @@ -/* A very simple decoding test case, using person.proto. - * Produces output compatible with protoc --decode. - * Reads the encoded data from stdin and prints the values - * to stdout as text. - * - * Run e.g. ./test_encode1 | ./test_decode1 - */ - -#include -#include -#include "person.pb.h" - -/* This function is called once from main(), it handles - the decoding and printing. */ -bool print_person(pb_istream_t *stream) -{ - int i; - Person person; - - if (!pb_decode(stream, Person_fields, &person)) - return false; - - /* Now the decoding is done, rest is just to print stuff out. */ - - printf("name: \"%s\"\n", person.name); - printf("id: %ld\n", (long)person.id); - - if (person.has_email) - printf("email: \"%s\"\n", person.email); - - for (i = 0; i < person.phone_count; i++) - { - Person_PhoneNumber *phone = &person.phone[i]; - printf("phone {\n"); - printf(" number: \"%s\"\n", phone->number); - - if (phone->has_type) - { - switch (phone->type) - { - case Person_PhoneType_WORK: - printf(" type: WORK\n"); - break; - - case Person_PhoneType_HOME: - printf(" type: HOME\n"); - break; - - case Person_PhoneType_MOBILE: - printf(" type: MOBILE\n"); - break; - } - } - printf("}\n"); - } - - return true; -} - -int main() -{ - /* Read the data into buffer */ - uint8_t buffer[512]; - size_t count = fread(buffer, 1, sizeof(buffer), stdin); - - if (!feof(stdin)) - { - printf("Message does not fit in buffer\n"); - return 1; - } - - /* Construct a pb_istream_t for reading from the buffer */ - pb_istream_t stream = pb_istream_from_buffer(buffer, count); - - /* Decode and print out the stuff */ - if (!print_person(&stream)) - { - printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); - return 1; - } else { - return 0; - } -} diff --git a/tests/test_decode2.c b/tests/test_decode2.c deleted file mode 100644 index 2142977e..00000000 --- a/tests/test_decode2.c +++ /dev/null @@ -1,83 +0,0 @@ -/* Same as test_decode1 but reads from stdin directly. - */ - -#include -#include -#include "person.pb.h" - -/* This function is called once from main(), it handles - the decoding and printing. - Ugly copy-paste from test_decode1.c. */ -bool print_person(pb_istream_t *stream) -{ - int i; - Person person; - - if (!pb_decode(stream, Person_fields, &person)) - return false; - - /* Now the decoding is done, rest is just to print stuff out. */ - - printf("name: \"%s\"\n", person.name); - printf("id: %ld\n", (long)person.id); - - if (person.has_email) - printf("email: \"%s\"\n", person.email); - - for (i = 0; i < person.phone_count; i++) - { - Person_PhoneNumber *phone = &person.phone[i]; - printf("phone {\n"); - printf(" number: \"%s\"\n", phone->number); - - if (phone->has_type) - { - switch (phone->type) - { - case Person_PhoneType_WORK: - printf(" type: WORK\n"); - break; - - case Person_PhoneType_HOME: - printf(" type: HOME\n"); - break; - - case Person_PhoneType_MOBILE: - printf(" type: MOBILE\n"); - break; - } - } - printf("}\n"); - } - - return true; -} - -/* This binds the pb_istream_t to stdin */ -bool callback(pb_istream_t *stream, uint8_t *buf, size_t count) -{ - FILE *file = (FILE*)stream->state; - bool status; - - status = (fread(buf, 1, count, file) == count); - - if (feof(file)) - stream->bytes_left = 0; - - return status; -} - -int main() -{ - /* Maximum size is specified to prevent infinite length messages from - * hanging this in the fuzz test. - */ - pb_istream_t stream = {&callback, stdin, 10000}; - if (!print_person(&stream)) - { - printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); - return 1; - } else { - return 0; - } -} diff --git a/tests/test_decode3.c b/tests/test_decode3.c deleted file mode 100644 index 55d025c0..00000000 --- a/tests/test_decode3.c +++ /dev/null @@ -1,197 +0,0 @@ -/* Tests the decoding of all types. - * This is the counterpart of test_encode3. - * Run e.g. ./test_encode3 | ./test_decode3 - */ - -#include -#include -#include -#include -#include "alltypes.pb.h" - -#define TEST(x) if (!(x)) { \ - printf("Test " #x " failed.\n"); \ - return false; \ - } - -/* This function is called once from main(), it handles - the decoding and checks the fields. */ -bool check_alltypes(pb_istream_t *stream, int mode) -{ - AllTypes alltypes; - - /* Fill with garbage to better detect initialization errors */ - memset(&alltypes, 0xAA, sizeof(alltypes)); - - if (!pb_decode(stream, AllTypes_fields, &alltypes)) - return false; - - TEST(alltypes.req_int32 == -1001); - TEST(alltypes.req_int64 == -1002); - TEST(alltypes.req_uint32 == 1003); - TEST(alltypes.req_uint64 == 1004); - TEST(alltypes.req_sint32 == -1005); - TEST(alltypes.req_sint64 == -1006); - TEST(alltypes.req_bool == true); - - TEST(alltypes.req_fixed32 == 1008); - TEST(alltypes.req_sfixed32 == -1009); - TEST(alltypes.req_float == 1010.0f); - - TEST(alltypes.req_fixed64 == 1011); - TEST(alltypes.req_sfixed64 == -1012); - TEST(alltypes.req_double == 1013.0f); - - TEST(strcmp(alltypes.req_string, "1014") == 0); - TEST(alltypes.req_bytes.size == 4); - TEST(memcmp(alltypes.req_bytes.bytes, "1015", 4) == 0); - TEST(strcmp(alltypes.req_submsg.substuff1, "1016") == 0); - TEST(alltypes.req_submsg.substuff2 == 1016); - TEST(alltypes.req_submsg.substuff3 == 3); - TEST(alltypes.req_enum == MyEnum_Truth); - - TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); - TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); - TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); - TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); - TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); - TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0); - TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); - - TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); - TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); - TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); - - TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); - TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0); - TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); - - TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); - TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); - TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); - - TEST(alltypes.rep_submsg_count == 5); - TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); - TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); - TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3); - - TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); - TEST(alltypes.rep_emptymsg_count == 5); - - if (mode == 0) - { - /* Expect default values */ - TEST(alltypes.has_opt_int32 == false); - TEST(alltypes.opt_int32 == 4041); - TEST(alltypes.has_opt_int64 == false); - TEST(alltypes.opt_int64 == 4042); - TEST(alltypes.has_opt_uint32 == false); - TEST(alltypes.opt_uint32 == 4043); - TEST(alltypes.has_opt_uint64 == false); - TEST(alltypes.opt_uint64 == 4044); - TEST(alltypes.has_opt_sint32 == false); - TEST(alltypes.opt_sint32 == 4045); - TEST(alltypes.has_opt_sint64 == false); - TEST(alltypes.opt_sint64 == 4046); - TEST(alltypes.has_opt_bool == false); - TEST(alltypes.opt_bool == false); - - TEST(alltypes.has_opt_fixed32 == false); - TEST(alltypes.opt_fixed32 == 4048); - TEST(alltypes.has_opt_sfixed32 == false); - TEST(alltypes.opt_sfixed32 == 4049); - TEST(alltypes.has_opt_float == false); - TEST(alltypes.opt_float == 4050.0f); - - TEST(alltypes.has_opt_fixed64 == false); - TEST(alltypes.opt_fixed64 == 4051); - TEST(alltypes.has_opt_sfixed64 == false); - TEST(alltypes.opt_sfixed64 == 4052); - TEST(alltypes.has_opt_double == false); - TEST(alltypes.opt_double == 4053.0); - - TEST(alltypes.has_opt_string == false); - TEST(strcmp(alltypes.opt_string, "4054") == 0); - TEST(alltypes.has_opt_bytes == false); - TEST(alltypes.opt_bytes.size == 4); - TEST(memcmp(alltypes.opt_bytes.bytes, "4055", 4) == 0); - TEST(alltypes.has_opt_submsg == false); - TEST(strcmp(alltypes.opt_submsg.substuff1, "1") == 0); - TEST(alltypes.opt_submsg.substuff2 == 2); - TEST(alltypes.opt_submsg.substuff3 == 3); - TEST(alltypes.has_opt_enum == false); - TEST(alltypes.opt_enum == MyEnum_Second); - TEST(alltypes.has_opt_emptymsg == false); - } - else - { - /* Expect filled-in values */ - TEST(alltypes.has_opt_int32 == true); - TEST(alltypes.opt_int32 == 3041); - TEST(alltypes.has_opt_int64 == true); - TEST(alltypes.opt_int64 == 3042); - TEST(alltypes.has_opt_uint32 == true); - TEST(alltypes.opt_uint32 == 3043); - TEST(alltypes.has_opt_uint64 == true); - TEST(alltypes.opt_uint64 == 3044); - TEST(alltypes.has_opt_sint32 == true); - TEST(alltypes.opt_sint32 == 3045); - TEST(alltypes.has_opt_sint64 == true); - TEST(alltypes.opt_sint64 == 3046); - TEST(alltypes.has_opt_bool == true); - TEST(alltypes.opt_bool == true); - - TEST(alltypes.has_opt_fixed32 == true); - TEST(alltypes.opt_fixed32 == 3048); - TEST(alltypes.has_opt_sfixed32 == true); - TEST(alltypes.opt_sfixed32 == 3049); - TEST(alltypes.has_opt_float == true); - TEST(alltypes.opt_float == 3050.0f); - - TEST(alltypes.has_opt_fixed64 == true); - TEST(alltypes.opt_fixed64 == 3051); - TEST(alltypes.has_opt_sfixed64 == true); - TEST(alltypes.opt_sfixed64 == 3052); - TEST(alltypes.has_opt_double == true); - TEST(alltypes.opt_double == 3053.0); - - TEST(alltypes.has_opt_string == true); - TEST(strcmp(alltypes.opt_string, "3054") == 0); - TEST(alltypes.has_opt_bytes == true); - TEST(alltypes.opt_bytes.size == 4); - TEST(memcmp(alltypes.opt_bytes.bytes, "3055", 4) == 0); - TEST(alltypes.has_opt_submsg == true); - TEST(strcmp(alltypes.opt_submsg.substuff1, "3056") == 0); - TEST(alltypes.opt_submsg.substuff2 == 3056); - TEST(alltypes.opt_submsg.substuff3 == 3); - TEST(alltypes.has_opt_enum == true); - TEST(alltypes.opt_enum == MyEnum_Truth); - TEST(alltypes.has_opt_emptymsg == true); - } - - TEST(alltypes.end == 1099); - - return true; -} - -int main(int argc, char **argv) -{ - /* Whether to expect the optional values or the default values. */ - int mode = (argc > 1) ? atoi(argv[1]) : 0; - - /* Read the data into buffer */ - uint8_t buffer[1024]; - size_t count = fread(buffer, 1, sizeof(buffer), stdin); - - /* Construct a pb_istream_t for reading from the buffer */ - pb_istream_t stream = pb_istream_from_buffer(buffer, count); - - /* Decode and print out the stuff */ - if (!check_alltypes(&stream, mode)) - { - printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); - return 1; - } else { - return 0; - } -} diff --git a/tests/test_decode_callbacks.c b/tests/test_decode_callbacks.c deleted file mode 100644 index b5056923..00000000 --- a/tests/test_decode_callbacks.c +++ /dev/null @@ -1,93 +0,0 @@ -/* Decoding testcase for callback fields. - * Run e.g. ./test_encode_callbacks | ./test_decode_callbacks - */ - -#include -#include -#include "callbacks.pb.h" - -bool print_string(pb_istream_t *stream, const pb_field_t *field, void **arg) -{ - uint8_t buffer[1024] = {0}; - - /* We could read block-by-block to avoid the large buffer... */ - if (stream->bytes_left > sizeof(buffer) - 1) - return false; - - if (!pb_read(stream, buffer, stream->bytes_left)) - return false; - - /* Print the string, in format comparable with protoc --decode. - * Format comes from the arg defined in main(). - */ - printf((char*)*arg, buffer); - return true; -} - -bool print_int32(pb_istream_t *stream, const pb_field_t *field, void **arg) -{ - uint64_t value; - if (!pb_decode_varint(stream, &value)) - return false; - - printf((char*)*arg, (long)value); - return true; -} - -bool print_fixed32(pb_istream_t *stream, const pb_field_t *field, void **arg) -{ - uint32_t value; - if (!pb_decode_fixed32(stream, &value)) - return false; - - printf((char*)*arg, (long)value); - return true; -} - -bool print_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg) -{ - uint64_t value; - if (!pb_decode_fixed64(stream, &value)) - return false; - - printf((char*)*arg, (long long)value); - return true; -} - -int main() -{ - uint8_t buffer[1024]; - size_t length = fread(buffer, 1, 1024, stdin); - pb_istream_t stream = pb_istream_from_buffer(buffer, length); - - /* Note: empty initializer list initializes the struct with all-0. - * This is recommended so that unused callbacks are set to NULL instead - * of crashing at runtime. - */ - TestMessage testmessage = {}; - - testmessage.submsg.stringvalue.funcs.decode = &print_string; - testmessage.submsg.stringvalue.arg = "submsg {\n stringvalue: \"%s\"\n"; - testmessage.submsg.int32value.funcs.decode = &print_int32; - testmessage.submsg.int32value.arg = " int32value: %ld\n"; - testmessage.submsg.fixed32value.funcs.decode = &print_fixed32; - testmessage.submsg.fixed32value.arg = " fixed32value: %ld\n"; - testmessage.submsg.fixed64value.funcs.decode = &print_fixed64; - testmessage.submsg.fixed64value.arg = " fixed64value: %lld\n}\n"; - - testmessage.stringvalue.funcs.decode = &print_string; - testmessage.stringvalue.arg = "stringvalue: \"%s\"\n"; - testmessage.int32value.funcs.decode = &print_int32; - testmessage.int32value.arg = "int32value: %ld\n"; - testmessage.fixed32value.funcs.decode = &print_fixed32; - testmessage.fixed32value.arg = "fixed32value: %ld\n"; - testmessage.fixed64value.funcs.decode = &print_fixed64; - testmessage.fixed64value.arg = "fixed64value: %lld\n"; - testmessage.repeatedstring.funcs.decode = &print_string; - testmessage.repeatedstring.arg = "repeatedstring: \"%s\"\n"; - - if (!pb_decode(&stream, TestMessage_fields, &testmessage)) - return 1; - - return 0; -} diff --git a/tests/test_decode_extensions.c b/tests/test_decode_extensions.c deleted file mode 100644 index ef6a0228..00000000 --- a/tests/test_decode_extensions.c +++ /dev/null @@ -1,43 +0,0 @@ -/* Test decoding of extension fields. */ - -#include -#include -#include -#include -#include "alltypes.pb.h" -#include "extensions.pb.h" - -#define TEST(x) if (!(x)) { \ - printf("Test " #x " failed.\n"); \ - return 2; \ - } - -int main(int argc, char **argv) -{ - uint8_t buffer[1024]; - size_t count = fread(buffer, 1, sizeof(buffer), stdin); - pb_istream_t stream = pb_istream_from_buffer(buffer, count); - - AllTypes alltypes = {}; - - int32_t extensionfield1; - pb_extension_t ext1 = {&AllTypes_extensionfield1, &extensionfield1, NULL}; - alltypes.extensions = &ext1; - - ExtensionMessage extensionfield2 = {}; - pb_extension_t ext2 = {&ExtensionMessage_AllTypes_extensionfield2, &extensionfield2, NULL}; - ext1.next = &ext2; - - if (!pb_decode(&stream, AllTypes_fields, &alltypes)) - { - printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); - return 1; - } - - TEST(extensionfield1 == 12345) - TEST(strcmp(extensionfield2.test1, "test") == 0) - TEST(extensionfield2.test2 == 54321) - - return 0; -} - diff --git a/tests/test_encode1.c b/tests/test_encode1.c deleted file mode 100644 index 742c99f4..00000000 --- a/tests/test_encode1.c +++ /dev/null @@ -1,33 +0,0 @@ -/* A very simple encoding test case using person.proto. - * Just puts constant data in the fields and encodes into - * buffer, which is then written to stdout. - */ - -#include -#include -#include "person.pb.h" - -int main() -{ - /* Initialize the structure with constants */ - Person person = {"Test Person 99", 99, true, "test@person.com", - 3, {{"555-12345678", true, Person_PhoneType_MOBILE}, - {"99-2342", false, 0}, - {"1234-5678", true, Person_PhoneType_WORK}, - }}; - - uint8_t buffer[512]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); - - /* Now encode it and check if we succeeded. */ - if (pb_encode(&stream, Person_fields, &person)) - { - fwrite(buffer, 1, stream.bytes_written, stdout); - return 0; /* Success */ - } - else - { - fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); - return 1; /* Failure */ - } -} diff --git a/tests/test_encode2.c b/tests/test_encode2.c deleted file mode 100644 index fd25c6cb..00000000 --- a/tests/test_encode2.c +++ /dev/null @@ -1,37 +0,0 @@ -/* Same as test_encode1.c, except writes directly to stdout. - */ - -#include -#include -#include "person.pb.h" - -/* This binds the pb_ostream_t into the stdout stream */ -bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) -{ - FILE *file = (FILE*) stream->state; - return fwrite(buf, 1, count, file) == count; -} - -int main() -{ - /* Initialize the structure with constants */ - Person person = {"Test Person 99", 99, true, "test@person.com", - 3, {{"555-12345678", true, Person_PhoneType_MOBILE}, - {"99-2342", false, 0}, - {"1234-5678", true, Person_PhoneType_WORK}, - }}; - - /* Prepare the stream, output goes directly to stdout */ - pb_ostream_t stream = {&streamcallback, stdout, SIZE_MAX, 0}; - - /* Now encode it and check if we succeeded. */ - if (pb_encode(&stream, Person_fields, &person)) - { - return 0; /* Success */ - } - else - { - fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); - return 1; /* Failure */ - } -} diff --git a/tests/test_encode3.c b/tests/test_encode3.c deleted file mode 100644 index 982ad3c8..00000000 --- a/tests/test_encode3.c +++ /dev/null @@ -1,130 +0,0 @@ -/* Attempts to test all the datatypes supported by ProtoBuf. - */ - -#include -#include -#include -#include -#include "alltypes.pb.h" - -int main(int argc, char **argv) -{ - int mode = (argc > 1) ? atoi(argv[1]) : 0; - - /* Initialize the structure with constants */ - AllTypes alltypes = {0}; - - alltypes.req_int32 = -1001; - alltypes.req_int64 = -1002; - alltypes.req_uint32 = 1003; - alltypes.req_uint64 = 1004; - alltypes.req_sint32 = -1005; - alltypes.req_sint64 = -1006; - alltypes.req_bool = true; - - alltypes.req_fixed32 = 1008; - alltypes.req_sfixed32 = -1009; - alltypes.req_float = 1010.0f; - - alltypes.req_fixed64 = 1011; - alltypes.req_sfixed64 = -1012; - alltypes.req_double = 1013.0; - - strcpy(alltypes.req_string, "1014"); - alltypes.req_bytes.size = 4; - memcpy(alltypes.req_bytes.bytes, "1015", 4); - strcpy(alltypes.req_submsg.substuff1, "1016"); - alltypes.req_submsg.substuff2 = 1016; - alltypes.req_enum = MyEnum_Truth; - - alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; - alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002; - alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; - alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004; - alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005; - alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006; - alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; - - alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; - alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009; - alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; - - alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011; - alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012; - alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0; - - alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); - alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4; - memcpy(alltypes.rep_bytes[4].bytes, "2015", 4); - - alltypes.rep_submsg_count = 5; - strcpy(alltypes.rep_submsg[4].substuff1, "2016"); - alltypes.rep_submsg[4].substuff2 = 2016; - alltypes.rep_submsg[4].has_substuff3 = true; - alltypes.rep_submsg[4].substuff3 = 2016; - - alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; - alltypes.rep_emptymsg_count = 5; - - if (mode != 0) - { - /* Fill in values for optional fields */ - alltypes.has_opt_int32 = true; - alltypes.opt_int32 = 3041; - alltypes.has_opt_int64 = true; - alltypes.opt_int64 = 3042; - alltypes.has_opt_uint32 = true; - alltypes.opt_uint32 = 3043; - alltypes.has_opt_uint64 = true; - alltypes.opt_uint64 = 3044; - alltypes.has_opt_sint32 = true; - alltypes.opt_sint32 = 3045; - alltypes.has_opt_sint64 = true; - alltypes.opt_sint64 = 3046; - alltypes.has_opt_bool = true; - alltypes.opt_bool = true; - - alltypes.has_opt_fixed32 = true; - alltypes.opt_fixed32 = 3048; - alltypes.has_opt_sfixed32 = true; - alltypes.opt_sfixed32 = 3049; - alltypes.has_opt_float = true; - alltypes.opt_float = 3050.0f; - - alltypes.has_opt_fixed64 = true; - alltypes.opt_fixed64 = 3051; - alltypes.has_opt_sfixed64 = true; - alltypes.opt_sfixed64 = 3052; - alltypes.has_opt_double = true; - alltypes.opt_double = 3053.0; - - alltypes.has_opt_string = true; - strcpy(alltypes.opt_string, "3054"); - alltypes.has_opt_bytes = true; - alltypes.opt_bytes.size = 4; - memcpy(alltypes.opt_bytes.bytes, "3055", 4); - alltypes.has_opt_submsg = true; - strcpy(alltypes.opt_submsg.substuff1, "3056"); - alltypes.opt_submsg.substuff2 = 3056; - alltypes.has_opt_enum = true; - alltypes.opt_enum = MyEnum_Truth; - alltypes.has_opt_emptymsg = true; - } - - alltypes.end = 1099; - - uint8_t buffer[1024]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); - - /* Now encode it and check if we succeeded. */ - if (pb_encode(&stream, AllTypes_fields, &alltypes)) - { - fwrite(buffer, 1, stream.bytes_written, stdout); - return 0; /* Success */ - } - else - { - fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); - return 1; /* Failure */ - } -} diff --git a/tests/test_encode_callbacks.c b/tests/test_encode_callbacks.c deleted file mode 100644 index 3bb6a45e..00000000 --- a/tests/test_encode_callbacks.c +++ /dev/null @@ -1,86 +0,0 @@ -/* Encoding testcase for callback fields */ - -#include -#include -#include -#include "callbacks.pb.h" - -bool encode_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) -{ - char *str = "Hello world!"; - - if (!pb_encode_tag_for_field(stream, field)) - return false; - - return pb_encode_string(stream, (uint8_t*)str, strlen(str)); -} - -bool encode_int32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) -{ - if (!pb_encode_tag_for_field(stream, field)) - return false; - - return pb_encode_varint(stream, 42); -} - -bool encode_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) -{ - if (!pb_encode_tag_for_field(stream, field)) - return false; - - uint32_t value = 42; - return pb_encode_fixed32(stream, &value); -} - -bool encode_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) -{ - if (!pb_encode_tag_for_field(stream, field)) - return false; - - uint64_t value = 42; - return pb_encode_fixed64(stream, &value); -} - -bool encode_repeatedstring(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) -{ - char *str[4] = {"Hello world!", "", "Test", "Test2"}; - int i; - - for (i = 0; i < 4; i++) - { - if (!pb_encode_tag_for_field(stream, field)) - return false; - - if (!pb_encode_string(stream, (uint8_t*)str[i], strlen(str[i]))) - return false; - } - return true; -} - -int main() -{ - uint8_t buffer[1024]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, 1024); - TestMessage testmessage = {}; - - testmessage.stringvalue.funcs.encode = &encode_string; - testmessage.int32value.funcs.encode = &encode_int32; - testmessage.fixed32value.funcs.encode = &encode_fixed32; - testmessage.fixed64value.funcs.encode = &encode_fixed64; - - testmessage.has_submsg = true; - testmessage.submsg.stringvalue.funcs.encode = &encode_string; - testmessage.submsg.int32value.funcs.encode = &encode_int32; - testmessage.submsg.fixed32value.funcs.encode = &encode_fixed32; - testmessage.submsg.fixed64value.funcs.encode = &encode_fixed64; - - testmessage.repeatedstring.funcs.encode = &encode_repeatedstring; - - if (!pb_encode(&stream, TestMessage_fields, &testmessage)) - return 1; - - if (fwrite(buffer, stream.bytes_written, 1, stdout) != 1) - return 2; - - return 0; -} diff --git a/tests/test_encode_extensions.c b/tests/test_encode_extensions.c deleted file mode 100644 index 8857f148..00000000 --- a/tests/test_encode_extensions.c +++ /dev/null @@ -1,38 +0,0 @@ -/* Tests extension fields. - */ - -#include -#include -#include -#include -#include "alltypes.pb.h" -#include "extensions.pb.h" - -int main(int argc, char **argv) -{ - AllTypes alltypes = {}; - - int32_t extensionfield1 = 12345; - pb_extension_t ext1 = {&AllTypes_extensionfield1, &extensionfield1, NULL}; - alltypes.extensions = &ext1; - - ExtensionMessage extensionfield2 = {"test", 54321}; - pb_extension_t ext2 = {&ExtensionMessage_AllTypes_extensionfield2, &extensionfield2, NULL}; - ext1.next = &ext2; - - uint8_t buffer[1024]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); - - /* Now encode it and check if we succeeded. */ - if (pb_encode(&stream, AllTypes_fields, &alltypes)) - { - fwrite(buffer, 1, stream.bytes_written, stdout); - return 0; /* Success */ - } - else - { - fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); - return 1; /* Failure */ - } -} - diff --git a/tests/test_missing_fields.c b/tests/test_missing_fields.c deleted file mode 100644 index 27741847..00000000 --- a/tests/test_missing_fields.c +++ /dev/null @@ -1,49 +0,0 @@ -/* Checks that missing required fields are detected properly */ - -#include -#include -#include -#include "missing_fields.pb.h" - -int main() -{ - uint8_t buffer[512] = {}; - - /* Create a message with one missing field */ - { - MissingField msg = {}; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); - if (!pb_encode(&stream, MissingField_fields, &msg)) - { - printf("Encode failed.\n"); - return 1; - } - } - - /* Test that it decodes properly if we don't require that field */ - { - MissingField msg = {}; - pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer)); - - if (!pb_decode(&stream, MissingField_fields, &msg)) - { - printf("Decode failed: %s\n", PB_GET_ERROR(&stream)); - return 2; - } - } - - /* Test that it does *not* decode properly if we require the field */ - { - AllFields msg = {}; - pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer)); - - if (pb_decode(&stream, AllFields_fields, &msg)) - { - printf("Decode didn't detect missing field.\n"); - return 3; - } - } - - return 0; /* All ok */ -} - diff --git a/tests/test_multiple_files.c b/tests/test_multiple_files.c deleted file mode 100644 index cb4e16d3..00000000 --- a/tests/test_multiple_files.c +++ /dev/null @@ -1,13 +0,0 @@ -/* - * Tests if still compile if typedefs are redfefined in STATIC_ASSERTS when - * proto file includes another poto file - */ - -#include -#include -#include "callbacks2.pb.h" - -int main() -{ - return 0; -} diff --git a/tests/testperson.pb b/tests/testperson.pb deleted file mode 100644 index d1eb8cfb..00000000 --- a/tests/testperson.pb +++ /dev/null @@ -1,3 +0,0 @@ - - Test Person7foobar@foobar.com" - 555-12345678 \ No newline at end of file diff --git a/tests/unittestproto.proto b/tests/unittestproto.proto deleted file mode 100644 index 7024942e..00000000 --- a/tests/unittestproto.proto +++ /dev/null @@ -1,32 +0,0 @@ -import 'nanopb.proto'; - -message IntegerArray { - repeated int32 data = 1 [(nanopb).max_count = 10]; -} - -message FloatArray { - repeated float data = 1 [(nanopb).max_count = 10]; -} - -message StringMessage { - required string data = 1 [(nanopb).max_size = 10]; -} - -message CallbackArray { - // We cheat a bit and use this message for testing other types, too. - // Nanopb does not care about the actual defined data type for callback - // fields. - repeated int32 data = 1; -} - -message IntegerContainer { - required IntegerArray submsg = 1; -} - -message CallbackContainer { - required CallbackArray submsg = 1; -} - -message CallbackContainerContainer { - required CallbackContainer submsg = 1; -} diff --git a/tests/unittests.h b/tests/unittests.h deleted file mode 100644 index c2b470ad..00000000 --- a/tests/unittests.h +++ /dev/null @@ -1,14 +0,0 @@ -#include - -#define COMMENT(x) printf("\n----" x "----\n"); -#define STR(x) #x -#define STR2(x) STR(x) -#define TEST(x) \ - if (!(x)) { \ - fprintf(stderr, "\033[31;1mFAILED:\033[22;39m " __FILE__ ":" STR2(__LINE__) " " #x "\n"); \ - status = 1; \ - } else { \ - printf("\033[32;1mOK:\033[22;39m " #x "\n"); \ - } - - -- cgit 1.2.3-korg From 4821e7f457ebd28aabcbdea726ebce11265f402f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 8 Sep 2013 19:55:05 +0300 Subject: Add support for running the nanopb generator as protoc plugin. Will be used to implement issue 47. For now, symlink nanopb_generator.py as protoc-gen-nanopb and use protoc --nanopb_out=. to call it. --- generator/Makefile | 3 + generator/nanopb_generator.py | 165 ++++++++++++++++++++++++++++-------------- generator/plugin.proto | 145 +++++++++++++++++++++++++++++++++++++ generator/plugin_pb2.py | 161 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 419 insertions(+), 55 deletions(-) mode change 100644 => 100755 generator/nanopb_generator.py create mode 100644 generator/plugin.proto create mode 100644 generator/plugin_pb2.py diff --git a/generator/Makefile b/generator/Makefile index 161ef387..f2b02a95 100644 --- a/generator/Makefile +++ b/generator/Makefile @@ -1,2 +1,5 @@ nanopb_pb2.py: nanopb.proto protoc --python_out=. -I /usr/include -I . nanopb.proto + +plugin_pb2.py: plugin.proto + protoc --python_out=. -I /usr/include -I . plugin.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py old mode 100644 new mode 100755 index 55e5ab65..d70938a1 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,3 +1,5 @@ +#!/usr/bin/python + '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' nanopb_version = "nanopb-0.2.3-dev" @@ -602,7 +604,7 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio # End of header yield '\n#endif\n' -def generate_source(headername, enums, messages, extensions): +def generate_source(headername, enums, messages, extensions, options): '''Generate content for a source file.''' yield '/* Automatically generated nanopb constant definitions */\n' @@ -780,73 +782,126 @@ optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", def optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[], help="Set generator option (max_size, max_count etc.).") -def process(filenames, options): - '''Process the files given on the command line.''' +def process_file(filename, fdesc, options): + '''Process a single file. + filename: The full path to the .proto or .pb source file, as string. + fdesc: The loaded FileDescriptorSet, or None to read from the input file. + options: Command line options as they come from OptionsParser. + + Returns a dict: + {'headername': Name of header file, + 'headerdata': Data for the .h header file, + 'sourcename': Name of the source code file, + 'sourcedata': Data for the .c source code file + } + ''' + toplevel_options = nanopb_pb2.NanoPBOptions() + for s in options.settings: + text_format.Merge(s, toplevel_options) + + if not fdesc: + data = open(filename, 'rb').read() + fdesc = descriptor.FileDescriptorSet.FromString(data).file[0] + + # Check if there is a separate .options file + try: + optfilename = options.options_file % os.path.splitext(filename)[0] + except TypeError: + # No %s specified, use the filename as-is + optfilename = options.options_file + + if options.verbose: + print 'Reading options from ' + optfilename + + if os.path.isfile(optfilename): + Globals.separate_options = read_options_file(open(optfilename, "rU")) + else: + Globals.separate_options = [] + + # Parse the file + file_options = get_nanopb_suboptions(fdesc, toplevel_options, Names([filename])) + enums, messages, extensions = parse_file(fdesc, file_options) + + # Decide the file names + noext = os.path.splitext(filename)[0] + headername = noext + '.' + options.extension + '.h' + sourcename = noext + '.' + options.extension + '.c' + headerbasename = os.path.basename(headername) + + # List of .proto files that should not be included in the C header file + # even if they are mentioned in the source .proto. + excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude + dependencies = [d for d in fdesc.dependency if d not in excludes] + + headerdata = ''.join(generate_header(dependencies, headerbasename, enums, + messages, extensions, options)) + + sourcedata = ''.join(generate_source(headerbasename, enums, + messages, extensions, options)) + + return {'headername': headername, 'headerdata': headerdata, + 'sourcename': sourcename, 'sourcedata': sourcedata} + +def main_cli(): + '''Main function when invoked directly from the command line.''' + + options, filenames = optparser.parse_args() if not filenames: optparser.print_help() - return False + sys.exit(1) if options.quiet: options.verbose = False Globals.verbose_options = options.verbose - toplevel_options = nanopb_pb2.NanoPBOptions() - for s in options.settings: - text_format.Merge(s, toplevel_options) - for filename in filenames: - data = open(filename, 'rb').read() - fdesc = descriptor.FileDescriptorSet.FromString(data) - - # Check if any separate options are specified - try: - optfilename = options.options_file % os.path.splitext(filename)[0] - except TypeError: - # No %s specified, use the filename as-is - optfilename = options.options_file - - if options.verbose: - print 'Reading options from ' + optfilename - - if os.path.isfile(optfilename): - Globals.separate_options = read_options_file(open(optfilename, "rU")) - else: - Globals.separate_options = [] - - # Parse the file - file_options = get_nanopb_suboptions(fdesc.file[0], toplevel_options, Names([filename])) - enums, messages, extensions = parse_file(fdesc.file[0], file_options) - - noext = os.path.splitext(filename)[0] - headername = noext + '.' + options.extension + '.h' - sourcename = noext + '.' + options.extension + '.c' - headerbasename = os.path.basename(headername) + results = process_file(filename, None, options) if not options.quiet: - print "Writing to " + headername + " and " + sourcename - - # List of .proto files that should not be included in the C header file - # even if they are mentioned in the source .proto. - excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude - dependencies = [d for d in fdesc.file[0].dependency if d not in excludes] - - header = open(headername, 'w') - for part in generate_header(dependencies, headerbasename, enums, - messages, extensions, options): - header.write(part) - - source = open(sourcename, 'w') - for part in generate_source(headerbasename, enums, messages, extensions): - source.write(part) + print "Writing to " + results['headername'] + " and " + results['sourcename'] + + open(results['headername'], 'w').write(results['headerdata']) + open(results['sourcename'], 'w').write(results['sourcedata']) - return True +def main_plugin(): + '''Main function when invoked as a protoc plugin.''' -if __name__ == '__main__': - options, filenames = optparser.parse_args() - status = process(filenames, options) + import plugin_pb2 + data = sys.stdin.read() + request = plugin_pb2.CodeGeneratorRequest.FromString(data) - if not status: - sys.exit(1) + import shlex + args = shlex.split(request.parameter) + options, dummy = optparser.parse_args(args) + + # We can't go printing stuff to stdout + Globals.verbose_options = False + options.verbose = False + options.quiet = True + + response = plugin_pb2.CodeGeneratorResponse() + for filename in request.file_to_generate: + for fdesc in request.proto_file: + if fdesc.name == filename: + results = process_file(filename, fdesc, options) + + f = response.file.add() + f.name = results['headername'] + f.content = results['headerdata'] + + f = response.file.add() + f.name = results['sourcename'] + f.content = results['sourcedata'] + + sys.stdout.write(response.SerializeToString()) + +if __name__ == '__main__': + # Check if we are running as a plugin under protoc + if 'protoc-gen-' in sys.argv[0]: + main_plugin() + else: + main_cli() + diff --git a/generator/plugin.proto b/generator/plugin.proto new file mode 100644 index 00000000..651ed10c --- /dev/null +++ b/generator/plugin.proto @@ -0,0 +1,145 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// +// WARNING: The plugin interface is currently EXPERIMENTAL and is subject to +// change. +// +// protoc (aka the Protocol Compiler) can be extended via plugins. A plugin is +// just a program that reads a CodeGeneratorRequest from stdin and writes a +// CodeGeneratorResponse to stdout. +// +// Plugins written using C++ can use google/protobuf/compiler/plugin.h instead +// of dealing with the raw protocol defined here. +// +// A plugin executable needs only to be placed somewhere in the path. The +// plugin should be named "protoc-gen-$NAME", and will then be used when the +// flag "--${NAME}_out" is passed to protoc. + +package google.protobuf.compiler; + +import "google/protobuf/descriptor.proto"; + +// An encoded CodeGeneratorRequest is written to the plugin's stdin. +message CodeGeneratorRequest { + // The .proto files that were explicitly listed on the command-line. The + // code generator should generate code only for these files. Each file's + // descriptor will be included in proto_file, below. + repeated string file_to_generate = 1; + + // The generator parameter passed on the command-line. + optional string parameter = 2; + + // FileDescriptorProtos for all files in files_to_generate and everything + // they import. The files will appear in topological order, so each file + // appears before any file that imports it. + // + // protoc guarantees that all proto_files will be written after + // the fields above, even though this is not technically guaranteed by the + // protobuf wire format. This theoretically could allow a plugin to stream + // in the FileDescriptorProtos and handle them one by one rather than read + // the entire set into memory at once. However, as of this writing, this + // is not similarly optimized on protoc's end -- it will store all fields in + // memory at once before sending them to the plugin. + repeated FileDescriptorProto proto_file = 15; +} + +// The plugin writes an encoded CodeGeneratorResponse to stdout. +message CodeGeneratorResponse { + // Error message. If non-empty, code generation failed. The plugin process + // should exit with status code zero even if it reports an error in this way. + // + // This should be used to indicate errors in .proto files which prevent the + // code generator from generating correct code. Errors which indicate a + // problem in protoc itself -- such as the input CodeGeneratorRequest being + // unparseable -- should be reported by writing a message to stderr and + // exiting with a non-zero status code. + optional string error = 1; + + // Represents a single generated file. + message File { + // The file name, relative to the output directory. The name must not + // contain "." or ".." components and must be relative, not be absolute (so, + // the file cannot lie outside the output directory). "/" must be used as + // the path separator, not "\". + // + // If the name is omitted, the content will be appended to the previous + // file. This allows the generator to break large files into small chunks, + // and allows the generated text to be streamed back to protoc so that large + // files need not reside completely in memory at one time. Note that as of + // this writing protoc does not optimize for this -- it will read the entire + // CodeGeneratorResponse before writing files to disk. + optional string name = 1; + + // If non-empty, indicates that the named file should already exist, and the + // content here is to be inserted into that file at a defined insertion + // point. This feature allows a code generator to extend the output + // produced by another code generator. The original generator may provide + // insertion points by placing special annotations in the file that look + // like: + // @@protoc_insertion_point(NAME) + // The annotation can have arbitrary text before and after it on the line, + // which allows it to be placed in a comment. NAME should be replaced with + // an identifier naming the point -- this is what other generators will use + // as the insertion_point. Code inserted at this point will be placed + // immediately above the line containing the insertion point (thus multiple + // insertions to the same point will come out in the order they were added). + // The double-@ is intended to make it unlikely that the generated code + // could contain things that look like insertion points by accident. + // + // For example, the C++ code generator places the following line in the + // .pb.h files that it generates: + // // @@protoc_insertion_point(namespace_scope) + // This line appears within the scope of the file's package namespace, but + // outside of any particular class. Another plugin can then specify the + // insertion_point "namespace_scope" to generate additional classes or + // other declarations that should be placed in this scope. + // + // Note that if the line containing the insertion point begins with + // whitespace, the same whitespace will be added to every line of the + // inserted text. This is useful for languages like Python, where + // indentation matters. In these languages, the insertion point comment + // should be indented the same amount as any inserted code will need to be + // in order to work correctly in that context. + // + // The code generator that generates the initial file and the one which + // inserts into it must both run as part of a single invocation of protoc. + // Code generators are executed in the order in which they appear on the + // command line. + // + // If |insertion_point| is present, |name| must also be present. + optional string insertion_point = 2; + + // The file contents. + optional string content = 15; + } + repeated File file = 15; +} diff --git a/generator/plugin_pb2.py b/generator/plugin_pb2.py new file mode 100644 index 00000000..2d8c09d4 --- /dev/null +++ b/generator/plugin_pb2.py @@ -0,0 +1,161 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! + +from google.protobuf import descriptor +from google.protobuf import message +from google.protobuf import reflection +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + + +import google.protobuf.descriptor_pb2 + +DESCRIPTOR = descriptor.FileDescriptor( + name='plugin.proto', + package='google.protobuf.compiler', + serialized_pb='\n\x0cplugin.proto\x12\x18google.protobuf.compiler\x1a google/protobuf/descriptor.proto\"}\n\x14\x43odeGeneratorRequest\x12\x18\n\x10\x66ile_to_generate\x18\x01 \x03(\t\x12\x11\n\tparameter\x18\x02 \x01(\t\x12\x38\n\nproto_file\x18\x0f \x03(\x0b\x32$.google.protobuf.FileDescriptorProto\"\xaa\x01\n\x15\x43odeGeneratorResponse\x12\r\n\x05\x65rror\x18\x01 \x01(\t\x12\x42\n\x04\x66ile\x18\x0f \x03(\x0b\x32\x34.google.protobuf.compiler.CodeGeneratorResponse.File\x1a>\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x17\n\x0finsertion_point\x18\x02 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x0f \x01(\t') + + + + +_CODEGENERATORREQUEST = descriptor.Descriptor( + name='CodeGeneratorRequest', + full_name='google.protobuf.compiler.CodeGeneratorRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='file_to_generate', full_name='google.protobuf.compiler.CodeGeneratorRequest.file_to_generate', index=0, + number=1, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='parameter', full_name='google.protobuf.compiler.CodeGeneratorRequest.parameter', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='proto_file', full_name='google.protobuf.compiler.CodeGeneratorRequest.proto_file', index=2, + number=15, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=76, + serialized_end=201, +) + + +_CODEGENERATORRESPONSE_FILE = descriptor.Descriptor( + name='File', + full_name='google.protobuf.compiler.CodeGeneratorResponse.File', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='name', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='insertion_point', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.insertion_point', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='content', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.content', index=2, + number=15, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=312, + serialized_end=374, +) + +_CODEGENERATORRESPONSE = descriptor.Descriptor( + name='CodeGeneratorResponse', + full_name='google.protobuf.compiler.CodeGeneratorResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + descriptor.FieldDescriptor( + name='error', full_name='google.protobuf.compiler.CodeGeneratorResponse.error', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=unicode("", "utf-8"), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + descriptor.FieldDescriptor( + name='file', full_name='google.protobuf.compiler.CodeGeneratorResponse.file', index=1, + number=15, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_CODEGENERATORRESPONSE_FILE, ], + enum_types=[ + ], + options=None, + is_extendable=False, + extension_ranges=[], + serialized_start=204, + serialized_end=374, +) + +_CODEGENERATORREQUEST.fields_by_name['proto_file'].message_type = google.protobuf.descriptor_pb2._FILEDESCRIPTORPROTO +_CODEGENERATORRESPONSE_FILE.containing_type = _CODEGENERATORRESPONSE; +_CODEGENERATORRESPONSE.fields_by_name['file'].message_type = _CODEGENERATORRESPONSE_FILE +DESCRIPTOR.message_types_by_name['CodeGeneratorRequest'] = _CODEGENERATORREQUEST +DESCRIPTOR.message_types_by_name['CodeGeneratorResponse'] = _CODEGENERATORRESPONSE + +class CodeGeneratorRequest(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + DESCRIPTOR = _CODEGENERATORREQUEST + + # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorRequest) + +class CodeGeneratorResponse(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + + class File(message.Message): + __metaclass__ = reflection.GeneratedProtocolMessageType + DESCRIPTOR = _CODEGENERATORRESPONSE_FILE + + # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse.File) + DESCRIPTOR = _CODEGENERATORRESPONSE + + # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse) + +# @@protoc_insertion_point(module_scope) -- cgit 1.2.3-korg From 5b9ad17dc2014d7506a7dde92281d8c36a1433e4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 9 Sep 2013 10:53:04 +0300 Subject: Move the declarations of _pb_ostream_t and _pb_istream_t before first use. Otherwise Microsoft Visual C++ threats them as C++ classes instead of plain structs, forbidding use in C linkage functions. Thanks to Markus Schwarzenberg for the patch. Update issue 84 Status: Started --- pb_decode.h | 66 +++++++++++++++++++++++++++++------------------------------ pb_encode.h | 68 ++++++++++++++++++++++++++++++------------------------------- 2 files changed, 67 insertions(+), 67 deletions(-) diff --git a/pb_decode.h b/pb_decode.h index 3da3f763..98a64cc0 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -12,6 +12,39 @@ extern "C" { #endif +/* Structure for defining custom input streams. You will need to provide + * a callback function to read the bytes from your storage, which can be + * for example a file or a network socket. + * + * The callback must conform to these rules: + * + * 1) Return false on IO errors. This will cause decoding to abort. + * 2) You can use state to store your own data (e.g. buffer pointer), + * and rely on pb_read to verify that no-body reads past bytes_left. + * 3) Your callback may be used with substreams, in which case bytes_left + * is different than from the main stream. Don't use bytes_left to compute + * any pointers. + */ +struct _pb_istream_t +{ +#ifdef PB_BUFFER_ONLY + /* Callback pointer is not used in buffer-only configuration. + * Having an int pointer here allows binary compatibility but + * gives an error if someone tries to assign callback function. + */ + int *callback; +#else + bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count); +#endif + + void *state; /* Free field for use by callback implementation */ + size_t bytes_left; + +#ifndef PB_NO_ERRMSG + const char *errmsg; +#endif +}; + /*************************** * Main decoding functions * ***************************/ @@ -66,39 +99,6 @@ pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); */ bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); -/* Structure for defining custom input streams. You will need to provide - * a callback function to read the bytes from your storage, which can be - * for example a file or a network socket. - * - * The callback must conform to these rules: - * - * 1) Return false on IO errors. This will cause decoding to abort. - * 2) You can use state to store your own data (e.g. buffer pointer), - * and rely on pb_read to verify that no-body reads past bytes_left. - * 3) Your callback may be used with substreams, in which case bytes_left - * is different than from the main stream. Don't use bytes_left to compute - * any pointers. - */ -struct _pb_istream_t -{ -#ifdef PB_BUFFER_ONLY - /* Callback pointer is not used in buffer-only configuration. - * Having an int pointer here allows binary compatibility but - * gives an error if someone tries to assign callback function. - */ - int *callback; -#else - bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count); -#endif - - void *state; /* Free field for use by callback implementation */ - size_t bytes_left; - -#ifndef PB_NO_ERRMSG - const char *errmsg; -#endif -}; - /************************************************ * Helper functions for writing field callbacks * diff --git a/pb_encode.h b/pb_encode.h index 04bdabee..3009820b 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -12,6 +12,40 @@ extern "C" { #endif +/* Structure for defining custom output streams. You will need to provide + * a callback function to write the bytes to your storage, which can be + * for example a file or a network socket. + * + * The callback must conform to these rules: + * + * 1) Return false on IO errors. This will cause encoding to abort. + * 2) You can use state to store your own data (e.g. buffer pointer). + * 3) pb_write will update bytes_written after your callback runs. + * 4) Substreams will modify max_size and bytes_written. Don't use them + * to calculate any pointers. + */ +struct _pb_ostream_t +{ +#ifdef PB_BUFFER_ONLY + /* Callback pointer is not used in buffer-only configuration. + * Having an int pointer here allows binary compatibility but + * gives an error if someone tries to assign callback function. + * Also, NULL pointer marks a 'sizing stream' that does not + * write anything. + */ + int *callback; +#else + bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); +#endif + void *state; /* Free field for use by callback implementation. */ + size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ + size_t bytes_written; /* Number of bytes written so far. */ + +#ifndef PB_NO_ERRMSG + const char *errmsg; +#endif +}; + /*************************** * Main encoding functions * ***************************/ @@ -70,40 +104,6 @@ pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize); */ bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); -/* Structure for defining custom output streams. You will need to provide - * a callback function to write the bytes to your storage, which can be - * for example a file or a network socket. - * - * The callback must conform to these rules: - * - * 1) Return false on IO errors. This will cause encoding to abort. - * 2) You can use state to store your own data (e.g. buffer pointer). - * 3) pb_write will update bytes_written after your callback runs. - * 4) Substreams will modify max_size and bytes_written. Don't use them - * to calculate any pointers. - */ -struct _pb_ostream_t -{ -#ifdef PB_BUFFER_ONLY - /* Callback pointer is not used in buffer-only configuration. - * Having an int pointer here allows binary compatibility but - * gives an error if someone tries to assign callback function. - * Also, NULL pointer marks a 'sizing stream' that does not - * write anything. - */ - int *callback; -#else - bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); -#endif - void *state; /* Free field for use by callback implementation. */ - size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ - size_t bytes_written; /* Number of bytes written so far. */ - -#ifndef PB_NO_ERRMSG - const char *errmsg; -#endif -}; - /************************************************ * Helper functions for writing field callbacks * -- cgit 1.2.3-korg From 4dccf28ba9c212b232147fd3823554d04b30c392 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 10 Sep 2013 11:34:57 +0300 Subject: Convert more test cases to scons --- tests/SConstruct | 72 +++----------------------------- tests/alltypes/SConscript | 12 ++++++ tests/cxx_main_program/SConscript | 20 +++++++++ tests/site_scons/site_init.py | 86 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 66 deletions(-) create mode 100644 tests/alltypes/SConscript create mode 100644 tests/cxx_main_program/SConscript create mode 100644 tests/site_scons/site_init.py diff --git a/tests/SConstruct b/tests/SConstruct index 0ec1e54e..b2d524cc 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -1,77 +1,17 @@ env = DefaultEnvironment() -env.Append(CPPPATH = ["#../", "#common"]) +# Add the builders defined in site_init.py +add_nanopb_builders(env) -# Build command for building .pb from .proto using protoc -def proto_actions(source, target, env, for_signature): - dirs = ' '.join(['-I' + env.GetBuildPath(d) for d in env['PROTOCPATH']]) - return '$PROTOC $PROTOCFLAGS %s -o%s %s' % (dirs, target[0], source[0]) +# Path to the files shared by tests, and to the nanopb core. +env.Append(CPPPATH = ["#../", "#common"]) -proto_file_builder = Builder(generator = proto_actions, - suffix = '.pb', - src_suffix = '.proto') -env.Append(BUILDERS = {'Proto': proto_file_builder}) -env.SetDefault(PROTOC = 'protoc') +# Path for finding nanopb.proto +env.Append(PROTOCPATH = ['#../generator', '/usr/include', '.']) # Define the include path to find nanopb.proto env.Append(PROTOCPATH = ['#../generator', '/usr/include', '.']) -# Build command for running nanopb generator -import os.path -def nanopb_targets(target, source, env): - basename = os.path.splitext(str(source[0]))[0] - target.append(basename + '.pb.h') - return target, source - -nanopb_file_builder = Builder(action = '$NANOPB_GENERATOR $NANOPB_FLAGS $SOURCE', - suffix = '.pb.c', - src_suffix = '.pb', - emitter = nanopb_targets) -env.Append(BUILDERS = {'Nanopb': nanopb_file_builder}) -env.SetDefault(NANOPB_GENERATOR = 'python ' + env.GetBuildPath("#../generator/nanopb_generator.py")) - -# Combined method to run both protoc and nanopb generator -def run_protoc_and_nanopb(env, source): - b1 = env.Proto(source) - b2 = env.Nanopb(source) - return b1 + b2 -env.AddMethod(run_protoc_and_nanopb, "NanopbProto") - -# Build command that runs a test program and saves the output -def run_test_actions(source, target, env, for_signature): - cmd = str(source[0]) # Name of binary - if len(source) > 1: - cmd += ' <' + str(source[1]) # Input file - cmd += ' >' + str(target[0]) - return cmd - -run_test_builder = Builder(generator = run_test_actions, - suffix = '.output') -env.Append(BUILDERS = {'RunTest': run_test_builder}) - -# Build command that decodes a message using protoc -def decode_actions(source, target, env, for_signature): - dirs = ' '.join(['-I' + env.GetBuildPath(d) for d in env['PROTOCPATH']]) - return '$PROTOC $PROTOCFLAGS %s --decode=%s %s <%s >%s' % (dirs, env['MESSAGE'], source[1], source[0], target[0]) - -decode_builder = Builder(generator = decode_actions, - suffix = '.decoded') -env.Append(BUILDERS = {'Decode': decode_builder}) - -# Build command that asserts that two files be equal -def compare_files(target, source, env): - data1 = open(str(source[0]), 'rb').read() - data2 = open(str(source[1]), 'rb').read() - if data1 == data2: -# open(str(target[0]), 'w').write('OK') - return 0 - else: - return "Test failed: %s and %s differ!" % (source[0], source[1]) - -compare_builder = Builder(action = compare_files, - suffix = '.equal') -env.Append(BUILDERS = {'Compare': compare_builder}) - # Now include the SConscript files from all subdirectories SConscript(Glob('*/SConscript'), exports = 'env') diff --git a/tests/alltypes/SConscript b/tests/alltypes/SConscript new file mode 100644 index 00000000..3c8adc48 --- /dev/null +++ b/tests/alltypes/SConscript @@ -0,0 +1,12 @@ +# Build and run a test that encodes and decodes a message that contains +# all of the Protocol Buffers data types. + +Import("env") + +env.NanopbProto("alltypes") +env.Program(["encode_alltypes.c", "alltypes.pb.c", "#common/pb_encode.o"]) +env.Program(["decode_alltypes.c", "alltypes.pb.c", "#common/pb_decode.o"]) + +env.RunTest("encode_alltypes") +env.RunTest(["decode_alltypes", "encode_alltypes.output"]) + diff --git a/tests/cxx_main_program/SConscript b/tests/cxx_main_program/SConscript new file mode 100644 index 00000000..b5482319 --- /dev/null +++ b/tests/cxx_main_program/SConscript @@ -0,0 +1,20 @@ +# Run the alltypes test case, but compile it as C++ instead. +# In fact, compile the entire nanopb using C++ compiler. + +Import("env") + +# Copy the files to .cxx extension in order to force C++ build. +c = Copy("$TARGET", "$SOURCE") +Command("pb_encode.cxx", "#../pb_encode.c", c) +Command("pb_decode.cxx", "#../pb_decode.c", c) +Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c) +Command("alltypes.pb.cxx", "#alltypes/alltypes.pb.c", c) +Command("encode_alltypes.cxx", "#alltypes/encode_alltypes.c", c) +Command("decode_alltypes.cxx", "#alltypes/decode_alltypes.c", c) + +# Now build and run the test normally. +env.Program(["encode_alltypes.cxx", "alltypes.pb.cxx", "pb_encode.cxx"]) +env.Program(["decode_alltypes.cxx", "alltypes.pb.cxx", "pb_decode.cxx"]) + +env.RunTest("encode_alltypes") +env.RunTest(["decode_alltypes", "encode_alltypes.output"]) diff --git a/tests/site_scons/site_init.py b/tests/site_scons/site_init.py new file mode 100644 index 00000000..1383067d --- /dev/null +++ b/tests/site_scons/site_init.py @@ -0,0 +1,86 @@ +import subprocess +import sys + +def add_nanopb_builders(env): + '''Add the necessary builder commands for nanopb tests.''' + + # Build command for building .pb from .proto using protoc + def proto_actions(source, target, env, for_signature): + dirs = ' '.join(['-I' + env.GetBuildPath(d) for d in env['PROTOCPATH']]) + return '$PROTOC $PROTOCFLAGS %s -o%s %s' % (dirs, target[0], source[0]) + + proto_file_builder = Builder(generator = proto_actions, + suffix = '.pb', + src_suffix = '.proto') + env.Append(BUILDERS = {'Proto': proto_file_builder}) + env.SetDefault(PROTOC = 'protoc') + + # Build command for running nanopb generator + import os.path + def nanopb_targets(target, source, env): + basename = os.path.splitext(str(source[0]))[0] + target.append(basename + '.pb.h') + return target, source + + nanopb_file_builder = Builder(action = '$NANOPB_GENERATOR $NANOPB_FLAGS $SOURCE', + suffix = '.pb.c', + src_suffix = '.pb', + emitter = nanopb_targets) + env.Append(BUILDERS = {'Nanopb': nanopb_file_builder}) + env.SetDefault(NANOPB_GENERATOR = 'python ' + env.GetBuildPath("#../generator/nanopb_generator.py")) + env.SetDefault(NANOPB_FLAGS = '-q') + + # Combined method to run both protoc and nanopb generator + def run_protoc_and_nanopb(env, source): + b1 = env.Proto(source) + b2 = env.Nanopb(source) + return b1 + b2 + env.AddMethod(run_protoc_and_nanopb, "NanopbProto") + + # Build command that runs a test program and saves the output + def run_test(target, source, env): + if len(source) > 1: + infile = open(str(source[1])) + else: + infile = None + + pipe = subprocess.Popen(str(source[0]), + stdin = infile, + stdout = open(str(target[0]), 'w'), + stderr = sys.stderr) + result = pipe.wait() + if result == 0: + print '\033[92m[ OK ]\033[0m Ran ' + str(source[0]) + else: + print '\033[91m[FAIL]\033[0m Program ' + str(source[0]) + ' returned ' + str(result) + return result + + run_test_builder = Builder(action = run_test, + suffix = '.output') + env.Append(BUILDERS = {'RunTest': run_test_builder}) + + # Build command that decodes a message using protoc + def decode_actions(source, target, env, for_signature): + dirs = ' '.join(['-I' + env.GetBuildPath(d) for d in env['PROTOCPATH']]) + return '$PROTOC $PROTOCFLAGS %s --decode=%s %s <%s >%s' % (dirs, env['MESSAGE'], source[1], source[0], target[0]) + + decode_builder = Builder(generator = decode_actions, + suffix = '.decoded') + env.Append(BUILDERS = {'Decode': decode_builder}) + + # Build command that asserts that two files be equal + def compare_files(target, source, env): + data1 = open(str(source[0]), 'rb').read() + data2 = open(str(source[1]), 'rb').read() + if data1 == data2: + print '\033[92m[ OK ]\033[0m Files equal: ' + str(source[0]) + ' and ' + str(source[1]) + return 0 + else: + print '\033[91m[FAIL]\033[0m Files differ: ' + str(source[0]) + ' and ' + str(source[1]) + return 1 + + compare_builder = Builder(action = compare_files, + suffix = '.equal') + env.Append(BUILDERS = {'Compare': compare_builder}) + + -- cgit 1.2.3-korg From e681dd0d75a4b6a7974cc898477f3a138f7872c2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 10 Sep 2013 12:39:39 +0300 Subject: Add an example pb_syshdr.h file for platforms without C99. This allows building the tests easily on Visual C++ in C mode. Also add checks to pb.h that the defined integer types are of the proper sizes. This may prevent some difficult to debug problems later.. --- compat/pb_syshdr.h | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ pb.h | 11 +++++++++ tests/SConstruct | 7 ++++++ 3 files changed, 89 insertions(+) create mode 100644 compat/pb_syshdr.h diff --git a/compat/pb_syshdr.h b/compat/pb_syshdr.h new file mode 100644 index 00000000..c170c8de --- /dev/null +++ b/compat/pb_syshdr.h @@ -0,0 +1,71 @@ +/* This is an example of a header file for platforms/compilers that do + * not come with stdint.h/stddef.h/stdbool.h/string.h. To use it, define + * PB_SYSTEM_HEADER as "pb_syshdr.h", including the quotes, and add the + * compat folder to your include path. + * + * It is very likely that you will need to customize this file to suit + * your platform. For any compiler that supports C99, this file should + * not be necessary. + */ + +#ifndef _PB_SYSHDR_H_ +#define _PB_SYSHDR_H_ + +/* stdint.h subset */ +/* You will need to modify these to match the word size of your platform. */ +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef signed short int16_t; +typedef unsigned short uint16_t; +typedef signed int int32_t; +typedef unsigned int uint32_t; +typedef signed long long int64_t; +typedef unsigned long long uint64_t; + +/* stddef.h subset */ +typedef uint32_t size_t; +#define offsetof(st, m) ((size_t)(&((st *)0)->m)) + +#ifndef NULL +#define NULL 0 +#endif + +/* stdbool.h subset */ +typedef int bool; +#define false 0 +#define true 1 + +/* string.h subset */ +/* Implementations are from the Public Domain C Library (PDCLib). */ +static size_t strlen( const char * s ) +{ + size_t rc = 0; + while ( s[rc] ) + { + ++rc; + } + return rc; +} + +static void * memcpy( void *s1, const void *s2, size_t n ) +{ + char * dest = (char *) s1; + const char * src = (const char *) s2; + while ( n-- ) + { + *dest++ = *src++; + } + return s1; +} + +static void * memset( void * s, int c, size_t n ) +{ + unsigned char * p = (unsigned char *) s; + while ( n-- ) + { + *p++ = (unsigned char) c; + } + return s; +} + +#endif diff --git a/pb.h b/pb.h index e3e68ce5..c63b3514 100644 --- a/pb.h +++ b/pb.h @@ -215,6 +215,17 @@ struct _pb_field_t { } pb_packed; PB_PACKED_STRUCT_END +/* Make sure that the standard integer types are of the expected sizes. + * All kinds of things may break otherwise.. atleast all fixed* types. */ +STATIC_ASSERT(sizeof(int8_t) == 1, INT8_T_WRONG_SIZE) +STATIC_ASSERT(sizeof(uint8_t) == 1, UINT8_T_WRONG_SIZE) +STATIC_ASSERT(sizeof(int16_t) == 2, INT16_T_WRONG_SIZE) +STATIC_ASSERT(sizeof(uint16_t) == 2, UINT16_T_WRONG_SIZE) +STATIC_ASSERT(sizeof(int32_t) == 4, INT32_T_WRONG_SIZE) +STATIC_ASSERT(sizeof(uint32_t) == 4, UINT32_T_WRONG_SIZE) +STATIC_ASSERT(sizeof(int64_t) == 8, INT64_T_WRONG_SIZE) +STATIC_ASSERT(sizeof(uint64_t) == 8, UINT64_T_WRONG_SIZE) + /* This structure is used for 'bytes' arrays. * It has the number of bytes in the beginning, and after that an array. * Note that actual structs used will have a different length of bytes array. diff --git a/tests/SConstruct b/tests/SConstruct index b2d524cc..a690aff0 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -12,6 +12,13 @@ env.Append(PROTOCPATH = ['#../generator', '/usr/include', '.']) # Define the include path to find nanopb.proto env.Append(PROTOCPATH = ['#../generator', '/usr/include', '.']) +# If the platform doesn't support C99, use our own header file instead. +conf = Configure(env) +if not conf.CheckCHeader('stdbool.h'): + conf.env.Append(CPPDEFINES = {'PB_SYSTEM_HEADER': '\\"pb_syshdr.h\\"'}) + conf.env.Append(CPPPATH = "#../compat") +env = conf.Finish() + # Now include the SConscript files from all subdirectories SConscript(Glob('*/SConscript'), exports = 'env') -- cgit 1.2.3-korg From f04ab838abd90fb70f7d6ef77fdacdf07f09ba4d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 10 Sep 2013 17:44:32 +0300 Subject: Build fixes for Windows/Visual C++ --- compat/pb_syshdr.h | 23 ++++++++++++++ tests/SConstruct | 52 +++++++++++++++++++++++++------ tests/alltypes/SConscript | 8 ++--- tests/alltypes/encode_alltypes.c | 26 +++++++++------- tests/basic_buffer/SConscript | 8 ++--- tests/basic_buffer/decode_buffer.c | 11 +++++-- tests/basic_buffer/encode_buffer.c | 11 +++++-- tests/common/test_helpers.h | 17 ++++++++++ tests/cxx_main_program/SConscript | 8 ++--- tests/decode_unittests/SConscript | 4 +-- tests/decode_unittests/decode_unittests.c | 2 +- tests/encode_unittests/SConscript | 4 +-- tests/site_scons/site_init.py | 16 +++++++--- 13 files changed, 141 insertions(+), 49 deletions(-) create mode 100644 tests/common/test_helpers.h diff --git a/compat/pb_syshdr.h b/compat/pb_syshdr.h index c170c8de..b69a1671 100644 --- a/compat/pb_syshdr.h +++ b/compat/pb_syshdr.h @@ -12,6 +12,9 @@ #define _PB_SYSHDR_H_ /* stdint.h subset */ +#ifdef HAVE_STDINT_H +#include +#else /* You will need to modify these to match the word size of your platform. */ typedef signed char int8_t; typedef unsigned char uint8_t; @@ -21,8 +24,13 @@ typedef signed int int32_t; typedef unsigned int uint32_t; typedef signed long long int64_t; typedef unsigned long long uint64_t; +#endif /* stddef.h subset */ +#ifdef HAVE_STDDEF_H +#include +#else + typedef uint32_t size_t; #define offsetof(st, m) ((size_t)(&((st *)0)->m)) @@ -30,12 +38,26 @@ typedef uint32_t size_t; #define NULL 0 #endif +#endif + /* stdbool.h subset */ +#ifdef HAVE_STDBOOL_H +#include +#else + +#ifndef __cplusplus typedef int bool; #define false 0 #define true 1 +#endif + +#endif /* string.h subset */ +#ifdef HAVE_STRING_H +#include +#else + /* Implementations are from the Public Domain C Library (PDCLib). */ static size_t strlen( const char * s ) { @@ -67,5 +89,6 @@ static void * memset( void * s, int c, size_t n ) } return s; } +#endif #endif diff --git a/tests/SConstruct b/tests/SConstruct index a690aff0..8bbb31d1 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -1,4 +1,5 @@ -env = DefaultEnvironment() +import os +env = Environment(ENV = {'PATH': os.environ['PATH']}) # Add the builders defined in site_init.py add_nanopb_builders(env) @@ -7,18 +8,49 @@ add_nanopb_builders(env) env.Append(CPPPATH = ["#../", "#common"]) # Path for finding nanopb.proto -env.Append(PROTOCPATH = ['#../generator', '/usr/include', '.']) +env.Append(PROTOCPATH = '#../generator') -# Define the include path to find nanopb.proto -env.Append(PROTOCPATH = ['#../generator', '/usr/include', '.']) +# Check the compilation environment, unless we are just cleaning up. +if not env.GetOption('clean'): + conf = Configure(env) -# If the platform doesn't support C99, use our own header file instead. -conf = Configure(env) -if not conf.CheckCHeader('stdbool.h'): - conf.env.Append(CPPDEFINES = {'PB_SYSTEM_HEADER': '\\"pb_syshdr.h\\"'}) - conf.env.Append(CPPPATH = "#../compat") -env = conf.Finish() + # If the platform doesn't support C99, use our own header file instead. + stdbool = conf.CheckCHeader('stdbool.h') + stdint = conf.CheckCHeader('stdint.h') + stddef = conf.CheckCHeader('stddef.h') + string = conf.CheckCHeader('string.h') + if not stdbool or not stdint or not stddef or not string: + conf.env.Append(CPPDEFINES = {'PB_SYSTEM_HEADER': '\\"pb_syshdr.h\\"'}) + conf.env.Append(CPPPATH = "#../compat") + + if stdbool: conf.env.Append(CPPDEFINES = {'HAVE_STDBOOL_H': 1}) + if stdint: conf.env.Append(CPPDEFINES = {'HAVE_STDINT_H': 1}) + if stddef: conf.env.Append(CPPDEFINES = {'HAVE_STDDEF_H': 1}) + if string: conf.env.Append(CPPDEFINES = {'HAVE_STRING_H': 1}) + + # Check if we can use pkg-config to find protobuf include path + status, output = conf.TryAction('pkg-config protobuf --variable=includedir > $TARGET') + if status: + conf.env.Append(PROTOCPATH = output.strip()) + else: + conf.env.Append(PROTOCPATH = '/usr/include') + + # End the config stuff + env = conf.Finish() +# Initialize the CCFLAGS according to the compiler +if 'cl' in env['CC']: + # Microsoft Visual C++ + + # Debug info on, warning level 2 for tests, warnings as errors + env.Append(CCFLAGS = '/Zi /W2 /WX') + env.Append(LINKFLAGS = '/DEBUG') + + # PB_RETURN_ERROR triggers C4127 because of while(0) + env.Append(CCFLAGS = '/wd4127') + + + # Now include the SConscript files from all subdirectories SConscript(Glob('*/SConscript'), exports = 'env') diff --git a/tests/alltypes/SConscript b/tests/alltypes/SConscript index 3c8adc48..8aa45b6b 100644 --- a/tests/alltypes/SConscript +++ b/tests/alltypes/SConscript @@ -4,9 +4,9 @@ Import("env") env.NanopbProto("alltypes") -env.Program(["encode_alltypes.c", "alltypes.pb.c", "#common/pb_encode.o"]) -env.Program(["decode_alltypes.c", "alltypes.pb.c", "#common/pb_decode.o"]) +enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "#common/pb_encode.o"]) +dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "#common/pb_decode.o"]) -env.RunTest("encode_alltypes") -env.RunTest(["decode_alltypes", "encode_alltypes.output"]) +env.RunTest(enc) +env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/alltypes/encode_alltypes.c b/tests/alltypes/encode_alltypes.c index 982ad3c8..802e1574 100644 --- a/tests/alltypes/encode_alltypes.c +++ b/tests/alltypes/encode_alltypes.c @@ -113,18 +113,20 @@ int main(int argc, char **argv) alltypes.end = 1099; - uint8_t buffer[1024]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); - - /* Now encode it and check if we succeeded. */ - if (pb_encode(&stream, AllTypes_fields, &alltypes)) - { - fwrite(buffer, 1, stream.bytes_written, stdout); - return 0; /* Success */ - } - else { - fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); - return 1; /* Failure */ + uint8_t buffer[1024]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } } } diff --git a/tests/basic_buffer/SConscript b/tests/basic_buffer/SConscript index 5b85e132..349fb14f 100644 --- a/tests/basic_buffer/SConscript +++ b/tests/basic_buffer/SConscript @@ -2,11 +2,11 @@ Import("env") -env.Program(["encode_buffer.c", "#common/person.pb.c", "#common/pb_encode.o"]) -env.Program(["decode_buffer.c", "#common/person.pb.c", "#common/pb_decode.o"]) +enc = env.Program(["encode_buffer.c", "#common/person.pb.c", "#common/pb_encode.o"]) +dec = env.Program(["decode_buffer.c", "#common/person.pb.c", "#common/pb_decode.o"]) -env.RunTest("encode_buffer") -env.RunTest(["decode_buffer", "encode_buffer.output"]) +env.RunTest(enc) +env.RunTest([dec, "encode_buffer.output"]) env.Decode(["encode_buffer.output", "#common/person.proto"], MESSAGE = "Person") env.Compare(["decode_buffer.output", "encode_buffer.decoded"]) diff --git a/tests/basic_buffer/decode_buffer.c b/tests/basic_buffer/decode_buffer.c index 56bbd8f8..d231c916 100644 --- a/tests/basic_buffer/decode_buffer.c +++ b/tests/basic_buffer/decode_buffer.c @@ -9,6 +9,7 @@ #include #include #include "person.pb.h" +#include "test_helpers.h" /* This function is called once from main(), it handles the decoding and printing. */ @@ -59,9 +60,13 @@ bool print_person(pb_istream_t *stream) int main() { - /* Read the data into buffer */ uint8_t buffer[512]; - size_t count = fread(buffer, 1, sizeof(buffer), stdin); + pb_istream_t stream; + size_t count; + + /* Read the data into buffer */ + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); if (!feof(stdin)) { @@ -70,7 +75,7 @@ int main() } /* Construct a pb_istream_t for reading from the buffer */ - pb_istream_t stream = pb_istream_from_buffer(buffer, count); + stream = pb_istream_from_buffer(buffer, count); /* Decode and print out the stuff */ if (!print_person(&stream)) diff --git a/tests/basic_buffer/encode_buffer.c b/tests/basic_buffer/encode_buffer.c index 742c99f4..d3e4f6e6 100644 --- a/tests/basic_buffer/encode_buffer.c +++ b/tests/basic_buffer/encode_buffer.c @@ -6,9 +6,13 @@ #include #include #include "person.pb.h" +#include "test_helpers.h" int main() { + uint8_t buffer[512]; + pb_ostream_t stream; + /* Initialize the structure with constants */ Person person = {"Test Person 99", 99, true, "test@person.com", 3, {{"555-12345678", true, Person_PhoneType_MOBILE}, @@ -16,12 +20,13 @@ int main() {"1234-5678", true, Person_PhoneType_WORK}, }}; - uint8_t buffer[512]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); /* Now encode it and check if we succeeded. */ if (pb_encode(&stream, Person_fields, &person)) - { + { + /* Write the result data to stdout */ + SET_BINARY_MODE(stdout); fwrite(buffer, 1, stream.bytes_written, stdout); return 0; /* Success */ } diff --git a/tests/common/test_helpers.h b/tests/common/test_helpers.h new file mode 100644 index 00000000..f77760a5 --- /dev/null +++ b/tests/common/test_helpers.h @@ -0,0 +1,17 @@ +/* Compatibility helpers for the test programs. */ + +#ifndef _TEST_HELPERS_H_ +#define _TEST_HELPERS_H_ + +#ifdef _WIN32 +#include +#include +#define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) + +#else +#define SET_BINARY_MODE(file) + +#endif + + +#endif diff --git a/tests/cxx_main_program/SConscript b/tests/cxx_main_program/SConscript index b5482319..0d0a25ca 100644 --- a/tests/cxx_main_program/SConscript +++ b/tests/cxx_main_program/SConscript @@ -13,8 +13,8 @@ Command("encode_alltypes.cxx", "#alltypes/encode_alltypes.c", c) Command("decode_alltypes.cxx", "#alltypes/decode_alltypes.c", c) # Now build and run the test normally. -env.Program(["encode_alltypes.cxx", "alltypes.pb.cxx", "pb_encode.cxx"]) -env.Program(["decode_alltypes.cxx", "alltypes.pb.cxx", "pb_decode.cxx"]) +enc = env.Program(["encode_alltypes.cxx", "alltypes.pb.cxx", "pb_encode.cxx"]) +dec = env.Program(["decode_alltypes.cxx", "alltypes.pb.cxx", "pb_decode.cxx"]) -env.RunTest("encode_alltypes") -env.RunTest(["decode_alltypes", "encode_alltypes.output"]) +env.RunTest(enc) +env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/decode_unittests/SConscript b/tests/decode_unittests/SConscript index 860d7738..5e0f8407 100644 --- a/tests/decode_unittests/SConscript +++ b/tests/decode_unittests/SConscript @@ -1,4 +1,4 @@ Import('env') -env.Program(["decode_unittests.c", "#common/unittestproto.pb.c", "#common/pb_decode.o"]) -env.RunTest('decode_unittests') +p = env.Program(["decode_unittests.c", "#common/unittestproto.pb.c", "#common/pb_decode.o"]) +env.RunTest(p) diff --git a/tests/decode_unittests/decode_unittests.c b/tests/decode_unittests/decode_unittests.c index 6ad05f00..9c447a57 100644 --- a/tests/decode_unittests/decode_unittests.c +++ b/tests/decode_unittests/decode_unittests.c @@ -291,7 +291,7 @@ int main() { pb_istream_t s; - IntegerContainer dest = {}; + IntegerContainer dest = {{0}}; COMMENT("Testing pb_decode_delimited") TEST((s = S("\x09\x0A\x07\x0A\x05\x01\x02\x03\x04\x05"), diff --git a/tests/encode_unittests/SConscript b/tests/encode_unittests/SConscript index 0864a910..6a5ffcff 100644 --- a/tests/encode_unittests/SConscript +++ b/tests/encode_unittests/SConscript @@ -1,5 +1,5 @@ # Build and run the stand-alone unit tests for the nanopb encoder part. Import('env') -env.Program(["encode_unittests.c", "#common/unittestproto.pb.c", "#common/pb_encode.o"]) -env.RunTest('encode_unittests') +p = env.Program(["encode_unittests.c", "#common/unittestproto.pb.c", "#common/pb_encode.o"]) +env.RunTest(p) diff --git a/tests/site_scons/site_init.py b/tests/site_scons/site_init.py index 1383067d..b69db646 100644 --- a/tests/site_scons/site_init.py +++ b/tests/site_scons/site_init.py @@ -1,6 +1,13 @@ import subprocess import sys +try: + # Make terminal colors work on windows + import colorama + colorama.init() +except ImportError: + pass + def add_nanopb_builders(env): '''Add the necessary builder commands for nanopb tests.''' @@ -14,6 +21,7 @@ def add_nanopb_builders(env): src_suffix = '.proto') env.Append(BUILDERS = {'Proto': proto_file_builder}) env.SetDefault(PROTOC = 'protoc') + env.SetDefault(PROTOCPATH = ['.']) # Build command for running nanopb generator import os.path @@ -50,9 +58,9 @@ def add_nanopb_builders(env): stderr = sys.stderr) result = pipe.wait() if result == 0: - print '\033[92m[ OK ]\033[0m Ran ' + str(source[0]) + print '\033[32m[ OK ]\033[0m Ran ' + str(source[0]) else: - print '\033[91m[FAIL]\033[0m Program ' + str(source[0]) + ' returned ' + str(result) + print '\033[31m[FAIL]\033[0m Program ' + str(source[0]) + ' returned ' + str(result) return result run_test_builder = Builder(action = run_test, @@ -73,10 +81,10 @@ def add_nanopb_builders(env): data1 = open(str(source[0]), 'rb').read() data2 = open(str(source[1]), 'rb').read() if data1 == data2: - print '\033[92m[ OK ]\033[0m Files equal: ' + str(source[0]) + ' and ' + str(source[1]) + print '\033[32m[ OK ]\033[0m Files equal: ' + str(source[0]) + ' and ' + str(source[1]) return 0 else: - print '\033[91m[FAIL]\033[0m Files differ: ' + str(source[0]) + ' and ' + str(source[1]) + print '\033[31m[FAIL]\033[0m Files differ: ' + str(source[0]) + ' and ' + str(source[1]) return 1 compare_builder = Builder(action = compare_files, -- cgit 1.2.3-korg From 696a01bf140e91661eae77663de99c78c95dcc73 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 9 Sep 2013 10:53:04 +0300 Subject: Move the declarations of _pb_ostream_t and _pb_istream_t before first use. Otherwise Microsoft Visual C++ threats them as C++ classes instead of plain structs, forbidding use in C linkage functions. Thanks to Markus Schwarzenberg for the patch. Update issue 84 Status: Started --- pb_decode.h | 66 +++++++++++++++++++++++++++++------------------------------ pb_encode.h | 68 ++++++++++++++++++++++++++++++------------------------------- 2 files changed, 67 insertions(+), 67 deletions(-) diff --git a/pb_decode.h b/pb_decode.h index 3da3f763..98a64cc0 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -12,6 +12,39 @@ extern "C" { #endif +/* Structure for defining custom input streams. You will need to provide + * a callback function to read the bytes from your storage, which can be + * for example a file or a network socket. + * + * The callback must conform to these rules: + * + * 1) Return false on IO errors. This will cause decoding to abort. + * 2) You can use state to store your own data (e.g. buffer pointer), + * and rely on pb_read to verify that no-body reads past bytes_left. + * 3) Your callback may be used with substreams, in which case bytes_left + * is different than from the main stream. Don't use bytes_left to compute + * any pointers. + */ +struct _pb_istream_t +{ +#ifdef PB_BUFFER_ONLY + /* Callback pointer is not used in buffer-only configuration. + * Having an int pointer here allows binary compatibility but + * gives an error if someone tries to assign callback function. + */ + int *callback; +#else + bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count); +#endif + + void *state; /* Free field for use by callback implementation */ + size_t bytes_left; + +#ifndef PB_NO_ERRMSG + const char *errmsg; +#endif +}; + /*************************** * Main decoding functions * ***************************/ @@ -66,39 +99,6 @@ pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); */ bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); -/* Structure for defining custom input streams. You will need to provide - * a callback function to read the bytes from your storage, which can be - * for example a file or a network socket. - * - * The callback must conform to these rules: - * - * 1) Return false on IO errors. This will cause decoding to abort. - * 2) You can use state to store your own data (e.g. buffer pointer), - * and rely on pb_read to verify that no-body reads past bytes_left. - * 3) Your callback may be used with substreams, in which case bytes_left - * is different than from the main stream. Don't use bytes_left to compute - * any pointers. - */ -struct _pb_istream_t -{ -#ifdef PB_BUFFER_ONLY - /* Callback pointer is not used in buffer-only configuration. - * Having an int pointer here allows binary compatibility but - * gives an error if someone tries to assign callback function. - */ - int *callback; -#else - bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count); -#endif - - void *state; /* Free field for use by callback implementation */ - size_t bytes_left; - -#ifndef PB_NO_ERRMSG - const char *errmsg; -#endif -}; - /************************************************ * Helper functions for writing field callbacks * diff --git a/pb_encode.h b/pb_encode.h index 04bdabee..3009820b 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -12,6 +12,40 @@ extern "C" { #endif +/* Structure for defining custom output streams. You will need to provide + * a callback function to write the bytes to your storage, which can be + * for example a file or a network socket. + * + * The callback must conform to these rules: + * + * 1) Return false on IO errors. This will cause encoding to abort. + * 2) You can use state to store your own data (e.g. buffer pointer). + * 3) pb_write will update bytes_written after your callback runs. + * 4) Substreams will modify max_size and bytes_written. Don't use them + * to calculate any pointers. + */ +struct _pb_ostream_t +{ +#ifdef PB_BUFFER_ONLY + /* Callback pointer is not used in buffer-only configuration. + * Having an int pointer here allows binary compatibility but + * gives an error if someone tries to assign callback function. + * Also, NULL pointer marks a 'sizing stream' that does not + * write anything. + */ + int *callback; +#else + bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); +#endif + void *state; /* Free field for use by callback implementation. */ + size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ + size_t bytes_written; /* Number of bytes written so far. */ + +#ifndef PB_NO_ERRMSG + const char *errmsg; +#endif +}; + /*************************** * Main encoding functions * ***************************/ @@ -70,40 +104,6 @@ pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize); */ bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); -/* Structure for defining custom output streams. You will need to provide - * a callback function to write the bytes to your storage, which can be - * for example a file or a network socket. - * - * The callback must conform to these rules: - * - * 1) Return false on IO errors. This will cause encoding to abort. - * 2) You can use state to store your own data (e.g. buffer pointer). - * 3) pb_write will update bytes_written after your callback runs. - * 4) Substreams will modify max_size and bytes_written. Don't use them - * to calculate any pointers. - */ -struct _pb_ostream_t -{ -#ifdef PB_BUFFER_ONLY - /* Callback pointer is not used in buffer-only configuration. - * Having an int pointer here allows binary compatibility but - * gives an error if someone tries to assign callback function. - * Also, NULL pointer marks a 'sizing stream' that does not - * write anything. - */ - int *callback; -#else - bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); -#endif - void *state; /* Free field for use by callback implementation. */ - size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ - size_t bytes_written; /* Number of bytes written so far. */ - -#ifndef PB_NO_ERRMSG - const char *errmsg; -#endif -}; - /************************************************ * Helper functions for writing field callbacks * -- cgit 1.2.3-korg From 0bbcb7b367998063637ee35c5d13716492cbf6a3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 10 Sep 2013 20:54:29 +0300 Subject: Compiler options for GCC, clang and tcc --- tests/SConstruct | 67 +++++++++++++++++++++++++++++++++++++++++++------ tests/common/SConscript | 7 ++++-- 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/tests/SConstruct b/tests/SConstruct index 8bbb31d1..92cb0c62 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -1,5 +1,26 @@ +Help(''' +Type 'scons' to build and run all the available test cases. +It will automatically detect your platform and C compiler and +build appropriately. + +You can modify the behavious using following options: +CC Name of C compiler +CXX Name of C++ compiler +CCFLAGS Flags to pass to the C compiler +CXXFLAGS Flags to pass to the C++ compiler + +For example, for a clang build, use: +scons CC=clang CXX=clang++ +''') + import os -env = Environment(ENV = {'PATH': os.environ['PATH']}) +env = Environment(ENV = os.environ) + +# Allow overriding the compiler with scons CC=??? +if 'CC' in ARGUMENTS: env.Replace(CC = ARGUMENTS['CC']) +if 'CXX' in ARGUMENTS: env.Replace(CXX = ARGUMENTS['CXX']) +if 'CFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CFLAGS']) +if 'CXXFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CXXFLAGS']) # Add the builders defined in site_init.py add_nanopb_builders(env) @@ -34,22 +55,54 @@ if not env.GetOption('clean'): conf.env.Append(PROTOCPATH = output.strip()) else: conf.env.Append(PROTOCPATH = '/usr/include') - + + # Check if libmudflap is available (only with GCC) + if 'gcc' in env['CC']: + if conf.CheckLib('mudflap'): + conf.env.Append(CCFLAGS = '-fmudflap') + conf.env.Append(LINKFLAGS = '-lmudflap -fmudflap') + # End the config stuff env = conf.Finish() # Initialize the CCFLAGS according to the compiler -if 'cl' in env['CC']: +if 'gcc' in env['CC']: + # GNU Compiler Collection + + # Debug info, warnings as errors + env.Append(CFLAGS = '-ansi -g -O0 -Wall -Werror --coverage -fstack-protector-all') + env.Append(LINKFLAGS = '--coverage') + + # More strict checks on the nanopb core + env.Append(CORECFLAGS = '-pedantic -Wextra -Wcast-qual -Wlogical-op -Wconversion') +elif 'clang' in env['CC']: + # CLang + env.Append(CFLAGS = '-ansi -g -O0 -Wall -Werror') + env.Append(CORECFLAGS = '-pedantic -Wextra -Wcast-qual -Wconversion') +elif 'cl' in env['CC']: # Microsoft Visual C++ # Debug info on, warning level 2 for tests, warnings as errors - env.Append(CCFLAGS = '/Zi /W2 /WX') + env.Append(CFLAGS = '/Zi /W2 /WX') env.Append(LINKFLAGS = '/DEBUG') - # PB_RETURN_ERROR triggers C4127 because of while(0) - env.Append(CCFLAGS = '/wd4127') - + # More strict checks on the nanopb core + env.Append(CORECFLAGS = '/W4 /Za') + # PB_RETURN_ERROR triggers C4127 because of while(0) + env.Append(CFLAGS = '/wd4127') +elif 'tcc' in env['CC']: + # Tiny C Compiler + env.Append(CFLAGS = '-Wall -Werror -g') + +env.SetDefault(CORECFLAGS = '') + +if 'clang++' in env['CXX']: + env.Append(CXXFLAGS = '-g -O0 -Wall -Werror -Wextra -Wno-missing-field-initializers') +elif 'g++' in env['CXX']: + env.Append(CXXFLAGS = '-g -O0 -Wall -Werror -Wextra -Wno-missing-field-initializers') +elif 'cl' in env['CXX']: + env.Append(CXXFLAGS = '/Zi /W2 /WX') # Now include the SConscript files from all subdirectories SConscript(Glob('*/SConscript'), exports = 'env') diff --git a/tests/common/SConscript b/tests/common/SConscript index ef3cdcaa..8130c85a 100644 --- a/tests/common/SConscript +++ b/tests/common/SConscript @@ -9,6 +9,9 @@ env.NanopbProto("unittestproto") env.NanopbProto("person") # Binaries of the pb_decode.c and pb_encode.c -env.Object("pb_decode.o", "#../pb_decode.c") -env.Object("pb_encode.o", "#../pb_encode.c") +# These are built using more strict warning flags. +strict = env.Clone() +strict.Append(CFLAGS = strict['CORECFLAGS']) +strict.Object("pb_decode.o", "#../pb_decode.c") +strict.Object("pb_encode.o", "#../pb_encode.c") -- cgit 1.2.3-korg From e2e9980627810fe0ee2b8f119bcf651f0f318a8a Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 10 Sep 2013 22:34:54 +0300 Subject: Move the rest of the tests to scons --- tests/backwards_compatibility/SConscript | 11 +++++++++++ tests/backwards_compatibility/alltypes_legacy.c | 2 +- tests/backwards_compatibility/decode_legacy.c | 10 +++++----- tests/backwards_compatibility/encode_legacy.c | 8 ++++---- tests/basic_stream/SConscript | 12 ++++++++++++ tests/callbacks/SConscript | 14 ++++++++++++++ tests/extensions/SConscript | 16 ++++++++++++++++ tests/extra_fields/SConscript | 10 ++++++++++ tests/extra_fields/person_with_extra_field.expected | 14 ++++++++++++++ tests/missing_fields/SConscript | 8 ++++++++ tests/multiple_files/SConscript | 13 +++++++++++++ tests/multiple_files/test_multiple_files.c | 3 +-- tests/no_messages/SConscript | 7 +++++++ tests/options/SConscript | 9 +++++++++ tests/site_scons/site_init.py | 21 +++++++++++++++++++-- tests/special_characters/SConscript | 7 +++++++ tests/special_characters/funny-proto+name.proto | 0 17 files changed, 151 insertions(+), 14 deletions(-) create mode 100644 tests/backwards_compatibility/SConscript create mode 100644 tests/basic_stream/SConscript create mode 100644 tests/callbacks/SConscript create mode 100644 tests/extensions/SConscript create mode 100644 tests/extra_fields/SConscript create mode 100644 tests/extra_fields/person_with_extra_field.expected create mode 100644 tests/missing_fields/SConscript create mode 100644 tests/multiple_files/SConscript create mode 100644 tests/no_messages/SConscript create mode 100644 tests/options/SConscript create mode 100644 tests/special_characters/SConscript delete mode 100644 tests/special_characters/funny-proto+name.proto diff --git a/tests/backwards_compatibility/SConscript b/tests/backwards_compatibility/SConscript new file mode 100644 index 00000000..5fb978f3 --- /dev/null +++ b/tests/backwards_compatibility/SConscript @@ -0,0 +1,11 @@ +# Check that the old generated .pb.c/.pb.h files are still compatible with the +# current version of nanopb. + +Import("env") + +enc = env.Program(["encode_legacy.c", "alltypes_legacy.c", "#common/pb_encode.o"]) +dec = env.Program(["decode_legacy.c", "alltypes_legacy.c", "#common/pb_decode.o"]) + +env.RunTest(enc) +env.RunTest([dec, "encode_legacy.output"]) + diff --git a/tests/backwards_compatibility/alltypes_legacy.c b/tests/backwards_compatibility/alltypes_legacy.c index b144b1e3..9134d5e6 100644 --- a/tests/backwards_compatibility/alltypes_legacy.c +++ b/tests/backwards_compatibility/alltypes_legacy.c @@ -5,7 +5,7 @@ * incompatible changes made to the generator in future versions. */ -#include "bc_alltypes.pb.h" +#include "alltypes_legacy.h" const char SubMessage_substuff1_default[16] = "1"; const int32_t SubMessage_substuff2_default = 2; diff --git a/tests/backwards_compatibility/decode_legacy.c b/tests/backwards_compatibility/decode_legacy.c index b74172fd..315b16ef 100644 --- a/tests/backwards_compatibility/decode_legacy.c +++ b/tests/backwards_compatibility/decode_legacy.c @@ -1,16 +1,16 @@ /* Tests the decoding of all types. - * This is a backwards-compatibility test, using bc_alltypes.pb.h. - * It is similar to test_decode3, but duplicated in order to allow - * test_decode3 to test any new features introduced later. + * This is a backwards-compatibility test, using alltypes_legacy.h. + * It is similar to decode_alltypes, but duplicated in order to allow + * decode_alltypes to test any new features introduced later. * - * Run e.g. ./bc_encode | ./bc_decode + * Run e.g. ./encode_legacy | ./decode_legacy */ #include #include #include #include -#include "bc_alltypes.pb.h" +#include "alltypes_legacy.h" #define TEST(x) if (!(x)) { \ printf("Test " #x " failed.\n"); \ diff --git a/tests/backwards_compatibility/encode_legacy.c b/tests/backwards_compatibility/encode_legacy.c index e84f0908..0e313093 100644 --- a/tests/backwards_compatibility/encode_legacy.c +++ b/tests/backwards_compatibility/encode_legacy.c @@ -1,14 +1,14 @@ /* Attempts to test all the datatypes supported by ProtoBuf. - * This is a backwards-compatibility test, using bc_alltypes.pb.h. - * It is similar to test_encode3, but duplicated in order to allow - * test_encode3 to test any new features introduced later. + * This is a backwards-compatibility test, using alltypes_legacy.h. + * It is similar to encode_alltypes, but duplicated in order to allow + * encode_alltypes to test any new features introduced later. */ #include #include #include #include -#include "bc_alltypes.pb.h" +#include "alltypes_legacy.h" int main(int argc, char **argv) { diff --git a/tests/basic_stream/SConscript b/tests/basic_stream/SConscript new file mode 100644 index 00000000..17382a98 --- /dev/null +++ b/tests/basic_stream/SConscript @@ -0,0 +1,12 @@ +# Build and run a basic round-trip test using direct stream encoding. + +Import("env") + +enc = env.Program(["encode_stream.c", "#common/person.pb.c", "#common/pb_encode.o"]) +dec = env.Program(["decode_stream.c", "#common/person.pb.c", "#common/pb_decode.o"]) + +env.RunTest(enc) +env.RunTest([dec, "encode_stream.output"]) +env.Decode(["encode_stream.output", "#common/person.proto"], MESSAGE = "Person") +env.Compare(["decode_stream.output", "encode_stream.decoded"]) + diff --git a/tests/callbacks/SConscript b/tests/callbacks/SConscript new file mode 100644 index 00000000..729fd65f --- /dev/null +++ b/tests/callbacks/SConscript @@ -0,0 +1,14 @@ +# Test the functionality of the callback fields. + +Import("env") + +env.NanopbProto("callbacks") +enc = env.Program(["encode_callbacks.c", "callbacks.pb.c", "#common/pb_encode.o"]) +dec = env.Program(["decode_callbacks.c", "callbacks.pb.c", "#common/pb_decode.o"]) + +env.RunTest(enc) +env.RunTest([dec, "encode_callbacks.output"]) + +env.Decode(["encode_callbacks.output", "callbacks.proto"], MESSAGE = "TestMessage") +env.Compare(["decode_callbacks.output", "encode_callbacks.decoded"]) + diff --git a/tests/extensions/SConscript b/tests/extensions/SConscript new file mode 100644 index 00000000..b48d6a66 --- /dev/null +++ b/tests/extensions/SConscript @@ -0,0 +1,16 @@ +# Test the support for extension fields. + +Import("env") + +# We use the files from the alltypes test case +incpath = env.Clone() +incpath.Append(PROTOCPATH = '#alltypes') +incpath.Append(CPPPATH = '#alltypes') + +incpath.NanopbProto("extensions") +enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb.o", "#common/pb_encode.o"]) +dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb.o", "#common/pb_decode.o"]) + +env.RunTest(enc) +env.RunTest([dec, "encode_extensions.output"]) + diff --git a/tests/extra_fields/SConscript b/tests/extra_fields/SConscript new file mode 100644 index 00000000..a6f0e751 --- /dev/null +++ b/tests/extra_fields/SConscript @@ -0,0 +1,10 @@ +# Test that the decoder properly handles unknown fields in the input. + +Import("env") + +dec = env.GetBuildPath('#basic_buffer/${PROGPREFIX}decode_buffer${PROGSUFFIX}') +env.RunTest('person_with_extra_field.output', [dec, "person_with_extra_field.pb"]) +env.Compare(["person_with_extra_field.output", "person_with_extra_field.expected"]) + +dec2 = env.GetBuildPath('#alltypes/${PROGPREFIX}decode_alltypes${PROGSUFFIX}') +env.RunTest('alltypes_with_extra_fields.output', [dec2, 'alltypes_with_extra_fields.pb']) diff --git a/tests/extra_fields/person_with_extra_field.expected b/tests/extra_fields/person_with_extra_field.expected new file mode 100644 index 00000000..da9c32df --- /dev/null +++ b/tests/extra_fields/person_with_extra_field.expected @@ -0,0 +1,14 @@ +name: "Test Person 99" +id: 99 +email: "test@person.com" +phone { + number: "555-12345678" + type: MOBILE +} +phone { + number: "99-2342" +} +phone { + number: "1234-5678" + type: WORK +} diff --git a/tests/missing_fields/SConscript b/tests/missing_fields/SConscript new file mode 100644 index 00000000..361b5502 --- /dev/null +++ b/tests/missing_fields/SConscript @@ -0,0 +1,8 @@ +# Check that the decoder properly detects when required fields are missing. + +Import("env") + +env.NanopbProto("missing_fields") +test = env.Program(["missing_fields.c", "missing_fields.pb.c", "#common/pb_encode.o", "#common/pb_decode.o"]) +env.RunTest(test) + diff --git a/tests/multiple_files/SConscript b/tests/multiple_files/SConscript new file mode 100644 index 00000000..6b4f6b69 --- /dev/null +++ b/tests/multiple_files/SConscript @@ -0,0 +1,13 @@ +# Test that multiple .proto files don't cause name collisions. + +Import("env") + +incpath = env.Clone() +incpath.Append(PROTOCPATH = '#multiple_files') + +incpath.NanopbProto("callbacks") +incpath.NanopbProto("callbacks2") +test = incpath.Program(["test_multiple_files.c", "callbacks.pb.c", "callbacks2.pb.c"]) + +env.RunTest(test) + diff --git a/tests/multiple_files/test_multiple_files.c b/tests/multiple_files/test_multiple_files.c index cb4e16d3..05722dc5 100644 --- a/tests/multiple_files/test_multiple_files.c +++ b/tests/multiple_files/test_multiple_files.c @@ -1,6 +1,5 @@ /* - * Tests if still compile if typedefs are redfefined in STATIC_ASSERTS when - * proto file includes another poto file + * Tests if this still compiles when multiple .proto files are involved. */ #include diff --git a/tests/no_messages/SConscript b/tests/no_messages/SConscript new file mode 100644 index 00000000..6492e2cf --- /dev/null +++ b/tests/no_messages/SConscript @@ -0,0 +1,7 @@ +# Test that a .proto file without any messages compiles fine. + +Import("env") + +env.NanopbProto("no_messages") +env.Object('no_messages.pb.c') + diff --git a/tests/options/SConscript b/tests/options/SConscript new file mode 100644 index 00000000..89a00fa5 --- /dev/null +++ b/tests/options/SConscript @@ -0,0 +1,9 @@ +# Test that the generator options work as expected. + +Import("env") + +env.NanopbProto("options") +env.Object('options.pb.c') + +env.Match(['options.pb.h', 'options.expected']) + diff --git a/tests/site_scons/site_init.py b/tests/site_scons/site_init.py index b69db646..86e50335 100644 --- a/tests/site_scons/site_init.py +++ b/tests/site_scons/site_init.py @@ -1,5 +1,6 @@ import subprocess import sys +import re try: # Make terminal colors work on windows @@ -13,8 +14,9 @@ def add_nanopb_builders(env): # Build command for building .pb from .proto using protoc def proto_actions(source, target, env, for_signature): - dirs = ' '.join(['-I' + env.GetBuildPath(d) for d in env['PROTOCPATH']]) - return '$PROTOC $PROTOCFLAGS %s -o%s %s' % (dirs, target[0], source[0]) + esc = env['ESCAPE'] + dirs = ' '.join(['-I' + esc(env.GetBuildPath(d)) for d in env['PROTOCPATH']]) + return '$PROTOC $PROTOCFLAGS %s -o%s %s' % (dirs, esc(str(target[0])), esc(str(source[0]))) proto_file_builder = Builder(generator = proto_actions, suffix = '.pb', @@ -91,4 +93,19 @@ def add_nanopb_builders(env): suffix = '.equal') env.Append(BUILDERS = {'Compare': compare_builder}) + # Build command that checks that each pattern in source2 is found in source1. + def match_files(target, source, env): + data = open(str(source[0]), 'rU').read() + patterns = open(str(source[1])) + for pattern in patterns: + if pattern.strip() and not re.search(pattern.strip(), data, re.MULTILINE): + print '\033[31m[FAIL]\033[0m Pattern not found in ' + str(source[0]) + ': ' + pattern + return 1 + else: + print '\033[32m[ OK ]\033[0m All patterns found in ' + str(source[0]) + return 0 + + match_builder = Builder(action = match_files, suffix = '.matched') + env.Append(BUILDERS = {'Match': match_builder}) + diff --git a/tests/special_characters/SConscript b/tests/special_characters/SConscript new file mode 100644 index 00000000..05dccaee --- /dev/null +++ b/tests/special_characters/SConscript @@ -0,0 +1,7 @@ +# Test that special characters in .proto filenames work. + +Import('env') + +env.Proto("funny-proto+name has.characters.proto") +env.Nanopb("funny-proto+name has.characters.pb.c", "funny-proto+name has.characters.pb") +env.Object("funny-proto+name has.characters.pb.c") diff --git a/tests/special_characters/funny-proto+name.proto b/tests/special_characters/funny-proto+name.proto deleted file mode 100644 index e69de29b..00000000 -- cgit 1.2.3-korg From 840e213b9fa244bce4c67789971c02ce16e66295 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 11 Sep 2013 09:53:51 +0300 Subject: Get rid of the ternary operator in the pb_field_t initialization. Some compilers where unable to detect that the ternary operator can be evaluated at the compile time. This commit does the evaluation on the Python side, which should fix the problem. The new .pb.c files are generated using PB_FIELD2() macro. The old PB_FIELD() macro remains, so that previously generated files keep working. --- generator/nanopb_generator.py | 3 ++- pb.h | 56 ++++++++++++++++++++++++++++--------------- 2 files changed, 39 insertions(+), 20 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index d70938a1..e463b6ca 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -246,13 +246,14 @@ class Field: '''Return the pb_field_t initializer to use in the constant array. prev_field_name is the name of the previous field or None. ''' - result = ' PB_FIELD(%3d, ' % self.tag + result = ' PB_FIELD2(%3d, ' % self.tag result += '%-8s, ' % self.pbtype result += '%s, ' % self.rules result += '%s, ' % self.allocation result += '%s, ' % self.struct_name result += '%s, ' % self.name result += '%s, ' % (prev_field_name or self.name) + result += '%s, ' % ("first" if not prev_field_name else "other") if self.pbtype == 'MESSAGE': result += '&%s_fields)' % self.submsgname diff --git a/pb.h b/pb.h index e3e68ce5..fe91ccd8 100644 --- a/pb.h +++ b/pb.h @@ -320,58 +320,66 @@ struct _pb_extension_t { }; /* These macros are used to declare pb_field_t's in the constant array. */ +/* Size of a structure member, in bytes. */ #define pb_membersize(st, m) (sizeof ((st*)0)->m) +/* Number of entries in an array. */ #define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0])) +/* Delta from start of one member to the start of another member. */ #define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2)) -#define pb_delta_end(st, m1, m2) (int)(offsetof(st, m1) == offsetof(st, m2) \ - ? offsetof(st, m1) \ - : offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) +/* Delta from start of structure to member. */ +#define pb_fielddelta_first(st, m1, m2) (offsetof(st, m1)) +/* Delta from end of one field to start of another field. */ +#define pb_fielddelta_other(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) +/* Choose between pb_fielddelta_first and pb_fielddelta_other (backwards compatibility) */ +#define pb_fielddelta_choose(st, m1, m2) (int)(offsetof(st, m1) == offsetof(st, m2) \ + ? pb_fielddelta_first(st, m1, m2) \ + : pb_fielddelta_other(st, m1, m2)) #define PB_LAST_FIELD {0,(pb_type_t) 0,0,0,0,0,0} /* Required fields are the simplest. They just have delta (padding) from * previous field end, and the size of the field. Pointer is used for * submessages and default values. */ -#define PB_REQUIRED_STATIC(tag, st, m, pm, ltype, ptr) \ +#define PB_REQUIRED_STATIC(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_STATIC | PB_HTYPE_REQUIRED | ltype, \ - pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr} + fd, 0, pb_membersize(st, m), 0, ptr} /* Optional fields add the delta to the has_ variable. */ -#define PB_OPTIONAL_STATIC(tag, st, m, pm, ltype, ptr) \ +#define PB_OPTIONAL_STATIC(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \ - pb_delta_end(st, m, pm), \ + fd, \ pb_delta(st, has_ ## m, m), \ pb_membersize(st, m), 0, ptr} /* Repeated fields have a _count field and also the maximum number of entries. */ -#define PB_REPEATED_STATIC(tag, st, m, pm, ltype, ptr) \ +#define PB_REPEATED_STATIC(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_STATIC | PB_HTYPE_REPEATED | ltype, \ - pb_delta_end(st, m, pm), \ + fd, \ pb_delta(st, m ## _count, m), \ pb_membersize(st, m[0]), \ pb_arraysize(st, m), ptr} /* Callbacks are much like required fields except with special datatype. */ -#define PB_REQUIRED_CALLBACK(tag, st, m, pm, ltype, ptr) \ +#define PB_REQUIRED_CALLBACK(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REQUIRED | ltype, \ - pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr} + fd, 0, pb_membersize(st, m), 0, ptr} -#define PB_OPTIONAL_CALLBACK(tag, st, m, pm, ltype, ptr) \ +#define PB_OPTIONAL_CALLBACK(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ - pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr} + fd, 0, pb_membersize(st, m), 0, ptr} -#define PB_REPEATED_CALLBACK(tag, st, m, pm, ltype, ptr) \ +#define PB_REPEATED_CALLBACK(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REPEATED | ltype, \ - pb_delta_end(st, m, pm), 0, pb_membersize(st, m), 0, ptr} + fd, 0, pb_membersize(st, m), 0, ptr} /* Optional extensions don't have the has_ field, as that would be redundant. */ -#define PB_OPTEXT_STATIC(tag, st, m, pm, ltype, ptr) \ +#define PB_OPTEXT_STATIC(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \ 0, \ 0, \ pb_membersize(st, m), 0, ptr} -#define PB_OPTEXT_CALLBACK(tag, st, m, pm, ltype, ptr) \ +#define PB_OPTEXT_CALLBACK(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ 0, 0, pb_membersize(st, m), 0, ptr} @@ -410,8 +418,18 @@ struct _pb_extension_t { */ #define PB_FIELD(tag, type, rules, allocation, message, field, prevfield, ptr) \ - PB_ ## rules ## _ ## allocation(tag, message, field, prevfield, \ - PB_LTYPE_MAP_ ## type, ptr) + PB_ ## rules ## _ ## allocation(tag, message, field, \ + pb_fielddelta_choose(message, field, prevfield), \ + PB_LTYPE_MAP_ ## type, ptr) + +/* This is a new version of the macro used by nanopb generator from + * version 0.2.3 onwards. It avoids the use of a ternary expression in + * the initialization, which confused some compilers. + */ +#define PB_FIELD2(tag, type, rules, allocation, message, field, prevfield, pos, ptr) \ + PB_ ## rules ## _ ## allocation(tag, message, field, \ + pb_fielddelta_ ## pos(message, field, prevfield), \ + PB_LTYPE_MAP_ ## type, ptr) /* These macros are used for giving out error messages. -- cgit 1.2.3-korg From b9f14bddf778a5ed27e3289b90a0657fec3e1a53 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 11 Sep 2013 13:16:20 +0300 Subject: Make all the tests ANSI C compatible. --- tests/SConstruct | 10 +++---- tests/backwards_compatibility/encode_legacy.c | 32 ++++++++++++--------- tests/basic_stream/decode_stream.c | 9 +++--- tests/basic_stream/encode_stream.c | 5 +++- tests/callbacks/decode_callbacks.c | 16 +++++++---- tests/callbacks/encode_callbacks.c | 14 ++++++--- tests/extensions/decode_extensions.c | 33 ++++++++++++++++------ tests/extensions/encode_extensions.c | 26 ++++++++++++----- tests/missing_fields/missing_fields.c | 9 +++--- .../funny-proto+name has.characters.proto | 0 10 files changed, 100 insertions(+), 54 deletions(-) create mode 100644 tests/special_characters/funny-proto+name has.characters.proto diff --git a/tests/SConstruct b/tests/SConstruct index 92cb0c62..26a513d5 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -70,15 +70,15 @@ if 'gcc' in env['CC']: # GNU Compiler Collection # Debug info, warnings as errors - env.Append(CFLAGS = '-ansi -g -O0 -Wall -Werror --coverage -fstack-protector-all') + env.Append(CFLAGS = '-ansi -pedantic -g -O0 -Wall -Werror --coverage -fstack-protector-all') env.Append(LINKFLAGS = '--coverage') # More strict checks on the nanopb core - env.Append(CORECFLAGS = '-pedantic -Wextra -Wcast-qual -Wlogical-op -Wconversion') + env.Append(CORECFLAGS = '-Wextra -Wcast-qual -Wlogical-op -Wconversion') elif 'clang' in env['CC']: # CLang - env.Append(CFLAGS = '-ansi -g -O0 -Wall -Werror') - env.Append(CORECFLAGS = '-pedantic -Wextra -Wcast-qual -Wconversion') + env.Append(CFLAGS = '-ansi -pedantic -g -O0 -Wall -Werror') + env.Append(CORECFLAGS = ' -Wextra -Wcast-qual -Wconversion') elif 'cl' in env['CC']: # Microsoft Visual C++ @@ -87,7 +87,7 @@ elif 'cl' in env['CC']: env.Append(LINKFLAGS = '/DEBUG') # More strict checks on the nanopb core - env.Append(CORECFLAGS = '/W4 /Za') + env.Append(CORECFLAGS = '/W4') # PB_RETURN_ERROR triggers C4127 because of while(0) env.Append(CFLAGS = '/wd4127') diff --git a/tests/backwards_compatibility/encode_legacy.c b/tests/backwards_compatibility/encode_legacy.c index 0e313093..5c9d41b3 100644 --- a/tests/backwards_compatibility/encode_legacy.c +++ b/tests/backwards_compatibility/encode_legacy.c @@ -9,6 +9,7 @@ #include #include #include "alltypes_legacy.h" +#include "test_helpers.h" int main(int argc, char **argv) { @@ -113,19 +114,22 @@ int main(int argc, char **argv) } alltypes.end = 1099; - - uint8_t buffer[1024]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); - - /* Now encode it and check if we succeeded. */ - if (pb_encode(&stream, AllTypes_fields, &alltypes)) - { - fwrite(buffer, 1, stream.bytes_written, stdout); - return 0; /* Success */ - } - else - { - fprintf(stderr, "Encoding failed!\n"); - return 1; /* Failure */ + + { + uint8_t buffer[1024]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed!\n"); + return 1; /* Failure */ + } } } diff --git a/tests/basic_stream/decode_stream.c b/tests/basic_stream/decode_stream.c index 2142977e..667bf3c5 100644 --- a/tests/basic_stream/decode_stream.c +++ b/tests/basic_stream/decode_stream.c @@ -4,6 +4,7 @@ #include #include #include "person.pb.h" +#include "test_helpers.h" /* This function is called once from main(), it handles the decoding and printing. @@ -69,10 +70,10 @@ bool callback(pb_istream_t *stream, uint8_t *buf, size_t count) int main() { - /* Maximum size is specified to prevent infinite length messages from - * hanging this in the fuzz test. - */ - pb_istream_t stream = {&callback, stdin, 10000}; + pb_istream_t stream = {&callback, NULL, SIZE_MAX}; + stream.state = stdin; + SET_BINARY_MODE(stdin); + if (!print_person(&stream)) { printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); diff --git a/tests/basic_stream/encode_stream.c b/tests/basic_stream/encode_stream.c index fd25c6cb..7f571c41 100644 --- a/tests/basic_stream/encode_stream.c +++ b/tests/basic_stream/encode_stream.c @@ -4,6 +4,7 @@ #include #include #include "person.pb.h" +#include "test_helpers.h" /* This binds the pb_ostream_t into the stdout stream */ bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) @@ -22,7 +23,9 @@ int main() }}; /* Prepare the stream, output goes directly to stdout */ - pb_ostream_t stream = {&streamcallback, stdout, SIZE_MAX, 0}; + pb_ostream_t stream = {&streamcallback, NULL, SIZE_MAX, 0}; + stream.state = stdout; + SET_BINARY_MODE(stdout); /* Now encode it and check if we succeeded. */ if (pb_encode(&stream, Person_fields, &person)) diff --git a/tests/callbacks/decode_callbacks.c b/tests/callbacks/decode_callbacks.c index b5056923..c8daed29 100644 --- a/tests/callbacks/decode_callbacks.c +++ b/tests/callbacks/decode_callbacks.c @@ -5,6 +5,7 @@ #include #include #include "callbacks.pb.h" +#include "test_helpers.h" bool print_string(pb_istream_t *stream, const pb_field_t *field, void **arg) { @@ -50,21 +51,24 @@ bool print_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg) if (!pb_decode_fixed64(stream, &value)) return false; - printf((char*)*arg, (long long)value); + printf((char*)*arg, (long)value); return true; } int main() { uint8_t buffer[1024]; - size_t length = fread(buffer, 1, 1024, stdin); - pb_istream_t stream = pb_istream_from_buffer(buffer, length); - + size_t length; + pb_istream_t stream; /* Note: empty initializer list initializes the struct with all-0. * This is recommended so that unused callbacks are set to NULL instead * of crashing at runtime. */ - TestMessage testmessage = {}; + TestMessage testmessage = {{{NULL}}}; + + SET_BINARY_MODE(stdin); + length = fread(buffer, 1, 1024, stdin); + stream = pb_istream_from_buffer(buffer, length); testmessage.submsg.stringvalue.funcs.decode = &print_string; testmessage.submsg.stringvalue.arg = "submsg {\n stringvalue: \"%s\"\n"; @@ -73,7 +77,7 @@ int main() testmessage.submsg.fixed32value.funcs.decode = &print_fixed32; testmessage.submsg.fixed32value.arg = " fixed32value: %ld\n"; testmessage.submsg.fixed64value.funcs.decode = &print_fixed64; - testmessage.submsg.fixed64value.arg = " fixed64value: %lld\n}\n"; + testmessage.submsg.fixed64value.arg = " fixed64value: %ld\n}\n"; testmessage.stringvalue.funcs.decode = &print_string; testmessage.stringvalue.arg = "stringvalue: \"%s\"\n"; diff --git a/tests/callbacks/encode_callbacks.c b/tests/callbacks/encode_callbacks.c index 3bb6a45e..6cb67b1e 100644 --- a/tests/callbacks/encode_callbacks.c +++ b/tests/callbacks/encode_callbacks.c @@ -4,6 +4,7 @@ #include #include #include "callbacks.pb.h" +#include "test_helpers.h" bool encode_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) { @@ -25,19 +26,21 @@ bool encode_int32(pb_ostream_t *stream, const pb_field_t *field, void * const *a bool encode_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) { + uint32_t value = 42; + if (!pb_encode_tag_for_field(stream, field)) return false; - uint32_t value = 42; return pb_encode_fixed32(stream, &value); } bool encode_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) { + uint64_t value = 42; + if (!pb_encode_tag_for_field(stream, field)) return false; - uint64_t value = 42; return pb_encode_fixed64(stream, &value); } @@ -60,8 +63,10 @@ bool encode_repeatedstring(pb_ostream_t *stream, const pb_field_t *field, void * int main() { uint8_t buffer[1024]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, 1024); - TestMessage testmessage = {}; + pb_ostream_t stream; + TestMessage testmessage = {{{NULL}}}; + + stream = pb_ostream_from_buffer(buffer, 1024); testmessage.stringvalue.funcs.encode = &encode_string; testmessage.int32value.funcs.encode = &encode_int32; @@ -79,6 +84,7 @@ int main() if (!pb_encode(&stream, TestMessage_fields, &testmessage)) return 1; + SET_BINARY_MODE(stdout); if (fwrite(buffer, stream.bytes_written, 1, stdout) != 1) return 2; diff --git a/tests/extensions/decode_extensions.c b/tests/extensions/decode_extensions.c index ef6a0228..f8ebbde0 100644 --- a/tests/extensions/decode_extensions.c +++ b/tests/extensions/decode_extensions.c @@ -6,6 +6,7 @@ #include #include "alltypes.pb.h" #include "extensions.pb.h" +#include "test_helpers.h" #define TEST(x) if (!(x)) { \ printf("Test " #x " failed.\n"); \ @@ -15,25 +16,39 @@ int main(int argc, char **argv) { uint8_t buffer[1024]; - size_t count = fread(buffer, 1, sizeof(buffer), stdin); - pb_istream_t stream = pb_istream_from_buffer(buffer, count); - - AllTypes alltypes = {}; + size_t count; + pb_istream_t stream; + AllTypes alltypes = {0}; int32_t extensionfield1; - pb_extension_t ext1 = {&AllTypes_extensionfield1, &extensionfield1, NULL}; - alltypes.extensions = &ext1; - - ExtensionMessage extensionfield2 = {}; - pb_extension_t ext2 = {&ExtensionMessage_AllTypes_extensionfield2, &extensionfield2, NULL}; + pb_extension_t ext1; + ExtensionMessage extensionfield2; + pb_extension_t ext2; + + /* Read the message data */ + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + stream = pb_istream_from_buffer(buffer, count); + + /* Add the extensions */ + alltypes.extensions = &ext1; + + ext1.type = &AllTypes_extensionfield1; + ext1.dest = &extensionfield1; ext1.next = &ext2; + ext2.type = &ExtensionMessage_AllTypes_extensionfield2; + ext2.dest = &extensionfield2; + ext2.next = NULL; + + /* Decode the message */ if (!pb_decode(&stream, AllTypes_fields, &alltypes)) { printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); return 1; } + /* Check that the extensions decoded properly */ TEST(extensionfield1 == 12345) TEST(strcmp(extensionfield2.test1, "test") == 0) TEST(extensionfield2.test2 == 54321) diff --git a/tests/extensions/encode_extensions.c b/tests/extensions/encode_extensions.c index 8857f148..dee3597d 100644 --- a/tests/extensions/encode_extensions.c +++ b/tests/extensions/encode_extensions.c @@ -7,25 +7,37 @@ #include #include "alltypes.pb.h" #include "extensions.pb.h" +#include "test_helpers.h" int main(int argc, char **argv) { - AllTypes alltypes = {}; + uint8_t buffer[1024]; + pb_ostream_t stream; + AllTypes alltypes = {0}; int32_t extensionfield1 = 12345; - pb_extension_t ext1 = {&AllTypes_extensionfield1, &extensionfield1, NULL}; + pb_extension_t ext1; + ExtensionMessage extensionfield2 = {"test", 54321}; + pb_extension_t ext2; + + /* Set up the extensions */ alltypes.extensions = &ext1; - ExtensionMessage extensionfield2 = {"test", 54321}; - pb_extension_t ext2 = {&ExtensionMessage_AllTypes_extensionfield2, &extensionfield2, NULL}; + ext1.type = &AllTypes_extensionfield1; + ext1.dest = &extensionfield1; ext1.next = &ext2; - uint8_t buffer[1024]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + ext2.type = &ExtensionMessage_AllTypes_extensionfield2; + ext2.dest = &extensionfield2; + ext2.next = NULL; + + /* Set up the output stream */ + stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); - /* Now encode it and check if we succeeded. */ + /* Now encode the message and check if we succeeded. */ if (pb_encode(&stream, AllTypes_fields, &alltypes)) { + SET_BINARY_MODE(stdout); fwrite(buffer, 1, stream.bytes_written, stdout); return 0; /* Success */ } diff --git a/tests/missing_fields/missing_fields.c b/tests/missing_fields/missing_fields.c index 27741847..b9a273a2 100644 --- a/tests/missing_fields/missing_fields.c +++ b/tests/missing_fields/missing_fields.c @@ -7,12 +7,13 @@ int main() { - uint8_t buffer[512] = {}; + uint8_t buffer[512]; /* Create a message with one missing field */ { - MissingField msg = {}; + MissingField msg = {0}; pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + if (!pb_encode(&stream, MissingField_fields, &msg)) { printf("Encode failed.\n"); @@ -22,7 +23,7 @@ int main() /* Test that it decodes properly if we don't require that field */ { - MissingField msg = {}; + MissingField msg = {0}; pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer)); if (!pb_decode(&stream, MissingField_fields, &msg)) @@ -34,7 +35,7 @@ int main() /* Test that it does *not* decode properly if we require the field */ { - AllFields msg = {}; + AllFields msg = {0}; pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer)); if (pb_decode(&stream, AllFields_fields, &msg)) diff --git a/tests/special_characters/funny-proto+name has.characters.proto b/tests/special_characters/funny-proto+name has.characters.proto new file mode 100644 index 00000000..e69de29b -- cgit 1.2.3-korg From d395768c8d3e63125c15950434fa255fb8c57717 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 11 Sep 2013 13:42:56 +0300 Subject: Windows build fixes --- tests/alltypes/decode_alltypes.c | 11 ++++++++--- tests/alltypes/encode_alltypes.c | 2 ++ tests/callbacks/decode_callbacks.c | 2 +- tests/extensions/SConscript | 4 ++-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/tests/alltypes/decode_alltypes.c b/tests/alltypes/decode_alltypes.c index 55d025c0..ee2e115c 100644 --- a/tests/alltypes/decode_alltypes.c +++ b/tests/alltypes/decode_alltypes.c @@ -8,6 +8,7 @@ #include #include #include "alltypes.pb.h" +#include "test_helpers.h" #define TEST(x) if (!(x)) { \ printf("Test " #x " failed.\n"); \ @@ -176,15 +177,19 @@ bool check_alltypes(pb_istream_t *stream, int mode) int main(int argc, char **argv) { + uint8_t buffer[1024]; + size_t count; + pb_istream_t stream; + /* Whether to expect the optional values or the default values. */ int mode = (argc > 1) ? atoi(argv[1]) : 0; /* Read the data into buffer */ - uint8_t buffer[1024]; - size_t count = fread(buffer, 1, sizeof(buffer), stdin); + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); /* Construct a pb_istream_t for reading from the buffer */ - pb_istream_t stream = pb_istream_from_buffer(buffer, count); + stream = pb_istream_from_buffer(buffer, count); /* Decode and print out the stuff */ if (!check_alltypes(&stream, mode)) diff --git a/tests/alltypes/encode_alltypes.c b/tests/alltypes/encode_alltypes.c index 802e1574..88fc10f0 100644 --- a/tests/alltypes/encode_alltypes.c +++ b/tests/alltypes/encode_alltypes.c @@ -6,6 +6,7 @@ #include #include #include "alltypes.pb.h" +#include "test_helpers.h" int main(int argc, char **argv) { @@ -120,6 +121,7 @@ int main(int argc, char **argv) /* Now encode it and check if we succeeded. */ if (pb_encode(&stream, AllTypes_fields, &alltypes)) { + SET_BINARY_MODE(stdout); fwrite(buffer, 1, stream.bytes_written, stdout); return 0; /* Success */ } diff --git a/tests/callbacks/decode_callbacks.c b/tests/callbacks/decode_callbacks.c index c8daed29..45724d0b 100644 --- a/tests/callbacks/decode_callbacks.c +++ b/tests/callbacks/decode_callbacks.c @@ -86,7 +86,7 @@ int main() testmessage.fixed32value.funcs.decode = &print_fixed32; testmessage.fixed32value.arg = "fixed32value: %ld\n"; testmessage.fixed64value.funcs.decode = &print_fixed64; - testmessage.fixed64value.arg = "fixed64value: %lld\n"; + testmessage.fixed64value.arg = "fixed64value: %ld\n"; testmessage.repeatedstring.funcs.decode = &print_string; testmessage.repeatedstring.arg = "repeatedstring: \"%s\"\n"; diff --git a/tests/extensions/SConscript b/tests/extensions/SConscript index b48d6a66..f632a9ae 100644 --- a/tests/extensions/SConscript +++ b/tests/extensions/SConscript @@ -8,8 +8,8 @@ incpath.Append(PROTOCPATH = '#alltypes') incpath.Append(CPPPATH = '#alltypes') incpath.NanopbProto("extensions") -enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb.o", "#common/pb_encode.o"]) -dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb.o", "#common/pb_decode.o"]) +enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb$OBJSUFFIX", "#common/pb_encode.o"]) +dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb$OBJSUFFIX", "#common/pb_decode.o"]) env.RunTest(enc) env.RunTest([dec, "encode_extensions.output"]) -- cgit 1.2.3-korg From 9f93d39f728c3e87b0ab482aa604c5cad4b1c86e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 11 Sep 2013 14:55:56 +0300 Subject: Add tests for different compilation options --- tests/buffer_only/SConscript | 23 +++++++++ tests/cxx_main_program/SConscript | 12 ++--- tests/field_size_16/SConscript | 24 ++++++++++ tests/field_size_16/alltypes.options | 3 ++ tests/field_size_16/alltypes.proto | 90 ++++++++++++++++++++++++++++++++++++ tests/field_size_32/SConscript | 24 ++++++++++ tests/field_size_32/alltypes.options | 3 ++ tests/field_size_32/alltypes.proto | 90 ++++++++++++++++++++++++++++++++++++ tests/no_errmsg/SConscript | 23 +++++++++ 9 files changed, 286 insertions(+), 6 deletions(-) create mode 100644 tests/buffer_only/SConscript create mode 100644 tests/field_size_16/SConscript create mode 100644 tests/field_size_16/alltypes.options create mode 100644 tests/field_size_16/alltypes.proto create mode 100644 tests/field_size_32/SConscript create mode 100644 tests/field_size_32/alltypes.options create mode 100644 tests/field_size_32/alltypes.proto create mode 100644 tests/no_errmsg/SConscript diff --git a/tests/buffer_only/SConscript b/tests/buffer_only/SConscript new file mode 100644 index 00000000..0770b2fc --- /dev/null +++ b/tests/buffer_only/SConscript @@ -0,0 +1,23 @@ +# Run the alltypes test case, but compile with PB_BUFFER_ONLY=1 + +Import("env") + +# Take copy of the files for custom build. +c = Copy("$TARGET", "$SOURCE") +env.Command("pb_encode.c", "#../pb_encode.c", c) +env.Command("pb_decode.c", "#../pb_decode.c", c) +env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c) +env.Command("alltypes.pb.c", "#alltypes/alltypes.pb.c", c) +env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c) +env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c) + +# Define the compilation options +opts = env.Clone() +opts.Append(CPPDEFINES = {'PB_BUFFER_ONLY': 1}) + +# Now build and run the test normally. +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"]) + +env.RunTest(enc) +env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/cxx_main_program/SConscript b/tests/cxx_main_program/SConscript index 0d0a25ca..055c5aee 100644 --- a/tests/cxx_main_program/SConscript +++ b/tests/cxx_main_program/SConscript @@ -5,12 +5,12 @@ Import("env") # Copy the files to .cxx extension in order to force C++ build. c = Copy("$TARGET", "$SOURCE") -Command("pb_encode.cxx", "#../pb_encode.c", c) -Command("pb_decode.cxx", "#../pb_decode.c", c) -Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c) -Command("alltypes.pb.cxx", "#alltypes/alltypes.pb.c", c) -Command("encode_alltypes.cxx", "#alltypes/encode_alltypes.c", c) -Command("decode_alltypes.cxx", "#alltypes/decode_alltypes.c", c) +env.Command("pb_encode.cxx", "#../pb_encode.c", c) +env.Command("pb_decode.cxx", "#../pb_decode.c", c) +env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c) +env.Command("alltypes.pb.cxx", "#alltypes/alltypes.pb.c", c) +env.Command("encode_alltypes.cxx", "#alltypes/encode_alltypes.c", c) +env.Command("decode_alltypes.cxx", "#alltypes/decode_alltypes.c", c) # Now build and run the test normally. enc = env.Program(["encode_alltypes.cxx", "alltypes.pb.cxx", "pb_encode.cxx"]) diff --git a/tests/field_size_16/SConscript b/tests/field_size_16/SConscript new file mode 100644 index 00000000..48d08e84 --- /dev/null +++ b/tests/field_size_16/SConscript @@ -0,0 +1,24 @@ +# Run the alltypes test case, but compile with PB_FIELD_16BIT=1. +# Also the .proto file has been modified to have high indexes. + +Import("env") + +# Take copy of the files for custom build. +c = Copy("$TARGET", "$SOURCE") +env.Command("pb_encode.c", "#../pb_encode.c", c) +env.Command("pb_decode.c", "#../pb_decode.c", c) +env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c) +env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c) + +env.NanopbProto("alltypes") + +# Define the compilation options +opts = env.Clone() +opts.Append(CPPDEFINES = {'PB_FIELD_16BIT': 1}) + +# Now build and run the test normally. +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"]) + +env.RunTest(enc) +env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/field_size_16/alltypes.options b/tests/field_size_16/alltypes.options new file mode 100644 index 00000000..b31e3cf0 --- /dev/null +++ b/tests/field_size_16/alltypes.options @@ -0,0 +1,3 @@ +* max_size:16 +* max_count:5 + diff --git a/tests/field_size_16/alltypes.proto b/tests/field_size_16/alltypes.proto new file mode 100644 index 00000000..b981760d --- /dev/null +++ b/tests/field_size_16/alltypes.proto @@ -0,0 +1,90 @@ +message SubMessage { + required string substuff1 = 1 [default = "1"]; + required int32 substuff2 = 2 [default = 2]; + optional fixed32 substuff3 = 65535 [default = 3]; +} + +message EmptyMessage { + +} + +enum MyEnum { + Zero = 0; + First = 1; + Second = 2; + Truth = 42; +} + +message AllTypes { + required int32 req_int32 = 1; + required int64 req_int64 = 2; + required uint32 req_uint32 = 3; + required uint64 req_uint64 = 4; + required sint32 req_sint32 = 5; + required sint64 req_sint64 = 6; + required bool req_bool = 7; + + required fixed32 req_fixed32 = 8; + required sfixed32 req_sfixed32= 9; + required float req_float = 10; + + required fixed64 req_fixed64 = 11; + required sfixed64 req_sfixed64= 12; + required double req_double = 13; + + required string req_string = 14; + required bytes req_bytes = 15; + required SubMessage req_submsg = 16; + required MyEnum req_enum = 17; + required EmptyMessage req_emptymsg = 18; + + + repeated int32 rep_int32 = 21; + repeated int64 rep_int64 = 22; + repeated uint32 rep_uint32 = 23; + repeated uint64 rep_uint64 = 24; + repeated sint32 rep_sint32 = 25; + repeated sint64 rep_sint64 = 26; + repeated bool rep_bool = 27; + + repeated fixed32 rep_fixed32 = 28; + repeated sfixed32 rep_sfixed32= 29; + repeated float rep_float = 30; + + repeated fixed64 rep_fixed64 = 10031; + repeated sfixed64 rep_sfixed64= 10032; + repeated double rep_double = 10033; + + repeated string rep_string = 10034; + repeated bytes rep_bytes = 10035; + repeated SubMessage rep_submsg = 10036; + repeated MyEnum rep_enum = 10037; + repeated EmptyMessage rep_emptymsg = 10038; + + optional int32 opt_int32 = 10041 [default = 4041]; + optional int64 opt_int64 = 10042 [default = 4042]; + optional uint32 opt_uint32 = 10043 [default = 4043]; + optional uint64 opt_uint64 = 10044 [default = 4044]; + optional sint32 opt_sint32 = 10045 [default = 4045]; + optional sint64 opt_sint64 = 10046 [default = 4046]; + optional bool opt_bool = 10047 [default = false]; + + optional fixed32 opt_fixed32 = 10048 [default = 4048]; + optional sfixed32 opt_sfixed32= 10049 [default = 4049]; + optional float opt_float = 10050 [default = 4050]; + + optional fixed64 opt_fixed64 = 10051 [default = 4051]; + optional sfixed64 opt_sfixed64= 10052 [default = 4052]; + optional double opt_double = 10053 [default = 4053]; + + optional string opt_string = 10054 [default = "4054"]; + optional bytes opt_bytes = 10055 [default = "4055"]; + optional SubMessage opt_submsg = 10056; + optional MyEnum opt_enum = 10057 [default = Second]; + optional EmptyMessage opt_emptymsg = 10058; + + // Just to make sure that the size of the fields has been calculated + // properly, i.e. otherwise a bug in last field might not be detected. + required int32 end = 10099; +} + diff --git a/tests/field_size_32/SConscript b/tests/field_size_32/SConscript new file mode 100644 index 00000000..a8584dc0 --- /dev/null +++ b/tests/field_size_32/SConscript @@ -0,0 +1,24 @@ +# Run the alltypes test case, but compile with PB_FIELD_32BIT=1. +# Also the .proto file has been modified to have high indexes. + +Import("env") + +# Take copy of the files for custom build. +c = Copy("$TARGET", "$SOURCE") +env.Command("pb_encode.c", "#../pb_encode.c", c) +env.Command("pb_decode.c", "#../pb_decode.c", c) +env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c) +env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c) + +env.NanopbProto("alltypes") + +# Define the compilation options +opts = env.Clone() +opts.Append(CPPDEFINES = {'PB_FIELD_32BIT': 1}) + +# Now build and run the test normally. +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"]) + +env.RunTest(enc) +env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/field_size_32/alltypes.options b/tests/field_size_32/alltypes.options new file mode 100644 index 00000000..b31e3cf0 --- /dev/null +++ b/tests/field_size_32/alltypes.options @@ -0,0 +1,3 @@ +* max_size:16 +* max_count:5 + diff --git a/tests/field_size_32/alltypes.proto b/tests/field_size_32/alltypes.proto new file mode 100644 index 00000000..3d1d856c --- /dev/null +++ b/tests/field_size_32/alltypes.proto @@ -0,0 +1,90 @@ +message SubMessage { + required string substuff1 = 1 [default = "1"]; + required int32 substuff2 = 2 [default = 2]; + optional fixed32 substuff3 = 12365535 [default = 3]; +} + +message EmptyMessage { + +} + +enum MyEnum { + Zero = 0; + First = 1; + Second = 2; + Truth = 42; +} + +message AllTypes { + required int32 req_int32 = 1; + required int64 req_int64 = 2; + required uint32 req_uint32 = 3; + required uint64 req_uint64 = 4; + required sint32 req_sint32 = 5; + required sint64 req_sint64 = 6; + required bool req_bool = 7; + + required fixed32 req_fixed32 = 8; + required sfixed32 req_sfixed32= 9; + required float req_float = 10; + + required fixed64 req_fixed64 = 11; + required sfixed64 req_sfixed64= 12; + required double req_double = 13; + + required string req_string = 14; + required bytes req_bytes = 15; + required SubMessage req_submsg = 16; + required MyEnum req_enum = 17; + required EmptyMessage req_emptymsg = 18; + + + repeated int32 rep_int32 = 21; + repeated int64 rep_int64 = 22; + repeated uint32 rep_uint32 = 23; + repeated uint64 rep_uint64 = 24; + repeated sint32 rep_sint32 = 25; + repeated sint64 rep_sint64 = 26; + repeated bool rep_bool = 27; + + repeated fixed32 rep_fixed32 = 28; + repeated sfixed32 rep_sfixed32= 29; + repeated float rep_float = 30; + + repeated fixed64 rep_fixed64 = 10031; + repeated sfixed64 rep_sfixed64= 10032; + repeated double rep_double = 10033; + + repeated string rep_string = 10034; + repeated bytes rep_bytes = 10035; + repeated SubMessage rep_submsg = 10036; + repeated MyEnum rep_enum = 10037; + repeated EmptyMessage rep_emptymsg = 10038; + + optional int32 opt_int32 = 10041 [default = 4041]; + optional int64 opt_int64 = 10042 [default = 4042]; + optional uint32 opt_uint32 = 10043 [default = 4043]; + optional uint64 opt_uint64 = 10044 [default = 4044]; + optional sint32 opt_sint32 = 10045 [default = 4045]; + optional sint64 opt_sint64 = 10046 [default = 4046]; + optional bool opt_bool = 10047 [default = false]; + + optional fixed32 opt_fixed32 = 10048 [default = 4048]; + optional sfixed32 opt_sfixed32= 10049 [default = 4049]; + optional float opt_float = 10050 [default = 4050]; + + optional fixed64 opt_fixed64 = 10051 [default = 4051]; + optional sfixed64 opt_sfixed64= 10052 [default = 4052]; + optional double opt_double = 10053 [default = 4053]; + + optional string opt_string = 10054 [default = "4054"]; + optional bytes opt_bytes = 10055 [default = "4055"]; + optional SubMessage opt_submsg = 10056; + optional MyEnum opt_enum = 10057 [default = Second]; + optional EmptyMessage opt_emptymsg = 10058; + + // Just to make sure that the size of the fields has been calculated + // properly, i.e. otherwise a bug in last field might not be detected. + required int32 end = 13432099; +} + diff --git a/tests/no_errmsg/SConscript b/tests/no_errmsg/SConscript new file mode 100644 index 00000000..870e8946 --- /dev/null +++ b/tests/no_errmsg/SConscript @@ -0,0 +1,23 @@ +# Run the alltypes test case, but compile with PB_NO_ERRMSG=1 + +Import("env") + +# Take copy of the files for custom build. +c = Copy("$TARGET", "$SOURCE") +env.Command("pb_encode.c", "#../pb_encode.c", c) +env.Command("pb_decode.c", "#../pb_decode.c", c) +env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c) +env.Command("alltypes.pb.c", "#alltypes/alltypes.pb.c", c) +env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c) +env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c) + +# Define the compilation options +opts = env.Clone() +opts.Append(CPPDEFINES = {'PB_NO_ERRMSG': 1}) + +# Now build and run the test normally. +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"]) + +env.RunTest(enc) +env.RunTest([dec, "encode_alltypes.output"]) -- cgit 1.2.3-korg From 2b728150360f4c15512579ff53bc3247d3b8d083 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 11 Sep 2013 16:45:52 +0300 Subject: Fix build error when path contains spaces --- tests/site_scons/site_init.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/site_scons/site_init.py b/tests/site_scons/site_init.py index 86e50335..1f811153 100644 --- a/tests/site_scons/site_init.py +++ b/tests/site_scons/site_init.py @@ -37,7 +37,8 @@ def add_nanopb_builders(env): src_suffix = '.pb', emitter = nanopb_targets) env.Append(BUILDERS = {'Nanopb': nanopb_file_builder}) - env.SetDefault(NANOPB_GENERATOR = 'python ' + env.GetBuildPath("#../generator/nanopb_generator.py")) + gen_path = env['ESCAPE'](env.GetBuildPath("#../generator/nanopb_generator.py")) + env.SetDefault(NANOPB_GENERATOR = 'python ' + gen_path) env.SetDefault(NANOPB_FLAGS = '-q') # Combined method to run both protoc and nanopb generator @@ -71,8 +72,10 @@ def add_nanopb_builders(env): # Build command that decodes a message using protoc def decode_actions(source, target, env, for_signature): - dirs = ' '.join(['-I' + env.GetBuildPath(d) for d in env['PROTOCPATH']]) - return '$PROTOC $PROTOCFLAGS %s --decode=%s %s <%s >%s' % (dirs, env['MESSAGE'], source[1], source[0], target[0]) + esc = env['ESCAPE'] + dirs = ' '.join(['-I' + esc(env.GetBuildPath(d)) for d in env['PROTOCPATH']]) + return '$PROTOC $PROTOCFLAGS %s --decode=%s %s <%s >%s' % ( + dirs, env['MESSAGE'], esc(str(source[1])), esc(str(source[0])), esc(str(target[0]))) decode_builder = Builder(generator = decode_actions, suffix = '.decoded') -- cgit 1.2.3-korg From 152c2c910c9cda6e0beb1863c510c678300bdd7b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 11 Sep 2013 16:51:53 +0300 Subject: Disable warning about uint64_t (long long) --- tests/SConstruct | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/SConstruct b/tests/SConstruct index 26a513d5..3f4d7702 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -73,11 +73,14 @@ if 'gcc' in env['CC']: env.Append(CFLAGS = '-ansi -pedantic -g -O0 -Wall -Werror --coverage -fstack-protector-all') env.Append(LINKFLAGS = '--coverage') + # We currently need uint64_t anyway, even though ANSI C90 otherwise.. + env.Append(CFLAGS = '-Wno-long-long') + # More strict checks on the nanopb core env.Append(CORECFLAGS = '-Wextra -Wcast-qual -Wlogical-op -Wconversion') elif 'clang' in env['CC']: # CLang - env.Append(CFLAGS = '-ansi -pedantic -g -O0 -Wall -Werror') + env.Append(CFLAGS = '-ansi -g -O0 -Wall -Werror') env.Append(CORECFLAGS = ' -Wextra -Wcast-qual -Wconversion') elif 'cl' in env['CC']: # Microsoft Visual C++ -- cgit 1.2.3-korg From 59cba0beeabac991b123dc6826e534f7838c5961 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 11 Sep 2013 17:33:50 +0300 Subject: Expand extra_fields test to cover field skipping in case of streams. --- tests/extra_fields/SConscript | 4 ++++ tests/extra_fields/person_with_extra_field.pb | Bin 90 -> 124 bytes 2 files changed, 4 insertions(+) diff --git a/tests/extra_fields/SConscript b/tests/extra_fields/SConscript index a6f0e751..9227e950 100644 --- a/tests/extra_fields/SConscript +++ b/tests/extra_fields/SConscript @@ -6,5 +6,9 @@ dec = env.GetBuildPath('#basic_buffer/${PROGPREFIX}decode_buffer${PROGSUFFIX}') env.RunTest('person_with_extra_field.output', [dec, "person_with_extra_field.pb"]) env.Compare(["person_with_extra_field.output", "person_with_extra_field.expected"]) +dec = env.GetBuildPath('#basic_stream/${PROGPREFIX}decode_stream${PROGSUFFIX}') +env.RunTest('person_with_extra_field_stream.output', [dec, "person_with_extra_field.pb"]) +env.Compare(["person_with_extra_field_stream.output", "person_with_extra_field.expected"]) + dec2 = env.GetBuildPath('#alltypes/${PROGPREFIX}decode_alltypes${PROGSUFFIX}') env.RunTest('alltypes_with_extra_fields.output', [dec2, 'alltypes_with_extra_fields.pb']) diff --git a/tests/extra_fields/person_with_extra_field.pb b/tests/extra_fields/person_with_extra_field.pb index ced3057c..ffb303dd 100644 Binary files a/tests/extra_fields/person_with_extra_field.pb and b/tests/extra_fields/person_with_extra_field.pb differ -- cgit 1.2.3-korg From 9ada7e752516260054525fca8e1f67efa321f682 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 13 Sep 2013 11:30:58 +0300 Subject: Fine-tune the naming of new macros before merging into master. Requires re-generation of files generated with dev_get_rid_of_ternary_operator. --- generator/nanopb_generator.py | 2 +- pb.h | 28 +++++++++++++++++----------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index e463b6ca..2e30b676 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -250,10 +250,10 @@ class Field: result += '%-8s, ' % self.pbtype result += '%s, ' % self.rules result += '%s, ' % self.allocation + result += '%s, ' % ("FIRST" if not prev_field_name else "OTHER") result += '%s, ' % self.struct_name result += '%s, ' % self.name result += '%s, ' % (prev_field_name or self.name) - result += '%s, ' % ("first" if not prev_field_name else "other") if self.pbtype == 'MESSAGE': result += '&%s_fields)' % self.submsgname diff --git a/pb.h b/pb.h index fe91ccd8..98b9bbd1 100644 --- a/pb.h +++ b/pb.h @@ -326,16 +326,19 @@ struct _pb_extension_t { #define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0])) /* Delta from start of one member to the start of another member. */ #define pb_delta(st, m1, m2) ((int)offsetof(st, m1) - (int)offsetof(st, m2)) -/* Delta from start of structure to member. */ -#define pb_fielddelta_first(st, m1, m2) (offsetof(st, m1)) -/* Delta from end of one field to start of another field. */ -#define pb_fielddelta_other(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) -/* Choose between pb_fielddelta_first and pb_fielddelta_other (backwards compatibility) */ -#define pb_fielddelta_choose(st, m1, m2) (int)(offsetof(st, m1) == offsetof(st, m2) \ - ? pb_fielddelta_first(st, m1, m2) \ - : pb_fielddelta_other(st, m1, m2)) +/* Marks the end of the field list */ #define PB_LAST_FIELD {0,(pb_type_t) 0,0,0,0,0,0} +/* Macros for filling in the data_offset field */ +/* data_offset for first field in a message */ +#define PB_DATAOFFSET_FIRST(st, m1, m2) (offsetof(st, m1)) +/* data_offset for subsequent fields */ +#define PB_DATAOFFSET_OTHER(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) +/* Choose first/other based on m1 == m2 (deprecated, remains for backwards compatibility) */ +#define PB_DATAOFFSET_CHOOSE(st, m1, m2) (int)(offsetof(st, m1) == offsetof(st, m2) \ + ? PB_DATAOFFSET_FIRST(st, m1, m2) \ + : PB_DATAOFFSET_OTHER(st, m1, m2)) + /* Required fields are the simplest. They just have delta (padding) from * previous field end, and the size of the field. Pointer is used for * submessages and default values. @@ -419,16 +422,19 @@ struct _pb_extension_t { #define PB_FIELD(tag, type, rules, allocation, message, field, prevfield, ptr) \ PB_ ## rules ## _ ## allocation(tag, message, field, \ - pb_fielddelta_choose(message, field, prevfield), \ + PB_DATAOFFSET_CHOOSE(message, field, prevfield), \ PB_LTYPE_MAP_ ## type, ptr) /* This is a new version of the macro used by nanopb generator from * version 0.2.3 onwards. It avoids the use of a ternary expression in * the initialization, which confused some compilers. + * + * - Placement: FIRST or OTHER, depending on if this is the first field in structure. + * */ -#define PB_FIELD2(tag, type, rules, allocation, message, field, prevfield, pos, ptr) \ +#define PB_FIELD2(tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ PB_ ## rules ## _ ## allocation(tag, message, field, \ - pb_fielddelta_ ## pos(message, field, prevfield), \ + PB_DATAOFFSET_ ## placement(message, field, prevfield), \ PB_LTYPE_MAP_ ## type, ptr) -- cgit 1.2.3-korg From f47410ea4b8ae43e19facd378be4cf1073e1813b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 13 Sep 2013 12:59:31 +0300 Subject: Move examples into subfolders, add READMEs --- example/Makefile | 14 --- example/client.c | 116 -------------------- example/common.c | 40 ------- example/common.h | 9 -- example/fileproto.options | 13 --- example/fileproto.proto | 18 ---- example/server.c | 131 ----------------------- example_avr_double/Makefile | 22 ---- example_avr_double/README.txt | 22 ---- example_avr_double/decode_double.c | 33 ------ example_avr_double/double_conversion.c | 123 --------------------- example_avr_double/double_conversion.h | 26 ----- example_avr_double/doubleproto.proto | 13 --- example_avr_double/encode_double.c | 25 ----- example_avr_double/test_conversions.c | 56 ---------- example_unions/Makefile | 17 --- example_unions/decode.c | 96 ----------------- example_unions/encode.c | 85 --------------- example_unions/unionproto.proto | 30 ------ examples/network_server/Makefile | 19 ++++ examples/network_server/README | 60 +++++++++++ examples/network_server/client.c | 116 ++++++++++++++++++++ examples/network_server/common.c | 40 +++++++ examples/network_server/common.h | 9 ++ examples/network_server/fileproto.options | 13 +++ examples/network_server/fileproto.proto | 18 ++++ examples/network_server/server.c | 131 +++++++++++++++++++++++ examples/using_double_on_avr/Makefile | 29 +++++ examples/using_double_on_avr/README | 25 +++++ examples/using_double_on_avr/decode_double.c | 33 ++++++ examples/using_double_on_avr/double_conversion.c | 123 +++++++++++++++++++++ examples/using_double_on_avr/double_conversion.h | 26 +++++ examples/using_double_on_avr/doubleproto.proto | 13 +++ examples/using_double_on_avr/encode_double.c | 25 +++++ examples/using_double_on_avr/test_conversions.c | 56 ++++++++++ examples/using_union_messages/Makefile | 22 ++++ examples/using_union_messages/README | 52 +++++++++ examples/using_union_messages/decode.c | 96 +++++++++++++++++ examples/using_union_messages/encode.c | 85 +++++++++++++++ examples/using_union_messages/unionproto.proto | 30 ++++++ 40 files changed, 1021 insertions(+), 889 deletions(-) delete mode 100644 example/Makefile delete mode 100644 example/client.c delete mode 100644 example/common.c delete mode 100644 example/common.h delete mode 100644 example/fileproto.options delete mode 100644 example/fileproto.proto delete mode 100644 example/server.c delete mode 100644 example_avr_double/Makefile delete mode 100644 example_avr_double/README.txt delete mode 100644 example_avr_double/decode_double.c delete mode 100644 example_avr_double/double_conversion.c delete mode 100644 example_avr_double/double_conversion.h delete mode 100644 example_avr_double/doubleproto.proto delete mode 100644 example_avr_double/encode_double.c delete mode 100644 example_avr_double/test_conversions.c delete mode 100644 example_unions/Makefile delete mode 100644 example_unions/decode.c delete mode 100644 example_unions/encode.c delete mode 100644 example_unions/unionproto.proto create mode 100644 examples/network_server/Makefile create mode 100644 examples/network_server/README create mode 100644 examples/network_server/client.c create mode 100644 examples/network_server/common.c create mode 100644 examples/network_server/common.h create mode 100644 examples/network_server/fileproto.options create mode 100644 examples/network_server/fileproto.proto create mode 100644 examples/network_server/server.c create mode 100644 examples/using_double_on_avr/Makefile create mode 100644 examples/using_double_on_avr/README create mode 100644 examples/using_double_on_avr/decode_double.c create mode 100644 examples/using_double_on_avr/double_conversion.c create mode 100644 examples/using_double_on_avr/double_conversion.h create mode 100644 examples/using_double_on_avr/doubleproto.proto create mode 100644 examples/using_double_on_avr/encode_double.c create mode 100644 examples/using_double_on_avr/test_conversions.c create mode 100644 examples/using_union_messages/Makefile create mode 100644 examples/using_union_messages/README create mode 100644 examples/using_union_messages/decode.c create mode 100644 examples/using_union_messages/encode.c create mode 100644 examples/using_union_messages/unionproto.proto diff --git a/example/Makefile b/example/Makefile deleted file mode 100644 index 14dd9fa4..00000000 --- a/example/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -CFLAGS=-ansi -Wall -Werror -I .. -g -O0 -DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h - -all: server client - -clean: - rm -f server client fileproto.pb.c fileproto.pb.h - -%: %.c $(DEPS) fileproto.pb.h fileproto.pb.c - $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c fileproto.pb.c common.c - -fileproto.pb.c fileproto.pb.h: fileproto.proto ../generator/nanopb_generator.py - protoc -I. -I../generator -I/usr/include -ofileproto.pb $< - python ../generator/nanopb_generator.py fileproto.pb diff --git a/example/client.c b/example/client.c deleted file mode 100644 index e6e9a2e0..00000000 --- a/example/client.c +++ /dev/null @@ -1,116 +0,0 @@ -/* This is a simple TCP client that connects to port 1234 and prints a list - * of files in a given directory. - * - * It directly deserializes and serializes messages from network, minimizing - * memory use. - * - * For flexibility, this example is implemented using posix api. - * In a real embedded system you would typically use some other kind of - * a communication and filesystem layer. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "fileproto.pb.h" -#include "common.h" - -bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void **arg) -{ - FileInfo fileinfo; - - if (!pb_decode(stream, FileInfo_fields, &fileinfo)) - return false; - - printf("%-10lld %s\n", (long long)fileinfo.inode, fileinfo.name); - - return true; -} - -bool listdir(int fd, char *path) -{ - ListFilesRequest request; - ListFilesResponse response; - pb_istream_t input = pb_istream_from_socket(fd); - pb_ostream_t output = pb_ostream_from_socket(fd); - uint8_t zero = 0; - - if (path == NULL) - { - request.has_path = false; - } - else - { - request.has_path = true; - if (strlen(path) + 1 > sizeof(request.path)) - { - fprintf(stderr, "Too long path.\n"); - return false; - } - - strcpy(request.path, path); - } - - if (!pb_encode(&output, ListFilesRequest_fields, &request)) - { - fprintf(stderr, "Encoding failed.\n"); - return false; - } - - /* We signal the end of request with a 0 tag. */ - pb_write(&output, &zero, 1); - - response.file.funcs.decode = &printfile_callback; - - if (!pb_decode(&input, ListFilesResponse_fields, &response)) - { - fprintf(stderr, "Decode failed: %s\n", PB_GET_ERROR(&input)); - return false; - } - - if (response.path_error) - { - fprintf(stderr, "Server reported error.\n"); - return false; - } - - return true; -} - -int main(int argc, char **argv) -{ - int sockfd; - struct sockaddr_in servaddr; - char *path = NULL; - - if (argc > 1) - path = argv[1]; - - sockfd = socket(AF_INET, SOCK_STREAM, 0); - - memset(&servaddr, 0, sizeof(servaddr)); - servaddr.sin_family = AF_INET; - servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - servaddr.sin_port = htons(1234); - - if (connect(sockfd, (struct sockaddr *)&servaddr, sizeof(servaddr)) != 0) - { - perror("connect"); - return 1; - } - - if (!listdir(sockfd, path)) - return 2; - - close(sockfd); - - return 0; -} diff --git a/example/common.c b/example/common.c deleted file mode 100644 index 04a5aa85..00000000 --- a/example/common.c +++ /dev/null @@ -1,40 +0,0 @@ -/* Simple binding of nanopb streams to TCP sockets. - */ - -#include -#include -#include -#include - -#include "common.h" - -static bool write_callback(pb_ostream_t *stream, const uint8_t *buf, size_t count) -{ - int fd = (intptr_t)stream->state; - return send(fd, buf, count, 0) == count; -} - -static bool read_callback(pb_istream_t *stream, uint8_t *buf, size_t count) -{ - int fd = (intptr_t)stream->state; - int result; - - result = recv(fd, buf, count, MSG_WAITALL); - - if (result == 0) - stream->bytes_left = 0; /* EOF */ - - return result == count; -} - -pb_ostream_t pb_ostream_from_socket(int fd) -{ - pb_ostream_t stream = {&write_callback, (void*)(intptr_t)fd, SIZE_MAX, 0}; - return stream; -} - -pb_istream_t pb_istream_from_socket(int fd) -{ - pb_istream_t stream = {&read_callback, (void*)(intptr_t)fd, SIZE_MAX}; - return stream; -} diff --git a/example/common.h b/example/common.h deleted file mode 100644 index 8dab3b7c..00000000 --- a/example/common.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef _PB_EXAMPLE_COMMON_H_ -#define _PB_EXAMPLE_COMMON_H_ - -#include - -pb_ostream_t pb_ostream_from_socket(int fd); -pb_istream_t pb_istream_from_socket(int fd); - -#endif \ No newline at end of file diff --git a/example/fileproto.options b/example/fileproto.options deleted file mode 100644 index 29a2ab0e..00000000 --- a/example/fileproto.options +++ /dev/null @@ -1,13 +0,0 @@ -# This file defines the nanopb-specific options for the messages defined -# in fileproto.proto. -# -# If you come from high-level programming background, the hardcoded -# maximum lengths may disgust you. However, if your microcontroller only -# has a few kB of ram to begin with, setting reasonable limits for -# filenames is ok. -# -# On the other hand, using the callback interface, it is not necessary -# to set a limit on the number of files in the response. - -ListFilesRequest.path max_size:128 -FileInfo.name max_size:128 diff --git a/example/fileproto.proto b/example/fileproto.proto deleted file mode 100644 index 3e70c492..00000000 --- a/example/fileproto.proto +++ /dev/null @@ -1,18 +0,0 @@ -// This defines protocol for a simple server that lists files. -// -// See also the nanopb-specific options in fileproto.options. - -message ListFilesRequest { - optional string path = 1 [default = "/"]; -} - -message FileInfo { - required uint64 inode = 1; - required string name = 2; -} - -message ListFilesResponse { - optional bool path_error = 1 [default = false]; - repeated FileInfo file = 2; -} - diff --git a/example/server.c b/example/server.c deleted file mode 100644 index 9a9c2644..00000000 --- a/example/server.c +++ /dev/null @@ -1,131 +0,0 @@ -/* This is a simple TCP server that listens on port 1234 and provides lists - * of files to clients, using a protocol defined in file_server.proto. - * - * It directly deserializes and serializes messages from network, minimizing - * memory use. - * - * For flexibility, this example is implemented using posix api. - * In a real embedded system you would typically use some other kind of - * a communication and filesystem layer. - */ - -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "fileproto.pb.h" -#include "common.h" - -bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) -{ - DIR *dir = (DIR*) *arg; - struct dirent *file; - FileInfo fileinfo; - - while ((file = readdir(dir)) != NULL) - { - fileinfo.inode = file->d_ino; - strncpy(fileinfo.name, file->d_name, sizeof(fileinfo.name)); - fileinfo.name[sizeof(fileinfo.name) - 1] = '\0'; - - if (!pb_encode_tag_for_field(stream, field)) - return false; - - if (!pb_encode_submessage(stream, FileInfo_fields, &fileinfo)) - return false; - } - - return true; -} - -void handle_connection(int connfd) -{ - ListFilesRequest request; - ListFilesResponse response; - pb_istream_t input = pb_istream_from_socket(connfd); - pb_ostream_t output = pb_ostream_from_socket(connfd); - DIR *directory; - - if (!pb_decode(&input, ListFilesRequest_fields, &request)) - { - printf("Decode failed: %s\n", PB_GET_ERROR(&input)); - return; - } - - directory = opendir(request.path); - - printf("Listing directory: %s\n", request.path); - - if (directory == NULL) - { - perror("opendir"); - - response.has_path_error = true; - response.path_error = true; - response.file.funcs.encode = NULL; - } - else - { - response.has_path_error = false; - response.file.funcs.encode = &listdir_callback; - response.file.arg = directory; - } - - if (!pb_encode(&output, ListFilesResponse_fields, &response)) - { - printf("Encoding failed.\n"); - } -} - -int main(int argc, char **argv) -{ - int listenfd, connfd; - struct sockaddr_in servaddr; - int reuse = 1; - - listenfd = socket(AF_INET, SOCK_STREAM, 0); - - setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); - - memset(&servaddr, 0, sizeof(servaddr)); - servaddr.sin_family = AF_INET; - servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); - servaddr.sin_port = htons(1234); - if (bind(listenfd, (struct sockaddr*)&servaddr, sizeof(servaddr)) != 0) - { - perror("bind"); - return 1; - } - - if (listen(listenfd, 5) != 0) - { - perror("listen"); - return 1; - } - - for(;;) - { - connfd = accept(listenfd, NULL, NULL); - - if (connfd < 0) - { - perror("accept"); - return 1; - } - - printf("Got connection.\n"); - - handle_connection(connfd); - - printf("Closing connection.\n"); - - close(connfd); - } -} diff --git a/example_avr_double/Makefile b/example_avr_double/Makefile deleted file mode 100644 index 74300fc6..00000000 --- a/example_avr_double/Makefile +++ /dev/null @@ -1,22 +0,0 @@ -CFLAGS=-Wall -Werror -I .. -g -O0 -DEPS=double_conversion.c ../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h - -all: run_tests - -clean: - rm -f test_conversions encode_double decode_double doubleproto.pb.c doubleproto.pb.h - -test_conversions: test_conversions.c double_conversion.c - $(CC) $(CFLAGS) -o $@ $^ - -%: %.c $(DEPS) doubleproto.pb.h doubleproto.pb.c - $(CC) $(CFLAGS) -o $@ $< double_conversion.c ../pb_decode.c ../pb_encode.c doubleproto.pb.c - -doubleproto.pb.c doubleproto.pb.h: doubleproto.proto ../generator/nanopb_generator.py - protoc -I. -I../generator -I/usr/include -odoubleproto.pb $< - python ../generator/nanopb_generator.py doubleproto.pb - -run_tests: test_conversions encode_double decode_double - ./test_conversions - ./encode_double | ./decode_double - diff --git a/example_avr_double/README.txt b/example_avr_double/README.txt deleted file mode 100644 index 0090d729..00000000 --- a/example_avr_double/README.txt +++ /dev/null @@ -1,22 +0,0 @@ -Some processors/compilers, such as AVR-GCC, do not support the double -datatype. Instead, they have sizeof(double) == 4. Because protocol -binary format uses the double encoding directly, this causes trouble -if the protocol in .proto requires double fields. - -This directory contains a solution to this problem. It uses uint64_t -to store the raw wire values, because its size is correct on all -platforms. The file double_conversion.c provides functions that -convert these values to/from floats, without relying on compiler -support. - -To use this method, you need to make two modifications to your code: - -1) Change all 'double' fields into 'fixed64' in the .proto. - -2) Whenever writing to a 'double' field, use float_to_double(). - -3) Whenever reading a 'double' field, use double_to_float(). - -The conversion routines should be as accurate as the float datatype can -be. Furthermore, they should handle all special values (NaN, inf, denormalized -numbers) correctly. There are testcases in test_conversions.c. diff --git a/example_avr_double/decode_double.c b/example_avr_double/decode_double.c deleted file mode 100644 index 5802eca7..00000000 --- a/example_avr_double/decode_double.c +++ /dev/null @@ -1,33 +0,0 @@ -/* Decodes a double value into a float variable. - * Used to read double values with AVR code, which doesn't support double directly. - */ - -#include -#include -#include "double_conversion.h" -#include "doubleproto.pb.h" - -int main() -{ - uint8_t buffer[32]; - size_t count = fread(buffer, 1, sizeof(buffer), stdin); - pb_istream_t stream = pb_istream_from_buffer(buffer, count); - - AVRDoubleMessage message; - pb_decode(&stream, AVRDoubleMessage_fields, &message); - - float v1 = double_to_float(message.field1); - float v2 = double_to_float(message.field2); - - printf("Values: %f %f\n", v1, v2); - - if (v1 == 1234.5678f && - v2 == 0.00001f) - { - return 0; - } - else - { - return 1; - } -} diff --git a/example_avr_double/double_conversion.c b/example_avr_double/double_conversion.c deleted file mode 100644 index cf79b9a0..00000000 --- a/example_avr_double/double_conversion.c +++ /dev/null @@ -1,123 +0,0 @@ -/* Conversion routines for platforms that do not support 'double' directly. */ - -#include "double_conversion.h" -#include - -typedef union { - float f; - uint32_t i; -} conversion_t; - -/* Note: IEE 754 standard specifies float formats as follows: - * Single precision: sign, 8-bit exp, 23-bit frac. - * Double precision: sign, 11-bit exp, 52-bit frac. - */ - -uint64_t float_to_double(float value) -{ - conversion_t in; - in.f = value; - uint8_t sign; - int16_t exponent; - uint64_t mantissa; - - /* Decompose input value */ - sign = (in.i >> 31) & 1; - exponent = ((in.i >> 23) & 0xFF) - 127; - mantissa = in.i & 0x7FFFFF; - - if (exponent == 128) - { - /* Special value (NaN etc.) */ - exponent = 1024; - } - else if (exponent == -127) - { - if (!mantissa) - { - /* Zero */ - exponent = -1023; - } - else - { - /* Denormalized */ - mantissa <<= 1; - while (!(mantissa & 0x800000)) - { - mantissa <<= 1; - exponent--; - } - mantissa &= 0x7FFFFF; - } - } - - /* Combine fields */ - mantissa <<= 29; - mantissa |= (uint64_t)(exponent + 1023) << 52; - mantissa |= (uint64_t)sign << 63; - - return mantissa; -} - -float double_to_float(uint64_t value) -{ - uint8_t sign; - int16_t exponent; - uint32_t mantissa; - conversion_t out; - - /* Decompose input value */ - sign = (value >> 63) & 1; - exponent = ((value >> 52) & 0x7FF) - 1023; - mantissa = (value >> 28) & 0xFFFFFF; /* Highest 24 bits */ - - /* Figure if value is in range representable by floats. */ - if (exponent == 1024) - { - /* Special value */ - exponent = 128; - } - else if (exponent > 127) - { - /* Too large */ - if (sign) - return -INFINITY; - else - return INFINITY; - } - else if (exponent < -150) - { - /* Too small */ - if (sign) - return -0.0f; - else - return 0.0f; - } - else if (exponent < -126) - { - /* Denormalized */ - mantissa |= 0x1000000; - mantissa >>= (-126 - exponent); - exponent = -127; - } - - /* Round off mantissa */ - mantissa = (mantissa + 1) >> 1; - - /* Check if mantissa went over 2.0 */ - if (mantissa & 0x800000) - { - exponent += 1; - mantissa &= 0x7FFFFF; - mantissa >>= 1; - } - - /* Combine fields */ - out.i = mantissa; - out.i |= (uint32_t)(exponent + 127) << 23; - out.i |= (uint32_t)sign << 31; - - return out.f; -} - - diff --git a/example_avr_double/double_conversion.h b/example_avr_double/double_conversion.h deleted file mode 100644 index 62b6a8ae..00000000 --- a/example_avr_double/double_conversion.h +++ /dev/null @@ -1,26 +0,0 @@ -/* AVR-GCC does not have real double datatype. Instead its double - * is equal to float, i.e. 32 bit value. If you need to communicate - * with other systems that use double in their .proto files, you - * need to do some conversion. - * - * These functions use bitwise operations to mangle floats into doubles - * and then store them in uint64_t datatype. - */ - -#ifndef DOUBLE_CONVERSION -#define DOUBLE_CONVERSION - -#include - -/* Convert native 4-byte float into a 8-byte double. */ -extern uint64_t float_to_double(float value); - -/* Convert 8-byte double into native 4-byte float. - * Values are rounded to nearest, 0.5 away from zero. - * Overflowing values are converted to Inf or -Inf. - */ -extern float double_to_float(uint64_t value); - - -#endif - diff --git a/example_avr_double/doubleproto.proto b/example_avr_double/doubleproto.proto deleted file mode 100644 index d8b7f2db..00000000 --- a/example_avr_double/doubleproto.proto +++ /dev/null @@ -1,13 +0,0 @@ -// A message containing doubles, as used by other applications. -message DoubleMessage { - required double field1 = 1; - required double field2 = 2; -} - -// A message containing doubles, but redefined using uint64_t. -// For use in AVR code. -message AVRDoubleMessage { - required fixed64 field1 = 1; - required fixed64 field2 = 2; -} - diff --git a/example_avr_double/encode_double.c b/example_avr_double/encode_double.c deleted file mode 100644 index cd532d46..00000000 --- a/example_avr_double/encode_double.c +++ /dev/null @@ -1,25 +0,0 @@ -/* Encodes a float value into a double on the wire. - * Used to emit doubles from AVR code, which doesn't support double directly. - */ - -#include -#include -#include "double_conversion.h" -#include "doubleproto.pb.h" - -int main() -{ - AVRDoubleMessage message = { - float_to_double(1234.5678f), - float_to_double(0.00001f) - }; - - uint8_t buffer[32]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); - - pb_encode(&stream, AVRDoubleMessage_fields, &message); - fwrite(buffer, 1, stream.bytes_written, stdout); - - return 0; -} - diff --git a/example_avr_double/test_conversions.c b/example_avr_double/test_conversions.c deleted file mode 100644 index 22620a6a..00000000 --- a/example_avr_double/test_conversions.c +++ /dev/null @@ -1,56 +0,0 @@ -#include "double_conversion.h" -#include -#include - -static const double testvalues[] = { - 0.0, -0.0, 0.1, -0.1, - M_PI, -M_PI, 123456.789, -123456.789, - INFINITY, -INFINITY, NAN, INFINITY - INFINITY, - 1e38, -1e38, 1e39, -1e39, - 1e-38, -1e-38, 1e-39, -1e-39, - 3.14159e-37,-3.14159e-37, 3.14159e-43, -3.14159e-43, - 1e-60, -1e-60, 1e-45, -1e-45, - 0.99999999999999, -0.99999999999999, 127.999999999999, -127.999999999999 -}; - -#define TESTVALUES_COUNT (sizeof(testvalues)/sizeof(testvalues[0])) - -int main() -{ - int status = 0; - int i; - for (i = 0; i < TESTVALUES_COUNT; i++) - { - double orig = testvalues[i]; - float expected_float = (float)orig; - double expected_double = (double)expected_float; - - float got_float = double_to_float(*(uint64_t*)&orig); - uint64_t got_double = float_to_double(got_float); - - uint32_t e1 = *(uint32_t*)&expected_float; - uint32_t g1 = *(uint32_t*)&got_float; - uint64_t e2 = *(uint64_t*)&expected_double; - uint64_t g2 = got_double; - - if (g1 != e1) - { - printf("%3d double_to_float fail: %08x != %08x\n", i, g1, e1); - status = 1; - } - - if (g2 != e2) - { - printf("%3d float_to_double fail: %016llx != %016llx\n", i, - (unsigned long long)g2, - (unsigned long long)e2); - status = 1; - } - } - - return status; -} - - - - diff --git a/example_unions/Makefile b/example_unions/Makefile deleted file mode 100644 index 29514ca9..00000000 --- a/example_unions/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -CFLAGS=-ansi -Wall -Werror -I .. -g -O0 -DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h - -all: encode decode - ./encode 1 | ./decode - ./encode 2 | ./decode - ./encode 3 | ./decode - -clean: - rm -f encode unionproto.pb.h unionproto.pb.c - -%: %.c $(DEPS) unionproto.pb.h unionproto.pb.c - $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c unionproto.pb.c - -unionproto.pb.h unionproto.pb.c: unionproto.proto ../generator/nanopb_generator.py - protoc -I. -I../generator -I/usr/include -ounionproto.pb $< - python ../generator/nanopb_generator.py unionproto.pb diff --git a/example_unions/decode.c b/example_unions/decode.c deleted file mode 100644 index b9f4af55..00000000 --- a/example_unions/decode.c +++ /dev/null @@ -1,96 +0,0 @@ -/* This program reads a message from stdin, detects its type and decodes it. - */ - -#include -#include -#include - -#include -#include "unionproto.pb.h" - -/* This function reads manually the first tag from the stream and finds the - * corresponding message type. It doesn't yet decode the actual message. - * - * Returns a pointer to the MsgType_fields array, as an identifier for the - * message type. Returns null if the tag is of unknown type or an error occurs. - */ -const pb_field_t* decode_unionmessage_type(pb_istream_t *stream) -{ - pb_wire_type_t wire_type; - uint32_t tag; - bool eof; - - while (pb_decode_tag(stream, &wire_type, &tag, &eof)) - { - if (wire_type == PB_WT_STRING) - { - const pb_field_t *field; - for (field = UnionMessage_fields; field->tag != 0; field++) - { - if (field->tag == tag && (field->type & PB_LTYPE_SUBMESSAGE)) - { - /* Found our field. */ - return field->ptr; - } - } - } - - /* Wasn't our field.. */ - pb_skip_field(stream, wire_type); - } - - return NULL; -} - -bool decode_unionmessage_contents(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) -{ - pb_istream_t substream; - bool status; - if (!pb_make_string_substream(stream, &substream)) - return false; - - status = pb_decode(&substream, fields, dest_struct); - pb_close_string_substream(stream, &substream); - return status; -} - -int main() -{ - /* Read the data into buffer */ - uint8_t buffer[512]; - size_t count = fread(buffer, 1, sizeof(buffer), stdin); - pb_istream_t stream = pb_istream_from_buffer(buffer, count); - - const pb_field_t *type = decode_unionmessage_type(&stream); - bool status = false; - - if (type == MsgType1_fields) - { - MsgType1 msg = {}; - status = decode_unionmessage_contents(&stream, MsgType1_fields, &msg); - printf("Got MsgType1: %d\n", msg.value); - } - else if (type == MsgType2_fields) - { - MsgType2 msg = {}; - status = decode_unionmessage_contents(&stream, MsgType2_fields, &msg); - printf("Got MsgType2: %s\n", msg.value ? "true" : "false"); - } - else if (type == MsgType3_fields) - { - MsgType3 msg = {}; - status = decode_unionmessage_contents(&stream, MsgType3_fields, &msg); - printf("Got MsgType3: %d %d\n", msg.value1, msg.value2); - } - - if (!status) - { - printf("Decode failed: %s\n", PB_GET_ERROR(&stream)); - return 1; - } - - return 0; -} - - - diff --git a/example_unions/encode.c b/example_unions/encode.c deleted file mode 100644 index e124bf91..00000000 --- a/example_unions/encode.c +++ /dev/null @@ -1,85 +0,0 @@ -/* This program takes a command line argument and encodes a message in - * one of MsgType1, MsgType2 or MsgType3. - */ - -#include -#include -#include - -#include -#include "unionproto.pb.h" - -/* This function is the core of the union encoding process. It handles - * the top-level pb_field_t array manually, in order to encode a correct - * field tag before the message. The pointer to MsgType_fields array is - * used as an unique identifier for the message type. - */ -bool encode_unionmessage(pb_ostream_t *stream, const pb_field_t messagetype[], const void *message) -{ - const pb_field_t *field; - for (field = UnionMessage_fields; field->tag != 0; field++) - { - if (field->ptr == messagetype) - { - /* This is our field, encode the message using it. */ - if (!pb_encode_tag_for_field(stream, field)) - return false; - - return pb_encode_submessage(stream, messagetype, message); - } - } - - /* Didn't find the field for messagetype */ - return false; -} - -int main(int argc, char **argv) -{ - if (argc != 2) - { - fprintf(stderr, "Usage: %s (1|2|3)\n", argv[0]); - return 1; - } - - uint8_t buffer[512]; - pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); - - bool status = false; - int msgtype = atoi(argv[1]); - if (msgtype == 1) - { - /* Send message of type 1 */ - MsgType1 msg = {42}; - status = encode_unionmessage(&stream, MsgType1_fields, &msg); - } - else if (msgtype == 2) - { - /* Send message of type 2 */ - MsgType2 msg = {true}; - status = encode_unionmessage(&stream, MsgType2_fields, &msg); - } - else if (msgtype == 3) - { - /* Send message of type 3 */ - MsgType3 msg = {3, 1415}; - status = encode_unionmessage(&stream, MsgType3_fields, &msg); - } - else - { - fprintf(stderr, "Unknown message type: %d\n", msgtype); - return 2; - } - - if (!status) - { - fprintf(stderr, "Encoding failed!\n"); - return 3; - } - else - { - fwrite(buffer, 1, stream.bytes_written, stdout); - return 0; /* Success */ - } -} - - diff --git a/example_unions/unionproto.proto b/example_unions/unionproto.proto deleted file mode 100644 index d7c9de2d..00000000 --- a/example_unions/unionproto.proto +++ /dev/null @@ -1,30 +0,0 @@ -// This is an example of how to handle 'union' style messages -// with nanopb, without allocating memory for all the message types. -// -// There is no official type in Protocol Buffers for describing unions, -// but they are commonly implemented by filling out exactly one of -// several optional fields. - -message MsgType1 -{ - required int32 value = 1; -} - -message MsgType2 -{ - required bool value = 1; -} - -message MsgType3 -{ - required int32 value1 = 1; - required int32 value2 = 2; -} - -message UnionMessage -{ - optional MsgType1 msg1 = 1; - optional MsgType2 msg2 = 2; - optional MsgType3 msg3 = 3; -} - diff --git a/examples/network_server/Makefile b/examples/network_server/Makefile new file mode 100644 index 00000000..981f2cf9 --- /dev/null +++ b/examples/network_server/Makefile @@ -0,0 +1,19 @@ +CFLAGS = -ansi -Wall -Werror -g -O0 + +# Path to the nanopb root folder +NANOPB_DIR = ../.. +DEPS = $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_decode.h \ + $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_encode.h $(NANOPB_DIR)/pb.h +CFLAGS += -I$(NANOPB_DIR) + +all: server client + +clean: + rm -f server client fileproto.pb.c fileproto.pb.h + +%: %.c $(DEPS) fileproto.pb.h fileproto.pb.c + $(CC) $(CFLAGS) -o $@ $< $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_encode.c fileproto.pb.c common.c + +fileproto.pb.c fileproto.pb.h: fileproto.proto $(NANOPB_DIR)/generator/nanopb_generator.py + protoc -ofileproto.pb $< + python $(NANOPB_DIR)/generator/nanopb_generator.py fileproto.pb diff --git a/examples/network_server/README b/examples/network_server/README new file mode 100644 index 00000000..7bdcbed5 --- /dev/null +++ b/examples/network_server/README @@ -0,0 +1,60 @@ +Nanopb example "network_server" +=============================== + +This example demonstrates the use of nanopb to communicate over network +connections. It consists of a server that sends file listings, and of +a client that requests the file list from the server. + +Example usage +------------- + +user@host:~/nanopb/examples/network_server$ make # Build the example +protoc -ofileproto.pb fileproto.proto +python ../../generator/nanopb_generator.py fileproto.pb +Writing to fileproto.pb.h and fileproto.pb.c +cc -ansi -Wall -Werror -I .. -g -O0 -I../.. -o server server.c + ../../pb_decode.c ../../pb_encode.c fileproto.pb.c common.c +cc -ansi -Wall -Werror -I .. -g -O0 -I../.. -o client client.c + ../../pb_decode.c ../../pb_encode.c fileproto.pb.c common.c + +user@host:~/nanopb/examples/network_server$ ./server & # Start the server on background +[1] 24462 + +petteri@oddish:~/nanopb/examples/network_server$ ./client /bin # Request the server to list /bin +Got connection. +Listing directory: /bin +1327119 bzdiff +1327126 bzless +1327147 ps +1327178 ntfsmove +1327271 mv +1327187 mount +1327259 false +1327266 tempfile +1327285 zfgrep +1327165 gzexe +1327204 nc.openbsd +1327260 uname + + +Details of implementation +------------------------- +fileproto.proto contains the portable Google Protocol Buffers protocol definition. +It could be used as-is to implement a server or a client in any other language, for +example Python or Java. + +fileproto.options contains the nanopb-specific options for the protocol file. This +sets the amount of space allocated for file names when decoding messages. + +common.c/h contains functions that allow nanopb to read and write directly from +network socket. This way there is no need to allocate a separate buffer to store +the message. + +server.c contains the code to open a listening socket, to respond to clients and +to list directory contents. + +client.c contains the code to connect to a server, to send a request and to print +the response message. + +The code is implemented using the POSIX socket api, but it should be easy enough +to port into any other socket api, such as lwip. diff --git a/examples/network_server/client.c b/examples/network_server/client.c new file mode 100644 index 00000000..e6e9a2e0 --- /dev/null +++ b/examples/network_server/client.c @@ -0,0 +1,116 @@ +/* This is a simple TCP client that connects to port 1234 and prints a list + * of files in a given directory. + * + * It directly deserializes and serializes messages from network, minimizing + * memory use. + * + * For flexibility, this example is implemented using posix api. + * In a real embedded system you would typically use some other kind of + * a communication and filesystem layer. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "fileproto.pb.h" +#include "common.h" + +bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + FileInfo fileinfo; + + if (!pb_decode(stream, FileInfo_fields, &fileinfo)) + return false; + + printf("%-10lld %s\n", (long long)fileinfo.inode, fileinfo.name); + + return true; +} + +bool listdir(int fd, char *path) +{ + ListFilesRequest request; + ListFilesResponse response; + pb_istream_t input = pb_istream_from_socket(fd); + pb_ostream_t output = pb_ostream_from_socket(fd); + uint8_t zero = 0; + + if (path == NULL) + { + request.has_path = false; + } + else + { + request.has_path = true; + if (strlen(path) + 1 > sizeof(request.path)) + { + fprintf(stderr, "Too long path.\n"); + return false; + } + + strcpy(request.path, path); + } + + if (!pb_encode(&output, ListFilesRequest_fields, &request)) + { + fprintf(stderr, "Encoding failed.\n"); + return false; + } + + /* We signal the end of request with a 0 tag. */ + pb_write(&output, &zero, 1); + + response.file.funcs.decode = &printfile_callback; + + if (!pb_decode(&input, ListFilesResponse_fields, &response)) + { + fprintf(stderr, "Decode failed: %s\n", PB_GET_ERROR(&input)); + return false; + } + + if (response.path_error) + { + fprintf(stderr, "Server reported error.\n"); + return false; + } + + return true; +} + +int main(int argc, char **argv) +{ + int sockfd; + struct sockaddr_in servaddr; + char *path = NULL; + + if (argc > 1) + path = argv[1]; + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + + memset(&servaddr, 0, sizeof(servaddr)); + servaddr.sin_family = AF_INET; + servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + servaddr.sin_port = htons(1234); + + if (connect(sockfd, (struct sockaddr *)&servaddr, sizeof(servaddr)) != 0) + { + perror("connect"); + return 1; + } + + if (!listdir(sockfd, path)) + return 2; + + close(sockfd); + + return 0; +} diff --git a/examples/network_server/common.c b/examples/network_server/common.c new file mode 100644 index 00000000..04a5aa85 --- /dev/null +++ b/examples/network_server/common.c @@ -0,0 +1,40 @@ +/* Simple binding of nanopb streams to TCP sockets. + */ + +#include +#include +#include +#include + +#include "common.h" + +static bool write_callback(pb_ostream_t *stream, const uint8_t *buf, size_t count) +{ + int fd = (intptr_t)stream->state; + return send(fd, buf, count, 0) == count; +} + +static bool read_callback(pb_istream_t *stream, uint8_t *buf, size_t count) +{ + int fd = (intptr_t)stream->state; + int result; + + result = recv(fd, buf, count, MSG_WAITALL); + + if (result == 0) + stream->bytes_left = 0; /* EOF */ + + return result == count; +} + +pb_ostream_t pb_ostream_from_socket(int fd) +{ + pb_ostream_t stream = {&write_callback, (void*)(intptr_t)fd, SIZE_MAX, 0}; + return stream; +} + +pb_istream_t pb_istream_from_socket(int fd) +{ + pb_istream_t stream = {&read_callback, (void*)(intptr_t)fd, SIZE_MAX}; + return stream; +} diff --git a/examples/network_server/common.h b/examples/network_server/common.h new file mode 100644 index 00000000..8dab3b7c --- /dev/null +++ b/examples/network_server/common.h @@ -0,0 +1,9 @@ +#ifndef _PB_EXAMPLE_COMMON_H_ +#define _PB_EXAMPLE_COMMON_H_ + +#include + +pb_ostream_t pb_ostream_from_socket(int fd); +pb_istream_t pb_istream_from_socket(int fd); + +#endif \ No newline at end of file diff --git a/examples/network_server/fileproto.options b/examples/network_server/fileproto.options new file mode 100644 index 00000000..29a2ab0e --- /dev/null +++ b/examples/network_server/fileproto.options @@ -0,0 +1,13 @@ +# This file defines the nanopb-specific options for the messages defined +# in fileproto.proto. +# +# If you come from high-level programming background, the hardcoded +# maximum lengths may disgust you. However, if your microcontroller only +# has a few kB of ram to begin with, setting reasonable limits for +# filenames is ok. +# +# On the other hand, using the callback interface, it is not necessary +# to set a limit on the number of files in the response. + +ListFilesRequest.path max_size:128 +FileInfo.name max_size:128 diff --git a/examples/network_server/fileproto.proto b/examples/network_server/fileproto.proto new file mode 100644 index 00000000..3e70c492 --- /dev/null +++ b/examples/network_server/fileproto.proto @@ -0,0 +1,18 @@ +// This defines protocol for a simple server that lists files. +// +// See also the nanopb-specific options in fileproto.options. + +message ListFilesRequest { + optional string path = 1 [default = "/"]; +} + +message FileInfo { + required uint64 inode = 1; + required string name = 2; +} + +message ListFilesResponse { + optional bool path_error = 1 [default = false]; + repeated FileInfo file = 2; +} + diff --git a/examples/network_server/server.c b/examples/network_server/server.c new file mode 100644 index 00000000..9a9c2644 --- /dev/null +++ b/examples/network_server/server.c @@ -0,0 +1,131 @@ +/* This is a simple TCP server that listens on port 1234 and provides lists + * of files to clients, using a protocol defined in file_server.proto. + * + * It directly deserializes and serializes messages from network, minimizing + * memory use. + * + * For flexibility, this example is implemented using posix api. + * In a real embedded system you would typically use some other kind of + * a communication and filesystem layer. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "fileproto.pb.h" +#include "common.h" + +bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + DIR *dir = (DIR*) *arg; + struct dirent *file; + FileInfo fileinfo; + + while ((file = readdir(dir)) != NULL) + { + fileinfo.inode = file->d_ino; + strncpy(fileinfo.name, file->d_name, sizeof(fileinfo.name)); + fileinfo.name[sizeof(fileinfo.name) - 1] = '\0'; + + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!pb_encode_submessage(stream, FileInfo_fields, &fileinfo)) + return false; + } + + return true; +} + +void handle_connection(int connfd) +{ + ListFilesRequest request; + ListFilesResponse response; + pb_istream_t input = pb_istream_from_socket(connfd); + pb_ostream_t output = pb_ostream_from_socket(connfd); + DIR *directory; + + if (!pb_decode(&input, ListFilesRequest_fields, &request)) + { + printf("Decode failed: %s\n", PB_GET_ERROR(&input)); + return; + } + + directory = opendir(request.path); + + printf("Listing directory: %s\n", request.path); + + if (directory == NULL) + { + perror("opendir"); + + response.has_path_error = true; + response.path_error = true; + response.file.funcs.encode = NULL; + } + else + { + response.has_path_error = false; + response.file.funcs.encode = &listdir_callback; + response.file.arg = directory; + } + + if (!pb_encode(&output, ListFilesResponse_fields, &response)) + { + printf("Encoding failed.\n"); + } +} + +int main(int argc, char **argv) +{ + int listenfd, connfd; + struct sockaddr_in servaddr; + int reuse = 1; + + listenfd = socket(AF_INET, SOCK_STREAM, 0); + + setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + + memset(&servaddr, 0, sizeof(servaddr)); + servaddr.sin_family = AF_INET; + servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + servaddr.sin_port = htons(1234); + if (bind(listenfd, (struct sockaddr*)&servaddr, sizeof(servaddr)) != 0) + { + perror("bind"); + return 1; + } + + if (listen(listenfd, 5) != 0) + { + perror("listen"); + return 1; + } + + for(;;) + { + connfd = accept(listenfd, NULL, NULL); + + if (connfd < 0) + { + perror("accept"); + return 1; + } + + printf("Got connection.\n"); + + handle_connection(connfd); + + printf("Closing connection.\n"); + + close(connfd); + } +} diff --git a/examples/using_double_on_avr/Makefile b/examples/using_double_on_avr/Makefile new file mode 100644 index 00000000..0b5383fa --- /dev/null +++ b/examples/using_double_on_avr/Makefile @@ -0,0 +1,29 @@ +CFLAGS = -Wall -Werror -g -O0 + +# Path to the nanopb root directory +NANOPB_DIR = ../.. +DEPS = double_conversion.c $(NANOPB_DIR)/pb.h \ + $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_decode.h \ + $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_encode.h +CFLAGS += -I$(NANOPB_DIR) + +all: run_tests + +clean: + rm -f test_conversions encode_double decode_double doubleproto.pb.c doubleproto.pb.h + +test_conversions: test_conversions.c double_conversion.c + $(CC) $(CFLAGS) -o $@ $^ + +%: %.c $(DEPS) doubleproto.pb.h doubleproto.pb.c + $(CC) $(CFLAGS) -o $@ $< double_conversion.c \ + $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_encode.c doubleproto.pb.c + +doubleproto.pb.c doubleproto.pb.h: doubleproto.proto $(NANOPB_DIR)/generator/nanopb_generator.py + protoc -odoubleproto.pb $< + python $(NANOPB_DIR)/generator/nanopb_generator.py doubleproto.pb + +run_tests: test_conversions encode_double decode_double + ./test_conversions + ./encode_double | ./decode_double + diff --git a/examples/using_double_on_avr/README b/examples/using_double_on_avr/README new file mode 100644 index 00000000..d9fcdfc6 --- /dev/null +++ b/examples/using_double_on_avr/README @@ -0,0 +1,25 @@ +Nanopb example "using_double_on_avr" +==================================== + +Some processors/compilers, such as AVR-GCC, do not support the double +datatype. Instead, they have sizeof(double) == 4. Because protocol +binary format uses the double encoding directly, this causes trouble +if the protocol in .proto requires double fields. + +This directory contains a solution to this problem. It uses uint64_t +to store the raw wire values, because its size is correct on all +platforms. The file double_conversion.c provides functions that +convert these values to/from floats, without relying on compiler +support. + +To use this method, you need to make some modifications to your code: + +1) Change all 'double' fields into 'fixed64' in the .proto. + +2) Whenever writing to a 'double' field, use float_to_double(). + +3) Whenever reading a 'double' field, use double_to_float(). + +The conversion routines are as accurate as the float datatype can +be. Furthermore, they should handle all special values (NaN, inf, denormalized +numbers) correctly. There are testcases in test_conversions.c. diff --git a/examples/using_double_on_avr/decode_double.c b/examples/using_double_on_avr/decode_double.c new file mode 100644 index 00000000..5802eca7 --- /dev/null +++ b/examples/using_double_on_avr/decode_double.c @@ -0,0 +1,33 @@ +/* Decodes a double value into a float variable. + * Used to read double values with AVR code, which doesn't support double directly. + */ + +#include +#include +#include "double_conversion.h" +#include "doubleproto.pb.h" + +int main() +{ + uint8_t buffer[32]; + size_t count = fread(buffer, 1, sizeof(buffer), stdin); + pb_istream_t stream = pb_istream_from_buffer(buffer, count); + + AVRDoubleMessage message; + pb_decode(&stream, AVRDoubleMessage_fields, &message); + + float v1 = double_to_float(message.field1); + float v2 = double_to_float(message.field2); + + printf("Values: %f %f\n", v1, v2); + + if (v1 == 1234.5678f && + v2 == 0.00001f) + { + return 0; + } + else + { + return 1; + } +} diff --git a/examples/using_double_on_avr/double_conversion.c b/examples/using_double_on_avr/double_conversion.c new file mode 100644 index 00000000..cf79b9a0 --- /dev/null +++ b/examples/using_double_on_avr/double_conversion.c @@ -0,0 +1,123 @@ +/* Conversion routines for platforms that do not support 'double' directly. */ + +#include "double_conversion.h" +#include + +typedef union { + float f; + uint32_t i; +} conversion_t; + +/* Note: IEE 754 standard specifies float formats as follows: + * Single precision: sign, 8-bit exp, 23-bit frac. + * Double precision: sign, 11-bit exp, 52-bit frac. + */ + +uint64_t float_to_double(float value) +{ + conversion_t in; + in.f = value; + uint8_t sign; + int16_t exponent; + uint64_t mantissa; + + /* Decompose input value */ + sign = (in.i >> 31) & 1; + exponent = ((in.i >> 23) & 0xFF) - 127; + mantissa = in.i & 0x7FFFFF; + + if (exponent == 128) + { + /* Special value (NaN etc.) */ + exponent = 1024; + } + else if (exponent == -127) + { + if (!mantissa) + { + /* Zero */ + exponent = -1023; + } + else + { + /* Denormalized */ + mantissa <<= 1; + while (!(mantissa & 0x800000)) + { + mantissa <<= 1; + exponent--; + } + mantissa &= 0x7FFFFF; + } + } + + /* Combine fields */ + mantissa <<= 29; + mantissa |= (uint64_t)(exponent + 1023) << 52; + mantissa |= (uint64_t)sign << 63; + + return mantissa; +} + +float double_to_float(uint64_t value) +{ + uint8_t sign; + int16_t exponent; + uint32_t mantissa; + conversion_t out; + + /* Decompose input value */ + sign = (value >> 63) & 1; + exponent = ((value >> 52) & 0x7FF) - 1023; + mantissa = (value >> 28) & 0xFFFFFF; /* Highest 24 bits */ + + /* Figure if value is in range representable by floats. */ + if (exponent == 1024) + { + /* Special value */ + exponent = 128; + } + else if (exponent > 127) + { + /* Too large */ + if (sign) + return -INFINITY; + else + return INFINITY; + } + else if (exponent < -150) + { + /* Too small */ + if (sign) + return -0.0f; + else + return 0.0f; + } + else if (exponent < -126) + { + /* Denormalized */ + mantissa |= 0x1000000; + mantissa >>= (-126 - exponent); + exponent = -127; + } + + /* Round off mantissa */ + mantissa = (mantissa + 1) >> 1; + + /* Check if mantissa went over 2.0 */ + if (mantissa & 0x800000) + { + exponent += 1; + mantissa &= 0x7FFFFF; + mantissa >>= 1; + } + + /* Combine fields */ + out.i = mantissa; + out.i |= (uint32_t)(exponent + 127) << 23; + out.i |= (uint32_t)sign << 31; + + return out.f; +} + + diff --git a/examples/using_double_on_avr/double_conversion.h b/examples/using_double_on_avr/double_conversion.h new file mode 100644 index 00000000..62b6a8ae --- /dev/null +++ b/examples/using_double_on_avr/double_conversion.h @@ -0,0 +1,26 @@ +/* AVR-GCC does not have real double datatype. Instead its double + * is equal to float, i.e. 32 bit value. If you need to communicate + * with other systems that use double in their .proto files, you + * need to do some conversion. + * + * These functions use bitwise operations to mangle floats into doubles + * and then store them in uint64_t datatype. + */ + +#ifndef DOUBLE_CONVERSION +#define DOUBLE_CONVERSION + +#include + +/* Convert native 4-byte float into a 8-byte double. */ +extern uint64_t float_to_double(float value); + +/* Convert 8-byte double into native 4-byte float. + * Values are rounded to nearest, 0.5 away from zero. + * Overflowing values are converted to Inf or -Inf. + */ +extern float double_to_float(uint64_t value); + + +#endif + diff --git a/examples/using_double_on_avr/doubleproto.proto b/examples/using_double_on_avr/doubleproto.proto new file mode 100644 index 00000000..d8b7f2db --- /dev/null +++ b/examples/using_double_on_avr/doubleproto.proto @@ -0,0 +1,13 @@ +// A message containing doubles, as used by other applications. +message DoubleMessage { + required double field1 = 1; + required double field2 = 2; +} + +// A message containing doubles, but redefined using uint64_t. +// For use in AVR code. +message AVRDoubleMessage { + required fixed64 field1 = 1; + required fixed64 field2 = 2; +} + diff --git a/examples/using_double_on_avr/encode_double.c b/examples/using_double_on_avr/encode_double.c new file mode 100644 index 00000000..cd532d46 --- /dev/null +++ b/examples/using_double_on_avr/encode_double.c @@ -0,0 +1,25 @@ +/* Encodes a float value into a double on the wire. + * Used to emit doubles from AVR code, which doesn't support double directly. + */ + +#include +#include +#include "double_conversion.h" +#include "doubleproto.pb.h" + +int main() +{ + AVRDoubleMessage message = { + float_to_double(1234.5678f), + float_to_double(0.00001f) + }; + + uint8_t buffer[32]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + pb_encode(&stream, AVRDoubleMessage_fields, &message); + fwrite(buffer, 1, stream.bytes_written, stdout); + + return 0; +} + diff --git a/examples/using_double_on_avr/test_conversions.c b/examples/using_double_on_avr/test_conversions.c new file mode 100644 index 00000000..22620a6a --- /dev/null +++ b/examples/using_double_on_avr/test_conversions.c @@ -0,0 +1,56 @@ +#include "double_conversion.h" +#include +#include + +static const double testvalues[] = { + 0.0, -0.0, 0.1, -0.1, + M_PI, -M_PI, 123456.789, -123456.789, + INFINITY, -INFINITY, NAN, INFINITY - INFINITY, + 1e38, -1e38, 1e39, -1e39, + 1e-38, -1e-38, 1e-39, -1e-39, + 3.14159e-37,-3.14159e-37, 3.14159e-43, -3.14159e-43, + 1e-60, -1e-60, 1e-45, -1e-45, + 0.99999999999999, -0.99999999999999, 127.999999999999, -127.999999999999 +}; + +#define TESTVALUES_COUNT (sizeof(testvalues)/sizeof(testvalues[0])) + +int main() +{ + int status = 0; + int i; + for (i = 0; i < TESTVALUES_COUNT; i++) + { + double orig = testvalues[i]; + float expected_float = (float)orig; + double expected_double = (double)expected_float; + + float got_float = double_to_float(*(uint64_t*)&orig); + uint64_t got_double = float_to_double(got_float); + + uint32_t e1 = *(uint32_t*)&expected_float; + uint32_t g1 = *(uint32_t*)&got_float; + uint64_t e2 = *(uint64_t*)&expected_double; + uint64_t g2 = got_double; + + if (g1 != e1) + { + printf("%3d double_to_float fail: %08x != %08x\n", i, g1, e1); + status = 1; + } + + if (g2 != e2) + { + printf("%3d float_to_double fail: %016llx != %016llx\n", i, + (unsigned long long)g2, + (unsigned long long)e2); + status = 1; + } + } + + return status; +} + + + + diff --git a/examples/using_union_messages/Makefile b/examples/using_union_messages/Makefile new file mode 100644 index 00000000..0f7b5206 --- /dev/null +++ b/examples/using_union_messages/Makefile @@ -0,0 +1,22 @@ +CFLAGS = -ansi -Wall -Werror -g -O0 + +# Path to the nanopb root folder +NANOPB_DIR = ../.. +DEPS = $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_decode.h \ + $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_encode.h $(NANOPB_DIR)/pb.h +CFLAGS += -I$(NANOPB_DIR) + +all: encode decode + ./encode 1 | ./decode + ./encode 2 | ./decode + ./encode 3 | ./decode + +clean: + rm -f encode unionproto.pb.h unionproto.pb.c + +%: %.c $(DEPS) unionproto.pb.h unionproto.pb.c + $(CC) $(CFLAGS) -o $@ $< $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_encode.c unionproto.pb.c + +unionproto.pb.h unionproto.pb.c: unionproto.proto $(NANOPB_DIR)/generator/nanopb_generator.py + protoc -ounionproto.pb $< + python $(NANOPB_DIR)/generator/nanopb_generator.py unionproto.pb diff --git a/examples/using_union_messages/README b/examples/using_union_messages/README new file mode 100644 index 00000000..7a1e75d4 --- /dev/null +++ b/examples/using_union_messages/README @@ -0,0 +1,52 @@ +Nanopb example "using_union_messages" +===================================== + +Union messages is a common technique in Google Protocol Buffers used to +represent a group of messages, only one of which is passed at a time. +It is described in Google's documentation: +https://developers.google.com/protocol-buffers/docs/techniques#union + +This directory contains an example on how to encode and decode union messages +with minimal memory usage. Usually, nanopb would allocate space to store +all of the possible messages at the same time, even though at most one of +them will be used at a time. + +By using some of the lower level nanopb APIs, we can manually generate the +top level message, so that we only need to allocate the one submessage that +we actually want. Similarly when decoding, we can manually read the tag of +the top level message, and only then allocate the memory for the submessage +after we already know its type. + + +Example usage +------------- + +Type `make` to run the example. It will build it and run commands like +following: + +./encode 1 | ./decode +Got MsgType1: 42 +./encode 2 | ./decode +Got MsgType2: true +./encode 3 | ./decode +Got MsgType3: 3 1415 + +This simply demonstrates that the "decode" program has correctly identified +the type of the received message, and managed to decode it. + + +Details of implementation +------------------------- + +unionproto.proto contains the protocol used in the example. It consists of +three messages: MsgType1, MsgType2 and MsgType3, which are collected together +into UnionMessage. + +encode.c takes one command line argument, which should be a number 1-3. It +then fills in and encodes the corresponding message, and writes it to stdout. + +decode.c reads a UnionMessage from stdin. Then it calls the function +decode_unionmessage_type() to determine the type of the message. After that, +the corresponding message is decoded and the contents of it printed to the +screen. + diff --git a/examples/using_union_messages/decode.c b/examples/using_union_messages/decode.c new file mode 100644 index 00000000..b9f4af55 --- /dev/null +++ b/examples/using_union_messages/decode.c @@ -0,0 +1,96 @@ +/* This program reads a message from stdin, detects its type and decodes it. + */ + +#include +#include +#include + +#include +#include "unionproto.pb.h" + +/* This function reads manually the first tag from the stream and finds the + * corresponding message type. It doesn't yet decode the actual message. + * + * Returns a pointer to the MsgType_fields array, as an identifier for the + * message type. Returns null if the tag is of unknown type or an error occurs. + */ +const pb_field_t* decode_unionmessage_type(pb_istream_t *stream) +{ + pb_wire_type_t wire_type; + uint32_t tag; + bool eof; + + while (pb_decode_tag(stream, &wire_type, &tag, &eof)) + { + if (wire_type == PB_WT_STRING) + { + const pb_field_t *field; + for (field = UnionMessage_fields; field->tag != 0; field++) + { + if (field->tag == tag && (field->type & PB_LTYPE_SUBMESSAGE)) + { + /* Found our field. */ + return field->ptr; + } + } + } + + /* Wasn't our field.. */ + pb_skip_field(stream, wire_type); + } + + return NULL; +} + +bool decode_unionmessage_contents(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + pb_istream_t substream; + bool status; + if (!pb_make_string_substream(stream, &substream)) + return false; + + status = pb_decode(&substream, fields, dest_struct); + pb_close_string_substream(stream, &substream); + return status; +} + +int main() +{ + /* Read the data into buffer */ + uint8_t buffer[512]; + size_t count = fread(buffer, 1, sizeof(buffer), stdin); + pb_istream_t stream = pb_istream_from_buffer(buffer, count); + + const pb_field_t *type = decode_unionmessage_type(&stream); + bool status = false; + + if (type == MsgType1_fields) + { + MsgType1 msg = {}; + status = decode_unionmessage_contents(&stream, MsgType1_fields, &msg); + printf("Got MsgType1: %d\n", msg.value); + } + else if (type == MsgType2_fields) + { + MsgType2 msg = {}; + status = decode_unionmessage_contents(&stream, MsgType2_fields, &msg); + printf("Got MsgType2: %s\n", msg.value ? "true" : "false"); + } + else if (type == MsgType3_fields) + { + MsgType3 msg = {}; + status = decode_unionmessage_contents(&stream, MsgType3_fields, &msg); + printf("Got MsgType3: %d %d\n", msg.value1, msg.value2); + } + + if (!status) + { + printf("Decode failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + + return 0; +} + + + diff --git a/examples/using_union_messages/encode.c b/examples/using_union_messages/encode.c new file mode 100644 index 00000000..e124bf91 --- /dev/null +++ b/examples/using_union_messages/encode.c @@ -0,0 +1,85 @@ +/* This program takes a command line argument and encodes a message in + * one of MsgType1, MsgType2 or MsgType3. + */ + +#include +#include +#include + +#include +#include "unionproto.pb.h" + +/* This function is the core of the union encoding process. It handles + * the top-level pb_field_t array manually, in order to encode a correct + * field tag before the message. The pointer to MsgType_fields array is + * used as an unique identifier for the message type. + */ +bool encode_unionmessage(pb_ostream_t *stream, const pb_field_t messagetype[], const void *message) +{ + const pb_field_t *field; + for (field = UnionMessage_fields; field->tag != 0; field++) + { + if (field->ptr == messagetype) + { + /* This is our field, encode the message using it. */ + if (!pb_encode_tag_for_field(stream, field)) + return false; + + return pb_encode_submessage(stream, messagetype, message); + } + } + + /* Didn't find the field for messagetype */ + return false; +} + +int main(int argc, char **argv) +{ + if (argc != 2) + { + fprintf(stderr, "Usage: %s (1|2|3)\n", argv[0]); + return 1; + } + + uint8_t buffer[512]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + bool status = false; + int msgtype = atoi(argv[1]); + if (msgtype == 1) + { + /* Send message of type 1 */ + MsgType1 msg = {42}; + status = encode_unionmessage(&stream, MsgType1_fields, &msg); + } + else if (msgtype == 2) + { + /* Send message of type 2 */ + MsgType2 msg = {true}; + status = encode_unionmessage(&stream, MsgType2_fields, &msg); + } + else if (msgtype == 3) + { + /* Send message of type 3 */ + MsgType3 msg = {3, 1415}; + status = encode_unionmessage(&stream, MsgType3_fields, &msg); + } + else + { + fprintf(stderr, "Unknown message type: %d\n", msgtype); + return 2; + } + + if (!status) + { + fprintf(stderr, "Encoding failed!\n"); + return 3; + } + else + { + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } +} + + diff --git a/examples/using_union_messages/unionproto.proto b/examples/using_union_messages/unionproto.proto new file mode 100644 index 00000000..d7c9de2d --- /dev/null +++ b/examples/using_union_messages/unionproto.proto @@ -0,0 +1,30 @@ +// This is an example of how to handle 'union' style messages +// with nanopb, without allocating memory for all the message types. +// +// There is no official type in Protocol Buffers for describing unions, +// but they are commonly implemented by filling out exactly one of +// several optional fields. + +message MsgType1 +{ + required int32 value = 1; +} + +message MsgType2 +{ + required bool value = 1; +} + +message MsgType3 +{ + required int32 value1 = 1; + required int32 value2 = 2; +} + +message UnionMessage +{ + optional MsgType1 msg1 = 1; + optional MsgType2 msg2 = 2; + optional MsgType3 msg3 = 3; +} + -- cgit 1.2.3-korg From 73142ba082e56b8c5ba75c055a20c200b623da83 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 13 Sep 2013 13:35:25 +0300 Subject: Add a new very simple example --- docs/index.rst | 29 ++++++++++++++----- examples/simple/Makefile | 22 ++++++++++++++ examples/simple/README | 30 +++++++++++++++++++ examples/simple/simple.c | 68 ++++++++++++++++++++++++++++++++++++++++++++ examples/simple/simple.proto | 7 +++++ 5 files changed, 149 insertions(+), 7 deletions(-) create mode 100644 examples/simple/Makefile create mode 100644 examples/simple/README create mode 100644 examples/simple/simple.c create mode 100644 examples/simple/simple.proto diff --git a/docs/index.rst b/docs/index.rst index e56ff4cc..27840f7d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -90,22 +90,37 @@ After that, buffer will contain the encoded message. The number of bytes in the message is stored in *stream.bytes_written*. You can feed the message to *protoc --decode=Example message.proto* to verify its validity. -For complete examples of the simple cases, see *tests/test_decode1.c* and *tests/test_encode1.c*. For an example with network interface, see the *example* subdirectory. +For a complete example of the simple case, see *example/simple.c*. +For a more complex example with network interface, see the *example/network_server* subdirectory. Compiler requirements ===================== -Nanopb should compile with most ansi-C compatible compilers. It however requires a few header files to be available: +Nanopb should compile with most ansi-C compatible compilers. It however +requires a few header files to be available: #) *string.h*, with these functions: *strlen*, *memcpy*, *memset* #) *stdint.h*, for definitions of *int32_t* etc. #) *stddef.h*, for definition of *size_t* #) *stdbool.h*, for definition of *bool* -If these header files do not come with your compiler, you should be able to find suitable replacements online. Mostly the requirements are very simple, just a few basic functions and typedefs. +If these header files do not come with your compiler, you can use the +file *compat/pb_syshdr.h* instead. It contains an example of how to provide +the dependencies. You may have to edit it a bit to suit your custom platform. -Alternatively, you can define *PB_SYSTEM_HEADER*, which should be the name of a single header file including all the necessary definitions. +To use the pb_syshdr.h, define *PB_SYSTEM_HEADER* to be the name of your custom +header file. It should provide all the dependencies listed above. -Debugging and testing -===================== -Extensive unittests are included under the *tests* folder. Just type *make* there to run the tests. +Running the test cases +====================== +Extensive unittests and test cases are included under the *tests* folder. + +To build the tests, you will need the `scons`__ build system. The tests should +be runnable on most platforms. Windows and Linux builds are regularly tested. + +__ http://www.scons.org/ + +In addition to the build system, you will also need a working Google Protocol +Buffers *protoc* compiler, and the Python bindings for Protocol Buffers. On +Debian-based systems, install the following packages: *protobuf-compiler*, +*python-protobuf* and *libprotobuf-dev*. diff --git a/examples/simple/Makefile b/examples/simple/Makefile new file mode 100644 index 00000000..2be3a047 --- /dev/null +++ b/examples/simple/Makefile @@ -0,0 +1,22 @@ +# Compiler flags to enable all warnings & debug info +CFLAGS = -Wall -Werror -g -O0 + +# Path to the nanopb root folder +NANOPB_DIR = ../.. +CFLAGS += -I$(NANOPB_DIR) + +# C source code files that are required +CSRC = simple.c # The main program +CSRC += simple.pb.c # The compiled protocol definition +CSRC += $(NANOPB_DIR)/pb_encode.c # The nanopb encoder +CSRC += $(NANOPB_DIR)/pb_decode.c # The nanopb decoder + +# Build rule for the main program +simple: $(CSRC) + $(CC) $(CFLAGS) -osimple $(CSRC) + +# Build rule for the protocol +simple.pb.c: simple.proto + protoc -osimple.pb simple.proto + python $(NANOPB_DIR)/generator/nanopb_generator.py simple.pb + diff --git a/examples/simple/README b/examples/simple/README new file mode 100644 index 00000000..d18c4183 --- /dev/null +++ b/examples/simple/README @@ -0,0 +1,30 @@ +Nanopb example "simple" +======================= + +This example demonstrates the very basic use of nanopb. It encodes and +decodes a simple message. + +The code uses four different API functions: + + * pb_ostream_from_buffer() to declare the output buffer that is to be used + * pb_encode() to encode a message + * pb_istream_from_buffer() to declare the input buffer that is to be used + * pb_decode() to decode a message + +Example usage +------------- + +On Linux, simply type "make" to build the example. After that, you can +run it with the command: ./simple + +On other platforms, you first have to compile the protocol definition using +the following two commands:: + + protoc -osimple.pb simple.proto + python nanopb_generator.py simple.pb + +After that, add the following four files to your project and compile: + + simple.c simple.pb.c pb_encode.c pb_decode.c + + diff --git a/examples/simple/simple.c b/examples/simple/simple.c new file mode 100644 index 00000000..31272301 --- /dev/null +++ b/examples/simple/simple.c @@ -0,0 +1,68 @@ +#include +#include +#include +#include "simple.pb.h" + +int main() +{ + /* This is the buffer where we will store our message. */ + uint8_t buffer[128]; + size_t message_length; + bool status; + + /* Encode our message */ + { + /* Allocate space on the stack to store the message data. + * + * Nanopb generates simple struct definitions for all the messages. + * - check out the contents of simple.pb.h! */ + SimpleMessage message; + + /* Create a stream that will write to our buffer. */ + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Fill in the lucky number */ + message.lucky_number = 13; + + /* Now we are ready to encode the message! */ + status = pb_encode(&stream, SimpleMessage_fields, &message); + message_length = stream.bytes_written; + + /* Then just check for any errors.. */ + if (!status) + { + printf("Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + } + + /* Now we could transmit the message over network, store it in a file or + * wrap it to a pigeon's leg. + */ + + /* But because we are lazy, we will just decode it immediately. */ + + { + /* Allocate space for the decoded message. */ + SimpleMessage message; + + /* Create a stream that reads from the buffer. */ + pb_istream_t stream = pb_istream_from_buffer(buffer, message_length); + + /* Now we are ready to decode the message. */ + status = pb_decode(&stream, SimpleMessage_fields, &message); + + /* Check for errors... */ + if (!status) + { + printf("Decoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + + /* Print the data contained in the message. */ + printf("Your lucky number was %d!\n", message.lucky_number); + } + + return 0; +} + diff --git a/examples/simple/simple.proto b/examples/simple/simple.proto new file mode 100644 index 00000000..26e72f46 --- /dev/null +++ b/examples/simple/simple.proto @@ -0,0 +1,7 @@ +// A very simple protocol definition, consisting of only +// one message. + +message SimpleMessage { + required int32 lucky_number = 1; +} + -- cgit 1.2.3-korg From 8accc25710cdb0839649d7356caa8aae1212cb8b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 13 Sep 2013 13:41:57 +0300 Subject: Update changelog --- CHANGELOG | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index 6dfefec8..eff979ab 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,12 @@ +nanopb-0.2.3 + Improve compatibility by removing ternary operator from initializations (issue 88) + Fix build error on Visual C++ (issue 84, patch by Markus Schwarzenberg) + Don't stop on unsupported extension fields (issue 83) + Add an example pb_syshdr.h file for non-C99 compilers + Reorganize tests and examples into subfolders (issue 63) + Switch from Makefiles to scons for building the tests + Make the tests buildable on Windows + nanopb-0.2.2 Add support for extension fields (issue 17) Fix unknown fields in empty message (issue 78) -- cgit 1.2.3-korg From 283a8f36cb1ad989765e0dda2071a2a15a12af10 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 18 Sep 2013 12:44:46 +0300 Subject: Publishing nanopb-0.2.3 --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 2e30b676..0eff79ff 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.3-dev" +nanopb_version = "nanopb-0.2.3" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index c0c25283..9974b65c 100644 --- a/pb.h +++ b/pb.h @@ -43,7 +43,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.3-dev +#define NANOPB_VERSION nanopb-0.2.3 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 2363af29a238fb4fa17474c979a0f45cde617a9e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 18 Sep 2013 12:47:32 +0300 Subject: Setting version to 0.2.4-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 0eff79ff..645d5fd7 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.3" +nanopb_version = "nanopb-0.2.4-dev" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index 9974b65c..8b501dec 100644 --- a/pb.h +++ b/pb.h @@ -43,7 +43,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.3 +#define NANOPB_VERSION nanopb-0.2.4-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 388d4de833cf4e2127b2ab0489cb6b14ecc0cbb5 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 26 Sep 2013 10:23:37 +0300 Subject: Add #defines for the maximum encoded message size. Update issue 89 Status: FixedInGit --- generator/nanopb_generator.py | 111 +++++++++++++++++++++++++----- tests/alltypes/encode_alltypes.c | 2 +- tests/basic_buffer/decode_buffer.c | 2 +- tests/basic_buffer/encode_buffer.c | 2 +- tests/encode_unittests/encode_unittests.c | 13 ++++ 5 files changed, 111 insertions(+), 19 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 645d5fd7..130ff932 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -38,22 +38,22 @@ except: import time import os.path -# Values are tuple (c type, pb type) +# Values are tuple (c type, pb type, encoded size) FieldD = descriptor.FieldDescriptorProto datatypes = { - FieldD.TYPE_BOOL: ('bool', 'BOOL'), - FieldD.TYPE_DOUBLE: ('double', 'DOUBLE'), - FieldD.TYPE_FIXED32: ('uint32_t', 'FIXED32'), - FieldD.TYPE_FIXED64: ('uint64_t', 'FIXED64'), - FieldD.TYPE_FLOAT: ('float', 'FLOAT'), - FieldD.TYPE_INT32: ('int32_t', 'INT32'), - FieldD.TYPE_INT64: ('int64_t', 'INT64'), - FieldD.TYPE_SFIXED32: ('int32_t', 'SFIXED32'), - FieldD.TYPE_SFIXED64: ('int64_t', 'SFIXED64'), - FieldD.TYPE_SINT32: ('int32_t', 'SINT32'), - FieldD.TYPE_SINT64: ('int64_t', 'SINT64'), - FieldD.TYPE_UINT32: ('uint32_t', 'UINT32'), - FieldD.TYPE_UINT64: ('uint64_t', 'UINT64') + FieldD.TYPE_BOOL: ('bool', 'BOOL', 1), + FieldD.TYPE_DOUBLE: ('double', 'DOUBLE', 8), + FieldD.TYPE_FIXED32: ('uint32_t', 'FIXED32', 4), + FieldD.TYPE_FIXED64: ('uint64_t', 'FIXED64', 8), + FieldD.TYPE_FLOAT: ('float', 'FLOAT', 4), + FieldD.TYPE_INT32: ('int32_t', 'INT32', 5), + FieldD.TYPE_INT64: ('int64_t', 'INT64', 10), + FieldD.TYPE_SFIXED32: ('int32_t', 'SFIXED32', 4), + FieldD.TYPE_SFIXED64: ('int64_t', 'SFIXED64', 8), + FieldD.TYPE_SINT32: ('int32_t', 'SINT32', 5), + FieldD.TYPE_SINT64: ('int64_t', 'SINT64', 10), + FieldD.TYPE_UINT32: ('uint32_t', 'UINT32', 5), + FieldD.TYPE_UINT64: ('uint64_t', 'UINT64', 10) } class Names: @@ -83,6 +83,17 @@ def names_from_type_name(type_name): raise NotImplementedError("Lookup of non-absolute type names is not supported") return Names(type_name[1:].split('.')) +def varint_max_size(max_value): + '''Returns the maximum number of bytes a varint can take when encoded.''' + for i in range(1, 11): + if (max_value >> (i * 7)) == 0: + return i + raise ValueError("Value too large for varint: " + str(max_value)) + +assert varint_max_size(0) == 1 +assert varint_max_size(127) == 1 +assert varint_max_size(128) == 2 + class Enum: def __init__(self, names, desc, enum_options): '''desc is EnumDescriptorProto''' @@ -113,6 +124,7 @@ class Field: self.max_size = None self.max_count = None self.array_decl = "" + self.enc_size = None # Parse field options if field_options.HasField("max_size"): @@ -141,12 +153,13 @@ class Field: # Decide the C data type to use in the struct. if datatypes.has_key(desc.type): - self.ctype, self.pbtype = datatypes[desc.type] + self.ctype, self.pbtype, self.enc_size = datatypes[desc.type] elif desc.type == FieldD.TYPE_ENUM: self.pbtype = 'ENUM' self.ctype = names_from_type_name(desc.type_name) if self.default is not None: self.default = self.ctype + self.default + self.enc_size = 5 # protoc rejects enum values > 32 bits elif desc.type == FieldD.TYPE_STRING: self.pbtype = 'STRING' if self.max_size is None: @@ -154,15 +167,18 @@ class Field: else: self.ctype = 'char' self.array_decl += '[%d]' % self.max_size + self.enc_size = varint_max_size(self.max_size) + self.max_size elif desc.type == FieldD.TYPE_BYTES: self.pbtype = 'BYTES' if self.max_size is None: can_be_static = False else: self.ctype = self.struct_name + self.name + 't' + self.enc_size = varint_max_size(self.max_size) + self.max_size elif desc.type == FieldD.TYPE_MESSAGE: self.pbtype = 'MESSAGE' self.ctype = self.submsgname = names_from_type_name(desc.type_name) + self.enc_size = None # Needs to be filled in after the message type is available else: raise NotImplementedError(desc.type) @@ -277,6 +293,42 @@ class Field: return max(self.tag, self.max_size, self.max_count) + def encoded_size(self, allmsgs): + '''Return the maximum size that this field can take when encoded, + including the field tag. If the size cannot be determined, returns + None.''' + + if self.allocation != 'STATIC': + return None + + encsize = self.enc_size + if self.pbtype == 'MESSAGE': + for msg in allmsgs: + if msg.name == self.submsgname: + encsize = msg.encoded_size(allmsgs) + if encsize is None: + return None # Submessage size is indeterminate + encsize += varint_max_size(encsize) # submsg length is encoded also + break + else: + # Submessage cannot be found, this currently occurs when + # the submessage type is defined in a different file. + return None + + if encsize is None: + raise RuntimeError("Could not determine encoded size for %s.%s" + % (self.struct_name, self.name)) + + encsize += varint_max_size(self.tag << 3) # Tag + wire type + + if self.rules == 'REPEATED': + # Decoders must be always able to handle unpacked arrays. + # Therefore we have to reserve space for it, even though + # we emit packed arrays ourselves. + encsize *= self.max_count + + return encsize + class ExtensionRange(Field): def __init__(self, struct_name, range_start, field_options): @@ -305,6 +357,12 @@ class ExtensionRange(Field): def tags(self): return '' + + def encoded_size(self, allmsgs): + # We exclude extensions from the count, because they cannot be known + # until runtime. Other option would be to return None here, but this + # way the value remains useful if extensions are not used. + return 0 class ExtensionField(Field): def __init__(self, struct_name, desc, field_options): @@ -429,6 +487,18 @@ class Message: result += ' PB_LAST_FIELD\n};' return result + def encoded_size(self, allmsgs): + '''Return the maximum size that this message can take when encoded. + If the size cannot be determined, returns None. + ''' + size = 0 + for field in self.fields: + fsize = field.encoded_size(allmsgs) + if fsize is None: + return None + size += fsize + + return size # --------------------------------------------------------------------------- @@ -597,8 +667,17 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio yield '/* Struct field encoding specification for nanopb */\n' for msg in messages: yield msg.fields_declaration() + '\n' + yield '\n' - yield '\n#ifdef __cplusplus\n' + yield '/* Maximum encoded size of messages (where known) */\n' + for msg in messages: + msize = msg.encoded_size(messages) + if msize is not None: + identifier = '%s_size' % msg.name + yield '#define %-40s %d\n' % (identifier, msize) + yield '\n' + + yield '#ifdef __cplusplus\n' yield '} /* extern "C" */\n' yield '#endif\n' diff --git a/tests/alltypes/encode_alltypes.c b/tests/alltypes/encode_alltypes.c index 88fc10f0..9a2c6f60 100644 --- a/tests/alltypes/encode_alltypes.c +++ b/tests/alltypes/encode_alltypes.c @@ -115,7 +115,7 @@ int main(int argc, char **argv) alltypes.end = 1099; { - uint8_t buffer[1024]; + uint8_t buffer[AllTypes_size]; pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); /* Now encode it and check if we succeeded. */ diff --git a/tests/basic_buffer/decode_buffer.c b/tests/basic_buffer/decode_buffer.c index d231c916..fae9e2fa 100644 --- a/tests/basic_buffer/decode_buffer.c +++ b/tests/basic_buffer/decode_buffer.c @@ -60,7 +60,7 @@ bool print_person(pb_istream_t *stream) int main() { - uint8_t buffer[512]; + uint8_t buffer[Person_size]; pb_istream_t stream; size_t count; diff --git a/tests/basic_buffer/encode_buffer.c b/tests/basic_buffer/encode_buffer.c index d3e4f6e6..c412c14e 100644 --- a/tests/basic_buffer/encode_buffer.c +++ b/tests/basic_buffer/encode_buffer.c @@ -10,7 +10,7 @@ int main() { - uint8_t buffer[512]; + uint8_t buffer[Person_size]; pb_ostream_t stream; /* Initialize the structure with constants */ diff --git a/tests/encode_unittests/encode_unittests.c b/tests/encode_unittests/encode_unittests.c index c3634ac8..32a37bf1 100644 --- a/tests/encode_unittests/encode_unittests.c +++ b/tests/encode_unittests/encode_unittests.c @@ -280,6 +280,19 @@ int main() TEST(!pb_encode(&s, CallbackContainerContainer_fields, &msg2)) } + { + uint8_t buffer[StringMessage_size]; + pb_ostream_t s; + StringMessage msg = {"0123456789"}; + + s = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + COMMENT("Test that StringMessage_size is correct") + + TEST(pb_encode(&s, StringMessage_fields, &msg)); + TEST(s.bytes_written == StringMessage_size); + } + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); -- cgit 1.2.3-korg From e83fbd18d3fa33af8a57198cddefe5fcef1b2f58 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 20 Oct 2013 21:42:00 +0300 Subject: Check array max size when encoding. Update issue 90 Status: FixedInGit --- pb_encode.c | 3 +++ tests/encode_unittests/encode_unittests.c | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/pb_encode.c b/pb_encode.c index d6ba7e34..4aced3cb 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -94,6 +94,9 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie if (count == 0) return true; + + if (count > field->array_size) + PB_RETURN_ERROR(stream, "array max size exceeded"); /* We always pack arrays if the datatype allows it. */ if (PB_LTYPE(field->type) <= PB_LTYPE_LAST_PACKABLE) diff --git a/tests/encode_unittests/encode_unittests.c b/tests/encode_unittests/encode_unittests.c index 32a37bf1..14bc62ee 100644 --- a/tests/encode_unittests/encode_unittests.c +++ b/tests/encode_unittests/encode_unittests.c @@ -223,6 +223,20 @@ int main() TEST(!pb_encode(&s, FloatArray_fields, &msg)) } + { + uint8_t buffer[50]; + pb_ostream_t s; + FloatArray msg = {1, {99.0f}}; + + COMMENT("Test array size limit in pb_encode") + + s = pb_ostream_from_buffer(buffer, sizeof(buffer)); + TEST((msg.data_count = 10) && pb_encode(&s, FloatArray_fields, &msg)) + + s = pb_ostream_from_buffer(buffer, sizeof(buffer)); + TEST((msg.data_count = 11) && !pb_encode(&s, FloatArray_fields, &msg)) + } + { uint8_t buffer[10]; pb_ostream_t s; -- cgit 1.2.3-korg From 2bfd497eea857d19a09054c5c54f3718b08dc8c9 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 20 Oct 2013 21:49:55 +0300 Subject: Define pb_size_t and pb_ssize_t data types. Use these in pb_field_t definition to clean up some #ifs, and also to prepare for solving issue #82. --- pb.h | 41 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/pb.h b/pb.h index 8b501dec..04a8288b 100644 --- a/pb.h +++ b/pb.h @@ -173,6 +173,20 @@ typedef uint8_t pb_type_t; #define PB_HTYPE(x) ((x) & PB_HTYPE_MASK) #define PB_LTYPE(x) ((x) & PB_LTYPE_MASK) +/* Data type used for storing sizes of struct fields + * and array counts. + */ +#if defined(PB_FIELD_32BIT) + typedef uint32_t pb_size_t; + typedef int32_t pb_ssize_t; +#elif defined(PB_FIELD_16BIT) + typedef uint16_t pb_size_t; + typedef int16_t pb_ssize_t; +#else + typedef uint8_t pb_size_t; + typedef int8_t pb_ssize_t; +#endif + /* This structure is used in auto-generated constants * to specify struct fields. * You can change field sizes if you need structures @@ -184,29 +198,12 @@ typedef uint8_t pb_type_t; PB_PACKED_STRUCT_START typedef struct _pb_field_t pb_field_t; struct _pb_field_t { - -#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) - uint8_t tag; + pb_size_t tag; pb_type_t type; - uint8_t data_offset; /* Offset of field data, relative to previous field. */ - int8_t size_offset; /* Offset of array size or has-boolean, relative to data */ - uint8_t data_size; /* Data size in bytes for a single item */ - uint8_t array_size; /* Maximum number of entries in array */ -#elif defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) - uint16_t tag; - pb_type_t type; - uint8_t data_offset; - int8_t size_offset; - uint16_t data_size; - uint16_t array_size; -#else - uint32_t tag; - pb_type_t type; - uint8_t data_offset; - int8_t size_offset; - uint32_t data_size; - uint32_t array_size; -#endif + pb_size_t data_offset; /* Offset of field data, relative to previous field. */ + pb_ssize_t size_offset; /* Offset of array size or has-boolean, relative to data */ + pb_size_t data_size; /* Data size in bytes for a single item */ + pb_size_t array_size; /* Maximum number of entries in array */ /* Field definitions for submessage * OR default value for all other non-array, non-callback types -- cgit 1.2.3-korg From 49bd3f35a0b6db0fa47d2e6e8fe9ddfb1bbcd58b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 23 Oct 2013 21:01:11 +0300 Subject: Generate message size #defines also for messages defined in multiple files. Add testcase for the same. --- generator/nanopb_generator.py | 58 ++++++++++++++++++++++++++++++++----- tests/message_sizes/SConscript | 11 +++++++ tests/message_sizes/dummy.c | 9 ++++++ tests/message_sizes/messages1.proto | 27 +++++++++++++++++ tests/message_sizes/messages2.proto | 8 +++++ 5 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 tests/message_sizes/SConscript create mode 100644 tests/message_sizes/dummy.c create mode 100644 tests/message_sizes/messages1.proto create mode 100644 tests/message_sizes/messages2.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 130ff932..ebb020b7 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -94,6 +94,44 @@ assert varint_max_size(0) == 1 assert varint_max_size(127) == 1 assert varint_max_size(128) == 2 +class EncodedSize: + '''Class used to represent the encoded size of a field or a message. + Consists of a combination of symbolic sizes and integer sizes.''' + def __init__(self, value = 0, symbols = []): + if isinstance(value, (str, Names)): + symbols = [str(value)] + value = 0 + self.value = value + self.symbols = symbols + + def __add__(self, other): + if isinstance(other, int): + return EncodedSize(self.value + other, self.symbols) + elif isinstance(other, (str, Names)): + return EncodedSize(self.value, self.symbols + [str(other)]) + elif isinstance(other, EncodedSize): + return EncodedSize(self.value + other.value, self.symbols + other.symbols) + else: + raise ValueError("Cannot add size: " + repr(other)) + + def __mul__(self, other): + if isinstance(other, int): + return EncodedSize(self.value * other, [str(other) + '*' + s for s in self.symbols]) + else: + raise ValueError("Cannot multiply size: " + repr(other)) + + def __str__(self): + if not self.symbols: + return str(self.value) + else: + return '(' + str(self.value) + ' + ' + ' + '.join(self.symbols) + ')' + + def upperlimit(self): + if not self.symbols: + return self.value + else: + return 2**32 - 1 + class Enum: def __init__(self, names, desc, enum_options): '''desc is EnumDescriptorProto''' @@ -301,23 +339,27 @@ class Field: if self.allocation != 'STATIC': return None - encsize = self.enc_size if self.pbtype == 'MESSAGE': for msg in allmsgs: if msg.name == self.submsgname: encsize = msg.encoded_size(allmsgs) if encsize is None: return None # Submessage size is indeterminate - encsize += varint_max_size(encsize) # submsg length is encoded also + + # Include submessage length prefix + encsize += varint_max_size(encsize.upperlimit()) break else: # Submessage cannot be found, this currently occurs when # the submessage type is defined in a different file. - return None - - if encsize is None: + # Instead of direct numeric value, reference the size that + # has been #defined in the other file. + encsize = EncodedSize(self.submsgname + 'size') + elif self.enc_size is None: raise RuntimeError("Could not determine encoded size for %s.%s" % (self.struct_name, self.name)) + else: + encsize = EncodedSize(self.enc_size) encsize += varint_max_size(self.tag << 3) # Tag + wire type @@ -362,7 +404,7 @@ class ExtensionRange(Field): # We exclude extensions from the count, because they cannot be known # until runtime. Other option would be to return None here, but this # way the value remains useful if extensions are not used. - return 0 + return EncodedSize(0) class ExtensionField(Field): def __init__(self, struct_name, desc, field_options): @@ -491,7 +533,7 @@ class Message: '''Return the maximum size that this message can take when encoded. If the size cannot be determined, returns None. ''' - size = 0 + size = EncodedSize(0) for field in self.fields: fsize = field.encoded_size(allmsgs) if fsize is None: @@ -674,7 +716,7 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio msize = msg.encoded_size(messages) if msize is not None: identifier = '%s_size' % msg.name - yield '#define %-40s %d\n' % (identifier, msize) + yield '#define %-40s %s\n' % (identifier, msize) yield '\n' yield '#ifdef __cplusplus\n' diff --git a/tests/message_sizes/SConscript b/tests/message_sizes/SConscript new file mode 100644 index 00000000..e7524e02 --- /dev/null +++ b/tests/message_sizes/SConscript @@ -0,0 +1,11 @@ +# Test the generation of message size #defines + +Import('env') + +incpath = env.Clone() +incpath.Append(PROTOCPATH = '#message_sizes') + +incpath.NanopbProto("messages1") +incpath.NanopbProto("messages2") + +incpath.Program(['dummy.c', 'messages1.pb.c', 'messages2.pb.c']) diff --git a/tests/message_sizes/dummy.c b/tests/message_sizes/dummy.c new file mode 100644 index 00000000..767ad463 --- /dev/null +++ b/tests/message_sizes/dummy.c @@ -0,0 +1,9 @@ +/* Just test that the file can be compiled successfully. */ + +#include "messages2.pb.h" + +int main() +{ + return xmit_size; +} + diff --git a/tests/message_sizes/messages1.proto b/tests/message_sizes/messages1.proto new file mode 100644 index 00000000..48af55a3 --- /dev/null +++ b/tests/message_sizes/messages1.proto @@ -0,0 +1,27 @@ +enum MessageStatus { + FAIL = 0; + OK = 1; +}; + +message MessageInfo { + required fixed32 msg_id = 1; + optional fixed32 interface_id = 2; +} + +message MessageResponseInfo { + required fixed64 interface_id = 1; + required fixed32 seq = 2; + required fixed32 msg_id = 3; +} + +message MessageHeader { + required MessageInfo info = 1; + optional MessageResponseInfo response_info = 2; + optional MessageResponse response = 3; +} + +message MessageResponse { + required MessageStatus status = 1; + required fixed32 seq = 2; +} + diff --git a/tests/message_sizes/messages2.proto b/tests/message_sizes/messages2.proto new file mode 100644 index 00000000..19fc11ef --- /dev/null +++ b/tests/message_sizes/messages2.proto @@ -0,0 +1,8 @@ +import 'nanopb.proto'; +import 'messages1.proto'; + +message xmit { + required MessageHeader header = 1; + required bytes data = 2 [(nanopb).max_size = 128]; +} + -- cgit 1.2.3-korg From 51f0e47295d537a113804f250cddef37a57237a8 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 23 Oct 2013 21:21:43 +0300 Subject: Fix the size of length prefix for messages in other files. --- generator/nanopb_generator.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index ebb020b7..0f14a041 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -355,6 +355,11 @@ class Field: # Instead of direct numeric value, reference the size that # has been #defined in the other file. encsize = EncodedSize(self.submsgname + 'size') + + # We will have to make a conservative assumption on the length + # prefix size, though. + encsize += 5 + elif self.enc_size is None: raise RuntimeError("Could not determine encoded size for %s.%s" % (self.struct_name, self.name)) -- cgit 1.2.3-korg From 0848255d4a4b7fa032e962592b15c57b30cd8483 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 24 Oct 2013 08:44:59 +0300 Subject: Handle also longs in EncodedSize --- generator/nanopb_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 0f14a041..aa2c1b38 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -105,7 +105,7 @@ class EncodedSize: self.symbols = symbols def __add__(self, other): - if isinstance(other, int): + if isinstance(other, (int, long)): return EncodedSize(self.value + other, self.symbols) elif isinstance(other, (str, Names)): return EncodedSize(self.value, self.symbols + [str(other)]) -- cgit 1.2.3-korg From 86d698315608c372868bb55f6d2c609600ac8e41 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 24 Oct 2013 09:52:40 +0300 Subject: Same fix for EncodedSize.__mul__ --- generator/nanopb_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index aa2c1b38..89647c2c 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -115,7 +115,7 @@ class EncodedSize: raise ValueError("Cannot add size: " + repr(other)) def __mul__(self, other): - if isinstance(other, int): + if isinstance(other, (int, long)): return EncodedSize(self.value * other, [str(other) + '*' + s for s in self.symbols]) else: raise ValueError("Cannot multiply size: " + repr(other)) -- cgit 1.2.3-korg From ed564186e14c79c767096f4b306dc3c6c5bd2e7d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 24 Oct 2013 21:45:39 +0300 Subject: Detect invalid sizes when encoding bytes fields. --- pb_encode.c | 5 ++++- tests/common/unittestproto.proto | 4 ++++ tests/encode_unittests/encode_unittests.c | 18 ++++++++++++++++-- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/pb_encode.c b/pb_encode.c index 4aced3cb..563c1bb7 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -521,7 +521,10 @@ bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, c bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) { const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src; - UNUSED(field); + + if (bytes->size + offsetof(pb_bytes_array_t, bytes) > field->data_size) + PB_RETURN_ERROR(stream, "bytes size exceeded"); + return pb_encode_string(stream, bytes->bytes, bytes->size); } diff --git a/tests/common/unittestproto.proto b/tests/common/unittestproto.proto index 7024942e..eb3e7dec 100644 --- a/tests/common/unittestproto.proto +++ b/tests/common/unittestproto.proto @@ -12,6 +12,10 @@ message StringMessage { required string data = 1 [(nanopb).max_size = 10]; } +message BytesMessage { + required bytes data = 1 [(nanopb).max_size = 16]; +} + message CallbackArray { // We cheat a bit and use this message for testing other types, too. // Nanopb does not care about the actual defined data type for callback diff --git a/tests/encode_unittests/encode_unittests.c b/tests/encode_unittests/encode_unittests.c index 14bc62ee..fd9a730c 100644 --- a/tests/encode_unittests/encode_unittests.c +++ b/tests/encode_unittests/encode_unittests.c @@ -172,9 +172,9 @@ int main() struct { size_t size; uint8_t bytes[5]; } value = {5, {'x', 'y', 'z', 'z', 'y'}}; COMMENT("Test pb_enc_bytes") - TEST(WRITES(pb_enc_bytes(&s, NULL, &value), "\x05xyzzy")) + TEST(WRITES(pb_enc_bytes(&s, &BytesMessage_fields[0], &value), "\x05xyzzy")) value.size = 0; - TEST(WRITES(pb_enc_bytes(&s, NULL, &value), "\x00")) + TEST(WRITES(pb_enc_bytes(&s, &BytesMessage_fields[0], &value), "\x00")) } { @@ -258,6 +258,20 @@ int main() "\x0A\x07\x0A\x05\x01\x02\x03\x04\x05")) } + { + uint8_t buffer[32]; + pb_ostream_t s; + BytesMessage msg = {{3, "xyz"}}; + + COMMENT("Test pb_encode with bytes message.") + TEST(WRITES(pb_encode(&s, BytesMessage_fields, &msg), + "\x0A\x03xyz")) + + msg.data.size = 17; /* More than maximum */ + TEST(!pb_encode(&s, BytesMessage_fields, &msg)) + } + + { uint8_t buffer[20]; pb_ostream_t s; -- cgit 1.2.3-korg From cd3af3272d170f4990ef745316faa7600571ef4e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 29 Oct 2013 15:32:51 +0200 Subject: Rename some internal functions to have unique names --- pb_decode.c | 4 ++-- pb_encode.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index e3be4127..9ad77e94 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -478,7 +478,7 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t /* Default handler for extension fields. Expects a pb_field_t structure * in extension->type->arg. */ -static bool checkreturn default_extension_handler(pb_istream_t *stream, +static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type) { const pb_field_t *field = (const pb_field_t*)extension->type->arg; @@ -513,7 +513,7 @@ static bool checkreturn decode_extension(pb_istream_t *stream, if (extension->type->decode) status = extension->type->decode(stream, extension, tag, wire_type); else - status = default_extension_handler(stream, extension, tag, wire_type); + status = default_extension_decoder(stream, extension, tag, wire_type); if (!status) return false; diff --git a/pb_encode.c b/pb_encode.c index 563c1bb7..e97d04fb 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -245,7 +245,7 @@ static bool checkreturn encode_field(pb_ostream_t *stream, /* Default handler for extension fields. Expects to have a pb_field_t * pointer in the extension->type->arg field. */ -static bool checkreturn default_extension_handler(pb_ostream_t *stream, +static bool checkreturn default_extension_encoder(pb_ostream_t *stream, const pb_extension_t *extension) { const pb_field_t *field = (const pb_field_t*)extension->type->arg; @@ -266,7 +266,7 @@ static bool checkreturn encode_extension_field(pb_ostream_t *stream, if (extension->type->encode) status = extension->type->encode(stream, extension); else - status = default_extension_handler(stream, extension); + status = default_extension_encoder(stream, extension); if (!status) return false; -- cgit 1.2.3-korg From c7b4ce0293a3e5bc1857ec76cee246d505b154e4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 29 Oct 2013 15:44:35 +0200 Subject: Add a definition of the security model to the documentation. --- docs/Makefile | 2 +- docs/menu.rst | 3 ++- docs/security.rst | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 docs/security.rst diff --git a/docs/Makefile b/docs/Makefile index cfd84ac0..e7143b25 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,4 +1,4 @@ -all: index.html concepts.html reference.html \ +all: index.html concepts.html reference.html security.html \ generator_flow.png %.png: %.svg diff --git a/docs/menu.rst b/docs/menu.rst index 4c643bc8..a49b22c1 100644 --- a/docs/menu.rst +++ b/docs/menu.rst @@ -3,8 +3,9 @@ 1) `Overview`_ 2) `Concepts`_ 3) `API reference`_ + 4) `Security model`_ .. _`Overview`: index.html .. _`Concepts`: concepts.html .. _`API reference`: reference.html - +.. _`Security model`: security.html diff --git a/docs/security.rst b/docs/security.rst new file mode 100644 index 00000000..e865f839 --- /dev/null +++ b/docs/security.rst @@ -0,0 +1,79 @@ +====================== +Nanopb: Security model +====================== + +.. include :: menu.rst + +.. contents :: + + + +Importance of security in a Protocol Buffers library +==================================================== +In the context of protocol buffers, security comes into play when decoding +untrusted data. Naturally, if the attacker can modify the contents of a +protocol buffers message, he can feed the application any values possible. +Therefore the application itself must be prepared to receive untrusted values. + +Where nanopb plays a part is preventing the attacker from running arbitrary +code on the target system. Mostly this means that there must not be any +possibility to cause buffer overruns, memory corruption or invalid pointers +by the means of crafting a malicious message. + +Division of trusted and untrusted data +====================================== +The following data is regarded as **trusted**. It must be under the control of +the application writer. Malicious data in these structures could cause +security issues, such as execution of arbitrary code: + +1. Callback and extension fields in message structures given to pb_encode() + and pb_decode(). These fields are memory pointers, and are generated + depending on the .proto file. +2. The automatically generated field definitions, i.e. *pb_field_t* lists. +3. Contents of the *pb_istream_t* and *pb_ostream_t* structures (this does not + mean the contents of the stream itself, just the stream definition). + +The following data is regarded as **untrusted**. Invalid/malicious data in +these will cause "garbage in, garbage out" behaviour. It will not cause +buffer overflows, information disclosure or other security problems: + +1. All data read from *pb_istream_t*. +2. All fields in message structures, except callbacks and extensions. + (Beginning with nanopb-0.2.4, in earlier versions the field sizes are partially unchecked.) + +Invariants +========== +The following invariants are maintained during operation, even if the +untrusted data has been maliciously crafted: + +1. Nanopb will never read more than *bytes_left* bytes from *pb_istream_t*. +2. Nanopb will never write more than *max_size* bytes to *pb_ostream_t*. +3. Nanopb will never access memory out of bounds of the message structure. +4. After pb_decode() returns successfully, the message structure will be + internally consistent: + + - The *count* fields of arrays will not exceed the array size. + - The *size* field of bytes will not exceed the allocated size. + - All string fields will have null terminator. + +5. After pb_encode() returns successfully, the resulting message is a valid + protocol buffers message. (Except if user-defined callbacks write incorrect + data.) + +Further considerations +====================== +Even if the nanopb library is free of any security issues, there are still +several possible attack vectors that the application author must consider. +The following list is not comprehensive: + +1. Stack usage may depend on the contents of the message. The message + definition places an upper bound on how much stack will be used. Tests + should be run with all fields present, to record the maximum possible + stack usage. +2. Callbacks can do anything. The code for the callbacks must be carefully + checked if they are used with untrusted data. +3. If using stream input, a maximum size should be set in *pb_istream_t* to + stop a denial of service attack from using an infinite message. +4. If using network sockets as streams, a timeout should be set to stop + denial of service attacks. + -- cgit 1.2.3-korg From 4d69cc2f3ef33af9b63495e59127884d48d12fb3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 29 Oct 2013 16:19:07 +0200 Subject: Cleanup of comments. --- pb_decode.c | 15 ++++++++------- pb_encode.c | 27 +++++++++++++++++++-------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 9ad77e94..370b2dc1 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -3,11 +3,13 @@ * 2011 Petteri Aimonen */ -/* The warn_unused_result attribute appeared first in gcc-3.4.0 */ +/* Use the GCC warn_unused_result attribute to check that all return values + * are propagated correctly. On other compilers and gcc before 3.4.0 just + * ignore the annotation. + */ #if !defined(__GNUC__) || ( __GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4) #define checkreturn #else - /* Verify that we remember to check all return values for proper error propagation */ #define checkreturn __attribute__((warn_unused_result)) #endif @@ -15,11 +17,10 @@ #include "pb.h" #include "pb_decode.h" -typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn; - /* --- Function pointers to field decoders --- * Order in the array must match pb_action_t LTYPE numbering. */ +typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn; static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { &pb_dec_varint, &pb_dec_svarint, @@ -32,9 +33,9 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { NULL /* extensions */ }; -/************** - * pb_istream * - **************/ +/******************************* + * pb_istream_t implementation * + *******************************/ static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count) { diff --git a/pb_encode.c b/pb_encode.c index e97d04fb..e933a0b1 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -7,19 +7,20 @@ #include "pb.h" #include "pb_encode.h" -/* The warn_unused_result attribute appeared first in gcc-3.4.0 */ +/* Use the GCC warn_unused_result attribute to check that all return values + * are propagated correctly. On other compilers and gcc before 3.4.0 just + * ignore the annotation. + */ #if !defined(__GNUC__) || ( __GNUC__ < 3) || (__GNUC__ == 3 && __GNUC_MINOR__ < 4) #define checkreturn #else - /* Verify that we remember to check all return values for proper error propagation */ #define checkreturn __attribute__((warn_unused_result)) #endif -typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, const void *src) checkreturn; - /* --- Function pointers to field encoders --- * Order in the array must match pb_action_t LTYPE numbering. */ +typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, const void *src) checkreturn; static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { &pb_enc_varint, &pb_enc_svarint, @@ -32,7 +33,9 @@ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { NULL /* extensions */ }; -/* pb_ostream_t implementation */ +/******************************* + * pb_ostream_t implementation * + *******************************/ static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) { @@ -49,7 +52,7 @@ pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize) { pb_ostream_t stream; #ifdef PB_BUFFER_ONLY - stream.callback = (void*)1; /* Just some marker value */ + stream.callback = (void*)1; /* Just a marker value */ #else stream.callback = &buf_write; #endif @@ -82,7 +85,9 @@ bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count return true; } -/* Main encoding stuff */ +/************************* + * Encode a single field * + *************************/ /* Encode a static array. Handles the size calculations and possible packing. */ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, @@ -277,6 +282,10 @@ static bool checkreturn encode_extension_field(pb_ostream_t *stream, return true; } +/********************* + * Encode all fields * + *********************/ + bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) { const pb_field_t *field = fields; @@ -319,7 +328,9 @@ bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const return pb_encode_submessage(stream, fields, src_struct); } -/* Helper functions */ +/******************** + * Helper functions * + ********************/ bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value) { uint8_t buffer[10]; -- cgit 1.2.3-korg From 0074deba9a2ff99c877abe1293f72a9ed76f46c1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 29 Oct 2013 16:24:50 +0200 Subject: Declare static functions before use. For compliance with MISRA C rules (issue 91). --- pb_decode.c | 43 +++++++++++++++++++++++++++++++------------ pb_encode.c | 13 ++++++++++++- 2 files changed, 43 insertions(+), 13 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 370b2dc1..f012e931 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -17,10 +17,40 @@ #include "pb.h" #include "pb_decode.h" +/************************************** + * Declarations internal to this file * + **************************************/ + +/* Iterator for pb_field_t list */ +typedef struct { + const pb_field_t *start; /* Start of the pb_field_t array */ + const pb_field_t *pos; /* Current position of the iterator */ + unsigned field_index; /* Zero-based index of the field. */ + unsigned required_field_index; /* Zero-based index that counts only the required fields */ + void *dest_struct; /* Pointer to the destination structure to decode to */ + void *pData; /* Pointer where to store current field value */ + void *pSize; /* Pointer where to store the size of current array field */ +} pb_field_iterator_t; + +typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn; + +static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count); +static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest); +static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, uint8_t *buf, size_t *size); +static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct); +static bool pb_field_next(pb_field_iterator_t *iter); +static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag); +static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter); +static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter); +static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter); +static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type); +static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_field_iterator_t *iter); +static bool checkreturn find_extension_field(pb_field_iterator_t *iter); +static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct); + /* --- Function pointers to field decoders --- * Order in the array must match pb_action_t LTYPE numbering. */ -typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn; static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { &pb_dec_varint, &pb_dec_svarint, @@ -278,17 +308,6 @@ void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) #endif } -/* Iterator for pb_field_t list */ -typedef struct { - const pb_field_t *start; /* Start of the pb_field_t array */ - const pb_field_t *pos; /* Current position of the iterator */ - unsigned field_index; /* Zero-based index of the field. */ - unsigned required_field_index; /* Zero-based index that counts only the required fields */ - void *dest_struct; /* Pointer to the destination structure to decode to */ - void *pData; /* Pointer where to store current field value */ - void *pSize; /* Pointer where to store the size of current array field */ -} pb_field_iterator_t; - static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct) { iter->start = iter->pos = fields; diff --git a/pb_encode.c b/pb_encode.c index e933a0b1..90236524 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -17,10 +17,21 @@ #define checkreturn __attribute__((warn_unused_result)) #endif +/************************************** + * Declarations internal to this file * + **************************************/ +typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, const void *src) checkreturn; + +static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); +static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, const void *pData, size_t count, pb_encoder_t func); +static bool checkreturn encode_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData); +static bool checkreturn default_extension_encoder(pb_ostream_t *stream, const pb_extension_t *extension); +static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData); + + /* --- Function pointers to field encoders --- * Order in the array must match pb_action_t LTYPE numbering. */ -typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, const void *src) checkreturn; static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { &pb_enc_varint, &pb_enc_svarint, -- cgit 1.2.3-korg From 287207841db5df93cf7ff9c71a5f1548deb26b09 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 29 Oct 2013 16:32:47 +0200 Subject: Remove the NANOPB_INTERNALS functions from public API. These have been deprecated since nanopb-0.1.6 (some since 0.1.3). Equivalent functions with better interface are available in the API. Update issue 91 Status: FixedInGit --- docs/reference.rst | 4 +++- pb_decode.c | 9 +++++++++ pb_decode.h | 19 ------------------- pb_encode.c | 8 +++++++- pb_encode.h | 18 ------------------ tests/SConstruct | 2 ++ tests/decode_unittests/SConscript | 2 +- tests/decode_unittests/decode_unittests.c | 4 ++-- tests/encode_unittests/SConscript | 2 +- tests/encode_unittests/encode_unittests.c | 4 ++-- 10 files changed, 27 insertions(+), 45 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 51556d35..32283734 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -25,7 +25,9 @@ __BIG_ENDIAN__ Set this if your platform stores integers and systems (different layout for ints and floats) are currently not supported. NANOPB_INTERNALS Set this to expose the field encoder functions - that are hidden since nanopb-0.1.3. + that are hidden since nanopb-0.1.3. Starting + with nanopb-0.2.4, this flag does nothing. Use + the newer functions that have better interface. PB_MAX_REQUIRED_FIELDS Maximum number of required fields to check for presence. Default value is 64. Increases stack usage 1 byte per every 8 fields. Compiler diff --git a/pb_decode.c b/pb_decode.c index f012e931..90fa18d2 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -47,6 +47,15 @@ static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_exten static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_field_iterator_t *iter); static bool checkreturn find_extension_field(pb_field_iterator_t *iter); static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct); +static bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest); +static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest); +static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest); +static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest); +static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); +static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); +static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest); +static bool checkreturn pb_skip_varint(pb_istream_t *stream); +static bool checkreturn pb_skip_string(pb_istream_t *stream); /* --- Function pointers to field decoders --- * Order in the array must match pb_action_t LTYPE numbering. diff --git a/pb_decode.h b/pb_decode.h index 98a64cc0..f71b5f1b 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -131,25 +131,6 @@ bool pb_decode_fixed64(pb_istream_t *stream, void *dest); bool pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream); void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream); - -/******************************* - * Internal / legacy functions * - *******************************/ - -#ifdef NANOPB_INTERNALS -bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest); - -bool pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); -bool pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest); - -bool pb_skip_varint(pb_istream_t *stream); -bool pb_skip_string(pb_istream_t *stream); -#endif - #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/pb_encode.c b/pb_encode.c index 90236524..ebf20de5 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -27,7 +27,13 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie static bool checkreturn encode_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData); static bool checkreturn default_extension_encoder(pb_ostream_t *stream, const pb_extension_t *extension); static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData); - +static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src); /* --- Function pointers to field encoders --- * Order in the array must match pb_action_t LTYPE numbering. diff --git a/pb_encode.h b/pb_encode.h index 3009820b..900994aa 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -143,24 +143,6 @@ bool pb_encode_fixed64(pb_ostream_t *stream, const void *value); */ bool pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); - -/******************************* - * Internal / legacy functions * - *******************************/ - -#ifdef NANOPB_INTERNALS -bool pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src); -bool pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); -bool pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src); -bool pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src); -bool pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src); -bool pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src); -#endif - -/* This function is not recommended for new programs. Use pb_encode_submessage() - * instead, it has the same functionality with a less confusing interface. */ -bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src); - #ifdef __cplusplus } /* extern "C" */ #endif diff --git a/tests/SConstruct b/tests/SConstruct index 3f4d7702..100fb544 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -78,6 +78,8 @@ if 'gcc' in env['CC']: # More strict checks on the nanopb core env.Append(CORECFLAGS = '-Wextra -Wcast-qual -Wlogical-op -Wconversion') + env.Append(CORECFLAGS = ' -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls') + env.Append(CORECFLAGS = ' -Wstack-protector') elif 'clang' in env['CC']: # CLang env.Append(CFLAGS = '-ansi -g -O0 -Wall -Werror') diff --git a/tests/decode_unittests/SConscript b/tests/decode_unittests/SConscript index 5e0f8407..9a639f7b 100644 --- a/tests/decode_unittests/SConscript +++ b/tests/decode_unittests/SConscript @@ -1,4 +1,4 @@ Import('env') -p = env.Program(["decode_unittests.c", "#common/unittestproto.pb.c", "#common/pb_decode.o"]) +p = env.Program(["decode_unittests.c", "#common/unittestproto.pb.c"]) env.RunTest(p) diff --git a/tests/decode_unittests/decode_unittests.c b/tests/decode_unittests/decode_unittests.c index 9c447a57..1be01913 100644 --- a/tests/decode_unittests/decode_unittests.c +++ b/tests/decode_unittests/decode_unittests.c @@ -1,8 +1,8 @@ -#define NANOPB_INTERNALS +/* This includes the whole .c file to get access to static functions. */ +#include "pb_decode.c" #include #include -#include "pb_decode.h" #include "unittests.h" #include "unittestproto.pb.h" diff --git a/tests/encode_unittests/SConscript b/tests/encode_unittests/SConscript index 6a5ffcff..7509ec03 100644 --- a/tests/encode_unittests/SConscript +++ b/tests/encode_unittests/SConscript @@ -1,5 +1,5 @@ # Build and run the stand-alone unit tests for the nanopb encoder part. Import('env') -p = env.Program(["encode_unittests.c", "#common/unittestproto.pb.c", "#common/pb_encode.o"]) +p = env.Program(["encode_unittests.c", "#common/unittestproto.pb.c"]) env.RunTest(p) diff --git a/tests/encode_unittests/encode_unittests.c b/tests/encode_unittests/encode_unittests.c index fd9a730c..edbc10a9 100644 --- a/tests/encode_unittests/encode_unittests.c +++ b/tests/encode_unittests/encode_unittests.c @@ -1,8 +1,8 @@ -#define NANOPB_INTERNALS +/* This includes the whole .c file to get access to static functions. */ +#include "pb_encode.c" #include #include -#include "pb_encode.h" #include "unittests.h" #include "unittestproto.pb.h" -- cgit 1.2.3-korg From dcdd7f98fdb000859a9e2812174a817431b1af34 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 2 Nov 2013 22:05:49 +0200 Subject: Generate #define tags for extension fields also. Update issue 93 Status: FixedInGit --- generator/nanopb_generator.py | 7 +++++++ tests/extensions/encode_extensions.c | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 89647c2c..3c248ac3 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -423,6 +423,11 @@ class ExtensionField(Field): self.skip = False self.rules = 'OPTEXT' + def tags(self): + '''Return the #define for the tag number of this field.''' + identifier = '%s_tag' % self.fullname + return '#define %-40s %d\n' % (identifier, self.tag) + def extension_decl(self): '''Declaration of the extension type in the .pb.h file''' if self.skip: @@ -709,6 +714,8 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio for msg in sort_dependencies(messages): for field in msg.fields: yield field.tags() + for extension in extensions: + yield extension.tags() yield '\n' yield '/* Struct field encoding specification for nanopb */\n' diff --git a/tests/extensions/encode_extensions.c b/tests/extensions/encode_extensions.c index dee3597d..00745826 100644 --- a/tests/extensions/encode_extensions.c +++ b/tests/extensions/encode_extensions.c @@ -46,5 +46,9 @@ int main(int argc, char **argv) fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); return 1; /* Failure */ } + + /* Check that the field tags are properly generated */ + (void)AllTypes_extensionfield1_tag; + (void)ExtensionMessage_AllTypes_extensionfield2_tag; } -- cgit 1.2.3-korg From 564bdc84488ac03120b6c51a14d9d7c2ee93a58c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 2 Nov 2013 22:11:27 +0200 Subject: Update changelog --- CHANGELOG | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG b/CHANGELOG index eff979ab..49a54acf 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,12 @@ +nanopb-0.2.4 + Remove the deprecated NANOPB_INTERNALS functions from public API. + Document the security model. + Check array and bytes max sizes when encoding (issue 90) + Add #defines for maximum encoded message size (issue 89) + Add #define tags for extension fields (issue 93) + Fix MISRA C violations (issue 91) + Clean up pb_field_t definition with typedefs. + nanopb-0.2.3 Improve compatibility by removing ternary operator from initializations (issue 88) Fix build error on Visual C++ (issue 84, patch by Markus Schwarzenberg) -- cgit 1.2.3-korg From d7af99434a15a508dc909ba053115fb112b0a247 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 2 Nov 2013 22:19:26 +0200 Subject: Add dates to changelog --- CHANGELOG | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 49a54acf..cc1d26a9 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,4 @@ -nanopb-0.2.4 +nanopb-0.2.4 (2013-11-xx) Remove the deprecated NANOPB_INTERNALS functions from public API. Document the security model. Check array and bytes max sizes when encoding (issue 90) @@ -7,7 +7,7 @@ nanopb-0.2.4 Fix MISRA C violations (issue 91) Clean up pb_field_t definition with typedefs. -nanopb-0.2.3 +nanopb-0.2.3 (2013-09-18) Improve compatibility by removing ternary operator from initializations (issue 88) Fix build error on Visual C++ (issue 84, patch by Markus Schwarzenberg) Don't stop on unsupported extension fields (issue 83) @@ -16,7 +16,7 @@ nanopb-0.2.3 Switch from Makefiles to scons for building the tests Make the tests buildable on Windows -nanopb-0.2.2 +nanopb-0.2.2 (2013-08-18) Add support for extension fields (issue 17) Fix unknown fields in empty message (issue 78) Include the field tags in the generated .pb.h file. @@ -25,7 +25,7 @@ nanopb-0.2.2 Documentation improvements (issues 12, 77 and others) Improved tests -nanopb-0.2.1 +nanopb-0.2.1 (2013-04-14) NOTE: The default callback function signature has changed. If you don't want to update your code, define PB_OLD_CALLBACK_STYLE. @@ -42,7 +42,7 @@ nanopb-0.2.1 Various new generator options Improved tests -nanopb-0.2.0 +nanopb-0.2.0 (2013-03-02) NOTE: This release requires you to regenerate all .pb.c files. Files generated by older versions will not compile anymore. @@ -55,7 +55,7 @@ nanopb-0.2.0 Add option to give file extension to generator (by Michael Haberler) Documentation updates -nanopb-0.1.9 +nanopb-0.1.9 (2013-02-13) Fixed error message bugs (issues 52, 56) Sanitize #ifndef filename (issue 50) Performance improvements @@ -66,13 +66,13 @@ nanopb-0.1.9 Added generator option to make message structs packed. (issue 49) Add more test cases. -nanopb-0.1.8 +nanopb-0.1.8 (2012-12-13) Fix bugs in the enum short names introduced in 0.1.7 (issues 42, 43) Fix STATIC_ASSERT macro when using multiple .proto files. (issue 41) Fix missing initialization of istream.errmsg Make tests/Makefile work for non-gcc compilers (issue 40) -nanopb-0.1.7 +nanopb-0.1.7 (2012-11-11) Remove "skip" mode from pb_istream_t callbacks. Example implementation had a bug. (issue 37) Add option to use shorter names for enum values (issue 38) Improve options support in generator (issues 12, 30) @@ -82,19 +82,19 @@ nanopb-0.1.7 Add buffer size check in example (issue 34) Fix build warnings on MS compilers (issue 33) -nanopb-0.1.6 +nanopb-0.1.6 (2012-09-02) Reorganize the field decoder interface (issue 2) Improve performance in submessage decoding (issue 28) Implement error messages in the decoder side (issue 7) Extended testcases (alltypes test is now complete). Fix some compiler warnings (issues 25, 26, 27, 32). -nanopb-0.1.5 +nanopb-0.1.5 (2012-08-04) Fix bug in decoder with packed arrays (issue 23). Extended testcases. Fix some compiler warnings. -nanopb-0.1.4 +nanopb-0.1.4 (2012-07-05) Add compile-time options for easy-to-use >255 field support. Improve the detection of missing required fields. Added example on how to handle union messages. @@ -102,20 +102,20 @@ nanopb-0.1.4 Fix problems that stopped the code from compiling with some compilers. Fix some compiler warnings. -nanopb-0.1.3 +nanopb-0.1.3 (2012-06-12) Refactor the field encoder interface. Improve generator error messages (issue 5) Add descriptor.proto into the #include exclusion list Fix some compiler warnings. -nanopb-0.1.2 +nanopb-0.1.2 (2012-02-15) Make the generator to generate include for other .proto files (issue 4). Fixed generator not working on Windows (issue 3) -nanopb-0.1.1 +nanopb-0.1.1 (2012-01-14) Fixed bug in encoder with 'bytes' fields (issue 1). Fixed a bug in the generator that caused a compiler error on sfixed32 and sfixed64 fields. Extended testcases. -nanopb-0.1.0 +nanopb-0.1.0 (2012-01-06) First stable release. -- cgit 1.2.3-korg From 935a26ab1cd2982e2478ac6f9b8decf7deb5fe5b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 7 Nov 2013 16:44:41 +0200 Subject: Publishing nanopb-0.2.4 --- CHANGELOG | 2 +- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index cc1d26a9..94f0c797 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,4 +1,4 @@ -nanopb-0.2.4 (2013-11-xx) +nanopb-0.2.4 (2013-11-07) Remove the deprecated NANOPB_INTERNALS functions from public API. Document the security model. Check array and bytes max sizes when encoding (issue 90) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 3c248ac3..9e857286 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.4-dev" +nanopb_version = "nanopb-0.2.4" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index 04a8288b..f6532176 100644 --- a/pb.h +++ b/pb.h @@ -43,7 +43,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.4-dev +#define NANOPB_VERSION nanopb-0.2.4 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 321ca6c1d5736edf6ec2ac0b97eae58227954fad Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 7 Nov 2013 16:47:14 +0200 Subject: Setting version to 0.2.5-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 9e857286..a4e4cc7b 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.4" +nanopb_version = "nanopb-0.2.5-dev" try: import google.protobuf.descriptor_pb2 as descriptor diff --git a/pb.h b/pb.h index f6532176..4e64dded 100644 --- a/pb.h +++ b/pb.h @@ -43,7 +43,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.4 +#define NANOPB_VERSION nanopb-0.2.5-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 171d64734a34674cd648ac7b8569b4b21285fed4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 11 Nov 2013 09:22:32 +0200 Subject: Fix path in FindNanopb.cmake. Update issue 94 Status: FixedInGit --- cmake/FindNanopb.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/FindNanopb.cmake b/cmake/FindNanopb.cmake index 9678fb18..ad96b5c3 100644 --- a/cmake/FindNanopb.cmake +++ b/cmake/FindNanopb.cmake @@ -148,7 +148,7 @@ function(NANOPB_GENERATE_CPP SRCS HDRS) "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" COMMAND python ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb - DEPENDS ${FIL_WE}.pb + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb" COMMENT "Running nanopb generator on ${FIL_WE}.pb" VERBATIM ) endforeach() -- cgit 1.2.3-korg From 5813144246f8f132f90bea117e578477914be0ea Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 13 Nov 2013 22:10:42 +0200 Subject: Make tests build in a separate folder, add to gitignore --- .gitignore | 19 ++++++++++++------- tests/SConstruct | 9 +++++++-- tests/alltypes/SConscript | 6 +++--- tests/backwards_compatibility/SConscript | 4 ++-- tests/basic_buffer/SConscript | 6 +++--- tests/basic_stream/SConscript | 6 +++--- tests/buffer_only/SConscript | 8 ++++---- tests/callbacks/SConscript | 4 ++-- tests/cxx_main_program/SConscript | 8 ++++---- tests/decode_unittests/SConscript | 2 +- tests/encode_unittests/SConscript | 2 +- tests/extensions/SConscript | 10 +++++----- tests/extra_fields/SConscript | 6 +++--- tests/field_size_16/SConscript | 6 +++--- tests/field_size_32/SConscript | 6 +++--- tests/missing_fields/SConscript | 2 +- tests/no_errmsg/SConscript | 8 ++++---- 17 files changed, 61 insertions(+), 51 deletions(-) diff --git a/.gitignore b/.gitignore index b83afeff..70f80a60 100644 --- a/.gitignore +++ b/.gitignore @@ -5,17 +5,22 @@ *.pb.c *.pb.h *.pb +*.pyc *~ *.tar.gz .sconsign.dblite +config.log +.sconf_temp +tests/build julkaisu.txt docs/*.html docs/generator_flow.png -example/client -example/server -example_avr_double/decode_double -example_avr_double/encode_double -example_avr_double/test_conversions -example_unions/decode -example_unions/encode +examples/simple/simple +examples/network_server/client +examples/network_server/server +examples/using_double_on_avr/decode_double +examples/using_double_on_avr/encode_double +examples/using_double_on_avr/test_conversions +examples/using_union_messages/decode +examples/using_union_messages/encode generator/nanopb_pb2.pyc diff --git a/tests/SConstruct b/tests/SConstruct index 100fb544..675989a0 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -26,7 +26,7 @@ if 'CXXFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CXXFLAGS']) add_nanopb_builders(env) # Path to the files shared by tests, and to the nanopb core. -env.Append(CPPPATH = ["#../", "#common"]) +env.Append(CPPPATH = ["#../", "$COMMON"]) # Path for finding nanopb.proto env.Append(PROTOCPATH = '#../generator') @@ -110,5 +110,10 @@ elif 'cl' in env['CXX']: env.Append(CXXFLAGS = '/Zi /W2 /WX') # Now include the SConscript files from all subdirectories -SConscript(Glob('*/SConscript'), exports = 'env') +import os.path +env['VARIANT_DIR'] = 'build' +env['BUILD'] = '#' + env['VARIANT_DIR'] +env['COMMON'] = '#' + env['VARIANT_DIR'] + '/common' +for subdir in Glob('*/SConscript'): + SConscript(subdir, exports = 'env', variant_dir = env['VARIANT_DIR'] + '/' + os.path.dirname(str(subdir))) diff --git a/tests/alltypes/SConscript b/tests/alltypes/SConscript index 8aa45b6b..1dc6f871 100644 --- a/tests/alltypes/SConscript +++ b/tests/alltypes/SConscript @@ -3,9 +3,9 @@ Import("env") -env.NanopbProto("alltypes") -enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "#common/pb_encode.o"]) -dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "#common/pb_decode.o"]) +env.NanopbProto(["alltypes", "alltypes.options"]) +enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) +dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_decode.o"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/backwards_compatibility/SConscript b/tests/backwards_compatibility/SConscript index 5fb978f3..777ef402 100644 --- a/tests/backwards_compatibility/SConscript +++ b/tests/backwards_compatibility/SConscript @@ -3,8 +3,8 @@ Import("env") -enc = env.Program(["encode_legacy.c", "alltypes_legacy.c", "#common/pb_encode.o"]) -dec = env.Program(["decode_legacy.c", "alltypes_legacy.c", "#common/pb_decode.o"]) +enc = env.Program(["encode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_encode.o"]) +dec = env.Program(["decode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_decode.o"]) env.RunTest(enc) env.RunTest([dec, "encode_legacy.output"]) diff --git a/tests/basic_buffer/SConscript b/tests/basic_buffer/SConscript index 349fb14f..2546aaa3 100644 --- a/tests/basic_buffer/SConscript +++ b/tests/basic_buffer/SConscript @@ -2,11 +2,11 @@ Import("env") -enc = env.Program(["encode_buffer.c", "#common/person.pb.c", "#common/pb_encode.o"]) -dec = env.Program(["decode_buffer.c", "#common/person.pb.c", "#common/pb_decode.o"]) +enc = env.Program(["encode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o"]) +dec = env.Program(["decode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o"]) env.RunTest(enc) env.RunTest([dec, "encode_buffer.output"]) -env.Decode(["encode_buffer.output", "#common/person.proto"], MESSAGE = "Person") +env.Decode(["encode_buffer.output", "$COMMON/person.proto"], MESSAGE = "Person") env.Compare(["decode_buffer.output", "encode_buffer.decoded"]) diff --git a/tests/basic_stream/SConscript b/tests/basic_stream/SConscript index 17382a98..46db8c43 100644 --- a/tests/basic_stream/SConscript +++ b/tests/basic_stream/SConscript @@ -2,11 +2,11 @@ Import("env") -enc = env.Program(["encode_stream.c", "#common/person.pb.c", "#common/pb_encode.o"]) -dec = env.Program(["decode_stream.c", "#common/person.pb.c", "#common/pb_decode.o"]) +enc = env.Program(["encode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o"]) +dec = env.Program(["decode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o"]) env.RunTest(enc) env.RunTest([dec, "encode_stream.output"]) -env.Decode(["encode_stream.output", "#common/person.proto"], MESSAGE = "Person") +env.Decode(["encode_stream.output", "$COMMON/person.proto"], MESSAGE = "Person") env.Compare(["decode_stream.output", "encode_stream.decoded"]) diff --git a/tests/buffer_only/SConscript b/tests/buffer_only/SConscript index 0770b2fc..db86d377 100644 --- a/tests/buffer_only/SConscript +++ b/tests/buffer_only/SConscript @@ -6,10 +6,10 @@ Import("env") c = Copy("$TARGET", "$SOURCE") env.Command("pb_encode.c", "#../pb_encode.c", c) env.Command("pb_decode.c", "#../pb_decode.c", c) -env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c) -env.Command("alltypes.pb.c", "#alltypes/alltypes.pb.c", c) -env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c) -env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c) +env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c) +env.Command("alltypes.pb.c", "$BUILD/alltypes/alltypes.pb.c", c) +env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c) +env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c) # Define the compilation options opts = env.Clone() diff --git a/tests/callbacks/SConscript b/tests/callbacks/SConscript index 729fd65f..9ec8a434 100644 --- a/tests/callbacks/SConscript +++ b/tests/callbacks/SConscript @@ -3,8 +3,8 @@ Import("env") env.NanopbProto("callbacks") -enc = env.Program(["encode_callbacks.c", "callbacks.pb.c", "#common/pb_encode.o"]) -dec = env.Program(["decode_callbacks.c", "callbacks.pb.c", "#common/pb_decode.o"]) +enc = env.Program(["encode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_encode.o"]) +dec = env.Program(["decode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_decode.o"]) env.RunTest(enc) env.RunTest([dec, "encode_callbacks.output"]) diff --git a/tests/cxx_main_program/SConscript b/tests/cxx_main_program/SConscript index 055c5aee..4bca1b1f 100644 --- a/tests/cxx_main_program/SConscript +++ b/tests/cxx_main_program/SConscript @@ -7,10 +7,10 @@ Import("env") c = Copy("$TARGET", "$SOURCE") env.Command("pb_encode.cxx", "#../pb_encode.c", c) env.Command("pb_decode.cxx", "#../pb_decode.c", c) -env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c) -env.Command("alltypes.pb.cxx", "#alltypes/alltypes.pb.c", c) -env.Command("encode_alltypes.cxx", "#alltypes/encode_alltypes.c", c) -env.Command("decode_alltypes.cxx", "#alltypes/decode_alltypes.c", c) +env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c) +env.Command("alltypes.pb.cxx", "$BUILD/alltypes/alltypes.pb.c", c) +env.Command("encode_alltypes.cxx", "$BUILD/alltypes/encode_alltypes.c", c) +env.Command("decode_alltypes.cxx", "$BUILD/alltypes/decode_alltypes.c", c) # Now build and run the test normally. enc = env.Program(["encode_alltypes.cxx", "alltypes.pb.cxx", "pb_encode.cxx"]) diff --git a/tests/decode_unittests/SConscript b/tests/decode_unittests/SConscript index 9a639f7b..369b9dc7 100644 --- a/tests/decode_unittests/SConscript +++ b/tests/decode_unittests/SConscript @@ -1,4 +1,4 @@ Import('env') -p = env.Program(["decode_unittests.c", "#common/unittestproto.pb.c"]) +p = env.Program(["decode_unittests.c", "$COMMON/unittestproto.pb.c"]) env.RunTest(p) diff --git a/tests/encode_unittests/SConscript b/tests/encode_unittests/SConscript index 7509ec03..bf6d1404 100644 --- a/tests/encode_unittests/SConscript +++ b/tests/encode_unittests/SConscript @@ -1,5 +1,5 @@ # Build and run the stand-alone unit tests for the nanopb encoder part. Import('env') -p = env.Program(["encode_unittests.c", "#common/unittestproto.pb.c"]) +p = env.Program(["encode_unittests.c", "$COMMON/unittestproto.pb.c"]) env.RunTest(p) diff --git a/tests/extensions/SConscript b/tests/extensions/SConscript index f632a9ae..26fc5a3f 100644 --- a/tests/extensions/SConscript +++ b/tests/extensions/SConscript @@ -4,12 +4,12 @@ Import("env") # We use the files from the alltypes test case incpath = env.Clone() -incpath.Append(PROTOCPATH = '#alltypes') -incpath.Append(CPPPATH = '#alltypes') +incpath.Append(PROTOCPATH = '$BUILD/alltypes') +incpath.Append(CPPPATH = '$BUILD/alltypes') -incpath.NanopbProto("extensions") -enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb$OBJSUFFIX", "#common/pb_encode.o"]) -dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "#alltypes/alltypes.pb$OBJSUFFIX", "#common/pb_decode.o"]) +incpath.NanopbProto(["extensions", "extensions.options"]) +enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_encode.o"]) +dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_decode.o"]) env.RunTest(enc) env.RunTest([dec, "encode_extensions.output"]) diff --git a/tests/extra_fields/SConscript b/tests/extra_fields/SConscript index 9227e950..797c4e6e 100644 --- a/tests/extra_fields/SConscript +++ b/tests/extra_fields/SConscript @@ -2,13 +2,13 @@ Import("env") -dec = env.GetBuildPath('#basic_buffer/${PROGPREFIX}decode_buffer${PROGSUFFIX}') +dec = env.GetBuildPath('$BUILD/basic_buffer/${PROGPREFIX}decode_buffer${PROGSUFFIX}') env.RunTest('person_with_extra_field.output', [dec, "person_with_extra_field.pb"]) env.Compare(["person_with_extra_field.output", "person_with_extra_field.expected"]) -dec = env.GetBuildPath('#basic_stream/${PROGPREFIX}decode_stream${PROGSUFFIX}') +dec = env.GetBuildPath('$BUILD/basic_stream/${PROGPREFIX}decode_stream${PROGSUFFIX}') env.RunTest('person_with_extra_field_stream.output', [dec, "person_with_extra_field.pb"]) env.Compare(["person_with_extra_field_stream.output", "person_with_extra_field.expected"]) -dec2 = env.GetBuildPath('#alltypes/${PROGPREFIX}decode_alltypes${PROGSUFFIX}') +dec2 = env.GetBuildPath('$BUILD/alltypes/${PROGPREFIX}decode_alltypes${PROGSUFFIX}') env.RunTest('alltypes_with_extra_fields.output', [dec2, 'alltypes_with_extra_fields.pb']) diff --git a/tests/field_size_16/SConscript b/tests/field_size_16/SConscript index 48d08e84..15dc0f5d 100644 --- a/tests/field_size_16/SConscript +++ b/tests/field_size_16/SConscript @@ -7,10 +7,10 @@ Import("env") c = Copy("$TARGET", "$SOURCE") env.Command("pb_encode.c", "#../pb_encode.c", c) env.Command("pb_decode.c", "#../pb_decode.c", c) -env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c) -env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c) +env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c) +env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c) -env.NanopbProto("alltypes") +env.NanopbProto(["alltypes", "alltypes.options"]) # Define the compilation options opts = env.Clone() diff --git a/tests/field_size_32/SConscript b/tests/field_size_32/SConscript index a8584dc0..25ef3546 100644 --- a/tests/field_size_32/SConscript +++ b/tests/field_size_32/SConscript @@ -7,10 +7,10 @@ Import("env") c = Copy("$TARGET", "$SOURCE") env.Command("pb_encode.c", "#../pb_encode.c", c) env.Command("pb_decode.c", "#../pb_decode.c", c) -env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c) -env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c) +env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c) +env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c) -env.NanopbProto("alltypes") +env.NanopbProto(["alltypes", "alltypes.options"]) # Define the compilation options opts = env.Clone() diff --git a/tests/missing_fields/SConscript b/tests/missing_fields/SConscript index 361b5502..9926efad 100644 --- a/tests/missing_fields/SConscript +++ b/tests/missing_fields/SConscript @@ -3,6 +3,6 @@ Import("env") env.NanopbProto("missing_fields") -test = env.Program(["missing_fields.c", "missing_fields.pb.c", "#common/pb_encode.o", "#common/pb_decode.o"]) +test = env.Program(["missing_fields.c", "missing_fields.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_decode.o"]) env.RunTest(test) diff --git a/tests/no_errmsg/SConscript b/tests/no_errmsg/SConscript index 870e8946..2b9815d1 100644 --- a/tests/no_errmsg/SConscript +++ b/tests/no_errmsg/SConscript @@ -6,10 +6,10 @@ Import("env") c = Copy("$TARGET", "$SOURCE") env.Command("pb_encode.c", "#../pb_encode.c", c) env.Command("pb_decode.c", "#../pb_decode.c", c) -env.Command("alltypes.pb.h", "#alltypes/alltypes.pb.h", c) -env.Command("alltypes.pb.c", "#alltypes/alltypes.pb.c", c) -env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", c) -env.Command("decode_alltypes.c", "#alltypes/decode_alltypes.c", c) +env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c) +env.Command("alltypes.pb.c", "$BUILD/alltypes/alltypes.pb.c", c) +env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c) +env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c) # Define the compilation options opts = env.Clone() -- cgit 1.2.3-korg From eff9e11150c0bfb6baf5d6bec2351034b72d95ed Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 14 Nov 2013 17:56:42 +0200 Subject: Optimize the common case of 1-byte reads for varints. For PB_BUFFER_ONLY configuration, this gives 20% speedup without increasing code size. --- pb_decode.c | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 90fa18d2..a887d155 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -124,6 +124,26 @@ bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) return true; } +/* Read a single byte from input stream. buf may not be NULL. + * This is an optimization for the varint decoding. */ +static bool checkreturn pb_readbyte(pb_istream_t *stream, uint8_t *buf) +{ + if (!stream->bytes_left) + PB_RETURN_ERROR(stream, "end-of-stream"); + +#ifndef PB_BUFFER_ONLY + if (!stream->callback(stream, buf, 1)) + PB_RETURN_ERROR(stream, "io error"); +#else + *buf = *(uint8_t*)stream->state; + stream->state = (uint8_t*)stream->state + 1; +#endif + + stream->bytes_left--; + + return true; +} + pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize) { pb_istream_t stream; @@ -149,7 +169,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) uint8_t byte; uint32_t result; - if (!pb_read(stream, &byte, 1)) + if (!pb_readbyte(stream, &byte)) return false; if (!(byte & 0x80)) @@ -168,7 +188,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) if (bitpos >= 32) PB_RETURN_ERROR(stream, "varint overflow"); - if (!pb_read(stream, &byte, 1)) + if (!pb_readbyte(stream, &byte)) return false; result |= (uint32_t)(byte & 0x7F) << bitpos; @@ -191,7 +211,7 @@ bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) if (bitpos >= 64) PB_RETURN_ERROR(stream, "varint overflow"); - if (!pb_read(stream, &byte, 1)) + if (!pb_readbyte(stream, &byte)) return false; result |= (uint64_t)(byte & 0x7F) << bitpos; -- cgit 1.2.3-korg From 9cc19a5e998d93fbe6d7a7c18fba1e37a36076d5 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 21 Dec 2013 12:14:20 +0200 Subject: Modify the alltypes test to check re-encoding through protoc. This way we can verify that the message is encoded exactly the same way as the official protobuf implementation would do it. --- tests/alltypes/SConscript | 23 +++++++++++++++++++++++ tests/alltypes/alltypes.proto | 28 ++++++++++++++-------------- tests/site_scons/site_init.py | 17 ++++++++++++++++- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/tests/alltypes/SConscript b/tests/alltypes/SConscript index 1dc6f871..9c9072ba 100644 --- a/tests/alltypes/SConscript +++ b/tests/alltypes/SConscript @@ -7,6 +7,29 @@ env.NanopbProto(["alltypes", "alltypes.options"]) enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_decode.o"]) +# Test the round-trip from nanopb encoder to nanopb decoder env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) +# Re-encode the data using protoc, and check that the results from nanopb +# match byte-per-byte to the protoc output. +env.Decode("encode_alltypes.output.decoded", + ["encode_alltypes.output", "alltypes.proto"], + MESSAGE='AllTypes') +env.Encode("encode_alltypes.output.recoded", + ["encode_alltypes.output.decoded", "alltypes.proto"], + MESSAGE='AllTypes') +env.Compare(["encode_alltypes.output", "encode_alltypes.output.recoded"]) + +# Do the same checks with the optional fields present. +env.RunTest("optionals.output", enc, ARGS = ['1']) +env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) +env.Decode("optionals.output.decoded", + ["optionals.output", "alltypes.proto"], + MESSAGE='AllTypes') +env.Encode("optionals.output.recoded", + ["optionals.output.decoded", "alltypes.proto"], + MESSAGE='AllTypes') +env.Compare(["optionals.output", "optionals.output.recoded"]) + + diff --git a/tests/alltypes/alltypes.proto b/tests/alltypes/alltypes.proto index a2cf8bbc..2bc8efcd 100644 --- a/tests/alltypes/alltypes.proto +++ b/tests/alltypes/alltypes.proto @@ -39,26 +39,26 @@ message AllTypes { required EmptyMessage req_emptymsg = 18; - repeated int32 rep_int32 = 21; - repeated int64 rep_int64 = 22; - repeated uint32 rep_uint32 = 23; - repeated uint64 rep_uint64 = 24; - repeated sint32 rep_sint32 = 25; - repeated sint64 rep_sint64 = 26; - repeated bool rep_bool = 27; + repeated int32 rep_int32 = 21 [packed = true]; + repeated int64 rep_int64 = 22 [packed = true]; + repeated uint32 rep_uint32 = 23 [packed = true]; + repeated uint64 rep_uint64 = 24 [packed = true]; + repeated sint32 rep_sint32 = 25 [packed = true]; + repeated sint64 rep_sint64 = 26 [packed = true]; + repeated bool rep_bool = 27 [packed = true]; - repeated fixed32 rep_fixed32 = 28; - repeated sfixed32 rep_sfixed32= 29; - repeated float rep_float = 30; + repeated fixed32 rep_fixed32 = 28 [packed = true]; + repeated sfixed32 rep_sfixed32= 29 [packed = true]; + repeated float rep_float = 30 [packed = true]; - repeated fixed64 rep_fixed64 = 31; - repeated sfixed64 rep_sfixed64= 32; - repeated double rep_double = 33; + repeated fixed64 rep_fixed64 = 31 [packed = true]; + repeated sfixed64 rep_sfixed64= 32 [packed = true]; + repeated double rep_double = 33 [packed = true]; repeated string rep_string = 34; repeated bytes rep_bytes = 35; repeated SubMessage rep_submsg = 36; - repeated MyEnum rep_enum = 37; + repeated MyEnum rep_enum = 37 [packed = true]; repeated EmptyMessage rep_emptymsg = 38; optional int32 opt_int32 = 41 [default = 4041]; diff --git a/tests/site_scons/site_init.py b/tests/site_scons/site_init.py index 1f811153..38aa1a44 100644 --- a/tests/site_scons/site_init.py +++ b/tests/site_scons/site_init.py @@ -55,7 +55,11 @@ def add_nanopb_builders(env): else: infile = None - pipe = subprocess.Popen(str(source[0]), + args = [str(source[0])] + if env.has_key('ARGS'): + args.extend(env['ARGS']) + + pipe = subprocess.Popen(args, stdin = infile, stdout = open(str(target[0]), 'w'), stderr = sys.stderr) @@ -81,6 +85,17 @@ def add_nanopb_builders(env): suffix = '.decoded') env.Append(BUILDERS = {'Decode': decode_builder}) + # Build command that encodes a message using protoc + def encode_actions(source, target, env, for_signature): + esc = env['ESCAPE'] + dirs = ' '.join(['-I' + esc(env.GetBuildPath(d)) for d in env['PROTOCPATH']]) + return '$PROTOC $PROTOCFLAGS %s --encode=%s %s <%s >%s' % ( + dirs, env['MESSAGE'], esc(str(source[1])), esc(str(source[0])), esc(str(target[0]))) + + encode_builder = Builder(generator = encode_actions, + suffix = '.encoded') + env.Append(BUILDERS = {'Encode': encode_builder}) + # Build command that asserts that two files be equal def compare_files(target, source, env): data1 = open(str(source[0]), 'rb').read() -- cgit 1.2.3-korg From ee5b12c537115b113ce01708d4a86a4062cdb182 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 21 Dec 2013 12:16:03 +0200 Subject: Add PB_LTYPE_UVARINT to fix encoding of negative int32 values. Apparently int32 values that are negative must be cast into int64 first before being encoded. Because uint32 still needs to be cast to uint64, the cases for int32 and uint32 had to be separated. Update issue 97 Status: FixedInGit --- pb.h | 30 +++++++++++++----------------- pb_decode.c | 24 +++++++++++++++++++++--- pb_encode.c | 23 +++++++++++++++++++++-- 3 files changed, 55 insertions(+), 22 deletions(-) diff --git a/pb.h b/pb.h index 4e64dded..a8e95e59 100644 --- a/pb.h +++ b/pb.h @@ -116,11 +116,6 @@ /* List of possible field types. These are used in the autogenerated code. * Least-significant 4 bits tell the scalar type * Most-significant 4 bits specify repeated/required/packed etc. - * - * INT32 and UINT32 are treated the same, as are (U)INT64 and (S)FIXED* - * These types are simply casted to correct field type when they are - * assigned to the memory pointer. - * SINT* is different, though, because it is zig-zag coded. */ typedef uint8_t pb_type_t; @@ -128,32 +123,33 @@ typedef uint8_t pb_type_t; /**** Field data types ****/ /* Numeric types */ -#define PB_LTYPE_VARINT 0x00 /* int32, uint32, int64, uint64, bool, enum */ -#define PB_LTYPE_SVARINT 0x01 /* sint32, sint64 */ -#define PB_LTYPE_FIXED32 0x02 /* fixed32, sfixed32, float */ -#define PB_LTYPE_FIXED64 0x03 /* fixed64, sfixed64, double */ +#define PB_LTYPE_VARINT 0x00 /* int32, int64, enum, bool */ +#define PB_LTYPE_UVARINT 0x01 /* uint32, uint64 */ +#define PB_LTYPE_SVARINT 0x02 /* sint32, sint64 */ +#define PB_LTYPE_FIXED32 0x03 /* fixed32, sfixed32, float */ +#define PB_LTYPE_FIXED64 0x04 /* fixed64, sfixed64, double */ /* Marker for last packable field type. */ -#define PB_LTYPE_LAST_PACKABLE 0x03 +#define PB_LTYPE_LAST_PACKABLE 0x04 /* Byte array with pre-allocated buffer. * data_size is the length of the allocated PB_BYTES_ARRAY structure. */ -#define PB_LTYPE_BYTES 0x04 +#define PB_LTYPE_BYTES 0x05 /* String with pre-allocated buffer. * data_size is the maximum length. */ -#define PB_LTYPE_STRING 0x05 +#define PB_LTYPE_STRING 0x06 /* Submessage * submsg_fields is pointer to field descriptions */ -#define PB_LTYPE_SUBMESSAGE 0x06 +#define PB_LTYPE_SUBMESSAGE 0x07 /* Extension pseudo-field * The field contains a pointer to pb_extension_t */ -#define PB_LTYPE_EXTENSION 0x07 +#define PB_LTYPE_EXTENSION 0x08 /* Number of declared LTYPES */ -#define PB_LTYPES_COUNT 8 +#define PB_LTYPES_COUNT 9 #define PB_LTYPE_MASK 0x0F /**** Field repetition rules ****/ @@ -410,8 +406,8 @@ struct _pb_extension_t { #define PB_LTYPE_MAP_SINT32 PB_LTYPE_SVARINT #define PB_LTYPE_MAP_SINT64 PB_LTYPE_SVARINT #define PB_LTYPE_MAP_STRING PB_LTYPE_STRING -#define PB_LTYPE_MAP_UINT32 PB_LTYPE_VARINT -#define PB_LTYPE_MAP_UINT64 PB_LTYPE_VARINT +#define PB_LTYPE_MAP_UINT32 PB_LTYPE_UVARINT +#define PB_LTYPE_MAP_UINT64 PB_LTYPE_UVARINT #define PB_LTYPE_MAP_EXTENSION PB_LTYPE_EXTENSION /* This is the actual macro used in field descriptions. diff --git a/pb_decode.c b/pb_decode.c index a887d155..30c124dd 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -47,7 +47,8 @@ static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_exten static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_field_iterator_t *iter); static bool checkreturn find_extension_field(pb_field_iterator_t *iter); static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct); -static bool pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest); +static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest); +static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, void *dest); static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest); static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest); static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest); @@ -62,6 +63,7 @@ static bool checkreturn pb_skip_string(pb_istream_t *stream); */ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { &pb_dec_varint, + &pb_dec_uvarint, &pb_dec_svarint, &pb_dec_fixed32, &pb_dec_fixed64, @@ -823,8 +825,24 @@ bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, vo switch (field->data_size) { - case 1: *(uint8_t*)dest = (uint8_t)value; break; - case 2: *(uint16_t*)dest = (uint16_t)value; break; + case 1: *(int8_t*)dest = (int8_t)value; break; + case 2: *(int16_t*)dest = (int16_t)value; break; + case 4: *(int32_t*)dest = (int32_t)value; break; + case 8: *(int64_t*)dest = (int64_t)value; break; + default: PB_RETURN_ERROR(stream, "invalid data_size"); + } + + return true; +} + +bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + uint64_t value; + if (!pb_decode_varint(stream, &value)) + return false; + + switch (field->data_size) + { case 4: *(uint32_t*)dest = (uint32_t)value; break; case 8: *(uint64_t*)dest = value; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); diff --git a/pb_encode.c b/pb_encode.c index ebf20de5..f6e08a57 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -28,6 +28,7 @@ static bool checkreturn encode_field(pb_ostream_t *stream, const pb_field_t *fie static bool checkreturn default_extension_encoder(pb_ostream_t *stream, const pb_extension_t *extension); static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData); static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src); static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src); @@ -40,6 +41,7 @@ static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t */ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { &pb_enc_varint, + &pb_enc_uvarint, &pb_enc_svarint, &pb_enc_fixed32, &pb_enc_fixed64, @@ -424,6 +426,7 @@ bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t switch (PB_LTYPE(field->type)) { case PB_LTYPE_VARINT: + case PB_LTYPE_UVARINT: case PB_LTYPE_SVARINT: wiretype = PB_WT_VARINT; break; @@ -505,13 +508,29 @@ bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fie /* Field encoders */ bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src) +{ + int64_t value = 0; + + /* Cases 1 and 2 are for compilers that have smaller types for bool + * or enums. */ + switch (field->data_size) + { + case 1: value = *(const int8_t*)src; break; + case 2: value = *(const int16_t*)src; break; + case 4: value = *(const int32_t*)src; break; + case 8: value = *(const int64_t*)src; break; + default: PB_RETURN_ERROR(stream, "invalid data_size"); + } + + return pb_encode_varint(stream, (uint64_t)value); +} + +bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, const void *src) { uint64_t value = 0; switch (field->data_size) { - case 1: value = *(const uint8_t*)src; break; - case 2: value = *(const uint16_t*)src; break; case 4: value = *(const uint32_t*)src; break; case 8: value = *(const uint64_t*)src; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); -- cgit 1.2.3-korg From cd9004089fd4ae42cc68fd15f085f42c48494cb4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 21 Dec 2013 12:41:20 +0200 Subject: Add test for extreme integer values (INT32_MAX etc.) in AllTypes. --- tests/alltypes/alltypes.proto | 21 +++++++++++++++++++++ tests/alltypes/decode_alltypes.c | 11 +++++++++++ tests/alltypes/encode_alltypes.c | 11 +++++++++++ tests/cxx_main_program/SConscript | 4 ++++ tests/extra_fields/SConscript | 4 +++- tests/field_size_16/alltypes.proto | 21 +++++++++++++++++++++ tests/field_size_32/alltypes.proto | 21 +++++++++++++++++++++ 7 files changed, 92 insertions(+), 1 deletion(-) diff --git a/tests/alltypes/alltypes.proto b/tests/alltypes/alltypes.proto index 2bc8efcd..234b7236 100644 --- a/tests/alltypes/alltypes.proto +++ b/tests/alltypes/alltypes.proto @@ -8,6 +8,24 @@ message EmptyMessage { } +enum HugeEnum { + Negative = -2147483647; /* protoc doesn't accept -2147483648 here */ + Positive = 2147483647; +} + +message Limits { + required int32 int32_min = 1; + required int32 int32_max = 2; + required uint32 uint32_min = 3; + required uint32 uint32_max = 4; + required int64 int64_min = 5; + required int64 int64_max = 6; + required uint64 uint64_min = 7; + required uint64 uint64_max = 8; + required HugeEnum enum_min = 9; + required HugeEnum enum_max = 10; +} + enum MyEnum { Zero = 0; First = 1; @@ -83,6 +101,9 @@ message AllTypes { optional MyEnum opt_enum = 57 [default = Second]; optional EmptyMessage opt_emptymsg = 58; + // Check that extreme integer values are handled correctly + required Limits req_limits = 98; + // Just to make sure that the size of the fields has been calculated // properly, i.e. otherwise a bug in last field might not be detected. required int32 end = 99; diff --git a/tests/alltypes/decode_alltypes.c b/tests/alltypes/decode_alltypes.c index ee2e115c..db72bb9f 100644 --- a/tests/alltypes/decode_alltypes.c +++ b/tests/alltypes/decode_alltypes.c @@ -170,6 +170,17 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.has_opt_emptymsg == true); } + TEST(alltypes.req_limits.int32_min == INT32_MIN); + TEST(alltypes.req_limits.int32_max == INT32_MAX); + TEST(alltypes.req_limits.uint32_min == 0); + TEST(alltypes.req_limits.uint32_max == UINT32_MAX); + TEST(alltypes.req_limits.int64_min == INT64_MIN); + TEST(alltypes.req_limits.int64_max == INT64_MAX); + TEST(alltypes.req_limits.uint64_min == 0); + TEST(alltypes.req_limits.uint64_max == UINT64_MAX); + TEST(alltypes.req_limits.enum_min == HugeEnum_Negative); + TEST(alltypes.req_limits.enum_max == HugeEnum_Positive); + TEST(alltypes.end == 1099); return true; diff --git a/tests/alltypes/encode_alltypes.c b/tests/alltypes/encode_alltypes.c index 9a2c6f60..fa8eec94 100644 --- a/tests/alltypes/encode_alltypes.c +++ b/tests/alltypes/encode_alltypes.c @@ -67,6 +67,17 @@ int main(int argc, char **argv) alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; alltypes.rep_emptymsg_count = 5; + alltypes.req_limits.int32_min = INT32_MIN; + alltypes.req_limits.int32_max = INT32_MAX; + alltypes.req_limits.uint32_min = 0; + alltypes.req_limits.uint32_max = UINT32_MAX; + alltypes.req_limits.int64_min = INT64_MIN; + alltypes.req_limits.int64_max = INT64_MAX; + alltypes.req_limits.uint64_min = 0; + alltypes.req_limits.uint64_max = UINT64_MAX; + alltypes.req_limits.enum_min = HugeEnum_Negative; + alltypes.req_limits.enum_max = HugeEnum_Positive; + if (mode != 0) { /* Fill in values for optional fields */ diff --git a/tests/cxx_main_program/SConscript b/tests/cxx_main_program/SConscript index 4bca1b1f..e78c6b35 100644 --- a/tests/cxx_main_program/SConscript +++ b/tests/cxx_main_program/SConscript @@ -3,6 +3,10 @@ Import("env") +# This is needed to get INT32_MIN etc. macros defined +env = env.Clone() +env.Append(CPPDEFINES = ['__STDC_LIMIT_MACROS']) + # Copy the files to .cxx extension in order to force C++ build. c = Copy("$TARGET", "$SOURCE") env.Command("pb_encode.cxx", "#../pb_encode.c", c) diff --git a/tests/extra_fields/SConscript b/tests/extra_fields/SConscript index 797c4e6e..75ac5c5e 100644 --- a/tests/extra_fields/SConscript +++ b/tests/extra_fields/SConscript @@ -10,5 +10,7 @@ dec = env.GetBuildPath('$BUILD/basic_stream/${PROGPREFIX}decode_stream${PROGSUFF env.RunTest('person_with_extra_field_stream.output', [dec, "person_with_extra_field.pb"]) env.Compare(["person_with_extra_field_stream.output", "person_with_extra_field.expected"]) -dec2 = env.GetBuildPath('$BUILD/alltypes/${PROGPREFIX}decode_alltypes${PROGSUFFIX}') +# This uses the backwards compatibility alltypes test, so that +# alltypes_with_extra_fields.pb doesn't have to be remade so often. +dec2 = env.GetBuildPath('$BUILD/backwards_compatibility/${PROGPREFIX}decode_legacy${PROGSUFFIX}') env.RunTest('alltypes_with_extra_fields.output', [dec2, 'alltypes_with_extra_fields.pb']) diff --git a/tests/field_size_16/alltypes.proto b/tests/field_size_16/alltypes.proto index b981760d..7494853d 100644 --- a/tests/field_size_16/alltypes.proto +++ b/tests/field_size_16/alltypes.proto @@ -8,6 +8,24 @@ message EmptyMessage { } +enum HugeEnum { + Negative = -2147483647; /* protoc doesn't accept -2147483648 here */ + Positive = 2147483647; +} + +message Limits { + required int32 int32_min = 1; + required int32 int32_max = 2; + required uint32 uint32_min = 3; + required uint32 uint32_max = 4; + required int64 int64_min = 5; + required int64 int64_max = 6; + required uint64 uint64_min = 7; + required uint64 uint64_max = 8; + required HugeEnum enum_min = 9; + required HugeEnum enum_max = 10; +} + enum MyEnum { Zero = 0; First = 1; @@ -83,6 +101,9 @@ message AllTypes { optional MyEnum opt_enum = 10057 [default = Second]; optional EmptyMessage opt_emptymsg = 10058; + // Check that extreme integer values are handled correctly + required Limits req_limits = 98; + // Just to make sure that the size of the fields has been calculated // properly, i.e. otherwise a bug in last field might not be detected. required int32 end = 10099; diff --git a/tests/field_size_32/alltypes.proto b/tests/field_size_32/alltypes.proto index 3d1d856c..17f17eeb 100644 --- a/tests/field_size_32/alltypes.proto +++ b/tests/field_size_32/alltypes.proto @@ -8,6 +8,24 @@ message EmptyMessage { } +enum HugeEnum { + Negative = -2147483647; /* protoc doesn't accept -2147483648 here */ + Positive = 2147483647; +} + +message Limits { + required int32 int32_min = 1; + required int32 int32_max = 2; + required uint32 uint32_min = 3; + required uint32 uint32_max = 4; + required int64 int64_min = 5; + required int64 int64_max = 6; + required uint64 uint64_min = 7; + required uint64 uint64_max = 8; + required HugeEnum enum_min = 9; + required HugeEnum enum_max = 10; +} + enum MyEnum { Zero = 0; First = 1; @@ -83,6 +101,9 @@ message AllTypes { optional MyEnum opt_enum = 10057 [default = Second]; optional EmptyMessage opt_emptymsg = 10058; + // Check that extreme integer values are handled correctly + required Limits req_limits = 98; + // Just to make sure that the size of the fields has been calculated // properly, i.e. otherwise a bug in last field might not be detected. required int32 end = 13432099; -- cgit 1.2.3-korg From ed93db08dc19290ac3232abd1e7480c3d18c84a5 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 21 Dec 2013 18:08:04 +0200 Subject: Fix problem with decode_legacy test case on Windows. --- tests/backwards_compatibility/decode_legacy.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/backwards_compatibility/decode_legacy.c b/tests/backwards_compatibility/decode_legacy.c index 315b16ef..9dfe437a 100644 --- a/tests/backwards_compatibility/decode_legacy.c +++ b/tests/backwards_compatibility/decode_legacy.c @@ -11,6 +11,7 @@ #include #include #include "alltypes_legacy.h" +#include "test_helpers.h" #define TEST(x) if (!(x)) { \ printf("Test " #x " failed.\n"); \ @@ -176,15 +177,19 @@ bool check_alltypes(pb_istream_t *stream, int mode) int main(int argc, char **argv) { + uint8_t buffer[1024]; + size_t count; + pb_istream_t stream; + /* Whether to expect the optional values or the default values. */ int mode = (argc > 1) ? atoi(argv[1]) : 0; /* Read the data into buffer */ - uint8_t buffer[1024]; - size_t count = fread(buffer, 1, sizeof(buffer), stdin); + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); /* Construct a pb_istream_t for reading from the buffer */ - pb_istream_t stream = pb_istream_from_buffer(buffer, count); + stream = pb_istream_from_buffer(buffer, count); /* Decode and print out the stuff */ if (!check_alltypes(&stream, mode)) -- cgit 1.2.3-korg From 4f37c083d532a782e478b2cef6a02d028613564d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 22 Dec 2013 23:24:02 +0200 Subject: Negative int32 values take 10 bytes now. --- generator/nanopb_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index a4e4cc7b..468c0d2e 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -46,7 +46,7 @@ datatypes = { FieldD.TYPE_FIXED32: ('uint32_t', 'FIXED32', 4), FieldD.TYPE_FIXED64: ('uint64_t', 'FIXED64', 8), FieldD.TYPE_FLOAT: ('float', 'FLOAT', 4), - FieldD.TYPE_INT32: ('int32_t', 'INT32', 5), + FieldD.TYPE_INT32: ('int32_t', 'INT32', 10), FieldD.TYPE_INT64: ('int64_t', 'INT64', 10), FieldD.TYPE_SFIXED32: ('int32_t', 'SFIXED32', 4), FieldD.TYPE_SFIXED64: ('int64_t', 'SFIXED64', 8), -- cgit 1.2.3-korg From 4ae3b2e5661b154cd1f7e545f6052d271306ff25 Mon Sep 17 00:00:00 2001 From: Martin Donath Date: Sun, 8 Dec 2013 23:25:32 +0100 Subject: Generating and encoding messages with dynamic allocaiton --- generator/nanopb.proto | 1 + generator/nanopb_generator.py | 62 +++++++--- generator/nanopb_pb2.py | 13 ++- pb.h | 17 +++ pb_encode.c | 67 ++++++++--- tests/alltypes_pointer/SConscript | 12 ++ tests/alltypes_pointer/alltypes.options | 2 + tests/alltypes_pointer/alltypes.proto | 93 +++++++++++++++ tests/alltypes_pointer/encode_alltypes_pointer.c | 142 +++++++++++++++++++++++ 9 files changed, 373 insertions(+), 36 deletions(-) create mode 100644 tests/alltypes_pointer/SConscript create mode 100644 tests/alltypes_pointer/alltypes.options create mode 100644 tests/alltypes_pointer/alltypes.proto create mode 100644 tests/alltypes_pointer/encode_alltypes_pointer.c diff --git a/generator/nanopb.proto b/generator/nanopb.proto index fe564b5c..2be2f803 100644 --- a/generator/nanopb.proto +++ b/generator/nanopb.proto @@ -12,6 +12,7 @@ option java_package = "fi.kapsi.koti.jpa.nanopb"; enum FieldType { FT_DEFAULT = 0; // Automatically decide field type, generate static field if possible. FT_CALLBACK = 1; // Always generate a callback field. + FT_POINTER = 4; // Always generate a dynamically allocated field. FT_STATIC = 2; // Generate a static field or raise an exception if not possible. FT_IGNORE = 3; // Ignore the field completely. } diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 468c0d2e..69548ad6 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -200,19 +200,27 @@ class Field: self.enc_size = 5 # protoc rejects enum values > 32 bits elif desc.type == FieldD.TYPE_STRING: self.pbtype = 'STRING' - if self.max_size is None: - can_be_static = False - else: + if field_options.type == nanopb_pb2.FT_POINTER: self.ctype = 'char' - self.array_decl += '[%d]' % self.max_size - self.enc_size = varint_max_size(self.max_size) + self.max_size + self.enc_size = None + else: + if self.max_size is None: + can_be_static = False + else: + self.ctype = 'char' + self.array_decl += '[%d]' % self.max_size + self.enc_size = varint_max_size(self.max_size) + self.max_size elif desc.type == FieldD.TYPE_BYTES: self.pbtype = 'BYTES' - if self.max_size is None: - can_be_static = False - else: + if field_options.type == nanopb_pb2.FT_POINTER: self.ctype = self.struct_name + self.name + 't' - self.enc_size = varint_max_size(self.max_size) + self.max_size + self.enc_size = None + else: + if self.max_size is None: + can_be_static = False + else: + self.ctype = self.struct_name + self.name + 't' + self.enc_size = varint_max_size(self.max_size) + self.max_size elif desc.type == FieldD.TYPE_MESSAGE: self.pbtype = 'MESSAGE' self.ctype = self.submsgname = names_from_type_name(desc.type_name) @@ -231,6 +239,8 @@ class Field: if field_options.type == nanopb_pb2.FT_STATIC: self.allocation = 'STATIC' + elif field_options.type == nanopb_pb2.FT_POINTER: + self.allocation = 'POINTER' elif field_options.type == nanopb_pb2.FT_CALLBACK: self.allocation = 'CALLBACK' self.ctype = 'pb_callback_t' @@ -242,21 +252,37 @@ class Field: return cmp(self.tag, other.tag) def __str__(self): - if self.rules == 'OPTIONAL' and self.allocation == 'STATIC': - result = ' bool has_' + self.name + ';\n' - elif self.rules == 'REPEATED' and self.allocation == 'STATIC': - result = ' size_t ' + self.name + '_count;\n' + result = '' + if self.allocation == 'POINTER': + if self.rules == 'REPEATED': + result += ' size_t ' + self.name + '_count;\n' + + # Use struct definition, so recursive submessages are possible + if self.pbtype == 'MESSAGE': + result += ' struct _%s *%s;' % (self.ctype, self.name) + + # String arrays need to be defined as pointers to pointers + elif self.rules == 'REPEATED' and self.pbtype == 'STRING': + result += ' %s **%s;' % (self.ctype, self.name) + else: + result += ' %s *%s;' % (self.ctype, self.name) else: - result = '' - result += ' %s %s%s;' % (self.ctype, self.name, self.array_decl) + if self.rules == 'OPTIONAL' and self.allocation == 'STATIC': + result += ' bool has_' + self.name + ';\n' + elif self.rules == 'REPEATED' and self.allocation == 'STATIC': + result += ' size_t ' + self.name + '_count;\n' + result += ' %s %s%s;' % (self.ctype, self.name, self.array_decl) return result def types(self): '''Return definitions for any special types this field might need.''' - if self.pbtype == 'BYTES' and self.allocation == 'STATIC': + if self.pbtype == 'BYTES' and (self.allocation == 'STATIC' or self.allocation == 'POINTER'): result = 'typedef struct {\n' result += ' size_t size;\n' - result += ' uint8_t bytes[%d];\n' % self.max_size + if self.allocation == 'POINTER': + result += ' uint8_t *bytes;\n' + else: + result += ' uint8_t bytes[%d];\n' % self.max_size result += '} %s;\n' % self.ctype else: result = None @@ -303,7 +329,7 @@ class Field: result = ' PB_FIELD2(%3d, ' % self.tag result += '%-8s, ' % self.pbtype result += '%s, ' % self.rules - result += '%s, ' % self.allocation + result += '%-8s, ' % self.allocation result += '%s, ' % ("FIRST" if not prev_field_name else "OTHER") result += '%s, ' % self.struct_name result += '%s, ' % self.name diff --git a/generator/nanopb_pb2.py b/generator/nanopb_pb2.py index 4ba18b26..ef1931f2 100644 --- a/generator/nanopb_pb2.py +++ b/generator/nanopb_pb2.py @@ -12,7 +12,7 @@ import google.protobuf.descriptor_pb2 DESCRIPTOR = descriptor.FileDescriptor( name='nanopb.proto', package='', - serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*J\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions') + serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*Z\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\x0e\n\nFT_POINTER\x10\x04\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptionsB\x1a\n\x18\x66i.kapsi.koti.jpa.nanopb') _FIELDTYPE = descriptor.EnumDescriptor( name='FieldType', @@ -29,23 +29,28 @@ _FIELDTYPE = descriptor.EnumDescriptor( options=None, type=None), descriptor.EnumValueDescriptor( - name='FT_STATIC', index=2, number=2, + name='FT_POINTER', index=2, number=4, options=None, type=None), descriptor.EnumValueDescriptor( - name='FT_IGNORE', index=3, number=3, + name='FT_STATIC', index=3, number=2, + options=None, + type=None), + descriptor.EnumValueDescriptor( + name='FT_IGNORE', index=4, number=3, options=None, type=None), ], containing_type=None, options=None, serialized_start=199, - serialized_end=273, + serialized_end=289, ) FT_DEFAULT = 0 FT_CALLBACK = 1 +FT_POINTER = 4 FT_STATIC = 2 FT_IGNORE = 3 diff --git a/pb.h b/pb.h index a8e95e59..d839be59 100644 --- a/pb.h +++ b/pb.h @@ -162,6 +162,7 @@ typedef uint8_t pb_type_t; /**** Field allocation types ****/ #define PB_ATYPE_STATIC 0x00 +#define PB_ATYPE_POINTER 0x80 #define PB_ATYPE_CALLBACK 0x40 #define PB_ATYPE_MASK 0xC0 @@ -366,6 +367,22 @@ struct _pb_extension_t { pb_membersize(st, m[0]), \ pb_arraysize(st, m), ptr} +/* Allocated fields carry the size of the actual data, not the pointer */ +#define PB_REQUIRED_POINTER(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_REQUIRED | ltype, \ + fd, 0, pb_membersize(st, m[0]), 0, ptr} + +/* Optional fields don't need a has_ variable, as information would be redundant */ +#define PB_OPTIONAL_POINTER(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m[0]), 0, ptr} + +#define PB_REPEATED_POINTER(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_REPEATED | ltype, \ + fd, \ + pb_delta(st, m ## _count, m), \ + pb_membersize(st, m[0]), 0, ptr} + /* Callbacks are much like required fields except with special datatype. */ #define PB_REQUIRED_CALLBACK(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REQUIRED | ltype, \ diff --git a/pb_encode.c b/pb_encode.c index f6e08a57..9c3ed2d4 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -118,8 +118,8 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie if (count == 0) return true; - - if (count > field->array_size) + + if (PB_ATYPE(field->type) != PB_ATYPE_POINTER && count > field->array_size) PB_RETURN_ERROR(stream, "array max size exceeded"); /* We always pack arrays if the datatype allows it. */ @@ -172,8 +172,19 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie { if (!pb_encode_tag_for_field(stream, field)) return false; - if (!func(stream, field, p)) - return false; + + /* Special case for strings */ + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER && + PB_LTYPE(field->type) == PB_LTYPE_STRING) + { + if (!func(stream, field, *(const void**)p)) + return false; + } + else + { + if (!func(stream, field, p)) + return false; + } p = (const char*)p + field->data_size; } } @@ -194,12 +205,19 @@ static bool checkreturn encode_static_field(pb_ostream_t *stream, if (field->size_offset) pSize = (const char*)pData + field->size_offset; + else if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + pSize = *(const void**)pData ? &dummy : pData; else pSize = &dummy; - + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + pData = *(const void**)pData; + switch (PB_HTYPE(field->type)) { case PB_HTYPE_REQUIRED: + if (!pData) + return false; if (!pb_encode_tag_for_field(stream, field)) return false; if (!func(stream, field, pData)) @@ -257,6 +275,7 @@ static bool checkreturn encode_field(pb_ostream_t *stream, switch (PB_ATYPE(field->type)) { case PB_ATYPE_STATIC: + case PB_ATYPE_POINTER: return encode_static_field(stream, field, pData); case PB_ATYPE_CALLBACK: @@ -314,7 +333,10 @@ bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], cons while (field->tag != 0) { pData = (const char*)pData + prev_size + field->data_offset; - prev_size = field->data_size; + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + prev_size = sizeof(const void*); + else + prev_size = field->data_size; /* Special case for static arrays */ if (PB_ATYPE(field->type) == PB_ATYPE_STATIC && @@ -569,10 +591,17 @@ bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, con { const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src; - if (bytes->size + offsetof(pb_bytes_array_t, bytes) > field->data_size) - PB_RETURN_ERROR(stream, "bytes size exceeded"); - - return pb_encode_string(stream, bytes->bytes, bytes->size); + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + { + return pb_encode_string(stream, *(const uint8_t**)bytes->bytes, bytes->size); + } + else + { + if (bytes->size + offsetof(pb_bytes_array_t, bytes) > field->data_size) + PB_RETURN_ERROR(stream, "bytes size exceeded"); + + return pb_encode_string(stream, bytes->bytes, bytes->size); + } } bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src) @@ -580,12 +609,22 @@ bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, co /* strnlen() is not always available, so just use a for-loop */ size_t size = 0; const char *p = (const char*)src; - while (size < field->data_size && *p != '\0') + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { - size++; - p++; + while (*p != '\0') + { + size++; + p++; + } + } + else + { + while (size < field->data_size && *p != '\0') + { + size++; + p++; + } } - return pb_encode_string(stream, (const uint8_t*)src, size); } diff --git a/tests/alltypes_pointer/SConscript b/tests/alltypes_pointer/SConscript new file mode 100644 index 00000000..b0e35042 --- /dev/null +++ b/tests/alltypes_pointer/SConscript @@ -0,0 +1,12 @@ +# Build and run a test that encodes and decodes a message that contains +# all of the Protocol Buffers data types. + +Import("env") + +env.NanopbProto(["alltypes", "alltypes.options"]) +enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) +# dec = env.Program(["decode_alltypes_pointer.c", "alltypes.pb.c", "$COMMON/pb_decode.o"]) + +env.RunTest(enc) +# env.RunTest([dec, "encode_alltypes.output"]) + diff --git a/tests/alltypes_pointer/alltypes.options b/tests/alltypes_pointer/alltypes.options new file mode 100644 index 00000000..330860aa --- /dev/null +++ b/tests/alltypes_pointer/alltypes.options @@ -0,0 +1,2 @@ +* type:FT_POINTER + diff --git a/tests/alltypes_pointer/alltypes.proto b/tests/alltypes_pointer/alltypes.proto new file mode 100644 index 00000000..a2cf8bbc --- /dev/null +++ b/tests/alltypes_pointer/alltypes.proto @@ -0,0 +1,93 @@ +message SubMessage { + required string substuff1 = 1 [default = "1"]; + required int32 substuff2 = 2 [default = 2]; + optional fixed32 substuff3 = 3 [default = 3]; +} + +message EmptyMessage { + +} + +enum MyEnum { + Zero = 0; + First = 1; + Second = 2; + Truth = 42; +} + +message AllTypes { + required int32 req_int32 = 1; + required int64 req_int64 = 2; + required uint32 req_uint32 = 3; + required uint64 req_uint64 = 4; + required sint32 req_sint32 = 5; + required sint64 req_sint64 = 6; + required bool req_bool = 7; + + required fixed32 req_fixed32 = 8; + required sfixed32 req_sfixed32= 9; + required float req_float = 10; + + required fixed64 req_fixed64 = 11; + required sfixed64 req_sfixed64= 12; + required double req_double = 13; + + required string req_string = 14; + required bytes req_bytes = 15; + required SubMessage req_submsg = 16; + required MyEnum req_enum = 17; + required EmptyMessage req_emptymsg = 18; + + + repeated int32 rep_int32 = 21; + repeated int64 rep_int64 = 22; + repeated uint32 rep_uint32 = 23; + repeated uint64 rep_uint64 = 24; + repeated sint32 rep_sint32 = 25; + repeated sint64 rep_sint64 = 26; + repeated bool rep_bool = 27; + + repeated fixed32 rep_fixed32 = 28; + repeated sfixed32 rep_sfixed32= 29; + repeated float rep_float = 30; + + repeated fixed64 rep_fixed64 = 31; + repeated sfixed64 rep_sfixed64= 32; + repeated double rep_double = 33; + + repeated string rep_string = 34; + repeated bytes rep_bytes = 35; + repeated SubMessage rep_submsg = 36; + repeated MyEnum rep_enum = 37; + repeated EmptyMessage rep_emptymsg = 38; + + optional int32 opt_int32 = 41 [default = 4041]; + optional int64 opt_int64 = 42 [default = 4042]; + optional uint32 opt_uint32 = 43 [default = 4043]; + optional uint64 opt_uint64 = 44 [default = 4044]; + optional sint32 opt_sint32 = 45 [default = 4045]; + optional sint64 opt_sint64 = 46 [default = 4046]; + optional bool opt_bool = 47 [default = false]; + + optional fixed32 opt_fixed32 = 48 [default = 4048]; + optional sfixed32 opt_sfixed32= 49 [default = 4049]; + optional float opt_float = 50 [default = 4050]; + + optional fixed64 opt_fixed64 = 51 [default = 4051]; + optional sfixed64 opt_sfixed64= 52 [default = 4052]; + optional double opt_double = 53 [default = 4053]; + + optional string opt_string = 54 [default = "4054"]; + optional bytes opt_bytes = 55 [default = "4055"]; + optional SubMessage opt_submsg = 56; + optional MyEnum opt_enum = 57 [default = Second]; + optional EmptyMessage opt_emptymsg = 58; + + // Just to make sure that the size of the fields has been calculated + // properly, i.e. otherwise a bug in last field might not be detected. + required int32 end = 99; + + + extensions 200 to 255; +} + diff --git a/tests/alltypes_pointer/encode_alltypes_pointer.c b/tests/alltypes_pointer/encode_alltypes_pointer.c new file mode 100644 index 00000000..cb2fe3d5 --- /dev/null +++ b/tests/alltypes_pointer/encode_alltypes_pointer.c @@ -0,0 +1,142 @@ +/* Attempts to test all the datatypes supported by ProtoBuf. + */ + +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "test_helpers.h" + +int main(int argc, char **argv) +{ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Initialize values to encode */ + int32_t value_int32 = -1000; + int64_t value_int64 = -10000000000; + + uint32_t value_uint32 = 1000; + uint64_t value_uint64 = 10000000000; + + bool value_bool = true; + float value_float = 1000.0f; + double value_double = 1000.0f; + + char *value_string = "1000"; + AllTypes_req_bytes_t value_req_bytes; + AllTypes_rep_bytes_t value_rep_bytes; + AllTypes_opt_bytes_t value_opt_bytes; + + SubMessage value_submessage = {0}; + MyEnum value_enum = MyEnum_Truth; + EmptyMessage value_empty_message = {0}; + + /* Initialize the structure with constants */ + AllTypes alltypes = {0}; + + alltypes.req_int32 = &value_int32; + alltypes.req_int64 = &value_int64; + alltypes.req_uint32 = &value_uint32; + alltypes.req_uint64 = &value_uint64; + alltypes.req_sint32 = &value_int32; + alltypes.req_sint64 = &value_int64; + alltypes.req_bool = &value_bool; + + alltypes.req_fixed32 = &value_uint32; + alltypes.req_sfixed32 = &value_int32; + alltypes.req_float = &value_float; + + alltypes.req_fixed64 = &value_uint64; + alltypes.req_sfixed64 = &value_int64; + alltypes.req_double = &value_double; + + value_req_bytes.bytes = (uint8_t*)"1000"; + value_req_bytes.size = 4; + + alltypes.req_string = value_string; + alltypes.req_bytes = &value_req_bytes; + + value_submessage.substuff1 = value_string; + value_submessage.substuff2 = &value_int32; + + alltypes.req_submsg = &value_submessage; + alltypes.req_enum = &value_enum; + alltypes.req_emptymsg = &value_empty_message; + + alltypes.rep_int32_count = 1; alltypes.rep_int32 = &value_int32; + alltypes.rep_int64_count = 1; alltypes.rep_int64 = &value_int64; + alltypes.rep_uint32_count = 1; alltypes.rep_uint32 = &value_uint32; + alltypes.rep_uint64_count = 1; alltypes.rep_uint64 = &value_uint64; + alltypes.rep_sint32_count = 1; alltypes.rep_sint32 = &value_int32; + alltypes.rep_sint64_count = 1; alltypes.rep_sint64 = &value_int64; + alltypes.rep_bool_count = 1; alltypes.rep_bool = &value_bool; + + alltypes.rep_fixed32_count = 1; alltypes.rep_fixed32 = &value_uint32; + alltypes.rep_sfixed32_count = 1; alltypes.rep_sfixed32 = &value_int32; + alltypes.rep_float_count = 1; alltypes.rep_float = &value_float; + + alltypes.rep_fixed64_count = 1; alltypes.rep_fixed64 = &value_uint64; + alltypes.rep_sfixed64_count = 1; alltypes.rep_sfixed64 = &value_int64; + alltypes.rep_double_count = 1; alltypes.rep_double = &value_double; + + value_rep_bytes.bytes = (uint8_t*)"1000"; + value_rep_bytes.size = 4; + + alltypes.rep_string_count = 1; alltypes.rep_string = (char **)&value_string; + alltypes.rep_bytes_count = 0; alltypes.rep_bytes = &value_rep_bytes; + + alltypes.rep_submsg_count = 1; alltypes.rep_submsg = &value_submessage; + alltypes.rep_enum_count = 1; alltypes.rep_enum = &value_enum; + alltypes.rep_emptymsg_count = 1; alltypes.rep_emptymsg = &value_empty_message; + + if (mode != 0) + { + /* Fill in values for optional fields */ + alltypes.opt_int32 = &value_int32; + alltypes.opt_int64 = &value_int64; + alltypes.opt_uint32 = &value_uint32; + alltypes.opt_uint64 = &value_uint64; + alltypes.opt_sint32 = &value_int32; + alltypes.opt_sint64 = &value_int64; + alltypes.opt_bool = &value_bool; + + alltypes.opt_fixed32 = &value_uint32; + alltypes.opt_sfixed32 = &value_int32; + alltypes.opt_float = &value_float; + + alltypes.opt_fixed64 = &value_uint64; + alltypes.opt_sfixed64 = &value_int64; + alltypes.opt_double = &value_double; + + value_opt_bytes.bytes = (uint8_t*)"1000"; + value_opt_bytes.size = 4; + + alltypes.opt_string = value_string; + alltypes.opt_bytes = &value_opt_bytes; + + alltypes.opt_submsg = &value_submessage; + alltypes.opt_enum = &value_enum; + alltypes.opt_emptymsg = &value_empty_message; + } + + alltypes.end = &value_int32; + + { + uint8_t buffer[4096]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + /*SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout);*/ /* TODO: use this to validate decoding, when implemented */ + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } + } +} -- cgit 1.2.3-korg From 414e637867c513ba9aa199e106c6d576e5b60287 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 9 Dec 2013 19:15:26 +0200 Subject: Const-correctness and small cleanup. --- pb_encode.c | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/pb_encode.c b/pb_encode.c index 9c3ed2d4..cbf7a667 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -173,11 +173,14 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie if (!pb_encode_tag_for_field(stream, field)) return false; - /* Special case for strings */ + /* Normally the data is stored directly in the array entries, but + * for pointer-type string fields, the array entries are actually + * string pointers. So we have to dereference once more to get to + * the character data. */ if (PB_ATYPE(field->type) == PB_ATYPE_POINTER && PB_LTYPE(field->type) == PB_LTYPE_STRING) { - if (!func(stream, field, *(const void**)p)) + if (!func(stream, field, *(const void* const*)p)) return false; } else @@ -199,19 +202,25 @@ static bool checkreturn encode_static_field(pb_ostream_t *stream, { pb_encoder_t func; const void *pSize; - bool dummy = true; + bool implicit_has = true; func = PB_ENCODERS[PB_LTYPE(field->type)]; if (field->size_offset) pSize = (const char*)pData + field->size_offset; - else if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) - pSize = *(const void**)pData ? &dummy : pData; else - pSize = &dummy; + pSize = &implicit_has; if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) - pData = *(const void**)pData; + { + /* pData is a pointer to the field, which contains pointer to + * the data. If the 2nd pointer is NULL, it is interpreted as if + * the has_field was false. + */ + + pData = *(const void* const*)pData; + implicit_has = (pData != NULL); + } switch (PB_HTYPE(field->type)) { @@ -606,25 +615,20 @@ bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, con bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - /* strnlen() is not always available, so just use a for-loop */ + /* strnlen() is not always available, so just use a loop */ size_t size = 0; + size_t max_size = field->data_size; const char *p = (const char*)src; + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + max_size = (size_t)-1; + + while (size < max_size && *p != '\0') { - while (*p != '\0') - { - size++; - p++; - } - } - else - { - while (size < field->data_size && *p != '\0') - { - size++; - p++; - } + size++; + p++; } + return pb_encode_string(stream, (const uint8_t*)src, size); } -- cgit 1.2.3-korg From 25516b059137be64735276bbe3a96029f4c926be Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 9 Dec 2013 19:19:12 +0200 Subject: Organize allocation logic in generator, add pb_bytes_ptr_t. Allocation decision is now made before the field data type is decided. This way the data type decisions can more cleanly account for the allocation type, i.e. FT_DEFAULT logic etc. Added pb_bytes_ptr_t for pointer-allocated bytes-fields. There is no point generating separate structs for these, as they would all be of the same type. --- generator/nanopb_generator.py | 87 ++++++++++++------------ pb.h | 14 +++- pb_encode.c | 6 +- tests/alltypes_pointer/encode_alltypes_pointer.c | 22 ++---- 4 files changed, 63 insertions(+), 66 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 69548ad6..d67fe94f 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -189,6 +189,34 @@ class Field: else: raise NotImplementedError(desc.label) + # Check if the field can be implemented with static allocation + # i.e. whether the data size is known. + if desc.type == FieldD.TYPE_STRING and self.max_size is None: + can_be_static = False + + if desc.type == FieldD.TYPE_BYTES and self.max_size is None: + can_be_static = False + + # Decide how the field data will be allocated + if field_options.type == nanopb_pb2.FT_DEFAULT: + if can_be_static: + field_options.type = nanopb_pb2.FT_STATIC + else: + field_options.type = nanopb_pb2.FT_CALLBACK + + if field_options.type == nanopb_pb2.FT_STATIC and not can_be_static: + raise Exception("Field %s is defined as static, but max_size or " + "max_count is not given." % self.name) + + if field_options.type == nanopb_pb2.FT_STATIC: + self.allocation = 'STATIC' + elif field_options.type == nanopb_pb2.FT_POINTER: + self.allocation = 'POINTER' + elif field_options.type == nanopb_pb2.FT_CALLBACK: + self.allocation = 'CALLBACK' + else: + raise NotImplementedError(field_options.type) + # Decide the C data type to use in the struct. if datatypes.has_key(desc.type): self.ctype, self.pbtype, self.enc_size = datatypes[desc.type] @@ -200,27 +228,18 @@ class Field: self.enc_size = 5 # protoc rejects enum values > 32 bits elif desc.type == FieldD.TYPE_STRING: self.pbtype = 'STRING' - if field_options.type == nanopb_pb2.FT_POINTER: + self.ctype = 'char' + if self.allocation == 'STATIC': self.ctype = 'char' - self.enc_size = None - else: - if self.max_size is None: - can_be_static = False - else: - self.ctype = 'char' - self.array_decl += '[%d]' % self.max_size - self.enc_size = varint_max_size(self.max_size) + self.max_size + self.array_decl += '[%d]' % self.max_size + self.enc_size = varint_max_size(self.max_size) + self.max_size elif desc.type == FieldD.TYPE_BYTES: self.pbtype = 'BYTES' - if field_options.type == nanopb_pb2.FT_POINTER: + if self.allocation == 'STATIC': self.ctype = self.struct_name + self.name + 't' - self.enc_size = None - else: - if self.max_size is None: - can_be_static = False - else: - self.ctype = self.struct_name + self.name + 't' - self.enc_size = varint_max_size(self.max_size) + self.max_size + self.enc_size = varint_max_size(self.max_size) + self.max_size + elif self.allocation == 'POINTER': + self.ctype = 'pb_bytes_ptr_t' elif desc.type == FieldD.TYPE_MESSAGE: self.pbtype = 'MESSAGE' self.ctype = self.submsgname = names_from_type_name(desc.type_name) @@ -228,26 +247,6 @@ class Field: else: raise NotImplementedError(desc.type) - if field_options.type == nanopb_pb2.FT_DEFAULT: - if can_be_static: - field_options.type = nanopb_pb2.FT_STATIC - else: - field_options.type = nanopb_pb2.FT_CALLBACK - - if field_options.type == nanopb_pb2.FT_STATIC and not can_be_static: - raise Exception("Field %s is defined as static, but max_size or max_count is not given." % self.name) - - if field_options.type == nanopb_pb2.FT_STATIC: - self.allocation = 'STATIC' - elif field_options.type == nanopb_pb2.FT_POINTER: - self.allocation = 'POINTER' - elif field_options.type == nanopb_pb2.FT_CALLBACK: - self.allocation = 'CALLBACK' - self.ctype = 'pb_callback_t' - self.array_decl = '' - else: - raise NotImplementedError(field_options.type) - def __cmp__(self, other): return cmp(self.tag, other.tag) @@ -257,15 +256,16 @@ class Field: if self.rules == 'REPEATED': result += ' size_t ' + self.name + '_count;\n' - # Use struct definition, so recursive submessages are possible if self.pbtype == 'MESSAGE': + # Use struct definition, so recursive submessages are possible result += ' struct _%s *%s;' % (self.ctype, self.name) - - # String arrays need to be defined as pointers to pointers elif self.rules == 'REPEATED' and self.pbtype == 'STRING': + # String arrays need to be defined as pointers to pointers result += ' %s **%s;' % (self.ctype, self.name) else: result += ' %s *%s;' % (self.ctype, self.name) + elif self.allocation == 'CALLBACK': + result += ' pb_callback_t %s;' % self.name else: if self.rules == 'OPTIONAL' and self.allocation == 'STATIC': result += ' bool has_' + self.name + ';\n' @@ -276,13 +276,10 @@ class Field: def types(self): '''Return definitions for any special types this field might need.''' - if self.pbtype == 'BYTES' and (self.allocation == 'STATIC' or self.allocation == 'POINTER'): + if self.pbtype == 'BYTES' and self.allocation == 'STATIC': result = 'typedef struct {\n' result += ' size_t size;\n' - if self.allocation == 'POINTER': - result += ' uint8_t *bytes;\n' - else: - result += ' uint8_t bytes[%d];\n' % self.max_size + result += ' uint8_t bytes[%d];\n' % self.max_size result += '} %s;\n' % self.ctype else: result = None diff --git a/pb.h b/pb.h index d839be59..76980e4c 100644 --- a/pb.h +++ b/pb.h @@ -228,9 +228,17 @@ struct _pb_bytes_array_t { size_t size; uint8_t bytes[1]; }; - typedef struct _pb_bytes_array_t pb_bytes_array_t; +/* Same, except for pointer-type fields. There is no need to variable struct + * length in this case. + */ +struct _pb_bytes_ptr_t { + size_t size; + uint8_t *bytes; +}; +typedef struct _pb_bytes_ptr_t pb_bytes_ptr_t; + /* This structure is used for giving the callback function. * It is stored in the message structure and filled in by the method that * calls pb_decode. @@ -377,10 +385,10 @@ struct _pb_extension_t { {tag, PB_ATYPE_POINTER | PB_HTYPE_OPTIONAL | ltype, \ fd, 0, pb_membersize(st, m[0]), 0, ptr} +/* Repeated fields have a _count field and a pointer to array of pointers */ #define PB_REPEATED_POINTER(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_POINTER | PB_HTYPE_REPEATED | ltype, \ - fd, \ - pb_delta(st, m ## _count, m), \ + fd, pb_delta(st, m ## _count, m), \ pb_membersize(st, m[0]), 0, ptr} /* Callbacks are much like required fields except with special datatype. */ diff --git a/pb_encode.c b/pb_encode.c index cbf7a667..8f01b10b 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -598,14 +598,14 @@ bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, c bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src; - if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { - return pb_encode_string(stream, *(const uint8_t**)bytes->bytes, bytes->size); + const pb_bytes_ptr_t *bytes = (const pb_bytes_ptr_t*)src; + return pb_encode_string(stream, bytes->bytes, bytes->size); } else { + const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src; if (bytes->size + offsetof(pb_bytes_array_t, bytes) > field->data_size) PB_RETURN_ERROR(stream, "bytes size exceeded"); diff --git a/tests/alltypes_pointer/encode_alltypes_pointer.c b/tests/alltypes_pointer/encode_alltypes_pointer.c index cb2fe3d5..7e648e39 100644 --- a/tests/alltypes_pointer/encode_alltypes_pointer.c +++ b/tests/alltypes_pointer/encode_alltypes_pointer.c @@ -24,9 +24,8 @@ int main(int argc, char **argv) double value_double = 1000.0f; char *value_string = "1000"; - AllTypes_req_bytes_t value_req_bytes; - AllTypes_rep_bytes_t value_rep_bytes; - AllTypes_opt_bytes_t value_opt_bytes; + + pb_bytes_ptr_t value_bytes = {4, (uint8_t*)"1000"}; SubMessage value_submessage = {0}; MyEnum value_enum = MyEnum_Truth; @@ -51,11 +50,9 @@ int main(int argc, char **argv) alltypes.req_sfixed64 = &value_int64; alltypes.req_double = &value_double; - value_req_bytes.bytes = (uint8_t*)"1000"; - value_req_bytes.size = 4; - alltypes.req_string = value_string; - alltypes.req_bytes = &value_req_bytes; + + alltypes.req_bytes = &value_bytes; value_submessage.substuff1 = value_string; value_submessage.substuff2 = &value_int32; @@ -80,11 +77,8 @@ int main(int argc, char **argv) alltypes.rep_sfixed64_count = 1; alltypes.rep_sfixed64 = &value_int64; alltypes.rep_double_count = 1; alltypes.rep_double = &value_double; - value_rep_bytes.bytes = (uint8_t*)"1000"; - value_rep_bytes.size = 4; - alltypes.rep_string_count = 1; alltypes.rep_string = (char **)&value_string; - alltypes.rep_bytes_count = 0; alltypes.rep_bytes = &value_rep_bytes; + alltypes.rep_bytes_count = 1; alltypes.rep_bytes = &value_bytes; alltypes.rep_submsg_count = 1; alltypes.rep_submsg = &value_submessage; alltypes.rep_enum_count = 1; alltypes.rep_enum = &value_enum; @@ -109,11 +103,9 @@ int main(int argc, char **argv) alltypes.opt_sfixed64 = &value_int64; alltypes.opt_double = &value_double; - value_opt_bytes.bytes = (uint8_t*)"1000"; - value_opt_bytes.size = 4; - alltypes.opt_string = value_string; - alltypes.opt_bytes = &value_opt_bytes; + + alltypes.opt_bytes = &value_bytes; alltypes.opt_submsg = &value_submessage; alltypes.opt_enum = &value_enum; -- cgit 1.2.3-korg From ef07635e3549b6a3ab6a6bafc12b7960ac6b1ab9 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 9 Dec 2013 19:25:19 +0200 Subject: Rename encode_static_field -> encode_basic_field. --- pb_encode.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pb_encode.c b/pb_encode.c index 8f01b10b..476b1119 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -195,9 +195,9 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie return true; } -/* Encode a field with static allocation, i.e. one whose data is stored - * in the structure itself. */ -static bool checkreturn encode_static_field(pb_ostream_t *stream, +/* Encode a field with static or pointer allocation, i.e. one whose data + * is available to the encoder directly. */ +static bool checkreturn encode_basic_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData) { pb_encoder_t func; @@ -285,7 +285,7 @@ static bool checkreturn encode_field(pb_ostream_t *stream, { case PB_ATYPE_STATIC: case PB_ATYPE_POINTER: - return encode_static_field(stream, field, pData); + return encode_basic_field(stream, field, pData); case PB_ATYPE_CALLBACK: return encode_callback_field(stream, field, pData); -- cgit 1.2.3-korg From bc4ded3b4314e0fbd181b3cc01635052b56d2b19 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 29 Dec 2013 18:34:57 +0200 Subject: Fix missing error messages in pb_encode.c --- pb_encode.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pb_encode.c b/pb_encode.c index 476b1119..c2d0e2c4 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -226,7 +226,7 @@ static bool checkreturn encode_basic_field(pb_ostream_t *stream, { case PB_HTYPE_REQUIRED: if (!pData) - return false; + PB_RETURN_ERROR(stream, "missing required field"); if (!pb_encode_tag_for_field(stream, field)) return false; if (!func(stream, field, pData)) @@ -499,7 +499,12 @@ bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fie bool status; if (!pb_encode(&substream, fields, src_struct)) + { +#ifndef PB_NO_ERRMSG + stream->errmsg = substream.errmsg; +#endif return false; + } size = substream.bytes_written; -- cgit 1.2.3-korg From 8da15d8b802b55d04bdd84761e3c15f03b97a650 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 29 Dec 2013 18:35:14 +0200 Subject: Improve alltypes_pointer test case --- tests/alltypes_pointer/SConscript | 14 +- tests/alltypes_pointer/alltypes.options | 1 + tests/alltypes_pointer/encode_alltypes_pointer.c | 217 +++++++++++++---------- 3 files changed, 137 insertions(+), 95 deletions(-) diff --git a/tests/alltypes_pointer/SConscript b/tests/alltypes_pointer/SConscript index b0e35042..81b9fa57 100644 --- a/tests/alltypes_pointer/SConscript +++ b/tests/alltypes_pointer/SConscript @@ -1,12 +1,18 @@ -# Build and run a test that encodes and decodes a message that contains -# all of the Protocol Buffers data types. +# Encode the AllTypes message using pointers for all fields, and verify the +# output against the normal AllTypes test case. Import("env") env.NanopbProto(["alltypes", "alltypes.options"]) enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) -# dec = env.Program(["decode_alltypes_pointer.c", "alltypes.pb.c", "$COMMON/pb_decode.o"]) +# Encode and compare results env.RunTest(enc) -# env.RunTest([dec, "encode_alltypes.output"]) +env.RunTest("decode_alltypes.output", ["$BUILD/alltypes/decode_alltypes", "encode_alltypes_pointer.output"]) +env.Compare(["encode_alltypes_pointer.output", "$BUILD/alltypes/encode_alltypes.output"]) + +# Do the same thing with the optional fields present +#env.RunTest("optionals.output", enc, ARGS = ['1']) +#env.RunTest("optionals.decout", ["$BUILD/alltypes/decode_alltypes", "optionals.output"], ARGS = ['1']) +#env.Compare(["optionals.output", "$BUILD/alltypes/optionals.output"]) diff --git a/tests/alltypes_pointer/alltypes.options b/tests/alltypes_pointer/alltypes.options index 330860aa..52abeb7f 100644 --- a/tests/alltypes_pointer/alltypes.options +++ b/tests/alltypes_pointer/alltypes.options @@ -1,2 +1,3 @@ +# Generate all fields as pointers. * type:FT_POINTER diff --git a/tests/alltypes_pointer/encode_alltypes_pointer.c b/tests/alltypes_pointer/encode_alltypes_pointer.c index 7e648e39..dabee737 100644 --- a/tests/alltypes_pointer/encode_alltypes_pointer.c +++ b/tests/alltypes_pointer/encode_alltypes_pointer.c @@ -12,107 +12,142 @@ int main(int argc, char **argv) { int mode = (argc > 1) ? atoi(argv[1]) : 0; - /* Initialize values to encode */ - int32_t value_int32 = -1000; - int64_t value_int64 = -10000000000; - - uint32_t value_uint32 = 1000; - uint64_t value_uint64 = 10000000000; - - bool value_bool = true; - float value_float = 1000.0f; - double value_double = 1000.0f; - - char *value_string = "1000"; + /* Values for required fields */ + int32_t req_int32 = -1001; + int64_t req_int64 = -1002; + uint32_t req_uint32 = 1003; + uint64_t req_uint64 = 1004; + int32_t req_sint32 = -1005; + int64_t req_sint64 = -1006; + bool req_bool = true; + uint32_t req_fixed32 = 1008; + int32_t req_sfixed32 = -1009; + float req_float = 1010.0f; + uint64_t req_fixed64 = 1011; + int64_t req_sfixed64 = -1012; + double req_double = 1013.0; + char* req_string = "1014"; + pb_bytes_ptr_t req_bytes = {4, (uint8_t*)"1015"}; + static int32_t req_substuff = 1016; + SubMessage req_submsg = {"1016", &req_substuff}; + MyEnum req_enum = MyEnum_Truth; + EmptyMessage req_emptymsg = {0}; - pb_bytes_ptr_t value_bytes = {4, (uint8_t*)"1000"}; + int32_t end = 1099; - SubMessage value_submessage = {0}; - MyEnum value_enum = MyEnum_Truth; - EmptyMessage value_empty_message = {0}; - - /* Initialize the structure with constants */ - AllTypes alltypes = {0}; + /* Values for repeated fields */ + int32_t rep_int32[5] = {0, 0, 0, 0, -2001}; + int64_t rep_int64[5] = {0, 0, 0, 0, -2002}; + uint32_t rep_uint32[5] = {0, 0, 0, 0, 2003}; + uint64_t rep_uint64[5] = {0, 0, 0, 0, 2004}; + int32_t rep_sint32[5] = {0, 0, 0, 0, -2005}; + int64_t rep_sint64[5] = {0, 0, 0, 0, -2006}; + bool rep_bool[5] = {false, false, false, false, true}; + uint32_t rep_fixed32[5] = {0, 0, 0, 0, 2008}; + int32_t rep_sfixed32[5] = {0, 0, 0, 0, -2009}; + float rep_float[5] = {0, 0, 0, 0, 2010.0f}; + uint64_t rep_fixed64[5] = {0, 0, 0, 0, 2011}; + int64_t rep_sfixed64[5] = {0, 0, 0, 0, -2012}; + double rep_double[5] = {0, 0, 0, 0, 2013.0f}; + char* rep_string[5] = {"", "", "", "", "2014"}; + pb_bytes_ptr_t rep_bytes[5] = {{0,0}, {0,0}, {0,0}, {0,0}, {4, (uint8_t*)"2015"}}; + static int32_t rep_sub2zero = 0; + static int32_t rep_substuff2 = 2016; + static uint32_t rep_substuff3 = 2016; + SubMessage rep_submsg[5] = {{"", &rep_sub2zero}, + {"", &rep_sub2zero}, + {"", &rep_sub2zero}, + {"", &rep_sub2zero}, + {"2016", &rep_substuff2, &rep_substuff3}}; + MyEnum rep_enum[5] = {0, 0, 0, 0, MyEnum_Truth}; + EmptyMessage rep_emptymsg[5] = {{0}, {0}, {0}, {0}, {0}}; - alltypes.req_int32 = &value_int32; - alltypes.req_int64 = &value_int64; - alltypes.req_uint32 = &value_uint32; - alltypes.req_uint64 = &value_uint64; - alltypes.req_sint32 = &value_int32; - alltypes.req_sint64 = &value_int64; - alltypes.req_bool = &value_bool; - - alltypes.req_fixed32 = &value_uint32; - alltypes.req_sfixed32 = &value_int32; - alltypes.req_float = &value_float; - - alltypes.req_fixed64 = &value_uint64; - alltypes.req_sfixed64 = &value_int64; - alltypes.req_double = &value_double; - - alltypes.req_string = value_string; - - alltypes.req_bytes = &value_bytes; + /* Values for optional fields */ + int32_t opt_int32 = 3041; + int64_t opt_int64 = 3042; + uint32_t opt_uint32 = 3043; + uint64_t opt_uint64 = 3044; + int32_t opt_sint32 = 3045; + int64_t opt_sint64 = 3046; + bool opt_bool = true; + uint32_t opt_fixed32 = 3048; + int32_t opt_sfixed32 = 3049; + float opt_float = 3050.0f; + uint64_t opt_fixed64 = 3051; + int64_t opt_sfixed64 = 3052; + double opt_double = 3053.0; + char* opt_string = "3054"; + pb_bytes_ptr_t opt_bytes = {4, (uint8_t*)"3055"}; + static int32_t opt_substuff = 3056; + SubMessage opt_submsg = {"3056", &opt_substuff}; + MyEnum opt_enum = MyEnum_Truth; + EmptyMessage opt_emptymsg = {0}; - value_submessage.substuff1 = value_string; - value_submessage.substuff2 = &value_int32; + /* Initialize the message struct with pointers to the fields. */ + AllTypes alltypes = {0}; - alltypes.req_submsg = &value_submessage; - alltypes.req_enum = &value_enum; - alltypes.req_emptymsg = &value_empty_message; - - alltypes.rep_int32_count = 1; alltypes.rep_int32 = &value_int32; - alltypes.rep_int64_count = 1; alltypes.rep_int64 = &value_int64; - alltypes.rep_uint32_count = 1; alltypes.rep_uint32 = &value_uint32; - alltypes.rep_uint64_count = 1; alltypes.rep_uint64 = &value_uint64; - alltypes.rep_sint32_count = 1; alltypes.rep_sint32 = &value_int32; - alltypes.rep_sint64_count = 1; alltypes.rep_sint64 = &value_int64; - alltypes.rep_bool_count = 1; alltypes.rep_bool = &value_bool; - - alltypes.rep_fixed32_count = 1; alltypes.rep_fixed32 = &value_uint32; - alltypes.rep_sfixed32_count = 1; alltypes.rep_sfixed32 = &value_int32; - alltypes.rep_float_count = 1; alltypes.rep_float = &value_float; + alltypes.req_int32 = &req_int32; + alltypes.req_int64 = &req_int64; + alltypes.req_uint32 = &req_uint32; + alltypes.req_uint64 = &req_uint64; + alltypes.req_sint32 = &req_sint32; + alltypes.req_sint64 = &req_sint64; + alltypes.req_bool = &req_bool; + alltypes.req_fixed32 = &req_fixed32; + alltypes.req_sfixed32 = &req_sfixed32; + alltypes.req_float = &req_float; + alltypes.req_fixed64 = &req_fixed64; + alltypes.req_sfixed64 = &req_sfixed64; + alltypes.req_double = &req_double; + alltypes.req_string = req_string; + alltypes.req_bytes = &req_bytes; + alltypes.req_submsg = &req_submsg; + alltypes.req_enum = &req_enum; + alltypes.req_emptymsg = &req_emptymsg; - alltypes.rep_fixed64_count = 1; alltypes.rep_fixed64 = &value_uint64; - alltypes.rep_sfixed64_count = 1; alltypes.rep_sfixed64 = &value_int64; - alltypes.rep_double_count = 1; alltypes.rep_double = &value_double; - - alltypes.rep_string_count = 1; alltypes.rep_string = (char **)&value_string; - alltypes.rep_bytes_count = 1; alltypes.rep_bytes = &value_bytes; - - alltypes.rep_submsg_count = 1; alltypes.rep_submsg = &value_submessage; - alltypes.rep_enum_count = 1; alltypes.rep_enum = &value_enum; - alltypes.rep_emptymsg_count = 1; alltypes.rep_emptymsg = &value_empty_message; + alltypes.rep_int32_count = 5; alltypes.rep_int32 = rep_int32; + alltypes.rep_int64_count = 5; alltypes.rep_int64 = rep_int64; + alltypes.rep_uint32_count = 5; alltypes.rep_uint32 = rep_uint32; + alltypes.rep_uint64_count = 5; alltypes.rep_uint64 = rep_uint64; + alltypes.rep_sint32_count = 5; alltypes.rep_sint32 = rep_sint32; + alltypes.rep_sint64_count = 5; alltypes.rep_sint64 = rep_sint64; + alltypes.rep_bool_count = 5; alltypes.rep_bool = rep_bool; + alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32 = rep_fixed32; + alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32 = rep_sfixed32; + alltypes.rep_float_count = 5; alltypes.rep_float = rep_float; + alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64 = rep_fixed64; + alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64 = rep_sfixed64; + alltypes.rep_double_count = 5; alltypes.rep_double = rep_double; + alltypes.rep_string_count = 5; alltypes.rep_string = rep_string; + alltypes.rep_bytes_count = 5; alltypes.rep_bytes = rep_bytes; + alltypes.rep_submsg_count = 5; alltypes.rep_submsg = rep_submsg; + alltypes.rep_enum_count = 5; alltypes.rep_enum = rep_enum; + alltypes.rep_emptymsg_count = 5; alltypes.rep_emptymsg = rep_emptymsg; if (mode != 0) { /* Fill in values for optional fields */ - alltypes.opt_int32 = &value_int32; - alltypes.opt_int64 = &value_int64; - alltypes.opt_uint32 = &value_uint32; - alltypes.opt_uint64 = &value_uint64; - alltypes.opt_sint32 = &value_int32; - alltypes.opt_sint64 = &value_int64; - alltypes.opt_bool = &value_bool; - - alltypes.opt_fixed32 = &value_uint32; - alltypes.opt_sfixed32 = &value_int32; - alltypes.opt_float = &value_float; - - alltypes.opt_fixed64 = &value_uint64; - alltypes.opt_sfixed64 = &value_int64; - alltypes.opt_double = &value_double; - - alltypes.opt_string = value_string; - - alltypes.opt_bytes = &value_bytes; - - alltypes.opt_submsg = &value_submessage; - alltypes.opt_enum = &value_enum; - alltypes.opt_emptymsg = &value_empty_message; + alltypes.opt_int32 = &opt_int32; + alltypes.opt_int64 = &opt_int64; + alltypes.opt_uint32 = &opt_uint32; + alltypes.opt_uint64 = &opt_uint64; + alltypes.opt_sint32 = &opt_sint32; + alltypes.opt_sint64 = &opt_sint64; + alltypes.opt_bool = &opt_bool; + alltypes.opt_fixed32 = &opt_fixed32; + alltypes.opt_sfixed32 = &opt_sfixed32; + alltypes.opt_float = &opt_float; + alltypes.opt_fixed64 = &opt_fixed64; + alltypes.opt_sfixed64 = &opt_sfixed64; + alltypes.opt_double = &opt_double; + alltypes.opt_string = opt_string; + alltypes.opt_bytes = &opt_bytes; + alltypes.opt_submsg = &opt_submsg; + alltypes.opt_enum = &opt_enum; + alltypes.opt_emptymsg = &opt_emptymsg; } - alltypes.end = &value_int32; + alltypes.end = &end; { uint8_t buffer[4096]; @@ -121,8 +156,8 @@ int main(int argc, char **argv) /* Now encode it and check if we succeeded. */ if (pb_encode(&stream, AllTypes_fields, &alltypes)) { - /*SET_BINARY_MODE(stdout); - fwrite(buffer, 1, stream.bytes_written, stdout);*/ /* TODO: use this to validate decoding, when implemented */ + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); return 0; /* Success */ } else -- cgit 1.2.3-korg From f4146edf01b8226709538a46919c666c37ebb50b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 29 Dec 2013 18:44:58 +0200 Subject: Bring alltypes_pointer testcase up to date --- tests/alltypes_pointer/SConscript | 9 ++- tests/alltypes_pointer/alltypes.proto | 93 ------------------------ tests/alltypes_pointer/encode_alltypes_pointer.c | 18 +++++ 3 files changed, 24 insertions(+), 96 deletions(-) delete mode 100644 tests/alltypes_pointer/alltypes.proto diff --git a/tests/alltypes_pointer/SConscript b/tests/alltypes_pointer/SConscript index 81b9fa57..11a287a8 100644 --- a/tests/alltypes_pointer/SConscript +++ b/tests/alltypes_pointer/SConscript @@ -3,6 +3,9 @@ Import("env") +c = Copy("$TARGET", "$SOURCE") +env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) + env.NanopbProto(["alltypes", "alltypes.options"]) enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) @@ -12,7 +15,7 @@ env.RunTest("decode_alltypes.output", ["$BUILD/alltypes/decode_alltypes", "encod env.Compare(["encode_alltypes_pointer.output", "$BUILD/alltypes/encode_alltypes.output"]) # Do the same thing with the optional fields present -#env.RunTest("optionals.output", enc, ARGS = ['1']) -#env.RunTest("optionals.decout", ["$BUILD/alltypes/decode_alltypes", "optionals.output"], ARGS = ['1']) -#env.Compare(["optionals.output", "$BUILD/alltypes/optionals.output"]) +env.RunTest("optionals.output", enc, ARGS = ['1']) +env.RunTest("optionals.decout", ["$BUILD/alltypes/decode_alltypes", "optionals.output"], ARGS = ['1']) +env.Compare(["optionals.output", "$BUILD/alltypes/optionals.output"]) diff --git a/tests/alltypes_pointer/alltypes.proto b/tests/alltypes_pointer/alltypes.proto deleted file mode 100644 index a2cf8bbc..00000000 --- a/tests/alltypes_pointer/alltypes.proto +++ /dev/null @@ -1,93 +0,0 @@ -message SubMessage { - required string substuff1 = 1 [default = "1"]; - required int32 substuff2 = 2 [default = 2]; - optional fixed32 substuff3 = 3 [default = 3]; -} - -message EmptyMessage { - -} - -enum MyEnum { - Zero = 0; - First = 1; - Second = 2; - Truth = 42; -} - -message AllTypes { - required int32 req_int32 = 1; - required int64 req_int64 = 2; - required uint32 req_uint32 = 3; - required uint64 req_uint64 = 4; - required sint32 req_sint32 = 5; - required sint64 req_sint64 = 6; - required bool req_bool = 7; - - required fixed32 req_fixed32 = 8; - required sfixed32 req_sfixed32= 9; - required float req_float = 10; - - required fixed64 req_fixed64 = 11; - required sfixed64 req_sfixed64= 12; - required double req_double = 13; - - required string req_string = 14; - required bytes req_bytes = 15; - required SubMessage req_submsg = 16; - required MyEnum req_enum = 17; - required EmptyMessage req_emptymsg = 18; - - - repeated int32 rep_int32 = 21; - repeated int64 rep_int64 = 22; - repeated uint32 rep_uint32 = 23; - repeated uint64 rep_uint64 = 24; - repeated sint32 rep_sint32 = 25; - repeated sint64 rep_sint64 = 26; - repeated bool rep_bool = 27; - - repeated fixed32 rep_fixed32 = 28; - repeated sfixed32 rep_sfixed32= 29; - repeated float rep_float = 30; - - repeated fixed64 rep_fixed64 = 31; - repeated sfixed64 rep_sfixed64= 32; - repeated double rep_double = 33; - - repeated string rep_string = 34; - repeated bytes rep_bytes = 35; - repeated SubMessage rep_submsg = 36; - repeated MyEnum rep_enum = 37; - repeated EmptyMessage rep_emptymsg = 38; - - optional int32 opt_int32 = 41 [default = 4041]; - optional int64 opt_int64 = 42 [default = 4042]; - optional uint32 opt_uint32 = 43 [default = 4043]; - optional uint64 opt_uint64 = 44 [default = 4044]; - optional sint32 opt_sint32 = 45 [default = 4045]; - optional sint64 opt_sint64 = 46 [default = 4046]; - optional bool opt_bool = 47 [default = false]; - - optional fixed32 opt_fixed32 = 48 [default = 4048]; - optional sfixed32 opt_sfixed32= 49 [default = 4049]; - optional float opt_float = 50 [default = 4050]; - - optional fixed64 opt_fixed64 = 51 [default = 4051]; - optional sfixed64 opt_sfixed64= 52 [default = 4052]; - optional double opt_double = 53 [default = 4053]; - - optional string opt_string = 54 [default = "4054"]; - optional bytes opt_bytes = 55 [default = "4055"]; - optional SubMessage opt_submsg = 56; - optional MyEnum opt_enum = 57 [default = Second]; - optional EmptyMessage opt_emptymsg = 58; - - // Just to make sure that the size of the fields has been calculated - // properly, i.e. otherwise a bug in last field might not be detected. - required int32 end = 99; - - - extensions 200 to 255; -} - diff --git a/tests/alltypes_pointer/encode_alltypes_pointer.c b/tests/alltypes_pointer/encode_alltypes_pointer.c index dabee737..6484ced8 100644 --- a/tests/alltypes_pointer/encode_alltypes_pointer.c +++ b/tests/alltypes_pointer/encode_alltypes_pointer.c @@ -83,6 +83,23 @@ int main(int argc, char **argv) MyEnum opt_enum = MyEnum_Truth; EmptyMessage opt_emptymsg = {0}; + /* Values for the Limits message. */ + static int32_t int32_min = INT32_MIN; + static int32_t int32_max = INT32_MAX; + static uint32_t uint32_min = 0; + static uint32_t uint32_max = UINT32_MAX; + static int64_t int64_min = INT64_MIN; + static int64_t int64_max = INT64_MAX; + static uint64_t uint64_min = 0; + static uint64_t uint64_max = UINT64_MAX; + static HugeEnum enum_min = HugeEnum_Negative; + static HugeEnum enum_max = HugeEnum_Positive; + Limits req_limits = {&int32_min, &int32_max, + &uint32_min, &uint32_max, + &int64_min, &int64_max, + &uint64_min, &uint64_max, + &enum_min, &enum_max}; + /* Initialize the message struct with pointers to the fields. */ AllTypes alltypes = {0}; @@ -104,6 +121,7 @@ int main(int argc, char **argv) alltypes.req_submsg = &req_submsg; alltypes.req_enum = &req_enum; alltypes.req_emptymsg = &req_emptymsg; + alltypes.req_limits = &req_limits; alltypes.rep_int32_count = 5; alltypes.rep_int32 = rep_int32; alltypes.rep_int64_count = 5; alltypes.rep_int64 = rep_int64; -- cgit 1.2.3-korg From 356535da9bd79252e5365a6e5648b7fcd28b367c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 15:07:12 +0200 Subject: Move the generator .proto files to a subdir, and get rid of precompiled versions. --- generator/Makefile | 5 -- generator/nanopb.proto | 69 ------------------ generator/nanopb_generator.py | 4 +- generator/nanopb_pb2.py | 163 ------------------------------------------ generator/plugin.proto | 145 ------------------------------------- generator/plugin_pb2.py | 161 ----------------------------------------- generator/proto/Makefile | 7 ++ generator/proto/__init__.py | 0 generator/proto/nanopb.proto | 69 ++++++++++++++++++ generator/proto/plugin.proto | 145 +++++++++++++++++++++++++++++++++++++ 10 files changed, 223 insertions(+), 545 deletions(-) delete mode 100644 generator/Makefile delete mode 100644 generator/nanopb.proto delete mode 100644 generator/nanopb_pb2.py delete mode 100644 generator/plugin.proto delete mode 100644 generator/plugin_pb2.py create mode 100644 generator/proto/Makefile create mode 100644 generator/proto/__init__.py create mode 100644 generator/proto/nanopb.proto create mode 100644 generator/proto/plugin.proto diff --git a/generator/Makefile b/generator/Makefile deleted file mode 100644 index f2b02a95..00000000 --- a/generator/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -nanopb_pb2.py: nanopb.proto - protoc --python_out=. -I /usr/include -I . nanopb.proto - -plugin_pb2.py: plugin.proto - protoc --python_out=. -I /usr/include -I . plugin.proto diff --git a/generator/nanopb.proto b/generator/nanopb.proto deleted file mode 100644 index 2be2f803..00000000 --- a/generator/nanopb.proto +++ /dev/null @@ -1,69 +0,0 @@ -// Custom options for defining: -// - Maximum size of string/bytes -// - Maximum number of elements in array -// -// These are used by nanopb to generate statically allocable structures -// for memory-limited environments. - -import "google/protobuf/descriptor.proto"; - -option java_package = "fi.kapsi.koti.jpa.nanopb"; - -enum FieldType { - FT_DEFAULT = 0; // Automatically decide field type, generate static field if possible. - FT_CALLBACK = 1; // Always generate a callback field. - FT_POINTER = 4; // Always generate a dynamically allocated field. - FT_STATIC = 2; // Generate a static field or raise an exception if not possible. - FT_IGNORE = 3; // Ignore the field completely. -} - -// This is the inner options message, which basically defines options for -// a field. When it is used in message or file scope, it applies to all -// fields. -message NanoPBOptions { - // Allocated size for 'bytes' and 'string' fields. - optional int32 max_size = 1; - - // Allocated number of entries in arrays ('repeated' fields) - optional int32 max_count = 2; - - // Force type of field (callback or static allocation) - optional FieldType type = 3 [default = FT_DEFAULT]; - - // Use long names for enums, i.e. EnumName_EnumValue. - optional bool long_names = 4 [default = true]; - - // Add 'packed' attribute to generated structs. - // Note: this cannot be used on CPUs that break on unaligned - // accesses to variables. - optional bool packed_struct = 5 [default = false]; -} - -// Extensions to protoc 'Descriptor' type in order to define options -// inside a .proto file. -// -// Protocol Buffers extension number registry -// -------------------------------- -// Project: Nanopb -// Contact: Petteri Aimonen -// Web site: http://kapsi.fi/~jpa/nanopb -// Extensions: 1010 (all types) -// -------------------------------- - -extend google.protobuf.FileOptions { - optional NanoPBOptions nanopb_fileopt = 1010; -} - -extend google.protobuf.MessageOptions { - optional NanoPBOptions nanopb_msgopt = 1010; -} - -extend google.protobuf.EnumOptions { - optional NanoPBOptions nanopb_enumopt = 1010; -} - -extend google.protobuf.FieldOptions { - optional NanoPBOptions nanopb = 1010; -} - - diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index d67fe94f..0002409b 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -16,7 +16,7 @@ except: raise try: - import nanopb_pb2 + import proto.nanopb_pb2 as nanopb_pb2 except: print print "***************************************************************" @@ -1025,7 +1025,7 @@ def main_cli(): def main_plugin(): '''Main function when invoked as a protoc plugin.''' - import plugin_pb2 + import proto.plugin_pb2 as plugin_pb2 data = sys.stdin.read() request = plugin_pb2.CodeGeneratorRequest.FromString(data) diff --git a/generator/nanopb_pb2.py b/generator/nanopb_pb2.py deleted file mode 100644 index ef1931f2..00000000 --- a/generator/nanopb_pb2.py +++ /dev/null @@ -1,163 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! - -from google.protobuf import descriptor -from google.protobuf import message -from google.protobuf import reflection -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - - -import google.protobuf.descriptor_pb2 - -DESCRIPTOR = descriptor.FileDescriptor( - name='nanopb.proto', - package='', - serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*Z\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\x0e\n\nFT_POINTER\x10\x04\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptionsB\x1a\n\x18\x66i.kapsi.koti.jpa.nanopb') - -_FIELDTYPE = descriptor.EnumDescriptor( - name='FieldType', - full_name='FieldType', - filename=None, - file=DESCRIPTOR, - values=[ - descriptor.EnumValueDescriptor( - name='FT_DEFAULT', index=0, number=0, - options=None, - type=None), - descriptor.EnumValueDescriptor( - name='FT_CALLBACK', index=1, number=1, - options=None, - type=None), - descriptor.EnumValueDescriptor( - name='FT_POINTER', index=2, number=4, - options=None, - type=None), - descriptor.EnumValueDescriptor( - name='FT_STATIC', index=3, number=2, - options=None, - type=None), - descriptor.EnumValueDescriptor( - name='FT_IGNORE', index=4, number=3, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=199, - serialized_end=289, -) - - -FT_DEFAULT = 0 -FT_CALLBACK = 1 -FT_POINTER = 4 -FT_STATIC = 2 -FT_IGNORE = 3 - -NANOPB_FILEOPT_FIELD_NUMBER = 1010 -nanopb_fileopt = descriptor.FieldDescriptor( - name='nanopb_fileopt', full_name='nanopb_fileopt', index=0, - number=1010, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=True, extension_scope=None, - options=None) -NANOPB_MSGOPT_FIELD_NUMBER = 1010 -nanopb_msgopt = descriptor.FieldDescriptor( - name='nanopb_msgopt', full_name='nanopb_msgopt', index=1, - number=1010, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=True, extension_scope=None, - options=None) -NANOPB_ENUMOPT_FIELD_NUMBER = 1010 -nanopb_enumopt = descriptor.FieldDescriptor( - name='nanopb_enumopt', full_name='nanopb_enumopt', index=2, - number=1010, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=True, extension_scope=None, - options=None) -NANOPB_FIELD_NUMBER = 1010 -nanopb = descriptor.FieldDescriptor( - name='nanopb', full_name='nanopb', index=3, - number=1010, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=True, extension_scope=None, - options=None) - - -_NANOPBOPTIONS = descriptor.Descriptor( - name='NanoPBOptions', - full_name='NanoPBOptions', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='max_size', full_name='NanoPBOptions.max_size', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='max_count', full_name='NanoPBOptions.max_count', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='type', full_name='NanoPBOptions.type', index=2, - number=3, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='long_names', full_name='NanoPBOptions.long_names', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=True, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='packed_struct', full_name='NanoPBOptions.packed_struct', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=51, - serialized_end=197, -) - -_NANOPBOPTIONS.fields_by_name['type'].enum_type = _FIELDTYPE -DESCRIPTOR.message_types_by_name['NanoPBOptions'] = _NANOPBOPTIONS - -class NanoPBOptions(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _NANOPBOPTIONS - - # @@protoc_insertion_point(class_scope:NanoPBOptions) - -nanopb_fileopt.message_type = _NANOPBOPTIONS -google.protobuf.descriptor_pb2.FileOptions.RegisterExtension(nanopb_fileopt) -nanopb_msgopt.message_type = _NANOPBOPTIONS -google.protobuf.descriptor_pb2.MessageOptions.RegisterExtension(nanopb_msgopt) -nanopb_enumopt.message_type = _NANOPBOPTIONS -google.protobuf.descriptor_pb2.EnumOptions.RegisterExtension(nanopb_enumopt) -nanopb.message_type = _NANOPBOPTIONS -google.protobuf.descriptor_pb2.FieldOptions.RegisterExtension(nanopb) -# @@protoc_insertion_point(module_scope) diff --git a/generator/plugin.proto b/generator/plugin.proto deleted file mode 100644 index 651ed10c..00000000 --- a/generator/plugin.proto +++ /dev/null @@ -1,145 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// http://code.google.com/p/protobuf/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Author: kenton@google.com (Kenton Varda) -// -// WARNING: The plugin interface is currently EXPERIMENTAL and is subject to -// change. -// -// protoc (aka the Protocol Compiler) can be extended via plugins. A plugin is -// just a program that reads a CodeGeneratorRequest from stdin and writes a -// CodeGeneratorResponse to stdout. -// -// Plugins written using C++ can use google/protobuf/compiler/plugin.h instead -// of dealing with the raw protocol defined here. -// -// A plugin executable needs only to be placed somewhere in the path. The -// plugin should be named "protoc-gen-$NAME", and will then be used when the -// flag "--${NAME}_out" is passed to protoc. - -package google.protobuf.compiler; - -import "google/protobuf/descriptor.proto"; - -// An encoded CodeGeneratorRequest is written to the plugin's stdin. -message CodeGeneratorRequest { - // The .proto files that were explicitly listed on the command-line. The - // code generator should generate code only for these files. Each file's - // descriptor will be included in proto_file, below. - repeated string file_to_generate = 1; - - // The generator parameter passed on the command-line. - optional string parameter = 2; - - // FileDescriptorProtos for all files in files_to_generate and everything - // they import. The files will appear in topological order, so each file - // appears before any file that imports it. - // - // protoc guarantees that all proto_files will be written after - // the fields above, even though this is not technically guaranteed by the - // protobuf wire format. This theoretically could allow a plugin to stream - // in the FileDescriptorProtos and handle them one by one rather than read - // the entire set into memory at once. However, as of this writing, this - // is not similarly optimized on protoc's end -- it will store all fields in - // memory at once before sending them to the plugin. - repeated FileDescriptorProto proto_file = 15; -} - -// The plugin writes an encoded CodeGeneratorResponse to stdout. -message CodeGeneratorResponse { - // Error message. If non-empty, code generation failed. The plugin process - // should exit with status code zero even if it reports an error in this way. - // - // This should be used to indicate errors in .proto files which prevent the - // code generator from generating correct code. Errors which indicate a - // problem in protoc itself -- such as the input CodeGeneratorRequest being - // unparseable -- should be reported by writing a message to stderr and - // exiting with a non-zero status code. - optional string error = 1; - - // Represents a single generated file. - message File { - // The file name, relative to the output directory. The name must not - // contain "." or ".." components and must be relative, not be absolute (so, - // the file cannot lie outside the output directory). "/" must be used as - // the path separator, not "\". - // - // If the name is omitted, the content will be appended to the previous - // file. This allows the generator to break large files into small chunks, - // and allows the generated text to be streamed back to protoc so that large - // files need not reside completely in memory at one time. Note that as of - // this writing protoc does not optimize for this -- it will read the entire - // CodeGeneratorResponse before writing files to disk. - optional string name = 1; - - // If non-empty, indicates that the named file should already exist, and the - // content here is to be inserted into that file at a defined insertion - // point. This feature allows a code generator to extend the output - // produced by another code generator. The original generator may provide - // insertion points by placing special annotations in the file that look - // like: - // @@protoc_insertion_point(NAME) - // The annotation can have arbitrary text before and after it on the line, - // which allows it to be placed in a comment. NAME should be replaced with - // an identifier naming the point -- this is what other generators will use - // as the insertion_point. Code inserted at this point will be placed - // immediately above the line containing the insertion point (thus multiple - // insertions to the same point will come out in the order they were added). - // The double-@ is intended to make it unlikely that the generated code - // could contain things that look like insertion points by accident. - // - // For example, the C++ code generator places the following line in the - // .pb.h files that it generates: - // // @@protoc_insertion_point(namespace_scope) - // This line appears within the scope of the file's package namespace, but - // outside of any particular class. Another plugin can then specify the - // insertion_point "namespace_scope" to generate additional classes or - // other declarations that should be placed in this scope. - // - // Note that if the line containing the insertion point begins with - // whitespace, the same whitespace will be added to every line of the - // inserted text. This is useful for languages like Python, where - // indentation matters. In these languages, the insertion point comment - // should be indented the same amount as any inserted code will need to be - // in order to work correctly in that context. - // - // The code generator that generates the initial file and the one which - // inserts into it must both run as part of a single invocation of protoc. - // Code generators are executed in the order in which they appear on the - // command line. - // - // If |insertion_point| is present, |name| must also be present. - optional string insertion_point = 2; - - // The file contents. - optional string content = 15; - } - repeated File file = 15; -} diff --git a/generator/plugin_pb2.py b/generator/plugin_pb2.py deleted file mode 100644 index 2d8c09d4..00000000 --- a/generator/plugin_pb2.py +++ /dev/null @@ -1,161 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! - -from google.protobuf import descriptor -from google.protobuf import message -from google.protobuf import reflection -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - - -import google.protobuf.descriptor_pb2 - -DESCRIPTOR = descriptor.FileDescriptor( - name='plugin.proto', - package='google.protobuf.compiler', - serialized_pb='\n\x0cplugin.proto\x12\x18google.protobuf.compiler\x1a google/protobuf/descriptor.proto\"}\n\x14\x43odeGeneratorRequest\x12\x18\n\x10\x66ile_to_generate\x18\x01 \x03(\t\x12\x11\n\tparameter\x18\x02 \x01(\t\x12\x38\n\nproto_file\x18\x0f \x03(\x0b\x32$.google.protobuf.FileDescriptorProto\"\xaa\x01\n\x15\x43odeGeneratorResponse\x12\r\n\x05\x65rror\x18\x01 \x01(\t\x12\x42\n\x04\x66ile\x18\x0f \x03(\x0b\x32\x34.google.protobuf.compiler.CodeGeneratorResponse.File\x1a>\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x17\n\x0finsertion_point\x18\x02 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x0f \x01(\t') - - - - -_CODEGENERATORREQUEST = descriptor.Descriptor( - name='CodeGeneratorRequest', - full_name='google.protobuf.compiler.CodeGeneratorRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='file_to_generate', full_name='google.protobuf.compiler.CodeGeneratorRequest.file_to_generate', index=0, - number=1, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='parameter', full_name='google.protobuf.compiler.CodeGeneratorRequest.parameter', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='proto_file', full_name='google.protobuf.compiler.CodeGeneratorRequest.proto_file', index=2, - number=15, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=76, - serialized_end=201, -) - - -_CODEGENERATORRESPONSE_FILE = descriptor.Descriptor( - name='File', - full_name='google.protobuf.compiler.CodeGeneratorResponse.File', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='name', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='insertion_point', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.insertion_point', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='content', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.content', index=2, - number=15, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=312, - serialized_end=374, -) - -_CODEGENERATORRESPONSE = descriptor.Descriptor( - name='CodeGeneratorResponse', - full_name='google.protobuf.compiler.CodeGeneratorResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='error', full_name='google.protobuf.compiler.CodeGeneratorResponse.error', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='file', full_name='google.protobuf.compiler.CodeGeneratorResponse.file', index=1, - number=15, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_CODEGENERATORRESPONSE_FILE, ], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=204, - serialized_end=374, -) - -_CODEGENERATORREQUEST.fields_by_name['proto_file'].message_type = google.protobuf.descriptor_pb2._FILEDESCRIPTORPROTO -_CODEGENERATORRESPONSE_FILE.containing_type = _CODEGENERATORRESPONSE; -_CODEGENERATORRESPONSE.fields_by_name['file'].message_type = _CODEGENERATORRESPONSE_FILE -DESCRIPTOR.message_types_by_name['CodeGeneratorRequest'] = _CODEGENERATORREQUEST -DESCRIPTOR.message_types_by_name['CodeGeneratorResponse'] = _CODEGENERATORRESPONSE - -class CodeGeneratorRequest(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _CODEGENERATORREQUEST - - # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorRequest) - -class CodeGeneratorResponse(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - - class File(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _CODEGENERATORRESPONSE_FILE - - # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse.File) - DESCRIPTOR = _CODEGENERATORRESPONSE - - # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse) - -# @@protoc_insertion_point(module_scope) diff --git a/generator/proto/Makefile b/generator/proto/Makefile new file mode 100644 index 00000000..2421175e --- /dev/null +++ b/generator/proto/Makefile @@ -0,0 +1,7 @@ +all: nanopb_pb2.py plugin_pb2.py + +nanopb_pb2.py: nanopb.proto + protoc --python_out=. -I /usr/include -I . nanopb.proto + +plugin_pb2.py: plugin.proto + protoc --python_out=. -I /usr/include -I . plugin.proto diff --git a/generator/proto/__init__.py b/generator/proto/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto new file mode 100644 index 00000000..2be2f803 --- /dev/null +++ b/generator/proto/nanopb.proto @@ -0,0 +1,69 @@ +// Custom options for defining: +// - Maximum size of string/bytes +// - Maximum number of elements in array +// +// These are used by nanopb to generate statically allocable structures +// for memory-limited environments. + +import "google/protobuf/descriptor.proto"; + +option java_package = "fi.kapsi.koti.jpa.nanopb"; + +enum FieldType { + FT_DEFAULT = 0; // Automatically decide field type, generate static field if possible. + FT_CALLBACK = 1; // Always generate a callback field. + FT_POINTER = 4; // Always generate a dynamically allocated field. + FT_STATIC = 2; // Generate a static field or raise an exception if not possible. + FT_IGNORE = 3; // Ignore the field completely. +} + +// This is the inner options message, which basically defines options for +// a field. When it is used in message or file scope, it applies to all +// fields. +message NanoPBOptions { + // Allocated size for 'bytes' and 'string' fields. + optional int32 max_size = 1; + + // Allocated number of entries in arrays ('repeated' fields) + optional int32 max_count = 2; + + // Force type of field (callback or static allocation) + optional FieldType type = 3 [default = FT_DEFAULT]; + + // Use long names for enums, i.e. EnumName_EnumValue. + optional bool long_names = 4 [default = true]; + + // Add 'packed' attribute to generated structs. + // Note: this cannot be used on CPUs that break on unaligned + // accesses to variables. + optional bool packed_struct = 5 [default = false]; +} + +// Extensions to protoc 'Descriptor' type in order to define options +// inside a .proto file. +// +// Protocol Buffers extension number registry +// -------------------------------- +// Project: Nanopb +// Contact: Petteri Aimonen +// Web site: http://kapsi.fi/~jpa/nanopb +// Extensions: 1010 (all types) +// -------------------------------- + +extend google.protobuf.FileOptions { + optional NanoPBOptions nanopb_fileopt = 1010; +} + +extend google.protobuf.MessageOptions { + optional NanoPBOptions nanopb_msgopt = 1010; +} + +extend google.protobuf.EnumOptions { + optional NanoPBOptions nanopb_enumopt = 1010; +} + +extend google.protobuf.FieldOptions { + optional NanoPBOptions nanopb = 1010; +} + + diff --git a/generator/proto/plugin.proto b/generator/proto/plugin.proto new file mode 100644 index 00000000..651ed10c --- /dev/null +++ b/generator/proto/plugin.proto @@ -0,0 +1,145 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// +// WARNING: The plugin interface is currently EXPERIMENTAL and is subject to +// change. +// +// protoc (aka the Protocol Compiler) can be extended via plugins. A plugin is +// just a program that reads a CodeGeneratorRequest from stdin and writes a +// CodeGeneratorResponse to stdout. +// +// Plugins written using C++ can use google/protobuf/compiler/plugin.h instead +// of dealing with the raw protocol defined here. +// +// A plugin executable needs only to be placed somewhere in the path. The +// plugin should be named "protoc-gen-$NAME", and will then be used when the +// flag "--${NAME}_out" is passed to protoc. + +package google.protobuf.compiler; + +import "google/protobuf/descriptor.proto"; + +// An encoded CodeGeneratorRequest is written to the plugin's stdin. +message CodeGeneratorRequest { + // The .proto files that were explicitly listed on the command-line. The + // code generator should generate code only for these files. Each file's + // descriptor will be included in proto_file, below. + repeated string file_to_generate = 1; + + // The generator parameter passed on the command-line. + optional string parameter = 2; + + // FileDescriptorProtos for all files in files_to_generate and everything + // they import. The files will appear in topological order, so each file + // appears before any file that imports it. + // + // protoc guarantees that all proto_files will be written after + // the fields above, even though this is not technically guaranteed by the + // protobuf wire format. This theoretically could allow a plugin to stream + // in the FileDescriptorProtos and handle them one by one rather than read + // the entire set into memory at once. However, as of this writing, this + // is not similarly optimized on protoc's end -- it will store all fields in + // memory at once before sending them to the plugin. + repeated FileDescriptorProto proto_file = 15; +} + +// The plugin writes an encoded CodeGeneratorResponse to stdout. +message CodeGeneratorResponse { + // Error message. If non-empty, code generation failed. The plugin process + // should exit with status code zero even if it reports an error in this way. + // + // This should be used to indicate errors in .proto files which prevent the + // code generator from generating correct code. Errors which indicate a + // problem in protoc itself -- such as the input CodeGeneratorRequest being + // unparseable -- should be reported by writing a message to stderr and + // exiting with a non-zero status code. + optional string error = 1; + + // Represents a single generated file. + message File { + // The file name, relative to the output directory. The name must not + // contain "." or ".." components and must be relative, not be absolute (so, + // the file cannot lie outside the output directory). "/" must be used as + // the path separator, not "\". + // + // If the name is omitted, the content will be appended to the previous + // file. This allows the generator to break large files into small chunks, + // and allows the generated text to be streamed back to protoc so that large + // files need not reside completely in memory at one time. Note that as of + // this writing protoc does not optimize for this -- it will read the entire + // CodeGeneratorResponse before writing files to disk. + optional string name = 1; + + // If non-empty, indicates that the named file should already exist, and the + // content here is to be inserted into that file at a defined insertion + // point. This feature allows a code generator to extend the output + // produced by another code generator. The original generator may provide + // insertion points by placing special annotations in the file that look + // like: + // @@protoc_insertion_point(NAME) + // The annotation can have arbitrary text before and after it on the line, + // which allows it to be placed in a comment. NAME should be replaced with + // an identifier naming the point -- this is what other generators will use + // as the insertion_point. Code inserted at this point will be placed + // immediately above the line containing the insertion point (thus multiple + // insertions to the same point will come out in the order they were added). + // The double-@ is intended to make it unlikely that the generated code + // could contain things that look like insertion points by accident. + // + // For example, the C++ code generator places the following line in the + // .pb.h files that it generates: + // // @@protoc_insertion_point(namespace_scope) + // This line appears within the scope of the file's package namespace, but + // outside of any particular class. Another plugin can then specify the + // insertion_point "namespace_scope" to generate additional classes or + // other declarations that should be placed in this scope. + // + // Note that if the line containing the insertion point begins with + // whitespace, the same whitespace will be added to every line of the + // inserted text. This is useful for languages like Python, where + // indentation matters. In these languages, the insertion point comment + // should be indented the same amount as any inserted code will need to be + // in order to work correctly in that context. + // + // The code generator that generates the initial file and the one which + // inserts into it must both run as part of a single invocation of protoc. + // Code generators are executed in the order in which they appear on the + // command line. + // + // If |insertion_point| is present, |name| must also be present. + optional string insertion_point = 2; + + // The file contents. + optional string content = 15; + } + repeated File file = 15; +} -- cgit 1.2.3-korg From 58b3a22d6d66c485bfd7c571c528e4c6d3619cb2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 15:46:41 +0200 Subject: Add script for making linux binary package --- .gitignore | 2 ++ tools/make_linux_package.sh | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100755 tools/make_linux_package.sh diff --git a/.gitignore b/.gitignore index 70f80a60..3bb09db1 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ *.pb.h *.pb *.pyc +*_pb2.py *~ *.tar.gz .sconsign.dblite @@ -13,6 +14,7 @@ config.log .sconf_temp tests/build julkaisu.txt +dist docs/*.html docs/generator_flow.png examples/simple/simple diff --git a/tools/make_linux_package.sh b/tools/make_linux_package.sh new file mode 100755 index 00000000..b0ad2dd2 --- /dev/null +++ b/tools/make_linux_package.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# Run this script in the top nanopb directory to create a binary package +# for Linux users. + +set -e +set -x + +VERSION=`git describe --always` +DEST=dist/$VERSION + +rm -rf $DEST +mkdir -p $DEST + +# Export the files from newest commit +git archive HEAD | tar x -C $DEST + +# Rebuild the Python .proto files +make -BC $DEST/generator/proto + +# Package the Python libraries +( cd $DEST/generator; bbfreeze nanopb_generator.py ) +mv $DEST/generator/dist $DEST/generator-bin + +# Package the protoc compiler +cp `which protoc` $DEST/generator-bin/protoc.bin +cat > $DEST/generator-bin/protoc << EOF +#!/bin/bash +SCRIPTDIR=\$(dirname \$(readlink -f \$0)) +export LD_LIBRARY_PATH=\$SCRIPTDIR +export PATH=\$SCRIPTDIR:\$PATH +exec \$SCRIPTDIR/protoc.bin "\$@" +EOF +chmod +x $DEST/generator-bin/protoc + +# Make the nanopb generator available as a protoc plugin +ln -s nanopb-generator $DEST/generator-bin/protoc-gen-nanopb + -- cgit 1.2.3-korg From 8c2dda67a100237bc6cdb3b814012079be2b3534 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 16:42:52 +0200 Subject: Add descriptor.proto to repository to avoid messing with protoc include path. --- generator/proto/Makefile | 4 +- generator/proto/descriptor.proto | 620 +++++++++++++++++++++++++++++++++++++++ generator/proto/nanopb.proto | 2 +- generator/proto/plugin.proto | 2 +- 4 files changed, 624 insertions(+), 4 deletions(-) create mode 100644 generator/proto/descriptor.proto diff --git a/generator/proto/Makefile b/generator/proto/Makefile index 2421175e..fea99a9c 100644 --- a/generator/proto/Makefile +++ b/generator/proto/Makefile @@ -1,7 +1,7 @@ all: nanopb_pb2.py plugin_pb2.py nanopb_pb2.py: nanopb.proto - protoc --python_out=. -I /usr/include -I . nanopb.proto + protoc --python_out=. nanopb.proto plugin_pb2.py: plugin.proto - protoc --python_out=. -I /usr/include -I . plugin.proto + protoc --python_out=. plugin.proto diff --git a/generator/proto/descriptor.proto b/generator/proto/descriptor.proto new file mode 100644 index 00000000..a785f79f --- /dev/null +++ b/generator/proto/descriptor.proto @@ -0,0 +1,620 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. +// +// The messages in this file describe the definitions found in .proto files. +// A valid .proto file can be translated directly to a FileDescriptorProto +// without any other information (e.g. without reading its imports). + + + +package google.protobuf; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DescriptorProtos"; + +// descriptor.proto must be optimized for speed because reflection-based +// algorithms don't work during bootstrapping. +option optimize_for = SPEED; + +// The protocol compiler can output a FileDescriptorSet containing the .proto +// files it parses. +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; +} + +// Describes a complete .proto file. +message FileDescriptorProto { + optional string name = 1; // file name, relative to root of source tree + optional string package = 2; // e.g. "foo", "foo.bar", etc. + + // Names of files imported by this file. + repeated string dependency = 3; + // Indexes of the public imported files in the dependency list above. + repeated int32 public_dependency = 10; + // Indexes of the weak imported files in the dependency list. + // For Google-internal migration only. Do not use. + repeated int32 weak_dependency = 11; + + // All top-level definitions in this file. + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + + optional FileOptions options = 8; + + // This field contains optional information about the original source code. + // You may safely remove this entire field whithout harming runtime + // functionality of the descriptors -- the information is needed only by + // development tools. + optional SourceCodeInfo source_code_info = 9; +} + +// Describes a message type. +message DescriptorProto { + optional string name = 1; + + repeated FieldDescriptorProto field = 2; + repeated FieldDescriptorProto extension = 6; + + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + } + repeated ExtensionRange extension_range = 5; + + optional MessageOptions options = 7; +} + +// Describes a field within a message. +message FieldDescriptorProto { + enum Type { + // 0 is reserved for errors. + // Order is weird for historical reasons. + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if + // negative values are likely. + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if + // negative values are likely. + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; // Tag-delimited aggregate. + TYPE_MESSAGE = 11; // Length-delimited aggregate. + + // New in version 2. + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; // Uses ZigZag encoding. + TYPE_SINT64 = 18; // Uses ZigZag encoding. + }; + + enum Label { + // 0 is reserved for errors + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + // TODO(sanjay): Should we add LABEL_MAP? + }; + + optional string name = 1; + optional int32 number = 3; + optional Label label = 4; + + // If type_name is set, this need not be set. If both this and type_name + // are set, this must be either TYPE_ENUM or TYPE_MESSAGE. + optional Type type = 5; + + // For message and enum types, this is the name of the type. If the name + // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping + // rules are used to find the type (i.e. first the nested types within this + // message are searched, then within the parent, on up to the root + // namespace). + optional string type_name = 6; + + // For extensions, this is the name of the type being extended. It is + // resolved in the same manner as type_name. + optional string extendee = 2; + + // For numeric types, contains the original text representation of the value. + // For booleans, "true" or "false". + // For strings, contains the default text contents (not escaped in any way). + // For bytes, contains the C escaped value. All bytes >= 128 are escaped. + // TODO(kenton): Base-64 encode? + optional string default_value = 7; + + optional FieldOptions options = 8; +} + +// Describes an enum type. +message EnumDescriptorProto { + optional string name = 1; + + repeated EnumValueDescriptorProto value = 2; + + optional EnumOptions options = 3; +} + +// Describes a value within an enum. +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + + optional EnumValueOptions options = 3; +} + +// Describes a service. +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + + optional ServiceOptions options = 3; +} + +// Describes a method of a service. +message MethodDescriptorProto { + optional string name = 1; + + // Input and output type names. These are resolved in the same way as + // FieldDescriptorProto.type_name, but must refer to a message type. + optional string input_type = 2; + optional string output_type = 3; + + optional MethodOptions options = 4; +} + + +// =================================================================== +// Options + +// Each of the definitions above may have "options" attached. These are +// just annotations which may cause code to be generated slightly differently +// or may contain hints for code that manipulates protocol messages. +// +// Clients may define custom options as extensions of the *Options messages. +// These extensions may not yet be known at parsing time, so the parser cannot +// store the values in them. Instead it stores them in a field in the *Options +// message called uninterpreted_option. This field must have the same name +// across all *Options messages. We then use this field to populate the +// extensions when we build a descriptor, at which point all protos have been +// parsed and so all extensions are known. +// +// Extension numbers for custom options may be chosen as follows: +// * For options which will only be used within a single application or +// organization, or for experimental options, use field numbers 50000 +// through 99999. It is up to you to ensure that you do not use the +// same number for multiple options. +// * For options which will be published and used publicly by multiple +// independent entities, e-mail protobuf-global-extension-registry@google.com +// to reserve extension numbers. Simply provide your project name (e.g. +// Object-C plugin) and your porject website (if available) -- there's no need +// to explain how you intend to use them. Usually you only need one extension +// number. You can declare multiple options with only one extension number by +// putting them in a sub-message. See the Custom Options section of the docs +// for examples: +// http://code.google.com/apis/protocolbuffers/docs/proto.html#options +// If this turns out to be popular, a web service will be set up +// to automatically assign option numbers. + + +message FileOptions { + + // Sets the Java package where classes generated from this .proto will be + // placed. By default, the proto package is used, but this is often + // inappropriate because proto packages do not normally start with backwards + // domain names. + optional string java_package = 1; + + + // If set, all the classes from the .proto file are wrapped in a single + // outer class with the given name. This applies to both Proto1 + // (equivalent to the old "--one_java_file" option) and Proto2 (where + // a .proto always translates to a single class, but you may want to + // explicitly choose the class name). + optional string java_outer_classname = 8; + + // If set true, then the Java code generator will generate a separate .java + // file for each top-level message, enum, and service defined in the .proto + // file. Thus, these types will *not* be nested inside the outer class + // named by java_outer_classname. However, the outer class will still be + // generated to contain the file's getDescriptor() method as well as any + // top-level extensions defined in the file. + optional bool java_multiple_files = 10 [default=false]; + + // If set true, then the Java code generator will generate equals() and + // hashCode() methods for all messages defined in the .proto file. This is + // purely a speed optimization, as the AbstractMessage base class includes + // reflection-based implementations of these methods. + optional bool java_generate_equals_and_hash = 20 [default=false]; + + // Generated classes can be optimized for speed or code size. + enum OptimizeMode { + SPEED = 1; // Generate complete code for parsing, serialization, + // etc. + CODE_SIZE = 2; // Use ReflectionOps to implement these methods. + LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime. + } + optional OptimizeMode optimize_for = 9 [default=SPEED]; + + // Sets the Go package where structs generated from this .proto will be + // placed. There is no default. + optional string go_package = 11; + + + + // Should generic services be generated in each language? "Generic" services + // are not specific to any particular RPC system. They are generated by the + // main code generators in each language (without additional plugins). + // Generic services were the only kind of service generation supported by + // early versions of proto2. + // + // Generic services are now considered deprecated in favor of using plugins + // that generate code specific to your particular RPC system. Therefore, + // these default to false. Old code which depends on generic services should + // explicitly set them to true. + optional bool cc_generic_services = 16 [default=false]; + optional bool java_generic_services = 17 [default=false]; + optional bool py_generic_services = 18 [default=false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message MessageOptions { + // Set true to use the old proto1 MessageSet wire format for extensions. + // This is provided for backwards-compatibility with the MessageSet wire + // format. You should not use this for any other reason: It's less + // efficient, has fewer features, and is more complicated. + // + // The message must be defined exactly as follows: + // message Foo { + // option message_set_wire_format = true; + // extensions 4 to max; + // } + // Note that the message cannot have any defined fields; MessageSets only + // have extensions. + // + // All extensions of your type must be singular messages; e.g. they cannot + // be int32s, enums, or repeated messages. + // + // Because this is an option, the above two restrictions are not enforced by + // the protocol compiler. + optional bool message_set_wire_format = 1 [default=false]; + + // Disables the generation of the standard "descriptor()" accessor, which can + // conflict with a field of the same name. This is meant to make migration + // from proto1 easier; new code should avoid fields named "descriptor". + optional bool no_standard_descriptor_accessor = 2 [default=false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message FieldOptions { + // The ctype option instructs the C++ code generator to use a different + // representation of the field than it normally would. See the specific + // options below. This option is not yet implemented in the open source + // release -- sorry, we'll try to include it in a future version! + optional CType ctype = 1 [default = STRING]; + enum CType { + // Default mode. + STRING = 0; + + CORD = 1; + + STRING_PIECE = 2; + } + // The packed option can be enabled for repeated primitive fields to enable + // a more efficient representation on the wire. Rather than repeatedly + // writing the tag and type for each element, the entire array is encoded as + // a single length-delimited blob. + optional bool packed = 2; + + + + // Should this field be parsed lazily? Lazy applies only to message-type + // fields. It means that when the outer message is initially parsed, the + // inner message's contents will not be parsed but instead stored in encoded + // form. The inner message will actually be parsed when it is first accessed. + // + // This is only a hint. Implementations are free to choose whether to use + // eager or lazy parsing regardless of the value of this option. However, + // setting this option true suggests that the protocol author believes that + // using lazy parsing on this field is worth the additional bookkeeping + // overhead typically needed to implement it. + // + // This option does not affect the public interface of any generated code; + // all method signatures remain the same. Furthermore, thread-safety of the + // interface is not affected by this option; const methods remain safe to + // call from multiple threads concurrently, while non-const methods continue + // to require exclusive access. + // + // + // Note that implementations may choose not to check required fields within + // a lazy sub-message. That is, calling IsInitialized() on the outher message + // may return true even if the inner message has missing required fields. + // This is necessary because otherwise the inner message would have to be + // parsed in order to perform the check, defeating the purpose of lazy + // parsing. An implementation which chooses not to check required fields + // must be consistent about it. That is, for any particular sub-message, the + // implementation must either *always* check its required fields, or *never* + // check its required fields, regardless of whether or not the message has + // been parsed. + optional bool lazy = 5 [default=false]; + + // Is this field deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for accessors, or it will be completely ignored; in the very least, this + // is a formalization for deprecating fields. + optional bool deprecated = 3 [default=false]; + + // EXPERIMENTAL. DO NOT USE. + // For "map" fields, the name of the field in the enclosed type that + // is the key for this map. For example, suppose we have: + // message Item { + // required string name = 1; + // required string value = 2; + // } + // message Config { + // repeated Item items = 1 [experimental_map_key="name"]; + // } + // In this situation, the map key for Item will be set to "name". + // TODO: Fully-implement this, then remove the "experimental_" prefix. + optional string experimental_map_key = 9; + + // For Google-internal migration only. Do not use. + optional bool weak = 10 [default=false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumOptions { + + // Set this option to false to disallow mapping different tag names to a same + // value. + optional bool allow_alias = 2 [default=true]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumValueOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message ServiceOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message MethodOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + + +// A message representing a option the parser does not recognize. This only +// appears in options protos created by the compiler::Parser class. +// DescriptorPool resolves these when building Descriptor objects. Therefore, +// options protos in descriptor objects (e.g. returned by Descriptor::options(), +// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions +// in them. +message UninterpretedOption { + // The name of the uninterpreted option. Each string represents a segment in + // a dot-separated name. is_extension is true iff a segment represents an + // extension (denoted with parentheses in options specs in .proto files). + // E.g.,{ ["foo", false], ["bar.baz", true], ["qux", false] } represents + // "foo.(bar.baz).qux". + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } + repeated NamePart name = 2; + + // The value of the uninterpreted option, in whatever type the tokenizer + // identified it as during parsing. Exactly one of these should be set. + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; +} + +// =================================================================== +// Optional source code info + +// Encapsulates information about the original source file from which a +// FileDescriptorProto was generated. +message SourceCodeInfo { + // A Location identifies a piece of source code in a .proto file which + // corresponds to a particular definition. This information is intended + // to be useful to IDEs, code indexers, documentation generators, and similar + // tools. + // + // For example, say we have a file like: + // message Foo { + // optional string foo = 1; + // } + // Let's look at just the field definition: + // optional string foo = 1; + // ^ ^^ ^^ ^ ^^^ + // a bc de f ghi + // We have the following locations: + // span path represents + // [a,i) [ 4, 0, 2, 0 ] The whole field definition. + // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). + // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). + // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). + // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). + // + // Notes: + // - A location may refer to a repeated field itself (i.e. not to any + // particular index within it). This is used whenever a set of elements are + // logically enclosed in a single code segment. For example, an entire + // extend block (possibly containing multiple extension definitions) will + // have an outer location whose path refers to the "extensions" repeated + // field without an index. + // - Multiple locations may have the same path. This happens when a single + // logical declaration is spread out across multiple places. The most + // obvious example is the "extend" block again -- there may be multiple + // extend blocks in the same scope, each of which will have the same path. + // - A location's span is not always a subset of its parent's span. For + // example, the "extendee" of an extension declaration appears at the + // beginning of the "extend" block and is shared by all extensions within + // the block. + // - Just because a location's span is a subset of some other location's span + // does not mean that it is a descendent. For example, a "group" defines + // both a type and a field in a single declaration. Thus, the locations + // corresponding to the type and field and their components will overlap. + // - Code which tries to interpret locations should probably be designed to + // ignore those that it doesn't understand, as more types of locations could + // be recorded in the future. + repeated Location location = 1; + message Location { + // Identifies which part of the FileDescriptorProto was defined at this + // location. + // + // Each element is a field number or an index. They form a path from + // the root FileDescriptorProto to the place where the definition. For + // example, this path: + // [ 4, 3, 2, 7, 1 ] + // refers to: + // file.message_type(3) // 4, 3 + // .field(7) // 2, 7 + // .name() // 1 + // This is because FileDescriptorProto.message_type has field number 4: + // repeated DescriptorProto message_type = 4; + // and DescriptorProto.field has field number 2: + // repeated FieldDescriptorProto field = 2; + // and FieldDescriptorProto.name has field number 1: + // optional string name = 1; + // + // Thus, the above path gives the location of a field name. If we removed + // the last element: + // [ 4, 3, 2, 7 ] + // this path refers to the whole field declaration (from the beginning + // of the label to the terminating semicolon). + repeated int32 path = 1 [packed=true]; + + // Always has exactly three or four elements: start line, start column, + // end line (optional, otherwise assumed same as start line), end column. + // These are packed into a single field for efficiency. Note that line + // and column numbers are zero-based -- typically you will want to add + // 1 to each before displaying to a user. + repeated int32 span = 2 [packed=true]; + + // If this SourceCodeInfo represents a complete declaration, these are any + // comments appearing before and after the declaration which appear to be + // attached to the declaration. + // + // A series of line comments appearing on consecutive lines, with no other + // tokens appearing on those lines, will be treated as a single comment. + // + // Only the comment content is provided; comment markers (e.g. //) are + // stripped out. For block comments, leading whitespace and an asterisk + // will be stripped from the beginning of each line other than the first. + // Newlines are included in the output. + // + // Examples: + // + // optional int32 foo = 1; // Comment attached to foo. + // // Comment attached to bar. + // optional int32 bar = 2; + // + // optional string baz = 3; + // // Comment attached to baz. + // // Another line attached to baz. + // + // // Comment attached to qux. + // // + // // Another line attached to qux. + // optional double qux = 4; + // + // optional string corge = 5; + // /* Block comment attached + // * to corge. Leading asterisks + // * will be removed. */ + // /* Block comment attached to + // * grault. */ + // optional int32 grault = 6; + optional string leading_comments = 3; + optional string trailing_comments = 4; + } +} diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index 2be2f803..2ea011f4 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -5,7 +5,7 @@ // These are used by nanopb to generate statically allocable structures // for memory-limited environments. -import "google/protobuf/descriptor.proto"; +import "descriptor.proto"; option java_package = "fi.kapsi.koti.jpa.nanopb"; diff --git a/generator/proto/plugin.proto b/generator/proto/plugin.proto index 651ed10c..d8b515e3 100644 --- a/generator/proto/plugin.proto +++ b/generator/proto/plugin.proto @@ -46,7 +46,7 @@ package google.protobuf.compiler; -import "google/protobuf/descriptor.proto"; +import "descriptor.proto"; // An encoded CodeGeneratorRequest is written to the plugin's stdin. message CodeGeneratorRequest { -- cgit 1.2.3-korg From ec517e2108f4e7cc31cffc4fdc2a11cebfabfc66 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 17:37:29 +0200 Subject: Add build script for windows package --- generator/nanopb_generator.py | 23 ++++++++++------------- generator/proto/Makefile | 9 +++------ tools/make_windows_package.sh | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 19 deletions(-) create mode 100644 tools/make_windows_package.sh diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 0002409b..86c554a2 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -4,7 +4,7 @@ nanopb_version = "nanopb-0.2.5-dev" try: - import google.protobuf.descriptor_pb2 as descriptor + import google, distutils.util # bbfreeze seems to need these import google.protobuf.text_format as text_format except: print @@ -15,18 +15,8 @@ except: print raise -try: - import proto.nanopb_pb2 as nanopb_pb2 -except: - print - print "***************************************************************" - print "*** Could not import the precompiled nanopb_pb2.py. ***" - print "*** Run 'make' in the 'generator' folder to update the file.***" - print "***************************************************************" - print - raise - - +import proto.nanopb_pb2 as nanopb_pb2 +import proto.descriptor_pb2 as descriptor @@ -1025,6 +1015,13 @@ def main_cli(): def main_plugin(): '''Main function when invoked as a protoc plugin.''' + import sys + if sys.platform == "win32": + import os, msvcrt + # Set stdin and stdout to binary mode + msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) + msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) + import proto.plugin_pb2 as plugin_pb2 data = sys.stdin.read() request = plugin_pb2.CodeGeneratorRequest.FromString(data) diff --git a/generator/proto/Makefile b/generator/proto/Makefile index fea99a9c..ee1390a5 100644 --- a/generator/proto/Makefile +++ b/generator/proto/Makefile @@ -1,7 +1,4 @@ -all: nanopb_pb2.py plugin_pb2.py +all: nanopb_pb2.py plugin_pb2.py descriptor_pb2.py -nanopb_pb2.py: nanopb.proto - protoc --python_out=. nanopb.proto - -plugin_pb2.py: plugin.proto - protoc --python_out=. plugin.proto +%_pb2.py: %.proto + protoc --python_out=. $< diff --git a/tools/make_windows_package.sh b/tools/make_windows_package.sh new file mode 100644 index 00000000..c3a050a9 --- /dev/null +++ b/tools/make_windows_package.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Run this script in the top nanopb directory to create a binary package +# for Windows users. This script is designed to run under MingW/MSYS bash + +set -e +set -x + +VERSION=`git describe --always` +DEST=dist/$VERSION + +rm -rf $DEST +mkdir -p $DEST + +# Export the files from newest commit +git archive HEAD | tar x -C $DEST + +# Rebuild the Python .proto files +make -BC $DEST/generator/proto + +# Make the nanopb generator available as a protoc plugin +cp $DEST/generator/nanopb_generator.py $DEST/generator/protoc-gen-nanopb.py + +# Package the Python libraries +( cd $DEST/generator; bbfreeze nanopb_generator.py protoc-gen-nanopb.py ) +mv $DEST/generator/dist $DEST/generator-bin + +# Remove temp file +rm $DEST/generator/protoc-gen-nanopb.py + +# Package the protoc compiler +cp `which protoc`.exe $DEST/generator-bin/ + -- cgit 1.2.3-korg From 480b39cc0adc749ad81183fad2bfce6548c56604 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 19:09:13 +0200 Subject: Package as .zip, convert linebreaks --- tools/make_windows_package.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/make_windows_package.sh b/tools/make_windows_package.sh index c3a050a9..3096003b 100644 --- a/tools/make_windows_package.sh +++ b/tools/make_windows_package.sh @@ -2,6 +2,7 @@ # Run this script in the top nanopb directory to create a binary package # for Windows users. This script is designed to run under MingW/MSYS bash +# and requires the following tools: git, make, zip, unix2dos set -e set -x @@ -31,3 +32,10 @@ rm $DEST/generator/protoc-gen-nanopb.py # Package the protoc compiler cp `which protoc`.exe $DEST/generator-bin/ +# Convert line breaks for convenience +find $DEST -name '*.c' -o -name '*.h' -o -name '*.txt' \ + -o -name '*.proto' -o -name '*.py' -o -name '*.options' \ + -exec unix2dos '{}' \; + +# Zip it all up +( cd dist; zip -r $VERSION.zip $VERSION ) -- cgit 1.2.3-korg From ef31774cd902a6f069b8f2b80b870167328f5acb Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 19:20:04 +0200 Subject: Moving files around Renamed READMEs to README.txt to be more friendly for Windows users. --- CHANGELOG | 121 ----------------- CHANGELOG.txt | 121 +++++++++++++++++ LICENSE | 20 --- LICENSE.txt | 20 +++ README | 11 -- README.txt | 11 ++ cmake/FindNanopb.cmake | 224 ------------------------------ compat/pb_syshdr.h | 94 ------------- examples/network_server/README | 60 --------- examples/network_server/README.txt | 60 +++++++++ examples/simple/README | 30 ----- examples/simple/README.txt | 30 +++++ examples/using_double_on_avr/README | 25 ---- examples/using_double_on_avr/README.txt | 25 ++++ examples/using_union_messages/README | 52 ------- examples/using_union_messages/README.txt | 52 +++++++ extra/FindNanopb.cmake | 225 +++++++++++++++++++++++++++++++ extra/pb_syshdr.h | 94 +++++++++++++ 18 files changed, 638 insertions(+), 637 deletions(-) delete mode 100644 CHANGELOG create mode 100644 CHANGELOG.txt delete mode 100644 LICENSE create mode 100644 LICENSE.txt delete mode 100644 README create mode 100644 README.txt delete mode 100644 cmake/FindNanopb.cmake delete mode 100644 compat/pb_syshdr.h delete mode 100644 examples/network_server/README create mode 100644 examples/network_server/README.txt delete mode 100644 examples/simple/README create mode 100644 examples/simple/README.txt delete mode 100644 examples/using_double_on_avr/README create mode 100644 examples/using_double_on_avr/README.txt delete mode 100644 examples/using_union_messages/README create mode 100644 examples/using_union_messages/README.txt create mode 100644 extra/FindNanopb.cmake create mode 100644 extra/pb_syshdr.h diff --git a/CHANGELOG b/CHANGELOG deleted file mode 100644 index 94f0c797..00000000 --- a/CHANGELOG +++ /dev/null @@ -1,121 +0,0 @@ -nanopb-0.2.4 (2013-11-07) - Remove the deprecated NANOPB_INTERNALS functions from public API. - Document the security model. - Check array and bytes max sizes when encoding (issue 90) - Add #defines for maximum encoded message size (issue 89) - Add #define tags for extension fields (issue 93) - Fix MISRA C violations (issue 91) - Clean up pb_field_t definition with typedefs. - -nanopb-0.2.3 (2013-09-18) - Improve compatibility by removing ternary operator from initializations (issue 88) - Fix build error on Visual C++ (issue 84, patch by Markus Schwarzenberg) - Don't stop on unsupported extension fields (issue 83) - Add an example pb_syshdr.h file for non-C99 compilers - Reorganize tests and examples into subfolders (issue 63) - Switch from Makefiles to scons for building the tests - Make the tests buildable on Windows - -nanopb-0.2.2 (2013-08-18) - Add support for extension fields (issue 17) - Fix unknown fields in empty message (issue 78) - Include the field tags in the generated .pb.h file. - Add pb_decode_delimited and pb_encode_delimited wrapper functions (issue 74) - Add a section in top of pb.h for changing compilation settings (issue 76) - Documentation improvements (issues 12, 77 and others) - Improved tests - -nanopb-0.2.1 (2013-04-14) - NOTE: The default callback function signature has changed. - If you don't want to update your code, define PB_OLD_CALLBACK_STYLE. - - Change the callback function to use void** (issue 69) - Add support for defining the nanopb options in a separate file (issue 12) - Add support for packed structs in IAR and MSVC (in addition to GCC) (issue 66) - Implement error message support for the encoder side (issue 7) - Handle unterminated strings when encoding (issue 68) - Fix bug with empty strings in repeated string callbacks (issue 73) - Fix regression in 0.2.0 with optional callback fields (issue 70) - Fix bugs with empty message types (issues 64, 65) - Fix some compiler warnings on clang (issue 67) - Some portability improvements (issues 60, 62) - Various new generator options - Improved tests - -nanopb-0.2.0 (2013-03-02) - NOTE: This release requires you to regenerate all .pb.c - files. Files generated by older versions will not - compile anymore. - - Reformat generated .pb.c files using macros (issue 58) - Rename PB_HTYPE_ARRAY -> PB_HTYPE_REPEATED - Separate PB_HTYPE to PB_ATYPE and PB_HTYPE - Move STATIC_ASSERTs to .pb.c file - Added CMake file (by Pavel Ilin) - Add option to give file extension to generator (by Michael Haberler) - Documentation updates - -nanopb-0.1.9 (2013-02-13) - Fixed error message bugs (issues 52, 56) - Sanitize #ifndef filename (issue 50) - Performance improvements - Add compile-time option PB_BUFFER_ONLY - Add Java package name to nanopb.proto - Check for sizeof(double) == 8 (issue 54) - Added generator option to ignore some fields. (issue 51) - Added generator option to make message structs packed. (issue 49) - Add more test cases. - -nanopb-0.1.8 (2012-12-13) - Fix bugs in the enum short names introduced in 0.1.7 (issues 42, 43) - Fix STATIC_ASSERT macro when using multiple .proto files. (issue 41) - Fix missing initialization of istream.errmsg - Make tests/Makefile work for non-gcc compilers (issue 40) - -nanopb-0.1.7 (2012-11-11) - Remove "skip" mode from pb_istream_t callbacks. Example implementation had a bug. (issue 37) - Add option to use shorter names for enum values (issue 38) - Improve options support in generator (issues 12, 30) - Add nanopb version number to generated files (issue 36) - Add extern "C" to generated headers (issue 35) - Add names for structs to allow forward declaration (issue 39) - Add buffer size check in example (issue 34) - Fix build warnings on MS compilers (issue 33) - -nanopb-0.1.6 (2012-09-02) - Reorganize the field decoder interface (issue 2) - Improve performance in submessage decoding (issue 28) - Implement error messages in the decoder side (issue 7) - Extended testcases (alltypes test is now complete). - Fix some compiler warnings (issues 25, 26, 27, 32). - -nanopb-0.1.5 (2012-08-04) - Fix bug in decoder with packed arrays (issue 23). - Extended testcases. - Fix some compiler warnings. - -nanopb-0.1.4 (2012-07-05) - Add compile-time options for easy-to-use >255 field support. - Improve the detection of missing required fields. - Added example on how to handle union messages. - Fix generator error with .proto without messages. - Fix problems that stopped the code from compiling with some compilers. - Fix some compiler warnings. - -nanopb-0.1.3 (2012-06-12) - Refactor the field encoder interface. - Improve generator error messages (issue 5) - Add descriptor.proto into the #include exclusion list - Fix some compiler warnings. - -nanopb-0.1.2 (2012-02-15) - Make the generator to generate include for other .proto files (issue 4). - Fixed generator not working on Windows (issue 3) - -nanopb-0.1.1 (2012-01-14) - Fixed bug in encoder with 'bytes' fields (issue 1). - Fixed a bug in the generator that caused a compiler error on sfixed32 and sfixed64 fields. - Extended testcases. - -nanopb-0.1.0 (2012-01-06) - First stable release. diff --git a/CHANGELOG.txt b/CHANGELOG.txt new file mode 100644 index 00000000..94f0c797 --- /dev/null +++ b/CHANGELOG.txt @@ -0,0 +1,121 @@ +nanopb-0.2.4 (2013-11-07) + Remove the deprecated NANOPB_INTERNALS functions from public API. + Document the security model. + Check array and bytes max sizes when encoding (issue 90) + Add #defines for maximum encoded message size (issue 89) + Add #define tags for extension fields (issue 93) + Fix MISRA C violations (issue 91) + Clean up pb_field_t definition with typedefs. + +nanopb-0.2.3 (2013-09-18) + Improve compatibility by removing ternary operator from initializations (issue 88) + Fix build error on Visual C++ (issue 84, patch by Markus Schwarzenberg) + Don't stop on unsupported extension fields (issue 83) + Add an example pb_syshdr.h file for non-C99 compilers + Reorganize tests and examples into subfolders (issue 63) + Switch from Makefiles to scons for building the tests + Make the tests buildable on Windows + +nanopb-0.2.2 (2013-08-18) + Add support for extension fields (issue 17) + Fix unknown fields in empty message (issue 78) + Include the field tags in the generated .pb.h file. + Add pb_decode_delimited and pb_encode_delimited wrapper functions (issue 74) + Add a section in top of pb.h for changing compilation settings (issue 76) + Documentation improvements (issues 12, 77 and others) + Improved tests + +nanopb-0.2.1 (2013-04-14) + NOTE: The default callback function signature has changed. + If you don't want to update your code, define PB_OLD_CALLBACK_STYLE. + + Change the callback function to use void** (issue 69) + Add support for defining the nanopb options in a separate file (issue 12) + Add support for packed structs in IAR and MSVC (in addition to GCC) (issue 66) + Implement error message support for the encoder side (issue 7) + Handle unterminated strings when encoding (issue 68) + Fix bug with empty strings in repeated string callbacks (issue 73) + Fix regression in 0.2.0 with optional callback fields (issue 70) + Fix bugs with empty message types (issues 64, 65) + Fix some compiler warnings on clang (issue 67) + Some portability improvements (issues 60, 62) + Various new generator options + Improved tests + +nanopb-0.2.0 (2013-03-02) + NOTE: This release requires you to regenerate all .pb.c + files. Files generated by older versions will not + compile anymore. + + Reformat generated .pb.c files using macros (issue 58) + Rename PB_HTYPE_ARRAY -> PB_HTYPE_REPEATED + Separate PB_HTYPE to PB_ATYPE and PB_HTYPE + Move STATIC_ASSERTs to .pb.c file + Added CMake file (by Pavel Ilin) + Add option to give file extension to generator (by Michael Haberler) + Documentation updates + +nanopb-0.1.9 (2013-02-13) + Fixed error message bugs (issues 52, 56) + Sanitize #ifndef filename (issue 50) + Performance improvements + Add compile-time option PB_BUFFER_ONLY + Add Java package name to nanopb.proto + Check for sizeof(double) == 8 (issue 54) + Added generator option to ignore some fields. (issue 51) + Added generator option to make message structs packed. (issue 49) + Add more test cases. + +nanopb-0.1.8 (2012-12-13) + Fix bugs in the enum short names introduced in 0.1.7 (issues 42, 43) + Fix STATIC_ASSERT macro when using multiple .proto files. (issue 41) + Fix missing initialization of istream.errmsg + Make tests/Makefile work for non-gcc compilers (issue 40) + +nanopb-0.1.7 (2012-11-11) + Remove "skip" mode from pb_istream_t callbacks. Example implementation had a bug. (issue 37) + Add option to use shorter names for enum values (issue 38) + Improve options support in generator (issues 12, 30) + Add nanopb version number to generated files (issue 36) + Add extern "C" to generated headers (issue 35) + Add names for structs to allow forward declaration (issue 39) + Add buffer size check in example (issue 34) + Fix build warnings on MS compilers (issue 33) + +nanopb-0.1.6 (2012-09-02) + Reorganize the field decoder interface (issue 2) + Improve performance in submessage decoding (issue 28) + Implement error messages in the decoder side (issue 7) + Extended testcases (alltypes test is now complete). + Fix some compiler warnings (issues 25, 26, 27, 32). + +nanopb-0.1.5 (2012-08-04) + Fix bug in decoder with packed arrays (issue 23). + Extended testcases. + Fix some compiler warnings. + +nanopb-0.1.4 (2012-07-05) + Add compile-time options for easy-to-use >255 field support. + Improve the detection of missing required fields. + Added example on how to handle union messages. + Fix generator error with .proto without messages. + Fix problems that stopped the code from compiling with some compilers. + Fix some compiler warnings. + +nanopb-0.1.3 (2012-06-12) + Refactor the field encoder interface. + Improve generator error messages (issue 5) + Add descriptor.proto into the #include exclusion list + Fix some compiler warnings. + +nanopb-0.1.2 (2012-02-15) + Make the generator to generate include for other .proto files (issue 4). + Fixed generator not working on Windows (issue 3) + +nanopb-0.1.1 (2012-01-14) + Fixed bug in encoder with 'bytes' fields (issue 1). + Fixed a bug in the generator that caused a compiler error on sfixed32 and sfixed64 fields. + Extended testcases. + +nanopb-0.1.0 (2012-01-06) + First stable release. diff --git a/LICENSE b/LICENSE deleted file mode 100644 index d11c9af1..00000000 --- a/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2011 Petteri Aimonen - -This software is provided 'as-is', without any express or -implied warranty. In no event will the authors be held liable -for any damages arising from the use of this software. - -Permission is granted to anyone to use this software for any -purpose, including commercial applications, and to alter it and -redistribute it freely, subject to the following restrictions: - -1. The origin of this software must not be misrepresented; you - must not claim that you wrote the original software. If you use - this software in a product, an acknowledgment in the product - documentation would be appreciated but is not required. - -2. Altered source versions must be plainly marked as such, and - must not be misrepresented as being the original software. - -3. This notice may not be removed or altered from any source - distribution. diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 00000000..d11c9af1 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,20 @@ +Copyright (c) 2011 Petteri Aimonen + +This software is provided 'as-is', without any express or +implied warranty. In no event will the authors be held liable +for any damages arising from the use of this software. + +Permission is granted to anyone to use this software for any +purpose, including commercial applications, and to alter it and +redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you + must not claim that you wrote the original software. If you use + this software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and + must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source + distribution. diff --git a/README b/README deleted file mode 100644 index 2a12b4f5..00000000 --- a/README +++ /dev/null @@ -1,11 +0,0 @@ -Nanopb is a small code-size Protocol Buffers implementation. - -Homepage: http://kapsi.fi/~jpa/nanopb/ - -To compile the library, you'll need these libraries: -protobuf-compiler python-protobuf libprotobuf-dev - -The only runtime dependencies are memset() and memcpy(). - -To run the tests, run make under the tests folder. -If it completes without error, everything is fine. diff --git a/README.txt b/README.txt new file mode 100644 index 00000000..2a12b4f5 --- /dev/null +++ b/README.txt @@ -0,0 +1,11 @@ +Nanopb is a small code-size Protocol Buffers implementation. + +Homepage: http://kapsi.fi/~jpa/nanopb/ + +To compile the library, you'll need these libraries: +protobuf-compiler python-protobuf libprotobuf-dev + +The only runtime dependencies are memset() and memcpy(). + +To run the tests, run make under the tests folder. +If it completes without error, everything is fine. diff --git a/cmake/FindNanopb.cmake b/cmake/FindNanopb.cmake deleted file mode 100644 index ad96b5c3..00000000 --- a/cmake/FindNanopb.cmake +++ /dev/null @@ -1,224 +0,0 @@ -# Locate and configure the nanopb library. -# -# The following varialbes have to be set: -# -# NANOPB_SRC_ROOT_FOLDER - Path to nanopb source folder -# -# The following variables can be set and are optional: -# -# -# PROTOBUF_SRC_ROOT_FOLDER - When compiling with MSVC, if this cache variable is set -# the protobuf-default VS project build locations -# (vsprojects/Debug & vsprojects/Release) will be searched -# for libraries and binaries. -# -# NANOPB_IMPORT_DIRS - List of additional directories to be searched for -# imported .proto files. -# -# NANOPB_GENERATE_CPP_APPEND_PATH - By default -I will be passed to protoc -# for each directory where a proto file is referenced. -# Set to FALSE if you want to disable this behaviour. -# -# Defines the following variables: -# -# NANOPB_FOUND - Found the nanopb library (source&header files, generator tool, protoc compiler tool) -# NANOPB_INCLUDE_DIRS - Include directories for Google Protocol Buffers -# -# The following cache variables are also available to set or use: -# NANOPB_GENERATOR_EXECUTABLE - The nanopb generator -# PROTOBUF_PROTOC_EXECUTABLE - The protoc compiler -# -# ==================================================================== -# -# NANOPB_GENERATE_CPP (public function) -# SRCS = Variable to define with autogenerated -# source files -# HDRS = Variable to define with autogenerated -# header files -# ARGN = proto files -# -# ==================================================================== -# Example: -# -# set(NANOPB_SRC_ROOT_FOLDER "/path/to/nanopb") -# set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${NANOPB_SRC_ROOT_FOLDER}/cmake) -# find_package( Nanopb REQUIRED ) -# include_directories(${NANOPB_INCLUDE_DIRS}) -# -# NANOPB_GENERATE_CPP(PROTO_SRCS PROTO_HDRS foo.proto) -# -# include_directories(${CMAKE_CURRENT_BINARY_DIR}) -# add_executable(bar bar.cc ${PROTO_SRCS} ${PROTO_HDRS}) -# -# ==================================================================== - -#============================================================================= -# Copyright 2009 Kitware, Inc. -# Copyright 2009-2011 Philip Lowman -# Copyright 2008 Esben Mose Hansen, Ange Optimization ApS -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# -# * Neither the names of Kitware, Inc., the Insight Software Consortium, -# nor the names of their contributors may be used to endorse or promote -# products derived from this software without specific prior written -# permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -# -#============================================================================= -# -# Changes -# 2013.01.31 - Pavlo Ilin - used Modules/FindProtobuf.cmake from cmake 2.8.10 to -# write FindNanopb.cmake -# -#============================================================================= - - -function(NANOPB_GENERATE_CPP SRCS HDRS) - if(NOT ARGN) - return() - endif() - - if(NANOPB_GENERATE_CPP_APPEND_PATH) - # Create an include path for each file specified - foreach(FIL ${ARGN}) - get_filename_component(ABS_FIL ${FIL} ABSOLUTE) - get_filename_component(ABS_PATH ${ABS_FIL} PATH) - - list(FIND _nanobp_include_path ${ABS_PATH} _contains_already) - if(${_contains_already} EQUAL -1) - list(APPEND _nanobp_include_path -I ${ABS_PATH}) - endif() - endforeach() - else() - set(_nanobp_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) - endif() - - if(DEFINED NANOPB_IMPORT_DIRS) - foreach(DIR ${NANOPB_IMPORT_DIRS}) - get_filename_component(ABS_PATH ${DIR} ABSOLUTE) - list(FIND _nanobp_include_path ${ABS_PATH} _contains_already) - if(${_contains_already} EQUAL -1) - list(APPEND _nanobp_include_path -I ${ABS_PATH}) - endif() - endforeach() - endif() - - set(${SRCS}) - set(${HDRS}) - get_filename_component(GENERATOR_PATH ${NANOPB_GENERATOR_EXECUTABLE} PATH) - - foreach(FIL ${ARGN}) - get_filename_component(ABS_FIL ${FIL} ABSOLUTE) - get_filename_component(FIL_WE ${FIL} NAME_WE) - - list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c") - list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") - - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb" - COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} - ARGS -I${GENERATOR_PATH} -I${CMAKE_CURRENT_BINARY_DIR} ${_nanobp_include_path} -o${FIL_WE}.pb ${ABS_FIL} - DEPENDS ${ABS_FIL} - COMMENT "Running C++ protocol buffer compiler on ${FIL}" - VERBATIM ) - - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c" - "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" - COMMAND python - ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb - DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb" - COMMENT "Running nanopb generator on ${FIL_WE}.pb" - VERBATIM ) - endforeach() - - set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) - set(${SRCS} ${${SRCS}} ${NANOPB_SRCS} PARENT_SCOPE) - set(${HDRS} ${${HDRS}} ${NANOPB_HDRS} PARENT_SCOPE) - -endfunction() - - - -# -# Main. -# - -# By default have NANOPB_GENERATE_CPP macro pass -I to protoc -# for each directory where a proto file is referenced. -if(NOT DEFINED NANOPB_GENERATE_CPP_APPEND_PATH) - set(NANOPB_GENERATE_CPP_APPEND_PATH TRUE) -endif() - -# Find the include directory -find_path(NANOPB_INCLUDE_DIRS - pb.h - PATHS ${NANOPB_SRC_ROOT_FOLDER} -) -mark_as_advanced(NANOPB_INCLUDE_DIRS) - -# Find nanopb source files -set(NANOPB_SRCS) -set(NANOPB_HDRS) -list(APPEND _nanopb_srcs pb_decode.c pb_encode.c) -list(APPEND _nanopb_hdrs pb_decode.h pb_encode.h pb.h) - -foreach(FIL ${_nanopb_srcs}) - find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_SRC_ROOT_FOLDER} ${NANOPB_INCLUDE_DIRS}) - list(APPEND NANOPB_SRCS "${${FIL}__nano_pb_file}") - mark_as_advanced(${FIL}__nano_pb_file) -endforeach() - -foreach(FIL ${_nanopb_hdrs}) - find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_INCLUDE_DIRS}) - mark_as_advanced(${FIL}__nano_pb_file) - list(APPEND NANOPB_HDRS "${${FIL}__nano_pb_file}") -endforeach() - -# Find the protoc Executable -find_program(PROTOBUF_PROTOC_EXECUTABLE - NAMES protoc - DOC "The Google Protocol Buffers Compiler" - PATHS - ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Release - ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Debug -) -mark_as_advanced(PROTOBUF_PROTOC_EXECUTABLE) - -# Find nanopb generator -find_file(NANOPB_GENERATOR_EXECUTABLE - NAMES nanopb_generator.py - DOC "nanopb generator" - PATHS - ${NANOPB_SRC_ROOT_FOLDER}/generator -) -mark_as_advanced(NANOPB_GENERATOR_EXECUTABLE) - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(NANOPB DEFAULT_MSG - NANOPB_INCLUDE_DIRS - NANOPB_SRCS NANOPB_HDRS - NANOPB_GENERATOR_EXECUTABLE - PROTOBUF_PROTOC_EXECUTABLE - ) diff --git a/compat/pb_syshdr.h b/compat/pb_syshdr.h deleted file mode 100644 index b69a1671..00000000 --- a/compat/pb_syshdr.h +++ /dev/null @@ -1,94 +0,0 @@ -/* This is an example of a header file for platforms/compilers that do - * not come with stdint.h/stddef.h/stdbool.h/string.h. To use it, define - * PB_SYSTEM_HEADER as "pb_syshdr.h", including the quotes, and add the - * compat folder to your include path. - * - * It is very likely that you will need to customize this file to suit - * your platform. For any compiler that supports C99, this file should - * not be necessary. - */ - -#ifndef _PB_SYSHDR_H_ -#define _PB_SYSHDR_H_ - -/* stdint.h subset */ -#ifdef HAVE_STDINT_H -#include -#else -/* You will need to modify these to match the word size of your platform. */ -typedef signed char int8_t; -typedef unsigned char uint8_t; -typedef signed short int16_t; -typedef unsigned short uint16_t; -typedef signed int int32_t; -typedef unsigned int uint32_t; -typedef signed long long int64_t; -typedef unsigned long long uint64_t; -#endif - -/* stddef.h subset */ -#ifdef HAVE_STDDEF_H -#include -#else - -typedef uint32_t size_t; -#define offsetof(st, m) ((size_t)(&((st *)0)->m)) - -#ifndef NULL -#define NULL 0 -#endif - -#endif - -/* stdbool.h subset */ -#ifdef HAVE_STDBOOL_H -#include -#else - -#ifndef __cplusplus -typedef int bool; -#define false 0 -#define true 1 -#endif - -#endif - -/* string.h subset */ -#ifdef HAVE_STRING_H -#include -#else - -/* Implementations are from the Public Domain C Library (PDCLib). */ -static size_t strlen( const char * s ) -{ - size_t rc = 0; - while ( s[rc] ) - { - ++rc; - } - return rc; -} - -static void * memcpy( void *s1, const void *s2, size_t n ) -{ - char * dest = (char *) s1; - const char * src = (const char *) s2; - while ( n-- ) - { - *dest++ = *src++; - } - return s1; -} - -static void * memset( void * s, int c, size_t n ) -{ - unsigned char * p = (unsigned char *) s; - while ( n-- ) - { - *p++ = (unsigned char) c; - } - return s; -} -#endif - -#endif diff --git a/examples/network_server/README b/examples/network_server/README deleted file mode 100644 index 7bdcbed5..00000000 --- a/examples/network_server/README +++ /dev/null @@ -1,60 +0,0 @@ -Nanopb example "network_server" -=============================== - -This example demonstrates the use of nanopb to communicate over network -connections. It consists of a server that sends file listings, and of -a client that requests the file list from the server. - -Example usage -------------- - -user@host:~/nanopb/examples/network_server$ make # Build the example -protoc -ofileproto.pb fileproto.proto -python ../../generator/nanopb_generator.py fileproto.pb -Writing to fileproto.pb.h and fileproto.pb.c -cc -ansi -Wall -Werror -I .. -g -O0 -I../.. -o server server.c - ../../pb_decode.c ../../pb_encode.c fileproto.pb.c common.c -cc -ansi -Wall -Werror -I .. -g -O0 -I../.. -o client client.c - ../../pb_decode.c ../../pb_encode.c fileproto.pb.c common.c - -user@host:~/nanopb/examples/network_server$ ./server & # Start the server on background -[1] 24462 - -petteri@oddish:~/nanopb/examples/network_server$ ./client /bin # Request the server to list /bin -Got connection. -Listing directory: /bin -1327119 bzdiff -1327126 bzless -1327147 ps -1327178 ntfsmove -1327271 mv -1327187 mount -1327259 false -1327266 tempfile -1327285 zfgrep -1327165 gzexe -1327204 nc.openbsd -1327260 uname - - -Details of implementation -------------------------- -fileproto.proto contains the portable Google Protocol Buffers protocol definition. -It could be used as-is to implement a server or a client in any other language, for -example Python or Java. - -fileproto.options contains the nanopb-specific options for the protocol file. This -sets the amount of space allocated for file names when decoding messages. - -common.c/h contains functions that allow nanopb to read and write directly from -network socket. This way there is no need to allocate a separate buffer to store -the message. - -server.c contains the code to open a listening socket, to respond to clients and -to list directory contents. - -client.c contains the code to connect to a server, to send a request and to print -the response message. - -The code is implemented using the POSIX socket api, but it should be easy enough -to port into any other socket api, such as lwip. diff --git a/examples/network_server/README.txt b/examples/network_server/README.txt new file mode 100644 index 00000000..7bdcbed5 --- /dev/null +++ b/examples/network_server/README.txt @@ -0,0 +1,60 @@ +Nanopb example "network_server" +=============================== + +This example demonstrates the use of nanopb to communicate over network +connections. It consists of a server that sends file listings, and of +a client that requests the file list from the server. + +Example usage +------------- + +user@host:~/nanopb/examples/network_server$ make # Build the example +protoc -ofileproto.pb fileproto.proto +python ../../generator/nanopb_generator.py fileproto.pb +Writing to fileproto.pb.h and fileproto.pb.c +cc -ansi -Wall -Werror -I .. -g -O0 -I../.. -o server server.c + ../../pb_decode.c ../../pb_encode.c fileproto.pb.c common.c +cc -ansi -Wall -Werror -I .. -g -O0 -I../.. -o client client.c + ../../pb_decode.c ../../pb_encode.c fileproto.pb.c common.c + +user@host:~/nanopb/examples/network_server$ ./server & # Start the server on background +[1] 24462 + +petteri@oddish:~/nanopb/examples/network_server$ ./client /bin # Request the server to list /bin +Got connection. +Listing directory: /bin +1327119 bzdiff +1327126 bzless +1327147 ps +1327178 ntfsmove +1327271 mv +1327187 mount +1327259 false +1327266 tempfile +1327285 zfgrep +1327165 gzexe +1327204 nc.openbsd +1327260 uname + + +Details of implementation +------------------------- +fileproto.proto contains the portable Google Protocol Buffers protocol definition. +It could be used as-is to implement a server or a client in any other language, for +example Python or Java. + +fileproto.options contains the nanopb-specific options for the protocol file. This +sets the amount of space allocated for file names when decoding messages. + +common.c/h contains functions that allow nanopb to read and write directly from +network socket. This way there is no need to allocate a separate buffer to store +the message. + +server.c contains the code to open a listening socket, to respond to clients and +to list directory contents. + +client.c contains the code to connect to a server, to send a request and to print +the response message. + +The code is implemented using the POSIX socket api, but it should be easy enough +to port into any other socket api, such as lwip. diff --git a/examples/simple/README b/examples/simple/README deleted file mode 100644 index d18c4183..00000000 --- a/examples/simple/README +++ /dev/null @@ -1,30 +0,0 @@ -Nanopb example "simple" -======================= - -This example demonstrates the very basic use of nanopb. It encodes and -decodes a simple message. - -The code uses four different API functions: - - * pb_ostream_from_buffer() to declare the output buffer that is to be used - * pb_encode() to encode a message - * pb_istream_from_buffer() to declare the input buffer that is to be used - * pb_decode() to decode a message - -Example usage -------------- - -On Linux, simply type "make" to build the example. After that, you can -run it with the command: ./simple - -On other platforms, you first have to compile the protocol definition using -the following two commands:: - - protoc -osimple.pb simple.proto - python nanopb_generator.py simple.pb - -After that, add the following four files to your project and compile: - - simple.c simple.pb.c pb_encode.c pb_decode.c - - diff --git a/examples/simple/README.txt b/examples/simple/README.txt new file mode 100644 index 00000000..d18c4183 --- /dev/null +++ b/examples/simple/README.txt @@ -0,0 +1,30 @@ +Nanopb example "simple" +======================= + +This example demonstrates the very basic use of nanopb. It encodes and +decodes a simple message. + +The code uses four different API functions: + + * pb_ostream_from_buffer() to declare the output buffer that is to be used + * pb_encode() to encode a message + * pb_istream_from_buffer() to declare the input buffer that is to be used + * pb_decode() to decode a message + +Example usage +------------- + +On Linux, simply type "make" to build the example. After that, you can +run it with the command: ./simple + +On other platforms, you first have to compile the protocol definition using +the following two commands:: + + protoc -osimple.pb simple.proto + python nanopb_generator.py simple.pb + +After that, add the following four files to your project and compile: + + simple.c simple.pb.c pb_encode.c pb_decode.c + + diff --git a/examples/using_double_on_avr/README b/examples/using_double_on_avr/README deleted file mode 100644 index d9fcdfc6..00000000 --- a/examples/using_double_on_avr/README +++ /dev/null @@ -1,25 +0,0 @@ -Nanopb example "using_double_on_avr" -==================================== - -Some processors/compilers, such as AVR-GCC, do not support the double -datatype. Instead, they have sizeof(double) == 4. Because protocol -binary format uses the double encoding directly, this causes trouble -if the protocol in .proto requires double fields. - -This directory contains a solution to this problem. It uses uint64_t -to store the raw wire values, because its size is correct on all -platforms. The file double_conversion.c provides functions that -convert these values to/from floats, without relying on compiler -support. - -To use this method, you need to make some modifications to your code: - -1) Change all 'double' fields into 'fixed64' in the .proto. - -2) Whenever writing to a 'double' field, use float_to_double(). - -3) Whenever reading a 'double' field, use double_to_float(). - -The conversion routines are as accurate as the float datatype can -be. Furthermore, they should handle all special values (NaN, inf, denormalized -numbers) correctly. There are testcases in test_conversions.c. diff --git a/examples/using_double_on_avr/README.txt b/examples/using_double_on_avr/README.txt new file mode 100644 index 00000000..d9fcdfc6 --- /dev/null +++ b/examples/using_double_on_avr/README.txt @@ -0,0 +1,25 @@ +Nanopb example "using_double_on_avr" +==================================== + +Some processors/compilers, such as AVR-GCC, do not support the double +datatype. Instead, they have sizeof(double) == 4. Because protocol +binary format uses the double encoding directly, this causes trouble +if the protocol in .proto requires double fields. + +This directory contains a solution to this problem. It uses uint64_t +to store the raw wire values, because its size is correct on all +platforms. The file double_conversion.c provides functions that +convert these values to/from floats, without relying on compiler +support. + +To use this method, you need to make some modifications to your code: + +1) Change all 'double' fields into 'fixed64' in the .proto. + +2) Whenever writing to a 'double' field, use float_to_double(). + +3) Whenever reading a 'double' field, use double_to_float(). + +The conversion routines are as accurate as the float datatype can +be. Furthermore, they should handle all special values (NaN, inf, denormalized +numbers) correctly. There are testcases in test_conversions.c. diff --git a/examples/using_union_messages/README b/examples/using_union_messages/README deleted file mode 100644 index 7a1e75d4..00000000 --- a/examples/using_union_messages/README +++ /dev/null @@ -1,52 +0,0 @@ -Nanopb example "using_union_messages" -===================================== - -Union messages is a common technique in Google Protocol Buffers used to -represent a group of messages, only one of which is passed at a time. -It is described in Google's documentation: -https://developers.google.com/protocol-buffers/docs/techniques#union - -This directory contains an example on how to encode and decode union messages -with minimal memory usage. Usually, nanopb would allocate space to store -all of the possible messages at the same time, even though at most one of -them will be used at a time. - -By using some of the lower level nanopb APIs, we can manually generate the -top level message, so that we only need to allocate the one submessage that -we actually want. Similarly when decoding, we can manually read the tag of -the top level message, and only then allocate the memory for the submessage -after we already know its type. - - -Example usage -------------- - -Type `make` to run the example. It will build it and run commands like -following: - -./encode 1 | ./decode -Got MsgType1: 42 -./encode 2 | ./decode -Got MsgType2: true -./encode 3 | ./decode -Got MsgType3: 3 1415 - -This simply demonstrates that the "decode" program has correctly identified -the type of the received message, and managed to decode it. - - -Details of implementation -------------------------- - -unionproto.proto contains the protocol used in the example. It consists of -three messages: MsgType1, MsgType2 and MsgType3, which are collected together -into UnionMessage. - -encode.c takes one command line argument, which should be a number 1-3. It -then fills in and encodes the corresponding message, and writes it to stdout. - -decode.c reads a UnionMessage from stdin. Then it calls the function -decode_unionmessage_type() to determine the type of the message. After that, -the corresponding message is decoded and the contents of it printed to the -screen. - diff --git a/examples/using_union_messages/README.txt b/examples/using_union_messages/README.txt new file mode 100644 index 00000000..7a1e75d4 --- /dev/null +++ b/examples/using_union_messages/README.txt @@ -0,0 +1,52 @@ +Nanopb example "using_union_messages" +===================================== + +Union messages is a common technique in Google Protocol Buffers used to +represent a group of messages, only one of which is passed at a time. +It is described in Google's documentation: +https://developers.google.com/protocol-buffers/docs/techniques#union + +This directory contains an example on how to encode and decode union messages +with minimal memory usage. Usually, nanopb would allocate space to store +all of the possible messages at the same time, even though at most one of +them will be used at a time. + +By using some of the lower level nanopb APIs, we can manually generate the +top level message, so that we only need to allocate the one submessage that +we actually want. Similarly when decoding, we can manually read the tag of +the top level message, and only then allocate the memory for the submessage +after we already know its type. + + +Example usage +------------- + +Type `make` to run the example. It will build it and run commands like +following: + +./encode 1 | ./decode +Got MsgType1: 42 +./encode 2 | ./decode +Got MsgType2: true +./encode 3 | ./decode +Got MsgType3: 3 1415 + +This simply demonstrates that the "decode" program has correctly identified +the type of the received message, and managed to decode it. + + +Details of implementation +------------------------- + +unionproto.proto contains the protocol used in the example. It consists of +three messages: MsgType1, MsgType2 and MsgType3, which are collected together +into UnionMessage. + +encode.c takes one command line argument, which should be a number 1-3. It +then fills in and encodes the corresponding message, and writes it to stdout. + +decode.c reads a UnionMessage from stdin. Then it calls the function +decode_unionmessage_type() to determine the type of the message. After that, +the corresponding message is decoded and the contents of it printed to the +screen. + diff --git a/extra/FindNanopb.cmake b/extra/FindNanopb.cmake new file mode 100644 index 00000000..c804e701 --- /dev/null +++ b/extra/FindNanopb.cmake @@ -0,0 +1,225 @@ +# This is an example script for use with CMake projects for locating and configuring +# the nanopb library. +# +# The following varialbes have to be set: +# +# NANOPB_SRC_ROOT_FOLDER - Path to nanopb source folder +# +# The following variables can be set and are optional: +# +# +# PROTOBUF_SRC_ROOT_FOLDER - When compiling with MSVC, if this cache variable is set +# the protobuf-default VS project build locations +# (vsprojects/Debug & vsprojects/Release) will be searched +# for libraries and binaries. +# +# NANOPB_IMPORT_DIRS - List of additional directories to be searched for +# imported .proto files. +# +# NANOPB_GENERATE_CPP_APPEND_PATH - By default -I will be passed to protoc +# for each directory where a proto file is referenced. +# Set to FALSE if you want to disable this behaviour. +# +# Defines the following variables: +# +# NANOPB_FOUND - Found the nanopb library (source&header files, generator tool, protoc compiler tool) +# NANOPB_INCLUDE_DIRS - Include directories for Google Protocol Buffers +# +# The following cache variables are also available to set or use: +# NANOPB_GENERATOR_EXECUTABLE - The nanopb generator +# PROTOBUF_PROTOC_EXECUTABLE - The protoc compiler +# +# ==================================================================== +# +# NANOPB_GENERATE_CPP (public function) +# SRCS = Variable to define with autogenerated +# source files +# HDRS = Variable to define with autogenerated +# header files +# ARGN = proto files +# +# ==================================================================== +# Example: +# +# set(NANOPB_SRC_ROOT_FOLDER "/path/to/nanopb") +# set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${NANOPB_SRC_ROOT_FOLDER}/cmake) +# find_package( Nanopb REQUIRED ) +# include_directories(${NANOPB_INCLUDE_DIRS}) +# +# NANOPB_GENERATE_CPP(PROTO_SRCS PROTO_HDRS foo.proto) +# +# include_directories(${CMAKE_CURRENT_BINARY_DIR}) +# add_executable(bar bar.cc ${PROTO_SRCS} ${PROTO_HDRS}) +# +# ==================================================================== + +#============================================================================= +# Copyright 2009 Kitware, Inc. +# Copyright 2009-2011 Philip Lowman +# Copyright 2008 Esben Mose Hansen, Ange Optimization ApS +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the names of Kitware, Inc., the Insight Software Consortium, +# nor the names of their contributors may be used to endorse or promote +# products derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +#============================================================================= +# +# Changes +# 2013.01.31 - Pavlo Ilin - used Modules/FindProtobuf.cmake from cmake 2.8.10 to +# write FindNanopb.cmake +# +#============================================================================= + + +function(NANOPB_GENERATE_CPP SRCS HDRS) + if(NOT ARGN) + return() + endif() + + if(NANOPB_GENERATE_CPP_APPEND_PATH) + # Create an include path for each file specified + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(ABS_PATH ${ABS_FIL} PATH) + + list(FIND _nanobp_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _nanobp_include_path -I ${ABS_PATH}) + endif() + endforeach() + else() + set(_nanobp_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + if(DEFINED NANOPB_IMPORT_DIRS) + foreach(DIR ${NANOPB_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND _nanobp_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _nanobp_include_path -I ${ABS_PATH}) + endif() + endforeach() + endif() + + set(${SRCS}) + set(${HDRS}) + get_filename_component(GENERATOR_PATH ${NANOPB_GENERATOR_EXECUTABLE} PATH) + + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c") + list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") + + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb" + COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + ARGS -I${GENERATOR_PATH} -I${CMAKE_CURRENT_BINARY_DIR} ${_nanobp_include_path} -o${FIL_WE}.pb ${ABS_FIL} + DEPENDS ${ABS_FIL} + COMMENT "Running C++ protocol buffer compiler on ${FIL}" + VERBATIM ) + + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" + COMMAND python + ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb" + COMMENT "Running nanopb generator on ${FIL_WE}.pb" + VERBATIM ) + endforeach() + + set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) + set(${SRCS} ${${SRCS}} ${NANOPB_SRCS} PARENT_SCOPE) + set(${HDRS} ${${HDRS}} ${NANOPB_HDRS} PARENT_SCOPE) + +endfunction() + + + +# +# Main. +# + +# By default have NANOPB_GENERATE_CPP macro pass -I to protoc +# for each directory where a proto file is referenced. +if(NOT DEFINED NANOPB_GENERATE_CPP_APPEND_PATH) + set(NANOPB_GENERATE_CPP_APPEND_PATH TRUE) +endif() + +# Find the include directory +find_path(NANOPB_INCLUDE_DIRS + pb.h + PATHS ${NANOPB_SRC_ROOT_FOLDER} +) +mark_as_advanced(NANOPB_INCLUDE_DIRS) + +# Find nanopb source files +set(NANOPB_SRCS) +set(NANOPB_HDRS) +list(APPEND _nanopb_srcs pb_decode.c pb_encode.c) +list(APPEND _nanopb_hdrs pb_decode.h pb_encode.h pb.h) + +foreach(FIL ${_nanopb_srcs}) + find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_SRC_ROOT_FOLDER} ${NANOPB_INCLUDE_DIRS}) + list(APPEND NANOPB_SRCS "${${FIL}__nano_pb_file}") + mark_as_advanced(${FIL}__nano_pb_file) +endforeach() + +foreach(FIL ${_nanopb_hdrs}) + find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_INCLUDE_DIRS}) + mark_as_advanced(${FIL}__nano_pb_file) + list(APPEND NANOPB_HDRS "${${FIL}__nano_pb_file}") +endforeach() + +# Find the protoc Executable +find_program(PROTOBUF_PROTOC_EXECUTABLE + NAMES protoc + DOC "The Google Protocol Buffers Compiler" + PATHS + ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Release + ${PROTOBUF_SRC_ROOT_FOLDER}/vsprojects/Debug +) +mark_as_advanced(PROTOBUF_PROTOC_EXECUTABLE) + +# Find nanopb generator +find_file(NANOPB_GENERATOR_EXECUTABLE + NAMES nanopb_generator.py + DOC "nanopb generator" + PATHS + ${NANOPB_SRC_ROOT_FOLDER}/generator +) +mark_as_advanced(NANOPB_GENERATOR_EXECUTABLE) + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(NANOPB DEFAULT_MSG + NANOPB_INCLUDE_DIRS + NANOPB_SRCS NANOPB_HDRS + NANOPB_GENERATOR_EXECUTABLE + PROTOBUF_PROTOC_EXECUTABLE + ) diff --git a/extra/pb_syshdr.h b/extra/pb_syshdr.h new file mode 100644 index 00000000..b9a97a4f --- /dev/null +++ b/extra/pb_syshdr.h @@ -0,0 +1,94 @@ +/* This is an example of a header file for platforms/compilers that do + * not come with stdint.h/stddef.h/stdbool.h/string.h. To use it, define + * PB_SYSTEM_HEADER as "pb_syshdr.h", including the quotes, and add the + * extra folder to your include path. + * + * It is very likely that you will need to customize this file to suit + * your platform. For any compiler that supports C99, this file should + * not be necessary. + */ + +#ifndef _PB_SYSHDR_H_ +#define _PB_SYSHDR_H_ + +/* stdint.h subset */ +#ifdef HAVE_STDINT_H +#include +#else +/* You will need to modify these to match the word size of your platform. */ +typedef signed char int8_t; +typedef unsigned char uint8_t; +typedef signed short int16_t; +typedef unsigned short uint16_t; +typedef signed int int32_t; +typedef unsigned int uint32_t; +typedef signed long long int64_t; +typedef unsigned long long uint64_t; +#endif + +/* stddef.h subset */ +#ifdef HAVE_STDDEF_H +#include +#else + +typedef uint32_t size_t; +#define offsetof(st, m) ((size_t)(&((st *)0)->m)) + +#ifndef NULL +#define NULL 0 +#endif + +#endif + +/* stdbool.h subset */ +#ifdef HAVE_STDBOOL_H +#include +#else + +#ifndef __cplusplus +typedef int bool; +#define false 0 +#define true 1 +#endif + +#endif + +/* string.h subset */ +#ifdef HAVE_STRING_H +#include +#else + +/* Implementations are from the Public Domain C Library (PDCLib). */ +static size_t strlen( const char * s ) +{ + size_t rc = 0; + while ( s[rc] ) + { + ++rc; + } + return rc; +} + +static void * memcpy( void *s1, const void *s2, size_t n ) +{ + char * dest = (char *) s1; + const char * src = (const char *) s2; + while ( n-- ) + { + *dest++ = *src++; + } + return s1; +} + +static void * memset( void * s, int c, size_t n ) +{ + unsigned char * p = (unsigned char *) s; + while ( n-- ) + { + *p++ = (unsigned char) c; + } + return s; +} +#endif + +#endif -- cgit 1.2.3-korg From 45c1a32e5046fe9323d4fd0213b474a554b2bbed Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 20:04:29 +0200 Subject: Rework the Makefiles to be compatible with binary packages. --- examples/network_server/Makefile | 16 ++++++---------- examples/simple/Makefile | 9 ++++----- examples/using_double_on_avr/Makefile | 19 ++++++------------- examples/using_union_messages/Makefile | 16 ++++++---------- extra/nanopb.mk | 29 +++++++++++++++++++++++++++++ generator/nanopb_generator.py | 17 ++++++++++++----- generator/protoc-gen-nanopb | 12 ++++++++++++ generator/protoc-gen-nanopb.bat | 11 +++++++++++ 8 files changed, 86 insertions(+), 43 deletions(-) create mode 100644 extra/nanopb.mk create mode 100755 generator/protoc-gen-nanopb create mode 100644 generator/protoc-gen-nanopb.bat diff --git a/examples/network_server/Makefile b/examples/network_server/Makefile index 981f2cf9..3c014009 100644 --- a/examples/network_server/Makefile +++ b/examples/network_server/Makefile @@ -1,9 +1,8 @@ -CFLAGS = -ansi -Wall -Werror -g -O0 +# Include the nanopb provided Makefile rules +include ../../extra/nanopb.mk -# Path to the nanopb root folder -NANOPB_DIR = ../.. -DEPS = $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_decode.h \ - $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_encode.h $(NANOPB_DIR)/pb.h +# Compiler flags to enable all warnings & debug info +CFLAGS = -ansi -Wall -Werror -g -O0 CFLAGS += -I$(NANOPB_DIR) all: server client @@ -11,9 +10,6 @@ all: server client clean: rm -f server client fileproto.pb.c fileproto.pb.h -%: %.c $(DEPS) fileproto.pb.h fileproto.pb.c - $(CC) $(CFLAGS) -o $@ $< $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_encode.c fileproto.pb.c common.c +%: %.c common.c fileproto.pb.c + $(CC) $(CFLAGS) -o $@ $^ $(NANOPB_CORE) -fileproto.pb.c fileproto.pb.h: fileproto.proto $(NANOPB_DIR)/generator/nanopb_generator.py - protoc -ofileproto.pb $< - python $(NANOPB_DIR)/generator/nanopb_generator.py fileproto.pb diff --git a/examples/simple/Makefile b/examples/simple/Makefile index 2be3a047..02a4c3ff 100644 --- a/examples/simple/Makefile +++ b/examples/simple/Makefile @@ -1,8 +1,8 @@ +# Include the nanopb provided Makefile rules +include ../../extra/nanopb.mk + # Compiler flags to enable all warnings & debug info CFLAGS = -Wall -Werror -g -O0 - -# Path to the nanopb root folder -NANOPB_DIR = ../.. CFLAGS += -I$(NANOPB_DIR) # C source code files that are required @@ -17,6 +17,5 @@ simple: $(CSRC) # Build rule for the protocol simple.pb.c: simple.proto - protoc -osimple.pb simple.proto - python $(NANOPB_DIR)/generator/nanopb_generator.py simple.pb + $(PROTOC) $(PROTOC_OPTS) --nanopb_out=. simple.proto diff --git a/examples/using_double_on_avr/Makefile b/examples/using_double_on_avr/Makefile index 0b5383fa..6f2203ae 100644 --- a/examples/using_double_on_avr/Makefile +++ b/examples/using_double_on_avr/Makefile @@ -1,10 +1,8 @@ -CFLAGS = -Wall -Werror -g -O0 +# Include the nanopb provided Makefile rules +include ../../extra/nanopb.mk -# Path to the nanopb root directory -NANOPB_DIR = ../.. -DEPS = double_conversion.c $(NANOPB_DIR)/pb.h \ - $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_decode.h \ - $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_encode.h +# Compiler flags to enable all warnings & debug info +CFLAGS = -Wall -Werror -g -O0 CFLAGS += -I$(NANOPB_DIR) all: run_tests @@ -15,13 +13,8 @@ clean: test_conversions: test_conversions.c double_conversion.c $(CC) $(CFLAGS) -o $@ $^ -%: %.c $(DEPS) doubleproto.pb.h doubleproto.pb.c - $(CC) $(CFLAGS) -o $@ $< double_conversion.c \ - $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_encode.c doubleproto.pb.c - -doubleproto.pb.c doubleproto.pb.h: doubleproto.proto $(NANOPB_DIR)/generator/nanopb_generator.py - protoc -odoubleproto.pb $< - python $(NANOPB_DIR)/generator/nanopb_generator.py doubleproto.pb +%: %.c double_conversion.c doubleproto.pb.c + $(CC) $(CFLAGS) -o $@ $^ $(NANOPB_CORE) run_tests: test_conversions encode_double decode_double ./test_conversions diff --git a/examples/using_union_messages/Makefile b/examples/using_union_messages/Makefile index 0f7b5206..618bca50 100644 --- a/examples/using_union_messages/Makefile +++ b/examples/using_union_messages/Makefile @@ -1,9 +1,8 @@ -CFLAGS = -ansi -Wall -Werror -g -O0 +# Include the nanopb provided Makefile rules +include ../../extra/nanopb.mk -# Path to the nanopb root folder -NANOPB_DIR = ../.. -DEPS = $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_decode.h \ - $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_encode.h $(NANOPB_DIR)/pb.h +# Compiler flags to enable all warnings & debug info +CFLAGS = -ansi -Wall -Werror -g -O0 CFLAGS += -I$(NANOPB_DIR) all: encode decode @@ -14,9 +13,6 @@ all: encode decode clean: rm -f encode unionproto.pb.h unionproto.pb.c -%: %.c $(DEPS) unionproto.pb.h unionproto.pb.c - $(CC) $(CFLAGS) -o $@ $< $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_encode.c unionproto.pb.c +%: %.c unionproto.pb.c + $(CC) $(CFLAGS) -o $@ $^ $(NANOPB_CORE) -unionproto.pb.h unionproto.pb.c: unionproto.proto $(NANOPB_DIR)/generator/nanopb_generator.py - protoc -ounionproto.pb $< - python $(NANOPB_DIR)/generator/nanopb_generator.py unionproto.pb diff --git a/extra/nanopb.mk b/extra/nanopb.mk new file mode 100644 index 00000000..b52c78c1 --- /dev/null +++ b/extra/nanopb.mk @@ -0,0 +1,29 @@ +# This is an include file for Makefiles. It provides rules for building +# .pb.c and .pb.h files out of .proto, as well the path to nanopb core. + +# Path to the nanopb root directory +NANOPB_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))../) + +# Files for the nanopb core +NANOPB_CORE = $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_decode.c + +# Check whether to use binary version of nanopb_generator or the +# system-supplied python interpreter. +ifneq "$(wildcard $(NANOPB_DIR)/generator-bin)" "" + # Binary package + PROTOC = $(NANOPB_DIR)/generator-bin/protoc + PROTOC_OPTS = +else + # Source only or git checkout + PROTOC = protoc + ifdef windir + PROTOC_OPTS = --plugin=nanopb=$(NANOPB_DIR)/generator/protoc-gen-nanopb.bat + else + PROTOC_OPTS = --plugin=nanopb=$(NANOPB_DIR)/generator/protoc-gen-nanopb + endif +endif + +# Rule for building .pb.c and .pb.h +%.pb.c %.pb.h: %.proto $(wildcard %.options) + $(PROTOC) $(PROTOC_OPTS) --nanopb_out=. $< + diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 86c554a2..7b3c9f80 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -15,10 +15,17 @@ except: print raise -import proto.nanopb_pb2 as nanopb_pb2 -import proto.descriptor_pb2 as descriptor - - +try: + import proto.nanopb_pb2 as nanopb_pb2 + import proto.descriptor_pb2 as descriptor +except: + print + print "********************************************************************" + print "*** Failed to import the protocol definitions for generator. ***" + print "*** You have to run 'make' in the nanopb/generator/proto folder. ***" + print "********************************************************************" + print + raise # --------------------------------------------------------------------------- @@ -1054,7 +1061,7 @@ def main_plugin(): if __name__ == '__main__': # Check if we are running as a plugin under protoc - if 'protoc-gen-' in sys.argv[0]: + if 'protoc-gen-' in sys.argv[0] or '--protoc-plugin' in sys.argv: main_plugin() else: main_cli() diff --git a/generator/protoc-gen-nanopb b/generator/protoc-gen-nanopb new file mode 100755 index 00000000..90be6a58 --- /dev/null +++ b/generator/protoc-gen-nanopb @@ -0,0 +1,12 @@ +#!/bin/sh + +# This file is used to invoke nanopb_generator.py as a plugin +# to protoc on Linux and other *nix-style systems. +# Use it like this: +# protoc --plugin=nanopb=..../protoc-gen-nanopb --nanopb_out=dir foo.proto +# +# Note that if you use the binary package of nanopb, the protoc +# path is already set up properly and there is no need to give +# --plugin= on the command line. + +exec python $(dirname $0)/nanopb_generator.py --protoc-plugin diff --git a/generator/protoc-gen-nanopb.bat b/generator/protoc-gen-nanopb.bat new file mode 100644 index 00000000..f70ca600 --- /dev/null +++ b/generator/protoc-gen-nanopb.bat @@ -0,0 +1,11 @@ +:: This file is used to invoke nanopb_generator.py as a plugin +:: to protoc on Windows. +:: Use it like this: +:: protoc --plugin=nanopb=..../protoc-gen-nanopb.bat --nanopb_out=dir foo.proto +:: +:: Note that if you use the binary package of nanopb, the protoc +:: path is already set up properly and there is no need to give +:: --plugin= on the command line. + +set mydir=%~dp0 +python %mydir%\nanopb_generator.py --protoc-plugin -- cgit 1.2.3-korg From c61660b3ea88c88834a764d08763b3a85514014d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 20:09:30 +0200 Subject: Makefile tuning --- examples/simple/Makefile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/examples/simple/Makefile b/examples/simple/Makefile index 02a4c3ff..ecee4525 100644 --- a/examples/simple/Makefile +++ b/examples/simple/Makefile @@ -13,9 +13,16 @@ CSRC += $(NANOPB_DIR)/pb_decode.c # The nanopb decoder # Build rule for the main program simple: $(CSRC) + @echo + @echo "*** Building the main binary..." $(CC) $(CFLAGS) -osimple $(CSRC) + @echo + @echo "*** Congrats! You can now run ./simple to see what it does!" + @echo # Build rule for the protocol simple.pb.c: simple.proto + @echo + @echo "*** Building the protocol definition..." $(PROTOC) $(PROTOC_OPTS) --nanopb_out=. simple.proto -- cgit 1.2.3-korg From 05e2dc20f32016be40b11cc3e02e1e62bbc3944f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 20:24:32 +0200 Subject: Actually no, it was a portability hassle --- examples/simple/Makefile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/examples/simple/Makefile b/examples/simple/Makefile index ecee4525..02a4c3ff 100644 --- a/examples/simple/Makefile +++ b/examples/simple/Makefile @@ -13,16 +13,9 @@ CSRC += $(NANOPB_DIR)/pb_decode.c # The nanopb decoder # Build rule for the main program simple: $(CSRC) - @echo - @echo "*** Building the main binary..." $(CC) $(CFLAGS) -osimple $(CSRC) - @echo - @echo "*** Congrats! You can now run ./simple to see what it does!" - @echo # Build rule for the protocol simple.pb.c: simple.proto - @echo - @echo "*** Building the protocol definition..." $(PROTOC) $(PROTOC_OPTS) --nanopb_out=. simple.proto -- cgit 1.2.3-korg From d344bda28ae7a1a590790e1346a73a5d159ce7dd Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 20:25:18 +0200 Subject: Fix protoc plugin path --- extra/nanopb.mk | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extra/nanopb.mk b/extra/nanopb.mk index b52c78c1..296f98e2 100644 --- a/extra/nanopb.mk +++ b/extra/nanopb.mk @@ -17,9 +17,9 @@ else # Source only or git checkout PROTOC = protoc ifdef windir - PROTOC_OPTS = --plugin=nanopb=$(NANOPB_DIR)/generator/protoc-gen-nanopb.bat + PROTOC_OPTS = --plugin=protoc-gen-nanopb=$(NANOPB_DIR)/generator/protoc-gen-nanopb.bat else - PROTOC_OPTS = --plugin=nanopb=$(NANOPB_DIR)/generator/protoc-gen-nanopb + PROTOC_OPTS = --plugin=protoc-gen-nanopb=$(NANOPB_DIR)/generator/protoc-gen-nanopb endif endif -- cgit 1.2.3-korg From d7cadaa46fe5979518a7ee2493250434f4e8ee40 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 20:37:18 +0200 Subject: Windows build fixes --- extra/nanopb.mk | 10 +++++++++- generator/protoc-gen-nanopb.bat | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/extra/nanopb.mk b/extra/nanopb.mk index 296f98e2..7576baee 100644 --- a/extra/nanopb.mk +++ b/extra/nanopb.mk @@ -7,6 +7,14 @@ NANOPB_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))../) # Files for the nanopb core NANOPB_CORE = $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_decode.c +# Check if we are running on Windows +ifdef windir +WINDOWS = 1 +endif +ifdef WINDIR +WINDOWS = 1 +endif + # Check whether to use binary version of nanopb_generator or the # system-supplied python interpreter. ifneq "$(wildcard $(NANOPB_DIR)/generator-bin)" "" @@ -16,7 +24,7 @@ ifneq "$(wildcard $(NANOPB_DIR)/generator-bin)" "" else # Source only or git checkout PROTOC = protoc - ifdef windir + ifdef WINDOWS PROTOC_OPTS = --plugin=protoc-gen-nanopb=$(NANOPB_DIR)/generator/protoc-gen-nanopb.bat else PROTOC_OPTS = --plugin=protoc-gen-nanopb=$(NANOPB_DIR)/generator/protoc-gen-nanopb diff --git a/generator/protoc-gen-nanopb.bat b/generator/protoc-gen-nanopb.bat index f70ca600..3eed057b 100644 --- a/generator/protoc-gen-nanopb.bat +++ b/generator/protoc-gen-nanopb.bat @@ -1,3 +1,4 @@ +@echo off :: This file is used to invoke nanopb_generator.py as a plugin :: to protoc on Windows. :: Use it like this: -- cgit 1.2.3-korg From 6e087731535bc938d48c0db09b7cc602c4769e8f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 20:57:50 +0200 Subject: README updates --- README.txt | 62 +++++++++++++++++++++++++++++++++++++++++----- examples/simple/README.txt | 5 ++-- 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/README.txt b/README.txt index 2a12b4f5..c6085d28 100644 --- a/README.txt +++ b/README.txt @@ -1,11 +1,61 @@ -Nanopb is a small code-size Protocol Buffers implementation. +Nanopb is a small code-size Protocol Buffers implementation in ansi C. It is +especially suitable for use in microcontrollers, but fits any memory +restricted system. Homepage: http://kapsi.fi/~jpa/nanopb/ -To compile the library, you'll need these libraries: -protobuf-compiler python-protobuf libprotobuf-dev -The only runtime dependencies are memset() and memcpy(). -To run the tests, run make under the tests folder. -If it completes without error, everything is fine. + +Using the nanopb library +======================== +To use the nanopb library, you need to do two things: + +1) Compile your .proto files for nanopb, using protoc. +2) Include pb_encode.c and pb_decode.c in your project. + +The easiest way to get started is to study the project in "examples/simple". +It contains a Makefile, which should work directly under most Linux systems. +However, for any other kind of build system, see the manual steps in +README.txt in that folder. + + + +Using the Protocol Buffers compiler (protoc) +============================================ +The nanopb generator is implemented as a plugin for the Google's own protoc +compiler. This has the advantage that there is no need to reimplement the +basic parsing of .proto files. However, it does mean that you need the +Google's protobuf library in order to run the generator. + +If you have downloaded a binary package for nanopb (either Windows or Linux +version), the 'protoc' binary is included in the 'generator-bin' folder. In +this case, you are ready to go. Simply run this command: + + generator-bin/protoc --nanopb_out=. myprotocol.proto + +However, if you are using a git checkout or a plain source distribution, you +need to provide your own version of protoc and the Google's protobuf library. +On Linux, the necessary packages are protobuf-compiler and python-protobuf. +On Windows, you can either build Google's protobuf library from source or use +one of the binary distributions of it. In either case, if you use a separate +protoc, you need to manually give the path to nanopb generator: + + protoc --plugin=protoc-gen-nanopb=nanopb/generator/protoc-gen-nanopb ... + + + +Running the tests +================= +If you want to perform further development of the nanopb core, or to verify +its functionality using your compiler and platform, you'll want to run the +test suite. The build rules for the test suite are implemented using Scons, +so you need to have that installed. To run the tests: + + cd tests + scons + +This will show the progress of various test cases. If the output does not +end in an error, the test cases were successful. + + diff --git a/examples/simple/README.txt b/examples/simple/README.txt index d18c4183..ee77bfc7 100644 --- a/examples/simple/README.txt +++ b/examples/simple/README.txt @@ -18,10 +18,9 @@ On Linux, simply type "make" to build the example. After that, you can run it with the command: ./simple On other platforms, you first have to compile the protocol definition using -the following two commands:: +the following command:: - protoc -osimple.pb simple.proto - python nanopb_generator.py simple.pb + ../../generator-bin/protoc --nanopb_out=. simple.proto After that, add the following four files to your project and compile: -- cgit 1.2.3-korg From 2abd27c91c05e124815b77898133e60ec4de16c9 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 21:20:12 +0200 Subject: Include Visual C++ runtime in the Windows package. --- tools/make_windows_package.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/make_windows_package.sh b/tools/make_windows_package.sh index 3096003b..4d4c4c94 100644 --- a/tools/make_windows_package.sh +++ b/tools/make_windows_package.sh @@ -30,7 +30,9 @@ mv $DEST/generator/dist $DEST/generator-bin rm $DEST/generator/protoc-gen-nanopb.py # Package the protoc compiler -cp `which protoc`.exe $DEST/generator-bin/ +cp `which protoc.exe` $DEST/generator-bin/ +cp `which MSVCR100.DLL` $DEST/generator-bin/ +cp `which MSVCP100.DLL` $DEST/generator-bin/ # Convert line breaks for convenience find $DEST -name '*.c' -o -name '*.h' -o -name '*.txt' \ -- cgit 1.2.3-korg From 1d42f72f409b4a464b28d2c706f4d0fd213a8541 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 21:43:50 +0200 Subject: Include MSVCR90.DLL also. --- tools/make_windows_package.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/make_windows_package.sh b/tools/make_windows_package.sh index 4d4c4c94..d8999a97 100644 --- a/tools/make_windows_package.sh +++ b/tools/make_windows_package.sh @@ -26,6 +26,10 @@ cp $DEST/generator/nanopb_generator.py $DEST/generator/protoc-gen-nanopb.py ( cd $DEST/generator; bbfreeze nanopb_generator.py protoc-gen-nanopb.py ) mv $DEST/generator/dist $DEST/generator-bin +# The python interpreter requires MSVCR90.dll. +# FIXME: Find a way around hardcoding this path +cp /c/windows/winsxs/x86_microsoft.vc90.crt_1fc8b3b9a1e18e3b_9.0.30729.4974_none_50940634bcb759cb/MSVCR90.DLL $DEST/generator-bin/ + # Remove temp file rm $DEST/generator/protoc-gen-nanopb.py -- cgit 1.2.3-korg From 94422ad456a659b2d78cc4956b937e3d1f98a187 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 22:19:54 +0200 Subject: Also include the manifest for MSVCR90.DLL --- tools/make_windows_package.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tools/make_windows_package.sh b/tools/make_windows_package.sh index d8999a97..375649fa 100644 --- a/tools/make_windows_package.sh +++ b/tools/make_windows_package.sh @@ -29,6 +29,14 @@ mv $DEST/generator/dist $DEST/generator-bin # The python interpreter requires MSVCR90.dll. # FIXME: Find a way around hardcoding this path cp /c/windows/winsxs/x86_microsoft.vc90.crt_1fc8b3b9a1e18e3b_9.0.30729.4974_none_50940634bcb759cb/MSVCR90.DLL $DEST/generator-bin/ +cat > $DEST/generator-bin/Microsoft.VC90.CRT.manifest < + + + + KSaO8M0iCtPF6YEr79P1dZsnomY= ojDmTgpYMFRKJYkPcM6ckpYkWUU= tVogb8kezDre2mXShlIqpp8ErIg= + +EOF # Remove temp file rm $DEST/generator/protoc-gen-nanopb.py -- cgit 1.2.3-korg From 8be8b7e41a27b0feb12b75fed42bee34258f1e35 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 22:25:52 +0200 Subject: Linux archive --- tools/make_linux_package.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/make_linux_package.sh b/tools/make_linux_package.sh index b0ad2dd2..0c8e826b 100755 --- a/tools/make_linux_package.sh +++ b/tools/make_linux_package.sh @@ -36,3 +36,6 @@ chmod +x $DEST/generator-bin/protoc # Make the nanopb generator available as a protoc plugin ln -s nanopb-generator $DEST/generator-bin/protoc-gen-nanopb +# Tar it all up +( cd dist; tar -czf $VERSION.tar.gz $VERSION ) + -- cgit 1.2.3-korg From 3bdbd4dc484945bfaa305d1df489a7805998139a Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 22:33:26 +0200 Subject: Include libprotoc --- tools/make_linux_package.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/make_linux_package.sh b/tools/make_linux_package.sh index 0c8e826b..c48a8074 100755 --- a/tools/make_linux_package.sh +++ b/tools/make_linux_package.sh @@ -24,6 +24,8 @@ mv $DEST/generator/dist $DEST/generator-bin # Package the protoc compiler cp `which protoc` $DEST/generator-bin/protoc.bin +LIBPROTOC=$(ldd `which protoc` | grep -o '/.*libprotoc[^ ]*') +cp $LIBPROTOC $DEST/generator-bin/ cat > $DEST/generator-bin/protoc << EOF #!/bin/bash SCRIPTDIR=\$(dirname \$(readlink -f \$0)) -- cgit 1.2.3-korg From 440a53f85965f35cd7bc401ef7e3fb9c4dfd1390 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 17 Nov 2013 22:37:48 +0200 Subject: Fix protoc-gen-nanopb in linux package --- tools/make_linux_package.sh | 11 +++++++---- tools/make_windows_package.sh | 6 +++--- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tools/make_linux_package.sh b/tools/make_linux_package.sh index c48a8074..fef0254b 100755 --- a/tools/make_linux_package.sh +++ b/tools/make_linux_package.sh @@ -18,10 +18,16 @@ git archive HEAD | tar x -C $DEST # Rebuild the Python .proto files make -BC $DEST/generator/proto +# Make the nanopb generator available as a protoc plugin +cp $DEST/generator/nanopb_generator.py $DEST/generator/protoc-gen-nanopb.py + # Package the Python libraries -( cd $DEST/generator; bbfreeze nanopb_generator.py ) +( cd $DEST/generator; bbfreeze nanopb_generator.py protoc-gen-nanopb.py ) mv $DEST/generator/dist $DEST/generator-bin +# Remove temp file +rm $DEST/generator/protoc-gen-nanopb.py + # Package the protoc compiler cp `which protoc` $DEST/generator-bin/protoc.bin LIBPROTOC=$(ldd `which protoc` | grep -o '/.*libprotoc[^ ]*') @@ -35,9 +41,6 @@ exec \$SCRIPTDIR/protoc.bin "\$@" EOF chmod +x $DEST/generator-bin/protoc -# Make the nanopb generator available as a protoc plugin -ln -s nanopb-generator $DEST/generator-bin/protoc-gen-nanopb - # Tar it all up ( cd dist; tar -czf $VERSION.tar.gz $VERSION ) diff --git a/tools/make_windows_package.sh b/tools/make_windows_package.sh index 375649fa..fe83bc44 100644 --- a/tools/make_windows_package.sh +++ b/tools/make_windows_package.sh @@ -26,6 +26,9 @@ cp $DEST/generator/nanopb_generator.py $DEST/generator/protoc-gen-nanopb.py ( cd $DEST/generator; bbfreeze nanopb_generator.py protoc-gen-nanopb.py ) mv $DEST/generator/dist $DEST/generator-bin +# Remove temp file +rm $DEST/generator/protoc-gen-nanopb.py + # The python interpreter requires MSVCR90.dll. # FIXME: Find a way around hardcoding this path cp /c/windows/winsxs/x86_microsoft.vc90.crt_1fc8b3b9a1e18e3b_9.0.30729.4974_none_50940634bcb759cb/MSVCR90.DLL $DEST/generator-bin/ @@ -38,9 +41,6 @@ cat > $DEST/generator-bin/Microsoft.VC90.CRT.manifest < EOF -# Remove temp file -rm $DEST/generator/protoc-gen-nanopb.py - # Package the protoc compiler cp `which protoc.exe` $DEST/generator-bin/ cp `which MSVCR100.DLL` $DEST/generator-bin/ -- cgit 1.2.3-korg From 1fc90ab4c4731ab521ee7f6b4f1f936f76520e2e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 23 Nov 2013 11:04:12 +0200 Subject: Add a SCons tool for nanopb protos. This tool script is useful both for building the tests and also for use in other applications. --- tests/SConstruct | 2 +- tests/common/SConscript | 4 +- tests/site_scons/site_init.py | 36 ---------- tests/site_scons/site_tools/nanopb.py | 125 ++++++++++++++++++++++++++++++++++ tests/special_characters/SConscript | 3 +- 5 files changed, 129 insertions(+), 41 deletions(-) create mode 100644 tests/site_scons/site_tools/nanopb.py diff --git a/tests/SConstruct b/tests/SConstruct index 675989a0..a44ee55f 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -14,7 +14,7 @@ scons CC=clang CXX=clang++ ''') import os -env = Environment(ENV = os.environ) +env = Environment(ENV = os.environ, tools = ['default', 'nanopb']) # Allow overriding the compiler with scons CC=??? if 'CC' in ARGUMENTS: env.Replace(CC = ARGUMENTS['CC']) diff --git a/tests/common/SConscript b/tests/common/SConscript index 8130c85a..144f1490 100644 --- a/tests/common/SConscript +++ b/tests/common/SConscript @@ -12,6 +12,6 @@ env.NanopbProto("person") # These are built using more strict warning flags. strict = env.Clone() strict.Append(CFLAGS = strict['CORECFLAGS']) -strict.Object("pb_decode.o", "#../pb_decode.c") -strict.Object("pb_encode.o", "#../pb_encode.c") +strict.Object("pb_decode.o", "$NANOPB/pb_decode.c") +strict.Object("pb_encode.o", "$NANOPB/pb_encode.c") diff --git a/tests/site_scons/site_init.py b/tests/site_scons/site_init.py index 38aa1a44..2226144f 100644 --- a/tests/site_scons/site_init.py +++ b/tests/site_scons/site_init.py @@ -11,42 +11,6 @@ except ImportError: def add_nanopb_builders(env): '''Add the necessary builder commands for nanopb tests.''' - - # Build command for building .pb from .proto using protoc - def proto_actions(source, target, env, for_signature): - esc = env['ESCAPE'] - dirs = ' '.join(['-I' + esc(env.GetBuildPath(d)) for d in env['PROTOCPATH']]) - return '$PROTOC $PROTOCFLAGS %s -o%s %s' % (dirs, esc(str(target[0])), esc(str(source[0]))) - - proto_file_builder = Builder(generator = proto_actions, - suffix = '.pb', - src_suffix = '.proto') - env.Append(BUILDERS = {'Proto': proto_file_builder}) - env.SetDefault(PROTOC = 'protoc') - env.SetDefault(PROTOCPATH = ['.']) - - # Build command for running nanopb generator - import os.path - def nanopb_targets(target, source, env): - basename = os.path.splitext(str(source[0]))[0] - target.append(basename + '.pb.h') - return target, source - - nanopb_file_builder = Builder(action = '$NANOPB_GENERATOR $NANOPB_FLAGS $SOURCE', - suffix = '.pb.c', - src_suffix = '.pb', - emitter = nanopb_targets) - env.Append(BUILDERS = {'Nanopb': nanopb_file_builder}) - gen_path = env['ESCAPE'](env.GetBuildPath("#../generator/nanopb_generator.py")) - env.SetDefault(NANOPB_GENERATOR = 'python ' + gen_path) - env.SetDefault(NANOPB_FLAGS = '-q') - - # Combined method to run both protoc and nanopb generator - def run_protoc_and_nanopb(env, source): - b1 = env.Proto(source) - b2 = env.Nanopb(source) - return b1 + b2 - env.AddMethod(run_protoc_and_nanopb, "NanopbProto") # Build command that runs a test program and saves the output def run_test(target, source, env): diff --git a/tests/site_scons/site_tools/nanopb.py b/tests/site_scons/site_tools/nanopb.py new file mode 100644 index 00000000..0ed70464 --- /dev/null +++ b/tests/site_scons/site_tools/nanopb.py @@ -0,0 +1,125 @@ +''' +Scons Builder for nanopb .proto definitions. + +This tool will locate the nanopb generator and use it to generate .pb.c and +.pb.h files from the .proto files. + +Basic example +------------- +# Build myproto.pb.c and myproto.pb.h from myproto.proto +myproto = env.NanopbProto("myproto") + +# Link nanopb core to the program +env.Append(CPPPATH = "$NANOB") +myprog = env.Program(["myprog.c", myproto, "$NANOPB/pb_encode.c", "$NANOPB/pb_decode.c"]) + +Configuration options +--------------------- +Normally, this script is used in the test environment of nanopb and it locates +the nanopb generator by a relative path. If this script is used in another +application, the path to nanopb root directory has to be defined: + +env.SetDefault(NANOPB = "path/to/nanopb") + +Additionally, the path to protoc and the options to give to protoc can be +defined manually: + +env.SetDefault(PROTOC = "path/to/protoc") +env.SetDefault(PROTOCFLAGS = "--plugin=protoc-gen-nanopb=path/to/protoc-gen-nanopb") +''' + +import SCons.Action +import SCons.Builder +import SCons.Util +import os.path + +class NanopbWarning(SCons.Warnings.Warning): + pass +SCons.Warnings.enableWarningClass(NanopbWarning) + +def _detect_nanopb(env): + '''Find the path to nanopb root directory.''' + if env.has_key('NANOPB'): + # Use nanopb dir given by user + return env['NANOPB'] + + p = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', '..')) + if os.path.isdir(p) and os.path.isfile(os.path.join(p, 'pb.h')): + # Assume we are running under tests/site_scons/site_tools + return p + + raise SCons.Errors.StopError(NanopbWarning, + "Could not find the nanopb root directory") + +def _detect_protoc(env): + '''Find the path to the protoc compiler.''' + if env.has_key('PROTOC'): + # Use protoc defined by user + return env['PROTOC'] + + p = _detect_nanopb(env) + p1 = os.path.join(p, 'generator-bin', 'protoc') + if os.path.exists(p1): + # Use protoc bundled with binary package + return p1 + + p = env.WhereIs('protoc') + if p: + # Use protoc from path + return p + + raise SCons.Errors.StopError(NanopbWarning, + "Could not find the protoc compiler") + +def _detect_protocflags(env): + '''Find the options to use for protoc.''' + if env.has_key('PROTOCFLAGS'): + return env['PROTOCFLAGS'] + + p = _detect_protoc(env) + n = _detect_nanopb(env) + if p == os.path.join(n, 'generator-bin', 'protoc'): + # Using the bundled protoc, no options needed + return '' + + e = env['ESCAPE'] + if env['PLATFORM'] == 'win32': + return e('--plugin=protoc-gen-nanopb=' + os.path.join(n, 'generator', 'protoc-gen-nanopb.bat')) + else: + return e('--plugin=protoc-gen-nanopb=' + os.path.join(n, 'generator', 'protoc-gen-nanopb')) + +def _nanopb_proto_actions(source, target, env, for_signature): + esc = env['ESCAPE'] + dirs = ' '.join(['-I' + esc(env.GetBuildPath(d)) for d in env['PROTOCPATH']]) + return '$PROTOC $PROTOCFLAGS %s --nanopb_out=. %s' % (dirs, esc(str(source[0]))) + +def _nanopb_proto_emitter(target, source, env): + basename = os.path.splitext(str(source[0]))[0] + target.append(basename + '.pb.h') + + if os.path.exists(basename + '.options'): + source.append(basename + '.options') + + return target, source + +_nanopb_proto_builder = SCons.Builder.Builder( + generator = _nanopb_proto_actions, + suffix = '.pb.c', + src_suffix = '.proto', + emitter = _nanopb_proto_emitter) + +def generate(env): + '''Add Builder for nanopb protos.''' + + env['NANOPB'] = _detect_nanopb(env) + env['PROTOC'] = _detect_protoc(env) + env['PROTOCFLAGS'] = _detect_protocflags(env) + + env.SetDefault(PROTOCPATH = ['.', os.path.join(env['NANOPB'], 'generator', 'proto')]) + + env.SetDefault(NANOPB_PROTO_CMD = '$PROTOC $PROTOC_OPTS --nanopb_out=. $SOURCE') + env['BUILDERS']['NanopbProto'] = _nanopb_proto_builder + +def exists(env): + return _detect_protoc(env) and _detect_protoc_opts(env) + diff --git a/tests/special_characters/SConscript b/tests/special_characters/SConscript index 05dccaee..2309cf2e 100644 --- a/tests/special_characters/SConscript +++ b/tests/special_characters/SConscript @@ -2,6 +2,5 @@ Import('env') -env.Proto("funny-proto+name has.characters.proto") -env.Nanopb("funny-proto+name has.characters.pb.c", "funny-proto+name has.characters.pb") +env.NanopbProto("funny-proto+name has.characters.proto") env.Object("funny-proto+name has.characters.pb.c") -- cgit 1.2.3-korg From dde71cb426addb6e847c2443ee1bd551d9eb0637 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 3 Dec 2013 18:44:25 +0200 Subject: Make generator print errors to stderr. Otherwise they won't be visible when run as a protoc plugin. --- generator/nanopb_generator.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 7b3c9f80..ab6f3cb5 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -3,28 +3,30 @@ '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' nanopb_version = "nanopb-0.2.5-dev" +import sys + try: import google, distutils.util # bbfreeze seems to need these import google.protobuf.text_format as text_format except: - print - print "*************************************************************" - print "*** Could not import the Google protobuf Python libraries ***" - print "*** Try installing package 'python-protobuf' or similar. ***" - print "*************************************************************" - print + sys.stderr.write(''' + ************************************************************* + *** Could not import the Google protobuf Python libraries *** + *** Try installing package 'python-protobuf' or similar. *** + ************************************************************* + ''' + '\n') raise try: import proto.nanopb_pb2 as nanopb_pb2 import proto.descriptor_pb2 as descriptor except: - print - print "********************************************************************" - print "*** Failed to import the protocol definitions for generator. ***" - print "*** You have to run 'make' in the nanopb/generator/proto folder. ***" - print "********************************************************************" - print + sys.stderr.write(''' + ******************************************************************** + *** Failed to import the protocol definitions for generator. *** + *** You have to run 'make' in the nanopb/generator/proto folder. *** + ******************************************************************** + ''' + '\n') raise -- cgit 1.2.3-korg From 6d0e0695d01bf1823a8aae2b3ec1d488e5215b76 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 3 Dec 2013 19:27:08 +0200 Subject: Check for supported GCC CCFLAGS when building tests. --- tests/SConstruct | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/tests/SConstruct b/tests/SConstruct index a44ee55f..9480c82c 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -33,7 +33,17 @@ env.Append(PROTOCPATH = '#../generator') # Check the compilation environment, unless we are just cleaning up. if not env.GetOption('clean'): - conf = Configure(env) + def check_ccflags(context, flags): + '''Check if given CCFLAGS are supported''' + context.Message('Checking support for CCFLAGS="%s"...' % flags) + oldflags = context.env['CCFLAGS'] + context.env.Append(CCFLAGS = flags) + result = context.TryCompile("int main() {return 0;}", '.c') + context.env.Replace(CCFLAGS = oldflags) + context.Result(result) + return result + + conf = Configure(env, custom_tests = {'CheckCCFLAGS': check_ccflags}) # If the platform doesn't support C99, use our own header file instead. stdbool = conf.CheckCHeader('stdbool.h') @@ -62,6 +72,14 @@ if not env.GetOption('clean'): conf.env.Append(CCFLAGS = '-fmudflap') conf.env.Append(LINKFLAGS = '-lmudflap -fmudflap') + # Check if we can use extra strict warning flags (only with GCC) + extra = '-Wcast-qual -Wlogical-op -Wconversion' + extra += ' -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls' + extra += ' -Wstack-protector' + if 'gcc' in env['CC']: + if conf.CheckCCFLAGS(extra): + conf.env.Append(CORECFLAGS = extra) + # End the config stuff env = conf.Finish() @@ -71,15 +89,11 @@ if 'gcc' in env['CC']: # Debug info, warnings as errors env.Append(CFLAGS = '-ansi -pedantic -g -O0 -Wall -Werror --coverage -fstack-protector-all') + env.Append(CORECFLAGS = '-Wextra') env.Append(LINKFLAGS = '--coverage') # We currently need uint64_t anyway, even though ANSI C90 otherwise.. env.Append(CFLAGS = '-Wno-long-long') - - # More strict checks on the nanopb core - env.Append(CORECFLAGS = '-Wextra -Wcast-qual -Wlogical-op -Wconversion') - env.Append(CORECFLAGS = ' -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls') - env.Append(CORECFLAGS = ' -Wstack-protector') elif 'clang' in env['CC']: # CLang env.Append(CFLAGS = '-ansi -g -O0 -Wall -Werror') -- cgit 1.2.3-korg From b1cb035373fc3b6888e3664ca0ca3f37a9c0e639 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 3 Dec 2013 19:30:39 +0200 Subject: Small fix for previous --- tests/SConstruct | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/SConstruct b/tests/SConstruct index 9480c82c..c1defe92 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -35,7 +35,7 @@ env.Append(PROTOCPATH = '#../generator') if not env.GetOption('clean'): def check_ccflags(context, flags): '''Check if given CCFLAGS are supported''' - context.Message('Checking support for CCFLAGS="%s"...' % flags) + context.Message('Checking support for CCFLAGS="%s"... ' % flags) oldflags = context.env['CCFLAGS'] context.env.Append(CCFLAGS = flags) result = context.TryCompile("int main() {return 0;}", '.c') @@ -75,7 +75,7 @@ if not env.GetOption('clean'): # Check if we can use extra strict warning flags (only with GCC) extra = '-Wcast-qual -Wlogical-op -Wconversion' extra += ' -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls' - extra += ' -Wstack-protector' + extra += ' -Wstack-protector ' if 'gcc' in env['CC']: if conf.CheckCCFLAGS(extra): conf.env.Append(CORECFLAGS = extra) -- cgit 1.2.3-korg From 435ccbfdf7e48cfd809c41049d9edf4d6c267566 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 3 Dec 2013 20:10:48 +0200 Subject: Fixes for pyinstaller --- generator/nanopb_generator.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index ab6f3cb5..6ccb96f9 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -6,7 +6,14 @@ nanopb_version = "nanopb-0.2.5-dev" import sys try: + # Add some dummy imports to keep packaging tools happy. import google, distutils.util # bbfreeze seems to need these + import pkg_resources # pyinstaller / protobuf 2.5 seem to need these +except: + # Don't care, we will error out later if it is actually important. + pass + +try: import google.protobuf.text_format as text_format except: sys.stderr.write(''' -- cgit 1.2.3-korg From 2b334015afd0bfd1888dc4b08f02778d96b2da8c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 3 Dec 2013 20:35:29 +0200 Subject: Add packaging script for Mac, add platform name to packages. --- tools/make_linux_package.sh | 2 +- tools/make_mac_package.sh | 49 +++++++++++++++++++++++++++++++++++++++++++ tools/make_windows_package.sh | 2 +- 3 files changed, 51 insertions(+), 2 deletions(-) create mode 100755 tools/make_mac_package.sh mode change 100644 => 100755 tools/make_windows_package.sh diff --git a/tools/make_linux_package.sh b/tools/make_linux_package.sh index fef0254b..342029aa 100755 --- a/tools/make_linux_package.sh +++ b/tools/make_linux_package.sh @@ -42,5 +42,5 @@ EOF chmod +x $DEST/generator-bin/protoc # Tar it all up -( cd dist; tar -czf $VERSION.tar.gz $VERSION ) +( cd dist; tar -czf $VERSION-linux.tar.gz $VERSION ) diff --git a/tools/make_mac_package.sh b/tools/make_mac_package.sh new file mode 100755 index 00000000..45ce71a0 --- /dev/null +++ b/tools/make_mac_package.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Run this script in the top nanopb directory to create a binary package +# for Mac OS X users. + +# Requires: protobuf, python-protobuf, pyinstaller + +set -e +set -x + +VERSION=`git describe --always` +DEST=dist/$VERSION + +rm -rf $DEST +mkdir -p $DEST + +# Export the files from newest commit +git archive HEAD | tar x -C $DEST + +# Rebuild the Python .proto files +make -BC $DEST/generator/proto + +# Package the Python libraries +( cd $DEST/generator; pyinstaller nanopb_generator.py ) +mv $DEST/generator/dist/nanopb_generator $DEST/generator-bin + +# Remove temp files +rm -rf $DEST/generator/dist $DEST/generator/build $DEST/generator/nanopb_generator.spec + +# Make the nanopb generator available as a protoc plugin +cp $DEST/generator-bin/nanopb_generator $DEST/generator-bin/protoc-gen-nanopb + +# Package the protoc compiler +cp `which protoc` $DEST/generator-bin/protoc.bin +LIBPROTOC=$(otool -L `which protoc` | grep -o '/.*libprotoc[^ ]*') +LIBPROTOBUF=$(otool -L `which protoc` | grep -o '/.*libprotobuf[^ ]*') +cp $LIBPROTOC $LIBPROTOBUF $DEST/generator-bin/ +cat > $DEST/generator-bin/protoc << EOF +#!/bin/bash +SCRIPTDIR=\$(cd \$(dirname \$0); pwd) +export DYLD_LIBRARY_PATH=\$SCRIPTDIR +export PATH=\$SCRIPTDIR:\$PATH +exec \$SCRIPTDIR/protoc.bin "\$@" +EOF +chmod +x $DEST/generator-bin/protoc + +# Tar it all up +( cd dist; tar -czf $VERSION-mac.tgz $VERSION ) + diff --git a/tools/make_windows_package.sh b/tools/make_windows_package.sh old mode 100644 new mode 100755 index fe83bc44..1a339a2b --- a/tools/make_windows_package.sh +++ b/tools/make_windows_package.sh @@ -52,4 +52,4 @@ find $DEST -name '*.c' -o -name '*.h' -o -name '*.txt' \ -exec unix2dos '{}' \; # Zip it all up -( cd dist; zip -r $VERSION.zip $VERSION ) +( cd dist; zip -r $VERSION-windows.zip $VERSION ) -- cgit 1.2.3-korg From 26f7341094bd023ff0e7b569eec8b207b9b8ab97 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 29 Dec 2013 20:25:36 +0200 Subject: Include platform name in also the directory name in binary pkgs --- tools/make_linux_package.sh | 4 ++-- tools/make_mac_package.sh | 4 ++-- tools/make_windows_package.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/make_linux_package.sh b/tools/make_linux_package.sh index 342029aa..1cbf478d 100755 --- a/tools/make_linux_package.sh +++ b/tools/make_linux_package.sh @@ -6,7 +6,7 @@ set -e set -x -VERSION=`git describe --always` +VERSION=`git describe --always`-linux-x86 DEST=dist/$VERSION rm -rf $DEST @@ -42,5 +42,5 @@ EOF chmod +x $DEST/generator-bin/protoc # Tar it all up -( cd dist; tar -czf $VERSION-linux.tar.gz $VERSION ) +( cd dist; tar -czf $VERSION.tar.gz $VERSION ) diff --git a/tools/make_mac_package.sh b/tools/make_mac_package.sh index 45ce71a0..c5a956d2 100755 --- a/tools/make_mac_package.sh +++ b/tools/make_mac_package.sh @@ -8,7 +8,7 @@ set -e set -x -VERSION=`git describe --always` +VERSION=`git describe --always`-macosx-x86 DEST=dist/$VERSION rm -rf $DEST @@ -45,5 +45,5 @@ EOF chmod +x $DEST/generator-bin/protoc # Tar it all up -( cd dist; tar -czf $VERSION-mac.tgz $VERSION ) +( cd dist; tar -czf $VERSION.tgz $VERSION ) diff --git a/tools/make_windows_package.sh b/tools/make_windows_package.sh index 1a339a2b..72de6f33 100755 --- a/tools/make_windows_package.sh +++ b/tools/make_windows_package.sh @@ -7,7 +7,7 @@ set -e set -x -VERSION=`git describe --always` +VERSION=`git describe --always`-windows-x86 DEST=dist/$VERSION rm -rf $DEST @@ -52,4 +52,4 @@ find $DEST -name '*.c' -o -name '*.h' -o -name '*.txt' \ -exec unix2dos '{}' \; # Zip it all up -( cd dist; zip -r $VERSION-windows.zip $VERSION ) +( cd dist; zip -r $VERSION.zip $VERSION ) -- cgit 1.2.3-korg From 156139f11293691d97b06db826b7b8ff8327317b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 29 Dec 2013 20:31:02 +0200 Subject: README updates --- README.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.txt b/README.txt index c6085d28..0f2ade8b 100644 --- a/README.txt +++ b/README.txt @@ -28,9 +28,9 @@ compiler. This has the advantage that there is no need to reimplement the basic parsing of .proto files. However, it does mean that you need the Google's protobuf library in order to run the generator. -If you have downloaded a binary package for nanopb (either Windows or Linux -version), the 'protoc' binary is included in the 'generator-bin' folder. In -this case, you are ready to go. Simply run this command: +If you have downloaded a binary package for nanopb (either Windows, Linux or +Mac OS X version), the 'protoc' binary is included in the 'generator-bin' +folder. In this case, you are ready to go. Simply run this command: generator-bin/protoc --nanopb_out=. myprotocol.proto -- cgit 1.2.3-korg From ae7b9a3bd3f54b5d0ba2db428944e38a3cd7b6a2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 29 Dec 2013 20:41:36 +0200 Subject: Fix path change: compat/ -> extra/ --- docs/index.rst | 7 ++++--- tests/SConstruct | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index 27840f7d..7bd1794e 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -104,11 +104,12 @@ requires a few header files to be available: #) *stdbool.h*, for definition of *bool* If these header files do not come with your compiler, you can use the -file *compat/pb_syshdr.h* instead. It contains an example of how to provide +file *extra/pb_syshdr.h* instead. It contains an example of how to provide the dependencies. You may have to edit it a bit to suit your custom platform. -To use the pb_syshdr.h, define *PB_SYSTEM_HEADER* to be the name of your custom -header file. It should provide all the dependencies listed above. +To use the pb_syshdr.h, define *PB_SYSTEM_HEADER* as *"pb_syshdr.h"* (including the quotes). +Similarly, you can provide a custom include file, which should provide all the dependencies +listed above. Running the test cases ====================== diff --git a/tests/SConstruct b/tests/SConstruct index c1defe92..566b4793 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -52,7 +52,7 @@ if not env.GetOption('clean'): string = conf.CheckCHeader('string.h') if not stdbool or not stdint or not stddef or not string: conf.env.Append(CPPDEFINES = {'PB_SYSTEM_HEADER': '\\"pb_syshdr.h\\"'}) - conf.env.Append(CPPPATH = "#../compat") + conf.env.Append(CPPPATH = "#../extra") if stdbool: conf.env.Append(CPPDEFINES = {'HAVE_STDBOOL_H': 1}) if stdint: conf.env.Append(CPPDEFINES = {'HAVE_STDINT_H': 1}) -- cgit 1.2.3-korg From 7c9a8a132bc98741b97a186b1436f84e853f6fa3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 29 Dec 2013 20:48:30 +0200 Subject: Fix protoc-gen-nanopb when the path contains spaces --- generator/protoc-gen-nanopb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/generator/protoc-gen-nanopb b/generator/protoc-gen-nanopb index 90be6a58..2de56216 100755 --- a/generator/protoc-gen-nanopb +++ b/generator/protoc-gen-nanopb @@ -9,4 +9,5 @@ # path is already set up properly and there is no need to give # --plugin= on the command line. -exec python $(dirname $0)/nanopb_generator.py --protoc-plugin +MYPATH=$(dirname "$0") +exec python "$MYPATH/nanopb_generator.py" --protoc-plugin -- cgit 1.2.3-korg From 10ef575d467db1f12406eee944212fde6c250089 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 29 Dec 2013 20:53:39 +0200 Subject: Fix build on Windows --- tests/alltypes_pointer/SConscript | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/alltypes_pointer/SConscript b/tests/alltypes_pointer/SConscript index 11a287a8..b577ca24 100644 --- a/tests/alltypes_pointer/SConscript +++ b/tests/alltypes_pointer/SConscript @@ -9,13 +9,15 @@ env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) env.NanopbProto(["alltypes", "alltypes.options"]) enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) +refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX" + # Encode and compare results env.RunTest(enc) -env.RunTest("decode_alltypes.output", ["$BUILD/alltypes/decode_alltypes", "encode_alltypes_pointer.output"]) +env.RunTest("decode_alltypes.output", [refdec, "encode_alltypes_pointer.output"]) env.Compare(["encode_alltypes_pointer.output", "$BUILD/alltypes/encode_alltypes.output"]) # Do the same thing with the optional fields present env.RunTest("optionals.output", enc, ARGS = ['1']) -env.RunTest("optionals.decout", ["$BUILD/alltypes/decode_alltypes", "optionals.output"], ARGS = ['1']) +env.RunTest("optionals.decout", [refdec, "optionals.output"], ARGS = ['1']) env.Compare(["optionals.output", "$BUILD/alltypes/optionals.output"]) -- cgit 1.2.3-korg From da8210b947821b2002ce7dfb34eba213a2da73a3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 29 Dec 2013 21:11:57 +0200 Subject: Fix handling spaces in directory name --- tests/site_scons/site_tools/nanopb.py | 4 ++-- tools/make_linux_package.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/site_scons/site_tools/nanopb.py b/tests/site_scons/site_tools/nanopb.py index 0ed70464..97837666 100644 --- a/tests/site_scons/site_tools/nanopb.py +++ b/tests/site_scons/site_tools/nanopb.py @@ -61,12 +61,12 @@ def _detect_protoc(env): p1 = os.path.join(p, 'generator-bin', 'protoc') if os.path.exists(p1): # Use protoc bundled with binary package - return p1 + return env['ESCAPE'](p1) p = env.WhereIs('protoc') if p: # Use protoc from path - return p + return env['ESCAPE'](p) raise SCons.Errors.StopError(NanopbWarning, "Could not find the protoc compiler") diff --git a/tools/make_linux_package.sh b/tools/make_linux_package.sh index 1cbf478d..332c281b 100755 --- a/tools/make_linux_package.sh +++ b/tools/make_linux_package.sh @@ -34,10 +34,10 @@ LIBPROTOC=$(ldd `which protoc` | grep -o '/.*libprotoc[^ ]*') cp $LIBPROTOC $DEST/generator-bin/ cat > $DEST/generator-bin/protoc << EOF #!/bin/bash -SCRIPTDIR=\$(dirname \$(readlink -f \$0)) +SCRIPTDIR=\$(dirname "\$0") export LD_LIBRARY_PATH=\$SCRIPTDIR export PATH=\$SCRIPTDIR:\$PATH -exec \$SCRIPTDIR/protoc.bin "\$@" +exec "\$SCRIPTDIR/protoc.bin" "\$@" EOF chmod +x $DEST/generator-bin/protoc -- cgit 1.2.3-korg From 125b49bf24883585d6826b8d72e14ccdfae41206 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 29 Dec 2013 21:36:06 +0200 Subject: Fix spaces in paths on Windows also --- generator/protoc-gen-nanopb.bat | 2 +- tests/site_scons/site_tools/nanopb.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/protoc-gen-nanopb.bat b/generator/protoc-gen-nanopb.bat index 3eed057b..7624984e 100644 --- a/generator/protoc-gen-nanopb.bat +++ b/generator/protoc-gen-nanopb.bat @@ -9,4 +9,4 @@ :: --plugin= on the command line. set mydir=%~dp0 -python %mydir%\nanopb_generator.py --protoc-plugin +python "%mydir%\nanopb_generator.py" --protoc-plugin diff --git a/tests/site_scons/site_tools/nanopb.py b/tests/site_scons/site_tools/nanopb.py index 97837666..9197e190 100644 --- a/tests/site_scons/site_tools/nanopb.py +++ b/tests/site_scons/site_tools/nanopb.py @@ -58,7 +58,7 @@ def _detect_protoc(env): return env['PROTOC'] p = _detect_nanopb(env) - p1 = os.path.join(p, 'generator-bin', 'protoc') + p1 = os.path.join(p, 'generator-bin', 'protoc' + env['PROGSUFFIX']) if os.path.exists(p1): # Use protoc bundled with binary package return env['ESCAPE'](p1) -- cgit 1.2.3-korg From bb51ee655bdb80b176d4d7a87dce88d62b0e827a Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 29 Dec 2013 21:43:22 +0200 Subject: Make the Mac OS X package a .zip instead --- tools/make_mac_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/make_mac_package.sh b/tools/make_mac_package.sh index c5a956d2..9e78910e 100755 --- a/tools/make_mac_package.sh +++ b/tools/make_mac_package.sh @@ -45,5 +45,5 @@ EOF chmod +x $DEST/generator-bin/protoc # Tar it all up -( cd dist; tar -czf $VERSION.tgz $VERSION ) +( cd dist; zip -r $VERSION.zip $VERSION ) -- cgit 1.2.3-korg From 585cd0356ee1ef3c48cf958c307de7dc651123a3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 29 Dec 2013 21:47:38 +0200 Subject: And same fixes on Mac also.. --- tools/make_mac_package.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/make_mac_package.sh b/tools/make_mac_package.sh index 9e78910e..21cd170d 100755 --- a/tools/make_mac_package.sh +++ b/tools/make_mac_package.sh @@ -37,10 +37,10 @@ LIBPROTOBUF=$(otool -L `which protoc` | grep -o '/.*libprotobuf[^ ]*') cp $LIBPROTOC $LIBPROTOBUF $DEST/generator-bin/ cat > $DEST/generator-bin/protoc << EOF #!/bin/bash -SCRIPTDIR=\$(cd \$(dirname \$0); pwd) +SCRIPTDIR=\$(dirname "\$0") export DYLD_LIBRARY_PATH=\$SCRIPTDIR export PATH=\$SCRIPTDIR:\$PATH -exec \$SCRIPTDIR/protoc.bin "\$@" +exec "\$SCRIPTDIR/protoc.bin" "\$@" EOF chmod +x $DEST/generator-bin/protoc -- cgit 1.2.3-korg From 18e71bbfe2907fa759fad09e8c5c13655e33a590 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 29 Dec 2013 22:08:13 +0200 Subject: Fix bundled protoc detection in scons script --- tests/site_scons/site_tools/nanopb.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/site_scons/site_tools/nanopb.py b/tests/site_scons/site_tools/nanopb.py index 9197e190..b3e58fa1 100644 --- a/tests/site_scons/site_tools/nanopb.py +++ b/tests/site_scons/site_tools/nanopb.py @@ -57,8 +57,8 @@ def _detect_protoc(env): # Use protoc defined by user return env['PROTOC'] - p = _detect_nanopb(env) - p1 = os.path.join(p, 'generator-bin', 'protoc' + env['PROGSUFFIX']) + n = _detect_nanopb(env) + p1 = os.path.join(n, 'generator-bin', 'protoc' + env['PROGSUFFIX']) if os.path.exists(p1): # Use protoc bundled with binary package return env['ESCAPE'](p1) @@ -78,7 +78,8 @@ def _detect_protocflags(env): p = _detect_protoc(env) n = _detect_nanopb(env) - if p == os.path.join(n, 'generator-bin', 'protoc'): + p1 = os.path.join(n, 'generator-bin', 'protoc' + env['PROGSUFFIX']) + if p == env['ESCAPE'](p1): # Using the bundled protoc, no options needed return '' -- cgit 1.2.3-korg From 4546bb92dc332ebbfb05a7517028f967b0657725 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 1 Jan 2014 11:20:07 +0200 Subject: Fix Makefiles for the examples The default rules for make were being selected before our own rules and the .pb.c was not being built. --- examples/network_server/Makefile | 2 ++ examples/using_double_on_avr/Makefile | 2 ++ examples/using_union_messages/Makefile | 2 ++ 3 files changed, 6 insertions(+) diff --git a/examples/network_server/Makefile b/examples/network_server/Makefile index 3c014009..2c7639a1 100644 --- a/examples/network_server/Makefile +++ b/examples/network_server/Makefile @@ -7,6 +7,8 @@ CFLAGS += -I$(NANOPB_DIR) all: server client +.SUFFIXES: + clean: rm -f server client fileproto.pb.c fileproto.pb.h diff --git a/examples/using_double_on_avr/Makefile b/examples/using_double_on_avr/Makefile index 6f2203ae..874a64bd 100644 --- a/examples/using_double_on_avr/Makefile +++ b/examples/using_double_on_avr/Makefile @@ -7,6 +7,8 @@ CFLAGS += -I$(NANOPB_DIR) all: run_tests +.SUFFIXES: + clean: rm -f test_conversions encode_double decode_double doubleproto.pb.c doubleproto.pb.h diff --git a/examples/using_union_messages/Makefile b/examples/using_union_messages/Makefile index 618bca50..66396a02 100644 --- a/examples/using_union_messages/Makefile +++ b/examples/using_union_messages/Makefile @@ -10,6 +10,8 @@ all: encode decode ./encode 2 | ./decode ./encode 3 | ./decode +.SUFFIXES: + clean: rm -f encode unionproto.pb.h unionproto.pb.c -- cgit 1.2.3-korg From 906c8283b5995eb7b27f4958a6a6502ae0deea07 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 1 Jan 2014 11:11:19 +0200 Subject: Publishing nanopb-0.2.5 --- CHANGELOG.txt | 7 +++++++ generator/nanopb_generator.py | 2 +- pb.h | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 94f0c797..f0c97c65 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,10 @@ +nanopb-0.2.5 (2014-01-01) + Fix a bug with encoding negative values in int32 fields (issue 97) + Create binary packages of the generator + dependencies (issue 47) + Add support for pointer-type fields to the encoder (part of issue 80) + Fixed path in FindNanopb.cmake (issue 94) + Improved tests + nanopb-0.2.4 (2013-11-07) Remove the deprecated NANOPB_INTERNALS functions from public API. Document the security model. diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 6ccb96f9..475e78ae 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.5-dev" +nanopb_version = "nanopb-0.2.5" import sys diff --git a/pb.h b/pb.h index 76980e4c..62e20715 100644 --- a/pb.h +++ b/pb.h @@ -43,7 +43,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.5-dev +#define NANOPB_VERSION nanopb-0.2.5 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 843fc4b11a5dde584143d4a18deb41556b80daa7 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 1 Jan 2014 11:44:47 +0200 Subject: Setting version to 0.2.6-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 475e78ae..dea9098b 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.5" +nanopb_version = "nanopb-0.2.6-dev" import sys diff --git a/pb.h b/pb.h index 62e20715..154e4047 100644 --- a/pb.h +++ b/pb.h @@ -43,7 +43,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.5 +#define NANOPB_VERSION nanopb-0.2.6-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From a46ed9f47523712861741c4bb1c42382933467be Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 3 Jan 2014 11:17:39 +0200 Subject: Fix problem with callback-type fields (issue #99). --- generator/nanopb_generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index dea9098b..094f8e85 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -169,6 +169,7 @@ class Field: self.max_count = None self.array_decl = "" self.enc_size = None + self.ctype = None # Parse field options if field_options.HasField("max_size"): -- cgit 1.2.3-korg From 5efeb392e00a57accb849904aba8dbc092c4076a Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 4 Feb 2014 20:11:52 +0200 Subject: Add strict-aliasing GCC warnings for core --- tests/SConstruct | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/SConstruct b/tests/SConstruct index 566b4793..abc6e7cb 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -74,6 +74,7 @@ if not env.GetOption('clean'): # Check if we can use extra strict warning flags (only with GCC) extra = '-Wcast-qual -Wlogical-op -Wconversion' + extra += ' -fstrict-aliasing -Wstrict-aliasing=1' extra += ' -Wmissing-prototypes -Wmissing-declarations -Wredundant-decls' extra += ' -Wstack-protector ' if 'gcc' in env['CC']: -- cgit 1.2.3-korg From a2f8112166d73aaf7e8fc877f0310202319d4639 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 4 Feb 2014 20:24:44 +0200 Subject: Add proper suffixes for integer default values. Update issue 102 Status: FixedInGit --- generator/nanopb_generator.py | 6 ++++++ tests/alltypes/alltypes.proto | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 094f8e85..4f8ed94c 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -315,6 +315,12 @@ class Field: data = self.default.decode('string_escape') data = ['0x%02x' % ord(c) for c in data] default = '{%d, {%s}}' % (len(data), ','.join(data)) + elif self.pbtype in ['FIXED32', 'UINT32']: + default += 'u' + elif self.pbtype in ['FIXED64', 'UINT64']: + default += 'ull' + elif self.pbtype in ['SFIXED64', 'INT64']: + default += 'll' if declaration_only: return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl) diff --git a/tests/alltypes/alltypes.proto b/tests/alltypes/alltypes.proto index 234b7236..6e5425a3 100644 --- a/tests/alltypes/alltypes.proto +++ b/tests/alltypes/alltypes.proto @@ -14,16 +14,16 @@ enum HugeEnum { } message Limits { - required int32 int32_min = 1; - required int32 int32_max = 2; - required uint32 uint32_min = 3; - required uint32 uint32_max = 4; - required int64 int64_min = 5; - required int64 int64_max = 6; - required uint64 uint64_min = 7; - required uint64 uint64_max = 8; - required HugeEnum enum_min = 9; - required HugeEnum enum_max = 10; + required int32 int32_min = 1 [default = 2147483647]; + required int32 int32_max = 2 [default = -2147483648]; + required uint32 uint32_min = 3 [default = 4294967295]; + required uint32 uint32_max = 4 [default = 0]; + required int64 int64_min = 5 [default = 9223372036854775807]; + required int64 int64_max = 6 [default = -9223372036854775807]; /* GCC gives a warning about -2**63, so just avoid the trouble for now.. */ + required uint64 uint64_min = 7 [default = 18446744073709551615]; + required uint64 uint64_max = 8 [default = 0]; + required HugeEnum enum_min = 9 [default = Positive]; + required HugeEnum enum_max = 10 [default = Negative]; } enum MyEnum { -- cgit 1.2.3-korg From 879860be196712990b1fb41cecd5a25229151e56 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 4 Feb 2014 20:34:57 +0200 Subject: Add coments to places where STATIC_ASSERT is used. Update issue 96 Status: FixedInGit --- generator/nanopb_generator.py | 34 ++++++++++++++++++++++++---------- pb.h | 16 +++++++++++++--- 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 4f8ed94c..e6efb247 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -822,6 +822,23 @@ def generate_source(headername, enums, messages, extensions, options): if worst > 255 or checks: yield '\n/* Check that field information fits in pb_field_t */\n' + if worst > 65535 or checks: + yield '#if !defined(PB_FIELD_32BIT)\n' + if worst > 65535: + yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field + else: + assertion = ' && '.join(str(c) + ' < 65536' for c in checks) + msgs = '_'.join(str(n) for n in checks_msgnames) + yield '/* If you get an error here, it means that you need to define PB_FIELD_32BIT\n' + yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n' + yield ' * \n' + yield ' * The reason you need to do this is that some of your messages contain tag\n' + yield ' * numbers or field sizes that are larger than what can fit in 8 or 16 bit\n' + yield ' * field descriptors.\n' + yield ' */\n' + yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) + yield '#endif\n\n' + if worst < 65536: yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n' if worst > 255: @@ -829,18 +846,15 @@ def generate_source(headername, enums, messages, extensions, options): else: assertion = ' && '.join(str(c) + ' < 256' for c in checks) msgs = '_'.join(str(n) for n in checks_msgnames) + yield '/* If you get an error here, it means that you need to define PB_FIELD_16BIT\n' + yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n' + yield ' * \n' + yield ' * The reason you need to do this is that some of your messages contain tag\n' + yield ' * numbers or field sizes that are larger than what can fit in the default\n' + yield ' * 8 bit descriptors.\n' + yield ' */\n' yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) yield '#endif\n\n' - - if worst > 65535 or checks: - yield '#if !defined(PB_FIELD_32BIT)\n' - if worst > 65535: - yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field - else: - assertion = ' && '.join(str(c) + ' < 65536' for c in checks) - msgs = '_'.join(str(n) for n in checks_msgnames) - yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) - yield '#endif\n' # Add check for sizeof(double) has_double = False diff --git a/pb.h b/pb.h index 154e4047..3faa67b2 100644 --- a/pb.h +++ b/pb.h @@ -96,8 +96,14 @@ #endif /* Compile-time assertion, used for checking compatible compilation options. - * If this fails on your compiler for some reason, use #define STATIC_ASSERT - * to disable it. */ + * If this does not work properly on your compiler, use #define STATIC_ASSERT + * to disable it. + * + * But before doing that, check carefully the error message / place where it + * comes from to see if the error has a real cause. Unfortunately the error + * message is not always very clear to read, but you can see the reason better + * in the place where the STATIC_ASSERT macro was called. + */ #ifndef STATIC_ASSERT #define STATIC_ASSERT(COND,MSG) typedef char STATIC_ASSERT_MSG(MSG, __LINE__, __COUNTER__)[(COND)?1:-1]; #define STATIC_ASSERT_MSG(MSG, LINE, COUNTER) STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) @@ -210,7 +216,11 @@ struct _pb_field_t { PB_PACKED_STRUCT_END /* Make sure that the standard integer types are of the expected sizes. - * All kinds of things may break otherwise.. atleast all fixed* types. */ + * All kinds of things may break otherwise.. atleast all fixed* types. + * + * If you get errors here, it probably means that your stdint.h is not + * correct for your platform. + */ STATIC_ASSERT(sizeof(int8_t) == 1, INT8_T_WRONG_SIZE) STATIC_ASSERT(sizeof(uint8_t) == 1, UINT8_T_WRONG_SIZE) STATIC_ASSERT(sizeof(int16_t) == 2, INT16_T_WRONG_SIZE) -- cgit 1.2.3-korg From d3ed0744d037e38ed8737a670fc05aec09f76379 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 5 Feb 2014 17:56:37 +0200 Subject: Extreme negative integer constants are a bit nasty to define in C, just avoid the trouble in tests for now. --- tests/alltypes/alltypes.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/alltypes/alltypes.proto b/tests/alltypes/alltypes.proto index 6e5425a3..db83c9a6 100644 --- a/tests/alltypes/alltypes.proto +++ b/tests/alltypes/alltypes.proto @@ -15,11 +15,11 @@ enum HugeEnum { message Limits { required int32 int32_min = 1 [default = 2147483647]; - required int32 int32_max = 2 [default = -2147483648]; + required int32 int32_max = 2 [default = -2147483647]; required uint32 uint32_min = 3 [default = 4294967295]; required uint32 uint32_max = 4 [default = 0]; required int64 int64_min = 5 [default = 9223372036854775807]; - required int64 int64_max = 6 [default = -9223372036854775807]; /* GCC gives a warning about -2**63, so just avoid the trouble for now.. */ + required int64 int64_max = 6 [default = -9223372036854775807]; required uint64 uint64_min = 7 [default = 18446744073709551615]; required uint64 uint64_max = 8 [default = 0]; required HugeEnum enum_min = 9 [default = Positive]; -- cgit 1.2.3-korg From 235219a29588fd19787baa7ff948ab52ae2499a4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 9 Feb 2014 19:28:50 +0200 Subject: Add AllTypes encoding test for callbacks (issue #99) Still need to do the decoding side. --- tests/alltypes_callback/SConscript | 20 ++ tests/alltypes_callback/alltypes.options | 3 + tests/alltypes_callback/encode_alltypes_callback.c | 397 +++++++++++++++++++++ 3 files changed, 420 insertions(+) create mode 100644 tests/alltypes_callback/SConscript create mode 100644 tests/alltypes_callback/alltypes.options create mode 100644 tests/alltypes_callback/encode_alltypes_callback.c diff --git a/tests/alltypes_callback/SConscript b/tests/alltypes_callback/SConscript new file mode 100644 index 00000000..e8d7704f --- /dev/null +++ b/tests/alltypes_callback/SConscript @@ -0,0 +1,20 @@ +# Test the AllTypes encoding & decoding using callbacks for all fields. + +Import("env") + +c = Copy("$TARGET", "$SOURCE") +env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) + +env.NanopbProto(["alltypes", "alltypes.options"]) +enc = env.Program(["encode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) + +refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX" + +# Encode and compare results +env.RunTest(enc) +env.RunTest("decode_alltypes.output", [refdec, "encode_alltypes_callback.output"]) + +# Do the same thing with the optional fields present +env.RunTest("optionals.output", enc, ARGS = ['1']) +env.RunTest("optionals.decout", [refdec, "optionals.output"], ARGS = ['1']) + diff --git a/tests/alltypes_callback/alltypes.options b/tests/alltypes_callback/alltypes.options new file mode 100644 index 00000000..a9c55ece --- /dev/null +++ b/tests/alltypes_callback/alltypes.options @@ -0,0 +1,3 @@ +# Generate all fields as callbacks. +AllTypes.* type:FT_CALLBACK +SubMessage.substuff1 max_size:16 diff --git a/tests/alltypes_callback/encode_alltypes_callback.c b/tests/alltypes_callback/encode_alltypes_callback.c new file mode 100644 index 00000000..10560b19 --- /dev/null +++ b/tests/alltypes_callback/encode_alltypes_callback.c @@ -0,0 +1,397 @@ +/* Attempts to test all the datatypes supported by ProtoBuf when used as callback fields. + * Note that normally there would be no reason to use callback fields for this, + * because each encoder defined here only gives a single field. + */ + +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "test_helpers.h" + +static bool write_varint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, (long)*arg); +} + +static bool write_svarint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, (long)*arg); +} + +static bool write_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_fixed32(stream, *arg); +} + +static bool write_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_fixed64(stream, *arg); +} + +static bool write_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, *arg, strlen(*arg)); +} + +static bool write_submsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, *arg); +} + +static bool write_emptymsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + EmptyMessage emptymsg = {0}; + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg); +} + +static bool write_repeated_varint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, (long)*arg); +} + +static bool write_repeated_svarint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, (long)*arg); +} + +static bool write_repeated_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + uint32_t dummy = 0; + + /* Make it a packed field */ + return pb_encode_tag(stream, PB_WT_STRING, field->tag) && + pb_encode_varint(stream, 5 * 4) && /* Number of bytes */ + pb_encode_fixed32(stream, &dummy) && + pb_encode_fixed32(stream, &dummy) && + pb_encode_fixed32(stream, &dummy) && + pb_encode_fixed32(stream, &dummy) && + pb_encode_fixed32(stream, *arg); +} + +static bool write_repeated_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + uint64_t dummy = 0; + + /* Make it a packed field */ + return pb_encode_tag(stream, PB_WT_STRING, field->tag) && + pb_encode_varint(stream, 5 * 8) && /* Number of bytes */ + pb_encode_fixed64(stream, &dummy) && + pb_encode_fixed64(stream, &dummy) && + pb_encode_fixed64(stream, &dummy) && + pb_encode_fixed64(stream, &dummy) && + pb_encode_fixed64(stream, *arg); +} + +static bool write_repeated_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, 0, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, 0, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, 0, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, 0, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, *arg, strlen(*arg)); +} + +static bool write_repeated_submsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + SubMessage dummy = {""}; + + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, &dummy) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, &dummy) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, &dummy) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, &dummy) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, *arg); +} + +static bool write_limits(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + Limits limits = {0}; + limits.int32_min = INT32_MIN; + limits.int32_max = INT32_MAX; + limits.uint32_min = 0; + limits.uint32_max = UINT32_MAX; + limits.int64_min = INT64_MIN; + limits.int64_max = INT64_MAX; + limits.uint64_min = 0; + limits.uint64_max = UINT64_MAX; + limits.enum_min = HugeEnum_Negative; + limits.enum_max = HugeEnum_Positive; + + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, Limits_fields, &limits); +} + +static bool write_repeated_emptymsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + EmptyMessage emptymsg = {0}; + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg); +} + +int main(int argc, char **argv) +{ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Values for use from callbacks through pointers. */ + uint32_t req_fixed32 = 1008; + int32_t req_sfixed32 = -1009; + float req_float = 1010.0f; + uint64_t req_fixed64 = 1011; + int64_t req_sfixed64 = -1012; + double req_double = 1013.0; + SubMessage req_submsg = {"1016", 1016}; + + uint32_t rep_fixed32 = 2008; + int32_t rep_sfixed32 = -2009; + float rep_float = 2010.0f; + uint64_t rep_fixed64 = 2011; + int64_t rep_sfixed64 = -2012; + double rep_double = 2013.0; + SubMessage rep_submsg = {"2016", 2016, true, 2016}; + + uint32_t opt_fixed32 = 3048; + int32_t opt_sfixed32 = 3049; + float opt_float = 3050.0f; + uint64_t opt_fixed64 = 3051; + int64_t opt_sfixed64 = 3052; + double opt_double = 3053.0f; + SubMessage opt_submsg = {"3056", 3056}; + + /* Bind callbacks for required fields */ + AllTypes alltypes = {{{0}}}; + + alltypes.req_int32.funcs.encode = &write_varint; + alltypes.req_int32.arg = (void*)-1001; + + alltypes.req_int64.funcs.encode = &write_varint; + alltypes.req_int64.arg = (void*)-1002; + + alltypes.req_uint32.funcs.encode = &write_varint; + alltypes.req_uint32.arg = (void*)1003; + + alltypes.req_uint32.funcs.encode = &write_varint; + alltypes.req_uint32.arg = (void*)1003; + + alltypes.req_uint64.funcs.encode = &write_varint; + alltypes.req_uint64.arg = (void*)1004; + + alltypes.req_sint32.funcs.encode = &write_svarint; + alltypes.req_sint32.arg = (void*)-1005; + + alltypes.req_sint64.funcs.encode = &write_svarint; + alltypes.req_sint64.arg = (void*)-1006; + + alltypes.req_bool.funcs.encode = &write_varint; + alltypes.req_bool.arg = (void*)true; + + alltypes.req_fixed32.funcs.encode = &write_fixed32; + alltypes.req_fixed32.arg = &req_fixed32; + + alltypes.req_sfixed32.funcs.encode = &write_fixed32; + alltypes.req_sfixed32.arg = &req_sfixed32; + + alltypes.req_float.funcs.encode = &write_fixed32; + alltypes.req_float.arg = &req_float; + + alltypes.req_fixed64.funcs.encode = &write_fixed64; + alltypes.req_fixed64.arg = &req_fixed64; + + alltypes.req_sfixed64.funcs.encode = &write_fixed64; + alltypes.req_sfixed64.arg = &req_sfixed64; + + alltypes.req_double.funcs.encode = &write_fixed64; + alltypes.req_double.arg = &req_double; + + alltypes.req_string.funcs.encode = &write_string; + alltypes.req_string.arg = "1014"; + + alltypes.req_bytes.funcs.encode = &write_string; + alltypes.req_bytes.arg = "1015"; + + alltypes.req_submsg.funcs.encode = &write_submsg; + alltypes.req_submsg.arg = &req_submsg; + + alltypes.req_enum.funcs.encode = &write_varint; + alltypes.req_enum.arg = (void*)MyEnum_Truth; + + alltypes.req_emptymsg.funcs.encode = &write_emptymsg; + + /* Bind callbacks for repeated fields */ + alltypes.rep_int32.funcs.encode = &write_repeated_varint; + alltypes.rep_int32.arg = (void*)-2001; + + alltypes.rep_int64.funcs.encode = &write_repeated_varint; + alltypes.rep_int64.arg = (void*)-2002; + + alltypes.rep_uint32.funcs.encode = &write_repeated_varint; + alltypes.rep_uint32.arg = (void*)2003; + + alltypes.rep_uint64.funcs.encode = &write_repeated_varint; + alltypes.rep_uint64.arg = (void*)2004; + + alltypes.rep_sint32.funcs.encode = &write_repeated_svarint; + alltypes.rep_sint32.arg = (void*)-2005; + + alltypes.rep_sint64.funcs.encode = &write_repeated_svarint; + alltypes.rep_sint64.arg = (void*)-2006; + + alltypes.rep_bool.funcs.encode = &write_repeated_varint; + alltypes.rep_bool.arg = (void*)true; + + alltypes.rep_fixed32.funcs.encode = &write_repeated_fixed32; + alltypes.rep_fixed32.arg = &rep_fixed32; + + alltypes.rep_sfixed32.funcs.encode = &write_repeated_fixed32; + alltypes.rep_sfixed32.arg = &rep_sfixed32; + + alltypes.rep_float.funcs.encode = &write_repeated_fixed32; + alltypes.rep_float.arg = &rep_float; + + alltypes.rep_fixed64.funcs.encode = &write_repeated_fixed64; + alltypes.rep_fixed64.arg = &rep_fixed64; + + alltypes.rep_sfixed64.funcs.encode = &write_repeated_fixed64; + alltypes.rep_sfixed64.arg = &rep_sfixed64; + + alltypes.rep_double.funcs.encode = &write_repeated_fixed64; + alltypes.rep_double.arg = &rep_double; + + alltypes.rep_string.funcs.encode = &write_repeated_string; + alltypes.rep_string.arg = "2014"; + + alltypes.rep_bytes.funcs.encode = &write_repeated_string; + alltypes.rep_bytes.arg = "2015"; + + alltypes.rep_submsg.funcs.encode = &write_repeated_submsg; + alltypes.rep_submsg.arg = &rep_submsg; + + alltypes.rep_enum.funcs.encode = &write_repeated_varint; + alltypes.rep_enum.arg = (void*)MyEnum_Truth; + + alltypes.rep_emptymsg.funcs.encode = &write_repeated_emptymsg; + + alltypes.req_limits.funcs.encode = &write_limits; + + /* Bind callbacks for optional fields */ + if (mode != 0) + { + alltypes.opt_int32.funcs.encode = &write_varint; + alltypes.opt_int32.arg = (void*)3041; + + alltypes.opt_int64.funcs.encode = &write_varint; + alltypes.opt_int64.arg = (void*)3042; + + alltypes.opt_uint32.funcs.encode = &write_varint; + alltypes.opt_uint32.arg = (void*)3043; + + alltypes.opt_uint64.funcs.encode = &write_varint; + alltypes.opt_uint64.arg = (void*)3044; + + alltypes.opt_sint32.funcs.encode = &write_svarint; + alltypes.opt_sint32.arg = (void*)3045; + + alltypes.opt_sint64.funcs.encode = &write_svarint; + alltypes.opt_sint64.arg = (void*)3046; + + alltypes.opt_bool.funcs.encode = &write_varint; + alltypes.opt_bool.arg = (void*)true; + + alltypes.opt_fixed32.funcs.encode = &write_fixed32; + alltypes.opt_fixed32.arg = &opt_fixed32; + + alltypes.opt_sfixed32.funcs.encode = &write_fixed32; + alltypes.opt_sfixed32.arg = &opt_sfixed32; + + alltypes.opt_float.funcs.encode = &write_fixed32; + alltypes.opt_float.arg = &opt_float; + + alltypes.opt_fixed64.funcs.encode = &write_fixed64; + alltypes.opt_fixed64.arg = &opt_fixed64; + + alltypes.opt_sfixed64.funcs.encode = &write_fixed64; + alltypes.opt_sfixed64.arg = &opt_sfixed64; + + alltypes.opt_double.funcs.encode = &write_fixed64; + alltypes.opt_double.arg = &opt_double; + + alltypes.opt_string.funcs.encode = &write_string; + alltypes.opt_string.arg = "3054"; + + alltypes.opt_bytes.funcs.encode = &write_string; + alltypes.opt_bytes.arg = "3055"; + + alltypes.opt_submsg.funcs.encode = &write_submsg; + alltypes.opt_submsg.arg = &opt_submsg; + + alltypes.opt_enum.funcs.encode = &write_varint; + alltypes.opt_enum.arg = (void*)MyEnum_Truth; + + alltypes.opt_emptymsg.funcs.encode = &write_emptymsg; + } + + alltypes.end.funcs.encode = &write_varint; + alltypes.end.arg = (void*)1099; + + { + uint8_t buffer[2048]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } + } +} -- cgit 1.2.3-korg From 057165966c8e1c47ede0b1e457282798f6a91065 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 10 Feb 2014 21:00:26 +0200 Subject: Add AllTypes decoding test with callbacks. Update issue 99 Status: FixedInGit --- tests/alltypes_callback/SConscript | 5 +- tests/alltypes_callback/decode_alltypes_callback.c | 423 +++++++++++++++++++++ 2 files changed, 427 insertions(+), 1 deletion(-) create mode 100644 tests/alltypes_callback/decode_alltypes_callback.c diff --git a/tests/alltypes_callback/SConscript b/tests/alltypes_callback/SConscript index e8d7704f..71b01600 100644 --- a/tests/alltypes_callback/SConscript +++ b/tests/alltypes_callback/SConscript @@ -7,14 +7,17 @@ env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) env.NanopbProto(["alltypes", "alltypes.options"]) enc = env.Program(["encode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) +dec = env.Program(["decode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_decode.o"]) refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX" # Encode and compare results env.RunTest(enc) env.RunTest("decode_alltypes.output", [refdec, "encode_alltypes_callback.output"]) +env.RunTest("decode_alltypes_callback.output", [dec, "encode_alltypes_callback.output"]) # Do the same thing with the optional fields present env.RunTest("optionals.output", enc, ARGS = ['1']) -env.RunTest("optionals.decout", [refdec, "optionals.output"], ARGS = ['1']) +env.RunTest("optionals.refdecout", [refdec, "optionals.output"], ARGS = ['1']) +env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) diff --git a/tests/alltypes_callback/decode_alltypes_callback.c b/tests/alltypes_callback/decode_alltypes_callback.c new file mode 100644 index 00000000..81274f60 --- /dev/null +++ b/tests/alltypes_callback/decode_alltypes_callback.c @@ -0,0 +1,423 @@ +/* Attempts to test all the datatypes supported by ProtoBuf when used as callback fields. + * Note that normally there would be no reason to use callback fields for this, + * because each encoder defined here only gives a single field. + */ + +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "test_helpers.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed (in field %d).\n", field->tag); \ + return false; \ + } + +static bool read_varint(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint64_t value; + if (!pb_decode_varint(stream, &value)) + return false; + + TEST((int64_t)value == (long)*arg); + return true; +} + +static bool read_svarint(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + int64_t value; + if (!pb_decode_svarint(stream, &value)) + return false; + + TEST(value == (long)*arg); + return true; +} + +static bool read_fixed32(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint32_t value; + if (!pb_decode_fixed32(stream, &value)) + return false; + + TEST(value == *(uint32_t*)*arg); + return true; +} + +static bool read_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint64_t value; + if (!pb_decode_fixed64(stream, &value)) + return false; + + TEST(value == *(uint64_t*)*arg); + return true; +} + +static bool read_string(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint8_t buf[16] = {0}; + size_t len = stream->bytes_left; + + if (len > sizeof(buf) - 1 || !pb_read(stream, buf, len)) + return false; + + TEST(strcmp((char*)buf, *arg) == 0); + return true; +} + +static bool read_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + SubMessage submsg = {""}; + + if (!pb_decode(stream, SubMessage_fields, &submsg)) + return false; + + TEST(memcmp(&submsg, *arg, sizeof(submsg))); + return true; +} + +static bool read_emptymsg(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + EmptyMessage emptymsg = {0}; + return pb_decode(stream, EmptyMessage_fields, &emptymsg); +} + +static bool read_repeated_varint(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + int32_t** expected = (int32_t**)arg; + uint64_t value; + if (!pb_decode_varint(stream, &value)) + return false; + + TEST(*(*expected)++ == value); + return true; +} + +static bool read_repeated_svarint(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + int32_t** expected = (int32_t**)arg; + int64_t value; + if (!pb_decode_svarint(stream, &value)) + return false; + + TEST(*(*expected)++ == value); + return true; +} + +static bool read_repeated_fixed32(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint32_t** expected = (uint32_t**)arg; + uint32_t value; + if (!pb_decode_fixed32(stream, &value)) + return false; + + TEST(*(*expected)++ == value); + return true; +} + +static bool read_repeated_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint64_t** expected = (uint64_t**)arg; + uint64_t value; + if (!pb_decode_fixed64(stream, &value)) + return false; + + TEST(*(*expected)++ == value); + return true; +} + +static bool read_repeated_string(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint8_t*** expected = (uint8_t***)arg; + uint8_t buf[16] = {0}; + size_t len = stream->bytes_left; + + if (len > sizeof(buf) - 1 || !pb_read(stream, buf, len)) + return false; + + TEST(strcmp((char*)*(*expected)++, (char*)buf) == 0); + return true; +} + +static bool read_repeated_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + SubMessage** expected = (SubMessage**)arg; + SubMessage decoded = {""}; + if (!pb_decode(stream, SubMessage_fields, &decoded)) + return false; + + TEST(memcmp((*expected)++, &decoded, sizeof(decoded)) == 0); + return true; +} + +static bool read_limits(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + Limits decoded = {0}; + if (!pb_decode(stream, Limits_fields, &decoded)) + return false; + + TEST(decoded.int32_min == INT32_MIN); + TEST(decoded.int32_max == INT32_MAX); + TEST(decoded.uint32_min == 0); + TEST(decoded.uint32_max == UINT32_MAX); + TEST(decoded.int64_min == INT64_MIN); + TEST(decoded.int64_max == INT64_MAX); + TEST(decoded.uint64_min == 0); + TEST(decoded.uint64_max == UINT64_MAX); + TEST(decoded.enum_min == HugeEnum_Negative); + TEST(decoded.enum_max == HugeEnum_Positive); + + return true; +} + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_alltypes(pb_istream_t *stream, int mode) +{ + /* Values for use from callbacks through pointers. */ + uint32_t req_fixed32 = 1008; + int32_t req_sfixed32 = -1009; + float req_float = 1010.0f; + uint64_t req_fixed64 = 1011; + int64_t req_sfixed64 = -1012; + double req_double = 1013.0; + SubMessage req_submsg = {"1016", 1016}; + + int32_t rep_int32[5] = {0, 0, 0, 0, -2001}; + int32_t rep_int64[5] = {0, 0, 0, 0, -2002}; + int32_t rep_uint32[5] = {0, 0, 0, 0, 2003}; + int32_t rep_uint64[5] = {0, 0, 0, 0, 2004}; + int32_t rep_sint32[5] = {0, 0, 0, 0, -2005}; + int32_t rep_sint64[5] = {0, 0, 0, 0, -2006}; + int32_t rep_bool[5] = {false, false, false, false, true}; + uint32_t rep_fixed32[5] = {0, 0, 0, 0, 2008}; + int32_t rep_sfixed32[5] = {0, 0, 0, 0, -2009}; + float rep_float[5] = {0, 0, 0, 0, 2010.0f}; + uint64_t rep_fixed64[5] = {0, 0, 0, 0, 2011}; + int64_t rep_sfixed64[5] = {0, 0, 0, 0, -2012}; + double rep_double[5] = {0, 0, 0, 0, 2013.0}; + char* rep_string[5] = {"", "", "", "", "2014"}; + char* rep_bytes[5] = {"", "", "", "", "2015"}; + SubMessage rep_submsg[5] = {{"", 0, 0, 3}, + {"", 0, 0, 3}, + {"", 0, 0, 3}, + {"", 0, 0, 3}, + {"2016", 2016, true, 2016}}; + int32_t rep_enum[5] = {0, 0, 0, 0, MyEnum_Truth}; + + uint32_t opt_fixed32 = 3048; + int32_t opt_sfixed32 = 3049; + float opt_float = 3050.0f; + uint64_t opt_fixed64 = 3051; + int64_t opt_sfixed64 = 3052; + double opt_double = 3053.0f; + SubMessage opt_submsg = {"3056", 3056}; + + /* Bind callbacks for required fields */ + AllTypes alltypes; + + /* Fill with garbage to better detect initialization errors */ + memset(&alltypes, 0xAA, sizeof(alltypes)); + + alltypes.req_int32.funcs.decode = &read_varint; + alltypes.req_int32.arg = (void*)-1001; + + alltypes.req_int64.funcs.decode = &read_varint; + alltypes.req_int64.arg = (void*)-1002; + + alltypes.req_uint32.funcs.decode = &read_varint; + alltypes.req_uint32.arg = (void*)1003; + + alltypes.req_uint32.funcs.decode = &read_varint; + alltypes.req_uint32.arg = (void*)1003; + + alltypes.req_uint64.funcs.decode = &read_varint; + alltypes.req_uint64.arg = (void*)1004; + + alltypes.req_sint32.funcs.decode = &read_svarint; + alltypes.req_sint32.arg = (void*)-1005; + + alltypes.req_sint64.funcs.decode = &read_svarint; + alltypes.req_sint64.arg = (void*)-1006; + + alltypes.req_bool.funcs.decode = &read_varint; + alltypes.req_bool.arg = (void*)true; + + alltypes.req_fixed32.funcs.decode = &read_fixed32; + alltypes.req_fixed32.arg = &req_fixed32; + + alltypes.req_sfixed32.funcs.decode = &read_fixed32; + alltypes.req_sfixed32.arg = &req_sfixed32; + + alltypes.req_float.funcs.decode = &read_fixed32; + alltypes.req_float.arg = &req_float; + + alltypes.req_fixed64.funcs.decode = &read_fixed64; + alltypes.req_fixed64.arg = &req_fixed64; + + alltypes.req_sfixed64.funcs.decode = &read_fixed64; + alltypes.req_sfixed64.arg = &req_sfixed64; + + alltypes.req_double.funcs.decode = &read_fixed64; + alltypes.req_double.arg = &req_double; + + alltypes.req_string.funcs.decode = &read_string; + alltypes.req_string.arg = "1014"; + + alltypes.req_bytes.funcs.decode = &read_string; + alltypes.req_bytes.arg = "1015"; + + alltypes.req_submsg.funcs.decode = &read_submsg; + alltypes.req_submsg.arg = &req_submsg; + + alltypes.req_enum.funcs.decode = &read_varint; + alltypes.req_enum.arg = (void*)MyEnum_Truth; + + alltypes.req_emptymsg.funcs.decode = &read_emptymsg; + + /* Bind callbacks for repeated fields */ + alltypes.rep_int32.funcs.decode = &read_repeated_varint; + alltypes.rep_int32.arg = rep_int32; + + alltypes.rep_int64.funcs.decode = &read_repeated_varint; + alltypes.rep_int64.arg = rep_int64; + + alltypes.rep_uint32.funcs.decode = &read_repeated_varint; + alltypes.rep_uint32.arg = rep_uint32; + + alltypes.rep_uint64.funcs.decode = &read_repeated_varint; + alltypes.rep_uint64.arg = rep_uint64; + + alltypes.rep_sint32.funcs.decode = &read_repeated_svarint; + alltypes.rep_sint32.arg = rep_sint32; + + alltypes.rep_sint64.funcs.decode = &read_repeated_svarint; + alltypes.rep_sint64.arg = rep_sint64; + + alltypes.rep_bool.funcs.decode = &read_repeated_varint; + alltypes.rep_bool.arg = rep_bool; + + alltypes.rep_fixed32.funcs.decode = &read_repeated_fixed32; + alltypes.rep_fixed32.arg = rep_fixed32; + + alltypes.rep_sfixed32.funcs.decode = &read_repeated_fixed32; + alltypes.rep_sfixed32.arg = rep_sfixed32; + + alltypes.rep_float.funcs.decode = &read_repeated_fixed32; + alltypes.rep_float.arg = rep_float; + + alltypes.rep_fixed64.funcs.decode = &read_repeated_fixed64; + alltypes.rep_fixed64.arg = rep_fixed64; + + alltypes.rep_sfixed64.funcs.decode = &read_repeated_fixed64; + alltypes.rep_sfixed64.arg = rep_sfixed64; + + alltypes.rep_double.funcs.decode = &read_repeated_fixed64; + alltypes.rep_double.arg = rep_double; + + alltypes.rep_string.funcs.decode = &read_repeated_string; + alltypes.rep_string.arg = rep_string; + + alltypes.rep_bytes.funcs.decode = &read_repeated_string; + alltypes.rep_bytes.arg = rep_bytes; + + alltypes.rep_submsg.funcs.decode = &read_repeated_submsg; + alltypes.rep_submsg.arg = rep_submsg; + + alltypes.rep_enum.funcs.decode = &read_repeated_varint; + alltypes.rep_enum.arg = rep_enum; + + alltypes.rep_emptymsg.funcs.decode = &read_emptymsg; + + alltypes.req_limits.funcs.decode = &read_limits; + + alltypes.end.funcs.decode = &read_varint; + alltypes.end.arg = (void*)1099; + + /* Bind callbacks for optional fields */ + if (mode == 1) + { + alltypes.opt_int32.funcs.decode = &read_varint; + alltypes.opt_int32.arg = (void*)3041; + + alltypes.opt_int64.funcs.decode = &read_varint; + alltypes.opt_int64.arg = (void*)3042; + + alltypes.opt_uint32.funcs.decode = &read_varint; + alltypes.opt_uint32.arg = (void*)3043; + + alltypes.opt_uint64.funcs.decode = &read_varint; + alltypes.opt_uint64.arg = (void*)3044; + + alltypes.opt_sint32.funcs.decode = &read_svarint; + alltypes.opt_sint32.arg = (void*)3045; + + alltypes.opt_sint64.funcs.decode = &read_svarint; + alltypes.opt_sint64.arg = (void*)3046; + + alltypes.opt_bool.funcs.decode = &read_varint; + alltypes.opt_bool.arg = (void*)true; + + alltypes.opt_fixed32.funcs.decode = &read_fixed32; + alltypes.opt_fixed32.arg = &opt_fixed32; + + alltypes.opt_sfixed32.funcs.decode = &read_fixed32; + alltypes.opt_sfixed32.arg = &opt_sfixed32; + + alltypes.opt_float.funcs.decode = &read_fixed32; + alltypes.opt_float.arg = &opt_float; + + alltypes.opt_fixed64.funcs.decode = &read_fixed64; + alltypes.opt_fixed64.arg = &opt_fixed64; + + alltypes.opt_sfixed64.funcs.decode = &read_fixed64; + alltypes.opt_sfixed64.arg = &opt_sfixed64; + + alltypes.opt_double.funcs.decode = &read_fixed64; + alltypes.opt_double.arg = &opt_double; + + alltypes.opt_string.funcs.decode = &read_string; + alltypes.opt_string.arg = "3054"; + + alltypes.opt_bytes.funcs.decode = &read_string; + alltypes.opt_bytes.arg = "3055"; + + alltypes.opt_submsg.funcs.decode = &read_submsg; + alltypes.opt_submsg.arg = &opt_submsg; + + alltypes.opt_enum.funcs.decode = &read_varint; + alltypes.opt_enum.arg = (void*)MyEnum_Truth; + + alltypes.opt_emptymsg.funcs.decode = &read_emptymsg; + } + + return pb_decode(stream, AllTypes_fields, &alltypes); +} + +int main(int argc, char **argv) +{ + uint8_t buffer[1024]; + size_t count; + pb_istream_t stream; + + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Read the data into buffer */ + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_alltypes(&stream, mode)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} -- cgit 1.2.3-korg From bd22cf27776f8981f1bf2fff4fcccf171a0ed2c8 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 12 Feb 2014 18:51:48 +0200 Subject: Improve status/error reporting in generator. Update issue 105 Status: FixedInGit --- generator/nanopb_generator.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index e6efb247..2f2e8531 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -36,7 +36,6 @@ except: ''' + '\n') raise - # --------------------------------------------------------------------------- # Generation of single fields # --------------------------------------------------------------------------- @@ -900,6 +899,7 @@ class Globals: '''Ugly global variables, should find a good way to pass these.''' verbose_options = False separate_options = [] + matched_namemasks = set() def get_nanopb_suboptions(subdesc, options, name): '''Get copy of options, and merge information from subdesc.''' @@ -910,6 +910,7 @@ def get_nanopb_suboptions(subdesc, options, name): dotname = '.'.join(name.parts) for namemask, options in Globals.separate_options: if fnmatch(dotname, namemask): + Globals.matched_namemasks.add(namemask) new_options.MergeFrom(options) # Handle options defined in .proto @@ -929,8 +930,8 @@ def get_nanopb_suboptions(subdesc, options, name): new_options.MergeFrom(ext) if Globals.verbose_options: - print "Options for " + dotname + ":" - print text_format.MessageToString(new_options) + sys.stderr.write("Options for " + dotname + ": ") + sys.stderr.write(text_format.MessageToString(new_options) + "\n") return new_options @@ -994,13 +995,14 @@ def process_file(filename, fdesc, options): # No %s specified, use the filename as-is optfilename = options.options_file - if options.verbose: - print 'Reading options from ' + optfilename - if os.path.isfile(optfilename): + if options.verbose: + sys.stderr.write('Reading options from ' + optfilename + '\n') + Globals.separate_options = read_options_file(open(optfilename, "rU")) else: Globals.separate_options = [] + Globals.matched_namemasks = set() # Parse the file file_options = get_nanopb_suboptions(fdesc, toplevel_options, Names([filename])) @@ -1023,6 +1025,14 @@ def process_file(filename, fdesc, options): sourcedata = ''.join(generate_source(headerbasename, enums, messages, extensions, options)) + # Check if there were any lines in .options that did not match a member + unmatched = [n for n,o in Globals.separate_options if n not in Globals.matched_namemasks] + if unmatched and not options.quiet: + sys.stderr.write("Following patterns in " + optfilename + " did not match any fields: " + + ', '.join(unmatched) + "\n") + if not Globals.verbose_options: + sys.stderr.write("Use protoc --nanopb-out=-v:. to see a list of the field names.\n") + return {'headername': headername, 'headerdata': headerdata, 'sourcename': sourcename, 'sourcedata': sourcedata} @@ -1044,7 +1054,8 @@ def main_cli(): results = process_file(filename, None, options) if not options.quiet: - print "Writing to " + results['headername'] + " and " + results['sourcename'] + sys.stderr.write("Writing to " + results['headername'] + " and " + + results['sourcename'] + "\n") open(results['headername'], 'w').write(results['headerdata']) open(results['sourcename'], 'w').write(results['sourcedata']) @@ -1067,10 +1078,7 @@ def main_plugin(): args = shlex.split(request.parameter) options, dummy = optparser.parse_args(args) - # We can't go printing stuff to stdout - Globals.verbose_options = False - options.verbose = False - options.quiet = True + Globals.verbose_options = options.verbose response = plugin_pb2.CodeGeneratorResponse() -- cgit 1.2.3-korg From fe0bf121eb606d9accbc0fc8be6b1ff6fd5f6f27 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 12 Feb 2014 19:22:01 +0200 Subject: Add test case for package names defined in .proto file. --- tests/package_name/SConscript | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 tests/package_name/SConscript diff --git a/tests/package_name/SConscript b/tests/package_name/SConscript new file mode 100644 index 00000000..8f1b9021 --- /dev/null +++ b/tests/package_name/SConscript @@ -0,0 +1,36 @@ +# Check that alltypes test case works also when the .proto file defines +# a package name. + +Import("env") + +# Build a modified alltypes.proto +def modify_proto(target, source, env): + '''Add a "package test.package;" directive to the beginning of the .proto file.''' + data = open(str(source[0]), 'r').read() + open(str(target[0]), 'w').write("package test.package;\n\n" + data) + return 0 + +env.Command("alltypes.proto", "#alltypes/alltypes.proto", modify_proto) +env.Command("alltypes.options", "#alltypes/alltypes.options", Copy("$TARGET", "$SOURCE")) +env.NanopbProto(["alltypes", "alltypes.options"]) + +# Build a modified encode_alltypes.c +def modify_c(target, source, env): + '''Add package name to type names in .c file.''' + + data = open(str(source[0]), 'r').read() + + type_names = ['AllTypes', 'MyEnum', 'HugeEnum'] + for name in type_names: + data = data.replace(name, 'test_package_' + name) + + open(str(target[0]), 'w').write(data) + return 0 +env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", modify_c) + +# Encode and compare results to original alltypes testcase +enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) +refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX" +env.RunTest(enc) +env.Compare(["encode_alltypes.output", "$BUILD/alltypes/encode_alltypes.output"]) + -- cgit 1.2.3-korg From 542463dbaa3acae564e4a1eddf264e1fe226c38a Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 12 Feb 2014 19:43:04 +0200 Subject: Move descriptor.proto to google/protobuf directory where it belongs. This fixes some interoperability problems when trying to compile nanopb.proto for e.g. the Google's C++ library. Update issue 104 Status: FixedInGit --- generator/nanopb_generator.py | 4 +- generator/proto/Makefile | 2 +- generator/proto/descriptor.proto | 620 ----------------------- generator/proto/google/protobuf/descriptor.proto | 620 +++++++++++++++++++++++ generator/proto/nanopb.proto | 2 +- generator/proto/plugin.proto | 2 +- 6 files changed, 625 insertions(+), 625 deletions(-) delete mode 100644 generator/proto/descriptor.proto create mode 100644 generator/proto/google/protobuf/descriptor.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 2f2e8531..7c60ebe9 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -15,6 +15,7 @@ except: try: import google.protobuf.text_format as text_format + import google.protobuf.descriptor_pb2 as descriptor except: sys.stderr.write(''' ************************************************************* @@ -26,7 +27,7 @@ except: try: import proto.nanopb_pb2 as nanopb_pb2 - import proto.descriptor_pb2 as descriptor + import proto.plugin_pb2 as plugin_pb2 except: sys.stderr.write(''' ******************************************************************** @@ -1070,7 +1071,6 @@ def main_plugin(): msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) - import proto.plugin_pb2 as plugin_pb2 data = sys.stdin.read() request = plugin_pb2.CodeGeneratorRequest.FromString(data) diff --git a/generator/proto/Makefile b/generator/proto/Makefile index ee1390a5..89bfe528 100644 --- a/generator/proto/Makefile +++ b/generator/proto/Makefile @@ -1,4 +1,4 @@ -all: nanopb_pb2.py plugin_pb2.py descriptor_pb2.py +all: nanopb_pb2.py plugin_pb2.py %_pb2.py: %.proto protoc --python_out=. $< diff --git a/generator/proto/descriptor.proto b/generator/proto/descriptor.proto deleted file mode 100644 index a785f79f..00000000 --- a/generator/proto/descriptor.proto +++ /dev/null @@ -1,620 +0,0 @@ -// Protocol Buffers - Google's data interchange format -// Copyright 2008 Google Inc. All rights reserved. -// http://code.google.com/p/protobuf/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Author: kenton@google.com (Kenton Varda) -// Based on original Protocol Buffers design by -// Sanjay Ghemawat, Jeff Dean, and others. -// -// The messages in this file describe the definitions found in .proto files. -// A valid .proto file can be translated directly to a FileDescriptorProto -// without any other information (e.g. without reading its imports). - - - -package google.protobuf; -option java_package = "com.google.protobuf"; -option java_outer_classname = "DescriptorProtos"; - -// descriptor.proto must be optimized for speed because reflection-based -// algorithms don't work during bootstrapping. -option optimize_for = SPEED; - -// The protocol compiler can output a FileDescriptorSet containing the .proto -// files it parses. -message FileDescriptorSet { - repeated FileDescriptorProto file = 1; -} - -// Describes a complete .proto file. -message FileDescriptorProto { - optional string name = 1; // file name, relative to root of source tree - optional string package = 2; // e.g. "foo", "foo.bar", etc. - - // Names of files imported by this file. - repeated string dependency = 3; - // Indexes of the public imported files in the dependency list above. - repeated int32 public_dependency = 10; - // Indexes of the weak imported files in the dependency list. - // For Google-internal migration only. Do not use. - repeated int32 weak_dependency = 11; - - // All top-level definitions in this file. - repeated DescriptorProto message_type = 4; - repeated EnumDescriptorProto enum_type = 5; - repeated ServiceDescriptorProto service = 6; - repeated FieldDescriptorProto extension = 7; - - optional FileOptions options = 8; - - // This field contains optional information about the original source code. - // You may safely remove this entire field whithout harming runtime - // functionality of the descriptors -- the information is needed only by - // development tools. - optional SourceCodeInfo source_code_info = 9; -} - -// Describes a message type. -message DescriptorProto { - optional string name = 1; - - repeated FieldDescriptorProto field = 2; - repeated FieldDescriptorProto extension = 6; - - repeated DescriptorProto nested_type = 3; - repeated EnumDescriptorProto enum_type = 4; - - message ExtensionRange { - optional int32 start = 1; - optional int32 end = 2; - } - repeated ExtensionRange extension_range = 5; - - optional MessageOptions options = 7; -} - -// Describes a field within a message. -message FieldDescriptorProto { - enum Type { - // 0 is reserved for errors. - // Order is weird for historical reasons. - TYPE_DOUBLE = 1; - TYPE_FLOAT = 2; - // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if - // negative values are likely. - TYPE_INT64 = 3; - TYPE_UINT64 = 4; - // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if - // negative values are likely. - TYPE_INT32 = 5; - TYPE_FIXED64 = 6; - TYPE_FIXED32 = 7; - TYPE_BOOL = 8; - TYPE_STRING = 9; - TYPE_GROUP = 10; // Tag-delimited aggregate. - TYPE_MESSAGE = 11; // Length-delimited aggregate. - - // New in version 2. - TYPE_BYTES = 12; - TYPE_UINT32 = 13; - TYPE_ENUM = 14; - TYPE_SFIXED32 = 15; - TYPE_SFIXED64 = 16; - TYPE_SINT32 = 17; // Uses ZigZag encoding. - TYPE_SINT64 = 18; // Uses ZigZag encoding. - }; - - enum Label { - // 0 is reserved for errors - LABEL_OPTIONAL = 1; - LABEL_REQUIRED = 2; - LABEL_REPEATED = 3; - // TODO(sanjay): Should we add LABEL_MAP? - }; - - optional string name = 1; - optional int32 number = 3; - optional Label label = 4; - - // If type_name is set, this need not be set. If both this and type_name - // are set, this must be either TYPE_ENUM or TYPE_MESSAGE. - optional Type type = 5; - - // For message and enum types, this is the name of the type. If the name - // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping - // rules are used to find the type (i.e. first the nested types within this - // message are searched, then within the parent, on up to the root - // namespace). - optional string type_name = 6; - - // For extensions, this is the name of the type being extended. It is - // resolved in the same manner as type_name. - optional string extendee = 2; - - // For numeric types, contains the original text representation of the value. - // For booleans, "true" or "false". - // For strings, contains the default text contents (not escaped in any way). - // For bytes, contains the C escaped value. All bytes >= 128 are escaped. - // TODO(kenton): Base-64 encode? - optional string default_value = 7; - - optional FieldOptions options = 8; -} - -// Describes an enum type. -message EnumDescriptorProto { - optional string name = 1; - - repeated EnumValueDescriptorProto value = 2; - - optional EnumOptions options = 3; -} - -// Describes a value within an enum. -message EnumValueDescriptorProto { - optional string name = 1; - optional int32 number = 2; - - optional EnumValueOptions options = 3; -} - -// Describes a service. -message ServiceDescriptorProto { - optional string name = 1; - repeated MethodDescriptorProto method = 2; - - optional ServiceOptions options = 3; -} - -// Describes a method of a service. -message MethodDescriptorProto { - optional string name = 1; - - // Input and output type names. These are resolved in the same way as - // FieldDescriptorProto.type_name, but must refer to a message type. - optional string input_type = 2; - optional string output_type = 3; - - optional MethodOptions options = 4; -} - - -// =================================================================== -// Options - -// Each of the definitions above may have "options" attached. These are -// just annotations which may cause code to be generated slightly differently -// or may contain hints for code that manipulates protocol messages. -// -// Clients may define custom options as extensions of the *Options messages. -// These extensions may not yet be known at parsing time, so the parser cannot -// store the values in them. Instead it stores them in a field in the *Options -// message called uninterpreted_option. This field must have the same name -// across all *Options messages. We then use this field to populate the -// extensions when we build a descriptor, at which point all protos have been -// parsed and so all extensions are known. -// -// Extension numbers for custom options may be chosen as follows: -// * For options which will only be used within a single application or -// organization, or for experimental options, use field numbers 50000 -// through 99999. It is up to you to ensure that you do not use the -// same number for multiple options. -// * For options which will be published and used publicly by multiple -// independent entities, e-mail protobuf-global-extension-registry@google.com -// to reserve extension numbers. Simply provide your project name (e.g. -// Object-C plugin) and your porject website (if available) -- there's no need -// to explain how you intend to use them. Usually you only need one extension -// number. You can declare multiple options with only one extension number by -// putting them in a sub-message. See the Custom Options section of the docs -// for examples: -// http://code.google.com/apis/protocolbuffers/docs/proto.html#options -// If this turns out to be popular, a web service will be set up -// to automatically assign option numbers. - - -message FileOptions { - - // Sets the Java package where classes generated from this .proto will be - // placed. By default, the proto package is used, but this is often - // inappropriate because proto packages do not normally start with backwards - // domain names. - optional string java_package = 1; - - - // If set, all the classes from the .proto file are wrapped in a single - // outer class with the given name. This applies to both Proto1 - // (equivalent to the old "--one_java_file" option) and Proto2 (where - // a .proto always translates to a single class, but you may want to - // explicitly choose the class name). - optional string java_outer_classname = 8; - - // If set true, then the Java code generator will generate a separate .java - // file for each top-level message, enum, and service defined in the .proto - // file. Thus, these types will *not* be nested inside the outer class - // named by java_outer_classname. However, the outer class will still be - // generated to contain the file's getDescriptor() method as well as any - // top-level extensions defined in the file. - optional bool java_multiple_files = 10 [default=false]; - - // If set true, then the Java code generator will generate equals() and - // hashCode() methods for all messages defined in the .proto file. This is - // purely a speed optimization, as the AbstractMessage base class includes - // reflection-based implementations of these methods. - optional bool java_generate_equals_and_hash = 20 [default=false]; - - // Generated classes can be optimized for speed or code size. - enum OptimizeMode { - SPEED = 1; // Generate complete code for parsing, serialization, - // etc. - CODE_SIZE = 2; // Use ReflectionOps to implement these methods. - LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime. - } - optional OptimizeMode optimize_for = 9 [default=SPEED]; - - // Sets the Go package where structs generated from this .proto will be - // placed. There is no default. - optional string go_package = 11; - - - - // Should generic services be generated in each language? "Generic" services - // are not specific to any particular RPC system. They are generated by the - // main code generators in each language (without additional plugins). - // Generic services were the only kind of service generation supported by - // early versions of proto2. - // - // Generic services are now considered deprecated in favor of using plugins - // that generate code specific to your particular RPC system. Therefore, - // these default to false. Old code which depends on generic services should - // explicitly set them to true. - optional bool cc_generic_services = 16 [default=false]; - optional bool java_generic_services = 17 [default=false]; - optional bool py_generic_services = 18 [default=false]; - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message MessageOptions { - // Set true to use the old proto1 MessageSet wire format for extensions. - // This is provided for backwards-compatibility with the MessageSet wire - // format. You should not use this for any other reason: It's less - // efficient, has fewer features, and is more complicated. - // - // The message must be defined exactly as follows: - // message Foo { - // option message_set_wire_format = true; - // extensions 4 to max; - // } - // Note that the message cannot have any defined fields; MessageSets only - // have extensions. - // - // All extensions of your type must be singular messages; e.g. they cannot - // be int32s, enums, or repeated messages. - // - // Because this is an option, the above two restrictions are not enforced by - // the protocol compiler. - optional bool message_set_wire_format = 1 [default=false]; - - // Disables the generation of the standard "descriptor()" accessor, which can - // conflict with a field of the same name. This is meant to make migration - // from proto1 easier; new code should avoid fields named "descriptor". - optional bool no_standard_descriptor_accessor = 2 [default=false]; - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message FieldOptions { - // The ctype option instructs the C++ code generator to use a different - // representation of the field than it normally would. See the specific - // options below. This option is not yet implemented in the open source - // release -- sorry, we'll try to include it in a future version! - optional CType ctype = 1 [default = STRING]; - enum CType { - // Default mode. - STRING = 0; - - CORD = 1; - - STRING_PIECE = 2; - } - // The packed option can be enabled for repeated primitive fields to enable - // a more efficient representation on the wire. Rather than repeatedly - // writing the tag and type for each element, the entire array is encoded as - // a single length-delimited blob. - optional bool packed = 2; - - - - // Should this field be parsed lazily? Lazy applies only to message-type - // fields. It means that when the outer message is initially parsed, the - // inner message's contents will not be parsed but instead stored in encoded - // form. The inner message will actually be parsed when it is first accessed. - // - // This is only a hint. Implementations are free to choose whether to use - // eager or lazy parsing regardless of the value of this option. However, - // setting this option true suggests that the protocol author believes that - // using lazy parsing on this field is worth the additional bookkeeping - // overhead typically needed to implement it. - // - // This option does not affect the public interface of any generated code; - // all method signatures remain the same. Furthermore, thread-safety of the - // interface is not affected by this option; const methods remain safe to - // call from multiple threads concurrently, while non-const methods continue - // to require exclusive access. - // - // - // Note that implementations may choose not to check required fields within - // a lazy sub-message. That is, calling IsInitialized() on the outher message - // may return true even if the inner message has missing required fields. - // This is necessary because otherwise the inner message would have to be - // parsed in order to perform the check, defeating the purpose of lazy - // parsing. An implementation which chooses not to check required fields - // must be consistent about it. That is, for any particular sub-message, the - // implementation must either *always* check its required fields, or *never* - // check its required fields, regardless of whether or not the message has - // been parsed. - optional bool lazy = 5 [default=false]; - - // Is this field deprecated? - // Depending on the target platform, this can emit Deprecated annotations - // for accessors, or it will be completely ignored; in the very least, this - // is a formalization for deprecating fields. - optional bool deprecated = 3 [default=false]; - - // EXPERIMENTAL. DO NOT USE. - // For "map" fields, the name of the field in the enclosed type that - // is the key for this map. For example, suppose we have: - // message Item { - // required string name = 1; - // required string value = 2; - // } - // message Config { - // repeated Item items = 1 [experimental_map_key="name"]; - // } - // In this situation, the map key for Item will be set to "name". - // TODO: Fully-implement this, then remove the "experimental_" prefix. - optional string experimental_map_key = 9; - - // For Google-internal migration only. Do not use. - optional bool weak = 10 [default=false]; - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message EnumOptions { - - // Set this option to false to disallow mapping different tag names to a same - // value. - optional bool allow_alias = 2 [default=true]; - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message EnumValueOptions { - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message ServiceOptions { - - // Note: Field numbers 1 through 32 are reserved for Google's internal RPC - // framework. We apologize for hoarding these numbers to ourselves, but - // we were already using them long before we decided to release Protocol - // Buffers. - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - -message MethodOptions { - - // Note: Field numbers 1 through 32 are reserved for Google's internal RPC - // framework. We apologize for hoarding these numbers to ourselves, but - // we were already using them long before we decided to release Protocol - // Buffers. - - // The parser stores options it doesn't recognize here. See above. - repeated UninterpretedOption uninterpreted_option = 999; - - // Clients can define custom options in extensions of this message. See above. - extensions 1000 to max; -} - - -// A message representing a option the parser does not recognize. This only -// appears in options protos created by the compiler::Parser class. -// DescriptorPool resolves these when building Descriptor objects. Therefore, -// options protos in descriptor objects (e.g. returned by Descriptor::options(), -// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions -// in them. -message UninterpretedOption { - // The name of the uninterpreted option. Each string represents a segment in - // a dot-separated name. is_extension is true iff a segment represents an - // extension (denoted with parentheses in options specs in .proto files). - // E.g.,{ ["foo", false], ["bar.baz", true], ["qux", false] } represents - // "foo.(bar.baz).qux". - message NamePart { - required string name_part = 1; - required bool is_extension = 2; - } - repeated NamePart name = 2; - - // The value of the uninterpreted option, in whatever type the tokenizer - // identified it as during parsing. Exactly one of these should be set. - optional string identifier_value = 3; - optional uint64 positive_int_value = 4; - optional int64 negative_int_value = 5; - optional double double_value = 6; - optional bytes string_value = 7; - optional string aggregate_value = 8; -} - -// =================================================================== -// Optional source code info - -// Encapsulates information about the original source file from which a -// FileDescriptorProto was generated. -message SourceCodeInfo { - // A Location identifies a piece of source code in a .proto file which - // corresponds to a particular definition. This information is intended - // to be useful to IDEs, code indexers, documentation generators, and similar - // tools. - // - // For example, say we have a file like: - // message Foo { - // optional string foo = 1; - // } - // Let's look at just the field definition: - // optional string foo = 1; - // ^ ^^ ^^ ^ ^^^ - // a bc de f ghi - // We have the following locations: - // span path represents - // [a,i) [ 4, 0, 2, 0 ] The whole field definition. - // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). - // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). - // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). - // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). - // - // Notes: - // - A location may refer to a repeated field itself (i.e. not to any - // particular index within it). This is used whenever a set of elements are - // logically enclosed in a single code segment. For example, an entire - // extend block (possibly containing multiple extension definitions) will - // have an outer location whose path refers to the "extensions" repeated - // field without an index. - // - Multiple locations may have the same path. This happens when a single - // logical declaration is spread out across multiple places. The most - // obvious example is the "extend" block again -- there may be multiple - // extend blocks in the same scope, each of which will have the same path. - // - A location's span is not always a subset of its parent's span. For - // example, the "extendee" of an extension declaration appears at the - // beginning of the "extend" block and is shared by all extensions within - // the block. - // - Just because a location's span is a subset of some other location's span - // does not mean that it is a descendent. For example, a "group" defines - // both a type and a field in a single declaration. Thus, the locations - // corresponding to the type and field and their components will overlap. - // - Code which tries to interpret locations should probably be designed to - // ignore those that it doesn't understand, as more types of locations could - // be recorded in the future. - repeated Location location = 1; - message Location { - // Identifies which part of the FileDescriptorProto was defined at this - // location. - // - // Each element is a field number or an index. They form a path from - // the root FileDescriptorProto to the place where the definition. For - // example, this path: - // [ 4, 3, 2, 7, 1 ] - // refers to: - // file.message_type(3) // 4, 3 - // .field(7) // 2, 7 - // .name() // 1 - // This is because FileDescriptorProto.message_type has field number 4: - // repeated DescriptorProto message_type = 4; - // and DescriptorProto.field has field number 2: - // repeated FieldDescriptorProto field = 2; - // and FieldDescriptorProto.name has field number 1: - // optional string name = 1; - // - // Thus, the above path gives the location of a field name. If we removed - // the last element: - // [ 4, 3, 2, 7 ] - // this path refers to the whole field declaration (from the beginning - // of the label to the terminating semicolon). - repeated int32 path = 1 [packed=true]; - - // Always has exactly three or four elements: start line, start column, - // end line (optional, otherwise assumed same as start line), end column. - // These are packed into a single field for efficiency. Note that line - // and column numbers are zero-based -- typically you will want to add - // 1 to each before displaying to a user. - repeated int32 span = 2 [packed=true]; - - // If this SourceCodeInfo represents a complete declaration, these are any - // comments appearing before and after the declaration which appear to be - // attached to the declaration. - // - // A series of line comments appearing on consecutive lines, with no other - // tokens appearing on those lines, will be treated as a single comment. - // - // Only the comment content is provided; comment markers (e.g. //) are - // stripped out. For block comments, leading whitespace and an asterisk - // will be stripped from the beginning of each line other than the first. - // Newlines are included in the output. - // - // Examples: - // - // optional int32 foo = 1; // Comment attached to foo. - // // Comment attached to bar. - // optional int32 bar = 2; - // - // optional string baz = 3; - // // Comment attached to baz. - // // Another line attached to baz. - // - // // Comment attached to qux. - // // - // // Another line attached to qux. - // optional double qux = 4; - // - // optional string corge = 5; - // /* Block comment attached - // * to corge. Leading asterisks - // * will be removed. */ - // /* Block comment attached to - // * grault. */ - // optional int32 grault = 6; - optional string leading_comments = 3; - optional string trailing_comments = 4; - } -} diff --git a/generator/proto/google/protobuf/descriptor.proto b/generator/proto/google/protobuf/descriptor.proto new file mode 100644 index 00000000..a785f79f --- /dev/null +++ b/generator/proto/google/protobuf/descriptor.proto @@ -0,0 +1,620 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// http://code.google.com/p/protobuf/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// Author: kenton@google.com (Kenton Varda) +// Based on original Protocol Buffers design by +// Sanjay Ghemawat, Jeff Dean, and others. +// +// The messages in this file describe the definitions found in .proto files. +// A valid .proto file can be translated directly to a FileDescriptorProto +// without any other information (e.g. without reading its imports). + + + +package google.protobuf; +option java_package = "com.google.protobuf"; +option java_outer_classname = "DescriptorProtos"; + +// descriptor.proto must be optimized for speed because reflection-based +// algorithms don't work during bootstrapping. +option optimize_for = SPEED; + +// The protocol compiler can output a FileDescriptorSet containing the .proto +// files it parses. +message FileDescriptorSet { + repeated FileDescriptorProto file = 1; +} + +// Describes a complete .proto file. +message FileDescriptorProto { + optional string name = 1; // file name, relative to root of source tree + optional string package = 2; // e.g. "foo", "foo.bar", etc. + + // Names of files imported by this file. + repeated string dependency = 3; + // Indexes of the public imported files in the dependency list above. + repeated int32 public_dependency = 10; + // Indexes of the weak imported files in the dependency list. + // For Google-internal migration only. Do not use. + repeated int32 weak_dependency = 11; + + // All top-level definitions in this file. + repeated DescriptorProto message_type = 4; + repeated EnumDescriptorProto enum_type = 5; + repeated ServiceDescriptorProto service = 6; + repeated FieldDescriptorProto extension = 7; + + optional FileOptions options = 8; + + // This field contains optional information about the original source code. + // You may safely remove this entire field whithout harming runtime + // functionality of the descriptors -- the information is needed only by + // development tools. + optional SourceCodeInfo source_code_info = 9; +} + +// Describes a message type. +message DescriptorProto { + optional string name = 1; + + repeated FieldDescriptorProto field = 2; + repeated FieldDescriptorProto extension = 6; + + repeated DescriptorProto nested_type = 3; + repeated EnumDescriptorProto enum_type = 4; + + message ExtensionRange { + optional int32 start = 1; + optional int32 end = 2; + } + repeated ExtensionRange extension_range = 5; + + optional MessageOptions options = 7; +} + +// Describes a field within a message. +message FieldDescriptorProto { + enum Type { + // 0 is reserved for errors. + // Order is weird for historical reasons. + TYPE_DOUBLE = 1; + TYPE_FLOAT = 2; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if + // negative values are likely. + TYPE_INT64 = 3; + TYPE_UINT64 = 4; + // Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if + // negative values are likely. + TYPE_INT32 = 5; + TYPE_FIXED64 = 6; + TYPE_FIXED32 = 7; + TYPE_BOOL = 8; + TYPE_STRING = 9; + TYPE_GROUP = 10; // Tag-delimited aggregate. + TYPE_MESSAGE = 11; // Length-delimited aggregate. + + // New in version 2. + TYPE_BYTES = 12; + TYPE_UINT32 = 13; + TYPE_ENUM = 14; + TYPE_SFIXED32 = 15; + TYPE_SFIXED64 = 16; + TYPE_SINT32 = 17; // Uses ZigZag encoding. + TYPE_SINT64 = 18; // Uses ZigZag encoding. + }; + + enum Label { + // 0 is reserved for errors + LABEL_OPTIONAL = 1; + LABEL_REQUIRED = 2; + LABEL_REPEATED = 3; + // TODO(sanjay): Should we add LABEL_MAP? + }; + + optional string name = 1; + optional int32 number = 3; + optional Label label = 4; + + // If type_name is set, this need not be set. If both this and type_name + // are set, this must be either TYPE_ENUM or TYPE_MESSAGE. + optional Type type = 5; + + // For message and enum types, this is the name of the type. If the name + // starts with a '.', it is fully-qualified. Otherwise, C++-like scoping + // rules are used to find the type (i.e. first the nested types within this + // message are searched, then within the parent, on up to the root + // namespace). + optional string type_name = 6; + + // For extensions, this is the name of the type being extended. It is + // resolved in the same manner as type_name. + optional string extendee = 2; + + // For numeric types, contains the original text representation of the value. + // For booleans, "true" or "false". + // For strings, contains the default text contents (not escaped in any way). + // For bytes, contains the C escaped value. All bytes >= 128 are escaped. + // TODO(kenton): Base-64 encode? + optional string default_value = 7; + + optional FieldOptions options = 8; +} + +// Describes an enum type. +message EnumDescriptorProto { + optional string name = 1; + + repeated EnumValueDescriptorProto value = 2; + + optional EnumOptions options = 3; +} + +// Describes a value within an enum. +message EnumValueDescriptorProto { + optional string name = 1; + optional int32 number = 2; + + optional EnumValueOptions options = 3; +} + +// Describes a service. +message ServiceDescriptorProto { + optional string name = 1; + repeated MethodDescriptorProto method = 2; + + optional ServiceOptions options = 3; +} + +// Describes a method of a service. +message MethodDescriptorProto { + optional string name = 1; + + // Input and output type names. These are resolved in the same way as + // FieldDescriptorProto.type_name, but must refer to a message type. + optional string input_type = 2; + optional string output_type = 3; + + optional MethodOptions options = 4; +} + + +// =================================================================== +// Options + +// Each of the definitions above may have "options" attached. These are +// just annotations which may cause code to be generated slightly differently +// or may contain hints for code that manipulates protocol messages. +// +// Clients may define custom options as extensions of the *Options messages. +// These extensions may not yet be known at parsing time, so the parser cannot +// store the values in them. Instead it stores them in a field in the *Options +// message called uninterpreted_option. This field must have the same name +// across all *Options messages. We then use this field to populate the +// extensions when we build a descriptor, at which point all protos have been +// parsed and so all extensions are known. +// +// Extension numbers for custom options may be chosen as follows: +// * For options which will only be used within a single application or +// organization, or for experimental options, use field numbers 50000 +// through 99999. It is up to you to ensure that you do not use the +// same number for multiple options. +// * For options which will be published and used publicly by multiple +// independent entities, e-mail protobuf-global-extension-registry@google.com +// to reserve extension numbers. Simply provide your project name (e.g. +// Object-C plugin) and your porject website (if available) -- there's no need +// to explain how you intend to use them. Usually you only need one extension +// number. You can declare multiple options with only one extension number by +// putting them in a sub-message. See the Custom Options section of the docs +// for examples: +// http://code.google.com/apis/protocolbuffers/docs/proto.html#options +// If this turns out to be popular, a web service will be set up +// to automatically assign option numbers. + + +message FileOptions { + + // Sets the Java package where classes generated from this .proto will be + // placed. By default, the proto package is used, but this is often + // inappropriate because proto packages do not normally start with backwards + // domain names. + optional string java_package = 1; + + + // If set, all the classes from the .proto file are wrapped in a single + // outer class with the given name. This applies to both Proto1 + // (equivalent to the old "--one_java_file" option) and Proto2 (where + // a .proto always translates to a single class, but you may want to + // explicitly choose the class name). + optional string java_outer_classname = 8; + + // If set true, then the Java code generator will generate a separate .java + // file for each top-level message, enum, and service defined in the .proto + // file. Thus, these types will *not* be nested inside the outer class + // named by java_outer_classname. However, the outer class will still be + // generated to contain the file's getDescriptor() method as well as any + // top-level extensions defined in the file. + optional bool java_multiple_files = 10 [default=false]; + + // If set true, then the Java code generator will generate equals() and + // hashCode() methods for all messages defined in the .proto file. This is + // purely a speed optimization, as the AbstractMessage base class includes + // reflection-based implementations of these methods. + optional bool java_generate_equals_and_hash = 20 [default=false]; + + // Generated classes can be optimized for speed or code size. + enum OptimizeMode { + SPEED = 1; // Generate complete code for parsing, serialization, + // etc. + CODE_SIZE = 2; // Use ReflectionOps to implement these methods. + LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime. + } + optional OptimizeMode optimize_for = 9 [default=SPEED]; + + // Sets the Go package where structs generated from this .proto will be + // placed. There is no default. + optional string go_package = 11; + + + + // Should generic services be generated in each language? "Generic" services + // are not specific to any particular RPC system. They are generated by the + // main code generators in each language (without additional plugins). + // Generic services were the only kind of service generation supported by + // early versions of proto2. + // + // Generic services are now considered deprecated in favor of using plugins + // that generate code specific to your particular RPC system. Therefore, + // these default to false. Old code which depends on generic services should + // explicitly set them to true. + optional bool cc_generic_services = 16 [default=false]; + optional bool java_generic_services = 17 [default=false]; + optional bool py_generic_services = 18 [default=false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message MessageOptions { + // Set true to use the old proto1 MessageSet wire format for extensions. + // This is provided for backwards-compatibility with the MessageSet wire + // format. You should not use this for any other reason: It's less + // efficient, has fewer features, and is more complicated. + // + // The message must be defined exactly as follows: + // message Foo { + // option message_set_wire_format = true; + // extensions 4 to max; + // } + // Note that the message cannot have any defined fields; MessageSets only + // have extensions. + // + // All extensions of your type must be singular messages; e.g. they cannot + // be int32s, enums, or repeated messages. + // + // Because this is an option, the above two restrictions are not enforced by + // the protocol compiler. + optional bool message_set_wire_format = 1 [default=false]; + + // Disables the generation of the standard "descriptor()" accessor, which can + // conflict with a field of the same name. This is meant to make migration + // from proto1 easier; new code should avoid fields named "descriptor". + optional bool no_standard_descriptor_accessor = 2 [default=false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message FieldOptions { + // The ctype option instructs the C++ code generator to use a different + // representation of the field than it normally would. See the specific + // options below. This option is not yet implemented in the open source + // release -- sorry, we'll try to include it in a future version! + optional CType ctype = 1 [default = STRING]; + enum CType { + // Default mode. + STRING = 0; + + CORD = 1; + + STRING_PIECE = 2; + } + // The packed option can be enabled for repeated primitive fields to enable + // a more efficient representation on the wire. Rather than repeatedly + // writing the tag and type for each element, the entire array is encoded as + // a single length-delimited blob. + optional bool packed = 2; + + + + // Should this field be parsed lazily? Lazy applies only to message-type + // fields. It means that when the outer message is initially parsed, the + // inner message's contents will not be parsed but instead stored in encoded + // form. The inner message will actually be parsed when it is first accessed. + // + // This is only a hint. Implementations are free to choose whether to use + // eager or lazy parsing regardless of the value of this option. However, + // setting this option true suggests that the protocol author believes that + // using lazy parsing on this field is worth the additional bookkeeping + // overhead typically needed to implement it. + // + // This option does not affect the public interface of any generated code; + // all method signatures remain the same. Furthermore, thread-safety of the + // interface is not affected by this option; const methods remain safe to + // call from multiple threads concurrently, while non-const methods continue + // to require exclusive access. + // + // + // Note that implementations may choose not to check required fields within + // a lazy sub-message. That is, calling IsInitialized() on the outher message + // may return true even if the inner message has missing required fields. + // This is necessary because otherwise the inner message would have to be + // parsed in order to perform the check, defeating the purpose of lazy + // parsing. An implementation which chooses not to check required fields + // must be consistent about it. That is, for any particular sub-message, the + // implementation must either *always* check its required fields, or *never* + // check its required fields, regardless of whether or not the message has + // been parsed. + optional bool lazy = 5 [default=false]; + + // Is this field deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for accessors, or it will be completely ignored; in the very least, this + // is a formalization for deprecating fields. + optional bool deprecated = 3 [default=false]; + + // EXPERIMENTAL. DO NOT USE. + // For "map" fields, the name of the field in the enclosed type that + // is the key for this map. For example, suppose we have: + // message Item { + // required string name = 1; + // required string value = 2; + // } + // message Config { + // repeated Item items = 1 [experimental_map_key="name"]; + // } + // In this situation, the map key for Item will be set to "name". + // TODO: Fully-implement this, then remove the "experimental_" prefix. + optional string experimental_map_key = 9; + + // For Google-internal migration only. Do not use. + optional bool weak = 10 [default=false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumOptions { + + // Set this option to false to disallow mapping different tag names to a same + // value. + optional bool allow_alias = 2 [default=true]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message EnumValueOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message ServiceOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + +message MethodOptions { + + // Note: Field numbers 1 through 32 are reserved for Google's internal RPC + // framework. We apologize for hoarding these numbers to ourselves, but + // we were already using them long before we decided to release Protocol + // Buffers. + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; +} + + +// A message representing a option the parser does not recognize. This only +// appears in options protos created by the compiler::Parser class. +// DescriptorPool resolves these when building Descriptor objects. Therefore, +// options protos in descriptor objects (e.g. returned by Descriptor::options(), +// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions +// in them. +message UninterpretedOption { + // The name of the uninterpreted option. Each string represents a segment in + // a dot-separated name. is_extension is true iff a segment represents an + // extension (denoted with parentheses in options specs in .proto files). + // E.g.,{ ["foo", false], ["bar.baz", true], ["qux", false] } represents + // "foo.(bar.baz).qux". + message NamePart { + required string name_part = 1; + required bool is_extension = 2; + } + repeated NamePart name = 2; + + // The value of the uninterpreted option, in whatever type the tokenizer + // identified it as during parsing. Exactly one of these should be set. + optional string identifier_value = 3; + optional uint64 positive_int_value = 4; + optional int64 negative_int_value = 5; + optional double double_value = 6; + optional bytes string_value = 7; + optional string aggregate_value = 8; +} + +// =================================================================== +// Optional source code info + +// Encapsulates information about the original source file from which a +// FileDescriptorProto was generated. +message SourceCodeInfo { + // A Location identifies a piece of source code in a .proto file which + // corresponds to a particular definition. This information is intended + // to be useful to IDEs, code indexers, documentation generators, and similar + // tools. + // + // For example, say we have a file like: + // message Foo { + // optional string foo = 1; + // } + // Let's look at just the field definition: + // optional string foo = 1; + // ^ ^^ ^^ ^ ^^^ + // a bc de f ghi + // We have the following locations: + // span path represents + // [a,i) [ 4, 0, 2, 0 ] The whole field definition. + // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). + // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). + // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). + // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). + // + // Notes: + // - A location may refer to a repeated field itself (i.e. not to any + // particular index within it). This is used whenever a set of elements are + // logically enclosed in a single code segment. For example, an entire + // extend block (possibly containing multiple extension definitions) will + // have an outer location whose path refers to the "extensions" repeated + // field without an index. + // - Multiple locations may have the same path. This happens when a single + // logical declaration is spread out across multiple places. The most + // obvious example is the "extend" block again -- there may be multiple + // extend blocks in the same scope, each of which will have the same path. + // - A location's span is not always a subset of its parent's span. For + // example, the "extendee" of an extension declaration appears at the + // beginning of the "extend" block and is shared by all extensions within + // the block. + // - Just because a location's span is a subset of some other location's span + // does not mean that it is a descendent. For example, a "group" defines + // both a type and a field in a single declaration. Thus, the locations + // corresponding to the type and field and their components will overlap. + // - Code which tries to interpret locations should probably be designed to + // ignore those that it doesn't understand, as more types of locations could + // be recorded in the future. + repeated Location location = 1; + message Location { + // Identifies which part of the FileDescriptorProto was defined at this + // location. + // + // Each element is a field number or an index. They form a path from + // the root FileDescriptorProto to the place where the definition. For + // example, this path: + // [ 4, 3, 2, 7, 1 ] + // refers to: + // file.message_type(3) // 4, 3 + // .field(7) // 2, 7 + // .name() // 1 + // This is because FileDescriptorProto.message_type has field number 4: + // repeated DescriptorProto message_type = 4; + // and DescriptorProto.field has field number 2: + // repeated FieldDescriptorProto field = 2; + // and FieldDescriptorProto.name has field number 1: + // optional string name = 1; + // + // Thus, the above path gives the location of a field name. If we removed + // the last element: + // [ 4, 3, 2, 7 ] + // this path refers to the whole field declaration (from the beginning + // of the label to the terminating semicolon). + repeated int32 path = 1 [packed=true]; + + // Always has exactly three or four elements: start line, start column, + // end line (optional, otherwise assumed same as start line), end column. + // These are packed into a single field for efficiency. Note that line + // and column numbers are zero-based -- typically you will want to add + // 1 to each before displaying to a user. + repeated int32 span = 2 [packed=true]; + + // If this SourceCodeInfo represents a complete declaration, these are any + // comments appearing before and after the declaration which appear to be + // attached to the declaration. + // + // A series of line comments appearing on consecutive lines, with no other + // tokens appearing on those lines, will be treated as a single comment. + // + // Only the comment content is provided; comment markers (e.g. //) are + // stripped out. For block comments, leading whitespace and an asterisk + // will be stripped from the beginning of each line other than the first. + // Newlines are included in the output. + // + // Examples: + // + // optional int32 foo = 1; // Comment attached to foo. + // // Comment attached to bar. + // optional int32 bar = 2; + // + // optional string baz = 3; + // // Comment attached to baz. + // // Another line attached to baz. + // + // // Comment attached to qux. + // // + // // Another line attached to qux. + // optional double qux = 4; + // + // optional string corge = 5; + // /* Block comment attached + // * to corge. Leading asterisks + // * will be removed. */ + // /* Block comment attached to + // * grault. */ + // optional int32 grault = 6; + optional string leading_comments = 3; + optional string trailing_comments = 4; + } +} diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index 2ea011f4..2be2f803 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -5,7 +5,7 @@ // These are used by nanopb to generate statically allocable structures // for memory-limited environments. -import "descriptor.proto"; +import "google/protobuf/descriptor.proto"; option java_package = "fi.kapsi.koti.jpa.nanopb"; diff --git a/generator/proto/plugin.proto b/generator/proto/plugin.proto index d8b515e3..651ed10c 100644 --- a/generator/proto/plugin.proto +++ b/generator/proto/plugin.proto @@ -46,7 +46,7 @@ package google.protobuf.compiler; -import "descriptor.proto"; +import "google/protobuf/descriptor.proto"; // An encoded CodeGeneratorRequest is written to the plugin's stdin. message CodeGeneratorRequest { -- cgit 1.2.3-korg From 586777b52feae5d10d2eb2fab9b2e366a5039731 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 12 Feb 2014 19:50:52 +0200 Subject: Update changelog --- CHANGELOG.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index f0c97c65..e6c88f2a 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,11 @@ +nanopb-0.2.6 (2014-02-15) + Fix generator error with bytes callback fields (issue 99) + Fix warnings about large integer constants (issue 102) + Add comments to where STATIC_ASSERT is used (issue 96) + Add warning about unknown field names on .options (issue 105) + Move descriptor.proto to google/protobuf subdirectory (issue 104) + Improved tests + nanopb-0.2.5 (2014-01-01) Fix a bug with encoding negative values in int32 fields (issue 97) Create binary packages of the generator + dependencies (issue 47) -- cgit 1.2.3-korg From 7f397b067f660efe98855c80bdd2500167303737 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 15 Feb 2014 16:40:38 +0200 Subject: Publishing nanopb-0.2.6 --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 7c60ebe9..a28e4710 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.6-dev" +nanopb_version = "nanopb-0.2.6" import sys diff --git a/pb.h b/pb.h index 3faa67b2..cf65bfb0 100644 --- a/pb.h +++ b/pb.h @@ -43,7 +43,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.6-dev +#define NANOPB_VERSION nanopb-0.2.6 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 842c960d5df4b7715b0c7d2c00adeddf84389421 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 15 Feb 2014 17:15:54 +0200 Subject: Setting version to 0.2.7-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index a28e4710..0926db29 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.6" +nanopb_version = "nanopb-0.2.7-dev" import sys diff --git a/pb.h b/pb.h index cf65bfb0..375b4a5a 100644 --- a/pb.h +++ b/pb.h @@ -43,7 +43,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.6 +#define NANOPB_VERSION nanopb-0.2.7-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 011a30af9c015b1edd67420d9ca947e9fb499e73 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 24 Feb 2014 21:09:25 +0200 Subject: Beginnings of malloc support in pb_decode --- pb.h | 4 + pb_decode.c | 143 ++++++++++++++- tests/alltypes_pointer/SConscript | 11 +- tests/alltypes_pointer/decode_alltypes_pointer.c | 210 +++++++++++++++++++++++ 4 files changed, 362 insertions(+), 6 deletions(-) create mode 100644 tests/alltypes_pointer/decode_alltypes_pointer.c diff --git a/pb.h b/pb.h index 375b4a5a..eb4f94e7 100644 --- a/pb.h +++ b/pb.h @@ -63,6 +63,10 @@ #include #include #include + +#ifdef PB_ENABLE_MALLOC +#include +#endif #endif /* Macro for defining packed structures (compiler dependent). diff --git a/pb_decode.c b/pb_decode.c index 30c124dd..4411e26a 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -465,6 +465,121 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t } } +#ifdef PB_ENABLE_MALLOC +/* Allocate storage for the field and store the pointer at iter->pData. + * array_size is the number of entries to reserve in an array. */ +static bool checkreturn allocate_field(pb_istream_t *stream, pb_field_iterator_t *iter, size_t array_size) +{ + void *ptr = *(void**)iter->pData; + size_t size = array_size * iter->pos->data_size; + + if (ptr == NULL) + { + /* First allocation */ + ptr = malloc(size); + if (ptr == NULL) + PB_RETURN_ERROR(stream, "malloc failed"); + } + else + { + /* Expand previous allocation */ + /* Note: on failure the old pointer will remain in the structure, + * the message must be freed by caller also on error return. */ + ptr = realloc(ptr, size); + if (ptr == NULL) + PB_RETURN_ERROR(stream, "realloc failed"); + } + + *(void**)iter->pData = ptr; + return true; +} +#endif + +static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) +{ +#ifndef PB_ENABLE_MALLOC + UNUSED(wire_type); + UNUSED(iter); + PB_RETURN_ERROR(stream, "no malloc support"); +#else + pb_type_t type; + pb_decoder_t func; + + type = iter->pos->type; + func = PB_DECODERS[PB_LTYPE(type)]; + + switch (PB_HTYPE(type)) + { + case PB_HTYPE_REQUIRED: + case PB_HTYPE_OPTIONAL: + if (!allocate_field(stream, iter, 1)) + return false; + return func(stream, iter->pos, iter->pData); + + case PB_HTYPE_REPEATED: + if (wire_type == PB_WT_STRING + && PB_LTYPE(type) <= PB_LTYPE_LAST_PACKABLE) + { + /* Packed array, multiple items come in at once. */ + bool status = true; + size_t *size = (size_t*)iter->pSize; + size_t allocated_size = *size; + void *pItem; + pb_istream_t substream; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + while (substream.bytes_left) + { + if (*size + 1 > allocated_size) + { + /* Allocate more storage. This tries to guess the + * number of remaining entries. */ + allocated_size += substream.bytes_left / iter->pos->data_size; + if (*size + 1 > allocated_size) + allocated_size++; /* Division gave zero. */ + + if (!allocate_field(&substream, iter, allocated_size)) + { + status = false; + break; + } + + /* Decode the array entry */ + pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); + if (!func(&substream, iter->pos, pItem)) + { + status = false; + break; + } + (*size)++; + } + } + pb_close_string_substream(stream, &substream); + + return status; + } + else + { + /* Normal repeated field, i.e. only one item at a time. */ + size_t *size = (size_t*)iter->pSize; + void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); + + if (!allocate_field(stream, iter, *size + 1)) + return false; + + + (*size)++; + return func(stream, iter->pos, pItem); + } + + default: + PB_RETURN_ERROR(stream, "invalid field type"); + } +#endif +} + static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) { pb_callback_t *pCallback = (pb_callback_t*)iter->pData; @@ -519,6 +634,9 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t case PB_ATYPE_STATIC: return decode_static_field(stream, wire_type, iter); + case PB_ATYPE_POINTER: + return decode_pointer_field(stream, wire_type, iter); + case PB_ATYPE_CALLBACK: return decode_callback_field(stream, wire_type, iter); @@ -597,45 +715,60 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str pb_field_iterator_t iter; pb_field_init(&iter, fields, dest_struct); - /* Initialize size/has fields and apply default values */ do { pb_type_t type; type = iter.pos->type; + /* Avoid crash on empty message types (zero fields) */ if (iter.pos->tag == 0) continue; if (PB_ATYPE(type) == PB_ATYPE_STATIC) { - /* Initialize the size field for optional/repeated fields to 0. */ if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL) { + /* Set has_field to false. Still initialize the optional field + * itself also. */ *(bool*)iter.pSize = false; } else if (PB_HTYPE(type) == PB_HTYPE_REPEATED) { + /* Set array count to 0, no need to initialize contents. */ *(size_t*)iter.pSize = 0; - continue; /* Array is empty, no need to initialize contents */ + continue; } - /* Initialize field contents to default value */ if (PB_LTYPE(iter.pos->type) == PB_LTYPE_SUBMESSAGE) { + /* Initialize submessage to defaults */ pb_message_set_to_defaults((const pb_field_t *) iter.pos->ptr, iter.pData); } else if (iter.pos->ptr != NULL) { + /* Initialize to default value */ memcpy(iter.pData, iter.pos->ptr, iter.pos->data_size); } else { + /* Initialize to zeros */ memset(iter.pData, 0, iter.pos->data_size); } } + else if (PB_ATYPE(type) == PB_ATYPE_POINTER) + { + /* Initialize the pointer to NULL. */ + *(void**)iter.pData = NULL; + + /* Initialize array count to 0. */ + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + { + *(size_t*)iter.pSize = 0; + } + } else if (PB_ATYPE(type) == PB_ATYPE_CALLBACK) { - continue; /* Don't overwrite callback */ + /* Don't overwrite callback */ } } while (pb_field_next(&iter)); } diff --git a/tests/alltypes_pointer/SConscript b/tests/alltypes_pointer/SConscript index b577ca24..f0103baa 100644 --- a/tests/alltypes_pointer/SConscript +++ b/tests/alltypes_pointer/SConscript @@ -3,18 +3,27 @@ Import("env") +# We need our own pb_decode.o for the malloc support +strict = env.Clone() +strict.Append(CFLAGS = strict['CORECFLAGS']) +strict.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1}); +strict.Object("pb_decode_with_malloc.o", "$NANOPB/pb_decode.c") + c = Copy("$TARGET", "$SOURCE") env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) env.NanopbProto(["alltypes", "alltypes.options"]) enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) +dec = env.Program(["decode_alltypes_pointer.c", "alltypes.pb.c", "pb_decode_with_malloc.o"]) refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX" # Encode and compare results env.RunTest(enc) -env.RunTest("decode_alltypes.output", [refdec, "encode_alltypes_pointer.output"]) +env.RunTest("decode_alltypes.output", [dec, "encode_alltypes_pointer.output"]) +env.RunTest("decode_alltypes_ref.output", [refdec, "encode_alltypes_pointer.output"]) env.Compare(["encode_alltypes_pointer.output", "$BUILD/alltypes/encode_alltypes.output"]) +env.Compare(["encode_alltypes_pointer_ref.output", "$BUILD/alltypes/encode_alltypes.output"]) # Do the same thing with the optional fields present env.RunTest("optionals.output", enc, ARGS = ['1']) diff --git a/tests/alltypes_pointer/decode_alltypes_pointer.c b/tests/alltypes_pointer/decode_alltypes_pointer.c new file mode 100644 index 00000000..32e34c58 --- /dev/null +++ b/tests/alltypes_pointer/decode_alltypes_pointer.c @@ -0,0 +1,210 @@ +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "test_helpers.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed.\n"); \ + return false; \ + } + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_alltypes(pb_istream_t *stream, int mode) +{ + AllTypes alltypes; + + /* Fill with garbage to better detect initialization errors */ + memset(&alltypes, 0xAA, sizeof(alltypes)); + + if (!pb_decode(stream, AllTypes_fields, &alltypes)) + return false; + + TEST(*alltypes.req_int32 == -1001); + TEST(*alltypes.req_int64 == -1002); + TEST(*alltypes.req_uint32 == 1003); + TEST(*alltypes.req_uint64 == 1004); + TEST(*alltypes.req_sint32 == -1005); + TEST(*alltypes.req_sint64 == -1006); + TEST(*alltypes.req_bool == true); + + TEST(*alltypes.req_fixed32 == 1008); + TEST(*alltypes.req_sfixed32 == -1009); + TEST(*alltypes.req_float == 1010.0f); + + TEST(*alltypes.req_fixed64 == 1011); + TEST(*alltypes.req_sfixed64 == -1012); + TEST(*alltypes.req_double == 1013.0f); + + TEST(strcmp(alltypes.req_string, "1014") == 0); + TEST(alltypes.req_bytes->size == 4); + TEST(memcmp(alltypes.req_bytes->bytes, "1015", 4) == 0); + TEST(strcmp(alltypes.req_submsg->substuff1, "1016") == 0); + TEST(*alltypes.req_submsg->substuff2 == 1016); + TEST(*alltypes.req_submsg->substuff3 == 3); + TEST(*alltypes.req_enum == MyEnum_Truth); + +#if 0 + TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); + TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); + TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); + TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); + TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); + TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0); + TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); + + TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); + TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); + TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); + + TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); + TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0); + TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); + + TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); + TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); + TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); + + TEST(alltypes.rep_submsg_count == 5); + TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); + TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); + TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3); + + TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); + TEST(alltypes.rep_emptymsg_count == 5); + + if (mode == 0) + { + /* Expect default values */ + TEST(alltypes.has_opt_int32 == false); + TEST(alltypes.opt_int32 == 4041); + TEST(alltypes.has_opt_int64 == false); + TEST(alltypes.opt_int64 == 4042); + TEST(alltypes.has_opt_uint32 == false); + TEST(alltypes.opt_uint32 == 4043); + TEST(alltypes.has_opt_uint64 == false); + TEST(alltypes.opt_uint64 == 4044); + TEST(alltypes.has_opt_sint32 == false); + TEST(alltypes.opt_sint32 == 4045); + TEST(alltypes.has_opt_sint64 == false); + TEST(alltypes.opt_sint64 == 4046); + TEST(alltypes.has_opt_bool == false); + TEST(alltypes.opt_bool == false); + + TEST(alltypes.has_opt_fixed32 == false); + TEST(alltypes.opt_fixed32 == 4048); + TEST(alltypes.has_opt_sfixed32 == false); + TEST(alltypes.opt_sfixed32 == 4049); + TEST(alltypes.has_opt_float == false); + TEST(alltypes.opt_float == 4050.0f); + + TEST(alltypes.has_opt_fixed64 == false); + TEST(alltypes.opt_fixed64 == 4051); + TEST(alltypes.has_opt_sfixed64 == false); + TEST(alltypes.opt_sfixed64 == 4052); + TEST(alltypes.has_opt_double == false); + TEST(alltypes.opt_double == 4053.0); + + TEST(alltypes.has_opt_string == false); + TEST(strcmp(alltypes.opt_string, "4054") == 0); + TEST(alltypes.has_opt_bytes == false); + TEST(alltypes.opt_bytes.size == 4); + TEST(memcmp(alltypes.opt_bytes.bytes, "4055", 4) == 0); + TEST(alltypes.has_opt_submsg == false); + TEST(strcmp(alltypes.opt_submsg.substuff1, "1") == 0); + TEST(alltypes.opt_submsg.substuff2 == 2); + TEST(alltypes.opt_submsg.substuff3 == 3); + TEST(alltypes.has_opt_enum == false); + TEST(alltypes.opt_enum == MyEnum_Second); + TEST(alltypes.has_opt_emptymsg == false); + } + else + { + /* Expect filled-in values */ + TEST(alltypes.has_opt_int32 == true); + TEST(alltypes.opt_int32 == 3041); + TEST(alltypes.has_opt_int64 == true); + TEST(alltypes.opt_int64 == 3042); + TEST(alltypes.has_opt_uint32 == true); + TEST(alltypes.opt_uint32 == 3043); + TEST(alltypes.has_opt_uint64 == true); + TEST(alltypes.opt_uint64 == 3044); + TEST(alltypes.has_opt_sint32 == true); + TEST(alltypes.opt_sint32 == 3045); + TEST(alltypes.has_opt_sint64 == true); + TEST(alltypes.opt_sint64 == 3046); + TEST(alltypes.has_opt_bool == true); + TEST(alltypes.opt_bool == true); + + TEST(alltypes.has_opt_fixed32 == true); + TEST(alltypes.opt_fixed32 == 3048); + TEST(alltypes.has_opt_sfixed32 == true); + TEST(alltypes.opt_sfixed32 == 3049); + TEST(alltypes.has_opt_float == true); + TEST(alltypes.opt_float == 3050.0f); + + TEST(alltypes.has_opt_fixed64 == true); + TEST(alltypes.opt_fixed64 == 3051); + TEST(alltypes.has_opt_sfixed64 == true); + TEST(alltypes.opt_sfixed64 == 3052); + TEST(alltypes.has_opt_double == true); + TEST(alltypes.opt_double == 3053.0); + + TEST(alltypes.has_opt_string == true); + TEST(strcmp(alltypes.opt_string, "3054") == 0); + TEST(alltypes.has_opt_bytes == true); + TEST(alltypes.opt_bytes.size == 4); + TEST(memcmp(alltypes.opt_bytes.bytes, "3055", 4) == 0); + TEST(alltypes.has_opt_submsg == true); + TEST(strcmp(alltypes.opt_submsg.substuff1, "3056") == 0); + TEST(alltypes.opt_submsg.substuff2 == 3056); + TEST(alltypes.opt_submsg.substuff3 == 3); + TEST(alltypes.has_opt_enum == true); + TEST(alltypes.opt_enum == MyEnum_Truth); + TEST(alltypes.has_opt_emptymsg == true); + } + + TEST(alltypes.req_limits.int32_min == INT32_MIN); + TEST(alltypes.req_limits.int32_max == INT32_MAX); + TEST(alltypes.req_limits.uint32_min == 0); + TEST(alltypes.req_limits.uint32_max == UINT32_MAX); + TEST(alltypes.req_limits.int64_min == INT64_MIN); + TEST(alltypes.req_limits.int64_max == INT64_MAX); + TEST(alltypes.req_limits.uint64_min == 0); + TEST(alltypes.req_limits.uint64_max == UINT64_MAX); + TEST(alltypes.req_limits.enum_min == HugeEnum_Negative); + TEST(alltypes.req_limits.enum_max == HugeEnum_Positive); + + TEST(alltypes.end == 1099); +#endif + + return true; +} + +int main(int argc, char **argv) +{ + uint8_t buffer[1024]; + size_t count; + pb_istream_t stream; + + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Read the data into buffer */ + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_alltypes(&stream, mode)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} -- cgit 1.2.3-korg From 48ac4613724ef0bfb7b0db61871127575b1d9002 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 25 Feb 2014 19:58:11 +0200 Subject: Bugfixes for dynamic allocation --- pb_decode.c | 52 +++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 4411e26a..30c36b36 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -359,6 +359,10 @@ static bool pb_field_next(pb_field_iterator_t *iter) { prev_size *= iter->pos->array_size; } + else if (PB_ATYPE(iter->pos->type) == PB_ATYPE_POINTER) + { + prev_size = sizeof(void*); + } if (iter->pos->tag == 0) return false; /* Only happens with empty message types */ @@ -512,9 +516,17 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ { case PB_HTYPE_REQUIRED: case PB_HTYPE_OPTIONAL: - if (!allocate_field(stream, iter, 1)) - return false; - return func(stream, iter->pos, iter->pData); + if (PB_LTYPE(type) == PB_LTYPE_STRING) + { + return func(stream, iter->pos, iter->pData); + } + else + { + if (!allocate_field(stream, iter, 1)) + return false; + + return func(stream, iter->pos, *(void**)iter->pData); + } case PB_HTYPE_REPEATED: if (wire_type == PB_WT_STRING @@ -545,16 +557,16 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ status = false; break; } + } - /* Decode the array entry */ - pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); - if (!func(&substream, iter->pos, pItem)) - { - status = false; - break; - } - (*size)++; + /* Decode the array entry */ + pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); + if (!func(&substream, iter->pos, pItem)) + { + status = false; + break; } + (*size)++; } pb_close_string_substream(stream, &substream); @@ -1036,8 +1048,22 @@ bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, vo return false; /* Check length, noting the null terminator */ - if (size + 1 > field->data_size) - PB_RETURN_ERROR(stream, "string overflow"); + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + { +#ifndef PB_ENABLE_MALLOC + PB_RETURN_ERROR(stream, "no malloc support"); +#else + *(void**)dest = realloc(*(void**)dest, size + 1); + if (*(void**)dest == NULL) + PB_RETURN_ERROR(stream, "out of memory"); + dest = *(void**)dest; +#endif + } + else + { + if (size + 1 > field->data_size) + PB_RETURN_ERROR(stream, "string overflow"); + } status = pb_read(stream, (uint8_t*)dest, size); *((uint8_t*)dest + size) = 0; -- cgit 1.2.3-korg From bf61d2337b4107b2c37c35bb41c7b809d8f3feb9 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 10 Mar 2014 18:19:38 +0200 Subject: More fixes for dynamic allocation --- pb_decode.c | 106 ++++++++++++++--------- tests/alltypes_pointer/SConscript | 1 - tests/alltypes_pointer/decode_alltypes_pointer.c | 52 ++++++----- 3 files changed, 93 insertions(+), 66 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 30c36b36..65e22af5 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -472,29 +472,19 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t #ifdef PB_ENABLE_MALLOC /* Allocate storage for the field and store the pointer at iter->pData. * array_size is the number of entries to reserve in an array. */ -static bool checkreturn allocate_field(pb_istream_t *stream, pb_field_iterator_t *iter, size_t array_size) +static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size) { - void *ptr = *(void**)iter->pData; - size_t size = array_size * iter->pos->data_size; + void *ptr = *(void**)pData; + size_t size = array_size * data_size; + /* Allocate new or expand previous allocation */ + /* Note: on failure the old pointer will remain in the structure, + * the message must be freed by caller also on error return. */ + ptr = realloc(ptr, size); if (ptr == NULL) - { - /* First allocation */ - ptr = malloc(size); - if (ptr == NULL) - PB_RETURN_ERROR(stream, "malloc failed"); - } - else - { - /* Expand previous allocation */ - /* Note: on failure the old pointer will remain in the structure, - * the message must be freed by caller also on error return. */ - ptr = realloc(ptr, size); - if (ptr == NULL) - PB_RETURN_ERROR(stream, "realloc failed"); - } + PB_RETURN_ERROR(stream, "realloc failed"); - *(void**)iter->pData = ptr; + *(void**)pData = ptr; return true; } #endif @@ -522,7 +512,7 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ } else { - if (!allocate_field(stream, iter, 1)) + if (!allocate_field(stream, iter->pData, iter->pos->data_size, 1)) return false; return func(stream, iter->pos, *(void**)iter->pData); @@ -547,12 +537,11 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ if (*size + 1 > allocated_size) { /* Allocate more storage. This tries to guess the - * number of remaining entries. */ - allocated_size += substream.bytes_left / iter->pos->data_size; - if (*size + 1 > allocated_size) - allocated_size++; /* Division gave zero. */ + * number of remaining entries. Round the division + * upwards. */ + allocated_size += (substream.bytes_left - 1) / iter->pos->data_size + 1; - if (!allocate_field(&substream, iter, allocated_size)) + if (!allocate_field(&substream, iter->pData, iter->pos->data_size, allocated_size)) { status = false; break; @@ -560,7 +549,7 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ } /* Decode the array entry */ - pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); + pItem = *(uint8_t**)iter->pData + iter->pos->data_size * (*size); if (!func(&substream, iter->pos, pItem)) { status = false; @@ -576,11 +565,26 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ { /* Normal repeated field, i.e. only one item at a time. */ size_t *size = (size_t*)iter->pSize; - void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); + void *pItem; - if (!allocate_field(stream, iter, *size + 1)) + if (!allocate_field(stream, iter->pData, iter->pos->data_size, *size + 1)) return false; + pItem = *(uint8_t**)iter->pData + iter->pos->data_size * (*size); + + /* Clear the new item in case it contains a pointer, or is a submessage. */ + if (PB_LTYPE(type) == PB_LTYPE_STRING) + { + *(char**)pItem = NULL; + } + else if (PB_LTYPE(type) == PB_LTYPE_BYTES) + { + memset(pItem, 0, iter->pos->data_size); + } + else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) + { + pb_message_set_to_defaults((const pb_field_t *) iter->pos->ptr, pItem); + } (*size)++; return func(stream, iter->pos, pItem); @@ -1026,42 +1030,62 @@ bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, v bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) { - pb_bytes_array_t *x = (pb_bytes_array_t*)dest; + uint32_t size; + size_t alloc_size; - uint32_t temp; - if (!pb_decode_varint32(stream, &temp)) + if (!pb_decode_varint32(stream, &size)) return false; - x->size = temp; - /* Check length, noting the space taken by the size_t header. */ - if (x->size > field->data_size - offsetof(pb_bytes_array_t, bytes)) - PB_RETURN_ERROR(stream, "bytes overflow"); + /* Space for the size_t header */ + alloc_size = size + offsetof(pb_bytes_array_t, bytes); - return pb_read(stream, x->bytes, x->size); + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + { +#ifndef PB_ENABLE_MALLOC + PB_RETURN_ERROR(stream, "no malloc support"); +#else + pb_bytes_ptr_t *bdest = (pb_bytes_ptr_t*)dest; + if (!allocate_field(stream, &bdest->bytes, alloc_size, 1)) + return false; + + bdest->size = size; + return pb_read(stream, bdest->bytes, size); +#endif + } + else + { + pb_bytes_array_t* bdest = (pb_bytes_array_t*)dest; + if (alloc_size > field->data_size) + PB_RETURN_ERROR(stream, "bytes overflow"); + bdest->size = size; + return pb_read(stream, bdest->bytes, size); + } } bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint32_t size; + size_t alloc_size; bool status; if (!pb_decode_varint32(stream, &size)) return false; - /* Check length, noting the null terminator */ + /* Space for null terminator */ + alloc_size = size + 1; + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { #ifndef PB_ENABLE_MALLOC PB_RETURN_ERROR(stream, "no malloc support"); #else - *(void**)dest = realloc(*(void**)dest, size + 1); - if (*(void**)dest == NULL) - PB_RETURN_ERROR(stream, "out of memory"); + if (!allocate_field(stream, dest, alloc_size, 1)) + return false; dest = *(void**)dest; #endif } else { - if (size + 1 > field->data_size) + if (alloc_size > field->data_size) PB_RETURN_ERROR(stream, "string overflow"); } diff --git a/tests/alltypes_pointer/SConscript b/tests/alltypes_pointer/SConscript index f0103baa..05b4e52d 100644 --- a/tests/alltypes_pointer/SConscript +++ b/tests/alltypes_pointer/SConscript @@ -23,7 +23,6 @@ env.RunTest(enc) env.RunTest("decode_alltypes.output", [dec, "encode_alltypes_pointer.output"]) env.RunTest("decode_alltypes_ref.output", [refdec, "encode_alltypes_pointer.output"]) env.Compare(["encode_alltypes_pointer.output", "$BUILD/alltypes/encode_alltypes.output"]) -env.Compare(["encode_alltypes_pointer_ref.output", "$BUILD/alltypes/encode_alltypes.output"]) # Do the same thing with the optional fields present env.RunTest("optionals.output", enc, ARGS = ['1']) diff --git a/tests/alltypes_pointer/decode_alltypes_pointer.c b/tests/alltypes_pointer/decode_alltypes_pointer.c index 32e34c58..3db48114 100644 --- a/tests/alltypes_pointer/decode_alltypes_pointer.c +++ b/tests/alltypes_pointer/decode_alltypes_pointer.c @@ -7,13 +7,14 @@ #define TEST(x) if (!(x)) { \ printf("Test " #x " failed.\n"); \ - return false; \ + status = false; \ } /* This function is called once from main(), it handles the decoding and checks the fields. */ bool check_alltypes(pb_istream_t *stream, int mode) { + bool status = true; AllTypes alltypes; /* Fill with garbage to better detect initialization errors */ @@ -22,28 +23,31 @@ bool check_alltypes(pb_istream_t *stream, int mode) if (!pb_decode(stream, AllTypes_fields, &alltypes)) return false; - TEST(*alltypes.req_int32 == -1001); - TEST(*alltypes.req_int64 == -1002); - TEST(*alltypes.req_uint32 == 1003); - TEST(*alltypes.req_uint64 == 1004); - TEST(*alltypes.req_sint32 == -1005); - TEST(*alltypes.req_sint64 == -1006); - TEST(*alltypes.req_bool == true); - - TEST(*alltypes.req_fixed32 == 1008); - TEST(*alltypes.req_sfixed32 == -1009); - TEST(*alltypes.req_float == 1010.0f); - - TEST(*alltypes.req_fixed64 == 1011); - TEST(*alltypes.req_sfixed64 == -1012); - TEST(*alltypes.req_double == 1013.0f); - - TEST(strcmp(alltypes.req_string, "1014") == 0); - TEST(alltypes.req_bytes->size == 4); - TEST(memcmp(alltypes.req_bytes->bytes, "1015", 4) == 0); - TEST(strcmp(alltypes.req_submsg->substuff1, "1016") == 0); - TEST(*alltypes.req_submsg->substuff2 == 1016); - TEST(*alltypes.req_submsg->substuff3 == 3); + TEST(alltypes.req_int32 && *alltypes.req_int32 == -1001); + TEST(alltypes.req_int64 && *alltypes.req_int64 == -1002); + TEST(alltypes.req_uint32 && *alltypes.req_uint32 == 1003); + TEST(alltypes.req_uint64 && *alltypes.req_uint64 == 1004); + TEST(alltypes.req_sint32 && *alltypes.req_sint32 == -1005); + TEST(alltypes.req_sint64 && *alltypes.req_sint64 == -1006); + TEST(alltypes.req_bool && *alltypes.req_bool == true); + + TEST(alltypes.req_fixed32 && *alltypes.req_fixed32 == 1008); + TEST(alltypes.req_sfixed32 && *alltypes.req_sfixed32 == -1009); + TEST(alltypes.req_float && *alltypes.req_float == 1010.0f); + + TEST(alltypes.req_fixed64 && *alltypes.req_fixed64 == 1011); + TEST(alltypes.req_sfixed64 && *alltypes.req_sfixed64 == -1012); + TEST(alltypes.req_double && *alltypes.req_double == 1013.0f); + + TEST(alltypes.req_string && strcmp(alltypes.req_string, "1014") == 0); + TEST(alltypes.req_bytes && alltypes.req_bytes->size == 4); + TEST(alltypes.req_bytes && alltypes.req_bytes->bytes + && memcmp(alltypes.req_bytes->bytes, "1015", 4) == 0); + TEST(alltypes.req_submsg && alltypes.req_submsg->substuff1 + && strcmp(alltypes.req_submsg->substuff1, "1016") == 0); + TEST(alltypes.req_submsg && alltypes.req_submsg->substuff2 + && *alltypes.req_submsg->substuff2 == 1016); + /* TEST(*alltypes.req_submsg->substuff3 == 3); Default values are not currently supported for pointer fields */ TEST(*alltypes.req_enum == MyEnum_Truth); #if 0 @@ -180,7 +184,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.end == 1099); #endif - return true; + return status; } int main(int argc, char **argv) -- cgit 1.2.3-korg From 9c196b89ba04733529edfe970af6307a34de1662 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 12 Mar 2014 21:08:35 +0200 Subject: Add pb_release() function --- pb_decode.c | 114 +++++++++++++++++++---- pb_decode.h | 8 ++ tests/SConstruct | 9 +- tests/alltypes_pointer/SConscript | 14 ++- tests/alltypes_pointer/decode_alltypes_pointer.c | 78 ++++++++++++---- 5 files changed, 185 insertions(+), 38 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 65e22af5..68497406 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -487,6 +487,23 @@ static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t *(void**)pData = ptr; return true; } + +/* Clear a newly allocated item in case it contains a pointer, or is a submessage. */ +static void initialize_pointer_field(void *pItem, pb_field_iterator_t *iter) +{ + if (PB_LTYPE(iter->pos->type) == PB_LTYPE_STRING) + { + *(char**)pItem = NULL; + } + else if (PB_LTYPE(iter->pos->type) == PB_LTYPE_BYTES) + { + memset(pItem, 0, iter->pos->data_size); + } + else if (PB_LTYPE(iter->pos->type) == PB_LTYPE_SUBMESSAGE) + { + pb_message_set_to_defaults((const pb_field_t *) iter->pos->ptr, pItem); + } +} #endif static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) @@ -515,6 +532,7 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ if (!allocate_field(stream, iter->pData, iter->pos->data_size, 1)) return false; + initialize_pointer_field(*(void**)iter->pData, iter); return func(stream, iter->pos, *(void**)iter->pData); } @@ -550,6 +568,7 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ /* Decode the array entry */ pItem = *(uint8_t**)iter->pData + iter->pos->data_size * (*size); + initialize_pointer_field(pItem, iter); if (!func(&substream, iter->pos, pItem)) { status = false; @@ -567,26 +586,12 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ size_t *size = (size_t*)iter->pSize; void *pItem; - if (!allocate_field(stream, iter->pData, iter->pos->data_size, *size + 1)) + (*size)++; + if (!allocate_field(stream, iter->pData, iter->pos->data_size, *size)) return false; - pItem = *(uint8_t**)iter->pData + iter->pos->data_size * (*size); - - /* Clear the new item in case it contains a pointer, or is a submessage. */ - if (PB_LTYPE(type) == PB_LTYPE_STRING) - { - *(char**)pItem = NULL; - } - else if (PB_LTYPE(type) == PB_LTYPE_BYTES) - { - memset(pItem, 0, iter->pos->data_size); - } - else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) - { - pb_message_set_to_defaults((const pb_field_t *) iter->pos->ptr, pItem); - } - - (*size)++; + pItem = *(uint8_t**)iter->pData + iter->pos->data_size * (*size - 1); + initialize_pointer_field(pItem, iter); return func(stream, iter->pos, pItem); } @@ -908,6 +913,79 @@ bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void * return status; } +#ifdef PB_ENABLE_MALLOC +void pb_release(const pb_field_t fields[], void *dest_struct) +{ + pb_field_iterator_t iter; + pb_field_init(&iter, fields, dest_struct); + + do + { + pb_type_t type; + type = iter.pos->type; + + /* Avoid crash on empty message types (zero fields) */ + if (iter.pos->tag == 0) + continue; + + if (PB_ATYPE(type) == PB_ATYPE_POINTER) + { + if (PB_LTYPE(type) == PB_LTYPE_STRING && + PB_HTYPE(type) == PB_HTYPE_REPEATED) + { + /* Release entries in repeated string array */ + void **pItem = *(void***)iter.pData; + size_t count = *(size_t*)iter.pSize; + while (count--) + { + free(*pItem); + *pItem++ = NULL; + } + } + else if (PB_LTYPE(type) == PB_LTYPE_BYTES) + { + /* Release entries in repeated bytes array */ + pb_bytes_ptr_t *pItem = *(pb_bytes_ptr_t**)iter.pData; + size_t count = (pItem ? 1 : 0); + + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + { + count = *(size_t*)iter.pSize; + } + + while (count--) + { + free(pItem->bytes); + pItem->bytes = NULL; + pItem++; + } + } + else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) + { + /* Release fields in submessages */ + void *pItem = *(void**)iter.pData; + size_t count = (pItem ? 1 : 0); + + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + { + count = *(size_t*)iter.pSize; + } + + while (count--) + { + pb_release((const pb_field_t*)iter.pos->ptr, pItem); + pItem = (uint8_t*)pItem + iter.pos->data_size; + } + } + + /* Release main item */ + free(*(void**)iter.pData); + *(void**)iter.pData = NULL; + } + } while (pb_field_next(&iter)); +} +#endif + /* Field decoders */ bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest) diff --git a/pb_decode.h b/pb_decode.h index f71b5f1b..2e1da5a6 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -82,6 +82,14 @@ bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *des */ bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); +#ifdef PB_ENABLE_MALLOC +/* Release any allocated pointer fields. If you use dynamic allocation, you should + * call this for any decoded message when you are done with it. You also need to + * free messages even if pb_decode() returned with error. + */ +void pb_release(const pb_field_t fields[], void *dest_struct); +#endif + /************************************** * Functions for manipulating streams * diff --git a/tests/SConstruct b/tests/SConstruct index abc6e7cb..eedb694e 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -58,7 +58,12 @@ if not env.GetOption('clean'): if stdint: conf.env.Append(CPPDEFINES = {'HAVE_STDINT_H': 1}) if stddef: conf.env.Append(CPPDEFINES = {'HAVE_STDDEF_H': 1}) if string: conf.env.Append(CPPDEFINES = {'HAVE_STRING_H': 1}) - + + # Check if we have mallinfo for memory leak tests + mallinfo = conf.CheckFunc('mallinfo', '#include \n') + if mallinfo: + conf.env.Append(CPPDEFINES = {'HAVE_MALLINFO': 1}) + # Check if we can use pkg-config to find protobuf include path status, output = conf.TryAction('pkg-config protobuf --variable=includedir > $TARGET') if status: @@ -70,7 +75,7 @@ if not env.GetOption('clean'): if 'gcc' in env['CC']: if conf.CheckLib('mudflap'): conf.env.Append(CCFLAGS = '-fmudflap') - conf.env.Append(LINKFLAGS = '-lmudflap -fmudflap') + conf.env.Append(LINKFLAGS = '-fmudflap') # Check if we can use extra strict warning flags (only with GCC) extra = '-Wcast-qual -Wlogical-op -Wconversion' diff --git a/tests/alltypes_pointer/SConscript b/tests/alltypes_pointer/SConscript index 05b4e52d..45985ff5 100644 --- a/tests/alltypes_pointer/SConscript +++ b/tests/alltypes_pointer/SConscript @@ -4,16 +4,26 @@ Import("env") # We need our own pb_decode.o for the malloc support +env = env.Clone() +env.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1}); + +# Disable libmudflap, because it will confuse e.g. valgrind +# and other memory leak detection tools. +if '-fmudflap' in env["CCFLAGS"]: + env["CCFLAGS"].remove("-fmudflap") + env["LINKFLAGS"].remove("-fmudflap") + env["LIBS"].remove("mudflap") + strict = env.Clone() strict.Append(CFLAGS = strict['CORECFLAGS']) -strict.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1}); strict.Object("pb_decode_with_malloc.o", "$NANOPB/pb_decode.c") +strict.Object("pb_encode_with_malloc.o", "$NANOPB/pb_encode.c") c = Copy("$TARGET", "$SOURCE") env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) env.NanopbProto(["alltypes", "alltypes.options"]) -enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) +enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "pb_encode_with_malloc.o"]) dec = env.Program(["decode_alltypes_pointer.c", "alltypes.pb.c", "pb_decode_with_malloc.o"]) refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX" diff --git a/tests/alltypes_pointer/decode_alltypes_pointer.c b/tests/alltypes_pointer/decode_alltypes_pointer.c index 3db48114..29495ac4 100644 --- a/tests/alltypes_pointer/decode_alltypes_pointer.c +++ b/tests/alltypes_pointer/decode_alltypes_pointer.c @@ -5,8 +5,12 @@ #include "alltypes.pb.h" #include "test_helpers.h" +#ifdef HAVE_MALLINFO +#include +#endif + #define TEST(x) if (!(x)) { \ - printf("Test " #x " failed.\n"); \ + fprintf(stderr, "Test " #x " failed.\n"); \ status = false; \ } @@ -21,7 +25,10 @@ bool check_alltypes(pb_istream_t *stream, int mode) memset(&alltypes, 0xAA, sizeof(alltypes)); if (!pb_decode(stream, AllTypes_fields, &alltypes)) + { + pb_release(AllTypes_fields, &alltypes); return false; + } TEST(alltypes.req_int32 && *alltypes.req_int32 == -1001); TEST(alltypes.req_int64 && *alltypes.req_int64 == -1002); @@ -184,31 +191,70 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.end == 1099); #endif + pb_release(AllTypes_fields, &alltypes); + return status; } int main(int argc, char **argv) { - uint8_t buffer[1024]; - size_t count; - pb_istream_t stream; - - /* Whether to expect the optional values or the default values. */ - int mode = (argc > 1) ? atoi(argv[1]) : 0; + bool status; + int orig_allocations; - /* Read the data into buffer */ - SET_BINARY_MODE(stdin); - count = fread(buffer, 1, sizeof(buffer), stdin); +#ifdef HAVE_MALLINFO + /* Dynamic library loader etc. may have some malloc()ed memory also. */ + { + struct mallinfo m = mallinfo(); + orig_allocations = m.uordblks; + } +#endif + + { + uint8_t buffer[1024]; + size_t count; + pb_istream_t stream; + + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Read the data into buffer */ + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + stream = pb_istream_from_buffer(buffer, count); + + /* Decode and verify the message */ + status = check_alltypes(&stream, mode); + + if (!status) + fprintf(stderr, "Parsing failed: %s\n", PB_GET_ERROR(&stream)); + } - /* Construct a pb_istream_t for reading from the buffer */ - stream = pb_istream_from_buffer(buffer, count); +#ifdef HAVE_MALLINFO + /* Check for memory leaks */ + { + struct mallinfo m = mallinfo(); + int leak = m.uordblks - orig_allocations; + + if (leak > 0) + { + fprintf(stderr, "Memory leak: %d bytes\n", leak); + return 1; + } + else + { + fprintf(stderr, "Ok, no memory leaks\n"); + } + } +#endif - /* Decode and print out the stuff */ - if (!check_alltypes(&stream, mode)) + if (!status) { - printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); return 1; - } else { + } + else + { return 0; } } -- cgit 1.2.3-korg From 9be2cfe968b4223f9d416aecd483f3b999bbab71 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 15 Mar 2014 08:45:58 +0200 Subject: Get rid of pb_bytes_ptr_t, just allocate pb_bytes_array_t dynamically. This makes the internal logic much simpler, and also keeps the datatypes more similar between STATIC/POINTER cases. It will still be a bit cumbersome to use because of variable length array member. Macros PB_BYTES_ARRAY_T(n) and PB_BYTES_ARRAY_T_ALLOCSIZE(n) have been added to make life a bit easier. This has the drawback that it is no longer as easy to use externally allocated byte array as input for bytes field in pointer mode. However, this is still easy to do using callbacks, so it shouldn't be a large issue. --- generator/nanopb_generator.py | 6 +-- pb.h | 12 ++--- pb_decode.c | 58 +++++++----------------- pb_encode.c | 42 ++++++++++------- tests/alltypes_pointer/SConscript | 4 +- tests/alltypes_pointer/decode_alltypes_pointer.c | 3 +- tests/alltypes_pointer/encode_alltypes_pointer.c | 11 +++-- 7 files changed, 59 insertions(+), 77 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 0926db29..c32b26ad 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -246,7 +246,7 @@ class Field: self.ctype = self.struct_name + self.name + 't' self.enc_size = varint_max_size(self.max_size) + self.max_size elif self.allocation == 'POINTER': - self.ctype = 'pb_bytes_ptr_t' + self.ctype = 'pb_bytes_array_t' elif desc.type == FieldD.TYPE_MESSAGE: self.pbtype = 'MESSAGE' self.ctype = self.submsgname = names_from_type_name(desc.type_name) @@ -266,8 +266,8 @@ class Field: if self.pbtype == 'MESSAGE': # Use struct definition, so recursive submessages are possible result += ' struct _%s *%s;' % (self.ctype, self.name) - elif self.rules == 'REPEATED' and self.pbtype == 'STRING': - # String arrays need to be defined as pointers to pointers + elif self.rules == 'REPEATED' and self.pbtype in ['STRING', 'BYTES']: + # String/bytes arrays need to be defined as pointers to pointers result += ' %s **%s;' % (self.ctype, self.name) else: result += ' %s *%s;' % (self.ctype, self.name) diff --git a/pb.h b/pb.h index eb4f94e7..d6cb1d40 100644 --- a/pb.h +++ b/pb.h @@ -238,21 +238,15 @@ STATIC_ASSERT(sizeof(uint64_t) == 8, UINT64_T_WRONG_SIZE) * It has the number of bytes in the beginning, and after that an array. * Note that actual structs used will have a different length of bytes array. */ +#define PB_BYTES_ARRAY_T(n) struct { size_t size; uint8_t bytes[n]; } +#define PB_BYTES_ARRAY_T_ALLOCSIZE(n) ((size_t)n + offsetof(pb_bytes_array_t, bytes)) + struct _pb_bytes_array_t { size_t size; uint8_t bytes[1]; }; typedef struct _pb_bytes_array_t pb_bytes_array_t; -/* Same, except for pointer-type fields. There is no need to variable struct - * length in this case. - */ -struct _pb_bytes_ptr_t { - size_t size; - uint8_t *bytes; -}; -typedef struct _pb_bytes_ptr_t pb_bytes_ptr_t; - /* This structure is used for giving the callback function. * It is stored in the message structure and filled in by the method that * calls pb_decode. diff --git a/pb_decode.c b/pb_decode.c index 68497406..81febb9b 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -491,13 +491,10 @@ static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t /* Clear a newly allocated item in case it contains a pointer, or is a submessage. */ static void initialize_pointer_field(void *pItem, pb_field_iterator_t *iter) { - if (PB_LTYPE(iter->pos->type) == PB_LTYPE_STRING) + if (PB_LTYPE(iter->pos->type) == PB_LTYPE_STRING || + PB_LTYPE(iter->pos->type) == PB_LTYPE_BYTES) { - *(char**)pItem = NULL; - } - else if (PB_LTYPE(iter->pos->type) == PB_LTYPE_BYTES) - { - memset(pItem, 0, iter->pos->data_size); + *(void**)pItem = NULL; } else if (PB_LTYPE(iter->pos->type) == PB_LTYPE_SUBMESSAGE) { @@ -523,7 +520,8 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ { case PB_HTYPE_REQUIRED: case PB_HTYPE_OPTIONAL: - if (PB_LTYPE(type) == PB_LTYPE_STRING) + if (PB_LTYPE(type) == PB_LTYPE_STRING || + PB_LTYPE(type) == PB_LTYPE_BYTES) { return func(stream, iter->pos, iter->pData); } @@ -930,10 +928,11 @@ void pb_release(const pb_field_t fields[], void *dest_struct) if (PB_ATYPE(type) == PB_ATYPE_POINTER) { - if (PB_LTYPE(type) == PB_LTYPE_STRING && - PB_HTYPE(type) == PB_HTYPE_REPEATED) + if (PB_HTYPE(type) == PB_HTYPE_REPEATED && + (PB_LTYPE(type) == PB_LTYPE_STRING || + PB_LTYPE(type) == PB_LTYPE_BYTES)) { - /* Release entries in repeated string array */ + /* Release entries in repeated string or bytes array */ void **pItem = *(void***)iter.pData; size_t count = *(size_t*)iter.pSize; while (count--) @@ -942,24 +941,6 @@ void pb_release(const pb_field_t fields[], void *dest_struct) *pItem++ = NULL; } } - else if (PB_LTYPE(type) == PB_LTYPE_BYTES) - { - /* Release entries in repeated bytes array */ - pb_bytes_ptr_t *pItem = *(pb_bytes_ptr_t**)iter.pData; - size_t count = (pItem ? 1 : 0); - - if (PB_HTYPE(type) == PB_HTYPE_REPEATED) - { - count = *(size_t*)iter.pSize; - } - - while (count--) - { - free(pItem->bytes); - pItem->bytes = NULL; - pItem++; - } - } else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) { /* Release fields in submessages */ @@ -1109,35 +1090,30 @@ bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, v bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint32_t size; - size_t alloc_size; + pb_bytes_array_t *bdest; if (!pb_decode_varint32(stream, &size)) return false; - /* Space for the size_t header */ - alloc_size = size + offsetof(pb_bytes_array_t, bytes); - if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { #ifndef PB_ENABLE_MALLOC PB_RETURN_ERROR(stream, "no malloc support"); #else - pb_bytes_ptr_t *bdest = (pb_bytes_ptr_t*)dest; - if (!allocate_field(stream, &bdest->bytes, alloc_size, 1)) + if (!allocate_field(stream, dest, PB_BYTES_ARRAY_T_ALLOCSIZE(size), 1)) return false; - - bdest->size = size; - return pb_read(stream, bdest->bytes, size); + bdest = *(pb_bytes_array_t**)dest; #endif } else { - pb_bytes_array_t* bdest = (pb_bytes_array_t*)dest; - if (alloc_size > field->data_size) + if (PB_BYTES_ARRAY_T_ALLOCSIZE(size) > field->data_size) PB_RETURN_ERROR(stream, "bytes overflow"); - bdest->size = size; - return pb_read(stream, bdest->bytes, size); + bdest = (pb_bytes_array_t*)dest; } + + bdest->size = size; + return pb_read(stream, bdest->bytes, size); } bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest) diff --git a/pb_encode.c b/pb_encode.c index c2d0e2c4..59e6f2a7 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -174,11 +174,12 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie return false; /* Normally the data is stored directly in the array entries, but - * for pointer-type string fields, the array entries are actually - * string pointers. So we have to dereference once more to get to - * the character data. */ + * for pointer-type string and bytes fields, the array entries are + * actually pointers themselves also. So we have to dereference once + * more to get to the actual data. */ if (PB_ATYPE(field->type) == PB_ATYPE_POINTER && - PB_LTYPE(field->type) == PB_LTYPE_STRING) + (PB_LTYPE(field->type) == PB_LTYPE_STRING || + PB_LTYPE(field->type) == PB_LTYPE_BYTES)) { if (!func(stream, field, *(const void* const*)p)) return false; @@ -603,19 +604,21 @@ bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, c bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src; + + if (src == NULL) { - const pb_bytes_ptr_t *bytes = (const pb_bytes_ptr_t*)src; - return pb_encode_string(stream, bytes->bytes, bytes->size); + /* Threat null pointer as an empty bytes field */ + return pb_encode_string(stream, NULL, 0); } - else + + if (PB_ATYPE(field->type) == PB_ATYPE_STATIC && + PB_BYTES_ARRAY_T_ALLOCSIZE(bytes->size) > field->data_size) { - const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src; - if (bytes->size + offsetof(pb_bytes_array_t, bytes) > field->data_size) - PB_RETURN_ERROR(stream, "bytes size exceeded"); - - return pb_encode_string(stream, bytes->bytes, bytes->size); + PB_RETURN_ERROR(stream, "bytes size exceeded"); } + + return pb_encode_string(stream, bytes->bytes, bytes->size); } bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src) @@ -628,10 +631,17 @@ bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, co if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) max_size = (size_t)-1; - while (size < max_size && *p != '\0') + if (src == NULL) { - size++; - p++; + size = 0; /* Threat null pointer as an empty string */ + } + else + { + while (size < max_size && *p != '\0') + { + size++; + p++; + } } return pb_encode_string(stream, (const uint8_t*)src, size); diff --git a/tests/alltypes_pointer/SConscript b/tests/alltypes_pointer/SConscript index 45985ff5..97e4267a 100644 --- a/tests/alltypes_pointer/SConscript +++ b/tests/alltypes_pointer/SConscript @@ -30,9 +30,11 @@ refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX" # Encode and compare results env.RunTest(enc) +env.Compare(["encode_alltypes_pointer.output", "$BUILD/alltypes/encode_alltypes.output"]) + +# Decode env.RunTest("decode_alltypes.output", [dec, "encode_alltypes_pointer.output"]) env.RunTest("decode_alltypes_ref.output", [refdec, "encode_alltypes_pointer.output"]) -env.Compare(["encode_alltypes_pointer.output", "$BUILD/alltypes/encode_alltypes.output"]) # Do the same thing with the optional fields present env.RunTest("optionals.output", enc, ARGS = ['1']) diff --git a/tests/alltypes_pointer/decode_alltypes_pointer.c b/tests/alltypes_pointer/decode_alltypes_pointer.c index 29495ac4..47d72685 100644 --- a/tests/alltypes_pointer/decode_alltypes_pointer.c +++ b/tests/alltypes_pointer/decode_alltypes_pointer.c @@ -48,8 +48,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.req_string && strcmp(alltypes.req_string, "1014") == 0); TEST(alltypes.req_bytes && alltypes.req_bytes->size == 4); - TEST(alltypes.req_bytes && alltypes.req_bytes->bytes - && memcmp(alltypes.req_bytes->bytes, "1015", 4) == 0); + TEST(alltypes.req_bytes && memcmp(&alltypes.req_bytes->bytes, "1015", 4) == 0); TEST(alltypes.req_submsg && alltypes.req_submsg->substuff1 && strcmp(alltypes.req_submsg->substuff1, "1016") == 0); TEST(alltypes.req_submsg && alltypes.req_submsg->substuff2 diff --git a/tests/alltypes_pointer/encode_alltypes_pointer.c b/tests/alltypes_pointer/encode_alltypes_pointer.c index 6484ced8..c128569b 100644 --- a/tests/alltypes_pointer/encode_alltypes_pointer.c +++ b/tests/alltypes_pointer/encode_alltypes_pointer.c @@ -27,7 +27,7 @@ int main(int argc, char **argv) int64_t req_sfixed64 = -1012; double req_double = 1013.0; char* req_string = "1014"; - pb_bytes_ptr_t req_bytes = {4, (uint8_t*)"1015"}; + PB_BYTES_ARRAY_T(4) req_bytes = {4, {'1', '0', '1', '5'}}; static int32_t req_substuff = 1016; SubMessage req_submsg = {"1016", &req_substuff}; MyEnum req_enum = MyEnum_Truth; @@ -50,7 +50,8 @@ int main(int argc, char **argv) int64_t rep_sfixed64[5] = {0, 0, 0, 0, -2012}; double rep_double[5] = {0, 0, 0, 0, 2013.0f}; char* rep_string[5] = {"", "", "", "", "2014"}; - pb_bytes_ptr_t rep_bytes[5] = {{0,0}, {0,0}, {0,0}, {0,0}, {4, (uint8_t*)"2015"}}; + static PB_BYTES_ARRAY_T(4) rep_bytes_4 = {4, {'2', '0', '1', '5'}}; + pb_bytes_array_t *rep_bytes[5]= {NULL, NULL, NULL, NULL, (pb_bytes_array_t*)&rep_bytes_4}; static int32_t rep_sub2zero = 0; static int32_t rep_substuff2 = 2016; static uint32_t rep_substuff3 = 2016; @@ -77,7 +78,7 @@ int main(int argc, char **argv) int64_t opt_sfixed64 = 3052; double opt_double = 3053.0; char* opt_string = "3054"; - pb_bytes_ptr_t opt_bytes = {4, (uint8_t*)"3055"}; + PB_BYTES_ARRAY_T(4) opt_bytes = {4, {'3', '0', '5', '5'}}; static int32_t opt_substuff = 3056; SubMessage opt_submsg = {"3056", &opt_substuff}; MyEnum opt_enum = MyEnum_Truth; @@ -117,7 +118,7 @@ int main(int argc, char **argv) alltypes.req_sfixed64 = &req_sfixed64; alltypes.req_double = &req_double; alltypes.req_string = req_string; - alltypes.req_bytes = &req_bytes; + alltypes.req_bytes = (pb_bytes_array_t*)&req_bytes; alltypes.req_submsg = &req_submsg; alltypes.req_enum = &req_enum; alltypes.req_emptymsg = &req_emptymsg; @@ -159,7 +160,7 @@ int main(int argc, char **argv) alltypes.opt_sfixed64 = &opt_sfixed64; alltypes.opt_double = &opt_double; alltypes.opt_string = opt_string; - alltypes.opt_bytes = &opt_bytes; + alltypes.opt_bytes = (pb_bytes_array_t*)&opt_bytes; alltypes.opt_submsg = &opt_submsg; alltypes.opt_enum = &opt_enum; alltypes.opt_emptymsg = &opt_emptymsg; -- cgit 1.2.3-korg From 108864963faf54762629a8bdf1f8bd614f0abd16 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 15 Mar 2014 09:39:27 +0200 Subject: Finish the alltypes_pointer testcase, use valgrind if available. --- tests/SConstruct | 5 - tests/alltypes_pointer/SConscript | 21 ++- tests/alltypes_pointer/decode_alltypes_pointer.c | 221 +++++++---------------- tests/site_scons/site_init.py | 11 +- 4 files changed, 90 insertions(+), 168 deletions(-) diff --git a/tests/SConstruct b/tests/SConstruct index eedb694e..8bf16ebe 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -59,11 +59,6 @@ if not env.GetOption('clean'): if stddef: conf.env.Append(CPPDEFINES = {'HAVE_STDDEF_H': 1}) if string: conf.env.Append(CPPDEFINES = {'HAVE_STRING_H': 1}) - # Check if we have mallinfo for memory leak tests - mallinfo = conf.CheckFunc('mallinfo', '#include \n') - if mallinfo: - conf.env.Append(CPPDEFINES = {'HAVE_MALLINFO': 1}) - # Check if we can use pkg-config to find protobuf include path status, output = conf.TryAction('pkg-config protobuf --variable=includedir > $TARGET') if status: diff --git a/tests/alltypes_pointer/SConscript b/tests/alltypes_pointer/SConscript index 97e4267a..e48d6aad 100644 --- a/tests/alltypes_pointer/SConscript +++ b/tests/alltypes_pointer/SConscript @@ -7,7 +7,7 @@ Import("env") env = env.Clone() env.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1}); -# Disable libmudflap, because it will confuse e.g. valgrind +# Disable libmudflap, because it will confuse valgrind # and other memory leak detection tools. if '-fmudflap' in env["CCFLAGS"]: env["CCFLAGS"].remove("-fmudflap") @@ -26,18 +26,23 @@ env.NanopbProto(["alltypes", "alltypes.options"]) enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "pb_encode_with_malloc.o"]) dec = env.Program(["decode_alltypes_pointer.c", "alltypes.pb.c", "pb_decode_with_malloc.o"]) -refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX" - -# Encode and compare results +# Encode and compare results to non-pointer alltypes test case env.RunTest(enc) env.Compare(["encode_alltypes_pointer.output", "$BUILD/alltypes/encode_alltypes.output"]) -# Decode -env.RunTest("decode_alltypes.output", [dec, "encode_alltypes_pointer.output"]) -env.RunTest("decode_alltypes_ref.output", [refdec, "encode_alltypes_pointer.output"]) +# Decode (under valgrind if available) +valgrind = env.WhereIs('valgrind') +kwargs = {} +if valgrind: + kwargs['COMMAND'] = valgrind + kwargs['ARGS'] = ["-q", dec[0].abspath] + +env.RunTest("decode_alltypes.output", [dec, "encode_alltypes_pointer.output"], **kwargs) # Do the same thing with the optional fields present env.RunTest("optionals.output", enc, ARGS = ['1']) -env.RunTest("optionals.decout", [refdec, "optionals.output"], ARGS = ['1']) env.Compare(["optionals.output", "$BUILD/alltypes/optionals.output"]) +kwargs['ARGS'] = kwargs.get('ARGS', []) + ['1'] +env.RunTest("optionals.decout", [dec, "optionals.output"], **kwargs) + diff --git a/tests/alltypes_pointer/decode_alltypes_pointer.c b/tests/alltypes_pointer/decode_alltypes_pointer.c index 47d72685..d0cdcde0 100644 --- a/tests/alltypes_pointer/decode_alltypes_pointer.c +++ b/tests/alltypes_pointer/decode_alltypes_pointer.c @@ -5,10 +5,6 @@ #include "alltypes.pb.h" #include "test_helpers.h" -#ifdef HAVE_MALLINFO -#include -#endif - #define TEST(x) if (!(x)) { \ fprintf(stderr, "Test " #x " failed.\n"); \ status = false; \ @@ -53,10 +49,8 @@ bool check_alltypes(pb_istream_t *stream, int mode) && strcmp(alltypes.req_submsg->substuff1, "1016") == 0); TEST(alltypes.req_submsg && alltypes.req_submsg->substuff2 && *alltypes.req_submsg->substuff2 == 1016); - /* TEST(*alltypes.req_submsg->substuff3 == 3); Default values are not currently supported for pointer fields */ TEST(*alltypes.req_enum == MyEnum_Truth); -#if 0 TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); @@ -74,121 +68,79 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); - TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); - TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); + TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4]->size == 4 && alltypes.rep_bytes[0]->size == 0); + TEST(memcmp(&alltypes.rep_bytes[4]->bytes, "2015", 4) == 0); TEST(alltypes.rep_submsg_count == 5); TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); - TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); - TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3); + TEST(*alltypes.rep_submsg[4].substuff2 == 2016 && *alltypes.rep_submsg[0].substuff2 == 0); + TEST(*alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == NULL); TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); TEST(alltypes.rep_emptymsg_count == 5); - + if (mode == 0) { - /* Expect default values */ - TEST(alltypes.has_opt_int32 == false); - TEST(alltypes.opt_int32 == 4041); - TEST(alltypes.has_opt_int64 == false); - TEST(alltypes.opt_int64 == 4042); - TEST(alltypes.has_opt_uint32 == false); - TEST(alltypes.opt_uint32 == 4043); - TEST(alltypes.has_opt_uint64 == false); - TEST(alltypes.opt_uint64 == 4044); - TEST(alltypes.has_opt_sint32 == false); - TEST(alltypes.opt_sint32 == 4045); - TEST(alltypes.has_opt_sint64 == false); - TEST(alltypes.opt_sint64 == 4046); - TEST(alltypes.has_opt_bool == false); - TEST(alltypes.opt_bool == false); + /* Expect that optional values are not present */ + TEST(alltypes.opt_int32 == NULL); + TEST(alltypes.opt_int64 == NULL); + TEST(alltypes.opt_uint32 == NULL); + TEST(alltypes.opt_uint64 == NULL); + TEST(alltypes.opt_sint32 == NULL); + TEST(alltypes.opt_sint64 == NULL); + TEST(alltypes.opt_bool == NULL); - TEST(alltypes.has_opt_fixed32 == false); - TEST(alltypes.opt_fixed32 == 4048); - TEST(alltypes.has_opt_sfixed32 == false); - TEST(alltypes.opt_sfixed32 == 4049); - TEST(alltypes.has_opt_float == false); - TEST(alltypes.opt_float == 4050.0f); + TEST(alltypes.opt_fixed32 == NULL); + TEST(alltypes.opt_sfixed32 == NULL); + TEST(alltypes.opt_float == NULL); + TEST(alltypes.opt_fixed64 == NULL); + TEST(alltypes.opt_sfixed64 == NULL); + TEST(alltypes.opt_double == NULL); - TEST(alltypes.has_opt_fixed64 == false); - TEST(alltypes.opt_fixed64 == 4051); - TEST(alltypes.has_opt_sfixed64 == false); - TEST(alltypes.opt_sfixed64 == 4052); - TEST(alltypes.has_opt_double == false); - TEST(alltypes.opt_double == 4053.0); - - TEST(alltypes.has_opt_string == false); - TEST(strcmp(alltypes.opt_string, "4054") == 0); - TEST(alltypes.has_opt_bytes == false); - TEST(alltypes.opt_bytes.size == 4); - TEST(memcmp(alltypes.opt_bytes.bytes, "4055", 4) == 0); - TEST(alltypes.has_opt_submsg == false); - TEST(strcmp(alltypes.opt_submsg.substuff1, "1") == 0); - TEST(alltypes.opt_submsg.substuff2 == 2); - TEST(alltypes.opt_submsg.substuff3 == 3); - TEST(alltypes.has_opt_enum == false); - TEST(alltypes.opt_enum == MyEnum_Second); - TEST(alltypes.has_opt_emptymsg == false); + TEST(alltypes.opt_string == NULL); + TEST(alltypes.opt_bytes == NULL); + TEST(alltypes.opt_submsg == NULL); + TEST(alltypes.opt_enum == NULL); } else { /* Expect filled-in values */ - TEST(alltypes.has_opt_int32 == true); - TEST(alltypes.opt_int32 == 3041); - TEST(alltypes.has_opt_int64 == true); - TEST(alltypes.opt_int64 == 3042); - TEST(alltypes.has_opt_uint32 == true); - TEST(alltypes.opt_uint32 == 3043); - TEST(alltypes.has_opt_uint64 == true); - TEST(alltypes.opt_uint64 == 3044); - TEST(alltypes.has_opt_sint32 == true); - TEST(alltypes.opt_sint32 == 3045); - TEST(alltypes.has_opt_sint64 == true); - TEST(alltypes.opt_sint64 == 3046); - TEST(alltypes.has_opt_bool == true); - TEST(alltypes.opt_bool == true); - - TEST(alltypes.has_opt_fixed32 == true); - TEST(alltypes.opt_fixed32 == 3048); - TEST(alltypes.has_opt_sfixed32 == true); - TEST(alltypes.opt_sfixed32 == 3049); - TEST(alltypes.has_opt_float == true); - TEST(alltypes.opt_float == 3050.0f); + TEST(alltypes.opt_int32 && *alltypes.opt_int32 == 3041); + TEST(alltypes.opt_int64 && *alltypes.opt_int64 == 3042); + TEST(alltypes.opt_uint32 && *alltypes.opt_uint32 == 3043); + TEST(alltypes.opt_uint64 && *alltypes.opt_uint64 == 3044); + TEST(alltypes.opt_sint32 && *alltypes.opt_sint32 == 3045); + TEST(alltypes.opt_sint64 && *alltypes.opt_sint64 == 3046); + TEST(alltypes.opt_bool && *alltypes.opt_bool == true); - TEST(alltypes.has_opt_fixed64 == true); - TEST(alltypes.opt_fixed64 == 3051); - TEST(alltypes.has_opt_sfixed64 == true); - TEST(alltypes.opt_sfixed64 == 3052); - TEST(alltypes.has_opt_double == true); - TEST(alltypes.opt_double == 3053.0); + TEST(alltypes.opt_fixed32 && *alltypes.opt_fixed32 == 3048); + TEST(alltypes.opt_sfixed32 && *alltypes.opt_sfixed32== 3049); + TEST(alltypes.opt_float && *alltypes.opt_float == 3050.0f); + TEST(alltypes.opt_fixed64 && *alltypes.opt_fixed64 == 3051); + TEST(alltypes.opt_sfixed64 && *alltypes.opt_sfixed64== 3052); + TEST(alltypes.opt_double && *alltypes.opt_double == 3053.0); - TEST(alltypes.has_opt_string == true); - TEST(strcmp(alltypes.opt_string, "3054") == 0); - TEST(alltypes.has_opt_bytes == true); - TEST(alltypes.opt_bytes.size == 4); - TEST(memcmp(alltypes.opt_bytes.bytes, "3055", 4) == 0); - TEST(alltypes.has_opt_submsg == true); - TEST(strcmp(alltypes.opt_submsg.substuff1, "3056") == 0); - TEST(alltypes.opt_submsg.substuff2 == 3056); - TEST(alltypes.opt_submsg.substuff3 == 3); - TEST(alltypes.has_opt_enum == true); - TEST(alltypes.opt_enum == MyEnum_Truth); - TEST(alltypes.has_opt_emptymsg == true); + TEST(alltypes.opt_string && strcmp(alltypes.opt_string, "3054") == 0); + TEST(alltypes.opt_bytes && alltypes.opt_bytes->size == 4); + TEST(alltypes.opt_bytes && memcmp(&alltypes.opt_bytes->bytes, "3055", 4) == 0); + TEST(alltypes.opt_submsg && strcmp(alltypes.opt_submsg->substuff1, "3056") == 0); + TEST(alltypes.opt_submsg && *alltypes.opt_submsg->substuff2 == 3056); + TEST(alltypes.opt_enum && *alltypes.opt_enum == MyEnum_Truth); + TEST(alltypes.opt_emptymsg); } - TEST(alltypes.req_limits.int32_min == INT32_MIN); - TEST(alltypes.req_limits.int32_max == INT32_MAX); - TEST(alltypes.req_limits.uint32_min == 0); - TEST(alltypes.req_limits.uint32_max == UINT32_MAX); - TEST(alltypes.req_limits.int64_min == INT64_MIN); - TEST(alltypes.req_limits.int64_max == INT64_MAX); - TEST(alltypes.req_limits.uint64_min == 0); - TEST(alltypes.req_limits.uint64_max == UINT64_MAX); - TEST(alltypes.req_limits.enum_min == HugeEnum_Negative); - TEST(alltypes.req_limits.enum_max == HugeEnum_Positive); - - TEST(alltypes.end == 1099); -#endif + TEST(alltypes.req_limits->int32_min && *alltypes.req_limits->int32_min == INT32_MIN); + TEST(alltypes.req_limits->int32_max && *alltypes.req_limits->int32_max == INT32_MAX); + TEST(alltypes.req_limits->uint32_min && *alltypes.req_limits->uint32_min == 0); + TEST(alltypes.req_limits->uint32_max && *alltypes.req_limits->uint32_max == UINT32_MAX); + TEST(alltypes.req_limits->int64_min && *alltypes.req_limits->int64_min == INT64_MIN); + TEST(alltypes.req_limits->int64_max && *alltypes.req_limits->int64_max == INT64_MAX); + TEST(alltypes.req_limits->uint64_min && *alltypes.req_limits->uint64_min == 0); + TEST(alltypes.req_limits->uint64_max && *alltypes.req_limits->uint64_max == UINT64_MAX); + TEST(alltypes.req_limits->enum_min && *alltypes.req_limits->enum_min == HugeEnum_Negative); + TEST(alltypes.req_limits->enum_max && *alltypes.req_limits->enum_max == HugeEnum_Positive); + + TEST(alltypes.end && *alltypes.end == 1099); pb_release(AllTypes_fields, &alltypes); @@ -197,59 +149,24 @@ bool check_alltypes(pb_istream_t *stream, int mode) int main(int argc, char **argv) { - bool status; - int orig_allocations; + uint8_t buffer[1024]; + size_t count; + pb_istream_t stream; -#ifdef HAVE_MALLINFO - /* Dynamic library loader etc. may have some malloc()ed memory also. */ - { - struct mallinfo m = mallinfo(); - orig_allocations = m.uordblks; - } -#endif - - { - uint8_t buffer[1024]; - size_t count; - pb_istream_t stream; - - /* Whether to expect the optional values or the default values. */ - int mode = (argc > 1) ? atoi(argv[1]) : 0; - - /* Read the data into buffer */ - SET_BINARY_MODE(stdin); - count = fread(buffer, 1, sizeof(buffer), stdin); - - /* Construct a pb_istream_t for reading from the buffer */ - stream = pb_istream_from_buffer(buffer, count); - - /* Decode and verify the message */ - status = check_alltypes(&stream, mode); - - if (!status) - fprintf(stderr, "Parsing failed: %s\n", PB_GET_ERROR(&stream)); - } + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; -#ifdef HAVE_MALLINFO - /* Check for memory leaks */ - { - struct mallinfo m = mallinfo(); - int leak = m.uordblks - orig_allocations; - - if (leak > 0) - { - fprintf(stderr, "Memory leak: %d bytes\n", leak); - return 1; - } - else - { - fprintf(stderr, "Ok, no memory leaks\n"); - } - } -#endif + /* Read the data into buffer */ + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + stream = pb_istream_from_buffer(buffer, count); - if (!status) + /* Decode and verify the message */ + if (!check_alltypes(&stream, mode)) { + fprintf(stderr, "Test failed: %s\n", PB_GET_ERROR(&stream)); return 1; } else diff --git a/tests/site_scons/site_init.py b/tests/site_scons/site_init.py index 2226144f..5fb06d6e 100644 --- a/tests/site_scons/site_init.py +++ b/tests/site_scons/site_init.py @@ -19,19 +19,24 @@ def add_nanopb_builders(env): else: infile = None - args = [str(source[0])] + if env.has_key("COMMAND"): + args = [env["COMMAND"]] + else: + args = [str(source[0])] + if env.has_key('ARGS'): args.extend(env['ARGS']) + print 'Command line: ' + str(args) pipe = subprocess.Popen(args, stdin = infile, stdout = open(str(target[0]), 'w'), stderr = sys.stderr) result = pipe.wait() if result == 0: - print '\033[32m[ OK ]\033[0m Ran ' + str(source[0]) + print '\033[32m[ OK ]\033[0m Ran ' + args[0] else: - print '\033[31m[FAIL]\033[0m Program ' + str(source[0]) + ' returned ' + str(result) + print '\033[31m[FAIL]\033[0m Program ' + args[0] + ' returned ' + str(result) return result run_test_builder = Builder(action = run_test, -- cgit 1.2.3-korg From ab62402059ff3752660ffc9f292cf210aef59be0 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 16 Mar 2014 15:52:19 +0200 Subject: Documentation updates --- docs/reference.rst | 51 +++++++++++++++++++++++- pb_decode.c | 10 ++++- pb_decode.h | 7 +++- tests/alltypes_pointer/decode_alltypes_pointer.c | 3 -- 4 files changed, 63 insertions(+), 8 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 32283734..79fa7333 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -28,6 +28,8 @@ NANOPB_INTERNALS Set this to expose the field encoder functions that are hidden since nanopb-0.1.3. Starting with nanopb-0.2.4, this flag does nothing. Use the newer functions that have better interface. +PB_ENABLE_MALLOC Set this to enable dynamic allocation support + in the decoder. PB_MAX_REQUIRED_FIELDS Maximum number of required fields to check for presence. Default value is 64. Increases stack usage 1 byte per every 8 fields. Compiler @@ -77,8 +79,9 @@ max_count Allocated number of entries in arrays (*repeated* fields). type Type of the generated field. Default value is *FT_DEFAULT*, which selects automatically. - You can use *FT_CALLBACK*, *FT_STATIC* or - *FT_IGNORE* to force a callback field, a static + You can use *FT_CALLBACK*, *FT_POINTER*, + *FT_STATIC* or *FT_IGNORE* to force a callback + field, a dynamically allocated field, a static field or to completely ignore the field. long_names Prefix the enum name to the enum value in definitions, i.e. *EnumName_EnumValue*. Enabled @@ -417,6 +420,17 @@ Encodes the contents of a structure as a protocol buffers message and writes it Normally pb_encode simply walks through the fields description array and serializes each field in turn. However, submessages must be serialized twice: first to calculate their size and then to actually write them to output. This causes some constraints for callback fields, which must return the same data on every call. +pb_encode_delimited +------------------- +Calculates the length of the message, encodes it as varint and then encodes the message. :: + + bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); + +(parameters are the same as for `pb_encode`_.) + +A common way to indicate the message length in Protocol Buffers is to prefix it with a varint. +This function does this, and it is compatible with *parseDelimitedFrom* in Google's protobuf library. + .. sidebar:: Encoding fields manually The functions with names *pb_encode_\** are used when dealing with callback fields. The typical reason for using callbacks is to have an array of unlimited size. In that case, `pb_encode`_ will call your callback function, which in turn will call *pb_encode_\** functions repeatedly to write out values. @@ -579,6 +593,10 @@ In addition to EOF, the pb_decode implementation supports terminating a message For optional fields, this function applies the default value and sets *has_* to false if the field is not present. +If *PB_ENABLE_MALLOC* is defined, this function may allocate storage for any pointer type fields. +In this case, you have to call `pb_release`_ to release the memory after you are done with the message. +On error return `pb_decode` will release the memory itself. + pb_decode_noinit ---------------- Same as `pb_decode`_, except does not apply the default values to fields. :: @@ -589,6 +607,35 @@ Same as `pb_decode`_, except does not apply the default values to fields. :: The destination structure should be filled with zeros before calling this function. Doing a *memset* manually can be slightly faster than using `pb_decode`_ if you don't need any default values. +In addition to decoding a single message, this function can be used to merge two messages, so that +values from previous message will remain if the new message does not contain a field. + +This function *will not* release the message even on error return. If you use *PB_ENABLE_MALLOC*, +you will need to call `pb_release`_ yourself. + +pb_decode_delimited +------------------- +Same as `pb_decode`_, except that it first reads a varint with the length of the message. :: + + bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +(parameters are the same as for `pb_decode`_.) + +A common method to indicate message size in Protocol Buffers is to prefix it with a varint. +This function is compatible with *writeDelimitedTo* in the Google's Protocol Buffers library. + +pb_release +---------- +Releases any dynamically allocated fields. + + void pb_release(const pb_field_t fields[], void *dest_struct); + +:fields: A field description array. Usually autogenerated. +:dest_struct: Pointer to structure where data will be stored. + +This function is only available if *PB_ENABLE_MALLOC* is defined. It will release any +pointer type fields in the structure and set the pointers to NULL. + pb_skip_varint -------------- Skip a varint_ encoded integer without decoding it. :: diff --git a/pb_decode.c b/pb_decode.c index 81febb9b..7938d70b 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -894,8 +894,16 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { + bool status; pb_message_set_to_defaults(fields, dest_struct); - return pb_decode_noinit(stream, fields, dest_struct); + status = pb_decode_noinit(stream, fields, dest_struct); + +#ifdef PB_ENABLE_MALLOC + if (!status) + pb_release(fields, dest_struct); +#endif + + return status; } bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) diff --git a/pb_decode.h b/pb_decode.h index 2e1da5a6..8dc67408 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -73,6 +73,9 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc * * This can also be used for 'merging' two messages, i.e. update only the * fields that exist in the new message. + * + * Note: If this function returns with an error, it will not release any + * dynamically allocated fields. You will need to call pb_release() yourself. */ bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); @@ -84,8 +87,8 @@ bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void * #ifdef PB_ENABLE_MALLOC /* Release any allocated pointer fields. If you use dynamic allocation, you should - * call this for any decoded message when you are done with it. You also need to - * free messages even if pb_decode() returned with error. + * call this for any successfully decoded message when you are done with it. If + * pb_decode() returns with an error, the message is already released. */ void pb_release(const pb_field_t fields[], void *dest_struct); #endif diff --git a/tests/alltypes_pointer/decode_alltypes_pointer.c b/tests/alltypes_pointer/decode_alltypes_pointer.c index d0cdcde0..889676b4 100644 --- a/tests/alltypes_pointer/decode_alltypes_pointer.c +++ b/tests/alltypes_pointer/decode_alltypes_pointer.c @@ -21,10 +21,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) memset(&alltypes, 0xAA, sizeof(alltypes)); if (!pb_decode(stream, AllTypes_fields, &alltypes)) - { - pb_release(AllTypes_fields, &alltypes); return false; - } TEST(alltypes.req_int32 && *alltypes.req_int32 == -1001); TEST(alltypes.req_int64 && *alltypes.req_int64 == -1002); -- cgit 1.2.3-korg From 607cb998b5472ef1df461995b33694e9a54bef23 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 17 Mar 2014 17:25:58 +0200 Subject: More configuration options for dynamic alloc --- pb.h | 14 ++++++++++++++ pb_decode.c | 6 +++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/pb.h b/pb.h index d6cb1d40..6f843854 100644 --- a/pb.h +++ b/pb.h @@ -10,6 +10,9 @@ * uncommenting the lines, or on the compiler command line. * *****************************************************************/ +/* Enable support for dynamically allocated fields */ +/* #define PB_ENABLE_MALLOC 1 */ + /* Define this if your CPU architecture is big endian, i.e. it * stores the most-significant byte first. */ /* #define __BIG_ENDIAN__ 1 */ @@ -340,6 +343,17 @@ struct _pb_extension_t { pb_extension_t *next; }; +/* Memory allocation functions to use. You can define pb_realloc and + * pb_free to custom functions if you want. */ +#ifdef PB_ENABLE_MALLOC +# ifndef pb_realloc +# define pb_realloc(ptr, size) realloc(ptr, size) +# endif +# ifndef pb_free +# define pb_free(ptr) free(ptr) +# endif +#endif + /* These macros are used to declare pb_field_t's in the constant array. */ /* Size of a structure member, in bytes. */ #define pb_membersize(st, m) (sizeof ((st*)0)->m) diff --git a/pb_decode.c b/pb_decode.c index 7938d70b..9a2abd6e 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -480,7 +480,7 @@ static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t /* Allocate new or expand previous allocation */ /* Note: on failure the old pointer will remain in the structure, * the message must be freed by caller also on error return. */ - ptr = realloc(ptr, size); + ptr = pb_realloc(ptr, size); if (ptr == NULL) PB_RETURN_ERROR(stream, "realloc failed"); @@ -945,7 +945,7 @@ void pb_release(const pb_field_t fields[], void *dest_struct) size_t count = *(size_t*)iter.pSize; while (count--) { - free(*pItem); + pb_free(*pItem); *pItem++ = NULL; } } @@ -968,7 +968,7 @@ void pb_release(const pb_field_t fields[], void *dest_struct) } /* Release main item */ - free(*(void**)iter.pData); + pb_free(*(void**)iter.pData); *(void**)iter.pData = NULL; } } while (pb_field_next(&iter)); -- cgit 1.2.3-korg From f4949119ada32e28959e25e46b4f3314805b5ed1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 18 Mar 2014 16:13:54 +0200 Subject: Add stdlib.h to pb_syshdr.h for dynamic allocation --- extra/pb_syshdr.h | 10 ++++++++++ tests/SConstruct | 2 ++ 2 files changed, 12 insertions(+) diff --git a/extra/pb_syshdr.h b/extra/pb_syshdr.h index b9a97a4f..1ff48230 100644 --- a/extra/pb_syshdr.h +++ b/extra/pb_syshdr.h @@ -53,6 +53,16 @@ typedef int bool; #endif +/* stdlib.h subset */ +#ifdef PB_ENABLE_MALLOC +#ifdef HAVE_STDLIB_H +#include +#else +void *realloc(void *ptr, size_t size); +void free(void *ptr); +#endif +#endif + /* string.h subset */ #ifdef HAVE_STRING_H #include diff --git a/tests/SConstruct b/tests/SConstruct index 8bf16ebe..b6b877f7 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -50,6 +50,7 @@ if not env.GetOption('clean'): stdint = conf.CheckCHeader('stdint.h') stddef = conf.CheckCHeader('stddef.h') string = conf.CheckCHeader('string.h') + stdlib = conf.CheckCHeader('stdlib.h') if not stdbool or not stdint or not stddef or not string: conf.env.Append(CPPDEFINES = {'PB_SYSTEM_HEADER': '\\"pb_syshdr.h\\"'}) conf.env.Append(CPPPATH = "#../extra") @@ -58,6 +59,7 @@ if not env.GetOption('clean'): if stdint: conf.env.Append(CPPDEFINES = {'HAVE_STDINT_H': 1}) if stddef: conf.env.Append(CPPDEFINES = {'HAVE_STDDEF_H': 1}) if string: conf.env.Append(CPPDEFINES = {'HAVE_STRING_H': 1}) + if stdlib: conf.env.Append(CPPDEFINES = {'HAVE_STDLIB_H': 1}) # Check if we can use pkg-config to find protobuf include path status, output = conf.TryAction('pkg-config protobuf --variable=includedir > $TARGET') -- cgit 1.2.3-korg From 6c90e824c4e8f54c71645b63b1ea33b1e028cb97 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 1 Apr 2014 16:47:53 +0300 Subject: Fix compile error when default value given for extension field. Update issue 111 Status: FixedInGit --- generator/nanopb_generator.py | 2 ++ tests/extensions/extensions.proto | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index c32b26ad..0660d20e 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -351,6 +351,8 @@ class Field: result += '0)' elif self.pbtype in ['BYTES', 'STRING'] and self.allocation != 'STATIC': result += '0)' # Arbitrary size default values not implemented + elif self.rules == 'OPTEXT': + result += '0)' # Default value for extensions is not implemented else: result += '&%s_default)' % (self.struct_name + self.name) diff --git a/tests/extensions/extensions.proto b/tests/extensions/extensions.proto index d85e8193..da8432e0 100644 --- a/tests/extensions/extensions.proto +++ b/tests/extensions/extensions.proto @@ -1,7 +1,7 @@ import 'alltypes.proto'; extend AllTypes { - optional int32 AllTypes_extensionfield1 = 255; + optional int32 AllTypes_extensionfield1 = 255 [default = 5]; } message ExtensionMessage { -- cgit 1.2.3-korg From 99434724d0375280abe44944b0c39f45991f26df Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 2 Apr 2014 20:59:01 +0300 Subject: Fix splint warnings, add splint test case --- pb_decode.c | 16 ++++++++-------- pb_encode.c | 6 +++--- tests/splint/SConscript | 13 +++++++++++++ tests/splint/splint.rc | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 tests/splint/SConscript create mode 100644 tests/splint/splint.rc diff --git a/pb_decode.c b/pb_decode.c index 9a2abd6e..1213336d 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -130,7 +130,7 @@ bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) * This is an optimization for the varint decoding. */ static bool checkreturn pb_readbyte(pb_istream_t *stream, uint8_t *buf) { - if (!stream->bytes_left) + if (stream->bytes_left == 0) PB_RETURN_ERROR(stream, "end-of-stream"); #ifndef PB_BUFFER_ONLY @@ -174,7 +174,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) if (!pb_readbyte(stream, &byte)) return false; - if (!(byte & 0x80)) + if ((byte & 0x80) == 0) { /* Quick case, 1 byte value */ result = byte; @@ -397,7 +397,7 @@ static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag) { return true; } - pb_field_next(iter); + (void)pb_field_next(iter); } while (iter->field_index != start); return false; @@ -435,7 +435,7 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t if (!pb_make_string_substream(stream, &substream)) return false; - while (substream.bytes_left && *size < iter->pos->array_size) + while (substream.bytes_left > 0 && *size < iter->pos->array_size) { void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); if (!func(&substream, iter->pos, pItem)) @@ -695,7 +695,7 @@ static bool checkreturn decode_extension(pb_istream_t *stream, pb_extension_t *extension = *(pb_extension_t* const *)iter->pData; size_t pos = stream->bytes_left; - while (extension && pos == stream->bytes_left) + while (extension != NULL && pos == stream->bytes_left) { bool status; if (extension->type->decode) @@ -722,7 +722,7 @@ static bool checkreturn find_extension_field(pb_field_iterator_t *iter) do { if (PB_LTYPE(iter->pos->type) == PB_LTYPE_EXTENSION) return true; - pb_field_next(iter); + (void)pb_field_next(iter); } while (iter->field_index != start); return false; @@ -798,7 +798,7 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { - uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {0}; /* Used to check for required fields */ + uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {0, 0, 0, 0, 0, 0, 0, 0}; uint32_t extension_range_start = 0; pb_field_iterator_t iter; @@ -874,7 +874,7 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ } while (pb_field_next(&iter)); /* Fixup if last field was also required. */ - if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.pos->tag) + if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.pos->tag != 0) req_field_count++; /* Check the whole bytes */ diff --git a/pb_encode.c b/pb_encode.c index 59e6f2a7..a5851582 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -405,9 +405,9 @@ bool checkreturn pb_encode_svarint(pb_ostream_t *stream, int64_t value) { uint64_t zigzagged; if (value < 0) - zigzagged = (uint64_t)(~(value << 1)); + zigzagged = ~((uint64_t)value << 1); else - zigzagged = (uint64_t)(value << 1); + zigzagged = (uint64_t)value << 1; return pb_encode_varint(stream, zigzagged); } @@ -448,7 +448,7 @@ bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value) bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, uint32_t field_number) { - uint64_t tag = wiretype | (field_number << 3); + uint64_t tag = ((uint64_t)field_number << 3) | wiretype; return pb_encode_varint(stream, tag); } diff --git a/tests/splint/SConscript b/tests/splint/SConscript new file mode 100644 index 00000000..c1432dde --- /dev/null +++ b/tests/splint/SConscript @@ -0,0 +1,13 @@ +# Check the nanopb core using splint + +Import('env') + +p = env.WhereIs('splint') + +if p: + env.Command('pb_decode.splint', '$NANOPB/pb_decode.c', + 'splint -f splint/splint.rc $SOURCE 2> $TARGET') + + env.Command('pb_encode.splint', '$NANOPB/pb_encode.c', + 'splint -f splint/splint.rc $SOURCE 2> $TARGET') + diff --git a/tests/splint/splint.rc b/tests/splint/splint.rc new file mode 100644 index 00000000..c77e210f --- /dev/null +++ b/tests/splint/splint.rc @@ -0,0 +1,36 @@ ++checks ++partial ++matchanyintegral ++strictlib +-isoreserved # to be fixed in 0.3 +-nullassign +-predboolint +-predboolptr ++ptrnegate +-switchloopbreak ++ignoresigns +-infloopsuncon +-type + +# splint's memory checks don't quite work without annotations +-mustfreeonly +-compmempass +-nullret +-observertrans +-statictrans +-compdestroy +-nullpass +-nullstate +-compdef +-usereleased +-temptrans +-dependenttrans +-kepttrans +-branchstate + +# These tests give false positives, compiler typically has +# better warnings for these. +-noret +-noeffect +-usedef + -- cgit 1.2.3-korg From 70dee34da6578ab76dbf00009357d679e154e04b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 2 Apr 2014 21:08:15 +0300 Subject: Add some missing 'static' specifiers Update issue 91 Status: FixedInGit --- docs/reference.rst | 4 ---- pb_decode.c | 17 ++++++++--------- pb_encode.c | 17 ++++++++--------- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 79fa7333..ccbf0a4d 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -24,10 +24,6 @@ __BIG_ENDIAN__ Set this if your platform stores integers and floats in big-endian format. Mixed-endian systems (different layout for ints and floats) are currently not supported. -NANOPB_INTERNALS Set this to expose the field encoder functions - that are hidden since nanopb-0.1.3. Starting - with nanopb-0.2.4, this flag does nothing. Use - the newer functions that have better interface. PB_ENABLE_MALLOC Set this to enable dynamic allocation support in the decoder. PB_MAX_REQUIRED_FIELDS Maximum number of required fields to check for diff --git a/pb_decode.c b/pb_decode.c index 1213336d..d8ac9ee2 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -13,7 +13,6 @@ #define checkreturn __attribute__((warn_unused_result)) #endif -#define NANOPB_INTERNALS #include "pb.h" #include "pb_decode.h" @@ -1033,7 +1032,7 @@ bool pb_decode_fixed64(pb_istream_t *stream, void *dest) #endif } -bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) +static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint64_t value; if (!pb_decode_varint(stream, &value)) @@ -1051,7 +1050,7 @@ bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, vo return true; } -bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, void *dest) +static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint64_t value; if (!pb_decode_varint(stream, &value)) @@ -1067,7 +1066,7 @@ bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, v return true; } -bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) +static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) { int64_t value; if (!pb_decode_svarint(stream, &value)) @@ -1083,19 +1082,19 @@ bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, v return true; } -bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest) +static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest) { UNUSED(field); return pb_decode_fixed32(stream, dest); } -bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest) +static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest) { UNUSED(field); return pb_decode_fixed64(stream, dest); } -bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) +static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint32_t size; pb_bytes_array_t *bdest; @@ -1124,7 +1123,7 @@ bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, voi return pb_read(stream, bdest->bytes, size); } -bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest) +static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint32_t size; size_t alloc_size; @@ -1156,7 +1155,7 @@ bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, vo return status; } -bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) +static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest) { bool status; pb_istream_t substream; diff --git a/pb_encode.c b/pb_encode.c index a5851582..1eb9473a 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -3,7 +3,6 @@ * 2011 Petteri Aimonen */ -#define NANOPB_INTERNALS #include "pb.h" #include "pb_encode.h" @@ -544,7 +543,7 @@ bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fie /* Field encoders */ -bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src) +static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src) { int64_t value = 0; @@ -562,7 +561,7 @@ bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, co return pb_encode_varint(stream, (uint64_t)value); } -bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, const void *src) +static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, const void *src) { uint64_t value = 0; @@ -576,7 +575,7 @@ bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, c return pb_encode_varint(stream, value); } -bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src) +static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src) { int64_t value = 0; @@ -590,19 +589,19 @@ bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, c return pb_encode_svarint(stream, value); } -bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src) +static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src) { UNUSED(field); return pb_encode_fixed64(stream, src); } -bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src) +static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src) { UNUSED(field); return pb_encode_fixed32(stream, src); } -bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) +static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) { const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src; @@ -621,7 +620,7 @@ bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, con return pb_encode_string(stream, bytes->bytes, bytes->size); } -bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src) +static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src) { /* strnlen() is not always available, so just use a loop */ size_t size = 0; @@ -647,7 +646,7 @@ bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, co return pb_encode_string(stream, (const uint8_t*)src, size); } -bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src) +static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src) { if (field->ptr == NULL) PB_RETURN_ERROR(stream, "invalid field descriptor"); -- cgit 1.2.3-korg From e5b855fec5d2977971f96d817728c7a3ee8077b2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 5 Apr 2014 11:11:05 +0300 Subject: Add a 'found' field to pb_extension_t. Update issue 112 Status: FixedInGit --- pb.h | 4 ++++ pb_decode.c | 3 +-- tests/extensions/decode_extensions.c | 2 ++ tests/splint/splint.rc | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pb.h b/pb.h index 6f843854..63d313dc 100644 --- a/pb.h +++ b/pb.h @@ -341,6 +341,10 @@ struct _pb_extension_t { * If this extension does not match a field, the next handler is * automatically called. */ pb_extension_t *next; + + /* The decoder sets this to true if the extension was found. + * Ignored for encoding. */ + bool found; }; /* Memory allocation functions to use. You can define pb_realloc and diff --git a/pb_decode.c b/pb_decode.c index d8ac9ee2..9a48c60f 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -670,7 +670,6 @@ static bool checkreturn default_extension_decoder(pb_istream_t *stream, { const pb_field_t *field = (const pb_field_t*)extension->type->arg; pb_field_iterator_t iter; - bool dummy; if (field->tag != tag) return true; @@ -681,7 +680,7 @@ static bool checkreturn default_extension_decoder(pb_istream_t *stream, iter.required_field_index = 0; iter.dest_struct = extension->dest; iter.pData = extension->dest; - iter.pSize = &dummy; + iter.pSize = &extension->found; return decode_field(stream, wire_type, &iter); } diff --git a/tests/extensions/decode_extensions.c b/tests/extensions/decode_extensions.c index f8ebbde0..e4374380 100644 --- a/tests/extensions/decode_extensions.c +++ b/tests/extensions/decode_extensions.c @@ -49,7 +49,9 @@ int main(int argc, char **argv) } /* Check that the extensions decoded properly */ + TEST(ext1.found) TEST(extensionfield1 == 12345) + TEST(ext2.found) TEST(strcmp(extensionfield2.test1, "test") == 0) TEST(extensionfield2.test2 == 54321) diff --git a/tests/splint/splint.rc b/tests/splint/splint.rc index c77e210f..421f5677 100644 --- a/tests/splint/splint.rc +++ b/tests/splint/splint.rc @@ -27,6 +27,7 @@ -dependenttrans -kepttrans -branchstate +-immediatetrans # These tests give false positives, compiler typically has # better warnings for these. -- cgit 1.2.3-korg From b63e582bdb34b4727a8eb551fc72ee0476047a46 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 5 Apr 2014 11:26:39 +0300 Subject: Add a convenience function pb_get_encoded_size() There is minimal size penalty from this, and it is probably much more intuitive to use than PB_OSTREAM_SIZING itself. This has been suggested before also, but I ended up refusing it back them. Reconsidering it now, I see that an intuitive API is much better than any amount of documentation explaining a non-intuitive API. Update issue 16 Status: FixedInGit --- pb_encode.c | 11 +++++++++++ pb_encode.h | 4 ++++ tests/encode_unittests/encode_unittests.c | 9 +++++++++ 3 files changed, 24 insertions(+) diff --git a/pb_encode.c b/pb_encode.c index 1eb9473a..dc5a2734 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -378,6 +378,17 @@ bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const return pb_encode_submessage(stream, fields, src_struct); } +bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], const void *src_struct) +{ + pb_ostream_t stream = PB_OSTREAM_SIZING; + + if (!pb_encode(&stream, fields, src_struct)) + return false; + + *size = stream.bytes_written; + return true; +} + /******************** * Helper functions * ********************/ diff --git a/pb_encode.h b/pb_encode.h index 900994aa..f82bac8f 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -71,6 +71,10 @@ bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_ */ bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); +/* Encode the message to get the size of the encoded data, but do not store + * the data. */ +bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], const void *src_struct); + /************************************** * Functions for manipulating streams * **************************************/ diff --git a/tests/encode_unittests/encode_unittests.c b/tests/encode_unittests/encode_unittests.c index edbc10a9..06935f9a 100644 --- a/tests/encode_unittests/encode_unittests.c +++ b/tests/encode_unittests/encode_unittests.c @@ -281,6 +281,15 @@ int main() TEST(WRITES(pb_encode_delimited(&s, IntegerContainer_fields, &msg), "\x09\x0A\x07\x0A\x05\x01\x02\x03\x04\x05")) } + + { + IntegerContainer msg = {{5, {1,2,3,4,5}}}; + size_t size; + + COMMENT("Test pb_get_encoded_size.") + TEST(pb_get_encoded_size(&size, IntegerContainer_fields, &msg) && + size == 9); + } { uint8_t buffer[10]; -- cgit 1.2.3-korg From 7880f308ea996292d5e28a81618370d79e2bdf26 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 5 Apr 2014 13:25:44 +0300 Subject: Fix unused parameter warning when building without errmsg. --- pb.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pb.h b/pb.h index 63d313dc..aca96ee9 100644 --- a/pb.h +++ b/pb.h @@ -500,7 +500,11 @@ struct _pb_extension_t { * messages if not used. */ #ifdef PB_NO_ERRMSG -#define PB_RETURN_ERROR(stream,msg) return false +#define PB_RETURN_ERROR(stream,msg) \ + do {\ + UNUSED(stream); \ + return false; \ + } while(0) #define PB_GET_ERROR(stream) "(errmsg disabled)" #else #define PB_RETURN_ERROR(stream,msg) \ -- cgit 1.2.3-korg From a8de6acf2df23104be8907b9e3ed8647b33df6e2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 5 Apr 2014 13:26:37 +0300 Subject: Add rule for building coverage summary using lcov. Also modified a few tests to be more compatible with coverage information, so that they use the same pb_encode/decode.c instead of making a copy. --- tests/Makefile | 15 +++++++++++++++ tests/SConstruct | 4 ++-- tests/buffer_only/SConscript | 12 ++++++++---- tests/field_size_16/SConscript | 12 ++++++++---- tests/field_size_32/SConscript | 12 ++++++++---- tests/no_errmsg/SConscript | 12 ++++++++---- 6 files changed, 49 insertions(+), 18 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index fb37e630..3c4e0b04 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -4,3 +4,18 @@ all: clean: scons -c +coverage: + rm -rf build coverage + + # LCOV does not like the newer gcov format + scons CC=gcc-4.6 CXX=gcc-4.6 + + # We are only interested in pb_encode.o and pb_decode.o + find build -name '*.gcda' -and \! \( -name '*pb_encode*' -or -name '*pb_decode*' \) -exec rm '{}' \; + + # Collect the data + mkdir build/coverage + lcov --base-directory . --directory build/ --gcov-tool gcov-4.6 -c -o build/coverage/nanopb.info + + # Generate HTML + genhtml -o build/coverage build/coverage/nanopb.info diff --git a/tests/SConstruct b/tests/SConstruct index b6b877f7..e7aa4717 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -91,9 +91,9 @@ if 'gcc' in env['CC']: # GNU Compiler Collection # Debug info, warnings as errors - env.Append(CFLAGS = '-ansi -pedantic -g -O0 -Wall -Werror --coverage -fstack-protector-all') + env.Append(CFLAGS = '-ansi -pedantic -g -O0 -Wall -Werror -fprofile-arcs -ftest-coverage -fstack-protector-all') env.Append(CORECFLAGS = '-Wextra') - env.Append(LINKFLAGS = '--coverage') + env.Append(LINKFLAGS = '-g --coverage') # We currently need uint64_t anyway, even though ANSI C90 otherwise.. env.Append(CFLAGS = '-Wno-long-long') diff --git a/tests/buffer_only/SConscript b/tests/buffer_only/SConscript index db86d377..cddbb04f 100644 --- a/tests/buffer_only/SConscript +++ b/tests/buffer_only/SConscript @@ -4,8 +4,6 @@ Import("env") # Take copy of the files for custom build. c = Copy("$TARGET", "$SOURCE") -env.Command("pb_encode.c", "#../pb_encode.c", c) -env.Command("pb_decode.c", "#../pb_decode.c", c) env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c) env.Command("alltypes.pb.c", "$BUILD/alltypes/alltypes.pb.c", c) env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c) @@ -15,9 +13,15 @@ env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c) opts = env.Clone() opts.Append(CPPDEFINES = {'PB_BUFFER_ONLY': 1}) +# Build new version of core +strict = opts.Clone() +strict.Append(CFLAGS = strict['CORECFLAGS']) +strict.Object("pb_decode_bufonly.o", "$NANOPB/pb_decode.c") +strict.Object("pb_encode_bufonly.o", "$NANOPB/pb_encode.c") + # Now build and run the test normally. -enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"]) -dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"]) +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_bufonly.o"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_bufonly.o"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/field_size_16/SConscript b/tests/field_size_16/SConscript index 15dc0f5d..8fee0049 100644 --- a/tests/field_size_16/SConscript +++ b/tests/field_size_16/SConscript @@ -5,8 +5,6 @@ Import("env") # Take copy of the files for custom build. c = Copy("$TARGET", "$SOURCE") -env.Command("pb_encode.c", "#../pb_encode.c", c) -env.Command("pb_decode.c", "#../pb_decode.c", c) env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c) env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c) @@ -16,9 +14,15 @@ env.NanopbProto(["alltypes", "alltypes.options"]) opts = env.Clone() opts.Append(CPPDEFINES = {'PB_FIELD_16BIT': 1}) +# Build new version of core +strict = opts.Clone() +strict.Append(CFLAGS = strict['CORECFLAGS']) +strict.Object("pb_decode_fields16.o", "$NANOPB/pb_decode.c") +strict.Object("pb_encode_fields16.o", "$NANOPB/pb_encode.c") + # Now build and run the test normally. -enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"]) -dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"]) +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields16.o"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields16.o"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/field_size_32/SConscript b/tests/field_size_32/SConscript index 25ef3546..2a64c6c5 100644 --- a/tests/field_size_32/SConscript +++ b/tests/field_size_32/SConscript @@ -5,8 +5,6 @@ Import("env") # Take copy of the files for custom build. c = Copy("$TARGET", "$SOURCE") -env.Command("pb_encode.c", "#../pb_encode.c", c) -env.Command("pb_decode.c", "#../pb_decode.c", c) env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c) env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c) @@ -16,9 +14,15 @@ env.NanopbProto(["alltypes", "alltypes.options"]) opts = env.Clone() opts.Append(CPPDEFINES = {'PB_FIELD_32BIT': 1}) +# Build new version of core +strict = opts.Clone() +strict.Append(CFLAGS = strict['CORECFLAGS']) +strict.Object("pb_decode_fields32.o", "$NANOPB/pb_decode.c") +strict.Object("pb_encode_fields32.o", "$NANOPB/pb_encode.c") + # Now build and run the test normally. -enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"]) -dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"]) +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields32.o"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields32.o"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/no_errmsg/SConscript b/tests/no_errmsg/SConscript index 2b9815d1..ed467051 100644 --- a/tests/no_errmsg/SConscript +++ b/tests/no_errmsg/SConscript @@ -4,8 +4,6 @@ Import("env") # Take copy of the files for custom build. c = Copy("$TARGET", "$SOURCE") -env.Command("pb_encode.c", "#../pb_encode.c", c) -env.Command("pb_decode.c", "#../pb_decode.c", c) env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c) env.Command("alltypes.pb.c", "$BUILD/alltypes/alltypes.pb.c", c) env.Command("encode_alltypes.c", "$BUILD/alltypes/encode_alltypes.c", c) @@ -15,9 +13,15 @@ env.Command("decode_alltypes.c", "$BUILD/alltypes/decode_alltypes.c", c) opts = env.Clone() opts.Append(CPPDEFINES = {'PB_NO_ERRMSG': 1}) +# Build new version of core +strict = opts.Clone() +strict.Append(CFLAGS = strict['CORECFLAGS']) +strict.Object("pb_decode_noerr.o", "$NANOPB/pb_decode.c") +strict.Object("pb_encode_noerr.o", "$NANOPB/pb_encode.c") + # Now build and run the test normally. -enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode.c"]) -dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode.c"]) +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_noerr.o"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_noerr.o"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) -- cgit 1.2.3-korg From c998ffe1178c26b513495c9144db24e89b9cbcc3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 7 Apr 2014 20:30:12 +0300 Subject: Update changelog --- CHANGELOG.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index e6c88f2a..6d643ade 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,11 @@ +nanopb-0.2.7 (2014-04-07) + Fix bug with default values for extension fields (issue 111) + Fix some MISRA-C warnings (issue 91) + Implemented optional malloc() support (issue 80) + Changed pointer-type bytes field datatype + Add a "found" field to pb_extension_t (issue 112) + Add convenience function pb_get_encoded_size() (issue 16) + nanopb-0.2.6 (2014-02-15) Fix generator error with bytes callback fields (issue 99) Fix warnings about large integer constants (issue 102) -- cgit 1.2.3-korg From 6d74c66adabf4bbe62f3c72bc5bb644eebf36d42 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 7 Apr 2014 20:30:42 +0300 Subject: Publishing nanopb-0.2.7 --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 0660d20e..b603a332 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.7-dev" +nanopb_version = "nanopb-0.2.7" import sys diff --git a/pb.h b/pb.h index aca96ee9..b5cb3ea4 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.7-dev +#define NANOPB_VERSION nanopb-0.2.7 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 938c7ac3f302ccfa3044ce0720aeed9a4336ac0e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 7 Apr 2014 20:45:04 +0300 Subject: Setting version to 0.2.8-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index b603a332..7b6f0ba9 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.7" +nanopb_version = "nanopb-0.2.8-dev" import sys diff --git a/pb.h b/pb.h index b5cb3ea4..fe6fb5b3 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.7 +#define NANOPB_VERSION nanopb-0.2.8-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 3e83d81b09606791b05715e6680ea57e65f06234 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 9 Apr 2014 19:28:57 +0300 Subject: Use -fsanitize=undefined when running tests with clang --- tests/SConstruct | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/SConstruct b/tests/SConstruct index e7aa4717..0ad833d1 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -33,13 +33,16 @@ env.Append(PROTOCPATH = '#../generator') # Check the compilation environment, unless we are just cleaning up. if not env.GetOption('clean'): - def check_ccflags(context, flags): + def check_ccflags(context, flags, linkflags = ''): '''Check if given CCFLAGS are supported''' context.Message('Checking support for CCFLAGS="%s"... ' % flags) oldflags = context.env['CCFLAGS'] + oldlinkflags = context.env['CCFLAGS'] context.env.Append(CCFLAGS = flags) + context.env.Append(LINKFLAGS = linkflags) result = context.TryCompile("int main() {return 0;}", '.c') context.env.Replace(CCFLAGS = oldflags) + context.env.Replace(LINKFLAGS = oldlinkflags) context.Result(result) return result @@ -83,6 +86,13 @@ if not env.GetOption('clean'): if conf.CheckCCFLAGS(extra): conf.env.Append(CORECFLAGS = extra) + # Check if we can use undefined behaviour sanitizer (only with clang) + extra = '-fsanitize=undefined ' + if 'clang' in env['CC']: + if conf.CheckCCFLAGS(extra, linkflags = extra): + conf.env.Append(CORECFLAGS = extra) + conf.env.Append(LINKFLAGS = extra) + # End the config stuff env = conf.Finish() @@ -119,9 +129,9 @@ elif 'tcc' in env['CC']: env.SetDefault(CORECFLAGS = '') -if 'clang++' in env['CXX']: +if 'clang' in env['CXX']: env.Append(CXXFLAGS = '-g -O0 -Wall -Werror -Wextra -Wno-missing-field-initializers') -elif 'g++' in env['CXX']: +elif 'g++' in env['CXX'] or 'gcc' in env['CXX']: env.Append(CXXFLAGS = '-g -O0 -Wall -Werror -Wextra -Wno-missing-field-initializers') elif 'cl' in env['CXX']: env.Append(CXXFLAGS = '/Zi /W2 /WX') -- cgit 1.2.3-korg From 1d249a48ea979729d1b818847dfbea3316cc527e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 9 Apr 2014 19:39:12 +0300 Subject: Fix bug in missing_fields test case --- tests/missing_fields/missing_fields.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/missing_fields/missing_fields.c b/tests/missing_fields/missing_fields.c index b9a273a2..8aded827 100644 --- a/tests/missing_fields/missing_fields.c +++ b/tests/missing_fields/missing_fields.c @@ -8,6 +8,7 @@ int main() { uint8_t buffer[512]; + size_t size; /* Create a message with one missing field */ { @@ -19,12 +20,14 @@ int main() printf("Encode failed.\n"); return 1; } + + size = stream.bytes_written; } /* Test that it decodes properly if we don't require that field */ { MissingField msg = {0}; - pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer)); + pb_istream_t stream = pb_istream_from_buffer(buffer, size); if (!pb_decode(&stream, MissingField_fields, &msg)) { @@ -36,7 +39,7 @@ int main() /* Test that it does *not* decode properly if we require the field */ { AllFields msg = {0}; - pb_istream_t stream = pb_istream_from_buffer(buffer, sizeof(buffer)); + pb_istream_t stream = pb_istream_from_buffer(buffer, size); if (pb_decode(&stream, AllFields_fields, &msg)) { -- cgit 1.2.3-korg From 3b36235cef2c1c5f3c79a33cb765de2ec1a2d01e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 15 Apr 2014 20:27:38 +0300 Subject: Remove -O0 from tests CFLAGS so that optimized builds can be tested also --- tests/SConstruct | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/SConstruct b/tests/SConstruct index 0ad833d1..b411b48e 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -101,7 +101,7 @@ if 'gcc' in env['CC']: # GNU Compiler Collection # Debug info, warnings as errors - env.Append(CFLAGS = '-ansi -pedantic -g -O0 -Wall -Werror -fprofile-arcs -ftest-coverage -fstack-protector-all') + env.Append(CFLAGS = '-ansi -pedantic -g -Wall -Werror -fprofile-arcs -ftest-coverage -fstack-protector-all') env.Append(CORECFLAGS = '-Wextra') env.Append(LINKFLAGS = '-g --coverage') @@ -109,7 +109,7 @@ if 'gcc' in env['CC']: env.Append(CFLAGS = '-Wno-long-long') elif 'clang' in env['CC']: # CLang - env.Append(CFLAGS = '-ansi -g -O0 -Wall -Werror') + env.Append(CFLAGS = '-ansi -g -Wall -Werror') env.Append(CORECFLAGS = ' -Wextra -Wcast-qual -Wconversion') elif 'cl' in env['CC']: # Microsoft Visual C++ @@ -130,9 +130,9 @@ elif 'tcc' in env['CC']: env.SetDefault(CORECFLAGS = '') if 'clang' in env['CXX']: - env.Append(CXXFLAGS = '-g -O0 -Wall -Werror -Wextra -Wno-missing-field-initializers') + env.Append(CXXFLAGS = '-g -Wall -Werror -Wextra -Wno-missing-field-initializers') elif 'g++' in env['CXX'] or 'gcc' in env['CXX']: - env.Append(CXXFLAGS = '-g -O0 -Wall -Werror -Wextra -Wno-missing-field-initializers') + env.Append(CXXFLAGS = '-g -Wall -Werror -Wextra -Wno-missing-field-initializers') elif 'cl' in env['CXX']: env.Append(CXXFLAGS = '/Zi /W2 /WX') -- cgit 1.2.3-korg From d177af163995c8663991e000f129327cb4dc2505 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 15 Apr 2014 20:30:50 +0300 Subject: Fix typos in scons command line options --- tests/SConstruct | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/SConstruct b/tests/SConstruct index b411b48e..7a278443 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -19,8 +19,8 @@ env = Environment(ENV = os.environ, tools = ['default', 'nanopb']) # Allow overriding the compiler with scons CC=??? if 'CC' in ARGUMENTS: env.Replace(CC = ARGUMENTS['CC']) if 'CXX' in ARGUMENTS: env.Replace(CXX = ARGUMENTS['CXX']) -if 'CFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CFLAGS']) -if 'CXXFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CXXFLAGS']) +if 'CCFLAGS' in ARGUMENTS: env.Append(CCFLAGS = ARGUMENTS['CCFLAGS']) +if 'CXXFLAGS' in ARGUMENTS: env.Append(CXXFLAGS = ARGUMENTS['CXXFLAGS']) # Add the builders defined in site_init.py add_nanopb_builders(env) -- cgit 1.2.3-korg From e6a57e512fb7ca0198ae2bb95616284387c7e4f9 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 18 Apr 2014 15:40:40 +0300 Subject: Add option to not add timestamps to .pb.h and .pb.c preambles. Patch by rusnakp. Update issue 115 Status: FixedInGit --- generator/nanopb_generator.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 7b6f0ba9..4d8d733f 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -707,7 +707,10 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio ''' yield '/* Automatically generated nanopb header */\n' - yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) + if options.notimestamp: + yield '/* Generated by %s */\n\n' % (nanopb_version) + else: + yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) symbol = make_identifier(headername) yield '#ifndef _PB_%s_\n' % symbol @@ -780,7 +783,10 @@ def generate_source(headername, enums, messages, extensions, options): '''Generate content for a source file.''' yield '/* Automatically generated nanopb constant definitions */\n' - yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) + if options.notimestamp: + yield '/* Generated by %s */\n\n' % (nanopb_version) + else: + yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) yield options.genformat % (headername) yield '\n' @@ -963,6 +969,8 @@ optparser.add_option("-Q", "--generated-include-format", dest="genformat", optparser.add_option("-L", "--library-include-format", dest="libformat", metavar="FORMAT", default='#include <%s>\n', help="Set format string to use for including the nanopb pb.h header. [default: %default]") +optparser.add_option("-T", "--no-timestamp", dest="notimestamp", action="store_true", default=False, + help="Don't add timestamp to .pb.h and .pb.c preambles") optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Don't print anything except errors.") optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", default=False, -- cgit 1.2.3-korg From ba2ab9ea65d029b2560c461be317f3cf0d19eb3e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 26 Apr 2014 20:11:54 +0300 Subject: Docs update, remove malloc from limitations list --- docs/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/index.rst b/docs/index.rst index 7bd1794e..d49abc07 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -47,7 +47,6 @@ Features and limitations **Limitations** -#) User must provide callbacks when decoding arrays or strings without maximum size. Malloc support could be added as a separate module. #) Some speed has been sacrificed for code size. #) Encoding is focused on writing to streams. For memory buffers only it could be made more efficient. #) The deprecated Protocol Buffers feature called "groups" is not supported. -- cgit 1.2.3-korg From 5ef128616baffd15bb904fc56f651d40901be429 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 17 May 2014 20:06:55 +0300 Subject: Fix security issue with PB_ENABLE_MALLOC. The multiplication in allocate_field could potentially overflow, leading to allocating too little memory. This could subsequently allow an attacker to cause a write past the buffer, overwriting other memory contents. The attack is possible if untrusted message data is decoded using nanopb, and the message type includes a pointer-type string or bytes field, or a repeated numeric field. Submessage fields are not affected. This issue only affects systems that have been compiled with PB_ENABLE_MALLOC enabled. Only version nanopb-0.2.7 is affected, as prior versions do not include this functionality. Update issue 117 Status: FixedInGit --- pb_decode.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 9a48c60f..d687cee3 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -470,11 +470,31 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t #ifdef PB_ENABLE_MALLOC /* Allocate storage for the field and store the pointer at iter->pData. - * array_size is the number of entries to reserve in an array. */ + * array_size is the number of entries to reserve in an array. + */ static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size) { void *ptr = *(void**)pData; - size_t size = array_size * data_size; + + /* Check for multiplication overflows. */ + size_t size = 0; + if (data_size > 0 && array_size > 0) + { + /* Avoid the costly division if the sizes are small enough. + * Multiplication is safe as long as only half of bits are set + * in either multiplicand. + */ + const size_t check_limit = (size_t)1 << (sizeof(size_t) * 4); + if (data_size >= check_limit || array_size >= check_limit) + { + if (SIZE_MAX / array_size < data_size) + { + PB_RETURN_ERROR(stream, "size too large"); + } + } + + size = array_size * data_size; + } /* Allocate new or expand previous allocation */ /* Note: on failure the old pointer will remain in the structure, -- cgit 1.2.3-korg From 9cf788de5463e42aaf8a24ac35a3541fdaf8c517 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 17 May 2014 20:28:33 +0300 Subject: Fix bug in alltypes test case that made fuzzing difficult. --- tests/alltypes/decode_alltypes.c | 1 + tests/alltypes_callback/decode_alltypes_callback.c | 1 + tests/alltypes_pointer/decode_alltypes_pointer.c | 1 + tests/field_size_16/alltypes.proto | 2 ++ tests/field_size_32/alltypes.proto | 2 ++ 5 files changed, 7 insertions(+) diff --git a/tests/alltypes/decode_alltypes.c b/tests/alltypes/decode_alltypes.c index db72bb9f..0202ebc1 100644 --- a/tests/alltypes/decode_alltypes.c +++ b/tests/alltypes/decode_alltypes.c @@ -23,6 +23,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) /* Fill with garbage to better detect initialization errors */ memset(&alltypes, 0xAA, sizeof(alltypes)); + alltypes.extensions = 0; if (!pb_decode(stream, AllTypes_fields, &alltypes)) return false; diff --git a/tests/alltypes_callback/decode_alltypes_callback.c b/tests/alltypes_callback/decode_alltypes_callback.c index 81274f60..b6017fe6 100644 --- a/tests/alltypes_callback/decode_alltypes_callback.c +++ b/tests/alltypes_callback/decode_alltypes_callback.c @@ -220,6 +220,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) /* Fill with garbage to better detect initialization errors */ memset(&alltypes, 0xAA, sizeof(alltypes)); + alltypes.extensions = 0; alltypes.req_int32.funcs.decode = &read_varint; alltypes.req_int32.arg = (void*)-1001; diff --git a/tests/alltypes_pointer/decode_alltypes_pointer.c b/tests/alltypes_pointer/decode_alltypes_pointer.c index 889676b4..1e719019 100644 --- a/tests/alltypes_pointer/decode_alltypes_pointer.c +++ b/tests/alltypes_pointer/decode_alltypes_pointer.c @@ -19,6 +19,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) /* Fill with garbage to better detect initialization errors */ memset(&alltypes, 0xAA, sizeof(alltypes)); + alltypes.extensions = 0; if (!pb_decode(stream, AllTypes_fields, &alltypes)) return false; diff --git a/tests/field_size_16/alltypes.proto b/tests/field_size_16/alltypes.proto index 7494853d..81693c00 100644 --- a/tests/field_size_16/alltypes.proto +++ b/tests/field_size_16/alltypes.proto @@ -107,5 +107,7 @@ message AllTypes { // Just to make sure that the size of the fields has been calculated // properly, i.e. otherwise a bug in last field might not be detected. required int32 end = 10099; + + extensions 200 to 255; } diff --git a/tests/field_size_32/alltypes.proto b/tests/field_size_32/alltypes.proto index 17f17eeb..81717bbc 100644 --- a/tests/field_size_32/alltypes.proto +++ b/tests/field_size_32/alltypes.proto @@ -107,5 +107,7 @@ message AllTypes { // Just to make sure that the size of the fields has been calculated // properly, i.e. otherwise a bug in last field might not be detected. required int32 end = 13432099; + + extensions 200 to 255; } -- cgit 1.2.3-korg From 916bcb3643b257ab2cb2cbc6a74da2c0cb74d275 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 20 May 2014 19:34:28 +0300 Subject: Publishing nanopb-0.2.8 --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 4d8d733f..e494f002 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.8-dev" +nanopb_version = "nanopb-0.2.8" import sys diff --git a/pb.h b/pb.h index fe6fb5b3..258ec925 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.8-dev +#define NANOPB_VERSION nanopb-0.2.8 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 2c51fb7771687cdd75377094d09c6c01ef66e2cc Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 20 May 2014 19:46:48 +0300 Subject: Update changelog for 0.2.8 --- CHANGELOG.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 6d643ade..e35ee3ab 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,9 @@ +nanopb-0.2.8 (2014-05-20) + Fix security issue with PB_ENABLE_MALLOC. (issue 117) + Add option to not add timestamps to .pb.h and .pb.c preambles. (issue 115) + Documentation updates + Improved tests + nanopb-0.2.7 (2014-04-07) Fix bug with default values for extension fields (issue 111) Fix some MISRA-C warnings (issue 91) -- cgit 1.2.3-korg From 2e9797af5820c41645c98c4d156ae24961835d4e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 20 May 2014 19:52:09 +0300 Subject: Setting version to 0.2.9-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index e494f002..1f263a74 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.8" +nanopb_version = "nanopb-0.2.9-dev" import sys diff --git a/pb.h b/pb.h index 258ec925..c66375df 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.8 +#define NANOPB_VERSION nanopb-0.2.9-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 8611958a7f5ec16261cbacaf62a0ea92fd9dd314 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 30 May 2014 13:45:48 +0300 Subject: Add PB_PACKED_STRUCT support for Keil MDK-ARM toolchain Patch from Jon Read. Update issue 119 Status: FixedInGit --- pb.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pb.h b/pb.h index c66375df..2c1851be 100644 --- a/pb.h +++ b/pb.h @@ -80,8 +80,8 @@ # define PB_PACKED_STRUCT_START # define PB_PACKED_STRUCT_END # define pb_packed __attribute__((packed)) -#elif defined(__ICCARM__) - /* For IAR ARM compiler */ +#elif defined(__ICCARM__) || defined(__CC_ARM) + /* For IAR ARM and Keil MDK-ARM compilers */ # define PB_PACKED_STRUCT_START _Pragma("pack(push, 1)") # define PB_PACKED_STRUCT_END _Pragma("pack(pop)") # define pb_packed -- cgit 1.2.3-korg From 8a857a7f7519414c2294cc9f3286ebe5dba5a6f3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 2 Jun 2014 21:09:06 +0300 Subject: Don't use SIZE_MAX macro, as it is not in C89. Update issue 120 Status: FixedInGit --- pb_decode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pb_decode.c b/pb_decode.c index d687cee3..8b782a6f 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -487,7 +487,8 @@ static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t const size_t check_limit = (size_t)1 << (sizeof(size_t) * 4); if (data_size >= check_limit || array_size >= check_limit) { - if (SIZE_MAX / array_size < data_size) + const size_t size_max = (size_t)-1; + if (size_max / array_size < data_size) { PB_RETURN_ERROR(stream, "size too large"); } -- cgit 1.2.3-korg From 99bc1d4f97710cc71612869d3d7e0f501a4626ff Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 2 Jun 2014 21:12:38 +0300 Subject: Make clearer that size = 0 in allocate_field() is not allowed. Back in design phase the code used realloc() for freeing the memory also. However, this is not entirely portable, and therefore the finished implementation used free() separately. There were some remnants of the size = 0 code in the allocate_field() code, which made it somewhat confusing. This change makes it clearer that size = 0 is not allowed (and not used by nanopb). --- pb_decode.c | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 8b782a6f..4e187256 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -471,36 +471,31 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t #ifdef PB_ENABLE_MALLOC /* Allocate storage for the field and store the pointer at iter->pData. * array_size is the number of entries to reserve in an array. + * Zero size is not allowed, use pb_free() for releasing. */ static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size) { void *ptr = *(void**)pData; - /* Check for multiplication overflows. */ - size_t size = 0; - if (data_size > 0 && array_size > 0) + /* Check for multiplication overflows. + * This code avoids the costly division if the sizes are small enough. + * Multiplication is safe as long as only half of bits are set + * in either multiplicand. + */ + const size_t check_limit = (size_t)1 << (sizeof(size_t) * 4); + if (data_size >= check_limit || array_size >= check_limit) { - /* Avoid the costly division if the sizes are small enough. - * Multiplication is safe as long as only half of bits are set - * in either multiplicand. - */ - const size_t check_limit = (size_t)1 << (sizeof(size_t) * 4); - if (data_size >= check_limit || array_size >= check_limit) + const size_t size_max = (size_t)-1; + if (size_max / array_size < data_size) { - const size_t size_max = (size_t)-1; - if (size_max / array_size < data_size) - { - PB_RETURN_ERROR(stream, "size too large"); - } + PB_RETURN_ERROR(stream, "size too large"); } - - size = array_size * data_size; } /* Allocate new or expand previous allocation */ /* Note: on failure the old pointer will remain in the structure, * the message must be freed by caller also on error return. */ - ptr = pb_realloc(ptr, size); + ptr = pb_realloc(ptr, array_size * data_size); if (ptr == NULL) PB_RETURN_ERROR(stream, "realloc failed"); -- cgit 1.2.3-korg From 788d2825b06426f50064dc5f29b07ce0be105b2b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 2 Jun 2014 21:20:57 +0300 Subject: Add unit tests for allocate_field(). --- tests/decode_unittests/decode_unittests.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/decode_unittests/decode_unittests.c b/tests/decode_unittests/decode_unittests.c index 1be01913..98055df3 100644 --- a/tests/decode_unittests/decode_unittests.c +++ b/tests/decode_unittests/decode_unittests.c @@ -1,4 +1,5 @@ /* This includes the whole .c file to get access to static functions. */ +#define PB_ENABLE_MALLOC #include "pb_decode.c" #include @@ -299,6 +300,28 @@ int main() dest.submsg.data_count == 5) } + { + pb_istream_t s = {0}; + void *data = NULL; + + COMMENT("Testing allocate_field") + TEST(allocate_field(&s, &data, 10, 10) && data != NULL); + TEST(allocate_field(&s, &data, 10, 20) && data != NULL); + + { + void *oldvalue = data; + size_t very_big = (size_t)-1; + size_t somewhat_big = very_big / 2 + 1; + size_t not_so_big = (size_t)1 << (4 * sizeof(size_t)); + + TEST(!allocate_field(&s, &data, very_big, 2) && data == oldvalue); + TEST(!allocate_field(&s, &data, somewhat_big, 2) && data == oldvalue); + TEST(!allocate_field(&s, &data, not_so_big, not_so_big) && data == oldvalue); + } + + pb_free(data); + } + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); -- cgit 1.2.3-korg From f2f9f8a9ed2f59e4ed229f09fc95f9168cc4b473 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 20 Jul 2014 14:02:56 +0300 Subject: Fix problem with .options file and extension fields. The options for an extension field were being looked up under wrong name (MessageName instead of MessageName.fieldname). Fixed the problem and added regression test. Created a new subfolder for regression test cases. Update issue 125 Status: FixedInGit --- generator/nanopb_generator.py | 2 +- tests/SConstruct | 2 +- tests/regression/issue_125/SConscript | 9 +++++++++ tests/regression/issue_125/extensionbug.expected | 3 +++ tests/regression/issue_125/extensionbug.options | 4 ++++ tests/regression/issue_125/extensionbug.proto | 16 ++++++++++++++++ 6 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 tests/regression/issue_125/SConscript create mode 100644 tests/regression/issue_125/extensionbug.expected create mode 100644 tests/regression/issue_125/extensionbug.options create mode 100644 tests/regression/issue_125/extensionbug.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 1f263a74..3bc8b39a 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -645,7 +645,7 @@ def parse_file(fdesc, file_options): enums.append(Enum(names, enum, enum_options)) for names, extension in iterate_extensions(fdesc, base_name): - field_options = get_nanopb_suboptions(extension, file_options, names) + field_options = get_nanopb_suboptions(extension, file_options, names + extension.name) if field_options.type != nanopb_pb2.FT_IGNORE: extensions.append(ExtensionField(names, extension, field_options)) diff --git a/tests/SConstruct b/tests/SConstruct index 7a278443..57167ccc 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -141,6 +141,6 @@ import os.path env['VARIANT_DIR'] = 'build' env['BUILD'] = '#' + env['VARIANT_DIR'] env['COMMON'] = '#' + env['VARIANT_DIR'] + '/common' -for subdir in Glob('*/SConscript'): +for subdir in Glob('*/SConscript') + Glob('regression/*/SConscript'): SConscript(subdir, exports = 'env', variant_dir = env['VARIANT_DIR'] + '/' + os.path.dirname(str(subdir))) diff --git a/tests/regression/issue_125/SConscript b/tests/regression/issue_125/SConscript new file mode 100644 index 00000000..f2155e63 --- /dev/null +++ b/tests/regression/issue_125/SConscript @@ -0,0 +1,9 @@ +# Regression test for Issue 125: Wrong identifier name for extension fields + +Import("env") + +env.NanopbProto(["extensionbug", "extensionbug.options"]) +env.Object('extensionbug.pb.c') + +env.Match(['extensionbug.pb.h', 'extensionbug.expected']) + diff --git a/tests/regression/issue_125/extensionbug.expected b/tests/regression/issue_125/extensionbug.expected new file mode 100644 index 00000000..fc213354 --- /dev/null +++ b/tests/regression/issue_125/extensionbug.expected @@ -0,0 +1,3 @@ +pb_extension_type_t Message2_extras +uint32_t field2 + diff --git a/tests/regression/issue_125/extensionbug.options b/tests/regression/issue_125/extensionbug.options new file mode 100644 index 00000000..30b464a4 --- /dev/null +++ b/tests/regression/issue_125/extensionbug.options @@ -0,0 +1,4 @@ +* type:FT_IGNORE + +Message2.extras type:FT_STATIC +Message2.field2 type:FT_STATIC diff --git a/tests/regression/issue_125/extensionbug.proto b/tests/regression/issue_125/extensionbug.proto new file mode 100644 index 00000000..c4ac6860 --- /dev/null +++ b/tests/regression/issue_125/extensionbug.proto @@ -0,0 +1,16 @@ +message Message1 +{ + optional uint32 fieldA = 1; + extensions 30 to max; +} + +message Message2 +{ + extend Message1 + { + optional Message2 extras = 30; + } + + optional uint32 field1 = 1; + optional uint32 field2 = 2; +} -- cgit 1.2.3-korg From 7f97ad549e8ef2050c662b8160745402e84e8006 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 20 Jul 2014 14:10:47 +0300 Subject: Give better messages about the .options file path. Update issue 124 Status: FixedInGit --- generator/nanopb_generator.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 3bc8b39a..c7371887 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1000,19 +1000,28 @@ def process_file(filename, fdesc, options): fdesc = descriptor.FileDescriptorSet.FromString(data).file[0] # Check if there is a separate .options file + had_abspath = False try: optfilename = options.options_file % os.path.splitext(filename)[0] except TypeError: # No %s specified, use the filename as-is optfilename = options.options_file - + had_abspath = True + if os.path.isfile(optfilename): if options.verbose: sys.stderr.write('Reading options from ' + optfilename + '\n') Globals.separate_options = read_options_file(open(optfilename, "rU")) else: + # If we are given a full filename and it does not exist, give an error. + # However, don't give error when we automatically look for .options file + # with the same name as .proto. + if options.verbose or had_abspath: + sys.stderr.write('Options file not found: ' + optfilename) + Globals.separate_options = [] + Globals.matched_namemasks = set() # Parse the file -- cgit 1.2.3-korg From 3cf9668c755560a69e4858dea4f3c415bf807441 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 20 Jul 2014 14:25:11 +0300 Subject: Do not automatically add a dot with generator -e option. Now -e option in generator is more versatile. Especially it avoids double-dot problem with some build systems. Given foobar.proto, we now get: -e .pb => foobar.pb.c (default) -e _pb => foobar_pb.c -e '' => foobar.c Note that if you have used -e option previously, you will have to prepend . to the argument to get the same filenames as before. Update issue 122 Status: FixedInGit --- generator/nanopb_generator.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index c7371887..8ef4f187 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -724,7 +724,7 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio for dependency in dependencies: noext = os.path.splitext(dependency)[0] - yield options.genformat % (noext + '.' + options.extension + '.h') + yield options.genformat % (noext + options.extension + '.h') yield '\n' yield '#ifdef __cplusplus\n' @@ -959,8 +959,8 @@ optparser = OptionParser( "Output will be written to file.pb.h and file.pb.c.") optparser.add_option("-x", dest="exclude", metavar="FILE", action="append", default=[], help="Exclude file from generated #include list.") -optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", default="pb", - help="Set extension to use instead of 'pb' for generated files. [default: %default]") +optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", default=".pb", + help="Set extension to use instead of '.pb' for generated files. [default: %default]") optparser.add_option("-f", "--options-file", dest="options_file", metavar="FILE", default="%s.options", help="Set name of a separate generator options file.") optparser.add_option("-Q", "--generated-include-format", dest="genformat", @@ -1027,11 +1027,11 @@ def process_file(filename, fdesc, options): # Parse the file file_options = get_nanopb_suboptions(fdesc, toplevel_options, Names([filename])) enums, messages, extensions = parse_file(fdesc, file_options) - + # Decide the file names noext = os.path.splitext(filename)[0] - headername = noext + '.' + options.extension + '.h' - sourcename = noext + '.' + options.extension + '.c' + headername = noext + options.extension + '.h' + sourcename = noext + options.extension + '.c' headerbasename = os.path.basename(headername) # List of .proto files that should not be included in the C header file -- cgit 1.2.3-korg From eaa3c7b157ffd2a308dfc6d35d79309a9aacbcef Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 20 Jul 2014 14:44:41 +0300 Subject: Cleanup and comment the code of network_server example. Update issue 123 Status: FixedInGit --- examples/network_server/client.c | 98 +++++++++++++++++++++++++--------------- examples/network_server/server.c | 85 ++++++++++++++++++++++------------ 2 files changed, 118 insertions(+), 65 deletions(-) diff --git a/examples/network_server/client.c b/examples/network_server/client.c index e6e9a2e0..00f6dab8 100644 --- a/examples/network_server/client.c +++ b/examples/network_server/client.c @@ -23,9 +23,13 @@ #include "fileproto.pb.h" #include "common.h" +/* This callback function will be called once for each filename received + * from the server. The filenames will be printed out immediately, so that + * no memory has to be allocated for them. + */ bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void **arg) { - FileInfo fileinfo; + FileInfo fileinfo = {}; if (!pb_decode(stream, FileInfo_fields, &fileinfo)) return false; @@ -35,51 +39,70 @@ bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void **ar return true; } +/* This function sends a request to socket 'fd' to list the files in + * directory given in 'path'. The results received from server will + * be printed to stdout. + */ bool listdir(int fd, char *path) { - ListFilesRequest request; - ListFilesResponse response; - pb_istream_t input = pb_istream_from_socket(fd); - pb_ostream_t output = pb_ostream_from_socket(fd); - uint8_t zero = 0; - - if (path == NULL) - { - request.has_path = false; - } - else + /* Construct and send the request to server */ { - request.has_path = true; - if (strlen(path) + 1 > sizeof(request.path)) + ListFilesRequest request = {}; + pb_ostream_t output = pb_ostream_from_socket(fd); + uint8_t zero = 0; + + /* In our protocol, path is optional. If it is not given, + * the server will list the root directory. */ + if (path == NULL) + { + request.has_path = false; + } + else { - fprintf(stderr, "Too long path.\n"); + request.has_path = true; + if (strlen(path) + 1 > sizeof(request.path)) + { + fprintf(stderr, "Too long path.\n"); + return false; + } + + strcpy(request.path, path); + } + + /* Encode the request. It is written to the socket immediately + * through our custom stream. */ + if (!pb_encode(&output, ListFilesRequest_fields, &request)) + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&output)); return false; } - strcpy(request.path, path); + /* We signal the end of request with a 0 tag. */ + pb_write(&output, &zero, 1); } - if (!pb_encode(&output, ListFilesRequest_fields, &request)) + /* Read back the response from server */ { - fprintf(stderr, "Encoding failed.\n"); - return false; - } - - /* We signal the end of request with a 0 tag. */ - pb_write(&output, &zero, 1); - - response.file.funcs.decode = &printfile_callback; - - if (!pb_decode(&input, ListFilesResponse_fields, &response)) - { - fprintf(stderr, "Decode failed: %s\n", PB_GET_ERROR(&input)); - return false; - } - - if (response.path_error) - { - fprintf(stderr, "Server reported error.\n"); - return false; + ListFilesResponse response = {}; + pb_istream_t input = pb_istream_from_socket(fd); + + /* Give a pointer to our callback function, which will handle the + * filenames as they arrive. */ + response.file.funcs.decode = &printfile_callback; + + if (!pb_decode(&input, ListFilesResponse_fields, &response)) + { + fprintf(stderr, "Decode failed: %s\n", PB_GET_ERROR(&input)); + return false; + } + + /* If the message from server decodes properly, but directory was + * not found on server side, we get path_error == true. */ + if (response.path_error) + { + fprintf(stderr, "Server reported error.\n"); + return false; + } } return true; @@ -96,6 +119,7 @@ int main(int argc, char **argv) sockfd = socket(AF_INET, SOCK_STREAM, 0); + /* Connect to server running on localhost:1234 */ memset(&servaddr, 0, sizeof(servaddr)); servaddr.sin_family = AF_INET; servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); @@ -107,9 +131,11 @@ int main(int argc, char **argv) return 1; } + /* Send the directory listing request */ if (!listdir(sockfd, path)) return 2; + /* Close connection */ close(sockfd); return 0; diff --git a/examples/network_server/server.c b/examples/network_server/server.c index 9a9c2644..46a5f38d 100644 --- a/examples/network_server/server.c +++ b/examples/network_server/server.c @@ -23,11 +23,16 @@ #include "fileproto.pb.h" #include "common.h" +/* This callback function will be called once during the encoding. + * It will write out any number of FileInfo entries, without consuming unnecessary memory. + * This is accomplished by fetching the filenames one at a time and encoding them + * immediately. + */ bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) { DIR *dir = (DIR*) *arg; struct dirent *file; - FileInfo fileinfo; + FileInfo fileinfo = {}; while ((file = readdir(dir)) != NULL) { @@ -35,9 +40,12 @@ bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, void * cons strncpy(fileinfo.name, file->d_name, sizeof(fileinfo.name)); fileinfo.name[sizeof(fileinfo.name) - 1] = '\0'; + /* This encodes the header for the field, based on the constant info + * from pb_field_t. */ if (!pb_encode_tag_for_field(stream, field)) return false; + /* This encodes the data for the field, based on our FileInfo structure. */ if (!pb_encode_submessage(stream, FileInfo_fields, &fileinfo)) return false; } @@ -45,43 +53,59 @@ bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, void * cons return true; } +/* Handle one arriving client connection. + * Clients are expected to send a ListFilesRequest, terminated by a '0'. + * Server will respond with a ListFilesResponse message. + */ void handle_connection(int connfd) { - ListFilesRequest request; - ListFilesResponse response; - pb_istream_t input = pb_istream_from_socket(connfd); - pb_ostream_t output = pb_ostream_from_socket(connfd); - DIR *directory; + DIR *directory = NULL; - if (!pb_decode(&input, ListFilesRequest_fields, &request)) + /* Decode the message from the client and open the requested directory. */ { - printf("Decode failed: %s\n", PB_GET_ERROR(&input)); - return; + ListFilesRequest request = {}; + pb_istream_t input = pb_istream_from_socket(connfd); + + if (!pb_decode(&input, ListFilesRequest_fields, &request)) + { + printf("Decode failed: %s\n", PB_GET_ERROR(&input)); + return; + } + + directory = opendir(request.path); + printf("Listing directory: %s\n", request.path); } - directory = opendir(request.path); - - printf("Listing directory: %s\n", request.path); - - if (directory == NULL) + /* List the files in the directory and transmit the response to client */ { - perror("opendir"); + ListFilesResponse response = {}; + pb_ostream_t output = pb_ostream_from_socket(connfd); - response.has_path_error = true; - response.path_error = true; - response.file.funcs.encode = NULL; - } - else - { - response.has_path_error = false; - response.file.funcs.encode = &listdir_callback; - response.file.arg = directory; + if (directory == NULL) + { + perror("opendir"); + + /* Directory was not found, transmit error status */ + response.has_path_error = true; + response.path_error = true; + response.file.funcs.encode = NULL; + } + else + { + /* Directory was found, transmit filenames */ + response.has_path_error = false; + response.file.funcs.encode = &listdir_callback; + response.file.arg = directory; + } + + if (!pb_encode(&output, ListFilesResponse_fields, &response)) + { + printf("Encoding failed: %s\n", PB_GET_ERROR(&output)); + } } - if (!pb_encode(&output, ListFilesResponse_fields, &response)) - { - printf("Encoding failed.\n"); - } + if (directory != NULL) + closedir(directory); } int main(int argc, char **argv) @@ -90,8 +114,8 @@ int main(int argc, char **argv) struct sockaddr_in servaddr; int reuse = 1; + /* Listen on localhost:1234 for TCP connections */ listenfd = socket(AF_INET, SOCK_STREAM, 0); - setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); memset(&servaddr, 0, sizeof(servaddr)); @@ -112,6 +136,7 @@ int main(int argc, char **argv) for(;;) { + /* Wait for a client */ connfd = accept(listenfd, NULL, NULL); if (connfd < 0) @@ -128,4 +153,6 @@ int main(int argc, char **argv) close(connfd); } + + return 0; } -- cgit 1.2.3-korg From 5749606f5d2f0a0c64cb6e021b7bd40b74ce45ee Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 20 Jul 2014 14:55:47 +0300 Subject: Add support for inverted patterns in test framework. --- tests/site_scons/site_init.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/site_scons/site_init.py b/tests/site_scons/site_init.py index 5fb06d6e..da5f6d65 100644 --- a/tests/site_scons/site_init.py +++ b/tests/site_scons/site_init.py @@ -85,9 +85,20 @@ def add_nanopb_builders(env): data = open(str(source[0]), 'rU').read() patterns = open(str(source[1])) for pattern in patterns: - if pattern.strip() and not re.search(pattern.strip(), data, re.MULTILINE): - print '\033[31m[FAIL]\033[0m Pattern not found in ' + str(source[0]) + ': ' + pattern - return 1 + if pattern.strip(): + invert = False + if pattern.startswith('! '): + invert = True + pattern = pattern[2:] + + status = re.search(pattern.strip(), data, re.MULTILINE) + + if not status and not invert: + print '\033[31m[FAIL]\033[0m Pattern not found in ' + str(source[0]) + ': ' + pattern + return 1 + elif status and invert: + print '\033[31m[FAIL]\033[0m Pattern should not exist, but does in ' + str(source[0]) + ': ' + pattern + return 1 else: print '\033[32m[ OK ]\033[0m All patterns found in ' + str(source[0]) return 0 -- cgit 1.2.3-korg From 1d7f60fec30678ec7403786808026444a4b901e6 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 20 Jul 2014 14:56:12 +0300 Subject: Add skip_message option to generator. Update issue 121 Status: FixedInGit --- generator/nanopb_generator.py | 4 ++++ generator/proto/nanopb.proto | 3 +++ tests/options/options.expected | 3 +++ tests/options/options.proto | 10 +++++++--- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 8ef4f187..2d2071e1 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -639,6 +639,10 @@ def parse_file(fdesc, file_options): for names, message in iterate_messages(fdesc, base_name): message_options = get_nanopb_suboptions(message, file_options, names) + + if message_options.skip_message: + continue + messages.append(Message(names, message, message_options)) for enum in message.enum_type: enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name) diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index 2be2f803..9a4d6578 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -37,6 +37,9 @@ message NanoPBOptions { // Note: this cannot be used on CPUs that break on unaligned // accesses to variables. optional bool packed_struct = 5 [default = false]; + + // Skip this message + optional bool skip_message = 6 [default = false]; } // Extensions to protoc 'Descriptor' type in order to define options diff --git a/tests/options/options.expected b/tests/options/options.expected index e6179a27..dbd279ba 100644 --- a/tests/options/options.expected +++ b/tests/options/options.expected @@ -5,3 +5,6 @@ pb_callback_t int32_callback; \sEnumValue1 = 1 Message5_EnumValue1 } pb_packed my_packed_struct; +! skipped_field +! SkippedMessage + diff --git a/tests/options/options.proto b/tests/options/options.proto index b5badcfd..a8e557be 100644 --- a/tests/options/options.proto +++ b/tests/options/options.proto @@ -63,11 +63,15 @@ message my_packed_struct } // Message with ignored field -// Note: doesn't really test if the field is missing in the output, -// but atleast tests that the output compiles. message Message6 { required int32 field1 = 1; - optional int32 field2 = 2 [(nanopb).type = FT_IGNORE]; + optional int32 skipped_field = 2 [(nanopb).type = FT_IGNORE]; } +// Message that is skipped +message SkippedMessage +{ + option (nanopb_msgopt).skip_message = true; + required int32 foo = 1; +} -- cgit 1.2.3-korg From ec3bff4ba110ffad31fccdbf74c4c180fc041414 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 4 Aug 2014 18:40:40 +0300 Subject: Generate #defines for initializing message structures. Usage like: MyMessage foo = MyMessage_init_default; MyMessage_init_default will initialize to default values defined in .proto. MyMessage_init_zero will initialize to null/zero values. Same results as {} or {0}, but will avoid compiler warnings by initializing everything explicitly. Update issue 79 Status: FixedInGit --- generator/nanopb_generator.py | 95 +++++++++++++++++++++++++++++++------- tests/alltypes/decode_alltypes.c | 3 +- tests/alltypes/encode_alltypes.c | 2 +- tests/basic_buffer/decode_buffer.c | 2 +- tests/basic_stream/decode_stream.c | 2 +- 5 files changed, 84 insertions(+), 20 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 2d2071e1..6031e893 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -292,35 +292,63 @@ class Field: result = None return result + def get_initializer(self, null_init): + '''Return literal expression for this field's default value.''' + + if self.pbtype == 'MESSAGE': + if null_init: + return '%s_init_zero' % self.ctype + else: + return '%s_init_default' % self.ctype + + if self.default is None or null_init: + if self.pbtype == 'STRING': + return '""' + elif self.pbtype == 'BYTES': + return '{0, {0}}' + elif self.pbtype == 'ENUM': + return '(%s)0' % self.ctype + else: + return '0' + + default = str(self.default) + + if self.pbtype == 'STRING': + default = default.encode('utf-8').encode('string_escape') + default = default.replace('"', '\\"') + default = '"' + default + '"' + elif self.pbtype == 'BYTES': + data = default.decode('string_escape') + data = ['0x%02x' % ord(c) for c in data] + if len(data) == 0: + default = '{0, {0}}' + else: + default = '{%d, {%s}}' % (len(data), ','.join(data)) + elif self.pbtype in ['FIXED32', 'UINT32']: + default += 'u' + elif self.pbtype in ['FIXED64', 'UINT64']: + default += 'ull' + elif self.pbtype in ['SFIXED64', 'INT64']: + default += 'll' + + return default + def default_decl(self, declaration_only = False): '''Return definition for this field's default value.''' if self.default is None: return None - ctype, default = self.ctype, self.default + ctype = self.ctype + default = self.get_initializer(False) array_decl = '' if self.pbtype == 'STRING': if self.allocation != 'STATIC': return None # Not implemented - array_decl = '[%d]' % self.max_size - default = str(self.default).encode('string_escape') - default = default.replace('"', '\\"') - default = '"' + default + '"' elif self.pbtype == 'BYTES': if self.allocation != 'STATIC': return None # Not implemented - - data = self.default.decode('string_escape') - data = ['0x%02x' % ord(c) for c in data] - default = '{%d, {%s}}' % (len(data), ','.join(data)) - elif self.pbtype in ['FIXED32', 'UINT32']: - default += 'u' - elif self.pbtype in ['FIXED64', 'UINT64']: - default += 'ull' - elif self.pbtype in ['SFIXED64', 'INT64']: - default += 'll' if declaration_only: return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl) @@ -442,7 +470,7 @@ class ExtensionRange(Field): def tags(self): return '' - + def encoded_size(self, allmsgs): # We exclude extensions from the count, because they cannot be known # until runtime. Other option would be to return None here, but this @@ -553,6 +581,32 @@ class Message: result += types + '\n' return result + def get_initializer(self, null_init): + if not self.ordered_fields: + return '{0}' + + parts = [] + for field in self.ordered_fields: + if field.allocation == 'STATIC': + if field.rules == 'REPEATED': + parts.append('0') + parts.append('{' + + ', '.join([field.get_initializer(null_init)] * field.max_count) + + '}') + elif field.rules == 'OPTIONAL': + parts.append('false') + parts.append(field.get_initializer(null_init)) + else: + parts.append(field.get_initializer(null_init)) + elif field.allocation == 'POINTER': + parts.append('NULL') + elif field.allocation == 'CALLBACK': + if field.pbtype == 'EXTENSION': + parts.append('NULL') + else: + parts.append('{{NULL}, NULL}') + return '{' + ', '.join(parts) + '}' + def default_decl(self, declaration_only = False): result = "" for field in self.fields: @@ -755,6 +809,15 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio yield msg.default_decl(True) yield '\n' + yield '/* Initializer values for message structs */\n' + for msg in messages: + identifier = '%s_init_default' % msg.name + yield '#define %-40s %s\n' % (identifier, msg.get_initializer(False)) + for msg in messages: + identifier = '%s_init_zero' % msg.name + yield '#define %-40s %s\n' % (identifier, msg.get_initializer(True)) + yield '\n' + yield '/* Field tags (for use in manual encoding/decoding) */\n' for msg in sort_dependencies(messages): for field in msg.fields: diff --git a/tests/alltypes/decode_alltypes.c b/tests/alltypes/decode_alltypes.c index 0202ebc1..67cb72c4 100644 --- a/tests/alltypes/decode_alltypes.c +++ b/tests/alltypes/decode_alltypes.c @@ -19,7 +19,8 @@ the decoding and checks the fields. */ bool check_alltypes(pb_istream_t *stream, int mode) { - AllTypes alltypes; + /* Uses _init_default to just make sure that it works. */ + AllTypes alltypes = AllTypes_init_default; /* Fill with garbage to better detect initialization errors */ memset(&alltypes, 0xAA, sizeof(alltypes)); diff --git a/tests/alltypes/encode_alltypes.c b/tests/alltypes/encode_alltypes.c index fa8eec94..deec7903 100644 --- a/tests/alltypes/encode_alltypes.c +++ b/tests/alltypes/encode_alltypes.c @@ -13,7 +13,7 @@ int main(int argc, char **argv) int mode = (argc > 1) ? atoi(argv[1]) : 0; /* Initialize the structure with constants */ - AllTypes alltypes = {0}; + AllTypes alltypes = AllTypes_init_zero; alltypes.req_int32 = -1001; alltypes.req_int64 = -1002; diff --git a/tests/basic_buffer/decode_buffer.c b/tests/basic_buffer/decode_buffer.c index fae9e2fa..291d164c 100644 --- a/tests/basic_buffer/decode_buffer.c +++ b/tests/basic_buffer/decode_buffer.c @@ -16,7 +16,7 @@ bool print_person(pb_istream_t *stream) { int i; - Person person; + Person person = Person_init_zero; if (!pb_decode(stream, Person_fields, &person)) return false; diff --git a/tests/basic_stream/decode_stream.c b/tests/basic_stream/decode_stream.c index 667bf3c5..798dcc50 100644 --- a/tests/basic_stream/decode_stream.c +++ b/tests/basic_stream/decode_stream.c @@ -12,7 +12,7 @@ bool print_person(pb_istream_t *stream) { int i; - Person person; + Person person = Person_init_zero; if (!pb_decode(stream, Person_fields, &person)) return false; -- cgit 1.2.3-korg From 4f76e649299828e88c300a6af9517e778c29e94e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 4 Aug 2014 19:13:39 +0300 Subject: Update changelog --- CHANGELOG.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index e35ee3ab..0282efa4 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,16 @@ +nanopb-0.2.9 (2014-08-xx) + NOTE: If you are using the -e option with the generator, you have + to prepend . to the argument to get the same behaviour as before. + + Do not automatically add a dot with generator -e option. (issue 122) + Fix problem with .options file and extension fields. (issue 125) + Don't use SIZE_MAX macro, as it is not in C89. (issue 120) + Generate #defines for initializing message structures. (issue 79) + Add skip_message option to generator. (issue 121) + Add PB_PACKED_STRUCT support for Keil MDK-ARM toolchain (issue 119) + Give better messages about the .options file path. (issue 124) + Improved tests + nanopb-0.2.8 (2014-05-20) Fix security issue with PB_ENABLE_MALLOC. (issue 117) Add option to not add timestamps to .pb.h and .pb.c preambles. (issue 115) -- cgit 1.2.3-korg From 2f05a35b5f5ce28650db096d61d7793ee982d073 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Aug 2014 22:01:04 +0300 Subject: Publishing nanopb-0.2.9 --- CHANGELOG.txt | 2 +- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 0282efa4..d8b3ec86 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,4 +1,4 @@ -nanopb-0.2.9 (2014-08-xx) +nanopb-0.2.9 (2014-08-09) NOTE: If you are using the -e option with the generator, you have to prepend . to the argument to get the same behaviour as before. diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 6031e893..6bc30aa7 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.9-dev" +nanopb_version = "nanopb-0.2.9" import sys diff --git a/pb.h b/pb.h index 2c1851be..210aa9fd 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.9-dev +#define NANOPB_VERSION nanopb-0.2.9 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 0150b98be60b1e123fa51c561a908c234b6379fe Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 9 Aug 2014 22:18:24 +0300 Subject: Setting version to 0.3.0-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 6bc30aa7..1d12ae3c 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.9" +nanopb_version = "nanopb-0.3.0-dev" import sys diff --git a/pb.h b/pb.h index 210aa9fd..50a07c54 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.9 +#define NANOPB_VERSION nanopb-0.3.0-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From a641e21b34aed824b6b919f7ab9937eaadf09473 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 10 Aug 2014 12:42:01 +0300 Subject: Separate field iterator logic from pb_decode to pb_common. --- pb_common.c | 88 +++++++++++++++++++ pb_common.h | 40 +++++++++ pb_decode.c | 140 +++++++----------------------- tests/alltypes/SConscript | 4 +- tests/alltypes_callback/SConscript | 4 +- tests/alltypes_pointer/SConscript | 5 +- tests/backwards_compatibility/SConscript | 4 +- tests/basic_buffer/SConscript | 4 +- tests/basic_stream/SConscript | 4 +- tests/buffer_only/SConscript | 5 +- tests/callbacks/SConscript | 4 +- tests/common/SConscript | 2 +- tests/cxx_main_program/SConscript | 5 +- tests/decode_unittests/decode_unittests.c | 1 + tests/encode_unittests/encode_unittests.c | 1 + tests/extensions/SConscript | 4 +- tests/field_size_16/SConscript | 5 +- tests/field_size_32/SConscript | 5 +- tests/missing_fields/SConscript | 2 +- tests/no_errmsg/SConscript | 5 +- tests/package_name/SConscript | 2 +- tests/splint/SConscript | 3 + 22 files changed, 199 insertions(+), 138 deletions(-) create mode 100644 pb_common.c create mode 100644 pb_common.h diff --git a/pb_common.c b/pb_common.c new file mode 100644 index 00000000..de217699 --- /dev/null +++ b/pb_common.c @@ -0,0 +1,88 @@ +/* pb_common.c: Common support functions for pb_encode.c and pb_decode.c. + * + * 2014 Petteri Aimonen + */ + +#include "pb_common.h" + +void pb_field_iter_begin(pb_field_iter_t *iter, const pb_field_t *fields, void *dest_struct) +{ + iter->start = fields; + iter->pos = fields; + iter->required_field_index = 0; + iter->dest_struct = dest_struct; + iter->pData = (char*)dest_struct + iter->pos->data_offset; + iter->pSize = (char*)iter->pData + iter->pos->size_offset; +} + +bool pb_field_iter_next(pb_field_iter_t *iter) +{ + const pb_field_t *prev_field = iter->pos; + + if (prev_field->tag == 0) + { + /* Handle empty message types, where the first field is already the terminator. + * In other cases, the iter->pos never points to the terminator. */ + return false; + } + + iter->pos++; + + if (iter->pos->tag == 0) + { + /* Wrapped back to beginning, reinitialize */ + pb_field_iter_begin(iter, iter->start, iter->dest_struct); + return false; + } + else + { + /* Increment the pointers based on previous field size */ + size_t prev_size = prev_field->data_size; + + if (PB_ATYPE(prev_field->type) == PB_ATYPE_STATIC && + PB_HTYPE(prev_field->type) == PB_HTYPE_REPEATED) + { + /* In static arrays, the data_size tells the size of a single entry and + * array_size is the number of entries */ + prev_size *= prev_field->array_size; + } + else if (PB_ATYPE(prev_field->type) == PB_ATYPE_POINTER) + { + /* Pointer fields always have a constant size in the main structure. + * The data_size only applies to the dynamically allocated area. */ + prev_size = sizeof(void*); + } + + if (PB_HTYPE(prev_field->type) == PB_HTYPE_REQUIRED) + { + /* Count the required fields, in order to check their presence in the + * decoder. */ + iter->required_field_index++; + } + + iter->pData = (char*)iter->pData + prev_size + iter->pos->data_offset; + iter->pSize = (char*)iter->pData + iter->pos->size_offset; + return true; + } +} + +bool pb_field_iter_find(pb_field_iter_t *iter, uint32_t tag) +{ + const pb_field_t *start = iter->pos; + + do { + if (iter->pos->tag == tag && + PB_LTYPE(iter->pos->type) != PB_LTYPE_EXTENSION) + { + /* Found the wanted field */ + return true; + } + + (void)pb_field_iter_next(iter); + } while (iter->pos != start); + + /* Searched all the way back to start, and found nothing. */ + return false; +} + + diff --git a/pb_common.h b/pb_common.h new file mode 100644 index 00000000..e85f000c --- /dev/null +++ b/pb_common.h @@ -0,0 +1,40 @@ +/* pb_common.h: Common support functions for pb_encode.c and pb_decode.c. + * These functions are rarely needed by applications directly. + */ + +#ifndef _PB_COMMON_H_ +#define _PB_COMMON_H_ + +#include "pb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Iterator for pb_field_t list */ +typedef struct { + const pb_field_t *start; /* Start of the pb_field_t array */ + const pb_field_t *pos; /* Current position of the iterator */ + unsigned required_field_index; /* Zero-based index that counts only the required fields */ + void *dest_struct; /* Pointer to the destination structure to decode to */ + void *pData; /* Pointer where to store current field value */ + void *pSize; /* Pointer where to store the size of current array field */ +} pb_field_iter_t; + +/* Initialize the field iterator structure to beginning. */ +void pb_field_iter_begin(pb_field_iter_t *iter, const pb_field_t *fields, void *dest_struct); + +/* Advance the iterator to the next field. + * Returns false when the iterator wraps back to the first field. */ +bool pb_field_iter_next(pb_field_iter_t *iter); + +/* Advance the iterator until it points at a field with the given tag. + * Returns false if no such field exists. */ +bool pb_field_iter_find(pb_field_iter_t *iter, uint32_t tag); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif + diff --git a/pb_decode.c b/pb_decode.c index 4e187256..40da1aa4 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -15,36 +15,23 @@ #include "pb.h" #include "pb_decode.h" +#include "pb_common.h" /************************************** * Declarations internal to this file * **************************************/ -/* Iterator for pb_field_t list */ -typedef struct { - const pb_field_t *start; /* Start of the pb_field_t array */ - const pb_field_t *pos; /* Current position of the iterator */ - unsigned field_index; /* Zero-based index of the field. */ - unsigned required_field_index; /* Zero-based index that counts only the required fields */ - void *dest_struct; /* Pointer to the destination structure to decode to */ - void *pData; /* Pointer where to store current field value */ - void *pSize; /* Pointer where to store the size of current array field */ -} pb_field_iterator_t; - typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn; static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count); static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest); static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, uint8_t *buf, size_t *size); -static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct); -static bool pb_field_next(pb_field_iterator_t *iter); -static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag); -static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter); -static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter); -static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter); +static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); +static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); +static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type); -static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_field_iterator_t *iter); -static bool checkreturn find_extension_field(pb_field_iterator_t *iter); +static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_field_iter_t *iter); +static bool checkreturn find_extension_field(pb_field_iter_t *iter); static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct); static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest); static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, void *dest); @@ -338,75 +325,11 @@ void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) #endif } -static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct) -{ - iter->start = iter->pos = fields; - iter->field_index = 0; - iter->required_field_index = 0; - iter->pData = (char*)dest_struct + iter->pos->data_offset; - iter->pSize = (char*)iter->pData + iter->pos->size_offset; - iter->dest_struct = dest_struct; -} - -static bool pb_field_next(pb_field_iterator_t *iter) -{ - bool notwrapped = true; - size_t prev_size = iter->pos->data_size; - - if (PB_ATYPE(iter->pos->type) == PB_ATYPE_STATIC && - PB_HTYPE(iter->pos->type) == PB_HTYPE_REPEATED) - { - prev_size *= iter->pos->array_size; - } - else if (PB_ATYPE(iter->pos->type) == PB_ATYPE_POINTER) - { - prev_size = sizeof(void*); - } - - if (iter->pos->tag == 0) - return false; /* Only happens with empty message types */ - - if (PB_HTYPE(iter->pos->type) == PB_HTYPE_REQUIRED) - iter->required_field_index++; - - iter->pos++; - iter->field_index++; - if (iter->pos->tag == 0) - { - iter->pos = iter->start; - iter->field_index = 0; - iter->required_field_index = 0; - iter->pData = iter->dest_struct; - prev_size = 0; - notwrapped = false; - } - - iter->pData = (char*)iter->pData + prev_size + iter->pos->data_offset; - iter->pSize = (char*)iter->pData + iter->pos->size_offset; - return notwrapped; -} - -static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag) -{ - unsigned start = iter->field_index; - - do { - if (iter->pos->tag == tag && - PB_LTYPE(iter->pos->type) != PB_LTYPE_EXTENSION) - { - return true; - } - (void)pb_field_next(iter); - } while (iter->field_index != start); - - return false; -} - /************************* * Decode a single field * *************************/ -static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) +static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter) { pb_type_t type; pb_decoder_t func; @@ -504,7 +427,7 @@ static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t } /* Clear a newly allocated item in case it contains a pointer, or is a submessage. */ -static void initialize_pointer_field(void *pItem, pb_field_iterator_t *iter) +static void initialize_pointer_field(void *pItem, pb_field_iter_t *iter) { if (PB_LTYPE(iter->pos->type) == PB_LTYPE_STRING || PB_LTYPE(iter->pos->type) == PB_LTYPE_BYTES) @@ -518,7 +441,7 @@ static void initialize_pointer_field(void *pItem, pb_field_iterator_t *iter) } #endif -static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) +static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter) { #ifndef PB_ENABLE_MALLOC UNUSED(wire_type); @@ -614,7 +537,7 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ #endif } -static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) +static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter) { pb_callback_t *pCallback = (pb_callback_t*)iter->pData; @@ -661,7 +584,7 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type } } -static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) +static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter) { switch (PB_ATYPE(iter->pos->type)) { @@ -685,16 +608,15 @@ static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type) { const pb_field_t *field = (const pb_field_t*)extension->type->arg; - pb_field_iterator_t iter; + pb_field_iter_t iter; if (field->tag != tag) return true; - iter.start = field; - iter.pos = field; - iter.field_index = 0; - iter.required_field_index = 0; - iter.dest_struct = extension->dest; + /* Fake a field iterator for the extension field. + * It is not actually safe to advance this iterator, but decode_field + * will not even try to. */ + pb_field_iter_begin(&iter, field, extension->dest); iter.pData = extension->dest; iter.pSize = &extension->found; @@ -704,7 +626,7 @@ static bool checkreturn default_extension_decoder(pb_istream_t *stream, /* Try to decode an unknown field as an extension field. Tries each extension * decoder in turn, until one of them handles the field or loop ends. */ static bool checkreturn decode_extension(pb_istream_t *stream, - uint32_t tag, pb_wire_type_t wire_type, pb_field_iterator_t *iter) + uint32_t tag, pb_wire_type_t wire_type, pb_field_iter_t *iter) { pb_extension_t *extension = *(pb_extension_t* const *)iter->pData; size_t pos = stream->bytes_left; @@ -729,15 +651,15 @@ static bool checkreturn decode_extension(pb_istream_t *stream, /* Step through the iterator until an extension field is found or until all * entries have been checked. There can be only one extension field per * message. Returns false if no extension field is found. */ -static bool checkreturn find_extension_field(pb_field_iterator_t *iter) +static bool checkreturn find_extension_field(pb_field_iter_t *iter) { - unsigned start = iter->field_index; + const pb_field_t *start = iter->pos; do { if (PB_LTYPE(iter->pos->type) == PB_LTYPE_EXTENSION) return true; - (void)pb_field_next(iter); - } while (iter->field_index != start); + (void)pb_field_iter_next(iter); + } while (iter->pos != start); return false; } @@ -745,8 +667,8 @@ static bool checkreturn find_extension_field(pb_field_iterator_t *iter) /* Initialize message fields to default values, recursively */ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct) { - pb_field_iterator_t iter; - pb_field_init(&iter, fields, dest_struct); + pb_field_iter_t iter; + pb_field_iter_begin(&iter, fields, dest_struct); do { @@ -803,7 +725,7 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str { /* Don't overwrite callback */ } - } while (pb_field_next(&iter)); + } while (pb_field_iter_next(&iter)); } /********************* @@ -814,9 +736,9 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ { uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {0, 0, 0, 0, 0, 0, 0, 0}; uint32_t extension_range_start = 0; - pb_field_iterator_t iter; + pb_field_iter_t iter; - pb_field_init(&iter, fields, dest_struct); + pb_field_iter_begin(&iter, fields, dest_struct); while (stream->bytes_left) { @@ -832,7 +754,7 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ return false; } - if (!pb_field_find(&iter, tag)) + if (!pb_field_iter_find(&iter, tag)) { /* No match found, check if it matches an extension. */ if (tag >= extension_range_start) @@ -885,7 +807,7 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ do { req_field_count = iter.required_field_index; last_type = iter.pos->type; - } while (pb_field_next(&iter)); + } while (pb_field_iter_next(&iter)); /* Fixup if last field was also required. */ if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.pos->tag != 0) @@ -936,8 +858,8 @@ bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void * #ifdef PB_ENABLE_MALLOC void pb_release(const pb_field_t fields[], void *dest_struct) { - pb_field_iterator_t iter; - pb_field_init(&iter, fields, dest_struct); + pb_field_iter_t iter; + pb_field_iter_begin(&iter, fields, dest_struct); do { @@ -985,7 +907,7 @@ void pb_release(const pb_field_t fields[], void *dest_struct) pb_free(*(void**)iter.pData); *(void**)iter.pData = NULL; } - } while (pb_field_next(&iter)); + } while (pb_field_iter_next(&iter)); } #endif diff --git a/tests/alltypes/SConscript b/tests/alltypes/SConscript index 9c9072ba..6c6238c6 100644 --- a/tests/alltypes/SConscript +++ b/tests/alltypes/SConscript @@ -4,8 +4,8 @@ Import("env") env.NanopbProto(["alltypes", "alltypes.options"]) -enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) -dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_decode.o"]) +enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) # Test the round-trip from nanopb encoder to nanopb decoder env.RunTest(enc) diff --git a/tests/alltypes_callback/SConscript b/tests/alltypes_callback/SConscript index 71b01600..a241f24e 100644 --- a/tests/alltypes_callback/SConscript +++ b/tests/alltypes_callback/SConscript @@ -6,8 +6,8 @@ c = Copy("$TARGET", "$SOURCE") env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) env.NanopbProto(["alltypes", "alltypes.options"]) -enc = env.Program(["encode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) -dec = env.Program(["decode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_decode.o"]) +enc = env.Program(["encode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX" diff --git a/tests/alltypes_pointer/SConscript b/tests/alltypes_pointer/SConscript index e48d6aad..8fcf197e 100644 --- a/tests/alltypes_pointer/SConscript +++ b/tests/alltypes_pointer/SConscript @@ -18,13 +18,14 @@ strict = env.Clone() strict.Append(CFLAGS = strict['CORECFLAGS']) strict.Object("pb_decode_with_malloc.o", "$NANOPB/pb_decode.c") strict.Object("pb_encode_with_malloc.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common_with_malloc.o", "$NANOPB/pb_common.c") c = Copy("$TARGET", "$SOURCE") env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) env.NanopbProto(["alltypes", "alltypes.options"]) -enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "pb_encode_with_malloc.o"]) -dec = env.Program(["decode_alltypes_pointer.c", "alltypes.pb.c", "pb_decode_with_malloc.o"]) +enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "pb_encode_with_malloc.o", "pb_common_with_malloc.o"]) +dec = env.Program(["decode_alltypes_pointer.c", "alltypes.pb.c", "pb_decode_with_malloc.o", "pb_common_with_malloc.o"]) # Encode and compare results to non-pointer alltypes test case env.RunTest(enc) diff --git a/tests/backwards_compatibility/SConscript b/tests/backwards_compatibility/SConscript index 777ef402..81b03182 100644 --- a/tests/backwards_compatibility/SConscript +++ b/tests/backwards_compatibility/SConscript @@ -3,8 +3,8 @@ Import("env") -enc = env.Program(["encode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_encode.o"]) -dec = env.Program(["decode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_decode.o"]) +enc = env.Program(["encode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) env.RunTest(enc) env.RunTest([dec, "encode_legacy.output"]) diff --git a/tests/basic_buffer/SConscript b/tests/basic_buffer/SConscript index 2546aaa3..acaf5ffa 100644 --- a/tests/basic_buffer/SConscript +++ b/tests/basic_buffer/SConscript @@ -2,8 +2,8 @@ Import("env") -enc = env.Program(["encode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o"]) -dec = env.Program(["decode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o"]) +enc = env.Program(["encode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) env.RunTest(enc) env.RunTest([dec, "encode_buffer.output"]) diff --git a/tests/basic_stream/SConscript b/tests/basic_stream/SConscript index 46db8c43..7d668562 100644 --- a/tests/basic_stream/SConscript +++ b/tests/basic_stream/SConscript @@ -2,8 +2,8 @@ Import("env") -enc = env.Program(["encode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o"]) -dec = env.Program(["decode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o"]) +enc = env.Program(["encode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) env.RunTest(enc) env.RunTest([dec, "encode_stream.output"]) diff --git a/tests/buffer_only/SConscript b/tests/buffer_only/SConscript index cddbb04f..55b747b0 100644 --- a/tests/buffer_only/SConscript +++ b/tests/buffer_only/SConscript @@ -18,10 +18,11 @@ strict = opts.Clone() strict.Append(CFLAGS = strict['CORECFLAGS']) strict.Object("pb_decode_bufonly.o", "$NANOPB/pb_decode.c") strict.Object("pb_encode_bufonly.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common_bufonly.o", "$NANOPB/pb_common.c") # Now build and run the test normally. -enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_bufonly.o"]) -dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_bufonly.o"]) +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_bufonly.o", "pb_common_bufonly.o"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_bufonly.o", "pb_common_bufonly.o"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/callbacks/SConscript b/tests/callbacks/SConscript index 9ec8a434..44521439 100644 --- a/tests/callbacks/SConscript +++ b/tests/callbacks/SConscript @@ -3,8 +3,8 @@ Import("env") env.NanopbProto("callbacks") -enc = env.Program(["encode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_encode.o"]) -dec = env.Program(["decode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_decode.o"]) +enc = env.Program(["encode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) env.RunTest(enc) env.RunTest([dec, "encode_callbacks.output"]) diff --git a/tests/common/SConscript b/tests/common/SConscript index 144f1490..f1dee0e2 100644 --- a/tests/common/SConscript +++ b/tests/common/SConscript @@ -14,4 +14,4 @@ strict = env.Clone() strict.Append(CFLAGS = strict['CORECFLAGS']) strict.Object("pb_decode.o", "$NANOPB/pb_decode.c") strict.Object("pb_encode.o", "$NANOPB/pb_encode.c") - +strict.Object("pb_common.o", "$NANOPB/pb_common.c") diff --git a/tests/cxx_main_program/SConscript b/tests/cxx_main_program/SConscript index e78c6b35..edb88127 100644 --- a/tests/cxx_main_program/SConscript +++ b/tests/cxx_main_program/SConscript @@ -11,14 +11,15 @@ env.Append(CPPDEFINES = ['__STDC_LIMIT_MACROS']) c = Copy("$TARGET", "$SOURCE") env.Command("pb_encode.cxx", "#../pb_encode.c", c) env.Command("pb_decode.cxx", "#../pb_decode.c", c) +env.Command("pb_common.cxx", "#../pb_common.c", c) env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c) env.Command("alltypes.pb.cxx", "$BUILD/alltypes/alltypes.pb.c", c) env.Command("encode_alltypes.cxx", "$BUILD/alltypes/encode_alltypes.c", c) env.Command("decode_alltypes.cxx", "$BUILD/alltypes/decode_alltypes.c", c) # Now build and run the test normally. -enc = env.Program(["encode_alltypes.cxx", "alltypes.pb.cxx", "pb_encode.cxx"]) -dec = env.Program(["decode_alltypes.cxx", "alltypes.pb.cxx", "pb_decode.cxx"]) +enc = env.Program(["encode_alltypes.cxx", "alltypes.pb.cxx", "pb_encode.cxx", "pb_common.cxx"]) +dec = env.Program(["decode_alltypes.cxx", "alltypes.pb.cxx", "pb_decode.cxx", "pb_common.cxx"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/decode_unittests/decode_unittests.c b/tests/decode_unittests/decode_unittests.c index 98055df3..59c4a074 100644 --- a/tests/decode_unittests/decode_unittests.c +++ b/tests/decode_unittests/decode_unittests.c @@ -1,5 +1,6 @@ /* This includes the whole .c file to get access to static functions. */ #define PB_ENABLE_MALLOC +#include "pb_common.c" #include "pb_decode.c" #include diff --git a/tests/encode_unittests/encode_unittests.c b/tests/encode_unittests/encode_unittests.c index 06935f9a..a5f868c9 100644 --- a/tests/encode_unittests/encode_unittests.c +++ b/tests/encode_unittests/encode_unittests.c @@ -1,4 +1,5 @@ /* This includes the whole .c file to get access to static functions. */ +#include "pb_common.c" #include "pb_encode.c" #include diff --git a/tests/extensions/SConscript b/tests/extensions/SConscript index 26fc5a3f..cf2e096e 100644 --- a/tests/extensions/SConscript +++ b/tests/extensions/SConscript @@ -8,8 +8,8 @@ incpath.Append(PROTOCPATH = '$BUILD/alltypes') incpath.Append(CPPPATH = '$BUILD/alltypes') incpath.NanopbProto(["extensions", "extensions.options"]) -enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_encode.o"]) -dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_decode.o"]) +enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb.o", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb.o", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) env.RunTest(enc) env.RunTest([dec, "encode_extensions.output"]) diff --git a/tests/field_size_16/SConscript b/tests/field_size_16/SConscript index 8fee0049..ffb29c4e 100644 --- a/tests/field_size_16/SConscript +++ b/tests/field_size_16/SConscript @@ -19,10 +19,11 @@ strict = opts.Clone() strict.Append(CFLAGS = strict['CORECFLAGS']) strict.Object("pb_decode_fields16.o", "$NANOPB/pb_decode.c") strict.Object("pb_encode_fields16.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common_fields16.o", "$NANOPB/pb_common.c") # Now build and run the test normally. -enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields16.o"]) -dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields16.o"]) +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields16.o", "pb_common_fields16.o"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields16.o", "pb_common_fields16.o"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/field_size_32/SConscript b/tests/field_size_32/SConscript index 2a64c6c5..0b8dc0e3 100644 --- a/tests/field_size_32/SConscript +++ b/tests/field_size_32/SConscript @@ -19,10 +19,11 @@ strict = opts.Clone() strict.Append(CFLAGS = strict['CORECFLAGS']) strict.Object("pb_decode_fields32.o", "$NANOPB/pb_decode.c") strict.Object("pb_encode_fields32.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common_fields32.o", "$NANOPB/pb_common.c") # Now build and run the test normally. -enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields32.o"]) -dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields32.o"]) +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields32.o", "pb_common_fields32.o"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields32.o", "pb_common_fields32.o"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/missing_fields/SConscript b/tests/missing_fields/SConscript index 9926efad..86ba0833 100644 --- a/tests/missing_fields/SConscript +++ b/tests/missing_fields/SConscript @@ -3,6 +3,6 @@ Import("env") env.NanopbProto("missing_fields") -test = env.Program(["missing_fields.c", "missing_fields.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_decode.o"]) +test = env.Program(["missing_fields.c", "missing_fields.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) env.RunTest(test) diff --git a/tests/no_errmsg/SConscript b/tests/no_errmsg/SConscript index ed467051..629bfa68 100644 --- a/tests/no_errmsg/SConscript +++ b/tests/no_errmsg/SConscript @@ -18,10 +18,11 @@ strict = opts.Clone() strict.Append(CFLAGS = strict['CORECFLAGS']) strict.Object("pb_decode_noerr.o", "$NANOPB/pb_decode.c") strict.Object("pb_encode_noerr.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common_noerr.o", "$NANOPB/pb_common.c") # Now build and run the test normally. -enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_noerr.o"]) -dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_noerr.o"]) +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_noerr.o", "pb_common_noerr.o"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_noerr.o", "pb_common_noerr.o"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/package_name/SConscript b/tests/package_name/SConscript index 8f1b9021..897bc99c 100644 --- a/tests/package_name/SConscript +++ b/tests/package_name/SConscript @@ -29,7 +29,7 @@ def modify_c(target, source, env): env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", modify_c) # Encode and compare results to original alltypes testcase -enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) +enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX" env.RunTest(enc) env.Compare(["encode_alltypes.output", "$BUILD/alltypes/encode_alltypes.output"]) diff --git a/tests/splint/SConscript b/tests/splint/SConscript index c1432dde..cd4b5b9d 100644 --- a/tests/splint/SConscript +++ b/tests/splint/SConscript @@ -11,3 +11,6 @@ if p: env.Command('pb_encode.splint', '$NANOPB/pb_encode.c', 'splint -f splint/splint.rc $SOURCE 2> $TARGET') + env.Command('pb_common.splint', '$NANOPB/pb_common.c', + 'splint -f splint/splint.rc $SOURCE 2> $TARGET') + -- cgit 1.2.3-korg From 7edf250a627b56c43e44de546b73bf113d9dea59 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 10 Aug 2014 13:01:09 +0300 Subject: Switch pb_encode to use the common iterator logic in pb_common.c Update issue 128 Status: FixedInGit --- pb_common.c | 6 ++++-- pb_common.h | 11 ++++++----- pb_decode.c | 22 ++++++++++------------ pb_encode.c | 45 +++++++++++++++++++++------------------------ tests/splint/splint.rc | 1 + 5 files changed, 42 insertions(+), 43 deletions(-) diff --git a/pb_common.c b/pb_common.c index de217699..a9cade63 100644 --- a/pb_common.c +++ b/pb_common.c @@ -5,7 +5,7 @@ #include "pb_common.h" -void pb_field_iter_begin(pb_field_iter_t *iter, const pb_field_t *fields, void *dest_struct) +bool pb_field_iter_begin(pb_field_iter_t *iter, const pb_field_t *fields, void *dest_struct) { iter->start = fields; iter->pos = fields; @@ -13,6 +13,8 @@ void pb_field_iter_begin(pb_field_iter_t *iter, const pb_field_t *fields, void * iter->dest_struct = dest_struct; iter->pData = (char*)dest_struct + iter->pos->data_offset; iter->pSize = (char*)iter->pData + iter->pos->size_offset; + + return (iter->pos->tag != 0); } bool pb_field_iter_next(pb_field_iter_t *iter) @@ -31,7 +33,7 @@ bool pb_field_iter_next(pb_field_iter_t *iter) if (iter->pos->tag == 0) { /* Wrapped back to beginning, reinitialize */ - pb_field_iter_begin(iter, iter->start, iter->dest_struct); + (void)pb_field_iter_begin(iter, iter->start, iter->dest_struct); return false; } else diff --git a/pb_common.h b/pb_common.h index e85f000c..01a37682 100644 --- a/pb_common.h +++ b/pb_common.h @@ -16,13 +16,14 @@ typedef struct { const pb_field_t *start; /* Start of the pb_field_t array */ const pb_field_t *pos; /* Current position of the iterator */ unsigned required_field_index; /* Zero-based index that counts only the required fields */ - void *dest_struct; /* Pointer to the destination structure to decode to */ - void *pData; /* Pointer where to store current field value */ - void *pSize; /* Pointer where to store the size of current array field */ + void *dest_struct; /* Pointer to start of the structure */ + void *pData; /* Pointer to current field value */ + void *pSize; /* Pointer to count/has field */ } pb_field_iter_t; -/* Initialize the field iterator structure to beginning. */ -void pb_field_iter_begin(pb_field_iter_t *iter, const pb_field_t *fields, void *dest_struct); +/* Initialize the field iterator structure to beginning. + * Returns false if the message type is empty. */ +bool pb_field_iter_begin(pb_field_iter_t *iter, const pb_field_t *fields, void *dest_struct); /* Advance the iterator to the next field. * Returns false when the iterator wraps back to the first field. */ diff --git a/pb_decode.c b/pb_decode.c index 40da1aa4..3e817cc8 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -616,7 +616,7 @@ static bool checkreturn default_extension_decoder(pb_istream_t *stream, /* Fake a field iterator for the extension field. * It is not actually safe to advance this iterator, but decode_field * will not even try to. */ - pb_field_iter_begin(&iter, field, extension->dest); + (void)pb_field_iter_begin(&iter, field, extension->dest); iter.pData = extension->dest; iter.pSize = &extension->found; @@ -668,16 +668,14 @@ static bool checkreturn find_extension_field(pb_field_iter_t *iter) static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct) { pb_field_iter_t iter; - pb_field_iter_begin(&iter, fields, dest_struct); + + if (!pb_field_iter_begin(&iter, fields, dest_struct)) + return; /* Empty message type */ do { pb_type_t type; type = iter.pos->type; - - /* Avoid crash on empty message types (zero fields) */ - if (iter.pos->tag == 0) - continue; if (PB_ATYPE(type) == PB_ATYPE_STATIC) { @@ -738,7 +736,9 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ uint32_t extension_range_start = 0; pb_field_iter_t iter; - pb_field_iter_begin(&iter, fields, dest_struct); + /* Return value ignored, as empty message types will be correctly handled by + * pb_field_iter_find() anyway. */ + (void)pb_field_iter_begin(&iter, fields, dest_struct); while (stream->bytes_left) { @@ -859,17 +859,15 @@ bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void * void pb_release(const pb_field_t fields[], void *dest_struct) { pb_field_iter_t iter; - pb_field_iter_begin(&iter, fields, dest_struct); + + if (!pb_field_iter_begin(&iter, fields, dest_struct)) + return; /* Empty message type */ do { pb_type_t type; type = iter.pos->type; - /* Avoid crash on empty message types (zero fields) */ - if (iter.pos->tag == 0) - continue; - if (PB_ATYPE(type) == PB_ATYPE_POINTER) { if (PB_HTYPE(type) == PB_HTYPE_REPEATED && diff --git a/pb_encode.c b/pb_encode.c index dc5a2734..6041c641 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -5,6 +5,7 @@ #include "pb.h" #include "pb_encode.h" +#include "pb_common.h" /* Use the GCC warn_unused_result attribute to check that all return values * are propagated correctly. On other compilers and gcc before 3.4.0 just @@ -333,42 +334,38 @@ static bool checkreturn encode_extension_field(pb_ostream_t *stream, * Encode all fields * *********************/ +static void *remove_const(const void *p) +{ + /* Note: this casts away const, in order to use the common field iterator + * logic for both encoding and decoding. */ + union { + void *p1; + const void *p2; + } t; + t.p2 = p; + return t.p1; +} + bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) { - const pb_field_t *field = fields; - const void *pData = src_struct; - size_t prev_size = 0; + pb_field_iter_t iter; + if (!pb_field_iter_begin(&iter, fields, remove_const(src_struct))) + return true; /* Empty message type */ - while (field->tag != 0) - { - pData = (const char*)pData + prev_size + field->data_offset; - if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) - prev_size = sizeof(const void*); - else - prev_size = field->data_size; - - /* Special case for static arrays */ - if (PB_ATYPE(field->type) == PB_ATYPE_STATIC && - PB_HTYPE(field->type) == PB_HTYPE_REPEATED) - { - prev_size *= field->array_size; - } - - if (PB_LTYPE(field->type) == PB_LTYPE_EXTENSION) + do { + if (PB_LTYPE(iter.pos->type) == PB_LTYPE_EXTENSION) { /* Special case for the extension field placeholder */ - if (!encode_extension_field(stream, field, pData)) + if (!encode_extension_field(stream, iter.pos, iter.pData)) return false; } else { /* Regular field */ - if (!encode_field(stream, field, pData)) + if (!encode_field(stream, iter.pos, iter.pData)) return false; } - - field++; - } + } while (pb_field_iter_next(&iter)); return true; } diff --git a/tests/splint/splint.rc b/tests/splint/splint.rc index 421f5677..e2b26888 100644 --- a/tests/splint/splint.rc +++ b/tests/splint/splint.rc @@ -28,6 +28,7 @@ -kepttrans -branchstate -immediatetrans +-mustfreefresh # These tests give false positives, compiler typically has # better warnings for these. -- cgit 1.2.3-korg From adf0ab81d400a49f5986d4de4b6e0dca617d5e13 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 10 Aug 2014 17:40:17 +0300 Subject: Add document detailing migration from old versions --- docs/Makefile | 2 +- docs/menu.rst | 2 + docs/migration.rst | 186 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 docs/migration.rst diff --git a/docs/Makefile b/docs/Makefile index e7143b25..0dbd97cf 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,4 +1,4 @@ -all: index.html concepts.html reference.html security.html \ +all: index.html concepts.html reference.html security.html migration.html \ generator_flow.png %.png: %.svg diff --git a/docs/menu.rst b/docs/menu.rst index a49b22c1..2c110def 100644 --- a/docs/menu.rst +++ b/docs/menu.rst @@ -4,8 +4,10 @@ 2) `Concepts`_ 3) `API reference`_ 4) `Security model`_ + 5) `Migration from older versions`_ .. _`Overview`: index.html .. _`Concepts`: concepts.html .. _`API reference`: reference.html .. _`Security model`: security.html +.. _`Migration from older versions`: migration.html diff --git a/docs/migration.rst b/docs/migration.rst new file mode 100644 index 00000000..9d41f5b1 --- /dev/null +++ b/docs/migration.rst @@ -0,0 +1,186 @@ +===================================== +Nanopb: Migration from older versions +===================================== + +.. include :: menu.rst + +This document details all the breaking changes that have been made to nanopb +since its initial release. For each change, the rationale and required +modifications of user applications are explained. Also any error indications +are included, in order to make it easier to find this document. + +.. contents :: + +Nanopb-0.3.0 (2014-09-xx) +========================= + +Separate field iterator logic to pb_common.c +-------------------------------------------- +**Rationale:** Originally, the field iteration logic was simple enough to be +duplicated in *pb_decode.c* and *pb_encode.c*. New field types have made the +logic more complex, which required the creation of a new file to contain the +common functionality. + +**Changes:** There is a new file, *pb_common.c*, which must be included in +builds. + +**Required actions:** Add *pb_common.c* to build rules. This file is always +required. Either *pb_decode.c* or *pb_encode.c* can still be left out if some +functionality is not needed. + +**Error indications:** Linker error: undefined reference to +*pb_field_iter_begin*, *pb_field_iter_next* or similar. + +Nanopb-0.2.9 (2014-08-09) +========================= + +Change semantics of generator -e option +--------------------------------------- +**Rationale:** Some compilers do not accept filenames with two dots (like +in default extension .pb.c). The *-e* option to the generator allowed changing +the extension, but not skipping the extra dot. + +**Changes:** The *-e* option in generator will no longer add the prepending +dot. The default value has been adjusted accordingly to *.pb.c* to keep the +default behaviour the same as before. + +**Required actions:** Only if using the generator -e option. Add dot before +the parameter value on the command line. + +**Error indications:** File not found when trying to compile generated files. + +Nanopb-0.2.7 (2014-04-07) +========================= + +Changed pointer-type bytes field datatype +----------------------------------------- +**Rationale:** In the initial pointer encoding support since nanopb-0.2.5, +the bytes type used a separate *pb_bytes_ptr_t* type to represent *bytes* +fields. This made it easy to encode data from a separate, user-allocated +buffer. However, it made the internal logic more complex and was inconsistent +with the other types. + +**Changes:** Dynamically allocated bytes fields now have the *pb_bytes_array_t* +type, just like statically allocated ones. + +**Required actions:** Only if using pointer-type fields with the bytes datatype. +Change any access to *msg->field.size* to *msg->field->size*. Change any +allocation to reserve space of amount *PB_BYTES_ARRAY_T_ALLOCSIZE(n)*. If the +data pointer was begin assigned from external source, implement the field using +a callback function instead. + +**Error indications:** Compiler error: unknown type name *pb_bytes_ptr_t*. + +Nanopb-0.2.4 (2013-11-07) +========================= + +Remove the NANOPB_INTERNALS compilation option +---------------------------------------------- +**Rationale:** Having the option in the headers required the functions to +be non-static, even if the option is not used. This caused errors on some +static analysis tools. + +**Changes:** The *#ifdef* and associated functions were removed from the +header. + +**Required actions:** Only if the *NANOPB_INTERNALS* option was previously +used. Actions are as listed under nanopb-0.1.3 and nanopb-0.1.6. + +**Error indications:** Compiler warning: implicit declaration of function +*pb_dec_string*, *pb_enc_string*, or similar. + +Nanopb-0.2.1 (2013-04-14) +========================= + +Callback function signature +--------------------------- +**Rationale:** Previously the auxilary data to field callbacks was passed +as *void\**. This allowed passing of any data, but made it unnecessarily +complex to return a pointer from callback. + +**Changes:** The callback function parameter was changed to *void\**. + +**Required actions:** You can continue using the old callback style by +defining *PB_OLD_CALLBACK_STYLE*. Recommended action is to: + + * Change the callback signatures to contain *void\** for decoders and + *void \* const \** for encoders. + * Change the callback function body to use *\*arg* instead of *arg*. + +**Error indications:** Compiler warning: assignment from incompatible +pointer type, when initializing *funcs.encode* or *funcs.decode*. + +Nanopb-0.2.0 (2013-03-02) +========================= + +Reformatted generated .pb.c file using macros +--------------------------------------------- +**Rationale:** Previously the generator made a list of C *pb_field_t* +initializers in the .pb.c file. This led to a need to regenerate all .pb.c +files after even small changes to the *pb_field_t* definition. + +**Changes:** Macros were added to pb.h which allow for cleaner definition +of the .pb.c contents. By changing the macro definitions, changes to the +field structure are possible without breaking compatibility with old .pb.c +files. + +**Required actions:** Regenerate all .pb.c files from the .proto sources. + +**Error indications:** Compiler warning: implicit declaration of function +*pb_delta_end*. + +Changed pb_type_t definitions +----------------------------- +**Rationale:** The *pb_type_t* was previously an enumeration type. This +caused warnings on some compilers when using bitwise operations to set flags +inside the values. + +**Changes:** The *pb_type_t* was changed to *typedef uint8_t*. The values +were changed to *#define*. Some value names were changed for consistency. + +**Required actions:** Only if you directly access the `pb_field_t` contents +in your own code, something which is not usually done. Needed changes: + + * Change *PB_HTYPE_ARRAY* to *PB_HTYPE_REPEATED*. + * Change *PB_HTYPE_CALLBACK* to *PB_ATYPE()* and *PB_ATYPE_CALLBACK*. + +**Error indications:** Compiler error: *PB_HTYPE_ARRAY* or *PB_HTYPE_CALLBACK* +undeclared. + +Nanopb-0.1.6 (2012-09-02) +========================= + +Refactored field decoder interface +---------------------------------- +**Rationale:** Similarly to field encoders in nanopb-0.1.3. + +**Changes:** New functions with names *pb_decode_\** were added. + +**Required actions:** By defining NANOPB_INTERNALS, you can still keep using +the old functions. Recommended action is to replace any calls with the newer +*pb_decode_\** equivalents. + +**Error indications:** Compiler warning: implicit declaration of function +*pb_dec_string*, *pb_dec_varint*, *pb_dec_submessage* or similar. + +Nanopb-0.1.3 (2012-06-12) +========================= + +Refactored field encoder interface +---------------------------------- +**Rationale:** The old *pb_enc_\** functions were designed mostly for the +internal use by the core. Because they are internally accessed through +function pointers, their signatures had to be common. This led to a confusing +interface for external users. + +**Changes:** New functions with names *pb_encode_\** were added. These have +easier to use interfaces. The old functions are now only thin wrappers for +the new interface. + +**Required actions:** By defining NANOPB_INTERNALS, you can still keep using +the old functions. Recommended action is to replace any calls with the newer +*pb_encode_\** equivalents. + +**Error indications:** Compiler warning: implicit declaration of function +*pb_enc_string*, *pb_enc_varint, *pb_enc_submessage* or similar. + -- cgit 1.2.3-korg From 0b517b07789049089e19b714311c596399d53f8e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 10 Aug 2014 17:44:27 +0300 Subject: Fix windows build error in tests --- tests/extensions/SConscript | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/extensions/SConscript b/tests/extensions/SConscript index cf2e096e..a2c87428 100644 --- a/tests/extensions/SConscript +++ b/tests/extensions/SConscript @@ -8,8 +8,8 @@ incpath.Append(PROTOCPATH = '$BUILD/alltypes') incpath.Append(CPPPATH = '$BUILD/alltypes') incpath.NanopbProto(["extensions", "extensions.options"]) -enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb.o", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) -dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb.o", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) +enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) env.RunTest(enc) env.RunTest([dec, "encode_extensions.output"]) -- cgit 1.2.3-korg From 1dd9f1900fca0c137324c05a9421f1ba180b2470 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 18 Aug 2014 20:09:52 +0300 Subject: Change the _count fields to use pb_size_t datatype. Update issue 82 Status: FixedInGit --- docs/migration.rst | 15 ++++++++++ generator/nanopb_generator.py | 9 ++---- pb.h | 7 +++-- pb_decode.c | 40 ++++++++++++++++++------- pb_encode.c | 4 +-- tests/backwards_compatibility/alltypes_legacy.h | 40 ++++++++++++------------- tests/decode_unittests/decode_unittests.c | 4 +-- tests/encode_unittests/encode_unittests.c | 2 +- 8 files changed, 77 insertions(+), 44 deletions(-) diff --git a/docs/migration.rst b/docs/migration.rst index 9d41f5b1..800bc1f9 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -31,6 +31,21 @@ functionality is not needed. **Error indications:** Linker error: undefined reference to *pb_field_iter_begin*, *pb_field_iter_next* or similar. +Change data type of field counts to pb_size_t +--------------------------------------------- +**Rationale:** Often nanopb is used with small arrays, such as 255 items or +less. Using a full *size_t* field to store the array count wastes memory if +there are many arrays. There already exists parameters *PB_FIELD_16BIT* and +*PB_FIELD_32BIT* which tell nanopb what is the maximum size of arrays in use. + +**Changes:** Generator will now use *pb_size_t* for the array *_count* fields. +The size of the type will be controlled by the *PB_FIELD_16BIT* and +*PB_FIELD_32BIT* compilation time options. + +**Required actions:** Regenerate all *.pb.h* files. In some cases casts to the +*pb_size_t* type may need to be added in the user code when accessing the +*_count* fields. + Nanopb-0.2.9 (2014-08-09) ========================= diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 1d12ae3c..88e9798d 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -261,7 +261,7 @@ class Field: result = '' if self.allocation == 'POINTER': if self.rules == 'REPEATED': - result += ' size_t ' + self.name + '_count;\n' + result += ' pb_size_t ' + self.name + '_count;\n' if self.pbtype == 'MESSAGE': # Use struct definition, so recursive submessages are possible @@ -277,17 +277,14 @@ class Field: if self.rules == 'OPTIONAL' and self.allocation == 'STATIC': result += ' bool has_' + self.name + ';\n' elif self.rules == 'REPEATED' and self.allocation == 'STATIC': - result += ' size_t ' + self.name + '_count;\n' + result += ' pb_size_t ' + self.name + '_count;\n' result += ' %s %s%s;' % (self.ctype, self.name, self.array_decl) return result def types(self): '''Return definitions for any special types this field might need.''' if self.pbtype == 'BYTES' and self.allocation == 'STATIC': - result = 'typedef struct {\n' - result += ' size_t size;\n' - result += ' uint8_t bytes[%d];\n' % self.max_size - result += '} %s;\n' % self.ctype + result = 'typedef PB_BYTES_ARRAY_T(%d) %s;\n' % (self.max_size, self.ctype) else: result = None return result diff --git a/pb.h b/pb.h index 50a07c54..c9ef48b9 100644 --- a/pb.h +++ b/pb.h @@ -187,12 +187,15 @@ typedef uint8_t pb_type_t; * and array counts. */ #if defined(PB_FIELD_32BIT) +#define PB_SIZE_MAX ((uint32_t)-1) typedef uint32_t pb_size_t; typedef int32_t pb_ssize_t; #elif defined(PB_FIELD_16BIT) +#define PB_SIZE_MAX ((uint16_t)-1) typedef uint16_t pb_size_t; typedef int16_t pb_ssize_t; #else +#define PB_SIZE_MAX ((uint8_t)-1) typedef uint8_t pb_size_t; typedef int8_t pb_ssize_t; #endif @@ -241,11 +244,11 @@ STATIC_ASSERT(sizeof(uint64_t) == 8, UINT64_T_WRONG_SIZE) * It has the number of bytes in the beginning, and after that an array. * Note that actual structs used will have a different length of bytes array. */ -#define PB_BYTES_ARRAY_T(n) struct { size_t size; uint8_t bytes[n]; } +#define PB_BYTES_ARRAY_T(n) struct { pb_size_t size; uint8_t bytes[n]; } #define PB_BYTES_ARRAY_T_ALLOCSIZE(n) ((size_t)n + offsetof(pb_bytes_array_t, bytes)) struct _pb_bytes_array_t { - size_t size; + pb_size_t size; uint8_t bytes[1]; }; typedef struct _pb_bytes_array_t pb_bytes_array_t; diff --git a/pb_decode.c b/pb_decode.c index 3e817cc8..63ec0de9 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -352,7 +352,7 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t { /* Packed array */ bool status = true; - size_t *size = (size_t*)iter->pSize; + pb_size_t *size = (pb_size_t*)iter->pSize; pb_istream_t substream; if (!pb_make_string_substream(stream, &substream)) return false; @@ -377,7 +377,7 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t else { /* Repeated field */ - size_t *size = (size_t*)iter->pSize; + pb_size_t *size = (pb_size_t*)iter->pSize; void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); if (*size >= iter->pos->array_size) PB_RETURN_ERROR(stream, "array overflow"); @@ -478,7 +478,7 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ { /* Packed array, multiple items come in at once. */ bool status = true; - size_t *size = (size_t*)iter->pSize; + pb_size_t *size = (pb_size_t*)iter->pSize; size_t allocated_size = *size; void *pItem; pb_istream_t substream; @@ -488,7 +488,7 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ while (substream.bytes_left) { - if (*size + 1 > allocated_size) + if ((size_t)*size + 1 > allocated_size) { /* Allocate more storage. This tries to guess the * number of remaining entries. Round the division @@ -510,6 +510,16 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ status = false; break; } + + if (*size == PB_SIZE_MAX) + { +#ifndef PB_NO_ERRMSG + stream->errmsg = "too many array entries"; +#endif + status = false; + break; + } + (*size)++; } pb_close_string_substream(stream, &substream); @@ -519,9 +529,12 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ else { /* Normal repeated field, i.e. only one item at a time. */ - size_t *size = (size_t*)iter->pSize; + pb_size_t *size = (pb_size_t*)iter->pSize; void *pItem; + if (*size == PB_SIZE_MAX) + PB_RETURN_ERROR(stream, "too many array entries"); + (*size)++; if (!allocate_field(stream, iter->pData, iter->pos->data_size, *size)) return false; @@ -688,7 +701,7 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str else if (PB_HTYPE(type) == PB_HTYPE_REPEATED) { /* Set array count to 0, no need to initialize contents. */ - *(size_t*)iter.pSize = 0; + *(pb_size_t*)iter.pSize = 0; continue; } @@ -716,7 +729,7 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str /* Initialize array count to 0. */ if (PB_HTYPE(type) == PB_HTYPE_REPEATED) { - *(size_t*)iter.pSize = 0; + *(pb_size_t*)iter.pSize = 0; } } else if (PB_ATYPE(type) == PB_ATYPE_CALLBACK) @@ -876,7 +889,7 @@ void pb_release(const pb_field_t fields[], void *dest_struct) { /* Release entries in repeated string or bytes array */ void **pItem = *(void***)iter.pData; - size_t count = *(size_t*)iter.pSize; + pb_size_t count = *(pb_size_t*)iter.pSize; while (count--) { pb_free(*pItem); @@ -887,11 +900,11 @@ void pb_release(const pb_field_t fields[], void *dest_struct) { /* Release fields in submessages */ void *pItem = *(void**)iter.pData; - size_t count = (pItem ? 1 : 0); + pb_size_t count = (pItem ? 1 : 0); if (PB_HTYPE(type) == PB_HTYPE_REPEATED) { - count = *(size_t*)iter.pSize; + count = *(pb_size_t*)iter.pSize; } while (count--) @@ -1054,7 +1067,12 @@ static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *fie bdest = (pb_bytes_array_t*)dest; } - bdest->size = size; + if (size > PB_SIZE_MAX) + { + PB_RETURN_ERROR(stream, "bytes overflow"); + } + + bdest->size = (pb_size_t)size; return pb_read(stream, bdest->bytes, size); } diff --git a/pb_encode.c b/pb_encode.c index 6041c641..3dce1c10 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -246,7 +246,7 @@ static bool checkreturn encode_basic_field(pb_ostream_t *stream, break; case PB_HTYPE_REPEATED: - if (!encode_array(stream, field, pData, *(const size_t*)pSize, func)) + if (!encode_array(stream, field, pData, *(const pb_size_t*)pSize, func)) return false; break; @@ -630,7 +630,6 @@ static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *fie static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - /* strnlen() is not always available, so just use a loop */ size_t size = 0; size_t max_size = field->data_size; const char *p = (const char*)src; @@ -644,6 +643,7 @@ static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *fi } else { + /* strnlen() is not always available, so just use a loop */ while (size < max_size && *p != '\0') { size++; diff --git a/tests/backwards_compatibility/alltypes_legacy.h b/tests/backwards_compatibility/alltypes_legacy.h index 037b3478..abdd97a7 100644 --- a/tests/backwards_compatibility/alltypes_legacy.h +++ b/tests/backwards_compatibility/alltypes_legacy.h @@ -29,17 +29,17 @@ typedef struct _SubMessage { } SubMessage; typedef struct { - size_t size; + pb_size_t size; uint8_t bytes[16]; } AllTypes_req_bytes_t; typedef struct { - size_t size; + pb_size_t size; uint8_t bytes[16]; } AllTypes_rep_bytes_t; typedef struct { - size_t size; + pb_size_t size; uint8_t bytes[16]; } AllTypes_opt_bytes_t; @@ -61,39 +61,39 @@ typedef struct _AllTypes { AllTypes_req_bytes_t req_bytes; SubMessage req_submsg; MyEnum req_enum; - size_t rep_int32_count; + pb_size_t rep_int32_count; int32_t rep_int32[5]; - size_t rep_int64_count; + pb_size_t rep_int64_count; int64_t rep_int64[5]; - size_t rep_uint32_count; + pb_size_t rep_uint32_count; uint32_t rep_uint32[5]; - size_t rep_uint64_count; + pb_size_t rep_uint64_count; uint64_t rep_uint64[5]; - size_t rep_sint32_count; + pb_size_t rep_sint32_count; int32_t rep_sint32[5]; - size_t rep_sint64_count; + pb_size_t rep_sint64_count; int64_t rep_sint64[5]; - size_t rep_bool_count; + pb_size_t rep_bool_count; bool rep_bool[5]; - size_t rep_fixed32_count; + pb_size_t rep_fixed32_count; uint32_t rep_fixed32[5]; - size_t rep_sfixed32_count; + pb_size_t rep_sfixed32_count; int32_t rep_sfixed32[5]; - size_t rep_float_count; + pb_size_t rep_float_count; float rep_float[5]; - size_t rep_fixed64_count; + pb_size_t rep_fixed64_count; uint64_t rep_fixed64[5]; - size_t rep_sfixed64_count; + pb_size_t rep_sfixed64_count; int64_t rep_sfixed64[5]; - size_t rep_double_count; + pb_size_t rep_double_count; double rep_double[5]; - size_t rep_string_count; + pb_size_t rep_string_count; char rep_string[5][16]; - size_t rep_bytes_count; + pb_size_t rep_bytes_count; AllTypes_rep_bytes_t rep_bytes[5]; - size_t rep_submsg_count; + pb_size_t rep_submsg_count; SubMessage rep_submsg[5]; - size_t rep_enum_count; + pb_size_t rep_enum_count; MyEnum rep_enum[5]; bool has_opt_int32; int32_t opt_int32; diff --git a/tests/decode_unittests/decode_unittests.c b/tests/decode_unittests/decode_unittests.c index 59c4a074..97212aff 100644 --- a/tests/decode_unittests/decode_unittests.c +++ b/tests/decode_unittests/decode_unittests.c @@ -170,7 +170,7 @@ int main() { pb_istream_t s; - struct { size_t size; uint8_t bytes[5]; } d; + struct { pb_size_t size; uint8_t bytes[5]; } d; pb_field_t f = {1, PB_LTYPE_BYTES, 0, 0, sizeof(d), 0, 0}; COMMENT("Test pb_dec_bytes") @@ -251,7 +251,7 @@ int main() { pb_istream_t s; CallbackArray dest; - struct { size_t size; uint8_t bytes[10]; } ref; + struct { pb_size_t size; uint8_t bytes[10]; } ref; dest.data.funcs.decode = &callback_check; dest.data.arg = &ref; diff --git a/tests/encode_unittests/encode_unittests.c b/tests/encode_unittests/encode_unittests.c index a5f868c9..78fbb8b1 100644 --- a/tests/encode_unittests/encode_unittests.c +++ b/tests/encode_unittests/encode_unittests.c @@ -170,7 +170,7 @@ int main() { uint8_t buffer[30]; pb_ostream_t s; - struct { size_t size; uint8_t bytes[5]; } value = {5, {'x', 'y', 'z', 'z', 'y'}}; + struct { pb_size_t size; uint8_t bytes[5]; } value = {5, {'x', 'y', 'z', 'z', 'y'}}; COMMENT("Test pb_enc_bytes") TEST(WRITES(pb_enc_bytes(&s, &BytesMessage_fields[0], &value), "\x05xyzzy")) -- cgit 1.2.3-korg From 62b4a8ecaac45ce62cc93e09261d60e6ceade1a9 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 18 Aug 2014 20:49:48 +0300 Subject: Rename UNUSED() and STATIC_ASSERT() macros with PB_ prefix. This avoids possible namespace conflicts with other macros. --- generator/nanopb_generator.py | 6 ++-- pb.h | 40 ++++++++++++++----------- pb_decode.c | 8 ++--- pb_encode.c | 6 ++-- tests/backwards_compatibility/alltypes_legacy.h | 6 ++-- 5 files changed, 35 insertions(+), 31 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 88e9798d..80fb93ca 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -908,7 +908,7 @@ def generate_source(headername, enums, messages, extensions, options): yield ' * numbers or field sizes that are larger than what can fit in 8 or 16 bit\n' yield ' * field descriptors.\n' yield ' */\n' - yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) + yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) yield '#endif\n\n' if worst < 65536: @@ -925,7 +925,7 @@ def generate_source(headername, enums, messages, extensions, options): yield ' * numbers or field sizes that are larger than what can fit in the default\n' yield ' * 8 bit descriptors.\n' yield ' */\n' - yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) + yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) yield '#endif\n\n' # Add check for sizeof(double) @@ -941,7 +941,7 @@ def generate_source(headername, enums, messages, extensions, options): yield ' * These are not directly supported by nanopb, but see example_avr_double.\n' yield ' * To get rid of this error, remove any double fields from your .proto.\n' yield ' */\n' - yield 'STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n' + yield 'PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n' yield '\n' diff --git a/pb.h b/pb.h index c9ef48b9..8ddf30ed 100644 --- a/pb.h +++ b/pb.h @@ -98,23 +98,27 @@ #endif /* Handly macro for suppressing unreferenced-parameter compiler warnings. */ -#ifndef UNUSED -#define UNUSED(x) (void)(x) +#ifndef PB_UNUSED +#define PB_UNUSED(x) (void)(x) #endif /* Compile-time assertion, used for checking compatible compilation options. - * If this does not work properly on your compiler, use #define STATIC_ASSERT - * to disable it. + * If this does not work properly on your compiler, use + * #define PB_NO_STATIC_ASSERT to disable it. * * But before doing that, check carefully the error message / place where it * comes from to see if the error has a real cause. Unfortunately the error * message is not always very clear to read, but you can see the reason better - * in the place where the STATIC_ASSERT macro was called. + * in the place where the PB_STATIC_ASSERT macro was called. */ -#ifndef STATIC_ASSERT -#define STATIC_ASSERT(COND,MSG) typedef char STATIC_ASSERT_MSG(MSG, __LINE__, __COUNTER__)[(COND)?1:-1]; -#define STATIC_ASSERT_MSG(MSG, LINE, COUNTER) STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) -#define STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) static_assertion_##MSG##LINE##COUNTER +#ifndef PB_NO_STATIC_ASSERT +#ifndef PB_STATIC_ASSERT +#define PB_STATIC_ASSERT(COND,MSG) typedef char PB_STATIC_ASSERT_MSG(MSG, __LINE__, __COUNTER__)[(COND)?1:-1]; +#define PB_STATIC_ASSERT_MSG(MSG, LINE, COUNTER) PB_STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) +#define PB_STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) pb_static_assertion_##MSG##LINE##COUNTER +#endif +#else +#define PB_STATIC_ASSERT(COND,MSG) #endif /* Number of required fields to keep track of. */ @@ -231,14 +235,14 @@ PB_PACKED_STRUCT_END * If you get errors here, it probably means that your stdint.h is not * correct for your platform. */ -STATIC_ASSERT(sizeof(int8_t) == 1, INT8_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(uint8_t) == 1, UINT8_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(int16_t) == 2, INT16_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(uint16_t) == 2, UINT16_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(int32_t) == 4, INT32_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(uint32_t) == 4, UINT32_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(int64_t) == 8, INT64_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(uint64_t) == 8, UINT64_T_WRONG_SIZE) +PB_STATIC_ASSERT(sizeof(int8_t) == 1, INT8_T_WRONG_SIZE) +PB_STATIC_ASSERT(sizeof(uint8_t) == 1, UINT8_T_WRONG_SIZE) +PB_STATIC_ASSERT(sizeof(int16_t) == 2, INT16_T_WRONG_SIZE) +PB_STATIC_ASSERT(sizeof(uint16_t) == 2, UINT16_T_WRONG_SIZE) +PB_STATIC_ASSERT(sizeof(int32_t) == 4, INT32_T_WRONG_SIZE) +PB_STATIC_ASSERT(sizeof(uint32_t) == 4, UINT32_T_WRONG_SIZE) +PB_STATIC_ASSERT(sizeof(int64_t) == 8, INT64_T_WRONG_SIZE) +PB_STATIC_ASSERT(sizeof(uint64_t) == 8, UINT64_T_WRONG_SIZE) /* This structure is used for 'bytes' arrays. * It has the number of bytes in the beginning, and after that an array. @@ -505,7 +509,7 @@ struct _pb_extension_t { #ifdef PB_NO_ERRMSG #define PB_RETURN_ERROR(stream,msg) \ do {\ - UNUSED(stream); \ + PB_UNUSED(stream); \ return false; \ } while(0) #define PB_GET_ERROR(stream) "(errmsg disabled)" diff --git a/pb_decode.c b/pb_decode.c index 63ec0de9..5d211023 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -444,8 +444,8 @@ static void initialize_pointer_field(void *pItem, pb_field_iter_t *iter) static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter) { #ifndef PB_ENABLE_MALLOC - UNUSED(wire_type); - UNUSED(iter); + PB_UNUSED(wire_type); + PB_UNUSED(iter); PB_RETURN_ERROR(stream, "no malloc support"); #else pb_type_t type; @@ -1032,13 +1032,13 @@ static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *f static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest) { - UNUSED(field); + PB_UNUSED(field); return pb_decode_fixed32(stream, dest); } static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest) { - UNUSED(field); + PB_UNUSED(field); return pb_decode_fixed64(stream, dest); } diff --git a/pb_encode.c b/pb_encode.c index 3dce1c10..cdd78955 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -311,7 +311,7 @@ static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData) { const pb_extension_t *extension = *(const pb_extension_t* const *)pData; - UNUSED(field); + PB_UNUSED(field); while (extension) { @@ -599,13 +599,13 @@ static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *f static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - UNUSED(field); + PB_UNUSED(field); return pb_encode_fixed64(stream, src); } static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - UNUSED(field); + PB_UNUSED(field); return pb_encode_fixed32(stream, src); } diff --git a/tests/backwards_compatibility/alltypes_legacy.h b/tests/backwards_compatibility/alltypes_legacy.h index abdd97a7..8b7e3bca 100644 --- a/tests/backwards_compatibility/alltypes_legacy.h +++ b/tests/backwards_compatibility/alltypes_legacy.h @@ -159,17 +159,17 @@ extern const pb_field_t AllTypes_fields[53]; /* Check that field information fits in pb_field_t */ #if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) -STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 256 && pb_membersize(AllTypes, rep_submsg[0]) < 256 && pb_membersize(AllTypes, opt_submsg) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_SubMessage_AllTypes) +PB_STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 256 && pb_membersize(AllTypes, rep_submsg[0]) < 256 && pb_membersize(AllTypes, opt_submsg) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_SubMessage_AllTypes) #endif #if !defined(PB_FIELD_32BIT) -STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 65536 && pb_membersize(AllTypes, rep_submsg[0]) < 65536 && pb_membersize(AllTypes, opt_submsg) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_SubMessage_AllTypes) +PB_STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 65536 && pb_membersize(AllTypes, rep_submsg[0]) < 65536 && pb_membersize(AllTypes, opt_submsg) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_SubMessage_AllTypes) #endif /* On some platforms (such as AVR), double is really float. * These are not directly supported by nanopb, but see example_avr_double. */ -STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES) +PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES) #ifdef __cplusplus } /* extern "C" */ -- cgit 1.2.3-korg From be0b9e047a007685aa10a268f9bf856e9a52ef58 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 18 Aug 2014 21:11:10 +0300 Subject: Rename poorly named identifier to avoid name conflicts. Update issue 106 Status: FixedInGit --- docs/migration.rst | 40 +++++++++++++++++++++++++++++++++++++++- generator/nanopb_generator.py | 4 ++-- pb.h | 28 ++++++++++++++-------------- pb_common.h | 9 +++++---- pb_decode.h | 6 +++--- pb_encode.h | 6 +++--- tests/splint/splint.rc | 1 - 7 files changed, 66 insertions(+), 28 deletions(-) diff --git a/docs/migration.rst b/docs/migration.rst index 800bc1f9..5ac52b3e 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -44,7 +44,45 @@ The size of the type will be controlled by the *PB_FIELD_16BIT* and **Required actions:** Regenerate all *.pb.h* files. In some cases casts to the *pb_size_t* type may need to be added in the user code when accessing the -*_count* fields. +*_count* fields. + +**Error indications:** Incorrect data at runtime, crashes. But note that other +changes in the same version already require regenerating the files and have +better indications of errors, so this is only an issue for development +versions. + +Renamed some macros and identifiers +----------------------------------- +**Rationale:** Some names in nanopb core were badly chosen and conflicted with +ISO C99 reserved names or lacked a prefix. While they haven't caused trouble +so far, it is reasonable to switch to non-conflicting names as these are rarely +used from user code. + +**Changes:** The following identifier names have changed: + + * Macros: + + * STATIC_ASSERT(x) -> PB_STATIC_ASSERT(x) + * UNUSED(x) -> PB_UNUSED(x) + + * Include guards: + + * _PB_filename_ -> PB_filename_INCLUDED + + * Structure forward declaration tags: + + * _pb_field_t -> pb_field_s + * _pb_bytes_array_t -> pb_bytes_array_s + * _pb_callback_t -> pb_callback_s + * _pb_extension_type_t -> pb_extension_type_s + * _pb_extension_t -> pb_extension_s + * _pb_istream_t -> pb_istream_s + * _pb_ostream_t -> pb_ostream_s + +**Required actions:** Regenerate all *.pb.c* files. If you use any of the above +identifiers in your application code, perform search-replace to the new name. + +**Error indications:** Compiler errors on lines with the macro/type names. Nanopb-0.2.9 (2014-08-09) ========================= diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 80fb93ca..50108148 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -768,8 +768,8 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) symbol = make_identifier(headername) - yield '#ifndef _PB_%s_\n' % symbol - yield '#define _PB_%s_\n' % symbol + yield '#ifndef PB_%s_INCLUDED\n' % symbol + yield '#define PB_%s_INCLUDED\n' % symbol try: yield options.libformat % ('pb.h') except TypeError: diff --git a/pb.h b/pb.h index 8ddf30ed..5edd6483 100644 --- a/pb.h +++ b/pb.h @@ -2,8 +2,8 @@ * stuff. For the high-level interface, see pb_encode.h and pb_decode.h. */ -#ifndef _PB_H_ -#define _PB_H_ +#ifndef PB_H_INCLUDED +#define PB_H_INCLUDED /***************************************************************** * Nanopb compilation time options. You can change these here by * @@ -213,8 +213,8 @@ typedef uint8_t pb_type_t; * PB_FIELD_32BIT. */ PB_PACKED_STRUCT_START -typedef struct _pb_field_t pb_field_t; -struct _pb_field_t { +typedef struct pb_field_s pb_field_t; +struct pb_field_s { pb_size_t tag; pb_type_t type; pb_size_t data_offset; /* Offset of field data, relative to previous field. */ @@ -251,11 +251,11 @@ PB_STATIC_ASSERT(sizeof(uint64_t) == 8, UINT64_T_WRONG_SIZE) #define PB_BYTES_ARRAY_T(n) struct { pb_size_t size; uint8_t bytes[n]; } #define PB_BYTES_ARRAY_T_ALLOCSIZE(n) ((size_t)n + offsetof(pb_bytes_array_t, bytes)) -struct _pb_bytes_array_t { +struct pb_bytes_array_s { pb_size_t size; uint8_t bytes[1]; }; -typedef struct _pb_bytes_array_t pb_bytes_array_t; +typedef struct pb_bytes_array_s pb_bytes_array_t; /* This structure is used for giving the callback function. * It is stored in the message structure and filled in by the method that @@ -275,10 +275,10 @@ typedef struct _pb_bytes_array_t pb_bytes_array_t; * * The callback can be null if you want to skip a field. */ -typedef struct _pb_istream_t pb_istream_t; -typedef struct _pb_ostream_t pb_ostream_t; -typedef struct _pb_callback_t pb_callback_t; -struct _pb_callback_t { +typedef struct pb_istream_s pb_istream_t; +typedef struct pb_ostream_s pb_ostream_t; +typedef struct pb_callback_s pb_callback_t; +struct pb_callback_s { #ifdef PB_OLD_CALLBACK_STYLE /* Deprecated since nanopb-0.2.1 */ union { @@ -311,9 +311,9 @@ typedef enum { * if you want to catch all unknown fields, you can also create a custom * pb_extension_type_t with your own callback. */ -typedef struct _pb_extension_type_t pb_extension_type_t; -typedef struct _pb_extension_t pb_extension_t; -struct _pb_extension_type_t { +typedef struct pb_extension_type_s pb_extension_type_t; +typedef struct pb_extension_s pb_extension_t; +struct pb_extension_type_s { /* Called for each unknown field in the message. * If you handle the field, read off all of its data and return true. * If you do not handle the field, do not read anything and return true. @@ -335,7 +335,7 @@ struct _pb_extension_type_t { const void *arg; }; -struct _pb_extension_t { +struct pb_extension_s { /* Type describing the extension field. Usually you'll initialize * this to a pointer to the automatically generated structure. */ const pb_extension_type_t *type; diff --git a/pb_common.h b/pb_common.h index 01a37682..60b3d374 100644 --- a/pb_common.h +++ b/pb_common.h @@ -2,8 +2,8 @@ * These functions are rarely needed by applications directly. */ -#ifndef _PB_COMMON_H_ -#define _PB_COMMON_H_ +#ifndef PB_COMMON_H_INCLUDED +#define PB_COMMON_H_INCLUDED #include "pb.h" @@ -12,14 +12,15 @@ extern "C" { #endif /* Iterator for pb_field_t list */ -typedef struct { +struct pb_field_iter_s { const pb_field_t *start; /* Start of the pb_field_t array */ const pb_field_t *pos; /* Current position of the iterator */ unsigned required_field_index; /* Zero-based index that counts only the required fields */ void *dest_struct; /* Pointer to start of the structure */ void *pData; /* Pointer to current field value */ void *pSize; /* Pointer to count/has field */ -} pb_field_iter_t; +}; +typedef struct pb_field_iter_s pb_field_iter_t; /* Initialize the field iterator structure to beginning. * Returns false if the message type is empty. */ diff --git a/pb_decode.h b/pb_decode.h index 8dc67408..3d433155 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -3,8 +3,8 @@ * field descriptions created by nanopb_generator.py. */ -#ifndef _PB_DECODE_H_ -#define _PB_DECODE_H_ +#ifndef PB_DECODE_H_INCLUDED +#define PB_DECODE_H_INCLUDED #include "pb.h" @@ -25,7 +25,7 @@ extern "C" { * is different than from the main stream. Don't use bytes_left to compute * any pointers. */ -struct _pb_istream_t +struct pb_istream_s { #ifdef PB_BUFFER_ONLY /* Callback pointer is not used in buffer-only configuration. diff --git a/pb_encode.h b/pb_encode.h index f82bac8f..e992c8dc 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -3,8 +3,8 @@ * field descriptions created by nanopb_generator.py. */ -#ifndef _PB_ENCODE_H_ -#define _PB_ENCODE_H_ +#ifndef PB_ENCODE_H_INCLUDED +#define PB_ENCODE_H_INCLUDED #include "pb.h" @@ -24,7 +24,7 @@ extern "C" { * 4) Substreams will modify max_size and bytes_written. Don't use them * to calculate any pointers. */ -struct _pb_ostream_t +struct pb_ostream_s { #ifdef PB_BUFFER_ONLY /* Callback pointer is not used in buffer-only configuration. diff --git a/tests/splint/splint.rc b/tests/splint/splint.rc index e2b26888..e47d3c21 100644 --- a/tests/splint/splint.rc +++ b/tests/splint/splint.rc @@ -2,7 +2,6 @@ +partial +matchanyintegral +strictlib --isoreserved # to be fixed in 0.3 -nullassign -predboolint -predboolptr -- cgit 1.2.3-korg From 3ed219382e55885a66a462db1cf06170ce80e606 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 19 Aug 2014 17:55:44 +0300 Subject: Add #if guard for .pb.h version. The version in PB_PROTO_HEADER_VERSION can be bumped whenever there is a breaking change to the generated files, and it will then alert to the difference. Update issue 129 Status: FixedInGit --- generator/nanopb_generator.py | 12 +- pb.h | 20 +-- tests/backwards_compatibility/alltypes_legacy.c | 196 ++++++++++++++------- tests/backwards_compatibility/alltypes_legacy.h | 160 +++++++++++++---- .../alltypes_legacy.options | 3 + .../backwards_compatibility/alltypes_legacy.proto | 108 ++++++++++++ 6 files changed, 384 insertions(+), 115 deletions(-) create mode 100644 tests/backwards_compatibility/alltypes_legacy.options create mode 100644 tests/backwards_compatibility/alltypes_legacy.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 50108148..5d2a360d 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -361,7 +361,7 @@ class Field: '''Return the pb_field_t initializer to use in the constant array. prev_field_name is the name of the previous field or None. ''' - result = ' PB_FIELD2(%3d, ' % self.tag + result = ' PB_FIELD(%3d, ' % self.tag result += '%-8s, ' % self.pbtype result += '%s, ' % self.rules result += '%-8s, ' % self.allocation @@ -782,6 +782,11 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio yield options.genformat % (noext + options.extension + '.h') yield '\n' + yield '#if PB_PROTO_HEADER_VERSION != 30\n' + yield '#error Regenerate this file with the current version of nanopb generator.\n' + yield '#endif\n' + yield '\n' + yield '#ifdef __cplusplus\n' yield 'extern "C" {\n' yield '#endif\n\n' @@ -854,6 +859,11 @@ def generate_source(headername, enums, messages, extensions, options): yield options.genformat % (headername) yield '\n' + yield '#if PB_PROTO_HEADER_VERSION != 30\n' + yield '#error Regenerate this file with the current version of nanopb generator.\n' + yield '#endif\n' + yield '\n' + for msg in messages: yield msg.default_decl(False) diff --git a/pb.h b/pb.h index 5edd6483..98f1d387 100644 --- a/pb.h +++ b/pb.h @@ -365,6 +365,9 @@ struct pb_extension_s { # endif #endif +/* This is used to inform about need to regenerate .pb.h/.pb.c files. */ +#define PB_PROTO_HEADER_VERSION 30 + /* These macros are used to declare pb_field_t's in the constant array. */ /* Size of a structure member, in bytes. */ #define pb_membersize(st, m) (sizeof ((st*)0)->m) @@ -476,26 +479,15 @@ struct pb_extension_s { * SINT32, SINT64, STRING, UINT32, UINT64 or EXTENSION * - Field rules: REQUIRED, OPTIONAL or REPEATED * - Allocation: STATIC or CALLBACK + * - Placement: FIRST or OTHER, depending on if this is the first field in structure. * - Message name * - Field name * - Previous field name (or field name again for first field) * - Pointer to default value or submsg fields. */ -#define PB_FIELD(tag, type, rules, allocation, message, field, prevfield, ptr) \ - PB_ ## rules ## _ ## allocation(tag, message, field, \ - PB_DATAOFFSET_CHOOSE(message, field, prevfield), \ - PB_LTYPE_MAP_ ## type, ptr) - -/* This is a new version of the macro used by nanopb generator from - * version 0.2.3 onwards. It avoids the use of a ternary expression in - * the initialization, which confused some compilers. - * - * - Placement: FIRST or OTHER, depending on if this is the first field in structure. - * - */ -#define PB_FIELD2(tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ - PB_ ## rules ## _ ## allocation(tag, message, field, \ +#define PB_FIELD(tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ + PB_ ## rules ## _ ## allocation(tag, message, field, \ PB_DATAOFFSET_ ## placement(message, field, prevfield), \ PB_LTYPE_MAP_ ## type, ptr) diff --git a/tests/backwards_compatibility/alltypes_legacy.c b/tests/backwards_compatibility/alltypes_legacy.c index 9134d5e6..7311fd45 100644 --- a/tests/backwards_compatibility/alltypes_legacy.c +++ b/tests/backwards_compatibility/alltypes_legacy.c @@ -1,27 +1,37 @@ /* Automatically generated nanopb constant definitions */ -/* Generated by 0.2.0-dev at Sun Feb 17 00:09:53 2013. */ -/* This is a file generated using nanopb-0.2.0-dev. - * It is used as a part of test suite in order to detect any - * incompatible changes made to the generator in future versions. - */ +/* Generated by nanopb-0.3.0-dev at Tue Aug 19 17:53:24 2014. */ #include "alltypes_legacy.h" +#if PB_PROTO_HEADER_VERSION != 30 +#error Regenerate this file with the current version of nanopb generator. +#endif + const char SubMessage_substuff1_default[16] = "1"; const int32_t SubMessage_substuff2_default = 2; -const uint32_t SubMessage_substuff3_default = 3; +const uint32_t SubMessage_substuff3_default = 3u; +const int32_t Limits_int32_min_default = 2147483647; +const int32_t Limits_int32_max_default = -2147483647; +const uint32_t Limits_uint32_min_default = 4294967295u; +const uint32_t Limits_uint32_max_default = 0u; +const int64_t Limits_int64_min_default = 9223372036854775807ll; +const int64_t Limits_int64_max_default = -9223372036854775807ll; +const uint64_t Limits_uint64_min_default = 18446744073709551615ull; +const uint64_t Limits_uint64_max_default = 0ull; +const HugeEnum Limits_enum_min_default = HugeEnum_Positive; +const HugeEnum Limits_enum_max_default = HugeEnum_Negative; const int32_t AllTypes_opt_int32_default = 4041; -const int64_t AllTypes_opt_int64_default = 4042; -const uint32_t AllTypes_opt_uint32_default = 4043; -const uint64_t AllTypes_opt_uint64_default = 4044; +const int64_t AllTypes_opt_int64_default = 4042ll; +const uint32_t AllTypes_opt_uint32_default = 4043u; +const uint64_t AllTypes_opt_uint64_default = 4044ull; const int32_t AllTypes_opt_sint32_default = 4045; const int64_t AllTypes_opt_sint64_default = 4046; const bool AllTypes_opt_bool_default = false; -const uint32_t AllTypes_opt_fixed32_default = 4048; +const uint32_t AllTypes_opt_fixed32_default = 4048u; const int32_t AllTypes_opt_sfixed32_default = 4049; const float AllTypes_opt_float_default = 4050; -const uint64_t AllTypes_opt_fixed64_default = 4051; -const int64_t AllTypes_opt_sfixed64_default = 4052; +const uint64_t AllTypes_opt_fixed64_default = 4051ull; +const int64_t AllTypes_opt_sfixed64_default = 4052ll; const double AllTypes_opt_double_default = 4053; const char AllTypes_opt_string_default[16] = "4054"; const AllTypes_opt_bytes_t AllTypes_opt_bytes_default = {4, {0x34,0x30,0x35,0x35}}; @@ -29,65 +39,115 @@ const MyEnum AllTypes_opt_enum_default = MyEnum_Second; const pb_field_t SubMessage_fields[4] = { - PB_FIELD( 1, STRING , REQUIRED, STATIC, SubMessage, substuff1, substuff1, &SubMessage_substuff1_default), - PB_FIELD( 2, INT32 , REQUIRED, STATIC, SubMessage, substuff2, substuff1, &SubMessage_substuff2_default), - PB_FIELD( 3, FIXED32 , OPTIONAL, STATIC, SubMessage, substuff3, substuff2, &SubMessage_substuff3_default), + PB_FIELD( 1, STRING , REQUIRED, STATIC , FIRST, SubMessage, substuff1, substuff1, &SubMessage_substuff1_default), + PB_FIELD( 2, INT32 , REQUIRED, STATIC , OTHER, SubMessage, substuff2, substuff1, &SubMessage_substuff2_default), + PB_FIELD( 3, FIXED32 , OPTIONAL, STATIC , OTHER, SubMessage, substuff3, substuff2, &SubMessage_substuff3_default), + PB_LAST_FIELD +}; + +const pb_field_t EmptyMessage_fields[1] = { + PB_LAST_FIELD +}; + +const pb_field_t Limits_fields[11] = { + PB_FIELD( 1, INT32 , REQUIRED, STATIC , FIRST, Limits, int32_min, int32_min, &Limits_int32_min_default), + PB_FIELD( 2, INT32 , REQUIRED, STATIC , OTHER, Limits, int32_max, int32_min, &Limits_int32_max_default), + PB_FIELD( 3, UINT32 , REQUIRED, STATIC , OTHER, Limits, uint32_min, int32_max, &Limits_uint32_min_default), + PB_FIELD( 4, UINT32 , REQUIRED, STATIC , OTHER, Limits, uint32_max, uint32_min, &Limits_uint32_max_default), + PB_FIELD( 5, INT64 , REQUIRED, STATIC , OTHER, Limits, int64_min, uint32_max, &Limits_int64_min_default), + PB_FIELD( 6, INT64 , REQUIRED, STATIC , OTHER, Limits, int64_max, int64_min, &Limits_int64_max_default), + PB_FIELD( 7, UINT64 , REQUIRED, STATIC , OTHER, Limits, uint64_min, int64_max, &Limits_uint64_min_default), + PB_FIELD( 8, UINT64 , REQUIRED, STATIC , OTHER, Limits, uint64_max, uint64_min, &Limits_uint64_max_default), + PB_FIELD( 9, ENUM , REQUIRED, STATIC , OTHER, Limits, enum_min, uint64_max, &Limits_enum_min_default), + PB_FIELD( 10, ENUM , REQUIRED, STATIC , OTHER, Limits, enum_max, enum_min, &Limits_enum_max_default), PB_LAST_FIELD }; -const pb_field_t AllTypes_fields[53] = { - PB_FIELD( 1, INT32 , REQUIRED, STATIC, AllTypes, req_int32, req_int32, 0), - PB_FIELD( 2, INT64 , REQUIRED, STATIC, AllTypes, req_int64, req_int32, 0), - PB_FIELD( 3, UINT32 , REQUIRED, STATIC, AllTypes, req_uint32, req_int64, 0), - PB_FIELD( 4, UINT64 , REQUIRED, STATIC, AllTypes, req_uint64, req_uint32, 0), - PB_FIELD( 5, SINT32 , REQUIRED, STATIC, AllTypes, req_sint32, req_uint64, 0), - PB_FIELD( 6, SINT64 , REQUIRED, STATIC, AllTypes, req_sint64, req_sint32, 0), - PB_FIELD( 7, BOOL , REQUIRED, STATIC, AllTypes, req_bool, req_sint64, 0), - PB_FIELD( 8, FIXED32 , REQUIRED, STATIC, AllTypes, req_fixed32, req_bool, 0), - PB_FIELD( 9, SFIXED32, REQUIRED, STATIC, AllTypes, req_sfixed32, req_fixed32, 0), - PB_FIELD( 10, FLOAT , REQUIRED, STATIC, AllTypes, req_float, req_sfixed32, 0), - PB_FIELD( 11, FIXED64 , REQUIRED, STATIC, AllTypes, req_fixed64, req_float, 0), - PB_FIELD( 12, SFIXED64, REQUIRED, STATIC, AllTypes, req_sfixed64, req_fixed64, 0), - PB_FIELD( 13, DOUBLE , REQUIRED, STATIC, AllTypes, req_double, req_sfixed64, 0), - PB_FIELD( 14, STRING , REQUIRED, STATIC, AllTypes, req_string, req_double, 0), - PB_FIELD( 15, BYTES , REQUIRED, STATIC, AllTypes, req_bytes, req_string, 0), - PB_FIELD( 16, MESSAGE , REQUIRED, STATIC, AllTypes, req_submsg, req_bytes, &SubMessage_fields), - PB_FIELD( 17, ENUM , REQUIRED, STATIC, AllTypes, req_enum, req_submsg, 0), - PB_FIELD( 21, INT32 , REPEATED, STATIC, AllTypes, rep_int32, req_enum, 0), - PB_FIELD( 22, INT64 , REPEATED, STATIC, AllTypes, rep_int64, rep_int32, 0), - PB_FIELD( 23, UINT32 , REPEATED, STATIC, AllTypes, rep_uint32, rep_int64, 0), - PB_FIELD( 24, UINT64 , REPEATED, STATIC, AllTypes, rep_uint64, rep_uint32, 0), - PB_FIELD( 25, SINT32 , REPEATED, STATIC, AllTypes, rep_sint32, rep_uint64, 0), - PB_FIELD( 26, SINT64 , REPEATED, STATIC, AllTypes, rep_sint64, rep_sint32, 0), - PB_FIELD( 27, BOOL , REPEATED, STATIC, AllTypes, rep_bool, rep_sint64, 0), - PB_FIELD( 28, FIXED32 , REPEATED, STATIC, AllTypes, rep_fixed32, rep_bool, 0), - PB_FIELD( 29, SFIXED32, REPEATED, STATIC, AllTypes, rep_sfixed32, rep_fixed32, 0), - PB_FIELD( 30, FLOAT , REPEATED, STATIC, AllTypes, rep_float, rep_sfixed32, 0), - PB_FIELD( 31, FIXED64 , REPEATED, STATIC, AllTypes, rep_fixed64, rep_float, 0), - PB_FIELD( 32, SFIXED64, REPEATED, STATIC, AllTypes, rep_sfixed64, rep_fixed64, 0), - PB_FIELD( 33, DOUBLE , REPEATED, STATIC, AllTypes, rep_double, rep_sfixed64, 0), - PB_FIELD( 34, STRING , REPEATED, STATIC, AllTypes, rep_string, rep_double, 0), - PB_FIELD( 35, BYTES , REPEATED, STATIC, AllTypes, rep_bytes, rep_string, 0), - PB_FIELD( 36, MESSAGE , REPEATED, STATIC, AllTypes, rep_submsg, rep_bytes, &SubMessage_fields), - PB_FIELD( 37, ENUM , REPEATED, STATIC, AllTypes, rep_enum, rep_submsg, 0), - PB_FIELD( 41, INT32 , OPTIONAL, STATIC, AllTypes, opt_int32, rep_enum, &AllTypes_opt_int32_default), - PB_FIELD( 42, INT64 , OPTIONAL, STATIC, AllTypes, opt_int64, opt_int32, &AllTypes_opt_int64_default), - PB_FIELD( 43, UINT32 , OPTIONAL, STATIC, AllTypes, opt_uint32, opt_int64, &AllTypes_opt_uint32_default), - PB_FIELD( 44, UINT64 , OPTIONAL, STATIC, AllTypes, opt_uint64, opt_uint32, &AllTypes_opt_uint64_default), - PB_FIELD( 45, SINT32 , OPTIONAL, STATIC, AllTypes, opt_sint32, opt_uint64, &AllTypes_opt_sint32_default), - PB_FIELD( 46, SINT64 , OPTIONAL, STATIC, AllTypes, opt_sint64, opt_sint32, &AllTypes_opt_sint64_default), - PB_FIELD( 47, BOOL , OPTIONAL, STATIC, AllTypes, opt_bool, opt_sint64, &AllTypes_opt_bool_default), - PB_FIELD( 48, FIXED32 , OPTIONAL, STATIC, AllTypes, opt_fixed32, opt_bool, &AllTypes_opt_fixed32_default), - PB_FIELD( 49, SFIXED32, OPTIONAL, STATIC, AllTypes, opt_sfixed32, opt_fixed32, &AllTypes_opt_sfixed32_default), - PB_FIELD( 50, FLOAT , OPTIONAL, STATIC, AllTypes, opt_float, opt_sfixed32, &AllTypes_opt_float_default), - PB_FIELD( 51, FIXED64 , OPTIONAL, STATIC, AllTypes, opt_fixed64, opt_float, &AllTypes_opt_fixed64_default), - PB_FIELD( 52, SFIXED64, OPTIONAL, STATIC, AllTypes, opt_sfixed64, opt_fixed64, &AllTypes_opt_sfixed64_default), - PB_FIELD( 53, DOUBLE , OPTIONAL, STATIC, AllTypes, opt_double, opt_sfixed64, &AllTypes_opt_double_default), - PB_FIELD( 54, STRING , OPTIONAL, STATIC, AllTypes, opt_string, opt_double, &AllTypes_opt_string_default), - PB_FIELD( 55, BYTES , OPTIONAL, STATIC, AllTypes, opt_bytes, opt_string, &AllTypes_opt_bytes_default), - PB_FIELD( 56, MESSAGE , OPTIONAL, STATIC, AllTypes, opt_submsg, opt_bytes, &SubMessage_fields), - PB_FIELD( 57, ENUM , OPTIONAL, STATIC, AllTypes, opt_enum, opt_submsg, &AllTypes_opt_enum_default), - PB_FIELD( 99, INT32 , REQUIRED, STATIC, AllTypes, end, opt_enum, 0), +const pb_field_t AllTypes_fields[54] = { + PB_FIELD( 1, INT32 , REQUIRED, STATIC , FIRST, AllTypes, req_int32, req_int32, 0), + PB_FIELD( 2, INT64 , REQUIRED, STATIC , OTHER, AllTypes, req_int64, req_int32, 0), + PB_FIELD( 3, UINT32 , REQUIRED, STATIC , OTHER, AllTypes, req_uint32, req_int64, 0), + PB_FIELD( 4, UINT64 , REQUIRED, STATIC , OTHER, AllTypes, req_uint64, req_uint32, 0), + PB_FIELD( 5, SINT32 , REQUIRED, STATIC , OTHER, AllTypes, req_sint32, req_uint64, 0), + PB_FIELD( 6, SINT64 , REQUIRED, STATIC , OTHER, AllTypes, req_sint64, req_sint32, 0), + PB_FIELD( 7, BOOL , REQUIRED, STATIC , OTHER, AllTypes, req_bool, req_sint64, 0), + PB_FIELD( 8, FIXED32 , REQUIRED, STATIC , OTHER, AllTypes, req_fixed32, req_bool, 0), + PB_FIELD( 9, SFIXED32, REQUIRED, STATIC , OTHER, AllTypes, req_sfixed32, req_fixed32, 0), + PB_FIELD( 10, FLOAT , REQUIRED, STATIC , OTHER, AllTypes, req_float, req_sfixed32, 0), + PB_FIELD( 11, FIXED64 , REQUIRED, STATIC , OTHER, AllTypes, req_fixed64, req_float, 0), + PB_FIELD( 12, SFIXED64, REQUIRED, STATIC , OTHER, AllTypes, req_sfixed64, req_fixed64, 0), + PB_FIELD( 13, DOUBLE , REQUIRED, STATIC , OTHER, AllTypes, req_double, req_sfixed64, 0), + PB_FIELD( 14, STRING , REQUIRED, STATIC , OTHER, AllTypes, req_string, req_double, 0), + PB_FIELD( 15, BYTES , REQUIRED, STATIC , OTHER, AllTypes, req_bytes, req_string, 0), + PB_FIELD( 16, MESSAGE , REQUIRED, STATIC , OTHER, AllTypes, req_submsg, req_bytes, &SubMessage_fields), + PB_FIELD( 17, ENUM , REQUIRED, STATIC , OTHER, AllTypes, req_enum, req_submsg, 0), + PB_FIELD( 21, INT32 , REPEATED, STATIC , OTHER, AllTypes, rep_int32, req_enum, 0), + PB_FIELD( 22, INT64 , REPEATED, STATIC , OTHER, AllTypes, rep_int64, rep_int32, 0), + PB_FIELD( 23, UINT32 , REPEATED, STATIC , OTHER, AllTypes, rep_uint32, rep_int64, 0), + PB_FIELD( 24, UINT64 , REPEATED, STATIC , OTHER, AllTypes, rep_uint64, rep_uint32, 0), + PB_FIELD( 25, SINT32 , REPEATED, STATIC , OTHER, AllTypes, rep_sint32, rep_uint64, 0), + PB_FIELD( 26, SINT64 , REPEATED, STATIC , OTHER, AllTypes, rep_sint64, rep_sint32, 0), + PB_FIELD( 27, BOOL , REPEATED, STATIC , OTHER, AllTypes, rep_bool, rep_sint64, 0), + PB_FIELD( 28, FIXED32 , REPEATED, STATIC , OTHER, AllTypes, rep_fixed32, rep_bool, 0), + PB_FIELD( 29, SFIXED32, REPEATED, STATIC , OTHER, AllTypes, rep_sfixed32, rep_fixed32, 0), + PB_FIELD( 30, FLOAT , REPEATED, STATIC , OTHER, AllTypes, rep_float, rep_sfixed32, 0), + PB_FIELD( 31, FIXED64 , REPEATED, STATIC , OTHER, AllTypes, rep_fixed64, rep_float, 0), + PB_FIELD( 32, SFIXED64, REPEATED, STATIC , OTHER, AllTypes, rep_sfixed64, rep_fixed64, 0), + PB_FIELD( 33, DOUBLE , REPEATED, STATIC , OTHER, AllTypes, rep_double, rep_sfixed64, 0), + PB_FIELD( 34, STRING , REPEATED, STATIC , OTHER, AllTypes, rep_string, rep_double, 0), + PB_FIELD( 35, BYTES , REPEATED, STATIC , OTHER, AllTypes, rep_bytes, rep_string, 0), + PB_FIELD( 36, MESSAGE , REPEATED, STATIC , OTHER, AllTypes, rep_submsg, rep_bytes, &SubMessage_fields), + PB_FIELD( 37, ENUM , REPEATED, STATIC , OTHER, AllTypes, rep_enum, rep_submsg, 0), + PB_FIELD( 41, INT32 , OPTIONAL, STATIC , OTHER, AllTypes, opt_int32, rep_enum, &AllTypes_opt_int32_default), + PB_FIELD( 42, INT64 , OPTIONAL, STATIC , OTHER, AllTypes, opt_int64, opt_int32, &AllTypes_opt_int64_default), + PB_FIELD( 43, UINT32 , OPTIONAL, STATIC , OTHER, AllTypes, opt_uint32, opt_int64, &AllTypes_opt_uint32_default), + PB_FIELD( 44, UINT64 , OPTIONAL, STATIC , OTHER, AllTypes, opt_uint64, opt_uint32, &AllTypes_opt_uint64_default), + PB_FIELD( 45, SINT32 , OPTIONAL, STATIC , OTHER, AllTypes, opt_sint32, opt_uint64, &AllTypes_opt_sint32_default), + PB_FIELD( 46, SINT64 , OPTIONAL, STATIC , OTHER, AllTypes, opt_sint64, opt_sint32, &AllTypes_opt_sint64_default), + PB_FIELD( 47, BOOL , OPTIONAL, STATIC , OTHER, AllTypes, opt_bool, opt_sint64, &AllTypes_opt_bool_default), + PB_FIELD( 48, FIXED32 , OPTIONAL, STATIC , OTHER, AllTypes, opt_fixed32, opt_bool, &AllTypes_opt_fixed32_default), + PB_FIELD( 49, SFIXED32, OPTIONAL, STATIC , OTHER, AllTypes, opt_sfixed32, opt_fixed32, &AllTypes_opt_sfixed32_default), + PB_FIELD( 50, FLOAT , OPTIONAL, STATIC , OTHER, AllTypes, opt_float, opt_sfixed32, &AllTypes_opt_float_default), + PB_FIELD( 51, FIXED64 , OPTIONAL, STATIC , OTHER, AllTypes, opt_fixed64, opt_float, &AllTypes_opt_fixed64_default), + PB_FIELD( 52, SFIXED64, OPTIONAL, STATIC , OTHER, AllTypes, opt_sfixed64, opt_fixed64, &AllTypes_opt_sfixed64_default), + PB_FIELD( 53, DOUBLE , OPTIONAL, STATIC , OTHER, AllTypes, opt_double, opt_sfixed64, &AllTypes_opt_double_default), + PB_FIELD( 54, STRING , OPTIONAL, STATIC , OTHER, AllTypes, opt_string, opt_double, &AllTypes_opt_string_default), + PB_FIELD( 55, BYTES , OPTIONAL, STATIC , OTHER, AllTypes, opt_bytes, opt_string, &AllTypes_opt_bytes_default), + PB_FIELD( 56, MESSAGE , OPTIONAL, STATIC , OTHER, AllTypes, opt_submsg, opt_bytes, &SubMessage_fields), + PB_FIELD( 57, ENUM , OPTIONAL, STATIC , OTHER, AllTypes, opt_enum, opt_submsg, &AllTypes_opt_enum_default), + PB_FIELD( 99, INT32 , REQUIRED, STATIC , OTHER, AllTypes, end, opt_enum, 0), + PB_FIELD(200, EXTENSION, OPTIONAL, CALLBACK, OTHER, AllTypes, extensions, end, 0), PB_LAST_FIELD }; + +/* Check that field information fits in pb_field_t */ +#if !defined(PB_FIELD_32BIT) +/* If you get an error here, it means that you need to define PB_FIELD_32BIT + * compile-time option. You can do that in pb.h or on compiler command line. + * + * The reason you need to do this is that some of your messages contain tag + * numbers or field sizes that are larger than what can fit in 8 or 16 bit + * field descriptors. + */ +PB_STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 65536 && pb_membersize(AllTypes, rep_submsg[0]) < 65536 && pb_membersize(AllTypes, opt_submsg) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_SubMessage_EmptyMessage_Limits_AllTypes) +#endif + +#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) +/* If you get an error here, it means that you need to define PB_FIELD_16BIT + * compile-time option. You can do that in pb.h or on compiler command line. + * + * The reason you need to do this is that some of your messages contain tag + * numbers or field sizes that are larger than what can fit in the default + * 8 bit descriptors. + */ +PB_STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 256 && pb_membersize(AllTypes, rep_submsg[0]) < 256 && pb_membersize(AllTypes, opt_submsg) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_SubMessage_EmptyMessage_Limits_AllTypes) +#endif + + +/* On some platforms (such as AVR), double is really float. + * These are not directly supported by nanopb, but see example_avr_double. + * To get rid of this error, remove any double fields from your .proto. + */ +PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES) + diff --git a/tests/backwards_compatibility/alltypes_legacy.h b/tests/backwards_compatibility/alltypes_legacy.h index 8b7e3bca..4e0a63be 100644 --- a/tests/backwards_compatibility/alltypes_legacy.h +++ b/tests/backwards_compatibility/alltypes_legacy.h @@ -1,18 +1,24 @@ /* Automatically generated nanopb header */ -/* This is a file generated using nanopb-0.2.0-dev. - * It is used as a part of test suite in order to detect any - * incompatible changes made to the generator in future versions. - */ +/* Generated by nanopb-0.3.0-dev at Tue Aug 19 17:53:24 2014. */ -#ifndef _PB_ALLTYPES_PB_H_ -#define _PB_ALLTYPES_PB_H_ +#ifndef PB_ALLTYPES_LEGACY_H_INCLUDED +#define PB_ALLTYPES_LEGACY_H_INCLUDED #include +#if PB_PROTO_HEADER_VERSION != 30 +#error Regenerate this file with the current version of nanopb generator. +#endif + #ifdef __cplusplus extern "C" { #endif /* Enum definitions */ +typedef enum _HugeEnum { + HugeEnum_Negative = -2147483647, + HugeEnum_Positive = 2147483647 +} HugeEnum; + typedef enum _MyEnum { MyEnum_Zero = 0, MyEnum_First = 1, @@ -21,6 +27,23 @@ typedef enum _MyEnum { } MyEnum; /* Struct definitions */ +typedef struct _EmptyMessage { + uint8_t dummy_field; +} EmptyMessage; + +typedef struct _Limits { + int32_t int32_min; + int32_t int32_max; + uint32_t uint32_min; + uint32_t uint32_max; + int64_t int64_min; + int64_t int64_max; + uint64_t uint64_min; + uint64_t uint64_max; + HugeEnum enum_min; + HugeEnum enum_max; +} Limits; + typedef struct _SubMessage { char substuff1[16]; int32_t substuff2; @@ -28,20 +51,11 @@ typedef struct _SubMessage { uint32_t substuff3; } SubMessage; -typedef struct { - pb_size_t size; - uint8_t bytes[16]; -} AllTypes_req_bytes_t; +typedef PB_BYTES_ARRAY_T(16) AllTypes_req_bytes_t; -typedef struct { - pb_size_t size; - uint8_t bytes[16]; -} AllTypes_rep_bytes_t; +typedef PB_BYTES_ARRAY_T(16) AllTypes_rep_bytes_t; -typedef struct { - pb_size_t size; - uint8_t bytes[16]; -} AllTypes_opt_bytes_t; +typedef PB_BYTES_ARRAY_T(16) AllTypes_opt_bytes_t; typedef struct _AllTypes { int32_t req_int32; @@ -130,12 +144,23 @@ typedef struct _AllTypes { bool has_opt_enum; MyEnum opt_enum; int32_t end; + pb_extension_t *extensions; } AllTypes; /* Default values for struct fields */ extern const char SubMessage_substuff1_default[16]; extern const int32_t SubMessage_substuff2_default; extern const uint32_t SubMessage_substuff3_default; +extern const int32_t Limits_int32_min_default; +extern const int32_t Limits_int32_max_default; +extern const uint32_t Limits_uint32_min_default; +extern const uint32_t Limits_uint32_max_default; +extern const int64_t Limits_int64_min_default; +extern const int64_t Limits_int64_max_default; +extern const uint64_t Limits_uint64_min_default; +extern const uint64_t Limits_uint64_max_default; +extern const HugeEnum Limits_enum_min_default; +extern const HugeEnum Limits_enum_max_default; extern const int32_t AllTypes_opt_int32_default; extern const int64_t AllTypes_opt_int64_default; extern const uint32_t AllTypes_opt_uint32_default; @@ -153,23 +178,94 @@ extern const char AllTypes_opt_string_default[16]; extern const AllTypes_opt_bytes_t AllTypes_opt_bytes_default; extern const MyEnum AllTypes_opt_enum_default; -/* Struct field encoding specification for nanopb */ -extern const pb_field_t SubMessage_fields[4]; -extern const pb_field_t AllTypes_fields[53]; +/* Initializer values for message structs */ +#define SubMessage_init_default {"1", 2, false, 3u} +#define EmptyMessage_init_default {0} +#define Limits_init_default {2147483647, -2147483647, 4294967295u, 0u, 9223372036854775807ll, -9223372036854775807ll, 18446744073709551615ull, 0ull, HugeEnum_Positive, HugeEnum_Negative} +#define AllTypes_init_default {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", {0, {0}}, SubMessage_init_default, (MyEnum)0, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {"", "", "", "", ""}, 0, {{0, {0}}, {0, {0}}, {0, {0}}, {0, {0}}, {0, {0}}}, 0, {SubMessage_init_default, SubMessage_init_default, SubMessage_init_default, SubMessage_init_default, SubMessage_init_default}, 0, {(MyEnum)0, (MyEnum)0, (MyEnum)0, (MyEnum)0, (MyEnum)0}, false, 4041, false, 4042ll, false, 4043u, false, 4044ull, false, 4045, false, 4046, false, false, false, 4048u, false, 4049, false, 4050, false, 4051ull, false, 4052ll, false, 4053, false, "4054", false, {4, {0x34,0x30,0x35,0x35}}, false, SubMessage_init_default, false, MyEnum_Second, 0, NULL} +#define SubMessage_init_zero {"", 0, false, 0} +#define EmptyMessage_init_zero {0} +#define Limits_init_zero {0, 0, 0, 0, 0, 0, 0, 0, (HugeEnum)0, (HugeEnum)0} +#define AllTypes_init_zero {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", {0, {0}}, SubMessage_init_zero, (MyEnum)0, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {"", "", "", "", ""}, 0, {{0, {0}}, {0, {0}}, {0, {0}}, {0, {0}}, {0, {0}}}, 0, {SubMessage_init_zero, SubMessage_init_zero, SubMessage_init_zero, SubMessage_init_zero, SubMessage_init_zero}, 0, {(MyEnum)0, (MyEnum)0, (MyEnum)0, (MyEnum)0, (MyEnum)0}, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, {0, {0}}, false, SubMessage_init_zero, false, (MyEnum)0, 0, NULL} -/* Check that field information fits in pb_field_t */ -#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) -PB_STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 256 && pb_membersize(AllTypes, rep_submsg[0]) < 256 && pb_membersize(AllTypes, opt_submsg) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_SubMessage_AllTypes) -#endif +/* Field tags (for use in manual encoding/decoding) */ +#define Limits_int32_min_tag 1 +#define Limits_int32_max_tag 2 +#define Limits_uint32_min_tag 3 +#define Limits_uint32_max_tag 4 +#define Limits_int64_min_tag 5 +#define Limits_int64_max_tag 6 +#define Limits_uint64_min_tag 7 +#define Limits_uint64_max_tag 8 +#define Limits_enum_min_tag 9 +#define Limits_enum_max_tag 10 +#define SubMessage_substuff1_tag 1 +#define SubMessage_substuff2_tag 2 +#define SubMessage_substuff3_tag 3 +#define AllTypes_req_int32_tag 1 +#define AllTypes_req_int64_tag 2 +#define AllTypes_req_uint32_tag 3 +#define AllTypes_req_uint64_tag 4 +#define AllTypes_req_sint32_tag 5 +#define AllTypes_req_sint64_tag 6 +#define AllTypes_req_bool_tag 7 +#define AllTypes_req_fixed32_tag 8 +#define AllTypes_req_sfixed32_tag 9 +#define AllTypes_req_float_tag 10 +#define AllTypes_req_fixed64_tag 11 +#define AllTypes_req_sfixed64_tag 12 +#define AllTypes_req_double_tag 13 +#define AllTypes_req_string_tag 14 +#define AllTypes_req_bytes_tag 15 +#define AllTypes_req_submsg_tag 16 +#define AllTypes_req_enum_tag 17 +#define AllTypes_rep_int32_tag 21 +#define AllTypes_rep_int64_tag 22 +#define AllTypes_rep_uint32_tag 23 +#define AllTypes_rep_uint64_tag 24 +#define AllTypes_rep_sint32_tag 25 +#define AllTypes_rep_sint64_tag 26 +#define AllTypes_rep_bool_tag 27 +#define AllTypes_rep_fixed32_tag 28 +#define AllTypes_rep_sfixed32_tag 29 +#define AllTypes_rep_float_tag 30 +#define AllTypes_rep_fixed64_tag 31 +#define AllTypes_rep_sfixed64_tag 32 +#define AllTypes_rep_double_tag 33 +#define AllTypes_rep_string_tag 34 +#define AllTypes_rep_bytes_tag 35 +#define AllTypes_rep_submsg_tag 36 +#define AllTypes_rep_enum_tag 37 +#define AllTypes_opt_int32_tag 41 +#define AllTypes_opt_int64_tag 42 +#define AllTypes_opt_uint32_tag 43 +#define AllTypes_opt_uint64_tag 44 +#define AllTypes_opt_sint32_tag 45 +#define AllTypes_opt_sint64_tag 46 +#define AllTypes_opt_bool_tag 47 +#define AllTypes_opt_fixed32_tag 48 +#define AllTypes_opt_sfixed32_tag 49 +#define AllTypes_opt_float_tag 50 +#define AllTypes_opt_fixed64_tag 51 +#define AllTypes_opt_sfixed64_tag 52 +#define AllTypes_opt_double_tag 53 +#define AllTypes_opt_string_tag 54 +#define AllTypes_opt_bytes_tag 55 +#define AllTypes_opt_submsg_tag 56 +#define AllTypes_opt_enum_tag 57 +#define AllTypes_end_tag 99 -#if !defined(PB_FIELD_32BIT) -PB_STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 65536 && pb_membersize(AllTypes, rep_submsg[0]) < 65536 && pb_membersize(AllTypes, opt_submsg) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_SubMessage_AllTypes) -#endif +/* Struct field encoding specification for nanopb */ +extern const pb_field_t SubMessage_fields[4]; +extern const pb_field_t EmptyMessage_fields[1]; +extern const pb_field_t Limits_fields[11]; +extern const pb_field_t AllTypes_fields[54]; -/* On some platforms (such as AVR), double is really float. - * These are not directly supported by nanopb, but see example_avr_double. - */ -PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES) +/* Maximum encoded size of messages (where known) */ +#define SubMessage_size 34 +#define EmptyMessage_size 0 +#define Limits_size 90 +#define AllTypes_size 1362 #ifdef __cplusplus } /* extern "C" */ diff --git a/tests/backwards_compatibility/alltypes_legacy.options b/tests/backwards_compatibility/alltypes_legacy.options new file mode 100644 index 00000000..b31e3cf0 --- /dev/null +++ b/tests/backwards_compatibility/alltypes_legacy.options @@ -0,0 +1,3 @@ +* max_size:16 +* max_count:5 + diff --git a/tests/backwards_compatibility/alltypes_legacy.proto b/tests/backwards_compatibility/alltypes_legacy.proto new file mode 100644 index 00000000..d7631eb6 --- /dev/null +++ b/tests/backwards_compatibility/alltypes_legacy.proto @@ -0,0 +1,108 @@ +message SubMessage { + required string substuff1 = 1 [default = "1"]; + required int32 substuff2 = 2 [default = 2]; + optional fixed32 substuff3 = 3 [default = 3]; +} + +message EmptyMessage { + +} + +enum HugeEnum { + Negative = -2147483647; /* protoc doesn't accept -2147483648 here */ + Positive = 2147483647; +} + +message Limits { + required int32 int32_min = 1 [default = 2147483647]; + required int32 int32_max = 2 [default = -2147483647]; + required uint32 uint32_min = 3 [default = 4294967295]; + required uint32 uint32_max = 4 [default = 0]; + required int64 int64_min = 5 [default = 9223372036854775807]; + required int64 int64_max = 6 [default = -9223372036854775807]; + required uint64 uint64_min = 7 [default = 18446744073709551615]; + required uint64 uint64_max = 8 [default = 0]; + required HugeEnum enum_min = 9 [default = Positive]; + required HugeEnum enum_max = 10 [default = Negative]; +} + +enum MyEnum { + Zero = 0; + First = 1; + Second = 2; + Truth = 42; +} + +message AllTypes { + required int32 req_int32 = 1; + required int64 req_int64 = 2; + required uint32 req_uint32 = 3; + required uint64 req_uint64 = 4; + required sint32 req_sint32 = 5; + required sint64 req_sint64 = 6; + required bool req_bool = 7; + + required fixed32 req_fixed32 = 8; + required sfixed32 req_sfixed32= 9; + required float req_float = 10; + + required fixed64 req_fixed64 = 11; + required sfixed64 req_sfixed64= 12; + required double req_double = 13; + + required string req_string = 14; + required bytes req_bytes = 15; + required SubMessage req_submsg = 16; + required MyEnum req_enum = 17; + + + repeated int32 rep_int32 = 21 [packed = true]; + repeated int64 rep_int64 = 22 [packed = true]; + repeated uint32 rep_uint32 = 23 [packed = true]; + repeated uint64 rep_uint64 = 24 [packed = true]; + repeated sint32 rep_sint32 = 25 [packed = true]; + repeated sint64 rep_sint64 = 26 [packed = true]; + repeated bool rep_bool = 27 [packed = true]; + + repeated fixed32 rep_fixed32 = 28 [packed = true]; + repeated sfixed32 rep_sfixed32= 29 [packed = true]; + repeated float rep_float = 30 [packed = true]; + + repeated fixed64 rep_fixed64 = 31 [packed = true]; + repeated sfixed64 rep_sfixed64= 32 [packed = true]; + repeated double rep_double = 33 [packed = true]; + + repeated string rep_string = 34; + repeated bytes rep_bytes = 35; + repeated SubMessage rep_submsg = 36; + repeated MyEnum rep_enum = 37 [packed = true]; + + optional int32 opt_int32 = 41 [default = 4041]; + optional int64 opt_int64 = 42 [default = 4042]; + optional uint32 opt_uint32 = 43 [default = 4043]; + optional uint64 opt_uint64 = 44 [default = 4044]; + optional sint32 opt_sint32 = 45 [default = 4045]; + optional sint64 opt_sint64 = 46 [default = 4046]; + optional bool opt_bool = 47 [default = false]; + + optional fixed32 opt_fixed32 = 48 [default = 4048]; + optional sfixed32 opt_sfixed32= 49 [default = 4049]; + optional float opt_float = 50 [default = 4050]; + + optional fixed64 opt_fixed64 = 51 [default = 4051]; + optional sfixed64 opt_sfixed64= 52 [default = 4052]; + optional double opt_double = 53 [default = 4053]; + + optional string opt_string = 54 [default = "4054"]; + optional bytes opt_bytes = 55 [default = "4055"]; + optional SubMessage opt_submsg = 56; + optional MyEnum opt_enum = 57 [default = Second]; + + // Just to make sure that the size of the fields has been calculated + // properly, i.e. otherwise a bug in last field might not be detected. + required int32 end = 99; + + + extensions 200 to 255; +} + -- cgit 1.2.3-korg From 8a95311b51e6b64835ba8141594d8e804241377e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 26 Aug 2014 18:05:10 +0300 Subject: Add pb_common.c to examples --- examples/simple/Makefile | 1 + extra/FindNanopb.cmake | 4 ++-- extra/nanopb.mk | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/simple/Makefile b/examples/simple/Makefile index 02a4c3ff..970a8650 100644 --- a/examples/simple/Makefile +++ b/examples/simple/Makefile @@ -10,6 +10,7 @@ CSRC = simple.c # The main program CSRC += simple.pb.c # The compiled protocol definition CSRC += $(NANOPB_DIR)/pb_encode.c # The nanopb encoder CSRC += $(NANOPB_DIR)/pb_decode.c # The nanopb decoder +CSRC += $(NANOPB_DIR)/pb_common.c # The nanopb common parts # Build rule for the main program simple: $(CSRC) diff --git a/extra/FindNanopb.cmake b/extra/FindNanopb.cmake index c804e701..65ab5884 100644 --- a/extra/FindNanopb.cmake +++ b/extra/FindNanopb.cmake @@ -182,8 +182,8 @@ mark_as_advanced(NANOPB_INCLUDE_DIRS) # Find nanopb source files set(NANOPB_SRCS) set(NANOPB_HDRS) -list(APPEND _nanopb_srcs pb_decode.c pb_encode.c) -list(APPEND _nanopb_hdrs pb_decode.h pb_encode.h pb.h) +list(APPEND _nanopb_srcs pb_decode.c pb_encode.c pb_common.c) +list(APPEND _nanopb_hdrs pb_decode.h pb_encode.h pb_common.h pb.h) foreach(FIL ${_nanopb_srcs}) find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_SRC_ROOT_FOLDER} ${NANOPB_INCLUDE_DIRS}) diff --git a/extra/nanopb.mk b/extra/nanopb.mk index 7576baee..5c2cff56 100644 --- a/extra/nanopb.mk +++ b/extra/nanopb.mk @@ -5,7 +5,7 @@ NANOPB_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))../) # Files for the nanopb core -NANOPB_CORE = $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_decode.c +NANOPB_CORE = $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_common.c # Check if we are running on Windows ifdef windir -- cgit 1.2.3-korg From e60dee698a617882659c84624d6a63d7941c4624 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 26 Aug 2014 18:08:01 +0300 Subject: Update changelog --- CHANGELOG.txt | 12 ++++++++++++ docs/migration.rst | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index d8b3ec86..1555baee 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,15 @@ +nanopb-0.3.0 (2014-08-26) + NOTE: See docs/migration.html or online at + http://koti.kapsi.fi/~jpa/nanopb/docs/migration.html + for changes in this version. Most importantly, you need to add + pb_common.c to the list of files to compile. + + Separated field iterator logic to pb_common.c (issue 128) + Change the _count fields to use pb_size_t datatype (issue 82) + Added PB_ prefix to macro names (issue 106) + Added #if version guard to generated files (issue 129) + Added migration document + nanopb-0.2.9 (2014-08-09) NOTE: If you are using the -e option with the generator, you have to prepend . to the argument to get the same behaviour as before. diff --git a/docs/migration.rst b/docs/migration.rst index 5ac52b3e..67f97b34 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -11,7 +11,7 @@ are included, in order to make it easier to find this document. .. contents :: -Nanopb-0.3.0 (2014-09-xx) +Nanopb-0.3.0 (2014-08-26) ========================= Separate field iterator logic to pb_common.c -- cgit 1.2.3-korg From 8dede6095ede2281879b3484fa848210bb63dcfc Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 26 Aug 2014 18:08:31 +0300 Subject: Publishing nanopb-0.3.0 --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 5d2a360d..4ad82470 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.0-dev" +nanopb_version = "nanopb-0.3.0" import sys diff --git a/pb.h b/pb.h index 98f1d387..0796c3bf 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.0-dev +#define NANOPB_VERSION nanopb-0.3.0 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 71b81ad5735e8db6f85795489a4bb2d8d7af8b7d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 26 Aug 2014 18:20:48 +0300 Subject: Setting version to 0.3.1-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 4ad82470..79dd1989 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.0" +nanopb_version = "nanopb-0.3.1-dev" import sys diff --git a/pb.h b/pb.h index 0796c3bf..fc301479 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.0 +#define NANOPB_VERSION nanopb-0.3.1-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 9e866b485319e2ede206b4eafa0b6235be0c4a38 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 26 Aug 2014 18:22:13 +0300 Subject: Add missing * in migration docs --- docs/migration.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/migration.rst b/docs/migration.rst index 67f97b34..5f7246f5 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -151,12 +151,12 @@ Callback function signature as *void\**. This allowed passing of any data, but made it unnecessarily complex to return a pointer from callback. -**Changes:** The callback function parameter was changed to *void\**. +**Changes:** The callback function parameter was changed to *void\*\**. **Required actions:** You can continue using the old callback style by defining *PB_OLD_CALLBACK_STYLE*. Recommended action is to: - * Change the callback signatures to contain *void\** for decoders and + * Change the callback signatures to contain *void\*\** for decoders and *void \* const \** for encoders. * Change the callback function body to use *\*arg* instead of *arg*. -- cgit 1.2.3-korg From df7234fd8b8779c9973b19b5a8fb22ee2e903982 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 28 Aug 2014 21:23:28 +0300 Subject: Fix cyclic messages support in generator. Beginnings of test. Update issue 130 Status: Started --- generator/nanopb_generator.py | 2 +- tests/cyclic_messages/SConscript | 11 ++ tests/cyclic_messages/cyclic.proto | 25 +++++ tests/cyclic_messages/cyclic_callback.options | 7 ++ tests/cyclic_messages/encode_cyclic_callback.c | 148 +++++++++++++++++++++++++ 5 files changed, 192 insertions(+), 1 deletion(-) create mode 100644 tests/cyclic_messages/SConscript create mode 100644 tests/cyclic_messages/cyclic.proto create mode 100644 tests/cyclic_messages/cyclic_callback.options create mode 100644 tests/cyclic_messages/encode_cyclic_callback.c diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 79dd1989..56dfdb08 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -546,7 +546,7 @@ class Message: def get_dependencies(self): '''Get list of type names that this structure refers to.''' - return [str(field.ctype) for field in self.fields] + return [str(field.ctype) for field in self.fields if field.allocation == 'STATIC'] def __str__(self): result = 'typedef struct _%s {\n' % self.name diff --git a/tests/cyclic_messages/SConscript b/tests/cyclic_messages/SConscript new file mode 100644 index 00000000..c782001c --- /dev/null +++ b/tests/cyclic_messages/SConscript @@ -0,0 +1,11 @@ +Import("env") + +# Encode cyclic messages with callback fields + +c = Copy("$TARGET", "$SOURCE") +env.Command("cyclic_callback.proto", "cyclic.proto", c) +env.NanopbProto(["cyclic_callback", "cyclic_callback.options"]) + +enc_callback = env.Program(["encode_cyclic_callback.c", "cyclic_callback.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) + + diff --git a/tests/cyclic_messages/cyclic.proto b/tests/cyclic_messages/cyclic.proto new file mode 100644 index 00000000..a9d158cb --- /dev/null +++ b/tests/cyclic_messages/cyclic.proto @@ -0,0 +1,25 @@ +// Test structures with cyclic references. +// These can only be handled in pointer/callback mode, +// see associated .options files. + +message TreeNode +{ + optional int32 leaf = 1; + optional TreeNode left = 2; + optional TreeNode right = 3; +} + +message Dictionary +{ + repeated KeyValuePair dictItem = 1; +} + +message KeyValuePair +{ + required string key = 1; + optional string stringValue = 2; + optional int32 intValue = 3; + optional Dictionary dictValue = 4; + optional TreeNode treeValue = 5; +} + diff --git a/tests/cyclic_messages/cyclic_callback.options b/tests/cyclic_messages/cyclic_callback.options new file mode 100644 index 00000000..fd4e1e1c --- /dev/null +++ b/tests/cyclic_messages/cyclic_callback.options @@ -0,0 +1,7 @@ +TreeNode.left type:FT_CALLBACK +TreeNode.right type:FT_CALLBACK + +Dictionary.data type:FT_CALLBACK +KeyValuePair.key max_size:8 +KeyValuePair.stringValue max_size:8 +KeyValuePair.treeValue type:FT_CALLBACK diff --git a/tests/cyclic_messages/encode_cyclic_callback.c b/tests/cyclic_messages/encode_cyclic_callback.c new file mode 100644 index 00000000..7f67e70c --- /dev/null +++ b/tests/cyclic_messages/encode_cyclic_callback.c @@ -0,0 +1,148 @@ +/* This program parses an input string in a format a bit like JSON: + * {'foobar': 1234, 'xyz': 'abc', 'tree': [[[1, 2], 3], [4, 5]]} + * and encodes it as protobuf + * + * Note: The string parsing here is not in any way intended to be robust + * nor safe against buffer overflows. It is just for this test. + */ + +#include +#include +#include +#include +#include "cyclic_callback.pb.h" + +static char *find_end_of_item(char *p) +{ + int depth = 0; + do { + if (*p == '[' || *p == '{') depth++; + if (*p == ']' || *p == '}') depth--; + p++; + } while (depth > 0 || (*p != ',' && *p != '}')); + + if (*p == '}') + return p; /* End of parent dict */ + + p++; + while (*p == ' ') p++; + return p; +} + +/* Parse a tree in format [[1 2] 3] and encode it directly to protobuf */ +static bool encode_tree(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + TreeNode tree = TreeNode_init_zero; + char *p = (char*)*arg; + + if (*p == '[') + { + /* This is a tree branch */ + p++; + tree.left.funcs.encode = encode_tree; + tree.left.arg = p; + + p = find_end_of_item(p); + tree.right.funcs.encode = encode_tree; + tree.right.arg = p; + } + else + { + /* This is a leaf node */ + tree.has_leaf = true; + tree.leaf = atoi(p); + } + + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, TreeNode_fields, &tree); +} + +/* Parse a dictionary in format {'name': value} and encode it directly to protobuf */ +static bool encode_dictionary(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + int textlen; + char *p = (char*)*arg; + if (*p == '{') p++; + while (*p != '}') + { + KeyValuePair pair = KeyValuePair_init_zero; + + if (*p != '\'') + PB_RETURN_ERROR(stream, "invalid key, missing quote"); + + p++; /* Starting quote of key */ + textlen = strchr(p, '\'') - p; + strncpy(pair.key, p, textlen); + pair.key[textlen] = 0; + p += textlen + 2; + + while (*p == ' ') p++; + + if (*p == '[') + { + /* Value is a tree */ + pair.treeValue.funcs.encode = encode_tree; + pair.treeValue.arg = p; + } + else if (*p == '\'') + { + /* Value is a string */ + pair.has_stringValue = true; + p++; + textlen = strchr(p, '\'') - p; + strncpy(pair.stringValue, p, textlen); + pair.stringValue[textlen] = 0; + } + else if (*p == '{') + { + /* Value is a dictionary */ + pair.has_dictValue = true; + pair.dictValue.dictItem.funcs.encode = encode_dictionary; + pair.dictValue.dictItem.arg = p; + } + else + { + /* Value is integer */ + pair.has_intValue = true; + pair.intValue = atoi(p); + } + + p = find_end_of_item(p); + + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!pb_encode_submessage(stream, KeyValuePair_fields, &pair)) + return false; + } + + return true; +} + + +int main(int argc, char *argv[]) +{ + uint8_t buffer[256]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + Dictionary dict = Dictionary_init_zero; + + if (argc <= 1) + { + fprintf(stderr, "Usage: %s \"{'foobar': 1234, ...}\"\n", argv[0]); + return 1; + } + + dict.dictItem.funcs.encode = encode_dictionary; + dict.dictItem.arg = argv[1]; + + if (!pb_encode(&stream, Dictionary_fields, &dict)) + { + fprintf(stderr, "Encoding error: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; +} + + -- cgit 1.2.3-korg From d82a264c416f5ce64c611b48888eb84316e3445f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 4 Sep 2014 21:19:54 +0300 Subject: Update security model with regards to pointer fields --- docs/security.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/security.rst b/docs/security.rst index e865f839..2d0affc5 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -26,9 +26,9 @@ The following data is regarded as **trusted**. It must be under the control of the application writer. Malicious data in these structures could cause security issues, such as execution of arbitrary code: -1. Callback and extension fields in message structures given to pb_encode() - and pb_decode(). These fields are memory pointers, and are generated - depending on the .proto file. +1. Callback, pointer and extension fields in message structures given to + pb_encode() and pb_decode(). These fields are memory pointers, and are + generated depending on the message definition in the .proto file. 2. The automatically generated field definitions, i.e. *pb_field_t* lists. 3. Contents of the *pb_istream_t* and *pb_ostream_t* structures (this does not mean the contents of the stream itself, just the stream definition). @@ -38,7 +38,7 @@ these will cause "garbage in, garbage out" behaviour. It will not cause buffer overflows, information disclosure or other security problems: 1. All data read from *pb_istream_t*. -2. All fields in message structures, except callbacks and extensions. +2. All fields in message structures, except callbacks, pointers and extensions. (Beginning with nanopb-0.2.4, in earlier versions the field sizes are partially unchecked.) Invariants @@ -76,4 +76,6 @@ The following list is not comprehensive: stop a denial of service attack from using an infinite message. 4. If using network sockets as streams, a timeout should be set to stop denial of service attacks. - +5. If using *malloc()* support, some method of limiting memory use should be + employed. This can be done by defining custom *pb_realloc()* function. + Nanopb will properly detect and handle failed memory allocations. -- cgit 1.2.3-korg From d0299d87ec109c83480f857d2a1ce66d52834926 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 7 Sep 2014 19:25:09 +0300 Subject: Code coverage results were ignoring the data from encode/decode unittests. Update issue 126 Status: Started --- tests/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 3c4e0b04..cee6bf67 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -10,12 +10,12 @@ coverage: # LCOV does not like the newer gcov format scons CC=gcc-4.6 CXX=gcc-4.6 - # We are only interested in pb_encode.o and pb_decode.o - find build -name '*.gcda' -and \! \( -name '*pb_encode*' -or -name '*pb_decode*' \) -exec rm '{}' \; - # Collect the data mkdir build/coverage lcov --base-directory . --directory build/ --gcov-tool gcov-4.6 -c -o build/coverage/nanopb.info + # Remove the test code from results + lcov -r build/coverage/nanopb.info '*tests*' -o build/coverage/nanopb.info + # Generate HTML genhtml -o build/coverage build/coverage/nanopb.info -- cgit 1.2.3-korg From cdbf51db08d86ea024605078f21e8c61fe0e5fd3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 7 Sep 2014 19:49:00 +0300 Subject: Fix compilation error with generated initializers for repeated pointer fields --- generator/nanopb_generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 56dfdb08..9ffed03a 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -596,6 +596,8 @@ class Message: else: parts.append(field.get_initializer(null_init)) elif field.allocation == 'POINTER': + if field.rules == 'REPEATED': + parts.append('0') parts.append('NULL') elif field.allocation == 'CALLBACK': if field.pbtype == 'EXTENSION': -- cgit 1.2.3-korg From 38613acdb42731cc4bc80e93cfbc3cd126976c24 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 7 Sep 2014 19:49:26 +0300 Subject: Add a few missing unit tests --- tests/common/unittestproto.proto | 5 +++++ tests/decode_unittests/decode_unittests.c | 14 ++++++++++++++ tests/encode_unittests/encode_unittests.c | 17 +++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/tests/common/unittestproto.proto b/tests/common/unittestproto.proto index eb3e7dec..0ecb1f0b 100644 --- a/tests/common/unittestproto.proto +++ b/tests/common/unittestproto.proto @@ -34,3 +34,8 @@ message CallbackContainer { message CallbackContainerContainer { required CallbackContainer submsg = 1; } + +message StringPointerContainer { + repeated string rep_str = 1 [(nanopb).type = FT_POINTER]; +} + diff --git a/tests/decode_unittests/decode_unittests.c b/tests/decode_unittests/decode_unittests.c index 97212aff..8c12f1cd 100644 --- a/tests/decode_unittests/decode_unittests.c +++ b/tests/decode_unittests/decode_unittests.c @@ -87,6 +87,20 @@ int main() pb_decode_varint(&s, (uint64_t*)&i) && i == -1)); TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_decode_varint(&s, &u) && u == UINT64_MAX)); + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), + !pb_decode_varint(&s, &u))); + } + + { + pb_istream_t s; + uint32_t u; + + COMMENT("Test pb_decode_varint32"); + TEST((s = S("\x00"), pb_decode_varint32(&s, &u) && u == 0)); + TEST((s = S("\x01"), pb_decode_varint32(&s, &u) && u == 1)); + TEST((s = S("\xAC\x02"), pb_decode_varint32(&s, &u) && u == 300)); + TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint32(&s, &u) && u == UINT32_MAX)); + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\x01"), !pb_decode_varint32(&s, &u))); } { diff --git a/tests/encode_unittests/encode_unittests.c b/tests/encode_unittests/encode_unittests.c index 78fbb8b1..583af5c6 100644 --- a/tests/encode_unittests/encode_unittests.c +++ b/tests/encode_unittests/encode_unittests.c @@ -331,6 +331,23 @@ int main() TEST(s.bytes_written == StringMessage_size); } + { + uint8_t buffer[128]; + pb_ostream_t s; + StringPointerContainer msg = StringPointerContainer_init_zero; + char *strs[1] = {NULL}; + char zstr[] = "Z"; + + COMMENT("Test string pointer encoding."); + + msg.rep_str = strs; + msg.rep_str_count = 1; + TEST(WRITES(pb_encode(&s, StringPointerContainer_fields, &msg), "\x0a\x00")) + + strs[0] = zstr; + TEST(WRITES(pb_encode(&s, StringPointerContainer_fields, &msg), "\x0a\x01Z")) + } + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); -- cgit 1.2.3-korg From 8189d538dd80a89bc6fa1672336b99c9c5cd076b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 7 Sep 2014 20:31:36 +0300 Subject: Add test case for simulated io errors. Update issue 126 Status: FixedInGit --- tests/io_errors/SConscript | 15 ++++ tests/io_errors/alltypes.options | 3 + tests/io_errors/io_errors.c | 140 ++++++++++++++++++++++++++++++ tests/io_errors_pointers/SConscript | 42 +++++++++ tests/io_errors_pointers/alltypes.options | 3 + 5 files changed, 203 insertions(+) create mode 100644 tests/io_errors/SConscript create mode 100644 tests/io_errors/alltypes.options create mode 100644 tests/io_errors/io_errors.c create mode 100644 tests/io_errors_pointers/SConscript create mode 100644 tests/io_errors_pointers/alltypes.options diff --git a/tests/io_errors/SConscript b/tests/io_errors/SConscript new file mode 100644 index 00000000..60146cc0 --- /dev/null +++ b/tests/io_errors/SConscript @@ -0,0 +1,15 @@ +# Simulate io errors when encoding and decoding + +Import("env") + +c = Copy("$TARGET", "$SOURCE") +env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) + +env.NanopbProto(["alltypes", "alltypes.options"]) + +ioerr = env.Program(["io_errors.c", "alltypes.pb.c", + "$COMMON/pb_encode.o", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) + +env.RunTest("io_errors.output", [ioerr, "$BUILD/alltypes/encode_alltypes.output"]) + + diff --git a/tests/io_errors/alltypes.options b/tests/io_errors/alltypes.options new file mode 100644 index 00000000..b31e3cf0 --- /dev/null +++ b/tests/io_errors/alltypes.options @@ -0,0 +1,3 @@ +* max_size:16 +* max_count:5 + diff --git a/tests/io_errors/io_errors.c b/tests/io_errors/io_errors.c new file mode 100644 index 00000000..76f35b08 --- /dev/null +++ b/tests/io_errors/io_errors.c @@ -0,0 +1,140 @@ +/* Simulate IO errors after each byte in a stream. + * Verifies proper error propagation. + */ + +#include +#include +#include +#include "alltypes.pb.h" +#include "test_helpers.h" + +typedef struct +{ + uint8_t *buffer; + size_t fail_after; +} faulty_stream_t; + +bool read_callback(pb_istream_t *stream, uint8_t *buf, size_t count) +{ + faulty_stream_t *state = stream->state; + + while (count--) + { + if (state->fail_after == 0) + PB_RETURN_ERROR(stream, "simulated"); + state->fail_after--; + *buf++ = *state->buffer++; + } + + return true; +} +bool write_callback(pb_ostream_t *stream, const uint8_t *buf, size_t count) +{ + faulty_stream_t *state = stream->state; + + while (count--) + { + if (state->fail_after == 0) + PB_RETURN_ERROR(stream, "simulated"); + state->fail_after--; + *state->buffer++ = *buf++; + } + + return true; +} + +int main() +{ + uint8_t buffer[2048]; + size_t msglen; + AllTypes msg = AllTypes_init_zero; + + /* Get some base data to run the tests with */ + SET_BINARY_MODE(stdin); + msglen = fread(buffer, 1, sizeof(buffer), stdin); + + /* Test IO errors on decoding */ + { + bool status; + pb_istream_t stream = {&read_callback, NULL, SIZE_MAX}; + faulty_stream_t fs; + size_t i; + + for (i = 0; i < msglen; i++) + { + stream.bytes_left = msglen; + stream.state = &fs; + fs.buffer = buffer; + fs.fail_after = i; + + status = pb_decode(&stream, AllTypes_fields, &msg); + if (status != false) + { + fprintf(stderr, "Unexpected success in decode\n"); + return 2; + } + else if (strcmp(stream.errmsg, "simulated") != 0) + { + fprintf(stderr, "Wrong error in decode: %s\n", stream.errmsg); + return 3; + } + } + + stream.bytes_left = msglen; + stream.state = &fs; + fs.buffer = buffer; + fs.fail_after = msglen; + status = pb_decode(&stream, AllTypes_fields, &msg); + + if (!status) + { + fprintf(stderr, "Decoding failed: %s\n", stream.errmsg); + return 4; + } + } + + /* Test IO errors on encoding */ + { + bool status; + pb_ostream_t stream = {&write_callback, NULL, SIZE_MAX, 0}; + faulty_stream_t fs; + size_t i; + + for (i = 0; i < msglen; i++) + { + stream.max_size = msglen; + stream.bytes_written = 0; + stream.state = &fs; + fs.buffer = buffer; + fs.fail_after = i; + + status = pb_encode(&stream, AllTypes_fields, &msg); + if (status != false) + { + fprintf(stderr, "Unexpected success in encode\n"); + return 5; + } + else if (strcmp(stream.errmsg, "simulated") != 0) + { + fprintf(stderr, "Wrong error in encode: %s\n", stream.errmsg); + return 6; + } + } + + stream.max_size = msglen; + stream.bytes_written = 0; + stream.state = &fs; + fs.buffer = buffer; + fs.fail_after = msglen; + status = pb_encode(&stream, AllTypes_fields, &msg); + + if (!status) + { + fprintf(stderr, "Encoding failed: %s\n", stream.errmsg); + return 7; + } + } + + return 0; +} + diff --git a/tests/io_errors_pointers/SConscript b/tests/io_errors_pointers/SConscript new file mode 100644 index 00000000..0b96177f --- /dev/null +++ b/tests/io_errors_pointers/SConscript @@ -0,0 +1,42 @@ +# Simulate io errors when encoding and decoding + +Import("env") + +# We need our own pb_decode.o for the malloc support +env = env.Clone() +env.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1}); + +# Disable libmudflap, because it will confuse valgrind +# and other memory leak detection tools. +if '-fmudflap' in env["CCFLAGS"]: + env["CCFLAGS"].remove("-fmudflap") + env["LINKFLAGS"].remove("-fmudflap") + env["LIBS"].remove("mudflap") + +strict = env.Clone() +strict.Append(CFLAGS = strict['CORECFLAGS']) +strict.Object("pb_decode_with_malloc.o", "$NANOPB/pb_decode.c") +strict.Object("pb_encode_with_malloc.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common_with_malloc.o", "$NANOPB/pb_common.c") + +c = Copy("$TARGET", "$SOURCE") +env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) +env.Command("io_errors.c", "#io_errors/io_errors.c", c) + +env.NanopbProto(["alltypes", "alltypes.options"]) + +ioerr = env.Program(["io_errors.c", "alltypes.pb.c", + "pb_encode_with_malloc.o", + "pb_decode_with_malloc.o", + "pb_common_with_malloc.o"]) + +# Run tests under valgrind if available +valgrind = env.WhereIs('valgrind') +kwargs = {} +if valgrind: + kwargs['COMMAND'] = valgrind + kwargs['ARGS'] = ["-q", ioerr[0].abspath] + +env.RunTest("io_errors.output", [ioerr, "$BUILD/alltypes/encode_alltypes.output"], **kwargs) + + diff --git a/tests/io_errors_pointers/alltypes.options b/tests/io_errors_pointers/alltypes.options new file mode 100644 index 00000000..52abeb7f --- /dev/null +++ b/tests/io_errors_pointers/alltypes.options @@ -0,0 +1,3 @@ +# Generate all fields as pointers. +* type:FT_POINTER + -- cgit 1.2.3-korg From 0dce9ef635f8af1b9aa07a43f610295bca8954da Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 6 Sep 2014 19:01:11 +0300 Subject: Add a better fuzz test. Attempts to verify all the properties defined in the security model, while also being portable and able to run on many platforms. --- tests/SConstruct | 3 +- tests/fuzztest/SConscript | 50 ++++ tests/fuzztest/alltypes_pointer.options | 3 + tests/fuzztest/alltypes_static.options | 3 + tests/fuzztest/fuzz_syshdr.h | 15 ++ tests/fuzztest/fuzztest.c | 431 ++++++++++++++++++++++++++++++++ tests/fuzztest/malloc_wrappers.c | 54 ++++ tests/fuzztest/malloc_wrappers.h | 7 + 8 files changed, 565 insertions(+), 1 deletion(-) create mode 100644 tests/fuzztest/SConscript create mode 100644 tests/fuzztest/alltypes_pointer.options create mode 100644 tests/fuzztest/alltypes_static.options create mode 100644 tests/fuzztest/fuzz_syshdr.h create mode 100644 tests/fuzztest/fuzztest.c create mode 100644 tests/fuzztest/malloc_wrappers.c create mode 100644 tests/fuzztest/malloc_wrappers.h diff --git a/tests/SConstruct b/tests/SConstruct index 57167ccc..1890670b 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -57,6 +57,7 @@ if not env.GetOption('clean'): if not stdbool or not stdint or not stddef or not string: conf.env.Append(CPPDEFINES = {'PB_SYSTEM_HEADER': '\\"pb_syshdr.h\\"'}) conf.env.Append(CPPPATH = "#../extra") + conf.env.Append(SYSHDR = '\\"pb_syshdr.h\\"') if stdbool: conf.env.Append(CPPDEFINES = {'HAVE_STDBOOL_H': 1}) if stdint: conf.env.Append(CPPDEFINES = {'HAVE_STDINT_H': 1}) @@ -101,7 +102,7 @@ if 'gcc' in env['CC']: # GNU Compiler Collection # Debug info, warnings as errors - env.Append(CFLAGS = '-ansi -pedantic -g -Wall -Werror -fprofile-arcs -ftest-coverage -fstack-protector-all') + env.Append(CFLAGS = '-ansi -pedantic -g -Wall -Werror -fprofile-arcs -ftest-coverage ') env.Append(CORECFLAGS = '-Wextra') env.Append(LINKFLAGS = '-g --coverage') diff --git a/tests/fuzztest/SConscript b/tests/fuzztest/SConscript new file mode 100644 index 00000000..36b62c56 --- /dev/null +++ b/tests/fuzztest/SConscript @@ -0,0 +1,50 @@ +# Run a fuzz test to verify robustness against corrupted/malicious data. + +Import("env") + +# We need our own pb_decode.o for the malloc support +env = env.Clone() +env.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1, + 'PB_SYSTEM_HEADER': '\\"fuzz_syshdr.h\\"'}) +env.Append(CPPPATH = ".") + +if 'SYSHDR' in env: + env.Append(CPPDEFINES = {'PB_OLD_SYSHDR': env['SYSHDR']}) + +# Disable libmudflap, because it will confuse valgrind +# and other memory leak detection tools. +if '-fmudflap' in env["CCFLAGS"]: + env["CCFLAGS"].remove("-fmudflap") + env["LINKFLAGS"].remove("-fmudflap") + env["LIBS"].remove("mudflap") + +strict = env.Clone() +strict.Append(CFLAGS = strict['CORECFLAGS']) +strict.Object("pb_decode_with_malloc.o", "$NANOPB/pb_decode.c") +strict.Object("pb_encode_with_malloc.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common_with_malloc.o", "$NANOPB/pb_common.c") + +# We want both pointer and static versions of the AllTypes message +env.Command("alltypes_static.proto", "#alltypes/alltypes.proto", + lambda target, source, env: + open(str(target[0]), 'w').write("package alltypes_static;\n" + + open(str(source[0])).read())) +env.Command("alltypes_pointer.proto", "#alltypes/alltypes.proto", + lambda target, source, env: + open(str(target[0]), 'w').write("package alltypes_pointer;\n" + + open(str(source[0])).read())) + +p1 = env.NanopbProto(["alltypes_pointer", "alltypes_pointer.options"]) +p2 = env.NanopbProto(["alltypes_static", "alltypes_static.options"]) +fuzz = env.Program(["fuzztest.c", + "alltypes_pointer.pb.c", + "alltypes_static.pb.c", + "pb_encode_with_malloc.o", + "pb_decode_with_malloc.o", + "pb_common_with_malloc.o", + "malloc_wrappers.c"]) +Depends([p1, p2, fuzz], ["fuzz_syshdr.h", "malloc_wrappers.h"]) + +env.RunTest(fuzz) + + diff --git a/tests/fuzztest/alltypes_pointer.options b/tests/fuzztest/alltypes_pointer.options new file mode 100644 index 00000000..52abeb7f --- /dev/null +++ b/tests/fuzztest/alltypes_pointer.options @@ -0,0 +1,3 @@ +# Generate all fields as pointers. +* type:FT_POINTER + diff --git a/tests/fuzztest/alltypes_static.options b/tests/fuzztest/alltypes_static.options new file mode 100644 index 00000000..1c10637c --- /dev/null +++ b/tests/fuzztest/alltypes_static.options @@ -0,0 +1,3 @@ +* max_size:32 +* max_count:8 +*.extensions type:FT_IGNORE diff --git a/tests/fuzztest/fuzz_syshdr.h b/tests/fuzztest/fuzz_syshdr.h new file mode 100644 index 00000000..d295d9ed --- /dev/null +++ b/tests/fuzztest/fuzz_syshdr.h @@ -0,0 +1,15 @@ +/* This is just a wrapper in order to get our own malloc wrappers into nanopb core. */ + +#define pb_realloc(ptr,size) counting_realloc(ptr,size) +#define pb_free(ptr) counting_free(ptr) + +#ifdef PB_OLD_SYSHDR +#include PB_OLD_SYSHDR +#else +#include +#include +#include +#include +#endif + +#include diff --git a/tests/fuzztest/fuzztest.c b/tests/fuzztest/fuzztest.c new file mode 100644 index 00000000..996ed450 --- /dev/null +++ b/tests/fuzztest/fuzztest.c @@ -0,0 +1,431 @@ +/* Fuzz testing for the nanopb core. + * Attempts to verify all the properties defined in the security model document. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "malloc_wrappers.h" +#include "alltypes_static.pb.h" +#include "alltypes_pointer.pb.h" + +static uint64_t random_seed; + +/* Uses xorshift64 here instead of rand() for both speed and + * reproducibility across platforms. */ +static uint32_t rand_word() +{ + random_seed ^= random_seed >> 12; + random_seed ^= random_seed << 25; + random_seed ^= random_seed >> 27; + return random_seed * 2685821657736338717ULL; +} + +/* Get a random integer in range, with approximately flat distribution. */ +static int rand_int(int min, int max) +{ + return rand_word() % (max + 1 - min) + min; +} + +static bool rand_bool() +{ + return rand_word() & 1; +} + +/* Get a random byte, with skewed distribution. + * Important corner cases like 0xFF, 0x00 and 0xFE occur more + * often than other values. */ +static uint8_t rand_byte() +{ + uint32_t w = rand_word(); + uint8_t b = w & 0xFF; + if (w & 0x100000) + b >>= (w >> 8) & 7; + if (w & 0x200000) + b <<= (w >> 12) & 7; + if (w & 0x400000) + b ^= 0xFF; + return b; +} + +/* Get a random length, with skewed distribution. + * Favors the shorter lengths, but always atleast 1. */ +static size_t rand_len(size_t max) +{ + uint32_t w = rand_word(); + size_t s; + if (w & 0x800000) + w &= 3; + else if (w & 0x400000) + w &= 15; + else if (w & 0x200000) + w &= 255; + + s = (w % max); + if (s == 0) + s = 1; + + return s; +} + +/* Fills a buffer with random data with skewed distribution. */ +static void rand_fill(uint8_t *buf, size_t count) +{ + while (count--) + *buf++ = rand_byte(); +} + +/* Fill with random protobuf-like data */ +static size_t rand_fill_protobuf(uint8_t *buf, size_t min_bytes, size_t max_bytes, int min_tag) +{ + pb_ostream_t stream = pb_ostream_from_buffer(buf, max_bytes); + + while(stream.bytes_written < min_bytes) + { + pb_wire_type_t wt = rand_int(0, 3); + if (wt == 3) wt = 5; /* Gap in values */ + + if (!pb_encode_tag(&stream, wt, rand_int(min_tag, min_tag + 512))) + break; + + if (wt == PB_WT_VARINT) + { + uint64_t value; + rand_fill((uint8_t*)&value, sizeof(value)); + pb_encode_varint(&stream, value); + } + else if (wt == PB_WT_64BIT) + { + uint64_t value; + rand_fill((uint8_t*)&value, sizeof(value)); + pb_encode_fixed64(&stream, &value); + } + else if (wt == PB_WT_32BIT) + { + uint32_t value; + rand_fill((uint8_t*)&value, sizeof(value)); + pb_encode_fixed32(&stream, &value); + } + else if (wt == PB_WT_STRING) + { + size_t len; + uint8_t *buf; + + if (min_bytes > stream.bytes_written) + len = rand_len(min_bytes - stream.bytes_written); + else + len = 0; + + buf = malloc(len); + pb_encode_varint(&stream, len); + rand_fill(buf, len); + pb_write(&stream, buf, len); + free(buf); + } + } + + return stream.bytes_written; +} + +/* Given a buffer of data, mess it up a bit */ +static void rand_mess(uint8_t *buf, size_t count) +{ + int m = rand_int(0, 3); + + if (m == 0) + { + /* Replace random substring */ + int s = rand_int(0, count - 1); + int l = rand_len(count - s); + rand_fill(buf + s, l); + } + else if (m == 1) + { + /* Swap random bytes */ + int a = rand_int(0, count - 1); + int b = rand_int(0, count - 1); + int x = buf[a]; + buf[a] = buf[b]; + buf[b] = x; + } + else if (m == 2) + { + /* Duplicate substring */ + int s = rand_int(0, count - 2); + int l = rand_len((count - s) / 2); + memcpy(buf + s + l, buf + s, l); + } + else if (m == 3) + { + /* Add random protobuf noise */ + int s = rand_int(0, count - 1); + int l = rand_len(count - s); + rand_fill_protobuf(buf + s, l, count - s, 1); + } +} + +/* Some default data to put in the message */ +static const alltypes_static_AllTypes initval = alltypes_static_AllTypes_init_default; + +#define BUFSIZE 4096 + +static bool do_static_encode(uint8_t *buffer, size_t *msglen) +{ + pb_ostream_t stream; + bool status; + + /* Allocate a message and fill it with defaults */ + alltypes_static_AllTypes *msg = malloc_with_check(sizeof(alltypes_static_AllTypes)); + memcpy(msg, &initval, sizeof(initval)); + + /* Apply randomness to the data before encoding */ + while (rand_int(0, 7)) + rand_mess((uint8_t*)msg, sizeof(alltypes_static_AllTypes)); + + stream = pb_ostream_from_buffer(buffer, BUFSIZE); + status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg); + assert(stream.bytes_written <= BUFSIZE); + assert(stream.bytes_written <= alltypes_static_AllTypes_size); + + *msglen = stream.bytes_written; + free_with_check(msg); + + return status; +} + +/* Append or prepend protobuf noise */ +static void do_protobuf_noise(uint8_t *buffer, size_t *msglen) +{ + int m = rand_int(0, 2); + size_t max_size = BUFSIZE - 32 - *msglen; + if (m == 1) + { + /* Prepend */ + uint8_t *tmp = malloc_with_check(BUFSIZE); + size_t s = rand_fill_protobuf(tmp, rand_len(max_size), BUFSIZE - *msglen, 512); + memmove(buffer + s, buffer, *msglen); + memcpy(buffer, tmp, s); + free_with_check(tmp); + *msglen += s; + } + else if (m == 2) + { + /* Append */ + size_t s = rand_fill_protobuf(buffer + *msglen, rand_len(max_size), BUFSIZE - *msglen, 512); + *msglen += s; + } +} + +static bool do_static_decode(uint8_t *buffer, size_t msglen, bool assert_success) +{ + pb_istream_t stream; + bool status; + + alltypes_static_AllTypes *msg = malloc_with_check(sizeof(alltypes_static_AllTypes)); + rand_fill((uint8_t*)msg, sizeof(alltypes_static_AllTypes)); + stream = pb_istream_from_buffer(buffer, msglen); + status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg); + + if (!status && assert_success) + { + /* Anything that was successfully encoded, should be decodeable. + * One exception: strings without null terminator are encoded up + * to end of buffer, but refused on decode because the terminator + * would not fit. */ + if (strcmp(stream.errmsg, "string overflow") != 0) + assert(status); + } + + free_with_check(msg); + return status; +} + +static bool do_pointer_decode(uint8_t *buffer, size_t msglen, bool assert_success) +{ + pb_istream_t stream; + bool status; + alltypes_pointer_AllTypes *msg; + + msg = malloc_with_check(sizeof(alltypes_pointer_AllTypes)); + memset(msg, 0, sizeof(alltypes_pointer_AllTypes)); + stream = pb_istream_from_buffer(buffer, msglen); + + assert(get_alloc_count() == 0); + status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg); + + if (assert_success) + assert(status); + + pb_release(alltypes_pointer_AllTypes_fields, msg); + assert(get_alloc_count() == 0); + + free_with_check(msg); + + return status; +} + +/* Do a decode -> encode -> decode -> encode roundtrip */ +static void do_static_roundtrip(uint8_t *buffer, size_t msglen) +{ + bool status; + uint8_t *buf2 = malloc_with_check(BUFSIZE); + uint8_t *buf3 = malloc_with_check(BUFSIZE); + size_t msglen2, msglen3; + alltypes_static_AllTypes *msg1 = malloc_with_check(sizeof(alltypes_static_AllTypes)); + alltypes_static_AllTypes *msg2 = malloc_with_check(sizeof(alltypes_static_AllTypes)); + memset(msg1, 0, sizeof(alltypes_static_AllTypes)); + memset(msg2, 0, sizeof(alltypes_static_AllTypes)); + + { + pb_istream_t stream = pb_istream_from_buffer(buffer, msglen); + status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg1); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf2, BUFSIZE); + status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg1); + assert(status); + msglen2 = stream.bytes_written; + } + + { + pb_istream_t stream = pb_istream_from_buffer(buf2, msglen2); + status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg2); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf3, BUFSIZE); + status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg2); + assert(status); + msglen3 = stream.bytes_written; + } + + assert(msglen2 == msglen3); + assert(memcmp(buf2, buf3, msglen2) == 0); + + free_with_check(msg1); + free_with_check(msg2); + free_with_check(buf2); + free_with_check(buf3); +} + +/* Do decode -> encode -> decode -> encode roundtrip */ +static void do_pointer_roundtrip(uint8_t *buffer, size_t msglen) +{ + bool status; + uint8_t *buf2 = malloc_with_check(BUFSIZE); + uint8_t *buf3 = malloc_with_check(BUFSIZE); + size_t msglen2, msglen3; + alltypes_pointer_AllTypes *msg1 = malloc_with_check(sizeof(alltypes_pointer_AllTypes)); + alltypes_pointer_AllTypes *msg2 = malloc_with_check(sizeof(alltypes_pointer_AllTypes)); + memset(msg1, 0, sizeof(alltypes_pointer_AllTypes)); + memset(msg2, 0, sizeof(alltypes_pointer_AllTypes)); + + { + pb_istream_t stream = pb_istream_from_buffer(buffer, msglen); + status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg1); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf2, BUFSIZE); + status = pb_encode(&stream, alltypes_pointer_AllTypes_fields, msg1); + assert(status); + msglen2 = stream.bytes_written; + } + + { + pb_istream_t stream = pb_istream_from_buffer(buf2, msglen2); + status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg2); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf3, BUFSIZE); + status = pb_encode(&stream, alltypes_pointer_AllTypes_fields, msg2); + assert(status); + msglen3 = stream.bytes_written; + } + + assert(msglen2 == msglen3); + assert(memcmp(buf2, buf3, msglen2) == 0); + + pb_release(alltypes_pointer_AllTypes_fields, msg1); + pb_release(alltypes_pointer_AllTypes_fields, msg2); + free_with_check(msg1); + free_with_check(msg2); + free_with_check(buf2); + free_with_check(buf3); +} + +static void run_iteration() +{ + uint8_t *buffer = malloc_with_check(BUFSIZE); + size_t msglen; + bool status; + + rand_fill(buffer, BUFSIZE); + + if (do_static_encode(buffer, &msglen)) + { + do_protobuf_noise(buffer, &msglen); + + status = do_static_decode(buffer, msglen, true); + + if (status) + do_static_roundtrip(buffer, msglen); + + status = do_pointer_decode(buffer, msglen, true); + + if (status) + do_pointer_roundtrip(buffer, msglen); + + /* Apply randomness to the encoded data */ + while (rand_bool()) + rand_mess(buffer, BUFSIZE); + + /* Apply randomness to encoded data length */ + if (rand_bool()) + msglen = rand_int(0, BUFSIZE); + + status = do_static_decode(buffer, msglen, false); + do_pointer_decode(buffer, msglen, status); + + if (status) + { + do_static_roundtrip(buffer, msglen); + do_pointer_roundtrip(buffer, msglen); + } + } + + free_with_check(buffer); +} + +int main(int argc, char **argv) +{ + int i; + if (argc > 1) + { + random_seed = atol(argv[1]); + } + else + { + random_seed = time(NULL); + } + + fprintf(stderr, "Random seed: %llu\n", (long long unsigned)random_seed); + + for (i = 0; i < 10000; i++) + { + run_iteration(); + } + + return 0; +} + diff --git a/tests/fuzztest/malloc_wrappers.c b/tests/fuzztest/malloc_wrappers.c new file mode 100644 index 00000000..ad69f1ce --- /dev/null +++ b/tests/fuzztest/malloc_wrappers.c @@ -0,0 +1,54 @@ +#include "malloc_wrappers.h" +#include +#include +#include + +static size_t alloc_count = 0; + +/* Allocate memory and place check values before and after. */ +void* malloc_with_check(size_t size) +{ + size_t size32 = (size + 3) / 4 + 3; + uint32_t *buf = malloc(size32 * sizeof(uint32_t)); + buf[0] = size32; + buf[1] = 0xDEADBEEF; + buf[size32 - 1] = 0xBADBAD; + return buf + 2; +} + +/* Free memory allocated with malloc_with_check() and do the checks. */ +void free_with_check(void *mem) +{ + uint32_t *buf = (uint32_t*)mem - 2; + assert(buf[1] == 0xDEADBEEF); + assert(buf[buf[0] - 1] == 0xBADBAD); + free(buf); +} + +/* Track memory usage */ +void* counting_realloc(void *ptr, size_t size) +{ + /* Don't allocate crazy amounts of RAM when fuzzing */ + if (size > 1000000) + return NULL; + + if (!ptr && size) + alloc_count++; + + return realloc(ptr, size); +} + +void counting_free(void *ptr) +{ + if (ptr) + { + assert(alloc_count > 0); + alloc_count--; + free(ptr); + } +} + +size_t get_alloc_count() +{ + return alloc_count; +} diff --git a/tests/fuzztest/malloc_wrappers.h b/tests/fuzztest/malloc_wrappers.h new file mode 100644 index 00000000..7eec7952 --- /dev/null +++ b/tests/fuzztest/malloc_wrappers.h @@ -0,0 +1,7 @@ +#include + +void* malloc_with_check(size_t size); +void free_with_check(void *mem); +void* counting_realloc(void *ptr, size_t size); +void counting_free(void *ptr); +size_t get_alloc_count(); -- cgit 1.2.3-korg From 13a07e35b6b5f813078bde6a1a17d05f017bf714 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 6 Sep 2014 18:21:58 +0300 Subject: Fix crash in pb_release() if called twice on same message. There was a double-free bug in pb_release() because it didn't set size fields to zero after deallocation. Most commonly this happens if pb_decode() fails, internally calls pb_release() and then application code also calls pb_release(). --- pb_decode.c | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 5d211023..ecd46dc1 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -895,22 +895,27 @@ void pb_release(const pb_field_t fields[], void *dest_struct) pb_free(*pItem); *pItem++ = NULL; } + *(pb_size_t*)iter.pSize = 0; } else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) { /* Release fields in submessages */ void *pItem = *(void**)iter.pData; - pb_size_t count = (pItem ? 1 : 0); - - if (PB_HTYPE(type) == PB_HTYPE_REPEATED) - { - count = *(pb_size_t*)iter.pSize; - } - - while (count--) + if (pItem) { - pb_release((const pb_field_t*)iter.pos->ptr, pItem); - pItem = (uint8_t*)pItem + iter.pos->data_size; + pb_size_t count = 1; + + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + { + count = *(pb_size_t*)iter.pSize; + *(pb_size_t*)iter.pSize = 0; + } + + while (count--) + { + pb_release((const pb_field_t*)iter.pos->ptr, pItem); + pItem = (uint8_t*)pItem + iter.pos->data_size; + } } } -- cgit 1.2.3-korg From 5e3edb54159d98b90b4926d870d01648f54fc1d6 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 6 Sep 2014 18:56:34 +0300 Subject: Fix memory leak with duplicated fields and PB_ENABLE_MALLOC. If a required or optional field appeared twice in the message data, pb_decode will overwrite the old data with new one. That is fine, but with submessage fields, it didn't release the allocated subfields before overwriting. This bug can manifest if all of the following conditions are true: 1. There is a message with a "optional" or "required" submessage field that has type:FT_POINTER. 2. The submessage contains atleast one field with type:FT_POINTER. 3. The message data to be decoded has the submessage field twice in it. --- pb_decode.c | 104 +++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 60 insertions(+), 44 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index ecd46dc1..1fbc65c5 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -44,6 +44,10 @@ static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t static bool checkreturn pb_skip_varint(pb_istream_t *stream); static bool checkreturn pb_skip_string(pb_istream_t *stream); +#ifdef PB_ENABLE_MALLOC +static void pb_release_single_field(const pb_field_iter_t *iter); +#endif + /* --- Function pointers to field decoders --- * Order in the array must match pb_action_t LTYPE numbering. */ @@ -458,6 +462,13 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ { case PB_HTYPE_REQUIRED: case PB_HTYPE_OPTIONAL: + if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE && + *(void**)iter->pData != NULL) + { + /* Duplicate field, have to release the old allocation first. */ + pb_release_single_field(iter); + } + if (PB_LTYPE(type) == PB_LTYPE_STRING || PB_LTYPE(type) == PB_LTYPE_BYTES) { @@ -869,60 +880,65 @@ bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void * } #ifdef PB_ENABLE_MALLOC -void pb_release(const pb_field_t fields[], void *dest_struct) +static void pb_release_single_field(const pb_field_iter_t *iter) { - pb_field_iter_t iter; - - if (!pb_field_iter_begin(&iter, fields, dest_struct)) - return; /* Empty message type */ - - do + pb_type_t type; + type = iter->pos->type; + + if (PB_ATYPE(type) == PB_ATYPE_POINTER) { - pb_type_t type; - type = iter.pos->type; - - if (PB_ATYPE(type) == PB_ATYPE_POINTER) + if (PB_HTYPE(type) == PB_HTYPE_REPEATED && + (PB_LTYPE(type) == PB_LTYPE_STRING || + PB_LTYPE(type) == PB_LTYPE_BYTES)) { - if (PB_HTYPE(type) == PB_HTYPE_REPEATED && - (PB_LTYPE(type) == PB_LTYPE_STRING || - PB_LTYPE(type) == PB_LTYPE_BYTES)) + /* Release entries in repeated string or bytes array */ + void **pItem = *(void***)iter->pData; + pb_size_t count = *(pb_size_t*)iter->pSize; + while (count--) { - /* Release entries in repeated string or bytes array */ - void **pItem = *(void***)iter.pData; - pb_size_t count = *(pb_size_t*)iter.pSize; - while (count--) - { - pb_free(*pItem); - *pItem++ = NULL; - } - *(pb_size_t*)iter.pSize = 0; + pb_free(*pItem); + *pItem++ = NULL; } - else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) + *(pb_size_t*)iter->pSize = 0; + } + else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) + { + /* Release fields in submessages */ + void *pItem = *(void**)iter->pData; + if (pItem) { - /* Release fields in submessages */ - void *pItem = *(void**)iter.pData; - if (pItem) + pb_size_t count = 1; + + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) { - pb_size_t count = 1; - - if (PB_HTYPE(type) == PB_HTYPE_REPEATED) - { - count = *(pb_size_t*)iter.pSize; - *(pb_size_t*)iter.pSize = 0; - } - - while (count--) - { - pb_release((const pb_field_t*)iter.pos->ptr, pItem); - pItem = (uint8_t*)pItem + iter.pos->data_size; - } + count = *(pb_size_t*)iter->pSize; + *(pb_size_t*)iter->pSize = 0; + } + + while (count--) + { + pb_release((const pb_field_t*)iter->pos->ptr, pItem); + pItem = (uint8_t*)pItem + iter->pos->data_size; } } - - /* Release main item */ - pb_free(*(void**)iter.pData); - *(void**)iter.pData = NULL; } + + /* Release main item */ + pb_free(*(void**)iter->pData); + *(void**)iter->pData = NULL; + } +} + +void pb_release(const pb_field_t fields[], void *dest_struct) +{ + pb_field_iter_t iter; + + if (!pb_field_iter_begin(&iter, fields, dest_struct)) + return; /* Empty message type */ + + do + { + pb_release_single_field(&iter); } while (pb_field_iter_next(&iter)); } #endif -- cgit 1.2.3-korg From d0466bdf439a190f4a17c502d8aeb40ef823b53e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 8 Sep 2014 17:33:05 +0300 Subject: Add just-to-be-sure check to allocate_field(). This check will help to detect bugs earlier, and is quite lightweight compared to malloc() anyway. --- pb_decode.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 1fbc65c5..37f30707 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -45,6 +45,7 @@ static bool checkreturn pb_skip_varint(pb_istream_t *stream); static bool checkreturn pb_skip_string(pb_istream_t *stream); #ifdef PB_ENABLE_MALLOC +static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size); static void pb_release_single_field(const pb_field_iter_t *iter); #endif @@ -404,18 +405,23 @@ static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t { void *ptr = *(void**)pData; + if (data_size == 0 || array_size == 0) + PB_RETURN_ERROR(stream, "invalid size"); + /* Check for multiplication overflows. * This code avoids the costly division if the sizes are small enough. * Multiplication is safe as long as only half of bits are set * in either multiplicand. */ - const size_t check_limit = (size_t)1 << (sizeof(size_t) * 4); - if (data_size >= check_limit || array_size >= check_limit) { - const size_t size_max = (size_t)-1; - if (size_max / array_size < data_size) + const size_t check_limit = (size_t)1 << (sizeof(size_t) * 4); + if (data_size >= check_limit || array_size >= check_limit) { - PB_RETURN_ERROR(stream, "size too large"); + const size_t size_max = (size_t)-1; + if (size_max / array_size < data_size) + { + PB_RETURN_ERROR(stream, "size too large"); + } } } -- cgit 1.2.3-korg From d2099cc8f1adb33d427a44a5e32ed27b647c7168 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 8 Sep 2014 17:34:16 +0300 Subject: Protect against size_t overflows in pb_dec_bytes/pb_dec_string. Possible consequences of bug: 1) Denial of service by causing a crash Possible when all of the following apply: - Untrusted data is passed to pb_decode() - The top-level message contains a static string field as the first field. Causes a single write of '0' byte to 1 byte before the message struct. 2) Remote code execution Possible when all of the following apply: - 64-bit platform - The message or a submessage contains a static/pointer string field. - Decoding directly from a custom pb_istream_t - bytes_left on the stream is set to larger than 4 GB Causes a write of up to 4 GB of data past the string field. 3) Possible heap corruption or remote code execution Possible when all of the following apply: - less than 64-bit platform - The message or a submessage contains a pointer-type bytes field. Causes a write of sizeof(pb_size_t) bytes of data past a 0-byte long malloc()ed buffer. On many malloc() implementations, this causes at most a crash. However, remote code execution through a controlled jump cannot be ruled out. -- Detailed analysis follows In the following consideration, I define "platform bitness" as equal to number of bits in size_t datatype. Therefore most 8-bit platforms are regarded as 16-bit for the purposes of this discussion. 1. The overflow in pb_dec_string The overflow happens in this computation: uint32_t size; size_t alloc_size; alloc_size = size + 1; There are two ways in which the overflow can occur: In the uint32_t addition, or in the cast to size_t. This depends on the platform bitness. On 32- and 64-bit platforms, the size has to be UINT32_MAX for the overflow to occur. In that case alloc_size will be 0. On 16-bit platforms, overflow will happen whenever size is more than UINT16_MAX, and resulting alloc_size is attacker controlled. For static fields, the alloc_size value is just checked against the field data size. For pointer fields, the alloc_size value is passed to malloc(). End result in both cases is the same, the storage is 0 or just a few bytes in length. On 16-bit platforms, another overflow occurs in the call to pb_read(), when passing the original size. An attacker will want the passed value to be larger than the alloc_size, therefore the only reasonable choice is to have size = UINT16_MAX and alloc_size = 0. Any larger multiple will truncate to the same values. At this point we have read atleast the tag and the string length of the message, i.e. atleast 3 bytes. The maximum initial value for stream bytes_left is SIZE_MAX, thus at this point at most SIZE_MAX-3 bytes are remaining. On 32-bit and 16-bit platforms this means that the size passed to pb_read() is always larger than the number of remaining bytes. This causes pb_read() to fail immediately, before reading any bytes. On 64-bit platforms, it is possible for the bytes_left value to be set to a value larger than UINT32_MAX, which is the wraparound point in size calculation. In this case pb_read() will succeed and write up to 4 GB of attacker controlled data over the RAM that comes after the string field. On all platforms, there is an unconditional write of a terminating null byte. Because the size of size_t typically reflects the size of the processor address space, a write at UINT16_MAX or UINT32_MAX bytes after the string field actually wraps back to before the string field. Consequently, on 32-bit and 16-bit platforms, the bug causes a single write of '0' byte at one byte before the string field. If the string field is in the middle of a message, this will just corrupt other data in the message struct. Because the message contents is attacker controlled anyway, this is a non-issue. However, if the string field is the first field in the top-level message, it can corrupt other data on the stack/heap before it. Typically a single '0' write at a location not controlled by attacker is enough only for a denial-of-service attack. When using pointer fields and malloc(), the attacker controlled alloc_size will cause a 0-size allocation to happen. By the same logic as before, on 32-bit and 16-bit platforms this causes a '0' byte write only. On 64-bit platforms, however, it will again allow up to 4 GB of malicious data to be written over memory, if the stream length allows the read. 2. The overflow in pb_dec_bytes This overflow happens in the PB_BYTES_ARRAY_T_ALLOCSIZE macro: The computation is done in size_t data type this time. This means that an overflow is possible only when n is larger than SIZE_MAX - offsetof(..). The offsetof value in this case is equal to sizeof(pb_size_t) bytes. Because the incoming size value is limited to 32 bits, no overflow can happen here on 64-bit platforms. The size will be passed to pb_read(). Like before, on 32-bit and 16-bit platforms the read will always fail before writing anything. This leaves only the write of bdest->size as exploitable. On statically allocated fields, the size field will always be allocated, regardless of alloc_size. In this case, no buffer overflow is possible here, but user code could possibly use the attacker controlled size value and read past a buffer. If the field is allocated through malloc(), this will allow a write of sizeof(pb_size_t) attacker controlled bytes to past a 0-byte long buffer. In typical malloc implementations, this will either fit in unused alignment padding area, or cause a heap corruption and a crash. Under very exceptional situation it could allow attacker to influence the behaviour of malloc(), possibly jumping into an attacker-controlled location and thus leading to remote code execution. --- pb_decode.c | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 37f30707..d1efd1b5 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -1072,32 +1072,35 @@ static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *f static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint32_t size; + size_t alloc_size; pb_bytes_array_t *bdest; if (!pb_decode_varint32(stream, &size)) return false; + if (size > PB_SIZE_MAX) + PB_RETURN_ERROR(stream, "bytes overflow"); + + alloc_size = PB_BYTES_ARRAY_T_ALLOCSIZE(size); + if (size > alloc_size) + PB_RETURN_ERROR(stream, "size too large"); + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { #ifndef PB_ENABLE_MALLOC PB_RETURN_ERROR(stream, "no malloc support"); #else - if (!allocate_field(stream, dest, PB_BYTES_ARRAY_T_ALLOCSIZE(size), 1)) + if (!allocate_field(stream, dest, alloc_size, 1)) return false; bdest = *(pb_bytes_array_t**)dest; #endif } else { - if (PB_BYTES_ARRAY_T_ALLOCSIZE(size) > field->data_size) + if (alloc_size > field->data_size) PB_RETURN_ERROR(stream, "bytes overflow"); bdest = (pb_bytes_array_t*)dest; } - - if (size > PB_SIZE_MAX) - { - PB_RETURN_ERROR(stream, "bytes overflow"); - } bdest->size = (pb_size_t)size; return pb_read(stream, bdest->bytes, size); @@ -1114,6 +1117,9 @@ static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *fi /* Space for null terminator */ alloc_size = size + 1; + if (alloc_size < size) + PB_RETURN_ERROR(stream, "size too large"); + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { #ifndef PB_ENABLE_MALLOC -- cgit 1.2.3-korg From 07e9ffb97b747b22250f66a2210898db57c266c1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 11 Sep 2014 17:58:53 +0300 Subject: Add a fuzz testing stub for ability to use external generators also --- tests/fuzztest/SConscript | 8 ++ tests/fuzztest/fuzzstub.c | 189 ++++++++++++++++++++++++++++++++++ tests/fuzztest/run_radamsa.sh | 12 +++ tests/fuzztest/sample_data/sample1.pb | Bin 0 -> 573 bytes tests/fuzztest/sample_data/sample2.pb | Bin 0 -> 466 bytes 5 files changed, 209 insertions(+) create mode 100644 tests/fuzztest/fuzzstub.c create mode 100755 tests/fuzztest/run_radamsa.sh create mode 100644 tests/fuzztest/sample_data/sample1.pb create mode 100644 tests/fuzztest/sample_data/sample2.pb diff --git a/tests/fuzztest/SConscript b/tests/fuzztest/SConscript index 36b62c56..6499714f 100644 --- a/tests/fuzztest/SConscript +++ b/tests/fuzztest/SConscript @@ -47,4 +47,12 @@ Depends([p1, p2, fuzz], ["fuzz_syshdr.h", "malloc_wrappers.h"]) env.RunTest(fuzz) +fuzzstub = env.Program(["fuzzstub.c", + "alltypes_pointer.pb.c", + "alltypes_static.pb.c", + "pb_encode_with_malloc.o", + "pb_decode_with_malloc.o", + "pb_common_with_malloc.o", + "malloc_wrappers.c"]) + diff --git a/tests/fuzztest/fuzzstub.c b/tests/fuzztest/fuzzstub.c new file mode 100644 index 00000000..50998416 --- /dev/null +++ b/tests/fuzztest/fuzzstub.c @@ -0,0 +1,189 @@ +/* Fuzz testing for the nanopb core. + * This can be used with external fuzzers, e.g. radamsa. + * It performs most of the same checks as fuzztest, but does not feature data generation. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "malloc_wrappers.h" +#include "alltypes_static.pb.h" +#include "alltypes_pointer.pb.h" + +#define BUFSIZE 4096 + +static bool do_static_decode(uint8_t *buffer, size_t msglen, bool assert_success) +{ + pb_istream_t stream; + bool status; + + alltypes_static_AllTypes *msg = malloc_with_check(sizeof(alltypes_static_AllTypes)); + stream = pb_istream_from_buffer(buffer, msglen); + status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg); + + if (!status && assert_success) + { + /* Anything that was successfully encoded, should be decodeable. + * One exception: strings without null terminator are encoded up + * to end of buffer, but refused on decode because the terminator + * would not fit. */ + if (strcmp(stream.errmsg, "string overflow") != 0) + assert(status); + } + + free_with_check(msg); + return status; +} + +static bool do_pointer_decode(uint8_t *buffer, size_t msglen, bool assert_success) +{ + pb_istream_t stream; + bool status; + alltypes_pointer_AllTypes *msg; + + msg = malloc_with_check(sizeof(alltypes_pointer_AllTypes)); + memset(msg, 0, sizeof(alltypes_pointer_AllTypes)); + stream = pb_istream_from_buffer(buffer, msglen); + + assert(get_alloc_count() == 0); + status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg); + + if (assert_success) + assert(status); + + pb_release(alltypes_pointer_AllTypes_fields, msg); + assert(get_alloc_count() == 0); + + free_with_check(msg); + + return status; +} + +/* Do a decode -> encode -> decode -> encode roundtrip */ +static void do_static_roundtrip(uint8_t *buffer, size_t msglen) +{ + bool status; + uint8_t *buf2 = malloc_with_check(BUFSIZE); + uint8_t *buf3 = malloc_with_check(BUFSIZE); + size_t msglen2, msglen3; + alltypes_static_AllTypes *msg1 = malloc_with_check(sizeof(alltypes_static_AllTypes)); + alltypes_static_AllTypes *msg2 = malloc_with_check(sizeof(alltypes_static_AllTypes)); + memset(msg1, 0, sizeof(alltypes_static_AllTypes)); + memset(msg2, 0, sizeof(alltypes_static_AllTypes)); + + { + pb_istream_t stream = pb_istream_from_buffer(buffer, msglen); + status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg1); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf2, BUFSIZE); + status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg1); + assert(status); + msglen2 = stream.bytes_written; + } + + { + pb_istream_t stream = pb_istream_from_buffer(buf2, msglen2); + status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg2); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf3, BUFSIZE); + status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg2); + assert(status); + msglen3 = stream.bytes_written; + } + + assert(msglen2 == msglen3); + assert(memcmp(buf2, buf3, msglen2) == 0); + + free_with_check(msg1); + free_with_check(msg2); + free_with_check(buf2); + free_with_check(buf3); +} + +/* Do decode -> encode -> decode -> encode roundtrip */ +static void do_pointer_roundtrip(uint8_t *buffer, size_t msglen) +{ + bool status; + uint8_t *buf2 = malloc_with_check(BUFSIZE); + uint8_t *buf3 = malloc_with_check(BUFSIZE); + size_t msglen2, msglen3; + alltypes_pointer_AllTypes *msg1 = malloc_with_check(sizeof(alltypes_pointer_AllTypes)); + alltypes_pointer_AllTypes *msg2 = malloc_with_check(sizeof(alltypes_pointer_AllTypes)); + memset(msg1, 0, sizeof(alltypes_pointer_AllTypes)); + memset(msg2, 0, sizeof(alltypes_pointer_AllTypes)); + + { + pb_istream_t stream = pb_istream_from_buffer(buffer, msglen); + status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg1); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf2, BUFSIZE); + status = pb_encode(&stream, alltypes_pointer_AllTypes_fields, msg1); + assert(status); + msglen2 = stream.bytes_written; + } + + { + pb_istream_t stream = pb_istream_from_buffer(buf2, msglen2); + status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg2); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf3, BUFSIZE); + status = pb_encode(&stream, alltypes_pointer_AllTypes_fields, msg2); + assert(status); + msglen3 = stream.bytes_written; + } + + assert(msglen2 == msglen3); + assert(memcmp(buf2, buf3, msglen2) == 0); + + pb_release(alltypes_pointer_AllTypes_fields, msg1); + pb_release(alltypes_pointer_AllTypes_fields, msg2); + free_with_check(msg1); + free_with_check(msg2); + free_with_check(buf2); + free_with_check(buf3); +} + +static void run_iteration() +{ + uint8_t *buffer = malloc_with_check(BUFSIZE); + size_t msglen; + bool status; + + msglen = fread(buffer, BUFSIZE, 1, stdin); + + status = do_static_decode(buffer, msglen, false); + + if (status) + do_static_roundtrip(buffer, msglen); + + status = do_pointer_decode(buffer, msglen, false); + + if (status) + do_pointer_roundtrip(buffer, msglen); + + free_with_check(buffer); +} + +int main(int argc, char **argv) +{ + run_iteration(); + + return 0; +} + diff --git a/tests/fuzztest/run_radamsa.sh b/tests/fuzztest/run_radamsa.sh new file mode 100755 index 00000000..52cd40a8 --- /dev/null +++ b/tests/fuzztest/run_radamsa.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +TMP=`tempfile` + +echo $TMP +while true +do + radamsa sample_data/* > $TMP + $1 < $TMP + test $? -gt 127 && break +done + diff --git a/tests/fuzztest/sample_data/sample1.pb b/tests/fuzztest/sample_data/sample1.pb new file mode 100644 index 00000000..07527885 Binary files /dev/null and b/tests/fuzztest/sample_data/sample1.pb differ diff --git a/tests/fuzztest/sample_data/sample2.pb b/tests/fuzztest/sample_data/sample2.pb new file mode 100644 index 00000000..cc89f91b Binary files /dev/null and b/tests/fuzztest/sample_data/sample2.pb differ -- cgit 1.2.3-korg From 8d7deb4952c2a9b78f62b7fdc469d319344b187b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 11 Sep 2014 19:26:32 +0300 Subject: Update changelog --- CHANGELOG.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 1555baee..6087a29f 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,11 @@ +nanopb-0.3.1 (2014-09-11) + Fix security issue due to size_t overflows. (issue 132) + Fix memory leak with duplicated fields and PB_ENABLE_MALLOC + Fix crash if pb_release() is called twice. + Fix cyclic message support (issue 130) + Fix error in generated initializers for repeated pointer fields. + Improve tests (issues 113, 126) + nanopb-0.3.0 (2014-08-26) NOTE: See docs/migration.html or online at http://koti.kapsi.fi/~jpa/nanopb/docs/migration.html -- cgit 1.2.3-korg From b947dc6e2c0d63a29e83ebf9c8af450d2531aef2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 11 Sep 2014 19:36:14 +0300 Subject: Publishing nanopb-0.3.1 --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 9ffed03a..b9cfcc3d 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.1-dev" +nanopb_version = "nanopb-0.3.1" import sys diff --git a/pb.h b/pb.h index fc301479..34d14b04 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.1-dev +#define NANOPB_VERSION nanopb-0.3.1 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From baf44b367f726484c315f204f1bf95ba40af86fa Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 16 Sep 2014 20:41:45 +0300 Subject: Set version to nanopb-0.3.2-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index b9cfcc3d..19771a7a 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.1" +nanopb_version = "nanopb-0.3.2-dev" import sys diff --git a/pb.h b/pb.h index 34d14b04..2582e250 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.1 +#define NANOPB_VERSION nanopb-0.3.2-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From cfc517f36b3263bb583751f4cee0e58eeff98c5f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 22 Dec 2014 20:52:40 +0200 Subject: Add compilation option to disable struct packing. Update issue 136 Status: FixedInGit --- docs/reference.rst | 3 +++ pb.h | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/reference.rst b/docs/reference.rst index ccbf0a4d..8018bdb9 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -24,6 +24,9 @@ __BIG_ENDIAN__ Set this if your platform stores integers and floats in big-endian format. Mixed-endian systems (different layout for ints and floats) are currently not supported. +PB_NO_PACKED_STRUCTS Disable packed structs. Increases RAM usage but + is necessary on some platforms that do not + support unaligned memory access. PB_ENABLE_MALLOC Set this to enable dynamic allocation support in the decoder. PB_MAX_REQUIRED_FIELDS Maximum number of required fields to check for diff --git a/pb.h b/pb.h index 2582e250..b8a957ae 100644 --- a/pb.h +++ b/pb.h @@ -17,6 +17,10 @@ * stores the most-significant byte first. */ /* #define __BIG_ENDIAN__ 1 */ +/* Define this if your CPU / compiler combination does not support + * unaligned memory access to packed structures. */ +/* #define PB_NO_PACKED_STRUCTS 1 */ + /* Increase the number of required fields that are tracked. * A compiler warning will tell if you need this. */ /* #define PB_MAX_REQUIRED_FIELDS 256 */ @@ -75,7 +79,12 @@ /* Macro for defining packed structures (compiler dependent). * This just reduces memory requirements, but is not required. */ -#if defined(__GNUC__) || defined(__clang__) +#if defined(PB_NO_PACKED_STRUCTS) + /* Disable struct packing */ +# define PB_PACKED_STRUCT_START +# define PB_PACKED_STRUCT_END +# define pb_packed +#elif defined(__GNUC__) || defined(__clang__) /* For GCC and clang */ # define PB_PACKED_STRUCT_START # define PB_PACKED_STRUCT_END -- cgit 1.2.3-korg From e5cbee84e12e87b342d7c57808b009387e1ba2bb Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 22 Dec 2014 22:52:36 +0200 Subject: Verify build with protobuf-3.0.0, fix problems. Also updated descriptor.proto from protobuf-3.0.0. --- generator/proto/google/protobuf/descriptor.proto | 144 +++++++++++++++++++---- generator/proto/nanopb.proto | 1 + generator/proto/plugin.proto | 5 +- tests/extensions/extensions.proto | 4 +- 4 files changed, 126 insertions(+), 28 deletions(-) diff --git a/generator/proto/google/protobuf/descriptor.proto b/generator/proto/google/protobuf/descriptor.proto index a785f79f..e17c0cc8 100644 --- a/generator/proto/google/protobuf/descriptor.proto +++ b/generator/proto/google/protobuf/descriptor.proto @@ -1,6 +1,6 @@ // Protocol Buffers - Google's data interchange format // Copyright 2008 Google Inc. All rights reserved. -// http://code.google.com/p/protobuf/ +// https://developers.google.com/protocol-buffers/ // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are @@ -37,6 +37,7 @@ // without any other information (e.g. without reading its imports). +syntax = "proto2"; package google.protobuf; option java_package = "com.google.protobuf"; @@ -74,10 +75,14 @@ message FileDescriptorProto { optional FileOptions options = 8; // This field contains optional information about the original source code. - // You may safely remove this entire field whithout harming runtime + // You may safely remove this entire field without harming runtime // functionality of the descriptors -- the information is needed only by // development tools. optional SourceCodeInfo source_code_info = 9; + + // The syntax of the proto file. + // The supported values are "proto2" and "proto3". + optional string syntax = 12; } // Describes a message type. @@ -96,6 +101,8 @@ message DescriptorProto { } repeated ExtensionRange extension_range = 5; + repeated OneofDescriptorProto oneof_decl = 8; + optional MessageOptions options = 7; } @@ -143,7 +150,7 @@ message FieldDescriptorProto { optional Label label = 4; // If type_name is set, this need not be set. If both this and type_name - // are set, this must be either TYPE_ENUM or TYPE_MESSAGE. + // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. optional Type type = 5; // For message and enum types, this is the name of the type. If the name @@ -164,9 +171,20 @@ message FieldDescriptorProto { // TODO(kenton): Base-64 encode? optional string default_value = 7; + // If set, gives the index of a oneof in the containing type's oneof_decl + // list. This field is a member of that oneof. Extensions of a oneof should + // not set this since the oneof to which they belong will be inferred based + // on the extension range containing the extension's field number. + optional int32 oneof_index = 9; + optional FieldOptions options = 8; } +// Describes a oneof. +message OneofDescriptorProto { + optional string name = 1; +} + // Describes an enum type. message EnumDescriptorProto { optional string name = 1; @@ -202,6 +220,11 @@ message MethodDescriptorProto { optional string output_type = 3; optional MethodOptions options = 4; + + // Identifies if client streams multiple client messages + optional bool client_streaming = 5 [default=false]; + // Identifies if server streams multiple server messages + optional bool server_streaming = 6 [default=false]; } @@ -233,7 +256,7 @@ message MethodDescriptorProto { // number. You can declare multiple options with only one extension number by // putting them in a sub-message. See the Custom Options section of the docs // for examples: -// http://code.google.com/apis/protocolbuffers/docs/proto.html#options +// https://developers.google.com/protocol-buffers/docs/proto#options // If this turns out to be popular, a web service will be set up // to automatically assign option numbers. @@ -263,11 +286,26 @@ message FileOptions { optional bool java_multiple_files = 10 [default=false]; // If set true, then the Java code generator will generate equals() and - // hashCode() methods for all messages defined in the .proto file. This is - // purely a speed optimization, as the AbstractMessage base class includes - // reflection-based implementations of these methods. + // hashCode() methods for all messages defined in the .proto file. + // - In the full runtime, this is purely a speed optimization, as the + // AbstractMessage base class includes reflection-based implementations of + // these methods. + //- In the lite runtime, setting this option changes the semantics of + // equals() and hashCode() to more closely match those of the full runtime; + // the generated methods compute their results based on field values rather + // than object identity. (Implementations should not assume that hashcodes + // will be consistent across runtimes or versions of the protocol compiler.) optional bool java_generate_equals_and_hash = 20 [default=false]; + // If set true, then the Java2 code generator will generate code that + // throws an exception whenever an attempt is made to assign a non-UTF-8 + // byte sequence to a string field. + // Message reflection will do the same. + // However, an extension field still accepts non-UTF-8 byte sequences. + // This option has no effect on when used with the lite runtime. + optional bool java_string_check_utf8 = 27 [default=false]; + + // Generated classes can be optimized for speed or code size. enum OptimizeMode { SPEED = 1; // Generate complete code for parsing, serialization, @@ -278,7 +316,10 @@ message FileOptions { optional OptimizeMode optimize_for = 9 [default=SPEED]; // Sets the Go package where structs generated from this .proto will be - // placed. There is no default. + // placed. If omitted, the Go package will be derived from the following: + // - The basename of the package import path, if provided. + // - Otherwise, the package statement in the .proto file, if present. + // - Otherwise, the basename of the .proto file, without extension. optional string go_package = 11; @@ -287,7 +328,7 @@ message FileOptions { // are not specific to any particular RPC system. They are generated by the // main code generators in each language (without additional plugins). // Generic services were the only kind of service generation supported by - // early versions of proto2. + // early versions of google.protobuf. // // Generic services are now considered deprecated in favor of using plugins // that generate code specific to your particular RPC system. Therefore, @@ -297,6 +338,18 @@ message FileOptions { optional bool java_generic_services = 17 [default=false]; optional bool py_generic_services = 18 [default=false]; + // Is this file deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for everything in the file, or it will be completely ignored; in the very + // least, this is a formalization for deprecating files. + optional bool deprecated = 23 [default=false]; + + + // Enables the use of arenas for the proto messages in this file. This applies + // only to generated classes for C++. + optional bool cc_enable_arenas = 31 [default=false]; + + // The parser stores options it doesn't recognize here. See above. repeated UninterpretedOption uninterpreted_option = 999; @@ -330,6 +383,35 @@ message MessageOptions { // from proto1 easier; new code should avoid fields named "descriptor". optional bool no_standard_descriptor_accessor = 2 [default=false]; + // Is this message deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the message, or it will be completely ignored; in the very least, + // this is a formalization for deprecating messages. + optional bool deprecated = 3 [default=false]; + + // Whether the message is an automatically generated map entry type for the + // maps field. + // + // For maps fields: + // map map_field = 1; + // The parsed descriptor looks like: + // message MapFieldEntry { + // option map_entry = true; + // optional KeyType key = 1; + // optional ValueType value = 2; + // } + // repeated MapFieldEntry map_field = 1; + // + // Implementations may choose not to generate the map_entry=true message, but + // use a native map in the target language to hold the keys and values. + // The reflection APIs in such implementions still need to work as + // if the field is a repeated message field. + // + // NOTE: Do not set the option in .proto files. Always use the maps syntax + // instead. The option should only be implicitly set by the proto compiler + // parser. + optional bool map_entry = 7; + // The parser stores options it doesn't recognize here. See above. repeated UninterpretedOption uninterpreted_option = 999; @@ -395,23 +477,11 @@ message FieldOptions { // is a formalization for deprecating fields. optional bool deprecated = 3 [default=false]; - // EXPERIMENTAL. DO NOT USE. - // For "map" fields, the name of the field in the enclosed type that - // is the key for this map. For example, suppose we have: - // message Item { - // required string name = 1; - // required string value = 2; - // } - // message Config { - // repeated Item items = 1 [experimental_map_key="name"]; - // } - // In this situation, the map key for Item will be set to "name". - // TODO: Fully-implement this, then remove the "experimental_" prefix. - optional string experimental_map_key = 9; - // For Google-internal migration only. Do not use. optional bool weak = 10 [default=false]; + + // The parser stores options it doesn't recognize here. See above. repeated UninterpretedOption uninterpreted_option = 999; @@ -421,9 +491,15 @@ message FieldOptions { message EnumOptions { - // Set this option to false to disallow mapping different tag names to a same + // Set this option to true to allow mapping different tag names to the same // value. - optional bool allow_alias = 2 [default=true]; + optional bool allow_alias = 2; + + // Is this enum deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum, or it will be completely ignored; in the very least, this + // is a formalization for deprecating enums. + optional bool deprecated = 3 [default=false]; // The parser stores options it doesn't recognize here. See above. repeated UninterpretedOption uninterpreted_option = 999; @@ -433,6 +509,12 @@ message EnumOptions { } message EnumValueOptions { + // Is this enum value deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum value, or it will be completely ignored; in the very least, + // this is a formalization for deprecating enum values. + optional bool deprecated = 1 [default=false]; + // The parser stores options it doesn't recognize here. See above. repeated UninterpretedOption uninterpreted_option = 999; @@ -447,6 +529,12 @@ message ServiceOptions { // we were already using them long before we decided to release Protocol // Buffers. + // Is this service deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the service, or it will be completely ignored; in the very least, + // this is a formalization for deprecating services. + optional bool deprecated = 33 [default=false]; + // The parser stores options it doesn't recognize here. See above. repeated UninterpretedOption uninterpreted_option = 999; @@ -461,6 +549,12 @@ message MethodOptions { // we were already using them long before we decided to release Protocol // Buffers. + // Is this method deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the method, or it will be completely ignored; in the very least, + // this is a formalization for deprecating methods. + optional bool deprecated = 33 [default=false]; + // The parser stores options it doesn't recognize here. See above. repeated UninterpretedOption uninterpreted_option = 999; diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index 9a4d6578..0716be4b 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -5,6 +5,7 @@ // These are used by nanopb to generate statically allocable structures // for memory-limited environments. +syntax = "proto2"; import "google/protobuf/descriptor.proto"; option java_package = "fi.kapsi.koti.jpa.nanopb"; diff --git a/generator/proto/plugin.proto b/generator/proto/plugin.proto index 651ed10c..e627289b 100644 --- a/generator/proto/plugin.proto +++ b/generator/proto/plugin.proto @@ -1,6 +1,6 @@ // Protocol Buffers - Google's data interchange format // Copyright 2008 Google Inc. All rights reserved. -// http://code.google.com/p/protobuf/ +// https://developers.google.com/protocol-buffers/ // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are @@ -44,7 +44,10 @@ // plugin should be named "protoc-gen-$NAME", and will then be used when the // flag "--${NAME}_out" is passed to protoc. +syntax = "proto2"; package google.protobuf.compiler; +option java_package = "com.google.protobuf.compiler"; +option java_outer_classname = "PluginProtos"; import "google/protobuf/descriptor.proto"; diff --git a/tests/extensions/extensions.proto b/tests/extensions/extensions.proto index da8432e0..79c01245 100644 --- a/tests/extensions/extensions.proto +++ b/tests/extensions/extensions.proto @@ -7,8 +7,8 @@ extend AllTypes { message ExtensionMessage { extend AllTypes { optional ExtensionMessage AllTypes_extensionfield2 = 254; - required ExtensionMessage AllTypes_extensionfield3 = 253; - repeated ExtensionMessage AllTypes_extensionfield4 = 252; + // required ExtensionMessage AllTypes_extensionfield3 = 253; // No longer allowed by protobuf 3 + repeated ExtensionMessage AllTypes_extensionfield4 = 252; } required string test1 = 1; -- cgit 1.2.3-korg From 8a28b70351baf09f2131fee2fc186a96d069cc2e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 26 Dec 2014 17:08:17 +0200 Subject: Move malloc_wrappers.c to tests/common --- tests/common/SConscript | 4 +++ tests/common/malloc_wrappers.c | 54 +++++++++++++++++++++++++++++++++++ tests/common/malloc_wrappers.h | 7 +++++ tests/common/malloc_wrappers_syshdr.h | 15 ++++++++++ tests/fuzztest/SConscript | 9 +++--- tests/fuzztest/fuzz_syshdr.h | 15 ---------- tests/fuzztest/fuzzstub.c | 2 +- tests/fuzztest/fuzztest.c | 2 +- tests/fuzztest/malloc_wrappers.c | 54 ----------------------------------- tests/fuzztest/malloc_wrappers.h | 7 ----- 10 files changed, 86 insertions(+), 83 deletions(-) create mode 100644 tests/common/malloc_wrappers.c create mode 100644 tests/common/malloc_wrappers.h create mode 100644 tests/common/malloc_wrappers_syshdr.h delete mode 100644 tests/fuzztest/fuzz_syshdr.h delete mode 100644 tests/fuzztest/malloc_wrappers.c delete mode 100644 tests/fuzztest/malloc_wrappers.h diff --git a/tests/common/SConscript b/tests/common/SConscript index f1dee0e2..4581bea1 100644 --- a/tests/common/SConscript +++ b/tests/common/SConscript @@ -15,3 +15,7 @@ strict.Append(CFLAGS = strict['CORECFLAGS']) strict.Object("pb_decode.o", "$NANOPB/pb_decode.c") strict.Object("pb_encode.o", "$NANOPB/pb_encode.c") strict.Object("pb_common.o", "$NANOPB/pb_common.c") + +mw = env.Object("malloc_wrappers.o", "malloc_wrappers.c") +Depends(mw, ["malloc_wrappers_syshdr.h"]) + diff --git a/tests/common/malloc_wrappers.c b/tests/common/malloc_wrappers.c new file mode 100644 index 00000000..ad69f1ce --- /dev/null +++ b/tests/common/malloc_wrappers.c @@ -0,0 +1,54 @@ +#include "malloc_wrappers.h" +#include +#include +#include + +static size_t alloc_count = 0; + +/* Allocate memory and place check values before and after. */ +void* malloc_with_check(size_t size) +{ + size_t size32 = (size + 3) / 4 + 3; + uint32_t *buf = malloc(size32 * sizeof(uint32_t)); + buf[0] = size32; + buf[1] = 0xDEADBEEF; + buf[size32 - 1] = 0xBADBAD; + return buf + 2; +} + +/* Free memory allocated with malloc_with_check() and do the checks. */ +void free_with_check(void *mem) +{ + uint32_t *buf = (uint32_t*)mem - 2; + assert(buf[1] == 0xDEADBEEF); + assert(buf[buf[0] - 1] == 0xBADBAD); + free(buf); +} + +/* Track memory usage */ +void* counting_realloc(void *ptr, size_t size) +{ + /* Don't allocate crazy amounts of RAM when fuzzing */ + if (size > 1000000) + return NULL; + + if (!ptr && size) + alloc_count++; + + return realloc(ptr, size); +} + +void counting_free(void *ptr) +{ + if (ptr) + { + assert(alloc_count > 0); + alloc_count--; + free(ptr); + } +} + +size_t get_alloc_count() +{ + return alloc_count; +} diff --git a/tests/common/malloc_wrappers.h b/tests/common/malloc_wrappers.h new file mode 100644 index 00000000..7eec7952 --- /dev/null +++ b/tests/common/malloc_wrappers.h @@ -0,0 +1,7 @@ +#include + +void* malloc_with_check(size_t size); +void free_with_check(void *mem); +void* counting_realloc(void *ptr, size_t size); +void counting_free(void *ptr); +size_t get_alloc_count(); diff --git a/tests/common/malloc_wrappers_syshdr.h b/tests/common/malloc_wrappers_syshdr.h new file mode 100644 index 00000000..d295d9ed --- /dev/null +++ b/tests/common/malloc_wrappers_syshdr.h @@ -0,0 +1,15 @@ +/* This is just a wrapper in order to get our own malloc wrappers into nanopb core. */ + +#define pb_realloc(ptr,size) counting_realloc(ptr,size) +#define pb_free(ptr) counting_free(ptr) + +#ifdef PB_OLD_SYSHDR +#include PB_OLD_SYSHDR +#else +#include +#include +#include +#include +#endif + +#include diff --git a/tests/fuzztest/SConscript b/tests/fuzztest/SConscript index 6499714f..346ccab8 100644 --- a/tests/fuzztest/SConscript +++ b/tests/fuzztest/SConscript @@ -5,8 +5,8 @@ Import("env") # We need our own pb_decode.o for the malloc support env = env.Clone() env.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1, - 'PB_SYSTEM_HEADER': '\\"fuzz_syshdr.h\\"'}) -env.Append(CPPPATH = ".") + 'PB_SYSTEM_HEADER': '\\"malloc_wrappers_syshdr.h\\"'}) +env.Append(CPPPATH = [".", "$COMMON"]) if 'SYSHDR' in env: env.Append(CPPDEFINES = {'PB_OLD_SYSHDR': env['SYSHDR']}) @@ -42,8 +42,7 @@ fuzz = env.Program(["fuzztest.c", "pb_encode_with_malloc.o", "pb_decode_with_malloc.o", "pb_common_with_malloc.o", - "malloc_wrappers.c"]) -Depends([p1, p2, fuzz], ["fuzz_syshdr.h", "malloc_wrappers.h"]) + "$COMMON/malloc_wrappers.o"]) env.RunTest(fuzz) @@ -53,6 +52,6 @@ fuzzstub = env.Program(["fuzzstub.c", "pb_encode_with_malloc.o", "pb_decode_with_malloc.o", "pb_common_with_malloc.o", - "malloc_wrappers.c"]) + "$COMMON/malloc_wrappers.o"]) diff --git a/tests/fuzztest/fuzz_syshdr.h b/tests/fuzztest/fuzz_syshdr.h deleted file mode 100644 index d295d9ed..00000000 --- a/tests/fuzztest/fuzz_syshdr.h +++ /dev/null @@ -1,15 +0,0 @@ -/* This is just a wrapper in order to get our own malloc wrappers into nanopb core. */ - -#define pb_realloc(ptr,size) counting_realloc(ptr,size) -#define pb_free(ptr) counting_free(ptr) - -#ifdef PB_OLD_SYSHDR -#include PB_OLD_SYSHDR -#else -#include -#include -#include -#include -#endif - -#include diff --git a/tests/fuzztest/fuzzstub.c b/tests/fuzztest/fuzzstub.c index 50998416..ce14b9ba 100644 --- a/tests/fuzztest/fuzzstub.c +++ b/tests/fuzztest/fuzzstub.c @@ -10,7 +10,7 @@ #include #include #include -#include "malloc_wrappers.h" +#include #include "alltypes_static.pb.h" #include "alltypes_pointer.pb.h" diff --git a/tests/fuzztest/fuzztest.c b/tests/fuzztest/fuzztest.c index 996ed450..d3701724 100644 --- a/tests/fuzztest/fuzztest.c +++ b/tests/fuzztest/fuzztest.c @@ -9,7 +9,7 @@ #include #include #include -#include "malloc_wrappers.h" +#include #include "alltypes_static.pb.h" #include "alltypes_pointer.pb.h" diff --git a/tests/fuzztest/malloc_wrappers.c b/tests/fuzztest/malloc_wrappers.c deleted file mode 100644 index ad69f1ce..00000000 --- a/tests/fuzztest/malloc_wrappers.c +++ /dev/null @@ -1,54 +0,0 @@ -#include "malloc_wrappers.h" -#include -#include -#include - -static size_t alloc_count = 0; - -/* Allocate memory and place check values before and after. */ -void* malloc_with_check(size_t size) -{ - size_t size32 = (size + 3) / 4 + 3; - uint32_t *buf = malloc(size32 * sizeof(uint32_t)); - buf[0] = size32; - buf[1] = 0xDEADBEEF; - buf[size32 - 1] = 0xBADBAD; - return buf + 2; -} - -/* Free memory allocated with malloc_with_check() and do the checks. */ -void free_with_check(void *mem) -{ - uint32_t *buf = (uint32_t*)mem - 2; - assert(buf[1] == 0xDEADBEEF); - assert(buf[buf[0] - 1] == 0xBADBAD); - free(buf); -} - -/* Track memory usage */ -void* counting_realloc(void *ptr, size_t size) -{ - /* Don't allocate crazy amounts of RAM when fuzzing */ - if (size > 1000000) - return NULL; - - if (!ptr && size) - alloc_count++; - - return realloc(ptr, size); -} - -void counting_free(void *ptr) -{ - if (ptr) - { - assert(alloc_count > 0); - alloc_count--; - free(ptr); - } -} - -size_t get_alloc_count() -{ - return alloc_count; -} diff --git a/tests/fuzztest/malloc_wrappers.h b/tests/fuzztest/malloc_wrappers.h deleted file mode 100644 index 7eec7952..00000000 --- a/tests/fuzztest/malloc_wrappers.h +++ /dev/null @@ -1,7 +0,0 @@ -#include - -void* malloc_with_check(size_t size); -void free_with_check(void *mem); -void* counting_realloc(void *ptr, size_t size); -void counting_free(void *ptr); -size_t get_alloc_count(); -- cgit 1.2.3-korg From 0f3c0f79bc83c3678c325f7e8e17dd031acaf77f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 26 Dec 2014 17:34:45 +0200 Subject: Move malloc support to tests/common directory --- tests/SConstruct | 8 +++++++- tests/alltypes_pointer/SConscript | 31 ++++++++++------------------ tests/common/SConscript | 27 ++++++++++++++++++++++++ tests/fuzztest/SConscript | 41 +++++++++---------------------------- tests/io_errors_pointers/SConscript | 26 +++++------------------ 5 files changed, 60 insertions(+), 73 deletions(-) diff --git a/tests/SConstruct b/tests/SConstruct index 1890670b..9092d4f8 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -136,12 +136,18 @@ elif 'g++' in env['CXX'] or 'gcc' in env['CXX']: env.Append(CXXFLAGS = '-g -Wall -Werror -Wextra -Wno-missing-field-initializers') elif 'cl' in env['CXX']: env.Append(CXXFLAGS = '/Zi /W2 /WX') - + # Now include the SConscript files from all subdirectories import os.path env['VARIANT_DIR'] = 'build' env['BUILD'] = '#' + env['VARIANT_DIR'] env['COMMON'] = '#' + env['VARIANT_DIR'] + '/common' + +# Include common/SConscript first to make sure its exports are available +# to other SConscripts. +SConscript("common/SConscript", exports = 'env', variant_dir = env['VARIANT_DIR'] + '/common') + for subdir in Glob('*/SConscript') + Glob('regression/*/SConscript'): + if str(subdir).startswith("common"): continue SConscript(subdir, exports = 'env', variant_dir = env['VARIANT_DIR'] + '/' + os.path.dirname(str(subdir))) diff --git a/tests/alltypes_pointer/SConscript b/tests/alltypes_pointer/SConscript index 8fcf197e..52856f6e 100644 --- a/tests/alltypes_pointer/SConscript +++ b/tests/alltypes_pointer/SConscript @@ -1,31 +1,22 @@ # Encode the AllTypes message using pointers for all fields, and verify the # output against the normal AllTypes test case. -Import("env") - -# We need our own pb_decode.o for the malloc support -env = env.Clone() -env.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1}); - -# Disable libmudflap, because it will confuse valgrind -# and other memory leak detection tools. -if '-fmudflap' in env["CCFLAGS"]: - env["CCFLAGS"].remove("-fmudflap") - env["LINKFLAGS"].remove("-fmudflap") - env["LIBS"].remove("mudflap") - -strict = env.Clone() -strict.Append(CFLAGS = strict['CORECFLAGS']) -strict.Object("pb_decode_with_malloc.o", "$NANOPB/pb_decode.c") -strict.Object("pb_encode_with_malloc.o", "$NANOPB/pb_encode.c") -strict.Object("pb_common_with_malloc.o", "$NANOPB/pb_common.c") +Import("env", "malloc_env") c = Copy("$TARGET", "$SOURCE") env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) env.NanopbProto(["alltypes", "alltypes.options"]) -enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "pb_encode_with_malloc.o", "pb_common_with_malloc.o"]) -dec = env.Program(["decode_alltypes_pointer.c", "alltypes.pb.c", "pb_decode_with_malloc.o", "pb_common_with_malloc.o"]) +enc = malloc_env.Program(["encode_alltypes_pointer.c", + "alltypes.pb.c", + "$COMMON/pb_encode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", + "$COMMON/malloc_wrappers.o"]) +dec = malloc_env.Program(["decode_alltypes_pointer.c", + "alltypes.pb.c", + "$COMMON/pb_decode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", + "$COMMON/malloc_wrappers.o"]) # Encode and compare results to non-pointer alltypes test case env.RunTest(enc) diff --git a/tests/common/SConscript b/tests/common/SConscript index 4581bea1..a202ca47 100644 --- a/tests/common/SConscript +++ b/tests/common/SConscript @@ -8,6 +8,7 @@ env.NanopbProto("unittestproto") # Protocol definitions for basic_buffer/stream tests env.NanopbProto("person") +#-------------------------------------------- # Binaries of the pb_decode.c and pb_encode.c # These are built using more strict warning flags. strict = env.Clone() @@ -16,6 +17,32 @@ strict.Object("pb_decode.o", "$NANOPB/pb_decode.c") strict.Object("pb_encode.o", "$NANOPB/pb_encode.c") strict.Object("pb_common.o", "$NANOPB/pb_common.c") +#----------------------------------------------- +# Binaries of pb_decode etc. with malloc support +# Uses malloc_wrappers.c to count allocations. +malloc_env = env.Clone() +malloc_env.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1, + 'PB_SYSTEM_HEADER': '\\"malloc_wrappers_syshdr.h\\"'}) +malloc_env.Append(CPPPATH = ["$COMMON"]) + +if 'SYSHDR' in malloc_env: + malloc_env.Append(CPPDEFINES = {'PB_OLD_SYSHDR': malloc_env['SYSHDR']}) + +# Disable libmudflap, because it will confuse valgrind +# and other memory leak detection tools. +if '-fmudflap' in env["CCFLAGS"]: + malloc_env["CCFLAGS"].remove("-fmudflap") + malloc_env["LINKFLAGS"].remove("-fmudflap") + malloc_env["LIBS"].remove("mudflap") + +malloc_strict = malloc_env.Clone() +malloc_strict.Append(CFLAGS = malloc_strict['CORECFLAGS']) +malloc_strict.Object("pb_decode_with_malloc.o", "$NANOPB/pb_decode.c") +malloc_strict.Object("pb_encode_with_malloc.o", "$NANOPB/pb_encode.c") +malloc_strict.Object("pb_common_with_malloc.o", "$NANOPB/pb_common.c") + mw = env.Object("malloc_wrappers.o", "malloc_wrappers.c") Depends(mw, ["malloc_wrappers_syshdr.h"]) +Export("malloc_env") + diff --git a/tests/fuzztest/SConscript b/tests/fuzztest/SConscript index 346ccab8..35b697f9 100644 --- a/tests/fuzztest/SConscript +++ b/tests/fuzztest/SConscript @@ -1,30 +1,9 @@ # Run a fuzz test to verify robustness against corrupted/malicious data. -Import("env") - -# We need our own pb_decode.o for the malloc support -env = env.Clone() -env.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1, - 'PB_SYSTEM_HEADER': '\\"malloc_wrappers_syshdr.h\\"'}) -env.Append(CPPPATH = [".", "$COMMON"]) - -if 'SYSHDR' in env: - env.Append(CPPDEFINES = {'PB_OLD_SYSHDR': env['SYSHDR']}) - -# Disable libmudflap, because it will confuse valgrind -# and other memory leak detection tools. -if '-fmudflap' in env["CCFLAGS"]: - env["CCFLAGS"].remove("-fmudflap") - env["LINKFLAGS"].remove("-fmudflap") - env["LIBS"].remove("mudflap") - -strict = env.Clone() -strict.Append(CFLAGS = strict['CORECFLAGS']) -strict.Object("pb_decode_with_malloc.o", "$NANOPB/pb_decode.c") -strict.Object("pb_encode_with_malloc.o", "$NANOPB/pb_encode.c") -strict.Object("pb_common_with_malloc.o", "$NANOPB/pb_common.c") +Import("env", "malloc_env") # We want both pointer and static versions of the AllTypes message +# Prefix them with package name. env.Command("alltypes_static.proto", "#alltypes/alltypes.proto", lambda target, source, env: open(str(target[0]), 'w').write("package alltypes_static;\n" @@ -36,22 +15,22 @@ env.Command("alltypes_pointer.proto", "#alltypes/alltypes.proto", p1 = env.NanopbProto(["alltypes_pointer", "alltypes_pointer.options"]) p2 = env.NanopbProto(["alltypes_static", "alltypes_static.options"]) -fuzz = env.Program(["fuzztest.c", +fuzz = malloc_env.Program(["fuzztest.c", "alltypes_pointer.pb.c", "alltypes_static.pb.c", - "pb_encode_with_malloc.o", - "pb_decode_with_malloc.o", - "pb_common_with_malloc.o", + "$COMMON/pb_encode_with_malloc.o", + "$COMMON/pb_decode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", "$COMMON/malloc_wrappers.o"]) env.RunTest(fuzz) -fuzzstub = env.Program(["fuzzstub.c", +fuzzstub = malloc_env.Program(["fuzzstub.c", "alltypes_pointer.pb.c", "alltypes_static.pb.c", - "pb_encode_with_malloc.o", - "pb_decode_with_malloc.o", - "pb_common_with_malloc.o", + "$COMMON/pb_encode_with_malloc.o", + "$COMMON/pb_decode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", "$COMMON/malloc_wrappers.o"]) diff --git a/tests/io_errors_pointers/SConscript b/tests/io_errors_pointers/SConscript index 0b96177f..8d23f603 100644 --- a/tests/io_errors_pointers/SConscript +++ b/tests/io_errors_pointers/SConscript @@ -1,23 +1,6 @@ # Simulate io errors when encoding and decoding -Import("env") - -# We need our own pb_decode.o for the malloc support -env = env.Clone() -env.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1}); - -# Disable libmudflap, because it will confuse valgrind -# and other memory leak detection tools. -if '-fmudflap' in env["CCFLAGS"]: - env["CCFLAGS"].remove("-fmudflap") - env["LINKFLAGS"].remove("-fmudflap") - env["LIBS"].remove("mudflap") - -strict = env.Clone() -strict.Append(CFLAGS = strict['CORECFLAGS']) -strict.Object("pb_decode_with_malloc.o", "$NANOPB/pb_decode.c") -strict.Object("pb_encode_with_malloc.o", "$NANOPB/pb_encode.c") -strict.Object("pb_common_with_malloc.o", "$NANOPB/pb_common.c") +Import("env", "malloc_env") c = Copy("$TARGET", "$SOURCE") env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) @@ -26,9 +9,10 @@ env.Command("io_errors.c", "#io_errors/io_errors.c", c) env.NanopbProto(["alltypes", "alltypes.options"]) ioerr = env.Program(["io_errors.c", "alltypes.pb.c", - "pb_encode_with_malloc.o", - "pb_decode_with_malloc.o", - "pb_common_with_malloc.o"]) + "$COMMON/pb_encode_with_malloc.o", + "$COMMON/pb_decode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", + "$COMMON/malloc_wrappers.o"]) # Run tests under valgrind if available valgrind = env.WhereIs('valgrind') -- cgit 1.2.3-korg From 980f899dd5ca1b4201536cdb56a723ba7777d82c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 26 Dec 2014 17:43:38 +0200 Subject: Include the field type in a comment for extension fields --- generator/nanopb_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 19771a7a..6dcc911f 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -498,7 +498,8 @@ class ExtensionField(Field): msg +=' type of extension fields is currently supported. */\n' return msg - return 'extern const pb_extension_type_t %s;\n' % self.fullname + return ('extern const pb_extension_type_t %s; /* field type: %s */\n' % + (self.fullname, str(self).strip())) def extension_def(self): '''Definition of the extension type in the .pb.c file''' -- cgit 1.2.3-korg From 418f7d88b3f58603fe03d0060b8aaba905ca56c8 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 26 Dec 2014 18:23:36 +0200 Subject: Add support for POINTER type in extensions --- pb.h | 6 ++++-- pb_decode.c | 8 ++++++++ pb_encode.c | 13 ++++++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/pb.h b/pb.h index b8a957ae..442d9658 100644 --- a/pb.h +++ b/pb.h @@ -456,9 +456,11 @@ struct pb_extension_s { 0, \ pb_membersize(st, m), 0, ptr} +#define PB_OPTEXT_POINTER(tag, st, m, fd, ltype, ptr) \ + PB_OPTIONAL_POINTER(tag, st, m, fd, ltype, ptr) + #define PB_OPTEXT_CALLBACK(tag, st, m, fd, ltype, ptr) \ - {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ - 0, 0, pb_membersize(st, m), 0, ptr} + PB_OPTIONAL_CALLBACK(tag, st, m, fd, ltype, ptr) /* The mapping from protobuf types to LTYPEs is done using these macros. */ #define PB_LTYPE_MAP_BOOL PB_LTYPE_VARINT diff --git a/pb_decode.c b/pb_decode.c index d1efd1b5..26d7c2b7 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -650,6 +650,14 @@ static bool checkreturn default_extension_decoder(pb_istream_t *stream, iter.pData = extension->dest; iter.pSize = &extension->found; + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + { + /* For pointer extensions, the pointer is stored directly + * in the extension structure. This avoids having an extra + * indirection. */ + iter.pData = &extension->dest; + } + return decode_field(stream, wire_type, &iter); } diff --git a/pb_encode.c b/pb_encode.c index cdd78955..5318361e 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -302,7 +302,18 @@ static bool checkreturn default_extension_encoder(pb_ostream_t *stream, const pb_extension_t *extension) { const pb_field_t *field = (const pb_field_t*)extension->type->arg; - return encode_field(stream, field, extension->dest); + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + { + /* For pointer extensions, the pointer is stored directly + * in the extension structure. This avoids having an extra + * indirection. */ + return encode_field(stream, field, &extension->dest); + } + else + { + return encode_field(stream, field, extension->dest); + } } /* Walk through all the registered extensions and give them a chance -- cgit 1.2.3-korg From 1515cfb5c24f64fd046c5b5f3cdfdcb153c72df7 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 26 Dec 2014 18:24:23 +0200 Subject: Add testcase for releasing memory in submessages/extensions --- tests/mem_release/SConscript | 13 +++++ tests/mem_release/mem_release.c | 112 ++++++++++++++++++++++++++++++++++++ tests/mem_release/mem_release.proto | 24 ++++++++ 3 files changed, 149 insertions(+) create mode 100644 tests/mem_release/SConscript create mode 100644 tests/mem_release/mem_release.c create mode 100644 tests/mem_release/mem_release.proto diff --git a/tests/mem_release/SConscript b/tests/mem_release/SConscript new file mode 100644 index 00000000..6754e285 --- /dev/null +++ b/tests/mem_release/SConscript @@ -0,0 +1,13 @@ +Import("env", "malloc_env") + +env.NanopbProto("mem_release.proto") + +test = malloc_env.Program(["mem_release.c", + "mem_release.pb.c", + "$COMMON/pb_encode_with_malloc.o", + "$COMMON/pb_decode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", + "$COMMON/malloc_wrappers.o"]) + +env.RunTest(test) + diff --git a/tests/mem_release/mem_release.c b/tests/mem_release/mem_release.c new file mode 100644 index 00000000..796df13e --- /dev/null +++ b/tests/mem_release/mem_release.c @@ -0,0 +1,112 @@ +/* Make sure that all fields are freed in various scenarios. */ + +#include +#include +#include +#include +#include +#include "mem_release.pb.h" + +#define TEST(x) if (!(x)) { \ + fprintf(stderr, "Test " #x " on line %d failed.\n", __LINE__); \ + return false; \ + } + +static char *test_str_arr[] = {"1", "2", ""}; +static SubMessage test_msg_arr[] = {SubMessage_init_zero, SubMessage_init_zero}; + +static bool do_test() +{ + uint8_t buffer[256]; + size_t msgsize; + + /* Construct a message with various fields filled in */ + { + TestMessage msg = TestMessage_init_zero; + pb_extension_t ext1, ext2; + pb_ostream_t stream; + msg.static_req_submsg.dynamic_str = "12345"; + msg.static_req_submsg.dynamic_str_arr_count = 3; + msg.static_req_submsg.dynamic_str_arr = test_str_arr; + msg.static_req_submsg.dynamic_submsg_count = 2; + msg.static_req_submsg.dynamic_submsg = test_msg_arr; + msg.static_req_submsg.dynamic_submsg[1].dynamic_str = "abc"; + msg.static_opt_submsg.dynamic_str = "abc"; + msg.has_static_opt_submsg = true; + msg.dynamic_submsg = &msg.static_req_submsg; + + msg.extensions = &ext1; + ext1.type = &dynamic_ext; + ext1.dest = &msg.static_req_submsg; + ext1.next = &ext2; + ext2.type = &static_ext; + ext2.dest = &msg.static_req_submsg; + ext2.next = NULL; + + stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + if (!pb_encode(&stream, TestMessage_fields, &msg)) + { + fprintf(stderr, "Encode failed: %s\n", PB_GET_ERROR(&stream)); + return false; + } + msgsize = stream.bytes_written; + } + + /* Output encoded message for debug */ + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, msgsize, stdout); + + /* Decode memory using dynamic allocation */ + { + TestMessage msg = TestMessage_init_zero; + pb_istream_t stream; + SubMessage ext2_dest; + pb_extension_t ext1, ext2; + + msg.extensions = &ext1; + ext1.type = &dynamic_ext; + ext1.dest = NULL; + ext1.next = &ext2; + ext2.type = &static_ext; + ext2.dest = &ext2_dest; + ext2.next = NULL; + + stream = pb_istream_from_buffer(buffer, msgsize); + if (!pb_decode(&stream, TestMessage_fields, &msg)) + { + fprintf(stderr, "Decode failed: %s\n", PB_GET_ERROR(&stream)); + return false; + } + + /* Make sure it encodes back to same data */ + { + uint8_t buffer2[256]; + pb_ostream_t ostream = pb_ostream_from_buffer(buffer2, sizeof(buffer2)); + TEST(pb_encode(&ostream, TestMessage_fields, &msg)); + TEST(ostream.bytes_written == msgsize); + TEST(memcmp(buffer, buffer2, msgsize) == 0); + } + + /* Make sure that malloc counters work */ + TEST(get_alloc_count() > 0); + + /* Make sure that pb_release releases everything */ + pb_release(TestMessage_fields, &msg); + TEST(get_alloc_count() == 0); + + /* Check that double-free is a no-op */ + pb_release(TestMessage_fields, &msg); + TEST(get_alloc_count() == 0); + } + + return true; +} + +int main() +{ + if (do_test()) + return 0; + else + return 1; +} + diff --git a/tests/mem_release/mem_release.proto b/tests/mem_release/mem_release.proto new file mode 100644 index 00000000..0db393ae --- /dev/null +++ b/tests/mem_release/mem_release.proto @@ -0,0 +1,24 @@ +syntax = "proto2"; +import "nanopb.proto"; + +message SubMessage +{ + optional string dynamic_str = 1 [(nanopb).type = FT_POINTER]; + repeated string dynamic_str_arr = 2 [(nanopb).type = FT_POINTER]; + repeated SubMessage dynamic_submsg = 3 [(nanopb).type = FT_POINTER]; +} + +message TestMessage +{ + required SubMessage static_req_submsg = 1 [(nanopb).type = FT_STATIC]; + optional SubMessage dynamic_submsg = 2 [(nanopb).type = FT_POINTER]; + optional SubMessage static_opt_submsg = 3 [(nanopb).type = FT_STATIC]; + extensions 100 to 200; +} + +extend TestMessage +{ + optional SubMessage dynamic_ext = 100 [(nanopb).type = FT_POINTER]; + optional SubMessage static_ext = 101 [(nanopb).type = FT_STATIC]; +} + -- cgit 1.2.3-korg From 58643217b1fa3eaf05768fabffac709cc8f277e8 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 26 Dec 2014 23:03:04 +0200 Subject: Fix bug in backwards_compatibility test case. The memset() filled also the extensions field, which was just waiting for a crash to happen. --- tests/backwards_compatibility/decode_legacy.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/backwards_compatibility/decode_legacy.c b/tests/backwards_compatibility/decode_legacy.c index 9dfe437a..5f5b6bbe 100644 --- a/tests/backwards_compatibility/decode_legacy.c +++ b/tests/backwards_compatibility/decode_legacy.c @@ -22,10 +22,7 @@ the decoding and checks the fields. */ bool check_alltypes(pb_istream_t *stream, int mode) { - AllTypes alltypes; - - /* Fill with garbage to better detect initialization errors */ - memset(&alltypes, 0xAA, sizeof(alltypes)); + AllTypes alltypes = {0}; if (!pb_decode(stream, AllTypes_fields, &alltypes)) return false; -- cgit 1.2.3-korg From 500883048860cc98745d69ae1d16d85523cd5291 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 26 Dec 2014 23:13:07 +0200 Subject: Initialize also extension fields to defaults in pb_decode(). This makes the behaviour more consistent with non-extension fields, and also makes sure that all 'found' fields of extensions are initially false. --- pb_decode.c | 137 ++++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 83 insertions(+), 54 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 26d7c2b7..f0fa1cce 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -29,9 +29,11 @@ static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); +static void iter_from_extension(pb_field_iter_t *iter, pb_extension_t *extension); static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type); static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_field_iter_t *iter); static bool checkreturn find_extension_field(pb_field_iter_t *iter); +static void pb_field_set_to_default(pb_field_iter_t *iter); static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct); static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest); static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, void *dest); @@ -632,32 +634,38 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t } } -/* Default handler for extension fields. Expects a pb_field_t structure - * in extension->type->arg. */ -static bool checkreturn default_extension_decoder(pb_istream_t *stream, - pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type) +static void iter_from_extension(pb_field_iter_t *iter, pb_extension_t *extension) { - const pb_field_t *field = (const pb_field_t*)extension->type->arg; - pb_field_iter_t iter; - - if (field->tag != tag) - return true; - /* Fake a field iterator for the extension field. * It is not actually safe to advance this iterator, but decode_field * will not even try to. */ - (void)pb_field_iter_begin(&iter, field, extension->dest); - iter.pData = extension->dest; - iter.pSize = &extension->found; + const pb_field_t *field = (const pb_field_t*)extension->type->arg; + (void)pb_field_iter_begin(iter, field, extension->dest); + iter->pData = extension->dest; + iter->pSize = &extension->found; if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { /* For pointer extensions, the pointer is stored directly * in the extension structure. This avoids having an extra * indirection. */ - iter.pData = &extension->dest; + iter->pData = &extension->dest; } +} + +/* Default handler for extension fields. Expects a pb_field_t structure + * in extension->type->arg. */ +static bool checkreturn default_extension_decoder(pb_istream_t *stream, + pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type) +{ + const pb_field_t *field = (const pb_field_t*)extension->type->arg; + pb_field_iter_t iter; + + if (field->tag != tag) + return true; + iter_from_extension(&iter, extension); + extension->found = true; return decode_field(stream, wire_type, &iter); } @@ -703,64 +711,85 @@ static bool checkreturn find_extension_field(pb_field_iter_t *iter) } /* Initialize message fields to default values, recursively */ -static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct) +static void pb_field_set_to_default(pb_field_iter_t *iter) { - pb_field_iter_t iter; - - if (!pb_field_iter_begin(&iter, fields, dest_struct)) - return; /* Empty message type */ + pb_type_t type; + type = iter->pos->type; - do + if (PB_LTYPE(type) == PB_LTYPE_EXTENSION) + { + pb_extension_t *ext = *(pb_extension_t* const *)iter->pData; + while (ext != NULL) + { + pb_field_iter_t ext_iter; + ext->found = false; + iter_from_extension(&ext_iter, ext); + pb_field_set_to_default(&ext_iter); + ext = ext->next; + } + } + else if (PB_ATYPE(type) == PB_ATYPE_STATIC) { - pb_type_t type; - type = iter.pos->type; + bool init_data = true; + if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL) + { + /* Set has_field to false. Still initialize the optional field + * itself also. */ + *(bool*)iter->pSize = false; + } + else if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + { + /* Set array count to 0, no need to initialize contents. */ + *(pb_size_t*)iter->pSize = 0; + init_data = false; + } - if (PB_ATYPE(type) == PB_ATYPE_STATIC) + if (init_data) { - if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL) - { - /* Set has_field to false. Still initialize the optional field - * itself also. */ - *(bool*)iter.pSize = false; - } - else if (PB_HTYPE(type) == PB_HTYPE_REPEATED) - { - /* Set array count to 0, no need to initialize contents. */ - *(pb_size_t*)iter.pSize = 0; - continue; - } - - if (PB_LTYPE(iter.pos->type) == PB_LTYPE_SUBMESSAGE) + if (PB_LTYPE(iter->pos->type) == PB_LTYPE_SUBMESSAGE) { /* Initialize submessage to defaults */ - pb_message_set_to_defaults((const pb_field_t *) iter.pos->ptr, iter.pData); + pb_message_set_to_defaults((const pb_field_t *) iter->pos->ptr, iter->pData); } - else if (iter.pos->ptr != NULL) + else if (iter->pos->ptr != NULL) { /* Initialize to default value */ - memcpy(iter.pData, iter.pos->ptr, iter.pos->data_size); + memcpy(iter->pData, iter->pos->ptr, iter->pos->data_size); } else { /* Initialize to zeros */ - memset(iter.pData, 0, iter.pos->data_size); - } - } - else if (PB_ATYPE(type) == PB_ATYPE_POINTER) - { - /* Initialize the pointer to NULL. */ - *(void**)iter.pData = NULL; - - /* Initialize array count to 0. */ - if (PB_HTYPE(type) == PB_HTYPE_REPEATED) - { - *(pb_size_t*)iter.pSize = 0; + memset(iter->pData, 0, iter->pos->data_size); } } - else if (PB_ATYPE(type) == PB_ATYPE_CALLBACK) + } + else if (PB_ATYPE(type) == PB_ATYPE_POINTER) + { + /* Initialize the pointer to NULL. */ + *(void**)iter->pData = NULL; + + /* Initialize array count to 0. */ + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) { - /* Don't overwrite callback */ + *(pb_size_t*)iter->pSize = 0; } + } + else if (PB_ATYPE(type) == PB_ATYPE_CALLBACK) + { + /* Don't overwrite callback */ + } +} + +static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct) +{ + pb_field_iter_t iter; + + if (!pb_field_iter_begin(&iter, fields, dest_struct)) + return; /* Empty message type */ + + do + { + pb_field_set_to_default(&iter); } while (pb_field_iter_next(&iter)); } -- cgit 1.2.3-korg From 88b2efe0477f4f9e313b5d7307dfd347b6893376 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 26 Dec 2014 23:14:39 +0200 Subject: Fix memory leaks with PB_ENABLE_MALLOC and certain submessage type combinations. There was a memory leak when: 1) A statically allocated submessage or 2) an extension field submessage contained A) a pointer-type field or B) a submessage that further contained a pointer-type field. This was because pb_release() didn't recurse into non-pointer fields. Update issue 138 Status: FixedInGit --- pb_decode.c | 65 ++++++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index f0fa1cce..367f073b 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -928,6 +928,47 @@ static void pb_release_single_field(const pb_field_iter_t *iter) pb_type_t type; type = iter->pos->type; + /* Release anything contained inside an extension or submsg. + * This has to be done even if the submsg itself is statically + * allocated. */ + if (PB_LTYPE(type) == PB_LTYPE_EXTENSION) + { + /* Release fields from all extensions in the linked list */ + pb_extension_t *ext = *(pb_extension_t**)iter->pData; + while (ext != NULL) + { + pb_field_iter_t ext_iter; + iter_from_extension(&ext_iter, ext); + pb_release_single_field(&ext_iter); + ext = ext->next; + } + } + else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) + { + /* Release fields in submessage or submsg array */ + void *pItem = iter->pData; + pb_size_t count = 1; + + if (PB_ATYPE(type) == PB_ATYPE_POINTER) + { + pItem = *(void**)iter->pData; + } + + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + { + count = *(pb_size_t*)iter->pSize; + } + + if (pItem) + { + while (count--) + { + pb_release((const pb_field_t*)iter->pos->ptr, pItem); + pItem = (uint8_t*)pItem + iter->pos->data_size; + } + } + } + if (PB_ATYPE(type) == PB_ATYPE_POINTER) { if (PB_HTYPE(type) == PB_HTYPE_REPEATED && @@ -942,28 +983,12 @@ static void pb_release_single_field(const pb_field_iter_t *iter) pb_free(*pItem); *pItem++ = NULL; } - *(pb_size_t*)iter->pSize = 0; } - else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) + + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) { - /* Release fields in submessages */ - void *pItem = *(void**)iter->pData; - if (pItem) - { - pb_size_t count = 1; - - if (PB_HTYPE(type) == PB_HTYPE_REPEATED) - { - count = *(pb_size_t*)iter->pSize; - *(pb_size_t*)iter->pSize = 0; - } - - while (count--) - { - pb_release((const pb_field_t*)iter->pos->ptr, pItem); - pItem = (uint8_t*)pItem + iter->pos->data_size; - } - } + /* We are going to release the array, so set the size to 0 */ + *(pb_size_t*)iter->pSize = 0; } /* Release main item */ -- cgit 1.2.3-korg From 7be7c7769f9532b2aca98e271540a565a6a321e6 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 27 Dec 2014 00:37:59 +0200 Subject: Fix build failure due to missing dependency in SConscript --- tests/common/SConscript | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/common/SConscript b/tests/common/SConscript index a202ca47..05e2f852 100644 --- a/tests/common/SConscript +++ b/tests/common/SConscript @@ -41,8 +41,8 @@ malloc_strict.Object("pb_decode_with_malloc.o", "$NANOPB/pb_decode.c") malloc_strict.Object("pb_encode_with_malloc.o", "$NANOPB/pb_encode.c") malloc_strict.Object("pb_common_with_malloc.o", "$NANOPB/pb_common.c") -mw = env.Object("malloc_wrappers.o", "malloc_wrappers.c") -Depends(mw, ["malloc_wrappers_syshdr.h"]) +malloc_env.Object("malloc_wrappers.o", "malloc_wrappers.c") +malloc_env.Depends("$NANOPB/pb.h", ["malloc_wrappers_syshdr.h", "malloc_wrappers.h"]) Export("malloc_env") -- cgit 1.2.3-korg From b0d31468da7f644684be897cef5b0602ca10af0f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 3 Jan 2015 10:59:19 +0200 Subject: Change PB_RETURN_ERROR() macro to avoid compiler warnings. Update issue 140 Status: FixedInGit --- pb.h | 21 ++++++++++----------- tests/SConstruct | 3 --- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/pb.h b/pb.h index 442d9658..5a3340aa 100644 --- a/pb.h +++ b/pb.h @@ -508,22 +508,21 @@ struct pb_extension_s { * is the true/false return value from functions. * Some code space can be saved by disabling the error * messages if not used. + * + * PB_SET_ERROR() sets the error message if none has been set yet. + * msg must be a constant string literal. + * PB_GET_ERROR() always returns a pointer to a string. + * PB_RETURN_ERROR() sets the error and returns false from current + * function. */ #ifdef PB_NO_ERRMSG -#define PB_RETURN_ERROR(stream,msg) \ - do {\ - PB_UNUSED(stream); \ - return false; \ - } while(0) +#define PB_SET_ERROR(stream, msg) PB_UNUSED(stream) #define PB_GET_ERROR(stream) "(errmsg disabled)" #else -#define PB_RETURN_ERROR(stream,msg) \ - do {\ - if ((stream)->errmsg == NULL) \ - (stream)->errmsg = (msg); \ - return false; \ - } while(0) +#define PB_SET_ERROR(stream, msg) (stream->errmsg = (stream)->errmsg ? (stream)->errmsg : (msg)) #define PB_GET_ERROR(stream) ((stream)->errmsg ? (stream)->errmsg : "(none)") #endif +#define PB_RETURN_ERROR(stream, msg) return PB_SET_ERROR(stream, msg), false + #endif diff --git a/tests/SConstruct b/tests/SConstruct index 9092d4f8..9c222daa 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -121,9 +121,6 @@ elif 'cl' in env['CC']: # More strict checks on the nanopb core env.Append(CORECFLAGS = '/W4') - - # PB_RETURN_ERROR triggers C4127 because of while(0) - env.Append(CFLAGS = '/wd4127') elif 'tcc' in env['CC']: # Tiny C Compiler env.Append(CFLAGS = '-Wall -Werror -g') -- cgit 1.2.3-korg From 50c67ecec4895f65ba684e4b46b4b70980a5be6a Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 4 Jan 2015 11:36:42 +0200 Subject: Add int_size option for generator. This allows overriding the integer field types to e.g. uint8_t for saving RAM. Update issue 139 Status: FixedInGit --- docs/reference.rst | 3 + generator/nanopb_generator.py | 44 ++++++++++----- generator/proto/nanopb.proto | 12 ++++ pb_decode.c | 4 ++ pb_encode.c | 6 +- tests/intsizes/SConscript | 12 ++++ tests/intsizes/intsizes.proto | 39 +++++++++++++ tests/intsizes/intsizes_unittests.c | 108 ++++++++++++++++++++++++++++++++++++ 8 files changed, 212 insertions(+), 16 deletions(-) create mode 100644 tests/intsizes/SConscript create mode 100644 tests/intsizes/intsizes.proto create mode 100644 tests/intsizes/intsizes_unittests.c diff --git a/docs/reference.rst b/docs/reference.rst index 8018bdb9..e16500ad 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -76,6 +76,8 @@ The generator behaviour can be adjusted using these options, defined in the max_size Allocated size for *bytes* and *string* fields. max_count Allocated number of entries in arrays (*repeated* fields). +int_size Override the integer type of a field. + (To use e.g. uint8_t to save RAM.) type Type of the generated field. Default value is *FT_DEFAULT*, which selects automatically. You can use *FT_CALLBACK*, *FT_POINTER*, @@ -88,6 +90,7 @@ long_names Prefix the enum name to the enum value in packed_struct Make the generated structures packed. NOTE: This cannot be used on CPUs that break on unaligned accesses to variables. +skip_message Skip the whole message from generation. ============================ ================================================ These options can be defined for the .proto files before they are converted diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 6dcc911f..85cb413b 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -44,22 +44,30 @@ except: import time import os.path -# Values are tuple (c type, pb type, encoded size) +# Values are tuple (c type, pb type, encoded size, int_size_allowed) FieldD = descriptor.FieldDescriptorProto datatypes = { - FieldD.TYPE_BOOL: ('bool', 'BOOL', 1), - FieldD.TYPE_DOUBLE: ('double', 'DOUBLE', 8), - FieldD.TYPE_FIXED32: ('uint32_t', 'FIXED32', 4), - FieldD.TYPE_FIXED64: ('uint64_t', 'FIXED64', 8), - FieldD.TYPE_FLOAT: ('float', 'FLOAT', 4), - FieldD.TYPE_INT32: ('int32_t', 'INT32', 10), - FieldD.TYPE_INT64: ('int64_t', 'INT64', 10), - FieldD.TYPE_SFIXED32: ('int32_t', 'SFIXED32', 4), - FieldD.TYPE_SFIXED64: ('int64_t', 'SFIXED64', 8), - FieldD.TYPE_SINT32: ('int32_t', 'SINT32', 5), - FieldD.TYPE_SINT64: ('int64_t', 'SINT64', 10), - FieldD.TYPE_UINT32: ('uint32_t', 'UINT32', 5), - FieldD.TYPE_UINT64: ('uint64_t', 'UINT64', 10) + FieldD.TYPE_BOOL: ('bool', 'BOOL', 1, False), + FieldD.TYPE_DOUBLE: ('double', 'DOUBLE', 8, False), + FieldD.TYPE_FIXED32: ('uint32_t', 'FIXED32', 4, False), + FieldD.TYPE_FIXED64: ('uint64_t', 'FIXED64', 8, False), + FieldD.TYPE_FLOAT: ('float', 'FLOAT', 4, False), + FieldD.TYPE_INT32: ('int32_t', 'INT32', 10, True), + FieldD.TYPE_INT64: ('int64_t', 'INT64', 10, True), + FieldD.TYPE_SFIXED32: ('int32_t', 'SFIXED32', 4, False), + FieldD.TYPE_SFIXED64: ('int64_t', 'SFIXED64', 8, False), + FieldD.TYPE_SINT32: ('int32_t', 'SINT32', 5, True), + FieldD.TYPE_SINT64: ('int64_t', 'SINT64', 10, True), + FieldD.TYPE_UINT32: ('uint32_t', 'UINT32', 5, True), + FieldD.TYPE_UINT64: ('uint64_t', 'UINT64', 10, True) +} + +# Integer size overrides (from .proto settings) +intsizes = { + nanopb_pb2.IS_8: 'int8_t', + nanopb_pb2.IS_16: 'int16_t', + nanopb_pb2.IS_32: 'int32_t', + nanopb_pb2.IS_64: 'int64_t', } class Names: @@ -226,7 +234,13 @@ class Field: # Decide the C data type to use in the struct. if datatypes.has_key(desc.type): - self.ctype, self.pbtype, self.enc_size = datatypes[desc.type] + self.ctype, self.pbtype, self.enc_size, isa = datatypes[desc.type] + + # Override the field size if user wants to use smaller integers + if isa and field_options.int_size != nanopb_pb2.IS_DEFAULT: + self.ctype = intsizes[field_options.int_size] + if desc.type == FieldD.TYPE_UINT32 or desc.type == FieldD.TYPE_UINT64: + self.ctype = 'u' + self.ctype; elif desc.type == FieldD.TYPE_ENUM: self.pbtype = 'ENUM' self.ctype = names_from_type_name(desc.type_name) diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index 0716be4b..1bde5967 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -18,6 +18,14 @@ enum FieldType { FT_IGNORE = 3; // Ignore the field completely. } +enum IntSize { + IS_DEFAULT = 0; // Default, 32/64bit based on type in .proto + IS_8 = 1; + IS_16 = 2; + IS_32 = 3; + IS_64 = 4; +} + // This is the inner options message, which basically defines options for // a field. When it is used in message or file scope, it applies to all // fields. @@ -28,6 +36,10 @@ message NanoPBOptions { // Allocated number of entries in arrays ('repeated' fields) optional int32 max_count = 2; + // Size of integer fields. Can save some memory if you don't need + // full 32 bits for the value. + optional IntSize int_size = 7 [default = IS_DEFAULT]; + // Force type of field (callback or static allocation) optional FieldType type = 3 [default = FT_DEFAULT]; diff --git a/pb_decode.c b/pb_decode.c index 367f073b..b5ec1ef0 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -1095,6 +1095,8 @@ static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *f switch (field->data_size) { + case 1: *(uint8_t*)dest = (uint8_t)value; break; + case 2: *(uint16_t*)dest = (uint16_t)value; break; case 4: *(uint32_t*)dest = (uint32_t)value; break; case 8: *(uint64_t*)dest = value; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); @@ -1111,6 +1113,8 @@ static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *f switch (field->data_size) { + case 1: *(int8_t*)dest = (int8_t)value; break; + case 2: *(int16_t*)dest = (int16_t)value; break; case 4: *(int32_t*)dest = (int32_t)value; break; case 8: *(int64_t*)dest = value; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); diff --git a/pb_encode.c b/pb_encode.c index 5318361e..cef98861 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -567,7 +567,7 @@ static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *fi int64_t value = 0; /* Cases 1 and 2 are for compilers that have smaller types for bool - * or enums. */ + * or enums, and for int_size option. */ switch (field->data_size) { case 1: value = *(const int8_t*)src; break; @@ -586,6 +586,8 @@ static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *f switch (field->data_size) { + case 1: value = *(const uint8_t*)src; break; + case 2: value = *(const uint16_t*)src; break; case 4: value = *(const uint32_t*)src; break; case 8: value = *(const uint64_t*)src; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); @@ -600,6 +602,8 @@ static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *f switch (field->data_size) { + case 1: value = *(const int8_t*)src; break; + case 2: value = *(const int16_t*)src; break; case 4: value = *(const int32_t*)src; break; case 8: value = *(const int64_t*)src; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); diff --git a/tests/intsizes/SConscript b/tests/intsizes/SConscript new file mode 100644 index 00000000..a90680bc --- /dev/null +++ b/tests/intsizes/SConscript @@ -0,0 +1,12 @@ +# Test that the int_size option in .proto works. + +Import('env') + +env.NanopbProto('intsizes') + +p = env.Program(["intsizes_unittests.c", + "intsizes.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_decode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) diff --git a/tests/intsizes/intsizes.proto b/tests/intsizes/intsizes.proto new file mode 100644 index 00000000..236bf183 --- /dev/null +++ b/tests/intsizes/intsizes.proto @@ -0,0 +1,39 @@ +/* Test the integer size overriding in nanopb options. + * This allows to use 8- and 16-bit integer variables, which are not supported + * directly by Google Protobuf. + * + * The int_size setting will override the number of bits, but keep the type + * otherwise. E.g. uint32 + IS_8 => uint8_t + */ + +import 'nanopb.proto'; + +message IntSizes { + required int32 req_int8 = 1 [(nanopb).int_size = IS_8]; + required uint32 req_uint8 = 2 [(nanopb).int_size = IS_8]; + required sint32 req_sint8 = 3 [(nanopb).int_size = IS_8]; + required int32 req_int16 = 4 [(nanopb).int_size = IS_16]; + required uint32 req_uint16 = 5 [(nanopb).int_size = IS_16]; + required sint32 req_sint16 = 6 [(nanopb).int_size = IS_16]; + required int32 req_int32 = 7 [(nanopb).int_size = IS_32]; + required uint32 req_uint32 = 8 [(nanopb).int_size = IS_32]; + required sint32 req_sint32 = 9 [(nanopb).int_size = IS_32]; + required int32 req_int64 = 10 [(nanopb).int_size = IS_64]; + required uint32 req_uint64 = 11 [(nanopb).int_size = IS_64]; + required sint32 req_sint64 = 12 [(nanopb).int_size = IS_64]; +} + +message DefaultSizes { + required int32 req_int8 = 1 ; + required uint32 req_uint8 = 2 ; + required sint32 req_sint8 = 3 ; + required int32 req_int16 = 4 ; + required uint32 req_uint16 = 5 ; + required sint32 req_sint16 = 6 ; + required int32 req_int32 = 7 ; + required uint32 req_uint32 = 8 ; + required sint32 req_sint32 = 9 ; + required int64 req_int64 = 10; + required uint64 req_uint64 = 11; + required sint64 req_sint64 = 12; +} diff --git a/tests/intsizes/intsizes_unittests.c b/tests/intsizes/intsizes_unittests.c new file mode 100644 index 00000000..29cc7ab0 --- /dev/null +++ b/tests/intsizes/intsizes_unittests.c @@ -0,0 +1,108 @@ +#include +#include +#include +#include +#include "unittests.h" +#include "intsizes.pb.h" + +#define S(x) pb_istream_from_buffer((uint8_t*)x, sizeof(x) - 1) + +/* This is a macro instead of function in order to get the actual values + * into the TEST() lines in output */ +#define TEST_ROUNDTRIP(int8, uint8, sint8, \ + int16, uint16, sint16, \ + int32, uint32, sint32, \ + int64, uint64, sint64, expected_result) \ +{ \ + uint8_t buffer1[128], buffer2[128]; \ + size_t msgsize; \ + DefaultSizes msg1 = DefaultSizes_init_zero; \ + IntSizes msg2 = IntSizes_init_zero; \ + \ + msg1.req_int8 = int8; \ + msg1.req_uint8 = uint8; \ + msg1.req_sint8 = sint8; \ + msg1.req_int16 = int16; \ + msg1.req_uint16 = uint16; \ + msg1.req_sint16 = sint16; \ + msg1.req_int32 = int32; \ + msg1.req_uint32 = uint32; \ + msg1.req_sint32 = sint32; \ + msg1.req_int64 = int64; \ + msg1.req_uint64 = uint64; \ + msg1.req_sint64 = sint64; \ + \ + { \ + pb_ostream_t s = pb_ostream_from_buffer(buffer1, sizeof(buffer1)); \ + TEST(pb_encode(&s, DefaultSizes_fields, &msg1)); \ + msgsize = s.bytes_written; \ + } \ + \ + { \ + pb_istream_t s = pb_istream_from_buffer(buffer1, msgsize); \ + TEST(pb_decode(&s, IntSizes_fields, &msg2) == expected_result); \ + if (expected_result) \ + { \ + TEST(msg2.req_int8 == int8); \ + TEST(msg2.req_uint8 == uint8); \ + TEST(msg2.req_sint8 == sint8); \ + TEST(msg2.req_int16 == int16); \ + TEST(msg2.req_uint16 == uint16); \ + TEST(msg2.req_sint16 == sint16); \ + TEST(msg2.req_int32 == int32); \ + TEST(msg2.req_uint32 == uint32); \ + TEST(msg2.req_sint32 == sint32); \ + TEST(msg2.req_int64 == int64); \ + TEST(msg2.req_uint64 == uint64); \ + TEST(msg2.req_sint64 == sint64); \ + } \ + } \ + \ + if (expected_result) \ + { \ + pb_ostream_t s = pb_ostream_from_buffer(buffer2, sizeof(buffer2)); \ + TEST(pb_encode(&s, IntSizes_fields, &msg2)); \ + TEST(s.bytes_written == msgsize); \ + TEST(memcmp(buffer1, buffer2, msgsize) == 0); \ + } \ +} + +int main() +{ + int status = 0; + + { + IntSizes msg = IntSizes_init_zero; + + COMMENT("Test field sizes"); + TEST(sizeof(msg.req_int8) == 1); + TEST(sizeof(msg.req_uint8) == 1); + TEST(sizeof(msg.req_sint8) == 1); + TEST(sizeof(msg.req_int16) == 2); + TEST(sizeof(msg.req_uint16) == 2); + TEST(sizeof(msg.req_sint16) == 2); + TEST(sizeof(msg.req_int32) == 4); + TEST(sizeof(msg.req_uint32) == 4); + TEST(sizeof(msg.req_sint32) == 4); + TEST(sizeof(msg.req_int64) == 8); + TEST(sizeof(msg.req_uint64) == 8); + TEST(sizeof(msg.req_sint64) == 8); + } + + COMMENT("Test roundtrip at maximum value"); + TEST_ROUNDTRIP(127, 255, 127, + 32767, 65535, 32767, + INT32_MAX, UINT32_MAX, INT32_MAX, + INT64_MAX, UINT64_MAX, INT64_MAX, true); + + COMMENT("Test roundtrip at minimum value"); + TEST_ROUNDTRIP(-128, 0, -128, + -32768, 0, -32768, + INT32_MIN, 0, INT32_MIN, + INT64_MIN, 0, INT64_MIN, true); + + if (status != 0) + fprintf(stdout, "\n\nSome tests FAILED!\n"); + + return status; +} \ No newline at end of file -- cgit 1.2.3-korg From a0f0440394ac3b38105dfad09366f95011c5d8d3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 4 Jan 2015 12:04:24 +0200 Subject: Detect too large varint values when decoding. Because Issue #139 now allows limiting integer fields, it is good to check the values received from other protobuf libraries against the lower limits. --- pb_decode.c | 50 ++++++++++++++++++++++--------- tests/decode_unittests/decode_unittests.c | 6 ++-- tests/intsizes/intsizes_unittests.c | 14 +++++++++ 3 files changed, 53 insertions(+), 17 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index b5ec1ef0..5982c8e5 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -1072,53 +1072,75 @@ bool pb_decode_fixed64(pb_istream_t *stream, void *dest) static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint64_t value; + int64_t svalue; + int64_t clamped; if (!pb_decode_varint(stream, &value)) return false; + /* See issue 97: Google's C++ protobuf allows negative varint values to + * be cast as int32_t, instead of the int64_t that should be used when + * encoding. Previous nanopb versions had a bug in encoding. In order to + * not break decoding of such messages, we cast <=32 bit fields to + * int32_t first to get the sign correct. + */ + if (field->data_size == 8) + svalue = (int64_t)value; + else + svalue = (int32_t)value; + switch (field->data_size) { - case 1: *(int8_t*)dest = (int8_t)value; break; - case 2: *(int16_t*)dest = (int16_t)value; break; - case 4: *(int32_t*)dest = (int32_t)value; break; - case 8: *(int64_t*)dest = (int64_t)value; break; + case 1: clamped = *(int8_t*)dest = (int8_t)svalue; break; + case 2: clamped = *(int16_t*)dest = (int16_t)svalue; break; + case 4: clamped = *(int32_t*)dest = (int32_t)svalue; break; + case 8: clamped = *(int64_t*)dest = svalue; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); } + + if (clamped != svalue) + PB_RETURN_ERROR(stream, "integer too large"); return true; } static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, void *dest) { - uint64_t value; + uint64_t value, clamped; if (!pb_decode_varint(stream, &value)) return false; switch (field->data_size) { - case 1: *(uint8_t*)dest = (uint8_t)value; break; - case 2: *(uint16_t*)dest = (uint16_t)value; break; - case 4: *(uint32_t*)dest = (uint32_t)value; break; - case 8: *(uint64_t*)dest = value; break; + case 1: clamped = *(uint8_t*)dest = (uint8_t)value; break; + case 2: clamped = *(uint16_t*)dest = (uint16_t)value; break; + case 4: clamped = *(uint32_t*)dest = (uint32_t)value; break; + case 8: clamped = *(uint64_t*)dest = value; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); } + if (clamped != value) + PB_RETURN_ERROR(stream, "integer too large"); + return true; } static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) { - int64_t value; + int64_t value, clamped; if (!pb_decode_svarint(stream, &value)) return false; switch (field->data_size) { - case 1: *(int8_t*)dest = (int8_t)value; break; - case 2: *(int16_t*)dest = (int16_t)value; break; - case 4: *(int32_t*)dest = (int32_t)value; break; - case 8: *(int64_t*)dest = value; break; + case 1: clamped = *(int8_t*)dest = (int8_t)value; break; + case 2: clamped = *(int16_t*)dest = (int16_t)value; break; + case 4: clamped = *(int32_t*)dest = (int32_t)value; break; + case 8: clamped = *(int64_t*)dest = value; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); } + + if (clamped != value) + PB_RETURN_ERROR(stream, "integer too large"); return true; } diff --git a/tests/decode_unittests/decode_unittests.c b/tests/decode_unittests/decode_unittests.c index 8c12f1cd..47f0fbdb 100644 --- a/tests/decode_unittests/decode_unittests.c +++ b/tests/decode_unittests/decode_unittests.c @@ -123,16 +123,16 @@ int main() } { - pb_istream_t s = S("\x01\xFF\xFF\x03"); + pb_istream_t s = S("\x01\x00"); pb_field_t f = {1, PB_LTYPE_VARINT, 0, 0, 4, 0, 0}; uint32_t d; COMMENT("Test pb_dec_varint using uint32_t") TEST(pb_dec_varint(&s, &f, &d) && d == 1) /* Verify that no more than data_size is written. */ - d = 0; + d = 0xFFFFFFFF; f.data_size = 1; - TEST(pb_dec_varint(&s, &f, &d) && (d == 0xFF || d == 0xFF000000)) + TEST(pb_dec_varint(&s, &f, &d) && (d == 0xFFFFFF00 || d == 0x00FFFFFF)) } { diff --git a/tests/intsizes/intsizes_unittests.c b/tests/intsizes/intsizes_unittests.c index 29cc7ab0..189825f7 100644 --- a/tests/intsizes/intsizes_unittests.c +++ b/tests/intsizes/intsizes_unittests.c @@ -101,6 +101,20 @@ int main() INT32_MIN, 0, INT32_MIN, INT64_MIN, 0, INT64_MIN, true); + COMMENT("Test overflow detection"); + TEST_ROUNDTRIP(-129, 0, -128, + -32768, 0, -32768, + INT32_MIN, 0, INT32_MIN, + INT64_MIN, 0, INT64_MIN, false); + TEST_ROUNDTRIP(127, 256, 127, + 32767, 65535, 32767, + INT32_MAX, UINT32_MAX, INT32_MAX, + INT64_MAX, UINT64_MAX, INT64_MAX, false); + TEST_ROUNDTRIP(-128, 0, -128, + -32768, 0, -32769, + INT32_MIN, 0, INT32_MIN, + INT64_MIN, 0, INT64_MIN, false); + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); -- cgit 1.2.3-korg From 7713d43bc3d448358a04393c4e44dd12a768bdea Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 4 Jan 2015 19:39:37 +0200 Subject: Implement support for oneofs (C unions). Basic test included, should probably add an oneof to the AllTypes test also. Update issue 131 Status: Started --- generator/nanopb_generator.py | 264 ++++++++++++++++++++++++++++++------------ pb.h | 18 +++ pb_common.c | 7 ++ pb_decode.c | 12 +- pb_encode.c | 11 ++ tests/oneof/SConscript | 22 ++++ tests/oneof/decode_oneof.c | 72 ++++++++++++ tests/oneof/encode_oneof.c | 64 ++++++++++ tests/oneof/oneof.proto | 18 +++ 9 files changed, 415 insertions(+), 73 deletions(-) create mode 100644 tests/oneof/SConscript create mode 100644 tests/oneof/decode_oneof.c create mode 100644 tests/oneof/encode_oneof.c create mode 100644 tests/oneof/oneof.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 85cb413b..60944038 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -171,6 +171,7 @@ class Field: '''desc is FieldDescriptorProto''' self.tag = desc.number self.struct_name = struct_name + self.union_name = None self.name = desc.name self.default = None self.max_size = None @@ -300,57 +301,91 @@ class Field: if self.pbtype == 'BYTES' and self.allocation == 'STATIC': result = 'typedef PB_BYTES_ARRAY_T(%d) %s;\n' % (self.max_size, self.ctype) else: - result = None + result = '' return result - def get_initializer(self, null_init): - '''Return literal expression for this field's default value.''' - + def get_dependencies(self): + '''Get list of type names used by this field.''' + if self.allocation == 'STATIC': + return [str(self.ctype)] + else: + return [] + + def get_initializer(self, null_init, inner_init_only = False): + '''Return literal expression for this field's default value. + null_init: If True, initialize to a 0 value instead of default from .proto + inner_init_only: If True, exclude initialization for any count/has fields + ''' + + inner_init = None if self.pbtype == 'MESSAGE': if null_init: - return '%s_init_zero' % self.ctype + inner_init = '%s_init_zero' % self.ctype else: - return '%s_init_default' % self.ctype - - if self.default is None or null_init: + inner_init = '%s_init_default' % self.ctype + elif self.default is None or null_init: if self.pbtype == 'STRING': - return '""' + inner_init = '""' elif self.pbtype == 'BYTES': - return '{0, {0}}' + inner_init = '{0, {0}}' elif self.pbtype == 'ENUM': - return '(%s)0' % self.ctype + inner_init = '(%s)0' % self.ctype else: - return '0' - - default = str(self.default) - - if self.pbtype == 'STRING': - default = default.encode('utf-8').encode('string_escape') - default = default.replace('"', '\\"') - default = '"' + default + '"' - elif self.pbtype == 'BYTES': - data = default.decode('string_escape') - data = ['0x%02x' % ord(c) for c in data] - if len(data) == 0: - default = '{0, {0}}' + inner_init = '0' + else: + if self.pbtype == 'STRING': + inner_init = self.default.encode('utf-8').encode('string_escape') + inner_init = inner_init.replace('"', '\\"') + inner_init = '"' + inner_init + '"' + elif self.pbtype == 'BYTES': + data = str(self.default).decode('string_escape') + data = ['0x%02x' % ord(c) for c in data] + if len(data) == 0: + inner_init = '{0, {0}}' + else: + inner_init = '{%d, {%s}}' % (len(data), ','.join(data)) + elif self.pbtype in ['FIXED32', 'UINT32']: + inner_init = str(self.default) + 'u' + elif self.pbtype in ['FIXED64', 'UINT64']: + inner_init = str(self.default) + 'ull' + elif self.pbtype in ['SFIXED64', 'INT64']: + inner_init = str(self.default) + 'll' else: - default = '{%d, {%s}}' % (len(data), ','.join(data)) - elif self.pbtype in ['FIXED32', 'UINT32']: - default += 'u' - elif self.pbtype in ['FIXED64', 'UINT64']: - default += 'ull' - elif self.pbtype in ['SFIXED64', 'INT64']: - default += 'll' + inner_init = str(self.default) - return default - + if inner_init_only: + return inner_init + + outer_init = None + if self.allocation == 'STATIC': + if self.rules == 'REPEATED': + outer_init = '0, {' + outer_init += ', '.join([inner_init] * self.max_count) + outer_init += '}' + elif self.rules == 'OPTIONAL': + outer_init = 'false, ' + inner_init + else: + outer_init = inner_init + elif self.allocation == 'POINTER': + if self.rules == 'REPEATED': + outer_init = '0, NULL' + else: + outer_init = 'NULL' + elif self.allocation == 'CALLBACK': + if self.pbtype == 'EXTENSION': + outer_init = 'NULL' + else: + outer_init = '{{NULL}, NULL}' + + return outer_init + def default_decl(self, declaration_only = False): '''Return definition for this field's default value.''' if self.default is None: return None ctype = self.ctype - default = self.get_initializer(False) + default = self.get_initializer(False, True) array_decl = '' if self.pbtype == 'STRING': @@ -375,7 +410,13 @@ class Field: '''Return the pb_field_t initializer to use in the constant array. prev_field_name is the name of the previous field or None. ''' - result = ' PB_FIELD(%3d, ' % self.tag + + if self.rules == 'ONEOF': + result = ' PB_ONEOF_FIELD(%s, ' % self.union_name + else: + result = ' PB_FIELD(' + + result += '%3d, ' % self.tag result += '%-8s, ' % self.pbtype result += '%s, ' % self.rules result += '%-8s, ' % self.allocation @@ -403,6 +444,8 @@ class Field: if self.pbtype == 'MESSAGE': if self.rules == 'REPEATED' and self.allocation == 'STATIC': return 'pb_membersize(%s, %s[0])' % (self.struct_name, self.name) + elif self.rules == 'ONEOF': + return 'pb_membersize(%s, %s.%s)' % (self.struct_name, self.union_name, self.name) else: return 'pb_membersize(%s, %s)' % (self.struct_name, self.name) @@ -534,6 +577,71 @@ class ExtensionField(Field): return result +# --------------------------------------------------------------------------- +# Generation of oneofs (unions) +# --------------------------------------------------------------------------- + +class OneOf(Field): + def __init__(self, oneof_desc): + self.name = oneof_desc.name + self.ctype = 'union' + self.fields = [] + + def add_field(self, field): + if field.allocation == 'CALLBACK': + raise Exception("Callback fields inside of oneof are not supported" + + " (field %s)" % field.fullname) + + field.union_name = self.name + field.rules = 'ONEOF' + self.fields.append(field) + self.fields.sort(key = lambda f: f.tag) + + # Sort by the lowest tag number inside union + self.tag = min([f.tag for f in self.fields]) + + def __cmp__(self, other): + return cmp(self.tag, other.tag) + + def __str__(self): + result = '' + if self.fields: + result += ' pb_size_t which_' + self.name + ";\n" + result += ' union {\n' + for f in self.fields: + result += ' ' + str(f).replace('\n', '\n ') + '\n' + result += ' } ' + self.name + ';' + return result + + def types(self): + return ''.join([f.types() for f in self.fields]) + + def get_dependencies(self): + deps = [] + for f in self.fields: + deps += f.get_dependencies() + return deps + + def get_initializer(self, null_init): + return '0, {' + self.fields[0].get_initializer(null_init) + '}' + + def default_decl(self, declaration_only = False): + return None + + def tags(self): + return '\n'.join([f.tags() for f in self.fields]) + + def pb_field_t(self, prev_field_name): + prev_field_name = prev_field_name or self.name + result = ',\n'.join([f.pb_field_t(prev_field_name) for f in self.fields]) + return result + + def largest_field_value(self): + return max([f.largest_field_value() for f in self.fields]) + + def encoded_size(self, allmsgs): + return max([f.encoded_size(allmsgs) for f in self.fields]) + # --------------------------------------------------------------------------- # Generation of messages (structures) # --------------------------------------------------------------------------- @@ -543,11 +651,24 @@ class Message: def __init__(self, names, desc, message_options): self.name = names self.fields = [] - + self.oneofs = [] + + if hasattr(desc, 'oneof_decl'): + for f in desc.oneof_decl: + oneof = OneOf(f) + self.oneofs.append(oneof) + self.fields.append(oneof) + for f in desc.field: field_options = get_nanopb_suboptions(f, message_options, self.name + f.name) - if field_options.type != nanopb_pb2.FT_IGNORE: - self.fields.append(Field(self.name, f, field_options)) + if field_options.type == nanopb_pb2.FT_IGNORE: + continue + + field = Field(self.name, f, field_options) + if hasattr(f, 'oneof_index') and f.HasField('oneof_index'): + self.oneofs[f.oneof_index].add_field(field) + else: + self.fields.append(field) if len(desc.extension_range) > 0: field_options = get_nanopb_suboptions(desc, message_options, self.name + 'extensions') @@ -561,7 +682,10 @@ class Message: def get_dependencies(self): '''Get list of type names that this structure refers to.''' - return [str(field.ctype) for field in self.fields if field.allocation == 'STATIC'] + deps = [] + for f in self.fields: + deps += f.get_dependencies() + return deps def __str__(self): result = 'typedef struct _%s {\n' % self.name @@ -586,39 +710,15 @@ class Message: return result def types(self): - result = "" - for field in self.fields: - types = field.types() - if types is not None: - result += types + '\n' - return result - + return ''.join([f.types() for f in self.fields]) + def get_initializer(self, null_init): if not self.ordered_fields: return '{0}' parts = [] for field in self.ordered_fields: - if field.allocation == 'STATIC': - if field.rules == 'REPEATED': - parts.append('0') - parts.append('{' - + ', '.join([field.get_initializer(null_init)] * field.max_count) - + '}') - elif field.rules == 'OPTIONAL': - parts.append('false') - parts.append(field.get_initializer(null_init)) - else: - parts.append(field.get_initializer(null_init)) - elif field.allocation == 'POINTER': - if field.rules == 'REPEATED': - parts.append('0') - parts.append('NULL') - elif field.allocation == 'CALLBACK': - if field.pbtype == 'EXTENSION': - parts.append('NULL') - else: - parts.append('{{NULL}, NULL}') + parts.append(field.get_initializer(null_init)) return '{' + ', '.join(parts) + '}' def default_decl(self, declaration_only = False): @@ -629,18 +729,39 @@ class Message: result += default + '\n' return result + def count_required_fields(self): + '''Returns number of required fields inside this message''' + count = 0 + for f in self.fields: + if f not in self.oneofs: + if f.rules == 'REQUIRED': + count += 1 + return count + + def count_all_fields(self): + count = 0 + for f in self.fields: + if f in self.oneofs: + count += len(f.fields) + else: + count += 1 + return count + def fields_declaration(self): - result = 'extern const pb_field_t %s_fields[%d];' % (self.name, len(self.fields) + 1) + result = 'extern const pb_field_t %s_fields[%d];' % (self.name, self.count_all_fields() + 1) return result def fields_definition(self): - result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, len(self.fields) + 1) + result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, self.count_all_fields() + 1) prev = None for field in self.ordered_fields: result += field.pb_field_t(prev) result += ',\n' - prev = field.name + if isinstance(field, OneOf): + prev = field.name + '.' + field.fields[-1].name + else: + prev = field.name result += ' PB_LAST_FIELD\n};' return result @@ -894,9 +1015,8 @@ def generate_source(headername, enums, messages, extensions, options): # Add checks for numeric limits if messages: - count_required_fields = lambda m: len([f for f in msg.fields if f.rules == 'REQUIRED']) - largest_msg = max(messages, key = count_required_fields) - largest_count = count_required_fields(largest_msg) + largest_msg = max(messages, key = lambda m: m.count_required_fields()) + largest_count = largest_msg.count_required_fields() if largest_count > 64: yield '\n/* Check that missing required fields will be properly detected */\n' yield '#if PB_MAX_REQUIRED_FIELDS < %d\n' % largest_count diff --git a/pb.h b/pb.h index 5a3340aa..af58dca3 100644 --- a/pb.h +++ b/pb.h @@ -183,6 +183,7 @@ typedef uint8_t pb_type_t; #define PB_HTYPE_REQUIRED 0x00 #define PB_HTYPE_OPTIONAL 0x10 #define PB_HTYPE_REPEATED 0x20 +#define PB_HTYPE_ONEOF 0x30 #define PB_HTYPE_MASK 0x30 /**** Field allocation types ****/ @@ -502,6 +503,23 @@ struct pb_extension_s { PB_DATAOFFSET_ ## placement(message, field, prevfield), \ PB_LTYPE_MAP_ ## type, ptr) +/* Field description for oneof fields. This requires taking into account the + * union name also, that's why a separate set of macros is needed. + */ +#define PB_ONEOF_STATIC(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, u.m), \ + pb_membersize(st, u.m), 0, ptr} + +#define PB_ONEOF_POINTER(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, u.m), \ + pb_membersize(st, u.m), 0, ptr} + +#define PB_ONEOF_FIELD(union_name, tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ + PB_ ## rules ## _ ## allocation(union_name, tag, message, field, \ + PB_DATAOFFSET_ ## placement(message, union_name.field, prevfield), \ + PB_LTYPE_MAP_ ## type, ptr) /* These macros are used for giving out error messages. * They are mostly a debugging aid; the main error information diff --git a/pb_common.c b/pb_common.c index a9cade63..98964850 100644 --- a/pb_common.c +++ b/pb_common.c @@ -54,6 +54,13 @@ bool pb_field_iter_next(pb_field_iter_t *iter) * The data_size only applies to the dynamically allocated area. */ prev_size = sizeof(void*); } + else if (PB_HTYPE(prev_field->type) == PB_HTYPE_ONEOF && + PB_HTYPE(iter->pos->type) == PB_HTYPE_ONEOF) + { + /* Don't advance pointers inside unions */ + prev_size = 0; + iter->pData = (char*)iter->pData - prev_field->data_offset; + } if (PB_HTYPE(prev_field->type) == PB_HTYPE_REQUIRED) { diff --git a/pb_decode.c b/pb_decode.c index 5982c8e5..542fdc4c 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -393,6 +393,10 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t return func(stream, iter->pos, pItem); } + case PB_HTYPE_ONEOF: + *(pb_size_t*)iter->pSize = iter->pos->tag; + return func(stream, iter->pos, iter->pData); + default: PB_RETURN_ERROR(stream, "invalid field type"); } @@ -470,6 +474,7 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ { case PB_HTYPE_REQUIRED: case PB_HTYPE_OPTIONAL: + case PB_HTYPE_ONEOF: if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE && *(void**)iter->pData != NULL) { @@ -477,6 +482,11 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ pb_release_single_field(iter); } + if (PB_HTYPE(type) == PB_HTYPE_ONEOF) + { + *(pb_size_t*)iter->pSize = iter->pos->tag; + } + if (PB_LTYPE(type) == PB_LTYPE_STRING || PB_LTYPE(type) == PB_LTYPE_BYTES) { @@ -562,7 +572,7 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ initialize_pointer_field(pItem, iter); return func(stream, iter->pos, pItem); } - + default: PB_RETURN_ERROR(stream, "invalid field type"); } diff --git a/pb_encode.c b/pb_encode.c index cef98861..cc372b8f 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -250,6 +250,17 @@ static bool checkreturn encode_basic_field(pb_ostream_t *stream, return false; break; + case PB_HTYPE_ONEOF: + if (*(const pb_size_t*)pSize == field->tag) + { + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!func(stream, field, pData)) + return false; + } + break; + default: PB_RETURN_ERROR(stream, "invalid field type"); } diff --git a/tests/oneof/SConscript b/tests/oneof/SConscript new file mode 100644 index 00000000..19845278 --- /dev/null +++ b/tests/oneof/SConscript @@ -0,0 +1,22 @@ +# Test the 'oneof' feature for generating C unions. + +Import('env') + +env.NanopbProto('oneof') + +enc = env.Program(['encode_oneof.c', + 'oneof.pb.c', + '$COMMON/pb_encode.o', + '$COMMON/pb_common.o']) + +dec = env.Program(['decode_oneof.c', + 'oneof.pb.c', + '$COMMON/pb_decode.o', + '$COMMON/pb_common.o']) + +env.RunTest("message1.pb", enc, ARGS = ['1']) +env.RunTest("message1.txt", [dec, 'message1.pb'], ARGS = ['1']) +env.RunTest("message2.pb", enc, ARGS = ['2']) +env.RunTest("message2.txt", [dec, 'message2.pb'], ARGS = ['2']) +env.RunTest("message3.pb", enc, ARGS = ['3']) +env.RunTest("message3.txt", [dec, 'message3.pb'], ARGS = ['3']) diff --git a/tests/oneof/decode_oneof.c b/tests/oneof/decode_oneof.c new file mode 100644 index 00000000..e94becc7 --- /dev/null +++ b/tests/oneof/decode_oneof.c @@ -0,0 +1,72 @@ +/* Decode a message using oneof fields */ + +#include +#include +#include +#include "oneof.pb.h" +#include "test_helpers.h" +#include "unittests.h" + +int main(int argc, char **argv) +{ + uint8_t buffer[OneOfMessage_size]; + OneOfMessage msg = OneOfMessage_init_zero; + pb_istream_t stream; + size_t count; + int option; + + if (argc != 2) + { + fprintf(stderr, "Usage: encode_oneof [number]\n"); + return 1; + } + option = atoi(argv[1]); + + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + if (!feof(stdin)) + { + printf("Message does not fit in buffer\n"); + return 1; + } + + stream = pb_istream_from_buffer(buffer, count); + + if (!pb_decode(&stream, OneOfMessage_fields, &msg)) + { + printf("Decoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + + { + int status = 0; + + /* Check that the basic fields work normally */ + TEST(msg.prefix == 123); + TEST(msg.suffix == 321); + + /* Check that we got the right oneof according to command line */ + if (option == 1) + { + TEST(msg.which_values == OneOfMessage_first_tag); + TEST(msg.values.first == 999); + } + else if (option == 2) + { + TEST(msg.which_values == OneOfMessage_second_tag); + TEST(strcmp(msg.values.second, "abcd") == 0); + } + else if (option == 3) + { + TEST(msg.which_values == OneOfMessage_third_tag); + TEST(msg.values.third.array[0] == 1); + TEST(msg.values.third.array[1] == 2); + TEST(msg.values.third.array[2] == 3); + TEST(msg.values.third.array[3] == 4); + TEST(msg.values.third.array[4] == 5); + } + + return status; + } +} \ No newline at end of file diff --git a/tests/oneof/encode_oneof.c b/tests/oneof/encode_oneof.c new file mode 100644 index 00000000..913d2d43 --- /dev/null +++ b/tests/oneof/encode_oneof.c @@ -0,0 +1,64 @@ +/* Encode a message using oneof fields */ + +#include +#include +#include +#include "oneof.pb.h" +#include "test_helpers.h" + +int main(int argc, char **argv) +{ + uint8_t buffer[OneOfMessage_size]; + OneOfMessage msg = OneOfMessage_init_zero; + pb_ostream_t stream; + int option; + + if (argc != 2) + { + fprintf(stderr, "Usage: encode_oneof [number]\n"); + return 1; + } + option = atoi(argv[1]); + + /* Prefix and suffix are used to test that the union does not disturb + * other fields in the same message. */ + msg.prefix = 123; + + /* We encode one of the 'values' fields based on command line argument */ + if (option == 1) + { + msg.which_values = OneOfMessage_first_tag; + msg.values.first = 999; + } + else if (option == 2) + { + msg.which_values = OneOfMessage_second_tag; + strcpy(msg.values.second, "abcd"); + } + else if (option == 3) + { + msg.which_values = OneOfMessage_third_tag; + msg.values.third.array_count = 5; + msg.values.third.array[0] = 1; + msg.values.third.array[1] = 2; + msg.values.third.array[2] = 3; + msg.values.third.array[3] = 4; + msg.values.third.array[4] = 5; + } + + msg.suffix = 321; + + stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + if (pb_encode(&stream, OneOfMessage_fields, &msg)) + { + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } +} diff --git a/tests/oneof/oneof.proto b/tests/oneof/oneof.proto new file mode 100644 index 00000000..a89ef131 --- /dev/null +++ b/tests/oneof/oneof.proto @@ -0,0 +1,18 @@ +import 'nanopb.proto'; + +message SubMessage +{ + repeated int32 array = 1 [(nanopb).max_count = 8]; +} + +message OneOfMessage +{ + required int32 prefix = 1; + oneof values + { + int32 first = 5; + string second = 6 [(nanopb).max_size = 8]; + SubMessage third = 7; + } + required int32 suffix = 99; +} -- cgit 1.2.3-korg From 77a71ceb6d8da3995c72bf8cb0656254c77deb8d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 4 Jan 2015 19:48:09 +0200 Subject: Fix build failure --- generator/nanopb_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 60944038..2a9ae008 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -520,7 +520,7 @@ class ExtensionRange(Field): return ' pb_extension_t *extensions;' def types(self): - return None + return '' def tags(self): return '' -- cgit 1.2.3-korg From 7135e2797a80f224541080117313088273f02cad Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 4 Jan 2015 20:00:37 +0200 Subject: Only run oneof test when protoc >= 2.6 is available --- tests/SConstruct | 7 +++++++ tests/oneof/SConscript | 35 ++++++++++++++++++++--------------- 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/tests/SConstruct b/tests/SConstruct index 9c222daa..61ce0c37 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -72,6 +72,13 @@ if not env.GetOption('clean'): else: conf.env.Append(PROTOCPATH = '/usr/include') + # Check protoc version + status, output = conf.TryAction('protoc --version') + if status: + conf.env.Append(PROTOC_VERSION = output) + else: + conf.env.Append(PROTOC_VERSION = "") + # Check if libmudflap is available (only with GCC) if 'gcc' in env['CC']: if conf.CheckLib('mudflap'): diff --git a/tests/oneof/SConscript b/tests/oneof/SConscript index 19845278..95f451cb 100644 --- a/tests/oneof/SConscript +++ b/tests/oneof/SConscript @@ -2,21 +2,26 @@ Import('env') -env.NanopbProto('oneof') +import re +version = re.search('([0-9]+).([0-9]+).([0-9]+)', 'libprotoc 3.0.0').groups() -enc = env.Program(['encode_oneof.c', - 'oneof.pb.c', - '$COMMON/pb_encode.o', - '$COMMON/pb_common.o']) +# Oneof is supported by protoc >= 2.6.0 +if int(version[0]) > 2 or (int(version[0]) == 2 and int(version[1]) >= 6): + env.NanopbProto('oneof') -dec = env.Program(['decode_oneof.c', - 'oneof.pb.c', - '$COMMON/pb_decode.o', - '$COMMON/pb_common.o']) + enc = env.Program(['encode_oneof.c', + 'oneof.pb.c', + '$COMMON/pb_encode.o', + '$COMMON/pb_common.o']) -env.RunTest("message1.pb", enc, ARGS = ['1']) -env.RunTest("message1.txt", [dec, 'message1.pb'], ARGS = ['1']) -env.RunTest("message2.pb", enc, ARGS = ['2']) -env.RunTest("message2.txt", [dec, 'message2.pb'], ARGS = ['2']) -env.RunTest("message3.pb", enc, ARGS = ['3']) -env.RunTest("message3.txt", [dec, 'message3.pb'], ARGS = ['3']) + dec = env.Program(['decode_oneof.c', + 'oneof.pb.c', + '$COMMON/pb_decode.o', + '$COMMON/pb_common.o']) + + env.RunTest("message1.pb", enc, ARGS = ['1']) + env.RunTest("message1.txt", [dec, 'message1.pb'], ARGS = ['1']) + env.RunTest("message2.pb", enc, ARGS = ['2']) + env.RunTest("message2.txt", [dec, 'message2.pb'], ARGS = ['2']) + env.RunTest("message3.pb", enc, ARGS = ['3']) + env.RunTest("message3.txt", [dec, 'message3.pb'], ARGS = ['3']) -- cgit 1.2.3-korg From cc3c8732fd3f8505f46802dd54bd4a21c79b72aa Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 4 Jan 2015 20:20:40 +0200 Subject: Actually make the protoc version check work --- tests/SConstruct | 6 ++---- tests/oneof/SConscript | 10 ++++++++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/SConstruct b/tests/SConstruct index 61ce0c37..d8ab9ab0 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -73,11 +73,9 @@ if not env.GetOption('clean'): conf.env.Append(PROTOCPATH = '/usr/include') # Check protoc version - status, output = conf.TryAction('protoc --version') + status, output = conf.TryAction('$PROTOC --version > $TARGET') if status: - conf.env.Append(PROTOC_VERSION = output) - else: - conf.env.Append(PROTOC_VERSION = "") + conf.env['PROTOC_VERSION'] = output # Check if libmudflap is available (only with GCC) if 'gcc' in env['CC']: diff --git a/tests/oneof/SConscript b/tests/oneof/SConscript index 95f451cb..22634fb0 100644 --- a/tests/oneof/SConscript +++ b/tests/oneof/SConscript @@ -3,10 +3,16 @@ Import('env') import re -version = re.search('([0-9]+).([0-9]+).([0-9]+)', 'libprotoc 3.0.0').groups() + +match = None +if 'PROTOC_VERSION' in env: + match = re.search('([0-9]+).([0-9]+).([0-9]+)', env['PROTOC_VERSION']) + +if match: + version = map(int, match.groups()) # Oneof is supported by protoc >= 2.6.0 -if int(version[0]) > 2 or (int(version[0]) == 2 and int(version[1]) >= 6): +if env.GetOption('clean') or (match and (version[0] > 2 or (version[0] == 2 and version[1] >= 6))): env.NanopbProto('oneof') enc = env.Program(['encode_oneof.c', -- cgit 1.2.3-korg From fa444be4243b2e1f0f6ee22798ddbf0c85e54422 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 5 Jan 2015 22:32:34 +0200 Subject: Allow using 8/16/32/64 as values in int_size setting --- generator/proto/nanopb.proto | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index 1bde5967..e830ec2c 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -20,10 +20,10 @@ enum FieldType { enum IntSize { IS_DEFAULT = 0; // Default, 32/64bit based on type in .proto - IS_8 = 1; - IS_16 = 2; - IS_32 = 3; - IS_64 = 4; + IS_8 = 8; + IS_16 = 16; + IS_32 = 32; + IS_64 = 64; } // This is the inner options message, which basically defines options for -- cgit 1.2.3-korg From 8ef0392231c071a92ae78113763c0b1d61eafb1f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 7 Jan 2015 18:59:44 +0200 Subject: Fix generator error with OneOfs --- generator/nanopb_generator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 2a9ae008..3f309a4d 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -582,7 +582,8 @@ class ExtensionField(Field): # --------------------------------------------------------------------------- class OneOf(Field): - def __init__(self, oneof_desc): + def __init__(self, struct_name, oneof_desc): + self.struct_name = struct_name self.name = oneof_desc.name self.ctype = 'union' self.fields = [] @@ -655,7 +656,7 @@ class Message: if hasattr(desc, 'oneof_decl'): for f in desc.oneof_decl: - oneof = OneOf(f) + oneof = OneOf(self.name, f) self.oneofs.append(oneof) self.fields.append(oneof) -- cgit 1.2.3-korg From 8d12fecc7e4fb6899eb0d013abe0d6a5e03447ce Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 11 Jan 2015 19:38:05 +0200 Subject: New generator options for oneofs: allow skipping or generating as normal 'optional' fields. The behaviour with no_unions:true is the same as of nanopb 0.3.1 and earlier. --- docs/migration.rst | 19 +++++++++++++++++++ generator/nanopb_generator.py | 30 ++++++++++++++++++++---------- generator/proto/nanopb.proto | 3 +++ 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/docs/migration.rst b/docs/migration.rst index 5f7246f5..d5ded643 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -11,6 +11,25 @@ are included, in order to make it easier to find this document. .. contents :: +Nanopb-0.3.2 (2015-01-xx) +========================= + +Add support for OneOfs +---------------------- +**Rationale:** Previously nanopb did not support the *oneof* construct in +*.proto* files. Those fields were generated as regular *optional* fields. + +**Changes:** OneOfs are now generated as C unions. Callback fields are not +supported inside oneof and generator gives an error. + +**Required actions:** The generator option *no_unions* can be used to restore old +behaviour and to allow callbacks to be used. To use unions, one change is +needed: use *which_xxxx* field to detect which field is present, instead +of *has_xxxx*. Compare the value against *MyStruct_myfield_tag*. + +**Error indications:** Generator error: "Callback fields inside of oneof are +not supported". Compiler error: "Message" has no member named "has_xxxx". + Nanopb-0.3.0 (2014-08-26) ========================= diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 3f309a4d..0d7be492 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -591,7 +591,7 @@ class OneOf(Field): def add_field(self, field): if field.allocation == 'CALLBACK': raise Exception("Callback fields inside of oneof are not supported" - + " (field %s)" % field.fullname) + + " (field %s)" % field.name) field.union_name = self.name field.rules = 'ONEOF' @@ -652,13 +652,20 @@ class Message: def __init__(self, names, desc, message_options): self.name = names self.fields = [] - self.oneofs = [] + self.oneofs = {} + no_unions = [] if hasattr(desc, 'oneof_decl'): - for f in desc.oneof_decl: - oneof = OneOf(self.name, f) - self.oneofs.append(oneof) - self.fields.append(oneof) + for i, f in enumerate(desc.oneof_decl): + oneof_options = get_nanopb_suboptions(desc, message_options, self.name + f.name) + if oneof_options.no_unions: + no_unions.append(i) # No union, but add fields normally + elif oneof_options.type == nanopb_pb2.FT_IGNORE: + pass # No union and skip fields also + else: + oneof = OneOf(self.name, f) + self.oneofs[i] = oneof + self.fields.append(oneof) for f in desc.field: field_options = get_nanopb_suboptions(f, message_options, self.name + f.name) @@ -666,8 +673,11 @@ class Message: continue field = Field(self.name, f, field_options) - if hasattr(f, 'oneof_index') and f.HasField('oneof_index'): - self.oneofs[f.oneof_index].add_field(field) + if (hasattr(f, 'oneof_index') and + f.HasField('oneof_index') and + f.oneof_index not in no_unions): + if f.oneof_index in self.oneofs: + self.oneofs[f.oneof_index].add_field(field) else: self.fields.append(field) @@ -734,7 +744,7 @@ class Message: '''Returns number of required fields inside this message''' count = 0 for f in self.fields: - if f not in self.oneofs: + if not isinstance(f, OneOf): if f.rules == 'REQUIRED': count += 1 return count @@ -742,7 +752,7 @@ class Message: def count_all_fields(self): count = 0 for f in self.fields: - if f in self.oneofs: + if isinstance(f, OneOf): count += len(f.fields) else: count += 1 diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index e830ec2c..a1b24936 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -53,6 +53,9 @@ message NanoPBOptions { // Skip this message optional bool skip_message = 6 [default = false]; + + // Generate oneof fields as normal optional fields instead of union. + optional bool no_unions = 8 [default = false]; } // Extensions to protoc 'Descriptor' type in order to define options -- cgit 1.2.3-korg From d2e023e3e5ef888e176292f149d1ba93ba270c94 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 11 Jan 2015 19:46:15 +0200 Subject: Bugfixes for oneof support. Fixes crashes / memory leaks when using pointer type fields. Also fixes initialization of which_oneof fields. --- pb.h | 2 +- pb_common.c | 20 ++++++++++---------- pb_decode.c | 17 +++++++++++++---- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/pb.h b/pb.h index af58dca3..fc846c94 100644 --- a/pb.h +++ b/pb.h @@ -514,7 +514,7 @@ struct pb_extension_s { #define PB_ONEOF_POINTER(u, tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_POINTER | PB_HTYPE_ONEOF | ltype, \ fd, pb_delta(st, which_ ## u, u.m), \ - pb_membersize(st, u.m), 0, ptr} + pb_membersize(st, u.m[0]), 0, ptr} #define PB_ONEOF_FIELD(union_name, tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ PB_ ## rules ## _ ## allocation(union_name, tag, message, field, \ diff --git a/pb_common.c b/pb_common.c index 98964850..385c0193 100644 --- a/pb_common.c +++ b/pb_common.c @@ -41,8 +41,15 @@ bool pb_field_iter_next(pb_field_iter_t *iter) /* Increment the pointers based on previous field size */ size_t prev_size = prev_field->data_size; - if (PB_ATYPE(prev_field->type) == PB_ATYPE_STATIC && - PB_HTYPE(prev_field->type) == PB_HTYPE_REPEATED) + if (PB_HTYPE(prev_field->type) == PB_HTYPE_ONEOF && + PB_HTYPE(iter->pos->type) == PB_HTYPE_ONEOF) + { + /* Don't advance pointers inside unions */ + prev_size = 0; + iter->pData = (char*)iter->pData - prev_field->data_offset; + } + else if (PB_ATYPE(prev_field->type) == PB_ATYPE_STATIC && + PB_HTYPE(prev_field->type) == PB_HTYPE_REPEATED) { /* In static arrays, the data_size tells the size of a single entry and * array_size is the number of entries */ @@ -54,14 +61,7 @@ bool pb_field_iter_next(pb_field_iter_t *iter) * The data_size only applies to the dynamically allocated area. */ prev_size = sizeof(void*); } - else if (PB_HTYPE(prev_field->type) == PB_HTYPE_ONEOF && - PB_HTYPE(iter->pos->type) == PB_HTYPE_ONEOF) - { - /* Don't advance pointers inside unions */ - prev_size = 0; - iter->pData = (char*)iter->pData - prev_field->data_offset; - } - + if (PB_HTYPE(prev_field->type) == PB_HTYPE_REQUIRED) { /* Count the required fields, in order to check their presence in the diff --git a/pb_decode.c b/pb_decode.c index 542fdc4c..0677594f 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -747,13 +747,15 @@ static void pb_field_set_to_default(pb_field_iter_t *iter) * itself also. */ *(bool*)iter->pSize = false; } - else if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + else if (PB_HTYPE(type) == PB_HTYPE_REPEATED || + PB_HTYPE(type) == PB_HTYPE_ONEOF) { - /* Set array count to 0, no need to initialize contents. */ + /* REPEATED: Set array count to 0, no need to initialize contents. + ONEOF: Set which_field to 0. */ *(pb_size_t*)iter->pSize = 0; init_data = false; } - + if (init_data) { if (PB_LTYPE(iter->pos->type) == PB_LTYPE_SUBMESSAGE) @@ -779,7 +781,8 @@ static void pb_field_set_to_default(pb_field_iter_t *iter) *(void**)iter->pData = NULL; /* Initialize array count to 0. */ - if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + if (PB_HTYPE(type) == PB_HTYPE_REPEATED || + PB_HTYPE(type) == PB_HTYPE_ONEOF) { *(pb_size_t*)iter->pSize = 0; } @@ -938,6 +941,12 @@ static void pb_release_single_field(const pb_field_iter_t *iter) pb_type_t type; type = iter->pos->type; + if (PB_HTYPE(type) == PB_HTYPE_ONEOF) + { + if (*(pb_size_t*)iter->pSize != iter->pos->tag) + return; /* This is not the current field in the union */ + } + /* Release anything contained inside an extension or submsg. * This has to be done even if the submsg itself is statically * allocated. */ -- cgit 1.2.3-korg From f4b3a1c2027e330c52e661252acba6f69742fca6 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 11 Jan 2015 19:47:27 +0200 Subject: Add oneofs to AllTypes test case --- tests/alltypes/alltypes.proto | 6 ++++++ tests/alltypes/decode_alltypes.c | 6 ++++++ tests/alltypes/encode_alltypes.c | 4 ++++ tests/alltypes_callback/alltypes.options | 1 + tests/alltypes_callback/decode_alltypes_callback.c | 5 +++++ tests/alltypes_callback/encode_alltypes_callback.c | 5 +++++ tests/alltypes_pointer/SConscript | 2 +- tests/alltypes_pointer/decode_alltypes_pointer.c | 6 ++++++ tests/alltypes_pointer/encode_alltypes_pointer.c | 6 ++++++ tests/field_size_16/alltypes.proto | 6 ++++++ tests/field_size_32/alltypes.proto | 6 ++++++ tests/io_errors_pointers/SConscript | 2 +- 12 files changed, 53 insertions(+), 2 deletions(-) diff --git a/tests/alltypes/alltypes.proto b/tests/alltypes/alltypes.proto index db83c9a6..28eaf0be 100644 --- a/tests/alltypes/alltypes.proto +++ b/tests/alltypes/alltypes.proto @@ -101,6 +101,12 @@ message AllTypes { optional MyEnum opt_enum = 57 [default = Second]; optional EmptyMessage opt_emptymsg = 58; + oneof oneof + { + SubMessage oneof_msg1 = 59; + EmptyMessage oneof_msg2 = 60; + } + // Check that extreme integer values are handled correctly required Limits req_limits = 98; diff --git a/tests/alltypes/decode_alltypes.c b/tests/alltypes/decode_alltypes.c index 67cb72c4..458e5112 100644 --- a/tests/alltypes/decode_alltypes.c +++ b/tests/alltypes/decode_alltypes.c @@ -125,6 +125,8 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.has_opt_enum == false); TEST(alltypes.opt_enum == MyEnum_Second); TEST(alltypes.has_opt_emptymsg == false); + + TEST(alltypes.which_oneof == 0); } else { @@ -170,6 +172,10 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.has_opt_enum == true); TEST(alltypes.opt_enum == MyEnum_Truth); TEST(alltypes.has_opt_emptymsg == true); + + TEST(alltypes.which_oneof == AllTypes_oneof_msg1_tag); + TEST(strcmp(alltypes.oneof.oneof_msg1.substuff1, "4059") == 0); + TEST(alltypes.oneof.oneof_msg1.substuff2 == 4059); } TEST(alltypes.req_limits.int32_min == INT32_MIN); diff --git a/tests/alltypes/encode_alltypes.c b/tests/alltypes/encode_alltypes.c index deec7903..16f4b298 100644 --- a/tests/alltypes/encode_alltypes.c +++ b/tests/alltypes/encode_alltypes.c @@ -121,6 +121,10 @@ int main(int argc, char **argv) alltypes.has_opt_enum = true; alltypes.opt_enum = MyEnum_Truth; alltypes.has_opt_emptymsg = true; + + alltypes.which_oneof = AllTypes_oneof_msg1_tag; + strcpy(alltypes.oneof.oneof_msg1.substuff1, "4059"); + alltypes.oneof.oneof_msg1.substuff2 = 4059; } alltypes.end = 1099; diff --git a/tests/alltypes_callback/alltypes.options b/tests/alltypes_callback/alltypes.options index a9c55ece..daee5224 100644 --- a/tests/alltypes_callback/alltypes.options +++ b/tests/alltypes_callback/alltypes.options @@ -1,3 +1,4 @@ # Generate all fields as callbacks. AllTypes.* type:FT_CALLBACK SubMessage.substuff1 max_size:16 +AllTypes.oneof no_unions:true diff --git a/tests/alltypes_callback/decode_alltypes_callback.c b/tests/alltypes_callback/decode_alltypes_callback.c index b6017fe6..c53ab6ed 100644 --- a/tests/alltypes_callback/decode_alltypes_callback.c +++ b/tests/alltypes_callback/decode_alltypes_callback.c @@ -214,6 +214,8 @@ bool check_alltypes(pb_istream_t *stream, int mode) int64_t opt_sfixed64 = 3052; double opt_double = 3053.0f; SubMessage opt_submsg = {"3056", 3056}; + + SubMessage oneof_msg1 = {"4059", 4059}; /* Bind callbacks for required fields */ AllTypes alltypes; @@ -392,6 +394,9 @@ bool check_alltypes(pb_istream_t *stream, int mode) alltypes.opt_enum.arg = (void*)MyEnum_Truth; alltypes.opt_emptymsg.funcs.decode = &read_emptymsg; + + alltypes.oneof_msg1.funcs.decode = &read_submsg; + alltypes.oneof_msg1.arg = &oneof_msg1; } return pb_decode(stream, AllTypes_fields, &alltypes); diff --git a/tests/alltypes_callback/encode_alltypes_callback.c b/tests/alltypes_callback/encode_alltypes_callback.c index 10560b19..abc43f58 100644 --- a/tests/alltypes_callback/encode_alltypes_callback.c +++ b/tests/alltypes_callback/encode_alltypes_callback.c @@ -202,6 +202,8 @@ int main(int argc, char **argv) double opt_double = 3053.0f; SubMessage opt_submsg = {"3056", 3056}; + SubMessage oneof_msg1 = {"4059", 4059}; + /* Bind callbacks for required fields */ AllTypes alltypes = {{{0}}}; @@ -372,6 +374,9 @@ int main(int argc, char **argv) alltypes.opt_enum.arg = (void*)MyEnum_Truth; alltypes.opt_emptymsg.funcs.encode = &write_emptymsg; + + alltypes.oneof_msg1.funcs.encode = &write_submsg; + alltypes.oneof_msg1.arg = &oneof_msg1; } alltypes.end.funcs.encode = &write_varint; diff --git a/tests/alltypes_pointer/SConscript b/tests/alltypes_pointer/SConscript index 52856f6e..b095ae03 100644 --- a/tests/alltypes_pointer/SConscript +++ b/tests/alltypes_pointer/SConscript @@ -27,7 +27,7 @@ valgrind = env.WhereIs('valgrind') kwargs = {} if valgrind: kwargs['COMMAND'] = valgrind - kwargs['ARGS'] = ["-q", dec[0].abspath] + kwargs['ARGS'] = ["-q", "--error-exitcode=99", dec[0].abspath] env.RunTest("decode_alltypes.output", [dec, "encode_alltypes_pointer.output"], **kwargs) diff --git a/tests/alltypes_pointer/decode_alltypes_pointer.c b/tests/alltypes_pointer/decode_alltypes_pointer.c index 1e719019..1dbb6c55 100644 --- a/tests/alltypes_pointer/decode_alltypes_pointer.c +++ b/tests/alltypes_pointer/decode_alltypes_pointer.c @@ -99,6 +99,8 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.opt_bytes == NULL); TEST(alltypes.opt_submsg == NULL); TEST(alltypes.opt_enum == NULL); + + TEST(alltypes.which_oneof == 0); } else { @@ -125,6 +127,10 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.opt_submsg && *alltypes.opt_submsg->substuff2 == 3056); TEST(alltypes.opt_enum && *alltypes.opt_enum == MyEnum_Truth); TEST(alltypes.opt_emptymsg); + + TEST(alltypes.which_oneof == AllTypes_oneof_msg1_tag); + TEST(alltypes.oneof.oneof_msg1 && strcmp(alltypes.oneof.oneof_msg1->substuff1, "4059") == 0); + TEST(alltypes.oneof.oneof_msg1->substuff2 && *alltypes.oneof.oneof_msg1->substuff2 == 4059); } TEST(alltypes.req_limits->int32_min && *alltypes.req_limits->int32_min == INT32_MIN); diff --git a/tests/alltypes_pointer/encode_alltypes_pointer.c b/tests/alltypes_pointer/encode_alltypes_pointer.c index c128569b..7b52662f 100644 --- a/tests/alltypes_pointer/encode_alltypes_pointer.c +++ b/tests/alltypes_pointer/encode_alltypes_pointer.c @@ -84,6 +84,9 @@ int main(int argc, char **argv) MyEnum opt_enum = MyEnum_Truth; EmptyMessage opt_emptymsg = {0}; + static int32_t oneof_substuff = 4059; + SubMessage oneof_msg1 = {"4059", &oneof_substuff}; + /* Values for the Limits message. */ static int32_t int32_min = INT32_MIN; static int32_t int32_max = INT32_MAX; @@ -164,6 +167,9 @@ int main(int argc, char **argv) alltypes.opt_submsg = &opt_submsg; alltypes.opt_enum = &opt_enum; alltypes.opt_emptymsg = &opt_emptymsg; + + alltypes.which_oneof = AllTypes_oneof_msg1_tag; + alltypes.oneof.oneof_msg1 = &oneof_msg1; } alltypes.end = &end; diff --git a/tests/field_size_16/alltypes.proto b/tests/field_size_16/alltypes.proto index 81693c00..039391fc 100644 --- a/tests/field_size_16/alltypes.proto +++ b/tests/field_size_16/alltypes.proto @@ -101,6 +101,12 @@ message AllTypes { optional MyEnum opt_enum = 10057 [default = Second]; optional EmptyMessage opt_emptymsg = 10058; + oneof oneof + { + SubMessage oneof_msg1 = 10059; + EmptyMessage oneof_msg2 = 10060; + } + // Check that extreme integer values are handled correctly required Limits req_limits = 98; diff --git a/tests/field_size_32/alltypes.proto b/tests/field_size_32/alltypes.proto index 81717bbc..5749e0d4 100644 --- a/tests/field_size_32/alltypes.proto +++ b/tests/field_size_32/alltypes.proto @@ -101,6 +101,12 @@ message AllTypes { optional MyEnum opt_enum = 10057 [default = Second]; optional EmptyMessage opt_emptymsg = 10058; + oneof oneof + { + SubMessage oneof_msg1 = 10059; + EmptyMessage oneof_msg2 = 10060; + } + // Check that extreme integer values are handled correctly required Limits req_limits = 98; diff --git a/tests/io_errors_pointers/SConscript b/tests/io_errors_pointers/SConscript index 8d23f603..03727df9 100644 --- a/tests/io_errors_pointers/SConscript +++ b/tests/io_errors_pointers/SConscript @@ -19,7 +19,7 @@ valgrind = env.WhereIs('valgrind') kwargs = {} if valgrind: kwargs['COMMAND'] = valgrind - kwargs['ARGS'] = ["-q", ioerr[0].abspath] + kwargs['ARGS'] = ["-q", "--error-exitcode=99", ioerr[0].abspath] env.RunTest("io_errors.output", [ioerr, "$BUILD/alltypes/encode_alltypes.output"], **kwargs) -- cgit 1.2.3-korg From e1c50496d9e7d1f54c4f9c344cca25bb85d61659 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 15 Jan 2015 18:58:08 +0200 Subject: Release memory when overwriting oneof fields. Update issue 131 Status: FixedInGit --- pb_decode.c | 39 ++++++++++++ tests/mem_release/mem_release.c | 117 +++++++++++++++++++++++++++++------- tests/mem_release/mem_release.proto | 10 +++ 3 files changed, 144 insertions(+), 22 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 0677594f..9d25dc61 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -48,6 +48,7 @@ static bool checkreturn pb_skip_string(pb_istream_t *stream); #ifdef PB_ENABLE_MALLOC static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size); +static bool checkreturn pb_release_union_field(pb_istream_t *stream, pb_field_iter_t *iter); static void pb_release_single_field(const pb_field_iter_t *iter); #endif @@ -628,6 +629,16 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter) { +#ifdef PB_ENABLE_MALLOC + /* When decoding an oneof field, check if there is old data that must be + * released first. */ + if (PB_HTYPE(iter->pos->type) == PB_HTYPE_ONEOF) + { + if (!pb_release_union_field(stream, iter)) + return false; + } +#endif + switch (PB_ATYPE(iter->pos->type)) { case PB_ATYPE_STATIC: @@ -936,6 +947,34 @@ bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void * } #ifdef PB_ENABLE_MALLOC +/* Given an oneof field, if there has already been a field inside this oneof, + * release it before overwriting with a different one. */ +static bool pb_release_union_field(pb_istream_t *stream, pb_field_iter_t *iter) +{ + pb_size_t old_tag = *(pb_size_t*)iter->pSize; /* Previous which_ value */ + pb_size_t new_tag = iter->pos->tag; /* New which_ value */ + + if (old_tag == 0) + return true; /* Ok, no old data in union */ + + if (old_tag == new_tag) + return true; /* Ok, old data is of same type => merge */ + + /* Release old data. The find can fail if the message struct contains + * invalid data. */ + if (!pb_field_iter_find(iter, old_tag)) + PB_RETURN_ERROR(stream, "invalid union tag"); + + pb_release_single_field(iter); + + /* Restore iterator to where it should be. + * This shouldn't fail unless the pb_field_t structure is corrupted. */ + if (!pb_field_iter_find(iter, new_tag)) + PB_RETURN_ERROR(stream, "iterator error"); + + return true; +} + static void pb_release_single_field(const pb_field_iter_t *iter) { pb_type_t type; diff --git a/tests/mem_release/mem_release.c b/tests/mem_release/mem_release.c index 796df13e..40fdc9e4 100644 --- a/tests/mem_release/mem_release.c +++ b/tests/mem_release/mem_release.c @@ -14,8 +14,31 @@ static char *test_str_arr[] = {"1", "2", ""}; static SubMessage test_msg_arr[] = {SubMessage_init_zero, SubMessage_init_zero}; +static pb_extension_t ext1, ext2; -static bool do_test() +static void fill_TestMessage(TestMessage *msg) +{ + msg->static_req_submsg.dynamic_str = "12345"; + msg->static_req_submsg.dynamic_str_arr_count = 3; + msg->static_req_submsg.dynamic_str_arr = test_str_arr; + msg->static_req_submsg.dynamic_submsg_count = 2; + msg->static_req_submsg.dynamic_submsg = test_msg_arr; + msg->static_req_submsg.dynamic_submsg[1].dynamic_str = "abc"; + msg->static_opt_submsg.dynamic_str = "abc"; + msg->has_static_opt_submsg = true; + msg->dynamic_submsg = &msg->static_req_submsg; + + msg->extensions = &ext1; + ext1.type = &dynamic_ext; + ext1.dest = &msg->static_req_submsg; + ext1.next = &ext2; + ext2.type = &static_ext; + ext2.dest = &msg->static_req_submsg; + ext2.next = NULL; +} + +/* Basic fields, nested submessages, extensions */ +static bool test_TestMessage() { uint8_t buffer[256]; size_t msgsize; @@ -23,25 +46,9 @@ static bool do_test() /* Construct a message with various fields filled in */ { TestMessage msg = TestMessage_init_zero; - pb_extension_t ext1, ext2; pb_ostream_t stream; - msg.static_req_submsg.dynamic_str = "12345"; - msg.static_req_submsg.dynamic_str_arr_count = 3; - msg.static_req_submsg.dynamic_str_arr = test_str_arr; - msg.static_req_submsg.dynamic_submsg_count = 2; - msg.static_req_submsg.dynamic_submsg = test_msg_arr; - msg.static_req_submsg.dynamic_submsg[1].dynamic_str = "abc"; - msg.static_opt_submsg.dynamic_str = "abc"; - msg.has_static_opt_submsg = true; - msg.dynamic_submsg = &msg.static_req_submsg; - - msg.extensions = &ext1; - ext1.type = &dynamic_ext; - ext1.dest = &msg.static_req_submsg; - ext1.next = &ext2; - ext2.type = &static_ext; - ext2.dest = &msg.static_req_submsg; - ext2.next = NULL; + + fill_TestMessage(&msg); stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); if (!pb_encode(&stream, TestMessage_fields, &msg)) @@ -61,8 +68,7 @@ static bool do_test() TestMessage msg = TestMessage_init_zero; pb_istream_t stream; SubMessage ext2_dest; - pb_extension_t ext1, ext2; - + msg.extensions = &ext1; ext1.type = &dynamic_ext; ext1.dest = NULL; @@ -102,9 +108,76 @@ static bool do_test() return true; } +/* Oneofs */ +static bool test_OneofMessage() +{ + uint8_t buffer[256]; + size_t msgsize; + + { + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Encode first with TestMessage */ + { + OneofMessage msg = OneofMessage_init_zero; + msg.which_msgs = OneofMessage_msg1_tag; + + fill_TestMessage(&msg.msgs.msg1); + + if (!pb_encode(&stream, OneofMessage_fields, &msg)) + { + fprintf(stderr, "Encode failed: %s\n", PB_GET_ERROR(&stream)); + return false; + } + } + + /* Encode second with SubMessage, invoking 'merge' behaviour */ + { + OneofMessage msg = OneofMessage_init_zero; + msg.which_msgs = OneofMessage_msg2_tag; + + msg.first = 999; + msg.msgs.msg2.dynamic_str = "ABCD"; + msg.last = 888; + + if (!pb_encode(&stream, OneofMessage_fields, &msg)) + { + fprintf(stderr, "Encode failed: %s\n", PB_GET_ERROR(&stream)); + return false; + } + } + msgsize = stream.bytes_written; + } + + { + OneofMessage msg = OneofMessage_init_zero; + pb_istream_t stream = pb_istream_from_buffer(buffer, msgsize); + if (!pb_decode(&stream, OneofMessage_fields, &msg)) + { + fprintf(stderr, "Decode failed: %s\n", PB_GET_ERROR(&stream)); + return false; + } + + TEST(msg.first == 999); + TEST(msg.which_msgs == OneofMessage_msg2_tag); + TEST(msg.msgs.msg2.dynamic_str); + TEST(strcmp(msg.msgs.msg2.dynamic_str, "ABCD") == 0); + TEST(msg.msgs.msg2.dynamic_str_arr == NULL); + TEST(msg.msgs.msg2.dynamic_submsg == NULL); + TEST(msg.last == 888); + + pb_release(OneofMessage_fields, &msg); + TEST(get_alloc_count() == 0); + pb_release(OneofMessage_fields, &msg); + TEST(get_alloc_count() == 0); + } + + return true; +} + int main() { - if (do_test()) + if (test_TestMessage() && test_OneofMessage()) return 0; else return 1; diff --git a/tests/mem_release/mem_release.proto b/tests/mem_release/mem_release.proto index 0db393ae..c3b38c8b 100644 --- a/tests/mem_release/mem_release.proto +++ b/tests/mem_release/mem_release.proto @@ -22,3 +22,13 @@ extend TestMessage optional SubMessage static_ext = 101 [(nanopb).type = FT_STATIC]; } +message OneofMessage +{ + required int32 first = 1; + oneof msgs + { + TestMessage msg1 = 2; + SubMessage msg2 = 3; + } + required int32 last = 4; +} -- cgit 1.2.3-korg From 24a45b0a9c1ea9c84d3f180cf1ffe35a600d611c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 15 Jan 2015 19:18:48 +0200 Subject: Fix clang compiler warning in intsizes unit test. --- tests/intsizes/intsizes_unittests.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/intsizes/intsizes_unittests.c b/tests/intsizes/intsizes_unittests.c index 189825f7..79ef0369 100644 --- a/tests/intsizes/intsizes_unittests.c +++ b/tests/intsizes/intsizes_unittests.c @@ -43,18 +43,18 @@ TEST(pb_decode(&s, IntSizes_fields, &msg2) == expected_result); \ if (expected_result) \ { \ - TEST(msg2.req_int8 == int8); \ - TEST(msg2.req_uint8 == uint8); \ - TEST(msg2.req_sint8 == sint8); \ - TEST(msg2.req_int16 == int16); \ - TEST(msg2.req_uint16 == uint16); \ - TEST(msg2.req_sint16 == sint16); \ - TEST(msg2.req_int32 == int32); \ - TEST(msg2.req_uint32 == uint32); \ - TEST(msg2.req_sint32 == sint32); \ - TEST(msg2.req_int64 == int64); \ - TEST(msg2.req_uint64 == uint64); \ - TEST(msg2.req_sint64 == sint64); \ + TEST( (int64_t)msg2.req_int8 == int8); \ + TEST((uint64_t)msg2.req_uint8 == uint8); \ + TEST( (int64_t)msg2.req_sint8 == sint8); \ + TEST( (int64_t)msg2.req_int16 == int16); \ + TEST((uint64_t)msg2.req_uint16 == uint16); \ + TEST( (int64_t)msg2.req_sint16 == sint16); \ + TEST( (int64_t)msg2.req_int32 == int32); \ + TEST((uint64_t)msg2.req_uint32 == uint32); \ + TEST( (int64_t)msg2.req_sint32 == sint32); \ + TEST( (int64_t)msg2.req_int64 == int64); \ + TEST((uint64_t)msg2.req_uint64 == uint64); \ + TEST( (int64_t)msg2.req_sint64 == sint64); \ } \ } \ \ @@ -114,7 +114,7 @@ int main() -32768, 0, -32769, INT32_MIN, 0, INT32_MIN, INT64_MIN, 0, INT64_MIN, false); - + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); -- cgit 1.2.3-korg From 0286a0746ad6f6d525400f720e30d9d13913a556 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 15 Jan 2015 19:34:49 +0200 Subject: Update changelog --- CHANGELOG.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 6087a29f..3346be3d 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,14 @@ +nanopb-0.3.2 (2015-01-xx) + Fix memory leaks with PB_ENABLE_MALLOC with some submessage hierarchies (issue 138) + Implement support for oneofs (C unions). (issue 131) + Add int_size option for generator (issue 139) + Add compilation option to disable struct packing. (issue 136) + Change PB_RETURN_ERROR() macro to avoid compiler warnings (issue 140) + Fix build problems with protoc 3.0.0 + Add support for POINTER type in extensions + Initialize also extension fields to defaults in pb_decode(). + Detect too large varint values when decoding. + nanopb-0.3.1 (2014-09-11) Fix security issue due to size_t overflows. (issue 132) Fix memory leak with duplicated fields and PB_ENABLE_MALLOC -- cgit 1.2.3-korg From d32d04ba1011806fe769f95dc2a88964f1102c6c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 23 Jan 2015 21:29:29 +0200 Subject: Fix encoded_size #defines for oneof messages. The sizes are represented as EncodedSize() instances, which cause max() operation to sort them by address instead of value. This caused pretty much random item to be selected for the maximum. Update issue 141 Status: FixedInGit --- generator/nanopb_generator.py | 12 ++++++- tests/regression/issue_141/SConscript | 8 +++++ tests/regression/issue_141/testproto.expected | 7 ++++ tests/regression/issue_141/testproto.proto | 50 +++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 tests/regression/issue_141/SConscript create mode 100644 tests/regression/issue_141/testproto.expected create mode 100644 tests/regression/issue_141/testproto.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 0d7be492..9750b670 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -641,7 +641,17 @@ class OneOf(Field): return max([f.largest_field_value() for f in self.fields]) def encoded_size(self, allmsgs): - return max([f.encoded_size(allmsgs) for f in self.fields]) + largest = EncodedSize(0) + for f in self.fields: + size = f.encoded_size(allmsgs) + if size is None: + return None + elif size.symbols: + return None # Cannot resolve maximum of symbols + elif size.value > largest.value: + largest = size + + return largest # --------------------------------------------------------------------------- # Generation of messages (structures) diff --git a/tests/regression/issue_141/SConscript b/tests/regression/issue_141/SConscript new file mode 100644 index 00000000..b6526bed --- /dev/null +++ b/tests/regression/issue_141/SConscript @@ -0,0 +1,8 @@ +# Regression test for issue 141: wrong encoded size #define for oneof messages + +Import("env") + +env.NanopbProto("testproto") +env.Object('testproto.pb.c') +env.Match(['testproto.pb.h', 'testproto.expected']) + diff --git a/tests/regression/issue_141/testproto.expected b/tests/regression/issue_141/testproto.expected new file mode 100644 index 00000000..75bc195c --- /dev/null +++ b/tests/regression/issue_141/testproto.expected @@ -0,0 +1,7 @@ +define SubMessage_size \s* 88 +define OneOfMessage_size \s* 113 +define topMessage_size \s* 70 +define MyMessage1_size \s* 46 +define MyMessage2_size \s* 8 +define MyMessage3_size \s* 5 +define MyMessage4_size \s* 18 diff --git a/tests/regression/issue_141/testproto.proto b/tests/regression/issue_141/testproto.proto new file mode 100644 index 00000000..21598b45 --- /dev/null +++ b/tests/regression/issue_141/testproto.proto @@ -0,0 +1,50 @@ +import 'nanopb.proto'; + +message SubMessage +{ + repeated int32 array = 1 [(nanopb).max_count = 8]; +} + +message OneOfMessage +{ + required int32 prefix = 1; + oneof values + { + int32 first = 5; + string second = 6 [(nanopb).max_size = 8]; + SubMessage third = 7; + } + required int32 suffix = 99; +} + +message topMessage { + required int32 start = 1; + oneof msg { + MyMessage1 msg1 = 2; + MyMessage2 msg2 = 3; + } + required int32 end = 4; +} + +message MyMessage1 { + required uint32 n1 = 1; + required uint32 n2 = 2; + required string s = 3 [(nanopb).max_size = 32]; +} + +message MyMessage2 { + required uint32 num = 1; + required bool b = 2; +} + +message MyMessage3 { + required bool bbb = 1; + required string ss = 2 [(nanopb).max_size = 1]; +} + +message MyMessage4 { + required bool bbbb = 1; + required string sss = 2 [(nanopb).max_size = 2]; + required uint32 num = 3; + required uint32 num2 = 4; +} -- cgit 1.2.3-korg From acd72917912aa99e9751766ec90e2d2156639a60 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 24 Jan 2015 17:33:01 +0200 Subject: Publishing nanopb-0.3.2 --- CHANGELOG.txt | 4 ++-- docs/migration.rst | 2 +- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 3346be3d..d10364f4 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,6 @@ -nanopb-0.3.2 (2015-01-xx) +nanopb-0.3.2 (2015-01-24) Fix memory leaks with PB_ENABLE_MALLOC with some submessage hierarchies (issue 138) - Implement support for oneofs (C unions). (issue 131) + Implement support for oneofs (C unions). (issues 131, 141) Add int_size option for generator (issue 139) Add compilation option to disable struct packing. (issue 136) Change PB_RETURN_ERROR() macro to avoid compiler warnings (issue 140) diff --git a/docs/migration.rst b/docs/migration.rst index d5ded643..ac92db88 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -11,7 +11,7 @@ are included, in order to make it easier to find this document. .. contents :: -Nanopb-0.3.2 (2015-01-xx) +Nanopb-0.3.2 (2015-01-24) ========================= Add support for OneOfs diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 9750b670..8c87ff53 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.2-dev" +nanopb_version = "nanopb-0.3.2" import sys diff --git a/pb.h b/pb.h index fc846c94..90da6233 100644 --- a/pb.h +++ b/pb.h @@ -50,7 +50,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.2-dev +#define NANOPB_VERSION nanopb-0.3.2 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 5aa8207ab1acf3b614b528b3d84e60d9266d35ae Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 24 Jan 2015 17:40:42 +0200 Subject: Setting version to nanopb-0.3.3-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 8c87ff53..520824ce 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.2" +nanopb_version = "nanopb-0.3.3-dev" import sys diff --git a/pb.h b/pb.h index 90da6233..47941ebf 100644 --- a/pb.h +++ b/pb.h @@ -50,7 +50,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.2 +#define NANOPB_VERSION nanopb-0.3.3-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 25b92c5b4e154130bf0787009046ea60175d28e8 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 27 Jan 2015 17:47:25 +0200 Subject: Fix generator bug when oneof is first field in a message. Added test case for the same. Update issue 142 Status: FixedInGit --- generator/nanopb_generator.py | 1 - tests/oneof/decode_oneof.c | 129 ++++++++++++++++++++++++++++++------------ tests/oneof/oneof.proto | 12 ++++ 3 files changed, 104 insertions(+), 38 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 520824ce..4c01502c 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -633,7 +633,6 @@ class OneOf(Field): return '\n'.join([f.tags() for f in self.fields]) def pb_field_t(self, prev_field_name): - prev_field_name = prev_field_name or self.name result = ',\n'.join([f.pb_field_t(prev_field_name) for f in self.fields]) return result diff --git a/tests/oneof/decode_oneof.c b/tests/oneof/decode_oneof.c index e94becc7..83b4702c 100644 --- a/tests/oneof/decode_oneof.c +++ b/tests/oneof/decode_oneof.c @@ -7,17 +7,92 @@ #include "test_helpers.h" #include "unittests.h" +/* Test the 'OneOfMessage' */ +int test_oneof_1(pb_istream_t *stream, int option) +{ + OneOfMessage msg = OneOfMessage_init_zero; + int status = 0; + + if (!pb_decode(stream, OneOfMessage_fields, &msg)) + { + printf("Decoding failed: %s\n", PB_GET_ERROR(stream)); + return 1; + } + + /* Check that the basic fields work normally */ + TEST(msg.prefix == 123); + TEST(msg.suffix == 321); + + /* Check that we got the right oneof according to command line */ + if (option == 1) + { + TEST(msg.which_values == OneOfMessage_first_tag); + TEST(msg.values.first == 999); + } + else if (option == 2) + { + TEST(msg.which_values == OneOfMessage_second_tag); + TEST(strcmp(msg.values.second, "abcd") == 0); + } + else if (option == 3) + { + TEST(msg.which_values == OneOfMessage_third_tag); + TEST(msg.values.third.array[0] == 1); + TEST(msg.values.third.array[1] == 2); + TEST(msg.values.third.array[2] == 3); + TEST(msg.values.third.array[3] == 4); + TEST(msg.values.third.array[4] == 5); + } + + return status; +} + + +/* Test the 'PlainOneOfMessage' */ +int test_oneof_2(pb_istream_t *stream, int option) +{ + PlainOneOfMessage msg = PlainOneOfMessage_init_zero; + int status = 0; + + if (!pb_decode(stream, PlainOneOfMessage_fields, &msg)) + { + printf("Decoding failed: %s\n", PB_GET_ERROR(stream)); + return 1; + } + + /* Check that we got the right oneof according to command line */ + if (option == 1) + { + TEST(msg.which_values == OneOfMessage_first_tag); + TEST(msg.values.first == 999); + } + else if (option == 2) + { + TEST(msg.which_values == OneOfMessage_second_tag); + TEST(strcmp(msg.values.second, "abcd") == 0); + } + else if (option == 3) + { + TEST(msg.which_values == OneOfMessage_third_tag); + TEST(msg.values.third.array[0] == 1); + TEST(msg.values.third.array[1] == 2); + TEST(msg.values.third.array[2] == 3); + TEST(msg.values.third.array[3] == 4); + TEST(msg.values.third.array[4] == 5); + } + + return status; +} + int main(int argc, char **argv) { uint8_t buffer[OneOfMessage_size]; - OneOfMessage msg = OneOfMessage_init_zero; - pb_istream_t stream; size_t count; int option; if (argc != 2) { - fprintf(stderr, "Usage: encode_oneof [number]\n"); + fprintf(stderr, "Usage: decode_oneof [number]\n"); return 1; } option = atoi(argv[1]); @@ -31,42 +106,22 @@ int main(int argc, char **argv) return 1; } - stream = pb_istream_from_buffer(buffer, count); - - if (!pb_decode(&stream, OneOfMessage_fields, &msg)) - { - printf("Decoding failed: %s\n", PB_GET_ERROR(&stream)); - return 1; - } - { int status = 0; + pb_istream_t stream; + + stream = pb_istream_from_buffer(buffer, count); + status = test_oneof_1(&stream, option); + + if (status != 0) + return status; - /* Check that the basic fields work normally */ - TEST(msg.prefix == 123); - TEST(msg.suffix == 321); - - /* Check that we got the right oneof according to command line */ - if (option == 1) - { - TEST(msg.which_values == OneOfMessage_first_tag); - TEST(msg.values.first == 999); - } - else if (option == 2) - { - TEST(msg.which_values == OneOfMessage_second_tag); - TEST(strcmp(msg.values.second, "abcd") == 0); - } - else if (option == 3) - { - TEST(msg.which_values == OneOfMessage_third_tag); - TEST(msg.values.third.array[0] == 1); - TEST(msg.values.third.array[1] == 2); - TEST(msg.values.third.array[2] == 3); - TEST(msg.values.third.array[3] == 4); - TEST(msg.values.third.array[4] == 5); - } - - return status; + stream = pb_istream_from_buffer(buffer, count); + status = test_oneof_2(&stream, option); + + if (status != 0) + return status; } + + return 0; } \ No newline at end of file diff --git a/tests/oneof/oneof.proto b/tests/oneof/oneof.proto index a89ef131..00f1ceca 100644 --- a/tests/oneof/oneof.proto +++ b/tests/oneof/oneof.proto @@ -5,6 +5,7 @@ message SubMessage repeated int32 array = 1 [(nanopb).max_count = 8]; } +/* Oneof in a message with other fields */ message OneOfMessage { required int32 prefix = 1; @@ -16,3 +17,14 @@ message OneOfMessage } required int32 suffix = 99; } + +/* Oneof in a message by itself */ +message PlainOneOfMessage +{ + oneof values + { + int32 first = 5; + string second = 6 [(nanopb).max_size = 8]; + SubMessage third = 7; + } +} \ No newline at end of file -- cgit 1.2.3-korg From d8d3b75e2e3b348d016f48cebc1be764061975d2 Mon Sep 17 00:00:00 2001 From: Oliver Lee Date: Fri, 13 Feb 2015 10:26:31 +0100 Subject: Updates for the CMake rule file. 1) Search explicitly for python2.7 In systems where python3 is default or in build cases where the user has already searched for and found python3 in CMake, store the python3 executable and search for python2.7. 2) Generate nanopb core protobuf files with CMake Generate python output files used in turn by the nanopb generator script. This removes the requirement of manually calling 'make' in the nanopb/generator/proto directory. 3) Use nanopb options file if it exists Look for nanopb options file and use in protobuf source and header generation if it exists. The options file must have the same name and path as the proto file, excluding the extension. --- extra/FindNanopb.cmake | 49 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/extra/FindNanopb.cmake b/extra/FindNanopb.cmake index 65ab5884..513182d8 100644 --- a/extra/FindNanopb.cmake +++ b/extra/FindNanopb.cmake @@ -128,10 +128,36 @@ function(NANOPB_GENERATE_CPP SRCS HDRS) set(${SRCS}) set(${HDRS}) get_filename_component(GENERATOR_PATH ${NANOPB_GENERATOR_EXECUTABLE} PATH) + set(GENERATOR_CORE_DIR ${GENERATOR_PATH}/proto) + set(GENERATOR_CORE_SRC + ${GENERATOR_CORE_DIR}/nanopb.proto + ${GENERATOR_CORE_DIR}/plugin.proto) + + set(GENERATOR_CORE_PYTHON_SRC) + foreach(FIL ${GENERATOR_CORE_SRC}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + + set(output "${GENERATOR_CORE_DIR}/${FIL_WE}_pb2.py") + set(GENERATOR_CORE_PYTHON_SRC ${GENERATOR_CORE_PYTHON_SRC} ${output}) + add_custom_command( + OUTPUT ${output} + COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + ARGS -I${GENERATOR_PATH}/proto + --python_out=${GENERATOR_CORE_DIR} ${ABS_FIL} + DEPENDS ${ABS_FIL} + VERBATIM) + endforeach() foreach(FIL ${ARGN}) get_filename_component(ABS_FIL ${FIL} ABSOLUTE) get_filename_component(FIL_WE ${FIL} NAME_WE) + get_filename_component(FIL_DIR ${FIL} PATH) + set(NANOPB_OPTIONS_FILE ${FIL_DIR}/${FIL_WE}.options) + set(NANOPB_OPTIONS) + if(EXISTS ${NANOPB_OPTIONS_FILE}) + set(NANOPB_OPTIONS -f ${NANOPB_OPTIONS_FILE}) + endif() list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c") list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") @@ -139,16 +165,18 @@ function(NANOPB_GENERATE_CPP SRCS HDRS) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb" COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} - ARGS -I${GENERATOR_PATH} -I${CMAKE_CURRENT_BINARY_DIR} ${_nanobp_include_path} -o${FIL_WE}.pb ${ABS_FIL} - DEPENDS ${ABS_FIL} + ARGS -I${GENERATOR_PATH} -I${GENERATOR_CORE_DIR} + -I${CMAKE_CURRENT_BINARY_DIR} ${_nanobp_include_path} + -o${FIL_WE}.pb ${ABS_FIL} + DEPENDS ${ABS_FIL} ${GENERATOR_CORE_PYTHON_SRC} COMMENT "Running C++ protocol buffer compiler on ${FIL}" VERBATIM ) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c" "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" - COMMAND python - ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb + COMMAND ${PYTHON2_EXECUTABLE} + ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb ${NANOPB_OPTIONS} DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb" COMMENT "Running nanopb generator on ${FIL_WE}.pb" VERBATIM ) @@ -216,6 +244,19 @@ find_file(NANOPB_GENERATOR_EXECUTABLE ) mark_as_advanced(NANOPB_GENERATOR_EXECUTABLE) +# If python3 has already been found, save it and look for python2.7 +if(${PYTHON_VERSION_MAJOR} EQUAL 3) + set(PYTHON3_EXECUTABLE ${PYTHON_EXECUTABLE}) + set(PYTHON_EXECUTABLE PYTHON_EXECUTABLE-NOTFOUND) +endif() + +find_package(PythonInterp 2.7 REQUIRED) +set(PYTHON2_EXECUTABLE ${PYTHON_EXECUTABLE}) + +if(${PYTHON_VERSION_MAJOR} EQUAL 3) + set(PYTHON_EXECUTABLE ${PYTHON3_EXECUTABLE}) +endif() + include(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(NANOPB DEFAULT_MSG NANOPB_INCLUDE_DIRS -- cgit 1.2.3-korg From 02bd49bc9379e3062a718b1c16d6e2c23ea2f1a0 Mon Sep 17 00:00:00 2001 From: Oliver Lee Date: Fri, 13 Feb 2015 17:14:00 +0100 Subject: Fix search for Python 2 with CMake Do not assume that Python has already been found by CMake. Fix value of CMake variable PYTHON_EXECUTABLE if Python 3 was found. Change minimum supported Python version to 2.6. This fixes a bug introduced by this commit: d8d3b75e2e3b348d016f48cebc1be764061975d2 --- extra/FindNanopb.cmake | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/extra/FindNanopb.cmake b/extra/FindNanopb.cmake index 513182d8..e65706ac 100644 --- a/extra/FindNanopb.cmake +++ b/extra/FindNanopb.cmake @@ -244,17 +244,16 @@ find_file(NANOPB_GENERATOR_EXECUTABLE ) mark_as_advanced(NANOPB_GENERATOR_EXECUTABLE) -# If python3 has already been found, save it and look for python2.7 -if(${PYTHON_VERSION_MAJOR} EQUAL 3) +# If python3 has already been found, save it and look for python2.6 +if(${PYTHON_VERSION_MAJOR} AND ${PYTHON_VERSION_MAJOR} EQUAL 3) set(PYTHON3_EXECUTABLE ${PYTHON_EXECUTABLE}) set(PYTHON_EXECUTABLE PYTHON_EXECUTABLE-NOTFOUND) -endif() - -find_package(PythonInterp 2.7 REQUIRED) -set(PYTHON2_EXECUTABLE ${PYTHON_EXECUTABLE}) - -if(${PYTHON_VERSION_MAJOR} EQUAL 3) + find_package(PythonInterp 2.6 REQUIRED) + set(PYTHON2_EXECUTABLE ${PYTHON_EXECUTABLE}) set(PYTHON_EXECUTABLE ${PYTHON3_EXECUTABLE}) +else() + find_package(PythonInterp 2.6 REQUIRED) + set(PYTHON2_EXECUTABLE ${PYTHON_EXECUTABLE}) endif() include(FindPackageHandleStandardArgs) -- cgit 1.2.3-korg From 7c00b90910d4e14e23e027eb951e6f2a8dcfae29 Mon Sep 17 00:00:00 2001 From: Oliver Lee Date: Fri, 13 Feb 2015 17:28:27 +0100 Subject: Add simple example built with CMake --- examples/cmake_simple/CMakeLists.txt | 17 +++++++++ examples/cmake_simple/README.txt | 25 +++++++++++++ examples/cmake_simple/simple.c | 68 ++++++++++++++++++++++++++++++++++++ examples/cmake_simple/simple.proto | 7 ++++ 4 files changed, 117 insertions(+) create mode 100644 examples/cmake_simple/CMakeLists.txt create mode 100644 examples/cmake_simple/README.txt create mode 100644 examples/cmake_simple/simple.c create mode 100644 examples/cmake_simple/simple.proto diff --git a/examples/cmake_simple/CMakeLists.txt b/examples/cmake_simple/CMakeLists.txt new file mode 100644 index 00000000..5439e8e6 --- /dev/null +++ b/examples/cmake_simple/CMakeLists.txt @@ -0,0 +1,17 @@ +cmake_minimum_required(VERSION 2.8.12) +project(NANOPB_CMAKE_SIMPLE C) + +set(NANOPB_SRC_ROOT_FOLDER ${CMAKE_CURRENT_SOURCE_DIR}/../..) +set(CMAKE_MODULE_PATH ${NANOPB_SRC_ROOT_FOLDER}/extra) +find_package(Nanopb REQUIRED) +include_directories(${NANOPB_INCLUDE_DIRS}) + +nanopb_generate_cpp(PROTO_SRCS PROTO_HDRS simple.proto) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) +#add_custom_target(generate_proto_sources DEPENDS ${PROTO_SRCS} ${PROTO_HDRS}) +set_source_files_properties(${PROTO_SRCS} ${PROTO_HDRS} + PROPERTIES GENERATED TRUE) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror -g -O0") + +add_executable(simple simple.c ${PROTO_SRCS} ${PROTO_HDRS}) diff --git a/examples/cmake_simple/README.txt b/examples/cmake_simple/README.txt new file mode 100644 index 00000000..e41b8408 --- /dev/null +++ b/examples/cmake_simple/README.txt @@ -0,0 +1,25 @@ +Nanopb example "simple" using CMake +======================= + +This example is the same as the simple nanopb example but built using CMake. + +Example usage +------------- + +On Linux, create a build directory and then call cmake: + + nanopb/examples/cmake_simple$ mkdir build + nanopb/examples/cmake_simple$ cd build/ + nanopb/examples/cmake_simple/build$ cmake .. + nanopb/examples/cmake_simple/build$ make + +After that, you can run it with the command: ./simple + +#On other platforms, you first have to compile the protocol definition using +#the following command:: +# +# ../../generator-bin/protoc --nanopb_out=. simple.proto +# +#After that, add the following four files to your project and compile: +# +# simple.c simple.pb.c pb_encode.c pb_decode.c diff --git a/examples/cmake_simple/simple.c b/examples/cmake_simple/simple.c new file mode 100644 index 00000000..31272301 --- /dev/null +++ b/examples/cmake_simple/simple.c @@ -0,0 +1,68 @@ +#include +#include +#include +#include "simple.pb.h" + +int main() +{ + /* This is the buffer where we will store our message. */ + uint8_t buffer[128]; + size_t message_length; + bool status; + + /* Encode our message */ + { + /* Allocate space on the stack to store the message data. + * + * Nanopb generates simple struct definitions for all the messages. + * - check out the contents of simple.pb.h! */ + SimpleMessage message; + + /* Create a stream that will write to our buffer. */ + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Fill in the lucky number */ + message.lucky_number = 13; + + /* Now we are ready to encode the message! */ + status = pb_encode(&stream, SimpleMessage_fields, &message); + message_length = stream.bytes_written; + + /* Then just check for any errors.. */ + if (!status) + { + printf("Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + } + + /* Now we could transmit the message over network, store it in a file or + * wrap it to a pigeon's leg. + */ + + /* But because we are lazy, we will just decode it immediately. */ + + { + /* Allocate space for the decoded message. */ + SimpleMessage message; + + /* Create a stream that reads from the buffer. */ + pb_istream_t stream = pb_istream_from_buffer(buffer, message_length); + + /* Now we are ready to decode the message. */ + status = pb_decode(&stream, SimpleMessage_fields, &message); + + /* Check for errors... */ + if (!status) + { + printf("Decoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + + /* Print the data contained in the message. */ + printf("Your lucky number was %d!\n", message.lucky_number); + } + + return 0; +} + diff --git a/examples/cmake_simple/simple.proto b/examples/cmake_simple/simple.proto new file mode 100644 index 00000000..26e72f46 --- /dev/null +++ b/examples/cmake_simple/simple.proto @@ -0,0 +1,7 @@ +// A very simple protocol definition, consisting of only +// one message. + +message SimpleMessage { + required int32 lucky_number = 1; +} + -- cgit 1.2.3-korg From 93364463ac9ffc2206abe9d770152f073c671c2e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 13 Feb 2015 18:42:35 +0200 Subject: Update cmake_simple example readme --- examples/cmake_simple/README.txt | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/examples/cmake_simple/README.txt b/examples/cmake_simple/README.txt index e41b8408..aa0f3f3a 100644 --- a/examples/cmake_simple/README.txt +++ b/examples/cmake_simple/README.txt @@ -15,11 +15,4 @@ On Linux, create a build directory and then call cmake: After that, you can run it with the command: ./simple -#On other platforms, you first have to compile the protocol definition using -#the following command:: -# -# ../../generator-bin/protoc --nanopb_out=. simple.proto -# -#After that, add the following four files to your project and compile: -# -# simple.c simple.pb.c pb_encode.c pb_decode.c +On other platforms supported by CMake, refer to CMake instructions. -- cgit 1.2.3-korg From b836ac29dd212f173b0607c826d23b3e3da3d61a Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 13 Feb 2015 18:57:46 +0200 Subject: Lower required CMake version in example --- examples/cmake_simple/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cmake_simple/CMakeLists.txt b/examples/cmake_simple/CMakeLists.txt index 5439e8e6..5edfb61f 100644 --- a/examples/cmake_simple/CMakeLists.txt +++ b/examples/cmake_simple/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8.12) +cmake_minimum_required(VERSION 2.8) project(NANOPB_CMAKE_SIMPLE C) set(NANOPB_SRC_ROOT_FOLDER ${CMAKE_CURRENT_SOURCE_DIR}/../..) -- cgit 1.2.3-korg From 651e97456b24524a73070c6ea1c9c2696c02bd53 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 22 Feb 2015 15:28:26 +0200 Subject: Include libprotobuf in linux binary package. Previously this got included by bbfreeze, but apparently no more. Update issue 146 Status: FixedInGit --- tools/make_linux_package.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/make_linux_package.sh b/tools/make_linux_package.sh index 332c281b..6598936a 100755 --- a/tools/make_linux_package.sh +++ b/tools/make_linux_package.sh @@ -31,7 +31,8 @@ rm $DEST/generator/protoc-gen-nanopb.py # Package the protoc compiler cp `which protoc` $DEST/generator-bin/protoc.bin LIBPROTOC=$(ldd `which protoc` | grep -o '/.*libprotoc[^ ]*') -cp $LIBPROTOC $DEST/generator-bin/ +LIBPROTOBUF=$(ldd `which protoc` | grep -o '/.*libprotobuf[^ ]*') +cp $LIBPROTOC $LIBPROTOBUF $DEST/generator-bin/ cat > $DEST/generator-bin/protoc << EOF #!/bin/bash SCRIPTDIR=\$(dirname "\$0") -- cgit 1.2.3-korg From 4a6580726ec2a0b71d8f01133d5c38a469eac497 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 26 Feb 2015 17:33:36 +0200 Subject: Fix generator error when long_names:false is combined with Oneofs. Update issue 147 Status: FixedInGit --- generator/nanopb_generator.py | 3 +++ tests/options/options.proto | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 4c01502c..7ee06523 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -587,6 +587,9 @@ class OneOf(Field): self.name = oneof_desc.name self.ctype = 'union' self.fields = [] + self.allocation = 'ONEOF' + self.default = None + self.rules = 'ONEOF' def add_field(self, field): if field.allocation == 'CALLBACK': diff --git a/tests/options/options.proto b/tests/options/options.proto index a8e557be..cdcffeb7 100644 --- a/tests/options/options.proto +++ b/tests/options/options.proto @@ -75,3 +75,12 @@ message SkippedMessage option (nanopb_msgopt).skip_message = true; required int32 foo = 1; } + +// Message with oneof field +message OneofMessage +{ + oneof foo { + int32 bar = 1; + } +} + -- cgit 1.2.3-korg From 5c16a116ec2db9d19e42c5682cbfbf4f2b87ad87 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 26 Feb 2015 18:16:25 +0200 Subject: Better error messages for syntax errors in .options file --- generator/nanopb_generator.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 7ee06523..82602908 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1126,14 +1126,28 @@ def read_options_file(infile): [(namemask, options), ...] ''' results = [] - for line in infile: + for i, line in enumerate(infile): line = line.strip() if not line or line.startswith('//') or line.startswith('#'): continue parts = line.split(None, 1) + + if len(parts) < 2: + sys.stderr.write("%s:%d: " % (infile.name, i + 1) + + "Option lines should have space between field name and options. " + + "Skipping line: '%s'\n" % line) + continue + opts = nanopb_pb2.NanoPBOptions() - text_format.Merge(parts[1], opts) + + try: + text_format.Merge(parts[1], opts) + except Exception, e: + sys.stderr.write("%s:%d: " % (infile.name, i + 1) + + "Unparseable option line: '%s'. " % line + + "Error: %s\n" % str(e)) + continue results.append((parts[0], opts)) return results -- cgit 1.2.3-korg From ef422656a57b3f472b192691a40f48d0d72f2927 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 7 Mar 2015 10:25:09 +0200 Subject: Fix oneof submessage initialization bug. Update issue 149 Status: FixedInGit --- pb_decode.c | 4 ++++ tests/oneof/decode_oneof.c | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 9d25dc61..4d8c7cbe 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -396,6 +396,10 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t case PB_HTYPE_ONEOF: *(pb_size_t*)iter->pSize = iter->pos->tag; + if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) + { + pb_message_set_to_defaults((const pb_field_t*)iter->pos->ptr, iter->pData); + } return func(stream, iter->pos, iter->pData); default: diff --git a/tests/oneof/decode_oneof.c b/tests/oneof/decode_oneof.c index 83b4702c..37075cd6 100644 --- a/tests/oneof/decode_oneof.c +++ b/tests/oneof/decode_oneof.c @@ -2,6 +2,7 @@ #include #include +#include #include #include "oneof.pb.h" #include "test_helpers.h" @@ -10,9 +11,12 @@ /* Test the 'OneOfMessage' */ int test_oneof_1(pb_istream_t *stream, int option) { - OneOfMessage msg = OneOfMessage_init_zero; + OneOfMessage msg; int status = 0; + /* To better catch initialization errors */ + memset(&msg, 0xAA, sizeof(msg)); + if (!pb_decode(stream, OneOfMessage_fields, &msg)) { printf("Decoding failed: %s\n", PB_GET_ERROR(stream)); @@ -124,4 +128,4 @@ int main(int argc, char **argv) } return 0; -} \ No newline at end of file +} -- cgit 1.2.3-korg From 6db1857cb326653c39c930abb9a9a6d447564e1e Mon Sep 17 00:00:00 2001 From: Michael Haberler Date: Sun, 22 Mar 2015 02:17:30 +0100 Subject: nanopb.proto: add msgid message-level option --- generator/proto/nanopb.proto | 3 +++ 1 file changed, 3 insertions(+) diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index a1b24936..3f2ed8e0 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -56,6 +56,9 @@ message NanoPBOptions { // Generate oneof fields as normal optional fields instead of union. optional bool no_unions = 8 [default = false]; + + // integer type tag for a message + optional uint32 msgid = 9; } // Extensions to protoc 'Descriptor' type in order to define options -- cgit 1.2.3-korg From 942e82a82b88a073ad9654f54c11497e48b8fe09 Mon Sep 17 00:00:00 2001 From: Michael Haberler Date: Sun, 22 Mar 2015 02:18:24 +0100 Subject: nanopb_generator.py: emit macros for msgid message type id use --- generator/nanopb_generator.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 82602908..d316fad2 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -667,6 +667,9 @@ class Message: self.oneofs = {} no_unions = [] + if message_options.msgid: + self.msgid = message_options.msgid + if hasattr(desc, 'oneof_decl'): for i, f in enumerate(desc.oneof_decl): oneof_options = get_nanopb_suboptions(desc, message_options, self.name + f.name) @@ -854,7 +857,7 @@ def parse_file(fdesc, file_options): if message_options.skip_message: continue - + messages.append(Message(names, message, message_options)) for enum in message.enum_type: enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name) @@ -917,7 +920,7 @@ def make_identifier(headername): result += '_' return result -def generate_header(dependencies, headername, enums, messages, extensions, options): +def generate_header(noext, dependencies, headername, enums, messages, extensions, options): '''Generate content for a header file. Generates strings, which should be concatenated and stored to file. ''' @@ -1001,7 +1004,30 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio identifier = '%s_size' % msg.name yield '#define %-40s %s\n' % (identifier, msize) yield '\n' - + + yield '/* helper macros for message type ids if set with */\n' + yield '/* option (nanopb_msgopt).msgid = ; */\n\n' + + yield '#ifdef PB_MSGID\n' + for msg in messages: + if hasattr(msg,'msgid'): + yield '#define PB_MSG_%d %s\n' % (msg.msgid, msg.name) + yield '\n' + + yield '#define %s_MESSAGES \\\n' % (noext.upper()) + + for msg in messages: + m = "-1" + msize = msg.encoded_size(messages) + if msize is not None: + m = msize + if hasattr(msg,'msgid'): + yield '\tPB_MSG(%d,%s,%s) \\\n' % (msg.msgid, m, msg.name) + yield '\n' + + yield '#endif\n\n' + + yield '#ifdef __cplusplus\n' yield '} /* extern "C" */\n' yield '#endif\n' @@ -1287,7 +1313,7 @@ def process_file(filename, fdesc, options): excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude dependencies = [d for d in fdesc.dependency if d not in excludes] - headerdata = ''.join(generate_header(dependencies, headerbasename, enums, + headerdata = ''.join(generate_header(noext,dependencies, headerbasename, enums, messages, extensions, options)) sourcedata = ''.join(generate_source(headerbasename, enums, -- cgit 1.2.3-korg From 8c14e736034b82d65b43db529161dd281a37816a Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 22 Mar 2015 11:17:57 +0200 Subject: Use make_identifier() to create the name for FOO_MESSAGES define. This handles special charaters like in "my-file.proto" properly. Also use headerbasename instead of full path, so that compiling files with relative path doesn't create symbols like FOO/BAR/BAZ_MESSAGES. --- generator/nanopb_generator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index d316fad2..4673ac67 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -920,7 +920,7 @@ def make_identifier(headername): result += '_' return result -def generate_header(noext, dependencies, headername, enums, messages, extensions, options): +def generate_header(dependencies, headername, enums, messages, extensions, options): '''Generate content for a header file. Generates strings, which should be concatenated and stored to file. ''' @@ -1014,7 +1014,8 @@ def generate_header(noext, dependencies, headername, enums, messages, extensions yield '#define PB_MSG_%d %s\n' % (msg.msgid, msg.name) yield '\n' - yield '#define %s_MESSAGES \\\n' % (noext.upper()) + symbol = make_identifier(headername.split('.')[0]) + yield '#define %s_MESSAGES \\\n' % symbol for msg in messages: m = "-1" @@ -1313,7 +1314,7 @@ def process_file(filename, fdesc, options): excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude dependencies = [d for d in fdesc.dependency if d not in excludes] - headerdata = ''.join(generate_header(noext,dependencies, headerbasename, enums, + headerdata = ''.join(generate_header(dependencies, headerbasename, enums, messages, extensions, options)) sourcedata = ''.join(generate_source(headerbasename, enums, -- cgit 1.2.3-korg From 62a7e4ff2b3487ec259de90db9215c4ad8838dd6 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 22 Mar 2015 11:19:43 +0200 Subject: Add simple test case for message ids --- tests/options/options.expected | 7 +++++++ tests/options/options.proto | 3 +++ 2 files changed, 10 insertions(+) diff --git a/tests/options/options.expected b/tests/options/options.expected index dbd279ba..ad74a525 100644 --- a/tests/options/options.expected +++ b/tests/options/options.expected @@ -7,4 +7,11 @@ Message5_EnumValue1 } pb_packed my_packed_struct; ! skipped_field ! SkippedMessage +#define PB_MSG_103 Message3 +#define PB_MSG_104 Message4 +#define PB_MSG_105 Message5 +#define OPTIONS_MESSAGES \\ +\s+PB_MSG\(103,[0-9]*,Message3\) \\ +\s+PB_MSG\(104,-1,Message4\) \\ +\s+PB_MSG\(105,[0-9]*,Message5\) \\ diff --git a/tests/options/options.proto b/tests/options/options.proto index cdcffeb7..b7050418 100644 --- a/tests/options/options.proto +++ b/tests/options/options.proto @@ -22,12 +22,14 @@ message Message2 // Field level options message Message3 { + option (nanopb_msgopt).msgid = 103; required string fieldsize = 1 [(nanopb).max_size = 40]; } // Forced callback field message Message4 { + option (nanopb_msgopt).msgid = 104; required int32 int32_callback = 1 [(nanopb).type = FT_CALLBACK]; } @@ -47,6 +49,7 @@ message EnumTest // Short enum names inside message message Message5 { + option (nanopb_msgopt).msgid = 105; enum Enum2 { option (nanopb_enumopt).long_names = false; -- cgit 1.2.3-korg From 02652148fc9648e258f091298d7fae6159069823 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 3 Apr 2015 19:44:23 +0300 Subject: Generate #defines for plain message ids --- generator/nanopb_generator.py | 10 +++++++--- tests/options/options.expected | 1 + 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 4673ac67..fc32e2c5 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1005,9 +1005,8 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio yield '#define %-40s %s\n' % (identifier, msize) yield '\n' - yield '/* helper macros for message type ids if set with */\n' - yield '/* option (nanopb_msgopt).msgid = ; */\n\n' - + yield '/* Message IDs (where set with "msgid" option) */\n' + yield '#ifdef PB_MSGID\n' for msg in messages: if hasattr(msg,'msgid'): @@ -1026,6 +1025,11 @@ def generate_header(dependencies, headername, enums, messages, extensions, optio yield '\tPB_MSG(%d,%s,%s) \\\n' % (msg.msgid, m, msg.name) yield '\n' + for msg in messages: + if hasattr(msg,'msgid'): + yield '#define %s_msgid %d\n' % (msg.name, msg.msgid) + yield '\n' + yield '#endif\n\n' diff --git a/tests/options/options.expected b/tests/options/options.expected index ad74a525..63ba0fd8 100644 --- a/tests/options/options.expected +++ b/tests/options/options.expected @@ -14,4 +14,5 @@ Message5_EnumValue1 \s+PB_MSG\(103,[0-9]*,Message3\) \\ \s+PB_MSG\(104,-1,Message4\) \\ \s+PB_MSG\(105,[0-9]*,Message5\) \\ +#define Message5_msgid 105 -- cgit 1.2.3-korg From 90833934a28c4ed5dc57ec649ee3d37dca57fb89 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 3 Apr 2015 19:46:57 +0300 Subject: Add basic docs for msgid functionality --- docs/concepts.rst | 19 +++++++++++++++++++ docs/reference.rst | 4 ++++ 2 files changed, 23 insertions(+) diff --git a/docs/concepts.rst b/docs/concepts.rst index 0df5ad66..b0fd43aa 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -299,6 +299,25 @@ An example of this is available in *tests/test_encode_extensions.c* and .. _`extension fields`: https://developers.google.com/protocol-buffers/docs/proto#extensions +Message framing +=============== +Protocol Buffers does not specify a method of framing the messages for transmission. +This is something that must be provided by the library user, as there is no one-size-fits-all +solution. Typical needs for a framing format are to: + +1. Encode the message length. +2. Encode the message type. +3. Perform any synchronization and error checking that may be needed depending on application. + +For example UDP packets already fullfill all the requirements, and TCP streams typically only +need a way to identify the message length and type. Lower level interfaces such as serial ports +may need a more robust frame format, such as HDLC (high-level data link control). + +Nanopb provides a few helpers to facilitate implementing framing formats: + +1. Functions *pb_encode_delimited* and *pb_decode_delimited* prefix the message data with a varint-encoded length. +2. Union messages and oneofs are supported in order to implement top-level container messages. +3. Message IDs can be specified using the *(nanopb_msgopt).msgid* option and can then be accessed from the header. Return values and error handling ================================ diff --git a/docs/reference.rst b/docs/reference.rst index e16500ad..296bc78c 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -91,6 +91,10 @@ packed_struct Make the generated structures packed. NOTE: This cannot be used on CPUs that break on unaligned accesses to variables. skip_message Skip the whole message from generation. +no_unions Generate 'oneof' fields as optional fields + instead of C unions. +msgid Specifies a unique id for this message type. + Can be used by user code as an identifier. ============================ ================================================ These options can be defined for the .proto files before they are converted -- cgit 1.2.3-korg From 3bcdd49eac3aabcc121173c0023834f8bcf0a3cb Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 3 Apr 2015 20:08:05 +0300 Subject: Always define enum long names so that cross-file references work. Update issue 118 Status: FixedInGit --- generator/nanopb_generator.py | 7 +++++++ tests/regression/issue_118/SConscript | 11 +++++++++++ tests/regression/issue_118/enumdef.proto | 6 ++++++ tests/regression/issue_118/enumuse.proto | 5 +++++ 4 files changed, 29 insertions(+) create mode 100644 tests/regression/issue_118/SConscript create mode 100644 tests/regression/issue_118/enumdef.proto create mode 100644 tests/regression/issue_118/enumuse.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index fc32e2c5..eebe5404 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -164,6 +164,13 @@ class Enum: result = 'typedef enum _%s {\n' % self.names result += ',\n'.join([" %s = %d" % x for x in self.values]) result += '\n} %s;' % self.names + + if not self.options.long_names: + # Define the long names always so that enum value references + # from other files work properly. + for i, x in enumerate(self.values): + result += '\n#define %s %s' % (self.value_longnames[i], x[0]) + return result class Field: diff --git a/tests/regression/issue_118/SConscript b/tests/regression/issue_118/SConscript new file mode 100644 index 00000000..c90e8497 --- /dev/null +++ b/tests/regression/issue_118/SConscript @@ -0,0 +1,11 @@ +# Regression test for Issue 118: Short enum names in imported proto files are not honoured + +Import("env") +env.Append(PROTOCPATH = "#regression/issue_118") + +env.NanopbProto("enumdef") +env.Object('enumdef.pb.c') + +env.NanopbProto(["enumuse", "enumdef.proto"]) +env.Object('enumuse.pb.c') + diff --git a/tests/regression/issue_118/enumdef.proto b/tests/regression/issue_118/enumdef.proto new file mode 100644 index 00000000..830d2988 --- /dev/null +++ b/tests/regression/issue_118/enumdef.proto @@ -0,0 +1,6 @@ +import 'nanopb.proto'; + +enum MyEnum { + option (nanopb_enumopt).long_names = false; + FOOBAR = 1; +} diff --git a/tests/regression/issue_118/enumuse.proto b/tests/regression/issue_118/enumuse.proto new file mode 100644 index 00000000..d778fb8f --- /dev/null +++ b/tests/regression/issue_118/enumuse.proto @@ -0,0 +1,5 @@ +import 'enumdef.proto'; + +message MyMessage { + required MyEnum myenum = 1 [default = FOOBAR]; +} -- cgit 1.2.3-korg From 88f2dc5810b88f88b516bb6f11fc3cba1c255655 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 3 Apr 2015 20:43:13 +0300 Subject: Improve comment support in .options files. Update issue 145 Status: FixedInGit --- generator/nanopb_generator.py | 9 +++++++-- tests/regression/issue_145/SConscript | 9 +++++++++ tests/regression/issue_145/comments.expected | 3 +++ tests/regression/issue_145/comments.options | 6 ++++++ tests/regression/issue_145/comments.proto | 5 +++++ 5 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 tests/regression/issue_145/SConscript create mode 100644 tests/regression/issue_145/comments.expected create mode 100644 tests/regression/issue_145/comments.options create mode 100644 tests/regression/issue_145/comments.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index eebe5404..b1ee04e0 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -4,6 +4,7 @@ nanopb_version = "nanopb-0.3.3-dev" import sys +import re try: # Add some dummy imports to keep packaging tools happy. @@ -1164,9 +1165,13 @@ def read_options_file(infile): [(namemask, options), ...] ''' results = [] - for i, line in enumerate(infile): + data = infile.read() + data = re.sub('/\*.*?\*/', '', data, flags = re.MULTILINE) + data = re.sub('//.*?$', '', data, flags = re.MULTILINE) + data = re.sub('#.*?$', '', data, flags = re.MULTILINE) + for i, line in enumerate(data.split('\n')): line = line.strip() - if not line or line.startswith('//') or line.startswith('#'): + if not line: continue parts = line.split(None, 1) diff --git a/tests/regression/issue_145/SConscript b/tests/regression/issue_145/SConscript new file mode 100644 index 00000000..0b793a7a --- /dev/null +++ b/tests/regression/issue_145/SConscript @@ -0,0 +1,9 @@ +# Regression test for Issue 145: Allow /* */ and // comments in .options files + +Import("env") + +env.NanopbProto(["comments", "comments.options"]) +env.Object('comments.pb.c') + +env.Match(['comments.pb.h', 'comments.expected']) + diff --git a/tests/regression/issue_145/comments.expected b/tests/regression/issue_145/comments.expected new file mode 100644 index 00000000..7f874587 --- /dev/null +++ b/tests/regression/issue_145/comments.expected @@ -0,0 +1,3 @@ +char foo\[5\]; +char bar\[16\]; + diff --git a/tests/regression/issue_145/comments.options b/tests/regression/issue_145/comments.options new file mode 100644 index 00000000..89959ba2 --- /dev/null +++ b/tests/regression/issue_145/comments.options @@ -0,0 +1,6 @@ +/* Block comment */ +# Line comment +// Line comment +DummyMessage.foo /* Block comment */ max_size:5 +DummyMessage.bar max_size:16 # Line comment ### + diff --git a/tests/regression/issue_145/comments.proto b/tests/regression/issue_145/comments.proto new file mode 100644 index 00000000..4e86b302 --- /dev/null +++ b/tests/regression/issue_145/comments.proto @@ -0,0 +1,5 @@ +message DummyMessage { + required string foo = 1; + required string bar = 2; +} + -- cgit 1.2.3-korg From b1b28517708ec6d03f7aba5d4f3c8252a7880d20 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 3 Apr 2015 21:00:57 +0300 Subject: Fix for test case build rules --- tests/regression/issue_118/SConscript | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/regression/issue_118/SConscript b/tests/regression/issue_118/SConscript index c90e8497..833d9dec 100644 --- a/tests/regression/issue_118/SConscript +++ b/tests/regression/issue_118/SConscript @@ -1,6 +1,7 @@ # Regression test for Issue 118: Short enum names in imported proto files are not honoured Import("env") +env = env.Clone() env.Append(PROTOCPATH = "#regression/issue_118") env.NanopbProto("enumdef") -- cgit 1.2.3-korg From 1cfe56060ce3a386c03239396d51a93baa56895b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 3 Apr 2015 21:07:14 +0300 Subject: Clear callbacks for union fields. Update issue 148 Status: FixedInGit --- pb_decode.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pb_decode.c b/pb_decode.c index 4d8c7cbe..b21bfe37 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -398,6 +398,9 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t *(pb_size_t*)iter->pSize = iter->pos->tag; if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) { + /* We memset to zero so that any callbacks are set to NULL. + * Then set any default values. */ + memset(iter->pData, 0, iter->pos->data_size); pb_message_set_to_defaults((const pb_field_t*)iter->pos->ptr, iter->pData); } return func(stream, iter->pos, iter->pData); -- cgit 1.2.3-korg From 400fba7f8f4a9758edaa02c67159fba0bbc60e8d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 3 Apr 2015 21:14:06 +0300 Subject: Update changelog --- CHANGELOG.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index d10364f4..a1875a09 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,15 @@ +nanopb-0.3.3 (2015-04-xx) + Fix missing files in Linux binary package (issue 146) + Fix generator bug when oneof is first field in a message. (issue 142) + Fix generator error when long_names:false is combined with Oneofs. (issue 147) + Fix oneof submessage initialization bug. (issue 149) + Don't crash when callback is inside oneof field. (issue 148) + Always define enum long names so that cross-file references work. (issue 118) + Add msgid generator option. (issue 151) + Improve comment support in .options files. (issue 145) + Updates for the CMake rule file, add cmake example. + Better error messages for syntax errors in .options file + nanopb-0.3.2 (2015-01-24) Fix memory leaks with PB_ENABLE_MALLOC with some submessage hierarchies (issue 138) Implement support for oneofs (C unions). (issues 131, 141) -- cgit 1.2.3-korg From 9e4ac1ba9b27ebb8d37a6c58ae11fa748ddb7384 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 8 Apr 2015 18:05:25 +0300 Subject: Fix problem with plugin options on Python 2.7.2 and older. Update issue 153 Status: FixedInGit --- generator/nanopb_generator.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index b1ee04e0..49ce4da6 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1384,8 +1384,15 @@ def main_plugin(): data = sys.stdin.read() request = plugin_pb2.CodeGeneratorRequest.FromString(data) + try: + # Versions of Python prior to 2.7.3 do not support unicode + # input to shlex.split(). Try to convert to str if possible. + params = str(request.parameter) + except UnicodeEncodeError: + params = request.parameter + import shlex - args = shlex.split(request.parameter) + args = shlex.split(params) options, dummy = optparser.parse_args(args) Globals.verbose_options = options.verbose -- cgit 1.2.3-korg From 2024c8e3dc8a197cbb0c842a6b48a64c7b71368d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 8 Apr 2015 18:38:03 +0300 Subject: Switch to .tar.gz format for Mac OS X packages. Update issue 154 Status: FixedInGit --- tools/make_mac_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/make_mac_package.sh b/tools/make_mac_package.sh index 21cd170d..32bba5cc 100755 --- a/tools/make_mac_package.sh +++ b/tools/make_mac_package.sh @@ -45,5 +45,5 @@ EOF chmod +x $DEST/generator-bin/protoc # Tar it all up -( cd dist; zip -r $VERSION.zip $VERSION ) +( cd dist; tar -czf $VERSION.tar.gz $VERSION ) -- cgit 1.2.3-korg From 3486f67e0de8cd17009be6df493af1d9447935c7 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 10 Apr 2015 20:46:33 +0300 Subject: Publishing nanopb-0.3.3 --- CHANGELOG.txt | 4 +++- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index a1875a09..8c6f8e64 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -3,7 +3,9 @@ nanopb-0.3.3 (2015-04-xx) Fix generator bug when oneof is first field in a message. (issue 142) Fix generator error when long_names:false is combined with Oneofs. (issue 147) Fix oneof submessage initialization bug. (issue 149) - Don't crash when callback is inside oneof field. (issue 148) + Fix problem with plugin options on Python 2.7.2 and older. (issue 153) + Fix crash when callback is inside oneof field. (issue 148) + Switch to .tar.gz format for Mac OS X packages. (issue 154) Always define enum long names so that cross-file references work. (issue 118) Add msgid generator option. (issue 151) Improve comment support in .options files. (issue 145) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 49ce4da6..364a22cf 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.3-dev" +nanopb_version = "nanopb-0.3.3" import sys import re diff --git a/pb.h b/pb.h index 47941ebf..d0fa871d 100644 --- a/pb.h +++ b/pb.h @@ -50,7 +50,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.3-dev +#define NANOPB_VERSION nanopb-0.3.3 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 47e744f3486812d38ba7ad72d2025bebd2995aae Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 10 Apr 2015 21:07:25 +0300 Subject: Setting version to 0.3.4-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 364a22cf..8fd45372 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.3" +nanopb_version = "nanopb-0.3.4-dev" import sys import re diff --git a/pb.h b/pb.h index d0fa871d..649a25e9 100644 --- a/pb.h +++ b/pb.h @@ -50,7 +50,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.3 +#define NANOPB_VERSION nanopb-0.3.4-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 0dcfa096e69c8ea081d934086ba2f11c8f5abde7 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 28 Apr 2015 18:14:24 +0300 Subject: Prefer python2 in generator/protoc-gen-nanopb. Update issue 155 Status: FixedInGit --- generator/protoc-gen-nanopb | 1 + 1 file changed, 1 insertion(+) diff --git a/generator/protoc-gen-nanopb b/generator/protoc-gen-nanopb index 2de56216..595c65f7 100755 --- a/generator/protoc-gen-nanopb +++ b/generator/protoc-gen-nanopb @@ -10,4 +10,5 @@ # --plugin= on the command line. MYPATH=$(dirname "$0") +PYTHON=$(which python2 || which python) exec python "$MYPATH/nanopb_generator.py" --protoc-plugin -- cgit 1.2.3-korg From 56fdc1fc56b98dda82561d90a615698d42ee59a0 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 28 Apr 2015 18:16:51 +0300 Subject: Fix for previous (issue #155) --- generator/protoc-gen-nanopb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/protoc-gen-nanopb b/generator/protoc-gen-nanopb index 595c65f7..6dc468d3 100755 --- a/generator/protoc-gen-nanopb +++ b/generator/protoc-gen-nanopb @@ -11,4 +11,4 @@ MYPATH=$(dirname "$0") PYTHON=$(which python2 || which python) -exec python "$MYPATH/nanopb_generator.py" --protoc-plugin +exec $PYTHON "$MYPATH/nanopb_generator.py" --protoc-plugin -- cgit 1.2.3-korg From 775df2b3902d87b7c816ea165ac5d1e83980a509 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 7 May 2015 19:59:56 +0300 Subject: Add download link to readme --- README.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.txt b/README.txt index 0f2ade8b..f68ac3d4 100644 --- a/README.txt +++ b/README.txt @@ -3,7 +3,7 @@ especially suitable for use in microcontrollers, but fits any memory restricted system. Homepage: http://kapsi.fi/~jpa/nanopb/ - +Downloads: http://koti.kapsi.fi/~jpa/nanopb/download/ -- cgit 1.2.3-korg From 01ac4d6f342760aa85901d041c0dbf837eccb77a Mon Sep 17 00:00:00 2001 From: Nicolas Colomer Date: Tue, 16 Jun 2015 13:47:34 +0200 Subject: PlatformIO Library Registry manifest file --- library.json | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 library.json diff --git a/library.json b/library.json new file mode 100644 index 00000000..534c3bf5 --- /dev/null +++ b/library.json @@ -0,0 +1,21 @@ +{ + "name": "Nanopb", + "keywords": "protocol buffers, protobuf, google", + "description": "Nanopb is a plain-C implementation of Google's Protocol Buffers data format. It is targeted at 32 bit microcontrollers, but is also fit for other embedded systems with tight (2-10 kB ROM, <1 kB RAM) memory constraints.", + "repository": { + "type": "git", + "url": "https://github.com/nanopb/nanopb.git" + }, + "authors": [{ + "name": "Petteri Aimonen", + "email": "jpa@nanopb.mail.kapsi.fi", + "url": "http://koti.kapsi.fi/jpa/nanopb/" + }], + "frameworks": [ + "arduino" + ], + "platforms": [ + "atmelavr", + "teensy" + ] +} -- cgit 1.2.3-korg From 404b1866e36c979b72dfcd57cf5ecca8058772e6 Mon Sep 17 00:00:00 2001 From: Ivan Kravets Date: Wed, 17 Jun 2015 13:05:01 +0300 Subject: Specify additional fields for @PlatformIO Registry --- library.json | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/library.json b/library.json index 534c3bf5..30a56c94 100644 --- a/library.json +++ b/library.json @@ -11,11 +11,12 @@ "email": "jpa@nanopb.mail.kapsi.fi", "url": "http://koti.kapsi.fi/jpa/nanopb/" }], - "frameworks": [ - "arduino" + "include": [ + "*.c", + "*.cpp", + "*.h" ], - "platforms": [ - "atmelavr", - "teensy" - ] + "examples": "examples/*/*.c", + "frameworks": "*", + "platforms": "*" } -- cgit 1.2.3-korg From c31f2527b14aa35ff7d1fb844255c7722d1dd7f7 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 1 Jul 2015 19:48:43 +0300 Subject: Add link to forum to the readme --- README.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/README.txt b/README.txt index f68ac3d4..184e8c14 100644 --- a/README.txt +++ b/README.txt @@ -4,6 +4,7 @@ restricted system. Homepage: http://kapsi.fi/~jpa/nanopb/ Downloads: http://koti.kapsi.fi/~jpa/nanopb/download/ +Forum: https://groups.google.com/forum/#!forum/nanopb -- cgit 1.2.3-korg From 5497a1dfc91a86965383cdd1652e348345400435 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 11 Jul 2015 09:23:47 +0300 Subject: Add note about running tests on Mac OS X --- README.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.txt b/README.txt index 184e8c14..e9f7a7d1 100644 --- a/README.txt +++ b/README.txt @@ -59,4 +59,7 @@ so you need to have that installed. To run the tests: This will show the progress of various test cases. If the output does not end in an error, the test cases were successful. - +Note: Mac OS X by default aliases 'clang' as 'gcc', while not actually +supporting the same command line options as gcc does. To run tests on +Mac OS X, use: "scons CC=clang CXX=clang". Same way can be used to run +tests with different compilers on any platform. -- cgit 1.2.3-korg From 612a51c608504f59fcd86c74eedac2903f088a0a Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 12 Sep 2015 12:15:27 +0300 Subject: Add packed_enum option to generator. This can be generally useful for saving RAM, but also makes it easier to test for issue #164. --- generator/nanopb_generator.py | 8 +++++++- generator/proto/nanopb.proto | 3 +++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 8fd45372..aaa0d2f1 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -160,11 +160,17 @@ class Enum: self.values = [(names + x.name, x.number) for x in desc.value] self.value_longnames = [self.names + x.name for x in desc.value] + self.packed = enum_options.packed_enum def __str__(self): result = 'typedef enum _%s {\n' % self.names result += ',\n'.join([" %s = %d" % x for x in self.values]) - result += '\n} %s;' % self.names + result += '\n}' + + if self.packed: + result += ' pb_packed' + + result += ' %s;' % self.names if not self.options.long_names: # Define the long names always so that enum value references diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index 3f2ed8e0..b8671bbd 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -51,6 +51,9 @@ message NanoPBOptions { // accesses to variables. optional bool packed_struct = 5 [default = false]; + // Add 'packed' attribute to generated enums. + optional bool packed_enum = 10 [default = false]; + // Skip this message optional bool skip_message = 6 [default = false]; -- cgit 1.2.3-korg From 1582038e37771ade214f0627c3ecbf6e1ba69946 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 12 Sep 2015 12:16:58 +0300 Subject: Add testcase for issue #164 --- tests/enum_sizes/SConscript | 12 +++++ tests/enum_sizes/enumsizes.proto | 84 ++++++++++++++++++++++++++++++++++ tests/enum_sizes/enumsizes_unittests.c | 72 +++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+) create mode 100644 tests/enum_sizes/SConscript create mode 100644 tests/enum_sizes/enumsizes.proto create mode 100644 tests/enum_sizes/enumsizes_unittests.c diff --git a/tests/enum_sizes/SConscript b/tests/enum_sizes/SConscript new file mode 100644 index 00000000..048592ed --- /dev/null +++ b/tests/enum_sizes/SConscript @@ -0,0 +1,12 @@ +# Test that different sizes of enum fields are properly encoded and decoded. + +Import('env') + +env.NanopbProto('enumsizes') + +p = env.Program(["enumsizes_unittests.c", + "enumsizes.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_decode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) diff --git a/tests/enum_sizes/enumsizes.proto b/tests/enum_sizes/enumsizes.proto new file mode 100644 index 00000000..f9ab0b7f --- /dev/null +++ b/tests/enum_sizes/enumsizes.proto @@ -0,0 +1,84 @@ +/* Test handling of enums with different value ranges. + * Depending on compiler and the packed_enum setting, the datatypes + * for enums can be either signed or unsigned. In past this has caused + * a bit of a problem for the encoder/decoder (issue #164). + */ + +import 'nanopb.proto'; + +option (nanopb_fileopt).long_names = false; + +enum UnpackedUint8 { + option (nanopb_enumopt).packed_enum = false; + UU8_MIN = 0; + UU8_MAX = 255; +} + +enum PackedUint8 { + option (nanopb_enumopt).packed_enum = true; + PU8_MIN = 0; + PU8_MAX = 255; +} + +enum UnpackedInt8 { + option (nanopb_enumopt).packed_enum = false; + UI8_MIN = -128; + UI8_MAX = 127; +} + +enum PackedInt8 { + option (nanopb_enumopt).packed_enum = true; + PI8_MIN = -128; + PI8_MAX = 127; +} + +enum UnpackedUint16 { + option (nanopb_enumopt).packed_enum = false; + UU16_MIN = 0; + UU16_MAX = 65535; +} + +enum PackedUint16 { + option (nanopb_enumopt).packed_enum = true; + PU16_MIN = 0; + PU16_MAX = 65535; +} + +enum UnpackedInt16 { + option (nanopb_enumopt).packed_enum = false; + UI16_MIN = -32768; + UI16_MAX = 32767; +} + +enum PackedInt16 { + option (nanopb_enumopt).packed_enum = true; + PI16_MIN = -32768; + PI16_MAX = 32767; +} + +/* Protobuf supports enums up to 32 bits. + * The 32 bit case is covered by HugeEnum in the alltypes test. + */ + +message PackedEnums { + required PackedUint8 u8_min = 1; + required PackedUint8 u8_max = 2; + required PackedInt8 i8_min = 3; + required PackedInt8 i8_max = 4; + required PackedUint16 u16_min = 5; + required PackedUint16 u16_max = 6; + required PackedInt16 i16_min = 7; + required PackedInt16 i16_max = 8; +} + +message UnpackedEnums { + required UnpackedUint8 u8_min = 1; + required UnpackedUint8 u8_max = 2; + required UnpackedInt8 i8_min = 3; + required UnpackedInt8 i8_max = 4; + required UnpackedUint16 u16_min = 5; + required UnpackedUint16 u16_max = 6; + required UnpackedInt16 i16_min = 7; + required UnpackedInt16 i16_max = 8; +} + diff --git a/tests/enum_sizes/enumsizes_unittests.c b/tests/enum_sizes/enumsizes_unittests.c new file mode 100644 index 00000000..5606895a --- /dev/null +++ b/tests/enum_sizes/enumsizes_unittests.c @@ -0,0 +1,72 @@ +#include +#include +#include +#include +#include "unittests.h" +#include "enumsizes.pb.h" + +int main() +{ + int status = 0; + + UnpackedEnums msg1 = { + UU8_MIN, UU8_MAX, + UI8_MIN, UI8_MAX, + UU16_MIN, UU16_MAX, + UI16_MIN, UI16_MAX, + }; + + PackedEnums msg2; + UnpackedEnums msg3; + uint8_t buf[256]; + size_t msgsize; + + COMMENT("Step 1: unpacked enums -> protobuf"); + { + pb_ostream_t s = pb_ostream_from_buffer(buf, sizeof(buf)); + TEST(pb_encode(&s, UnpackedEnums_fields, &msg1)); + msgsize = s.bytes_written; + } + + COMMENT("Step 2: protobuf -> packed enums"); + { + pb_istream_t s = pb_istream_from_buffer(buf, msgsize); + TEST(pb_decode(&s, PackedEnums_fields, &msg2)); + + TEST(msg1.u8_min == (int)msg2.u8_min); + TEST(msg1.u8_max == (int)msg2.u8_max); + TEST(msg1.i8_min == (int)msg2.i8_min); + TEST(msg1.i8_max == (int)msg2.i8_max); + TEST(msg1.u16_min == (int)msg2.u16_min); + TEST(msg1.u16_max == (int)msg2.u16_max); + TEST(msg1.i16_min == (int)msg2.i16_min); + TEST(msg1.i16_max == (int)msg2.i16_max); + } + + COMMENT("Step 3: packed enums -> protobuf"); + { + pb_ostream_t s = pb_ostream_from_buffer(buf, sizeof(buf)); + TEST(pb_encode(&s, PackedEnums_fields, &msg2)); + msgsize = s.bytes_written; + } + + COMMENT("Step 4: protobuf -> unpacked enums"); + { + pb_istream_t s = pb_istream_from_buffer(buf, msgsize); + TEST(pb_decode(&s, UnpackedEnums_fields, &msg3)); + + TEST(msg1.u8_min == (int)msg3.u8_min); + TEST(msg1.u8_max == (int)msg3.u8_max); + TEST(msg1.i8_min == (int)msg3.i8_min); + TEST(msg1.i8_max == (int)msg3.i8_max); + TEST(msg1.u16_min == (int)msg2.u16_min); + TEST(msg1.u16_max == (int)msg2.u16_max); + TEST(msg1.i16_min == (int)msg2.i16_min); + TEST(msg1.i16_max == (int)msg2.i16_max); + } + + if (status != 0) + fprintf(stdout, "\n\nSome tests FAILED!\n"); + + return status; +} -- cgit 1.2.3-korg From 708084e7883a95dd7fd315cdc909f6664491c043 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 12 Sep 2015 13:04:22 +0300 Subject: Fix handling of unsigned 8- or 16-bit enums. Previously unsigned enums would throw errors on decoding if the value went outside the signed range (issue #164). Currently only helps for enums defined within the same file, but solving issue #165 will make it work for multiple files also. --- generator/nanopb_generator.py | 17 ++++++++++++++++- pb.h | 3 ++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index aaa0d2f1..2b1d63e5 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -162,6 +162,12 @@ class Enum: self.value_longnames = [self.names + x.name for x in desc.value] self.packed = enum_options.packed_enum + def has_negative(self): + for n, v in self.values: + if v < 0: + return True + return False + def __str__(self): result = 'typedef enum _%s {\n' % self.names result += ',\n'.join([" %s = %d" % x for x in self.values]) @@ -342,7 +348,7 @@ class Field: inner_init = '""' elif self.pbtype == 'BYTES': inner_init = '{0, {0}}' - elif self.pbtype == 'ENUM': + elif self.pbtype in ('ENUM', 'UENUM'): inner_init = '(%s)0' % self.ctype else: inner_init = '0' @@ -600,6 +606,7 @@ class OneOf(Field): self.struct_name = struct_name self.name = oneof_desc.name self.ctype = 'union' + self.pbtype = 'oneof' self.fields = [] self.allocation = 'ONEOF' self.default = None @@ -891,6 +898,14 @@ def parse_file(fdesc, file_options): idx = enum.value_longnames.index(field.default) field.default = enum.values[idx][0] + # Fix field data types where enums have negative values. + for enum in enums: + if not enum.has_negative(): + for message in messages: + for field in message.fields: + if field.pbtype == 'ENUM' and field.ctype == enum.names: + field.pbtype = 'UENUM' + return enums, messages, extensions def toposort2(data): diff --git a/pb.h b/pb.h index 649a25e9..ef2a166f 100644 --- a/pb.h +++ b/pb.h @@ -468,6 +468,7 @@ struct pb_extension_s { #define PB_LTYPE_MAP_BYTES PB_LTYPE_BYTES #define PB_LTYPE_MAP_DOUBLE PB_LTYPE_FIXED64 #define PB_LTYPE_MAP_ENUM PB_LTYPE_VARINT +#define PB_LTYPE_MAP_UENUM PB_LTYPE_UVARINT #define PB_LTYPE_MAP_FIXED32 PB_LTYPE_FIXED32 #define PB_LTYPE_MAP_FIXED64 PB_LTYPE_FIXED64 #define PB_LTYPE_MAP_FLOAT PB_LTYPE_FIXED32 @@ -486,7 +487,7 @@ struct pb_extension_s { /* This is the actual macro used in field descriptions. * It takes these arguments: * - Field tag number - * - Field type: BOOL, BYTES, DOUBLE, ENUM, FIXED32, FIXED64, + * - Field type: BOOL, BYTES, DOUBLE, ENUM, UENUM, FIXED32, FIXED64, * FLOAT, INT32, INT64, MESSAGE, SFIXED32, SFIXED64 * SINT32, SINT64, STRING, UINT32, UINT64 or EXTENSION * - Field rules: REQUIRED, OPTIONAL or REPEATED -- cgit 1.2.3-korg From 936cfdc675c2dc3580c2459e8b1773a1d0bf9a8b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 12 Sep 2015 13:27:56 +0300 Subject: Expand the multiple_files test case to include oneofs and enums --- tests/multiple_files/SConscript | 6 +++--- tests/multiple_files/callbacks.proto | 16 ---------------- tests/multiple_files/callbacks2.proto | 9 --------- tests/multiple_files/multifile1.proto | 28 ++++++++++++++++++++++++++++ tests/multiple_files/multifile2.proto | 21 +++++++++++++++++++++ tests/multiple_files/test_multiple_files.c | 14 ++++++++++++-- 6 files changed, 64 insertions(+), 30 deletions(-) delete mode 100644 tests/multiple_files/callbacks.proto delete mode 100644 tests/multiple_files/callbacks2.proto create mode 100644 tests/multiple_files/multifile1.proto create mode 100644 tests/multiple_files/multifile2.proto diff --git a/tests/multiple_files/SConscript b/tests/multiple_files/SConscript index 6b4f6b69..c3a82eb4 100644 --- a/tests/multiple_files/SConscript +++ b/tests/multiple_files/SConscript @@ -5,9 +5,9 @@ Import("env") incpath = env.Clone() incpath.Append(PROTOCPATH = '#multiple_files') -incpath.NanopbProto("callbacks") -incpath.NanopbProto("callbacks2") -test = incpath.Program(["test_multiple_files.c", "callbacks.pb.c", "callbacks2.pb.c"]) +incpath.NanopbProto("multifile1") +incpath.NanopbProto("multifile2") +test = incpath.Program(["test_multiple_files.c", "multifile1.pb.c", "multifile2.pb.c"]) env.RunTest(test) diff --git a/tests/multiple_files/callbacks.proto b/tests/multiple_files/callbacks.proto deleted file mode 100644 index ccd1edd8..00000000 --- a/tests/multiple_files/callbacks.proto +++ /dev/null @@ -1,16 +0,0 @@ -message SubMessage { - optional string stringvalue = 1; - repeated int32 int32value = 2; - repeated fixed32 fixed32value = 3; - repeated fixed64 fixed64value = 4; -} - -message TestMessage { - optional string stringvalue = 1; - repeated int32 int32value = 2; - repeated fixed32 fixed32value = 3; - repeated fixed64 fixed64value = 4; - optional SubMessage submsg = 5; - repeated string repeatedstring = 6; -} - diff --git a/tests/multiple_files/callbacks2.proto b/tests/multiple_files/callbacks2.proto deleted file mode 100644 index 9a55e15a..00000000 --- a/tests/multiple_files/callbacks2.proto +++ /dev/null @@ -1,9 +0,0 @@ -// Test if including generated header file for this file + implicit include of -// callbacks.pb.h still compiles. Used with test_compiles.c. -import "callbacks.proto"; - -message Callback2Message { - required TestMessage tstmsg = 1; - required SubMessage submsg = 2; -} - diff --git a/tests/multiple_files/multifile1.proto b/tests/multiple_files/multifile1.proto new file mode 100644 index 00000000..79cf0387 --- /dev/null +++ b/tests/multiple_files/multifile1.proto @@ -0,0 +1,28 @@ +message SubMessage { + optional string stringvalue = 1; + repeated int32 int32value = 2; + repeated fixed32 fixed32value = 3; + repeated fixed64 fixed64value = 4; +} + +message TestMessage { + optional string stringvalue = 1; + repeated int32 int32value = 2; + repeated fixed32 fixed32value = 3; + repeated fixed64 fixed64value = 4; + optional SubMessage submsg = 5; + repeated string repeatedstring = 6; +} + +enum SignedEnum { + SE_MIN = -128; + SE_MAX = 127; +} + +enum UnsignedEnum { + UE_MIN = 0; + UE_MAX = 255; +} + + + diff --git a/tests/multiple_files/multifile2.proto b/tests/multiple_files/multifile2.proto new file mode 100644 index 00000000..c5dcd406 --- /dev/null +++ b/tests/multiple_files/multifile2.proto @@ -0,0 +1,21 @@ +// Test if including generated header file for this file + implicit include of +// multifile2.pb.h still compiles. Used with test_compiles.c. +import "multifile1.proto"; + +message Callback2Message { + required TestMessage tstmsg = 1; + required SubMessage submsg = 2; +} + +message OneofMessage { + oneof msgs { + TestMessage tstmsg = 1; + SubMessage submsg = 2; + } +} + +message Enums { + required SignedEnum senum = 1; + required UnsignedEnum uenum = 2; +} + diff --git a/tests/multiple_files/test_multiple_files.c b/tests/multiple_files/test_multiple_files.c index 05722dc5..5134f03f 100644 --- a/tests/multiple_files/test_multiple_files.c +++ b/tests/multiple_files/test_multiple_files.c @@ -4,9 +4,19 @@ #include #include -#include "callbacks2.pb.h" +#include "unittests.h" +#include "multifile2.pb.h" + +/* Check that the size definition is obtained properly */ +static const int foo = OneofMessage_size; int main() { - return 0; + int status = 0; + + /* Check that enum signedness is detected properly */ + TEST(PB_LTYPE(Enums_fields[0].type) == PB_LTYPE_VARINT); + TEST(PB_LTYPE(Enums_fields[0].type) == PB_LTYPE_UVARINT); + + return status; } -- cgit 1.2.3-korg From 35dff3367452f89a1d8d483d0f8f601d89d78937 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 12 Sep 2015 14:46:00 +0300 Subject: Refactor the generator logic into a ProtoFile class. In preparation for multi-file support in generator. No functional changes yet. --- generator/nanopb_generator.py | 582 +++++++++++++++++++++--------------------- 1 file changed, 297 insertions(+), 285 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 2b1d63e5..6d06c2cd 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -471,7 +471,7 @@ class Field: return max(self.tag, self.max_size, self.max_count) - def encoded_size(self, allmsgs): + def encoded_size(self, dependencies): '''Return the maximum size that this field can take when encoded, including the field tag. If the size cannot be determined, returns None.''' @@ -480,15 +480,14 @@ class Field: return None if self.pbtype == 'MESSAGE': - for msg in allmsgs: - if msg.name == self.submsgname: - encsize = msg.encoded_size(allmsgs) - if encsize is None: - return None # Submessage size is indeterminate - - # Include submessage length prefix - encsize += varint_max_size(encsize.upperlimit()) - break + if str(self.submsgname) in dependencies: + submsg = dependencies[str(self.submsgname)] + encsize = submsg.encoded_size(dependencies) + if encsize is None: + return None # Submessage size is indeterminate + + # Include submessage length prefix + encsize += varint_max_size(encsize.upperlimit()) else: # Submessage cannot be found, this currently occurs when # the submessage type is defined in a different file. @@ -831,7 +830,6 @@ class Message: # Processing of entire .proto files # --------------------------------------------------------------------------- - def iterate_messages(desc, names = Names()): '''Recursively find all messages. For each, yield name, DescriptorProto.''' if hasattr(desc, 'message_type'): @@ -857,57 +855,6 @@ def iterate_extensions(desc, names = Names()): for extension in subdesc.extension: yield subname, extension -def parse_file(fdesc, file_options): - '''Takes a FileDescriptorProto and returns tuple (enums, messages, extensions).''' - - enums = [] - messages = [] - extensions = [] - - if fdesc.package: - base_name = Names(fdesc.package.split('.')) - else: - base_name = Names() - - for enum in fdesc.enum_type: - enum_options = get_nanopb_suboptions(enum, file_options, base_name + enum.name) - enums.append(Enum(base_name, enum, enum_options)) - - for names, message in iterate_messages(fdesc, base_name): - message_options = get_nanopb_suboptions(message, file_options, names) - - if message_options.skip_message: - continue - - messages.append(Message(names, message, message_options)) - for enum in message.enum_type: - enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name) - enums.append(Enum(names, enum, enum_options)) - - for names, extension in iterate_extensions(fdesc, base_name): - field_options = get_nanopb_suboptions(extension, file_options, names + extension.name) - if field_options.type != nanopb_pb2.FT_IGNORE: - extensions.append(ExtensionField(names, extension, field_options)) - - # Fix field default values where enum short names are used. - for enum in enums: - if not enum.options.long_names: - for message in messages: - for field in message.fields: - if field.default in enum.value_longnames: - idx = enum.value_longnames.index(field.default) - field.default = enum.values[idx][0] - - # Fix field data types where enums have negative values. - for enum in enums: - if not enum.has_negative(): - for message in messages: - for field in message.fields: - if field.pbtype == 'ENUM' and field.ctype == enum.names: - field.pbtype = 'UENUM' - - return enums, messages, extensions - def toposort2(data): '''Topological sort. From http://code.activestate.com/recipes/577413-topological-sort/ @@ -949,231 +896,299 @@ def make_identifier(headername): result += '_' return result -def generate_header(dependencies, headername, enums, messages, extensions, options): - '''Generate content for a header file. - Generates strings, which should be concatenated and stored to file. - ''' - - yield '/* Automatically generated nanopb header */\n' - if options.notimestamp: - yield '/* Generated by %s */\n\n' % (nanopb_version) - else: - yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) +class ProtoFile: + def __init__(self, fdesc, file_options): + '''Takes a FileDescriptorProto and parses it.''' + self.fdesc = fdesc + self.file_options = file_options + self.dependencies = {} + self.parse() + + # Some of types used in this file probably come from the file itself. + # Thus it has implicit dependency on itself. + self.add_dependency(self) + + def parse(self): + self.enums = [] + self.messages = [] + self.extensions = [] + + if self.fdesc.package: + base_name = Names(self.fdesc.package.split('.')) + else: + base_name = Names() - symbol = make_identifier(headername) - yield '#ifndef PB_%s_INCLUDED\n' % symbol - yield '#define PB_%s_INCLUDED\n' % symbol - try: - yield options.libformat % ('pb.h') - except TypeError: - # no %s specified - use whatever was passed in as options.libformat - yield options.libformat - yield '\n' + for enum in self.fdesc.enum_type: + enum_options = get_nanopb_suboptions(enum, self.file_options, base_name + enum.name) + self.enums.append(Enum(base_name, enum, enum_options)) + + for names, message in iterate_messages(self.fdesc, base_name): + message_options = get_nanopb_suboptions(message, self.file_options, names) + + if message_options.skip_message: + continue + + self.messages.append(Message(names, message, message_options)) + for enum in message.enum_type: + enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name) + self.enums.append(Enum(names, enum, enum_options)) + + for names, extension in iterate_extensions(self.fdesc, base_name): + field_options = get_nanopb_suboptions(extension, self.file_options, names + extension.name) + if field_options.type != nanopb_pb2.FT_IGNORE: + self.extensions.append(ExtensionField(names, extension, field_options)) - for dependency in dependencies: - noext = os.path.splitext(dependency)[0] - yield options.genformat % (noext + options.extension + '.h') + def add_dependency(self, other): + for enum in other.enums: + self.dependencies[str(enum.names)] = enum + + for msg in other.messages: + self.dependencies[str(msg.name)] = msg + + # Fix field default values where enum short names are used. + for enum in other.enums: + if not enum.options.long_names: + for message in self.messages: + for field in message.fields: + if field.default in enum.value_longnames: + idx = enum.value_longnames.index(field.default) + field.default = enum.values[idx][0] + + # Fix field data types where enums have negative values. + for enum in other.enums: + if not enum.has_negative(): + for message in self.messages: + for field in message.fields: + if field.pbtype == 'ENUM' and field.ctype == enum.names: + field.pbtype = 'UENUM' + + def generate_header(self, includes, headername, options): + '''Generate content for a header file. + Generates strings, which should be concatenated and stored to file. + ''' + + yield '/* Automatically generated nanopb header */\n' + if options.notimestamp: + yield '/* Generated by %s */\n\n' % (nanopb_version) + else: + yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) + + symbol = make_identifier(headername) + yield '#ifndef PB_%s_INCLUDED\n' % symbol + yield '#define PB_%s_INCLUDED\n' % symbol + try: + yield options.libformat % ('pb.h') + except TypeError: + # no %s specified - use whatever was passed in as options.libformat + yield options.libformat yield '\n' - - yield '#if PB_PROTO_HEADER_VERSION != 30\n' - yield '#error Regenerate this file with the current version of nanopb generator.\n' - yield '#endif\n' - yield '\n' - - yield '#ifdef __cplusplus\n' - yield 'extern "C" {\n' - yield '#endif\n\n' - - yield '/* Enum definitions */\n' - for enum in enums: - yield str(enum) + '\n\n' - - yield '/* Struct definitions */\n' - for msg in sort_dependencies(messages): - yield msg.types() - yield str(msg) + '\n\n' - - if extensions: - yield '/* Extensions */\n' - for extension in extensions: - yield extension.extension_decl() + + for incfile in includes: + noext = os.path.splitext(incfile)[0] + yield options.genformat % (noext + options.extension + '.h') + yield '\n' + + yield '#if PB_PROTO_HEADER_VERSION != 30\n' + yield '#error Regenerate this file with the current version of nanopb generator.\n' + yield '#endif\n' yield '\n' + + yield '#ifdef __cplusplus\n' + yield 'extern "C" {\n' + yield '#endif\n\n' - yield '/* Default values for struct fields */\n' - for msg in messages: - yield msg.default_decl(True) - yield '\n' - - yield '/* Initializer values for message structs */\n' - for msg in messages: - identifier = '%s_init_default' % msg.name - yield '#define %-40s %s\n' % (identifier, msg.get_initializer(False)) - for msg in messages: - identifier = '%s_init_zero' % msg.name - yield '#define %-40s %s\n' % (identifier, msg.get_initializer(True)) - yield '\n' - - yield '/* Field tags (for use in manual encoding/decoding) */\n' - for msg in sort_dependencies(messages): - for field in msg.fields: - yield field.tags() - for extension in extensions: - yield extension.tags() - yield '\n' - - yield '/* Struct field encoding specification for nanopb */\n' - for msg in messages: - yield msg.fields_declaration() + '\n' - yield '\n' - - yield '/* Maximum encoded size of messages (where known) */\n' - for msg in messages: - msize = msg.encoded_size(messages) - if msize is not None: - identifier = '%s_size' % msg.name - yield '#define %-40s %s\n' % (identifier, msize) - yield '\n' - - yield '/* Message IDs (where set with "msgid" option) */\n' - - yield '#ifdef PB_MSGID\n' - for msg in messages: - if hasattr(msg,'msgid'): - yield '#define PB_MSG_%d %s\n' % (msg.msgid, msg.name) - yield '\n' - - symbol = make_identifier(headername.split('.')[0]) - yield '#define %s_MESSAGES \\\n' % symbol - - for msg in messages: - m = "-1" - msize = msg.encoded_size(messages) - if msize is not None: - m = msize - if hasattr(msg,'msgid'): - yield '\tPB_MSG(%d,%s,%s) \\\n' % (msg.msgid, m, msg.name) - yield '\n' - - for msg in messages: - if hasattr(msg,'msgid'): - yield '#define %s_msgid %d\n' % (msg.name, msg.msgid) - yield '\n' - - yield '#endif\n\n' - - - yield '#ifdef __cplusplus\n' - yield '} /* extern "C" */\n' - yield '#endif\n' - - # End of header - yield '\n#endif\n' + if self.enums: + yield '/* Enum definitions */\n' + for enum in self.enums: + yield str(enum) + '\n\n' + + if self.messages: + yield '/* Struct definitions */\n' + for msg in sort_dependencies(self.messages): + yield msg.types() + yield str(msg) + '\n\n' + + if self.extensions: + yield '/* Extensions */\n' + for extension in self.extensions: + yield extension.extension_decl() + yield '\n' + + if self.messages: + yield '/* Default values for struct fields */\n' + for msg in self.messages: + yield msg.default_decl(True) + yield '\n' + + yield '/* Initializer values for message structs */\n' + for msg in self.messages: + identifier = '%s_init_default' % msg.name + yield '#define %-40s %s\n' % (identifier, msg.get_initializer(False)) + for msg in self.messages: + identifier = '%s_init_zero' % msg.name + yield '#define %-40s %s\n' % (identifier, msg.get_initializer(True)) + yield '\n' + + yield '/* Field tags (for use in manual encoding/decoding) */\n' + for msg in sort_dependencies(self.messages): + for field in msg.fields: + yield field.tags() + for extension in self.extensions: + yield extension.tags() + yield '\n' + + yield '/* Struct field encoding specification for nanopb */\n' + for msg in self.messages: + yield msg.fields_declaration() + '\n' + yield '\n' + + yield '/* Maximum encoded size of messages (where known) */\n' + for msg in self.messages: + msize = msg.encoded_size(self.dependencies) + if msize is not None: + identifier = '%s_size' % msg.name + yield '#define %-40s %s\n' % (identifier, msize) + yield '\n' + + yield '/* Message IDs (where set with "msgid" option) */\n' + + yield '#ifdef PB_MSGID\n' + for msg in self.messages: + if hasattr(msg,'msgid'): + yield '#define PB_MSG_%d %s\n' % (msg.msgid, msg.name) + yield '\n' + + symbol = make_identifier(headername.split('.')[0]) + yield '#define %s_MESSAGES \\\n' % symbol + + for msg in self.messages: + m = "-1" + msize = msg.encoded_size(self.dependencies) + if msize is not None: + m = msize + if hasattr(msg,'msgid'): + yield '\tPB_MSG(%d,%s,%s) \\\n' % (msg.msgid, m, msg.name) + yield '\n' + + for msg in self.messages: + if hasattr(msg,'msgid'): + yield '#define %s_msgid %d\n' % (msg.name, msg.msgid) + yield '\n' -def generate_source(headername, enums, messages, extensions, options): - '''Generate content for a source file.''' - - yield '/* Automatically generated nanopb constant definitions */\n' - if options.notimestamp: - yield '/* Generated by %s */\n\n' % (nanopb_version) - else: - yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) - yield options.genformat % (headername) - yield '\n' - - yield '#if PB_PROTO_HEADER_VERSION != 30\n' - yield '#error Regenerate this file with the current version of nanopb generator.\n' - yield '#endif\n' - yield '\n' - - for msg in messages: - yield msg.default_decl(False) - - yield '\n\n' - - for msg in messages: - yield msg.fields_definition() + '\n\n' - - for ext in extensions: - yield ext.extension_def() + '\n' - - # Add checks for numeric limits - if messages: - largest_msg = max(messages, key = lambda m: m.count_required_fields()) - largest_count = largest_msg.count_required_fields() - if largest_count > 64: - yield '\n/* Check that missing required fields will be properly detected */\n' - yield '#if PB_MAX_REQUIRED_FIELDS < %d\n' % largest_count - yield '#error Properly detecting missing required fields in %s requires \\\n' % largest_msg.name - yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count - yield '#endif\n' - - worst = 0 - worst_field = '' - checks = [] - checks_msgnames = [] - for msg in messages: - checks_msgnames.append(msg.name) - for field in msg.fields: - status = field.largest_field_value() - if isinstance(status, (str, unicode)): - checks.append(status) - elif status > worst: - worst = status - worst_field = str(field.struct_name) + '.' + str(field.name) - - if worst > 255 or checks: - yield '\n/* Check that field information fits in pb_field_t */\n' - - if worst > 65535 or checks: - yield '#if !defined(PB_FIELD_32BIT)\n' - if worst > 65535: - yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field - else: - assertion = ' && '.join(str(c) + ' < 65536' for c in checks) - msgs = '_'.join(str(n) for n in checks_msgnames) - yield '/* If you get an error here, it means that you need to define PB_FIELD_32BIT\n' - yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n' - yield ' * \n' - yield ' * The reason you need to do this is that some of your messages contain tag\n' - yield ' * numbers or field sizes that are larger than what can fit in 8 or 16 bit\n' - yield ' * field descriptors.\n' - yield ' */\n' - yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) yield '#endif\n\n' + + yield '#ifdef __cplusplus\n' + yield '} /* extern "C" */\n' + yield '#endif\n' + + # End of header + yield '\n#endif\n' + + def generate_source(self, headername, options): + '''Generate content for a source file.''' + + yield '/* Automatically generated nanopb constant definitions */\n' + if options.notimestamp: + yield '/* Generated by %s */\n\n' % (nanopb_version) + else: + yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) + yield options.genformat % (headername) + yield '\n' + + yield '#if PB_PROTO_HEADER_VERSION != 30\n' + yield '#error Regenerate this file with the current version of nanopb generator.\n' + yield '#endif\n' + yield '\n' + + for msg in self.messages: + yield msg.default_decl(False) + + yield '\n\n' + + for msg in self.messages: + yield msg.fields_definition() + '\n\n' + + for ext in self.extensions: + yield ext.extension_def() + '\n' + + # Add checks for numeric limits + if self.messages: + largest_msg = max(self.messages, key = lambda m: m.count_required_fields()) + largest_count = largest_msg.count_required_fields() + if largest_count > 64: + yield '\n/* Check that missing required fields will be properly detected */\n' + yield '#if PB_MAX_REQUIRED_FIELDS < %d\n' % largest_count + yield '#error Properly detecting missing required fields in %s requires \\\n' % largest_msg.name + yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count + yield '#endif\n' + + worst = 0 + worst_field = '' + checks = [] + checks_msgnames = [] + for msg in self.messages: + checks_msgnames.append(msg.name) + for field in msg.fields: + status = field.largest_field_value() + if isinstance(status, (str, unicode)): + checks.append(status) + elif status > worst: + worst = status + worst_field = str(field.struct_name) + '.' + str(field.name) + + if worst > 255 or checks: + yield '\n/* Check that field information fits in pb_field_t */\n' + + if worst > 65535 or checks: + yield '#if !defined(PB_FIELD_32BIT)\n' + if worst > 65535: + yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field + else: + assertion = ' && '.join(str(c) + ' < 65536' for c in checks) + msgs = '_'.join(str(n) for n in checks_msgnames) + yield '/* If you get an error here, it means that you need to define PB_FIELD_32BIT\n' + yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n' + yield ' * \n' + yield ' * The reason you need to do this is that some of your messages contain tag\n' + yield ' * numbers or field sizes that are larger than what can fit in 8 or 16 bit\n' + yield ' * field descriptors.\n' + yield ' */\n' + yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) + yield '#endif\n\n' + + if worst < 65536: + yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n' + if worst > 255: + yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field + else: + assertion = ' && '.join(str(c) + ' < 256' for c in checks) + msgs = '_'.join(str(n) for n in checks_msgnames) + yield '/* If you get an error here, it means that you need to define PB_FIELD_16BIT\n' + yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n' + yield ' * \n' + yield ' * The reason you need to do this is that some of your messages contain tag\n' + yield ' * numbers or field sizes that are larger than what can fit in the default\n' + yield ' * 8 bit descriptors.\n' + yield ' */\n' + yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) + yield '#endif\n\n' + + # Add check for sizeof(double) + has_double = False + for msg in self.messages: + for field in msg.fields: + if field.ctype == 'double': + has_double = True + + if has_double: + yield '\n' + yield '/* On some platforms (such as AVR), double is really float.\n' + yield ' * These are not directly supported by nanopb, but see example_avr_double.\n' + yield ' * To get rid of this error, remove any double fields from your .proto.\n' + yield ' */\n' + yield 'PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n' - if worst < 65536: - yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n' - if worst > 255: - yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field - else: - assertion = ' && '.join(str(c) + ' < 256' for c in checks) - msgs = '_'.join(str(n) for n in checks_msgnames) - yield '/* If you get an error here, it means that you need to define PB_FIELD_16BIT\n' - yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n' - yield ' * \n' - yield ' * The reason you need to do this is that some of your messages contain tag\n' - yield ' * numbers or field sizes that are larger than what can fit in the default\n' - yield ' * 8 bit descriptors.\n' - yield ' */\n' - yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) - yield '#endif\n\n' - - # Add check for sizeof(double) - has_double = False - for msg in messages: - for field in msg.fields: - if field.ctype == 'double': - has_double = True - - if has_double: yield '\n' - yield '/* On some platforms (such as AVR), double is really float.\n' - yield ' * These are not directly supported by nanopb, but see example_avr_double.\n' - yield ' * To get rid of this error, remove any double fields from your .proto.\n' - yield ' */\n' - yield 'PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n' - - yield '\n' # --------------------------------------------------------------------------- # Options parsing for the .proto files @@ -1338,7 +1353,7 @@ def process_file(filename, fdesc, options): # Parse the file file_options = get_nanopb_suboptions(fdesc, toplevel_options, Names([filename])) - enums, messages, extensions = parse_file(fdesc, file_options) + f = ProtoFile(fdesc, file_options) # Decide the file names noext = os.path.splitext(filename)[0] @@ -1349,13 +1364,10 @@ def process_file(filename, fdesc, options): # List of .proto files that should not be included in the C header file # even if they are mentioned in the source .proto. excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude - dependencies = [d for d in fdesc.dependency if d not in excludes] + includes = [d for d in fdesc.dependency if d not in excludes] - headerdata = ''.join(generate_header(dependencies, headerbasename, enums, - messages, extensions, options)) - - sourcedata = ''.join(generate_source(headerbasename, enums, - messages, extensions, options)) + headerdata = ''.join(f.generate_header(includes, headerbasename, options)) + sourcedata = ''.join(f.generate_source(headerbasename, options)) # Check if there were any lines in .options that did not match a member unmatched = [n for n,o in Globals.separate_options if n not in Globals.matched_namemasks] -- cgit 1.2.3-korg From 0b29baf5deaa4213c08ee71fa55d3d0b2ed709e4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 12 Sep 2015 15:45:37 +0300 Subject: Make the generator understand included files (issue #165). This will allow message sizes and enum options to be available across the include files. Currently searching for .options files for included files may not work for all path combinations, this is related to issue #116. Should probably make a pull request to protoc about that. --- generator/nanopb_generator.py | 88 ++++++++++++++++++++---------- tests/multiple_files/SConscript | 2 +- tests/multiple_files/multifile1.options | 1 + tests/multiple_files/multifile1.proto | 4 ++ tests/multiple_files/multifile2.proto | 3 +- tests/multiple_files/test_multiple_files.c | 8 +-- 6 files changed, 70 insertions(+), 36 deletions(-) create mode 100644 tests/multiple_files/multifile1.options diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 6d06c2cd..f9d98484 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -490,7 +490,8 @@ class Field: encsize += varint_max_size(encsize.upperlimit()) else: # Submessage cannot be found, this currently occurs when - # the submessage type is defined in a different file. + # the submessage type is defined in a different file and + # not using the protoc plugin. # Instead of direct numeric value, reference the size that # has been #defined in the other file. encsize = EncodedSize(self.submsgname + 'size') @@ -544,7 +545,7 @@ class ExtensionRange(Field): def tags(self): return '' - def encoded_size(self, allmsgs): + def encoded_size(self, dependencies): # We exclude extensions from the count, because they cannot be known # until runtime. Other option would be to return None here, but this # way the value remains useful if extensions are not used. @@ -662,10 +663,10 @@ class OneOf(Field): def largest_field_value(self): return max([f.largest_field_value() for f in self.fields]) - def encoded_size(self, allmsgs): + def encoded_size(self, dependencies): largest = EncodedSize(0) for f in self.fields: - size = f.encoded_size(allmsgs) + size = f.encoded_size(dependencies) if size is None: return None elif size.symbols: @@ -812,13 +813,13 @@ class Message: result += ' PB_LAST_FIELD\n};' return result - def encoded_size(self, allmsgs): + def encoded_size(self, dependencies): '''Return the maximum size that this message can take when encoded. If the size cannot be determined, returns None. ''' size = EncodedSize(0) for field in self.fields: - fsize = field.encoded_size(allmsgs) + fsize = field.encoded_size(dependencies) if fsize is None: return None size += fsize @@ -1290,6 +1291,9 @@ optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", help="Set extension to use instead of '.pb' for generated files. [default: %default]") optparser.add_option("-f", "--options-file", dest="options_file", metavar="FILE", default="%s.options", help="Set name of a separate generator options file.") +optparser.add_option("-I", "--options-path", dest="options_path", metavar="DIR", + action="append", default = [], + help="Search for .options files additionally in this path") optparser.add_option("-Q", "--generated-include-format", dest="genformat", metavar="FORMAT", default='#include "%s"\n', help="Set format string to use for including other .pb.h files. [default: %default]") @@ -1305,19 +1309,8 @@ optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", def optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[], help="Set generator option (max_size, max_count etc.).") -def process_file(filename, fdesc, options): - '''Process a single file. - filename: The full path to the .proto or .pb source file, as string. - fdesc: The loaded FileDescriptorSet, or None to read from the input file. - options: Command line options as they come from OptionsParser. - - Returns a dict: - {'headername': Name of header file, - 'headerdata': Data for the .h header file, - 'sourcename': Name of the source code file, - 'sourcedata': Data for the .c source code file - } - ''' +def parse_file(filename, fdesc, options): + '''Parse a single file. Returns a ProtoFile instance.''' toplevel_options = nanopb_pb2.NanoPBOptions() for s in options.settings: text_format.Merge(s, toplevel_options) @@ -1335,18 +1328,20 @@ def process_file(filename, fdesc, options): optfilename = options.options_file had_abspath = True - if os.path.isfile(optfilename): - if options.verbose: - sys.stderr.write('Reading options from ' + optfilename + '\n') - - Globals.separate_options = read_options_file(open(optfilename, "rU")) + paths = ['.'] + options.options_path + for p in paths: + if os.path.isfile(os.path.join(p, optfilename)): + optfilename = os.path.join(p, optfilename) + if options.verbose: + sys.stderr.write('Reading options from ' + optfilename + '\n') + Globals.separate_options = read_options_file(open(optfilename, "rU")) + break else: # If we are given a full filename and it does not exist, give an error. # However, don't give error when we automatically look for .options file # with the same name as .proto. if options.verbose or had_abspath: - sys.stderr.write('Options file not found: ' + optfilename) - + sys.stderr.write('Options file not found: ' + optfilename + '\n') Globals.separate_options = [] Globals.matched_namemasks = set() @@ -1354,6 +1349,29 @@ def process_file(filename, fdesc, options): # Parse the file file_options = get_nanopb_suboptions(fdesc, toplevel_options, Names([filename])) f = ProtoFile(fdesc, file_options) + f.optfilename = optfilename + + return f + +def process_file(filename, fdesc, options, other_files = {}): + '''Process a single file. + filename: The full path to the .proto or .pb source file, as string. + fdesc: The loaded FileDescriptorSet, or None to read from the input file. + options: Command line options as they come from OptionsParser. + + Returns a dict: + {'headername': Name of header file, + 'headerdata': Data for the .h header file, + 'sourcename': Name of the source code file, + 'sourcedata': Data for the .c source code file + } + ''' + f = parse_file(filename, fdesc, options) + + # Provide dependencies if available + for dep in f.fdesc.dependency: + if dep in other_files: + f.add_dependency(other_files[dep]) # Decide the file names noext = os.path.splitext(filename)[0] @@ -1364,7 +1382,7 @@ def process_file(filename, fdesc, options): # List of .proto files that should not be included in the C header file # even if they are mentioned in the source .proto. excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude - includes = [d for d in fdesc.dependency if d not in excludes] + includes = [d for d in f.fdesc.dependency if d not in excludes] headerdata = ''.join(f.generate_header(includes, headerbasename, options)) sourcedata = ''.join(f.generate_source(headerbasename, options)) @@ -1372,7 +1390,7 @@ def process_file(filename, fdesc, options): # Check if there were any lines in .options that did not match a member unmatched = [n for n,o in Globals.separate_options if n not in Globals.matched_namemasks] if unmatched and not options.quiet: - sys.stderr.write("Following patterns in " + optfilename + " did not match any fields: " + sys.stderr.write("Following patterns in " + f.optfilename + " did not match any fields: " + ', '.join(unmatched) + "\n") if not Globals.verbose_options: sys.stderr.write("Use protoc --nanopb-out=-v:. to see a list of the field names.\n") @@ -1432,10 +1450,22 @@ def main_plugin(): response = plugin_pb2.CodeGeneratorResponse() + # Google's protoc does not currently indicate the full path of proto files. + # Instead always add the main file path to the search dirs, that works for + # the common case. + import os.path + options.options_path.append(os.path.dirname(request.file_to_generate[0])) + + # Process any include files first, in order to have them + # available as dependencies + other_files = {} + for fdesc in request.proto_file: + other_files[fdesc.name] = parse_file(fdesc.name, fdesc, options) + for filename in request.file_to_generate: for fdesc in request.proto_file: if fdesc.name == filename: - results = process_file(filename, fdesc, options) + results = process_file(filename, fdesc, options, other_files) f = response.file.add() f.name = results['headername'] diff --git a/tests/multiple_files/SConscript b/tests/multiple_files/SConscript index c3a82eb4..1689f482 100644 --- a/tests/multiple_files/SConscript +++ b/tests/multiple_files/SConscript @@ -5,7 +5,7 @@ Import("env") incpath = env.Clone() incpath.Append(PROTOCPATH = '#multiple_files') -incpath.NanopbProto("multifile1") +incpath.NanopbProto(["multifile1", "multifile1.options"]) incpath.NanopbProto("multifile2") test = incpath.Program(["test_multiple_files.c", "multifile1.pb.c", "multifile2.pb.c"]) diff --git a/tests/multiple_files/multifile1.options b/tests/multiple_files/multifile1.options new file mode 100644 index 00000000..c44d2669 --- /dev/null +++ b/tests/multiple_files/multifile1.options @@ -0,0 +1 @@ +StaticMessage.repint32 max_count:5 diff --git a/tests/multiple_files/multifile1.proto b/tests/multiple_files/multifile1.proto index 79cf0387..d804b67d 100644 --- a/tests/multiple_files/multifile1.proto +++ b/tests/multiple_files/multifile1.proto @@ -14,6 +14,10 @@ message TestMessage { repeated string repeatedstring = 6; } +message StaticMessage { + repeated fixed32 repint32 = 1; +} + enum SignedEnum { SE_MIN = -128; SE_MAX = 127; diff --git a/tests/multiple_files/multifile2.proto b/tests/multiple_files/multifile2.proto index c5dcd406..66cb8a0b 100644 --- a/tests/multiple_files/multifile2.proto +++ b/tests/multiple_files/multifile2.proto @@ -9,8 +9,7 @@ message Callback2Message { message OneofMessage { oneof msgs { - TestMessage tstmsg = 1; - SubMessage submsg = 2; + StaticMessage tstmsg = 1; } } diff --git a/tests/multiple_files/test_multiple_files.c b/tests/multiple_files/test_multiple_files.c index 5134f03f..292b8d7c 100644 --- a/tests/multiple_files/test_multiple_files.c +++ b/tests/multiple_files/test_multiple_files.c @@ -7,16 +7,16 @@ #include "unittests.h" #include "multifile2.pb.h" -/* Check that the size definition is obtained properly */ -static const int foo = OneofMessage_size; - int main() { int status = 0; + /* Test that included file options are properly loaded */ + TEST(OneofMessage_size == 27); + /* Check that enum signedness is detected properly */ TEST(PB_LTYPE(Enums_fields[0].type) == PB_LTYPE_VARINT); - TEST(PB_LTYPE(Enums_fields[0].type) == PB_LTYPE_UVARINT); + TEST(PB_LTYPE(Enums_fields[1].type) == PB_LTYPE_UVARINT); return status; } -- cgit 1.2.3-korg From 6e72df4808aa138f1396ad098ce2d06a6feba882 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 13 Sep 2015 11:38:54 +0300 Subject: Fix maximum encoded size for negative enums (issue #166). --- generator/nanopb_generator.py | 16 +++++++++- tests/regression/issue_166/SConscript | 13 ++++++++ tests/regression/issue_166/enum_encoded_size.c | 43 ++++++++++++++++++++++++++ tests/regression/issue_166/enums.proto | 16 ++++++++++ 4 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 tests/regression/issue_166/SConscript create mode 100644 tests/regression/issue_166/enum_encoded_size.c create mode 100644 tests/regression/issue_166/enums.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index f9d98484..3a5fac5a 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -100,11 +100,14 @@ def names_from_type_name(type_name): def varint_max_size(max_value): '''Returns the maximum number of bytes a varint can take when encoded.''' + if max_value < 0: + max_value = 2**64 - max_value for i in range(1, 11): if (max_value >> (i * 7)) == 0: return i raise ValueError("Value too large for varint: " + str(max_value)) +assert varint_max_size(-1) == 10 assert varint_max_size(0) == 1 assert varint_max_size(127) == 1 assert varint_max_size(128) == 2 @@ -168,6 +171,9 @@ class Enum: return True return False + def encoded_size(self): + return max([varint_max_size(v) for n,v in self.values]) + def __str__(self): result = 'typedef enum _%s {\n' % self.names result += ',\n'.join([" %s = %d" % x for x in self.values]) @@ -267,7 +273,7 @@ class Field: self.ctype = names_from_type_name(desc.type_name) if self.default is not None: self.default = self.ctype + self.default - self.enc_size = 5 # protoc rejects enum values > 32 bits + self.enc_size = None # Needs to be filled in when enum values are known elif desc.type == FieldD.TYPE_STRING: self.pbtype = 'STRING' self.ctype = 'char' @@ -500,6 +506,14 @@ class Field: # prefix size, though. encsize += 5 + elif self.pbtype in ['ENUM', 'UENUM']: + if str(self.ctype) in dependencies: + enumtype = dependencies[str(self.ctype)] + encsize = enumtype.encoded_size() + else: + # Conservative assumption + encsize = 10 + elif self.enc_size is None: raise RuntimeError("Could not determine encoded size for %s.%s" % (self.struct_name, self.name)) diff --git a/tests/regression/issue_166/SConscript b/tests/regression/issue_166/SConscript new file mode 100644 index 00000000..c50b9193 --- /dev/null +++ b/tests/regression/issue_166/SConscript @@ -0,0 +1,13 @@ +# Verify that the maximum encoded size is calculated properly +# for enums. + +Import('env') + +env.NanopbProto('enums') + +p = env.Program(["enum_encoded_size.c", + "enums.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) + diff --git a/tests/regression/issue_166/enum_encoded_size.c b/tests/regression/issue_166/enum_encoded_size.c new file mode 100644 index 00000000..84e1c7de --- /dev/null +++ b/tests/regression/issue_166/enum_encoded_size.c @@ -0,0 +1,43 @@ +#include +#include +#include +#include "unittests.h" +#include "enums.pb.h" + +int main() +{ + int status = 0; + + uint8_t buf[256]; + SignedMsg msg1; + UnsignedMsg msg2; + pb_ostream_t s; + + { + COMMENT("Test negative value of signed enum"); + /* Negative value should take up the maximum size */ + msg1.value = SignedEnum_SE_MIN; + s = pb_ostream_from_buffer(buf, sizeof(buf)); + TEST(pb_encode(&s, SignedMsg_fields, &msg1)); + TEST(s.bytes_written == SignedMsg_size); + + COMMENT("Test positive value of signed enum"); + /* Positive value should be smaller */ + msg1.value = SignedEnum_SE_MAX; + s = pb_ostream_from_buffer(buf, sizeof(buf)); + TEST(pb_encode(&s, SignedMsg_fields, &msg1)); + TEST(s.bytes_written < SignedMsg_size); + } + + { + COMMENT("Test positive value of unsigned enum"); + /* This should take up the maximum size */ + msg2.value = UnsignedEnum_UE_MAX; + s = pb_ostream_from_buffer(buf, sizeof(buf)); + TEST(pb_encode(&s, UnsignedMsg_fields, &msg2)); + TEST(s.bytes_written == UnsignedMsg_size); + } + + return status; +} + diff --git a/tests/regression/issue_166/enums.proto b/tests/regression/issue_166/enums.proto new file mode 100644 index 00000000..a0964ab4 --- /dev/null +++ b/tests/regression/issue_166/enums.proto @@ -0,0 +1,16 @@ +enum SignedEnum { + SE_MIN = -1; + SE_MAX = 255; +} + +enum UnsignedEnum { + UE_MAX = 65536; +} + +message SignedMsg { + required SignedEnum value = 1; +} + +message UnsignedMsg { + required UnsignedEnum value = 1; +} -- cgit 1.2.3-korg From b03be4fa4d863e6beb362cfef6d834d14c96b7e1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 20 Sep 2015 14:12:19 +0300 Subject: Add syntax specification to .proto files (issue #167) Eliminates a warning on protoc 3.0. --- examples/cmake_simple/simple.proto | 2 ++ examples/network_server/fileproto.proto | 2 ++ examples/simple/simple.proto | 2 ++ examples/using_double_on_avr/doubleproto.proto | 2 ++ examples/using_union_messages/unionproto.proto | 2 ++ tests/alltypes/alltypes.proto | 3 +++ tests/backwards_compatibility/alltypes_legacy.proto | 2 ++ tests/callbacks/callbacks.proto | 2 ++ tests/common/person.proto | 2 ++ tests/common/unittestproto.proto | 2 ++ tests/cyclic_messages/cyclic.proto | 2 ++ tests/enum_sizes/enumsizes.proto | 2 ++ tests/extensions/extensions.proto | 2 ++ tests/field_size_16/alltypes.proto | 2 ++ tests/field_size_32/alltypes.proto | 2 ++ tests/fuzztest/SConscript | 15 +++++++++------ tests/intsizes/intsizes.proto | 2 ++ tests/message_sizes/messages1.proto | 2 ++ tests/message_sizes/messages2.proto | 2 ++ tests/missing_fields/missing_fields.proto | 2 ++ tests/multiple_files/multifile1.proto | 2 ++ tests/multiple_files/multifile2.proto | 2 ++ tests/no_messages/no_messages.proto | 2 ++ tests/oneof/oneof.proto | 2 ++ tests/options/options.proto | 2 ++ tests/package_name/SConscript | 16 +++++++++------- tests/regression/issue_118/enumdef.proto | 2 ++ tests/regression/issue_118/enumuse.proto | 2 ++ tests/regression/issue_125/extensionbug.proto | 2 ++ tests/regression/issue_141/testproto.proto | 2 ++ tests/regression/issue_145/comments.proto | 2 ++ tests/regression/issue_166/enums.proto | 2 ++ .../funny-proto+name has.characters.proto | 1 + 33 files changed, 80 insertions(+), 13 deletions(-) diff --git a/examples/cmake_simple/simple.proto b/examples/cmake_simple/simple.proto index 26e72f46..5c73a3b2 100644 --- a/examples/cmake_simple/simple.proto +++ b/examples/cmake_simple/simple.proto @@ -1,6 +1,8 @@ // A very simple protocol definition, consisting of only // one message. +syntax = "proto2"; + message SimpleMessage { required int32 lucky_number = 1; } diff --git a/examples/network_server/fileproto.proto b/examples/network_server/fileproto.proto index 3e70c492..5640b8d5 100644 --- a/examples/network_server/fileproto.proto +++ b/examples/network_server/fileproto.proto @@ -2,6 +2,8 @@ // // See also the nanopb-specific options in fileproto.options. +syntax = "proto2"; + message ListFilesRequest { optional string path = 1 [default = "/"]; } diff --git a/examples/simple/simple.proto b/examples/simple/simple.proto index 26e72f46..5c73a3b2 100644 --- a/examples/simple/simple.proto +++ b/examples/simple/simple.proto @@ -1,6 +1,8 @@ // A very simple protocol definition, consisting of only // one message. +syntax = "proto2"; + message SimpleMessage { required int32 lucky_number = 1; } diff --git a/examples/using_double_on_avr/doubleproto.proto b/examples/using_double_on_avr/doubleproto.proto index d8b7f2db..72d3f9c1 100644 --- a/examples/using_double_on_avr/doubleproto.proto +++ b/examples/using_double_on_avr/doubleproto.proto @@ -1,4 +1,6 @@ // A message containing doubles, as used by other applications. +syntax = "proto2"; + message DoubleMessage { required double field1 = 1; required double field2 = 2; diff --git a/examples/using_union_messages/unionproto.proto b/examples/using_union_messages/unionproto.proto index d7c9de2d..209df0d2 100644 --- a/examples/using_union_messages/unionproto.proto +++ b/examples/using_union_messages/unionproto.proto @@ -5,6 +5,8 @@ // but they are commonly implemented by filling out exactly one of // several optional fields. +syntax = "proto2"; + message MsgType1 { required int32 value = 1; diff --git a/tests/alltypes/alltypes.proto b/tests/alltypes/alltypes.proto index 28eaf0be..3995c552 100644 --- a/tests/alltypes/alltypes.proto +++ b/tests/alltypes/alltypes.proto @@ -1,3 +1,6 @@ +syntax = "proto2"; +// package name placeholder + message SubMessage { required string substuff1 = 1 [default = "1"]; required int32 substuff2 = 2 [default = 2]; diff --git a/tests/backwards_compatibility/alltypes_legacy.proto b/tests/backwards_compatibility/alltypes_legacy.proto index d7631eb6..f5bc35ce 100644 --- a/tests/backwards_compatibility/alltypes_legacy.proto +++ b/tests/backwards_compatibility/alltypes_legacy.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + message SubMessage { required string substuff1 = 1 [default = "1"]; required int32 substuff2 = 2 [default = 2]; diff --git a/tests/callbacks/callbacks.proto b/tests/callbacks/callbacks.proto index ccd1edd8..96ac744d 100644 --- a/tests/callbacks/callbacks.proto +++ b/tests/callbacks/callbacks.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + message SubMessage { optional string stringvalue = 1; repeated int32 int32value = 2; diff --git a/tests/common/person.proto b/tests/common/person.proto index dafcf934..becefdf3 100644 --- a/tests/common/person.proto +++ b/tests/common/person.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + import "nanopb.proto"; message Person { diff --git a/tests/common/unittestproto.proto b/tests/common/unittestproto.proto index 0ecb1f0b..23b5b97f 100644 --- a/tests/common/unittestproto.proto +++ b/tests/common/unittestproto.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + import 'nanopb.proto'; message IntegerArray { diff --git a/tests/cyclic_messages/cyclic.proto b/tests/cyclic_messages/cyclic.proto index a9d158cb..8cab0b14 100644 --- a/tests/cyclic_messages/cyclic.proto +++ b/tests/cyclic_messages/cyclic.proto @@ -2,6 +2,8 @@ // These can only be handled in pointer/callback mode, // see associated .options files. +syntax = "proto2"; + message TreeNode { optional int32 leaf = 1; diff --git a/tests/enum_sizes/enumsizes.proto b/tests/enum_sizes/enumsizes.proto index f9ab0b7f..a85d4160 100644 --- a/tests/enum_sizes/enumsizes.proto +++ b/tests/enum_sizes/enumsizes.proto @@ -4,6 +4,8 @@ * a bit of a problem for the encoder/decoder (issue #164). */ +syntax = "proto2"; + import 'nanopb.proto'; option (nanopb_fileopt).long_names = false; diff --git a/tests/extensions/extensions.proto b/tests/extensions/extensions.proto index 79c01245..fcd5b43b 100644 --- a/tests/extensions/extensions.proto +++ b/tests/extensions/extensions.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + import 'alltypes.proto'; extend AllTypes { diff --git a/tests/field_size_16/alltypes.proto b/tests/field_size_16/alltypes.proto index 039391fc..ba1ec383 100644 --- a/tests/field_size_16/alltypes.proto +++ b/tests/field_size_16/alltypes.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + message SubMessage { required string substuff1 = 1 [default = "1"]; required int32 substuff2 = 2 [default = 2]; diff --git a/tests/field_size_32/alltypes.proto b/tests/field_size_32/alltypes.proto index 5749e0d4..02ee1a6a 100644 --- a/tests/field_size_32/alltypes.proto +++ b/tests/field_size_32/alltypes.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + message SubMessage { required string substuff1 = 1 [default = "1"]; required int32 substuff2 = 2 [default = 2]; diff --git a/tests/fuzztest/SConscript b/tests/fuzztest/SConscript index 35b697f9..973148c2 100644 --- a/tests/fuzztest/SConscript +++ b/tests/fuzztest/SConscript @@ -2,16 +2,19 @@ Import("env", "malloc_env") +def set_pkgname(src, dst, pkgname): + data = open(str(src)).read() + placeholder = '// package name placeholder' + assert placeholder in data + data = data.replace(placeholder, 'package %s;' % pkgname) + open(str(dst), 'w').write(data) + # We want both pointer and static versions of the AllTypes message # Prefix them with package name. env.Command("alltypes_static.proto", "#alltypes/alltypes.proto", - lambda target, source, env: - open(str(target[0]), 'w').write("package alltypes_static;\n" - + open(str(source[0])).read())) + lambda target, source, env: set_pkgname(source[0], target[0], 'alltypes_static')) env.Command("alltypes_pointer.proto", "#alltypes/alltypes.proto", - lambda target, source, env: - open(str(target[0]), 'w').write("package alltypes_pointer;\n" - + open(str(source[0])).read())) + lambda target, source, env: set_pkgname(source[0], target[0], 'alltypes_pointer')) p1 = env.NanopbProto(["alltypes_pointer", "alltypes_pointer.options"]) p2 = env.NanopbProto(["alltypes_static", "alltypes_static.options"]) diff --git a/tests/intsizes/intsizes.proto b/tests/intsizes/intsizes.proto index 236bf183..91444d41 100644 --- a/tests/intsizes/intsizes.proto +++ b/tests/intsizes/intsizes.proto @@ -6,6 +6,8 @@ * otherwise. E.g. uint32 + IS_8 => uint8_t */ +syntax = "proto2"; + import 'nanopb.proto'; message IntSizes { diff --git a/tests/message_sizes/messages1.proto b/tests/message_sizes/messages1.proto index 48af55a3..b66fad71 100644 --- a/tests/message_sizes/messages1.proto +++ b/tests/message_sizes/messages1.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + enum MessageStatus { FAIL = 0; OK = 1; diff --git a/tests/message_sizes/messages2.proto b/tests/message_sizes/messages2.proto index 19fc11ef..67614080 100644 --- a/tests/message_sizes/messages2.proto +++ b/tests/message_sizes/messages2.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + import 'nanopb.proto'; import 'messages1.proto'; diff --git a/tests/missing_fields/missing_fields.proto b/tests/missing_fields/missing_fields.proto index cbb23ba1..cc5e550b 100644 --- a/tests/missing_fields/missing_fields.proto +++ b/tests/missing_fields/missing_fields.proto @@ -1,5 +1,7 @@ /* Test for one missing field among many */ +syntax = "proto2"; + message AllFields { required int32 field1 = 1; diff --git a/tests/multiple_files/multifile1.proto b/tests/multiple_files/multifile1.proto index d804b67d..18f2c672 100644 --- a/tests/multiple_files/multifile1.proto +++ b/tests/multiple_files/multifile1.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + message SubMessage { optional string stringvalue = 1; repeated int32 int32value = 2; diff --git a/tests/multiple_files/multifile2.proto b/tests/multiple_files/multifile2.proto index 66cb8a0b..4af45fd9 100644 --- a/tests/multiple_files/multifile2.proto +++ b/tests/multiple_files/multifile2.proto @@ -1,5 +1,7 @@ // Test if including generated header file for this file + implicit include of // multifile2.pb.h still compiles. Used with test_compiles.c. +syntax = "proto2"; + import "multifile1.proto"; message Callback2Message { diff --git a/tests/no_messages/no_messages.proto b/tests/no_messages/no_messages.proto index 279216b0..45bb2e66 100644 --- a/tests/no_messages/no_messages.proto +++ b/tests/no_messages/no_messages.proto @@ -1,5 +1,7 @@ /* Test that a file without any messages works. */ +syntax = "proto2"; + enum Test { First = 1; } diff --git a/tests/oneof/oneof.proto b/tests/oneof/oneof.proto index 00f1ceca..b4fe56f2 100644 --- a/tests/oneof/oneof.proto +++ b/tests/oneof/oneof.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + import 'nanopb.proto'; message SubMessage diff --git a/tests/options/options.proto b/tests/options/options.proto index b7050418..aa722b52 100644 --- a/tests/options/options.proto +++ b/tests/options/options.proto @@ -2,6 +2,8 @@ * options.expected lists the patterns that are searched for in the output. */ +syntax = "proto2"; + import "nanopb.proto"; // File level options diff --git a/tests/package_name/SConscript b/tests/package_name/SConscript index 897bc99c..4afc5037 100644 --- a/tests/package_name/SConscript +++ b/tests/package_name/SConscript @@ -3,14 +3,16 @@ Import("env") -# Build a modified alltypes.proto -def modify_proto(target, source, env): - '''Add a "package test.package;" directive to the beginning of the .proto file.''' - data = open(str(source[0]), 'r').read() - open(str(target[0]), 'w').write("package test.package;\n\n" + data) - return 0 +def set_pkgname(src, dst, pkgname): + data = open(str(src)).read() + placeholder = '// package name placeholder' + assert placeholder in data + data = data.replace(placeholder, 'package %s;' % pkgname) + open(str(dst), 'w').write(data) -env.Command("alltypes.proto", "#alltypes/alltypes.proto", modify_proto) +# Build a modified alltypes.proto +env.Command("alltypes.proto", "#alltypes/alltypes.proto", + lambda target, source, env: set_pkgname(source[0], target[0], 'test.package')) env.Command("alltypes.options", "#alltypes/alltypes.options", Copy("$TARGET", "$SOURCE")) env.NanopbProto(["alltypes", "alltypes.options"]) diff --git a/tests/regression/issue_118/enumdef.proto b/tests/regression/issue_118/enumdef.proto index 830d2988..46845bc9 100644 --- a/tests/regression/issue_118/enumdef.proto +++ b/tests/regression/issue_118/enumdef.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + import 'nanopb.proto'; enum MyEnum { diff --git a/tests/regression/issue_118/enumuse.proto b/tests/regression/issue_118/enumuse.proto index d778fb8f..4afc4521 100644 --- a/tests/regression/issue_118/enumuse.proto +++ b/tests/regression/issue_118/enumuse.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + import 'enumdef.proto'; message MyMessage { diff --git a/tests/regression/issue_125/extensionbug.proto b/tests/regression/issue_125/extensionbug.proto index c4ac6860..fd1e74f1 100644 --- a/tests/regression/issue_125/extensionbug.proto +++ b/tests/regression/issue_125/extensionbug.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + message Message1 { optional uint32 fieldA = 1; diff --git a/tests/regression/issue_141/testproto.proto b/tests/regression/issue_141/testproto.proto index 21598b45..a445c68a 100644 --- a/tests/regression/issue_141/testproto.proto +++ b/tests/regression/issue_141/testproto.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + import 'nanopb.proto'; message SubMessage diff --git a/tests/regression/issue_145/comments.proto b/tests/regression/issue_145/comments.proto index 4e86b302..621779f5 100644 --- a/tests/regression/issue_145/comments.proto +++ b/tests/regression/issue_145/comments.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + message DummyMessage { required string foo = 1; required string bar = 2; diff --git a/tests/regression/issue_166/enums.proto b/tests/regression/issue_166/enums.proto index a0964ab4..36948044 100644 --- a/tests/regression/issue_166/enums.proto +++ b/tests/regression/issue_166/enums.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + enum SignedEnum { SE_MIN = -1; SE_MAX = 255; diff --git a/tests/special_characters/funny-proto+name has.characters.proto b/tests/special_characters/funny-proto+name has.characters.proto index e69de29b..26b2cb1b 100644 --- a/tests/special_characters/funny-proto+name has.characters.proto +++ b/tests/special_characters/funny-proto+name has.characters.proto @@ -0,0 +1 @@ +syntax="proto2"; -- cgit 1.2.3-korg From aa316910644779259894ba57b528260eb9da1cb5 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 20 Sep 2015 14:37:39 +0300 Subject: Update changelog --- CHANGELOG.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 8c6f8e64..9c574291 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,4 +1,12 @@ -nanopb-0.3.3 (2015-04-xx) +nanopb-0.3.4 (2015-09-xx) + Fix handling of unsigned 8- and 16-bit enums (issue 164) + Fix generator on systems where python = python3. (issue 155) + Make the generator better handle imported .protos (issue 165) + Add packed_enum option to generator. + Add syntax= line to .proto files (issue 167) + Add PlatformIO registry manifest file. (pr 156) + +nanopb-0.3.3 (2015-04-10) Fix missing files in Linux binary package (issue 146) Fix generator bug when oneof is first field in a message. (issue 142) Fix generator error when long_names:false is combined with Oneofs. (issue 147) -- cgit 1.2.3-korg From b222209e1bf80b2e855817d8f9287c5e2dc1eb04 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 19 Sep 2015 14:42:03 -0700 Subject: generator: Don't force python2 Python2 is being phased out of the default python interpreter. Arch Linux has moved some time ago and upcoming Debian realeases will follow. My distro, Arch, doesn't have a python2-protobuf version, but does have a python3 version. With a python 2 & 3 compatible generator the exact interpreter can be ignored and can use the system default as most users expect. Update issue #155 --- generator/protoc-gen-nanopb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/generator/protoc-gen-nanopb b/generator/protoc-gen-nanopb index 6dc468d3..358f97cf 100755 --- a/generator/protoc-gen-nanopb +++ b/generator/protoc-gen-nanopb @@ -10,5 +10,4 @@ # --plugin= on the command line. MYPATH=$(dirname "$0") -PYTHON=$(which python2 || which python) -exec $PYTHON "$MYPATH/nanopb_generator.py" --protoc-plugin +exec "$MYPATH/nanopb_generator.py" --protoc-plugin -- cgit 1.2.3-korg From 1f86b707e3a845f2c403e77173d0859dbf6aa4b0 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 19 Sep 2015 15:19:28 -0700 Subject: cmake: Don't explicitly look for python2 * Use the system python binary and make sure the generator works with both instead. --- extra/FindNanopb.cmake | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/extra/FindNanopb.cmake b/extra/FindNanopb.cmake index e65706ac..1fd25334 100644 --- a/extra/FindNanopb.cmake +++ b/extra/FindNanopb.cmake @@ -175,7 +175,7 @@ function(NANOPB_GENERATE_CPP SRCS HDRS) add_custom_command( OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c" "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" - COMMAND ${PYTHON2_EXECUTABLE} + COMMAND ${PYTHON_EXECUTABLE} ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb ${NANOPB_OPTIONS} DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb" COMMENT "Running nanopb generator on ${FIL_WE}.pb" @@ -244,17 +244,7 @@ find_file(NANOPB_GENERATOR_EXECUTABLE ) mark_as_advanced(NANOPB_GENERATOR_EXECUTABLE) -# If python3 has already been found, save it and look for python2.6 -if(${PYTHON_VERSION_MAJOR} AND ${PYTHON_VERSION_MAJOR} EQUAL 3) - set(PYTHON3_EXECUTABLE ${PYTHON_EXECUTABLE}) - set(PYTHON_EXECUTABLE PYTHON_EXECUTABLE-NOTFOUND) - find_package(PythonInterp 2.6 REQUIRED) - set(PYTHON2_EXECUTABLE ${PYTHON_EXECUTABLE}) - set(PYTHON_EXECUTABLE ${PYTHON3_EXECUTABLE}) -else() - find_package(PythonInterp 2.6 REQUIRED) - set(PYTHON2_EXECUTABLE ${PYTHON_EXECUTABLE}) -endif() +find_package(PythonInterp REQUIRED) include(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(NANOPB DEFAULT_MSG -- cgit 1.2.3-korg From 02367d6d77b20cb8bb1dcac4a70ae7d6849a2e1f Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 19 Sep 2015 14:12:36 -0700 Subject: generator: Run python's 2to3 converter * Invoked with `2to3 -w nanopb_generator.py` * No other changes. --- generator/nanopb_generator.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 3a5fac5a..17cb674f 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -5,6 +5,7 @@ nanopb_version = "nanopb-0.3.4-dev" import sys import re +from functools import reduce try: # Add some dummy imports to keep packaging tools happy. @@ -82,7 +83,7 @@ class Names: return '_'.join(self.parts) def __add__(self, other): - if isinstance(other, (str, unicode)): + if isinstance(other, str): return Names(self.parts + (other,)) elif isinstance(other, tuple): return Names(self.parts + other) @@ -123,7 +124,7 @@ class EncodedSize: self.symbols = symbols def __add__(self, other): - if isinstance(other, (int, long)): + if isinstance(other, int): return EncodedSize(self.value + other, self.symbols) elif isinstance(other, (str, Names)): return EncodedSize(self.value, self.symbols + [str(other)]) @@ -133,7 +134,7 @@ class EncodedSize: raise ValueError("Cannot add size: " + repr(other)) def __mul__(self, other): - if isinstance(other, (int, long)): + if isinstance(other, int): return EncodedSize(self.value * other, [str(other) + '*' + s for s in self.symbols]) else: raise ValueError("Cannot multiply size: " + repr(other)) @@ -260,7 +261,7 @@ class Field: raise NotImplementedError(field_options.type) # Decide the C data type to use in the struct. - if datatypes.has_key(desc.type): + if desc.type in datatypes: self.ctype, self.pbtype, self.enc_size, isa = datatypes[desc.type] # Override the field size if user wants to use smaller integers @@ -875,17 +876,17 @@ def toposort2(data): From http://code.activestate.com/recipes/577413-topological-sort/ This function is under the MIT license. ''' - for k, v in data.items(): + for k, v in list(data.items()): v.discard(k) # Ignore self dependencies - extra_items_in_deps = reduce(set.union, data.values(), set()) - set(data.keys()) + extra_items_in_deps = reduce(set.union, list(data.values()), set()) - set(data.keys()) data.update(dict([(item, set()) for item in extra_items_in_deps])) while True: - ordered = set(item for item,dep in data.items() if not dep) + ordered = set(item for item,dep in list(data.items()) if not dep) if not ordered: break for item in sorted(ordered): yield item - data = dict([(item, (dep - ordered)) for item,dep in data.items() + data = dict([(item, (dep - ordered)) for item,dep in list(data.items()) if item not in ordered]) assert not data, "A cyclic dependency exists amongst %r" % data @@ -1145,7 +1146,7 @@ class ProtoFile: checks_msgnames.append(msg.name) for field in msg.fields: status = field.largest_field_value() - if isinstance(status, (str, unicode)): + if isinstance(status, str): checks.append(status) elif status > worst: worst = status @@ -1237,7 +1238,7 @@ def read_options_file(infile): try: text_format.Merge(parts[1], opts) - except Exception, e: + except Exception as e: sys.stderr.write("%s:%d: " % (infile.name, i + 1) + "Unparseable option line: '%s'. " % line + "Error: %s\n" % str(e)) -- cgit 1.2.3-korg From 2731fe3e6a4ba6fd858947e343160c95c96042e2 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 19 Sep 2015 14:30:21 -0700 Subject: generator: Use python2/3 binary read/write method * This works with python2 and python3 --- generator/nanopb_generator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 17cb674f..c2acb060 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1440,14 +1440,15 @@ def main_cli(): def main_plugin(): '''Main function when invoked as a protoc plugin.''' - import sys + import io, sys if sys.platform == "win32": import os, msvcrt # Set stdin and stdout to binary mode msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) - data = sys.stdin.read() + data = io.open(sys.stdin.fileno(), "rb").read() + request = plugin_pb2.CodeGeneratorRequest.FromString(data) try: @@ -1490,7 +1491,7 @@ def main_plugin(): f.name = results['sourcename'] f.content = results['sourcedata'] - sys.stdout.write(response.SerializeToString()) + io.open(sys.stdout.fileno(), "wb").write(response.SerializeToString()) if __name__ == '__main__': # Check if we are running as a plugin under protoc -- cgit 1.2.3-korg From 56134e87657714b50d9837f846410fc30fa9fe7b Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 19 Sep 2015 14:36:55 -0700 Subject: generator: More exhaustive field size checking Create a FieldMaxSize class that: * Accumlates all C assertions * Handles the checking of the longest simple field * Also python3 doesn't support max(None) --- generator/nanopb_generator.py | 51 ++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index c2acb060..240fa15f 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -193,6 +193,24 @@ class Enum: return result +class FieldMaxSize: + def __init__(self, worst = 0, checks = [], field_name = 'undefined'): + if isinstance(worst, list): + self.worst = max(i for i in worst if i is not None) + else: + self.worst = worst + + self.worst_field = field_name + self.checks = checks + + def extend(self, extend, field_name = None): + self.worst = max(self.worst, extend.worst) + + if self.worst == extend.worst: + self.worst_field = extend.worst_field + + self.checks.extend(extend.checks) + class Field: def __init__(self, struct_name, desc, field_options): '''desc is FieldDescriptorProto''' @@ -468,15 +486,18 @@ class Field: def largest_field_value(self): '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly. Returns numeric value or a C-expression for assert.''' + check = [] if self.pbtype == 'MESSAGE': if self.rules == 'REPEATED' and self.allocation == 'STATIC': - return 'pb_membersize(%s, %s[0])' % (self.struct_name, self.name) + check.append('pb_membersize(%s, %s[0])' % (self.struct_name, self.name)) elif self.rules == 'ONEOF': - return 'pb_membersize(%s, %s.%s)' % (self.struct_name, self.union_name, self.name) + check.append('pb_membersize(%s, %s.%s)' % (self.struct_name, self.union_name, self.name)) else: - return 'pb_membersize(%s, %s)' % (self.struct_name, self.name) + check.append('pb_membersize(%s, %s)' % (self.struct_name, self.name)) - return max(self.tag, self.max_size, self.max_count) + return FieldMaxSize([self.tag, self.max_size, self.max_count], + check, + ('%s.%s' % (self.struct_name, self.name))) def encoded_size(self, dependencies): '''Return the maximum size that this field can take when encoded, @@ -676,7 +697,10 @@ class OneOf(Field): return result def largest_field_value(self): - return max([f.largest_field_value() for f in self.fields]) + largest = FieldMaxSize() + for f in self.fields: + largest.extend(f.largest_field_value()) + return largest def encoded_size(self, dependencies): largest = EncodedSize(0) @@ -1137,20 +1161,17 @@ class ProtoFile: yield '#error Properly detecting missing required fields in %s requires \\\n' % largest_msg.name yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count yield '#endif\n' - - worst = 0 - worst_field = '' - checks = [] + + max_field = FieldMaxSize() checks_msgnames = [] for msg in self.messages: checks_msgnames.append(msg.name) for field in msg.fields: - status = field.largest_field_value() - if isinstance(status, str): - checks.append(status) - elif status > worst: - worst = status - worst_field = str(field.struct_name) + '.' + str(field.name) + max_field.extend(field.largest_field_value()) + + worst = max_field.worst + worst_field = max_field.worst_field + checks = max_field.checks if worst > 255 or checks: yield '\n/* Check that field information fits in pb_field_t */\n' -- cgit 1.2.3-korg From 0d7ef5f936afc32bfe0aaec8b92667d4c3a026a0 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 19 Sep 2015 15:03:13 -0700 Subject: generator: Remove cmp() to work with Python3 * Python3 ignores cmp() and __cmp__() and only needs __lt__() for sorting. Delete and update as appropriate. * Ref: https://docs.python.org/3.0/whatsnew/3.0.html#ordering-comparisons --- generator/nanopb_generator.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 240fa15f..f7d43221 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -314,8 +314,8 @@ class Field: else: raise NotImplementedError(desc.type) - def __cmp__(self, other): - return cmp(self.tag, other.tag) + def __lt__(self, other): + return self.tag < other.tag def __str__(self): result = '' @@ -661,9 +661,6 @@ class OneOf(Field): # Sort by the lowest tag number inside union self.tag = min([f.tag for f in self.fields]) - def __cmp__(self, other): - return cmp(self.tag, other.tag) - def __str__(self): result = '' if self.fields: -- cgit 1.2.3-korg From 03e3af597f0adf64b21731b820c997116a16cdcd Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 19 Sep 2015 15:13:05 -0700 Subject: generator: Strings are utf-8 by default in python3 * Not sure how to handle this case in python2, seems to work * Python 3 doesn't need this since all strings are utf-8 --- generator/nanopb_generator.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index f7d43221..78140a4c 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -379,12 +379,10 @@ class Field: inner_init = '0' else: if self.pbtype == 'STRING': - inner_init = self.default.encode('utf-8').encode('string_escape') - inner_init = inner_init.replace('"', '\\"') + inner_init = self.default.replace('"', '\\"') inner_init = '"' + inner_init + '"' elif self.pbtype == 'BYTES': - data = str(self.default).decode('string_escape') - data = ['0x%02x' % ord(c) for c in data] + data = ['0x%02x' % ord(c) for c in self.default] if len(data) == 0: inner_init = '{0, {0}}' else: -- cgit 1.2.3-korg From e02c65389493d195ee868bf927923108a753bf57 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sun, 20 Sep 2015 17:22:25 -0700 Subject: generator: Attempt to simplify the str/unicode madness * This is a shot in the dark. --- generator/nanopb_generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 78140a4c..91a820f3 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,7 @@ #!/usr/bin/python +from __future__ import unicode_literals + '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' nanopb_version = "nanopb-0.3.4-dev" -- cgit 1.2.3-korg From 67cafac8f2f4797e195c5ee1e1fcf691129f8694 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sun, 20 Sep 2015 18:39:45 -0700 Subject: generator: Fix strange unicode/str issue in python2 * Work around this by checking the appropriate class for the given * environment. --- generator/nanopb_generator.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 91a820f3..df973386 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -85,7 +85,14 @@ class Names: return '_'.join(self.parts) def __add__(self, other): - if isinstance(other, str): + # The fdesc names are unicode and need to be handled for + # python2 and python3 + try: + realstr = unicode + except NameError: + realstr = str + + if isinstance(other, realstr): return Names(self.parts + (other,)) elif isinstance(other, tuple): return Names(self.parts + other) -- cgit 1.2.3-korg From 59d6add7d268b6252ee207db3378a72f5fb1bca5 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sun, 20 Sep 2015 21:49:48 -0700 Subject: travis-ci: Initial working build * Initial working build for Travis CI build system. --- .travis.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..5848c424 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,28 @@ +language: c + +compiler: + - gcc + - clang + +before_install: + - export PATH=$HOME/.local/bin:$HOME/protobuf/bin:$PATH + - export MAKEFLAGS=-j$(grep processor /proc/cpuinfo | wc -l) + - $CC --version + - python --version + - lsb_release -a + +cache: + directories: + - $HOME/protobuf +install: + - pip install --user protobuf + - test \! -d $HOME/protobuf + && curl -L https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.bz2 | tar xjf - + && pushd protobuf-2.6.1 + && ./configure --prefix=$HOME/protobuf && make && make install + && popd + || true # True if test is false as the cache exists + +script: + - pushd generator/proto && make && popd + - pushd tests && scons && popd -- cgit 1.2.3-korg From 9c9f7f14e71c3b49877cca8eccd448cabea6306d Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Mon, 21 Sep 2015 08:16:17 -0700 Subject: generator: Use search $PATH for python * Search $PATH for the python binary so that this works better with things like virtualenv as used on Travis CI --- generator/nanopb_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index df973386..82b79270 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python from __future__ import unicode_literals -- cgit 1.2.3-korg From 47bd244f2e7772c2259f2c37ddf9480959c2272a Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sun, 20 Sep 2015 22:58:54 -0700 Subject: travis-ci: Build C/C++/Python matrix * Test a number of C compilers * Travis CI can't handle three languages (C, C++, Python) * Add support for swapping python2/3 binaries * Scons has made no attempts to support python3 yet: * Build the matrix manually * Scons doesn't inherit $CC/$CXX from parent environement, so pass compiler flags directly --- .travis.yml | 48 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5848c424..6b8db516 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,49 @@ -language: c +# Travis CI has no ability to handle 3 langauges (c, c++, python) +# and it overrides $CC/$CXX if language is set to c/c++ (only one, not both). +# +# Set language to python since at least the result of that is something useful. +language: python + +python: + - "2.7" + - "3.4" + +# Manage the C/C++ compiler manually +env: + - CC=gcc CXX=g++ + - CC=gcc-4.8 CXX=g++-4.8 + - CC=gcc-4.9 CXX=g++-4.9 + - CC=gcc-5 CXX=g++-5 + - CC=clang CXX=clang++ + +addons: + apt: + sources: + - ubuntu-toolchain-r-test + packages: + - gcc-4.8 + - g++-4.8 + - gcc-4.9 + - g++-4.9 + - gcc-5 + - g++-5 -compiler: - - gcc - - clang before_install: - export PATH=$HOME/.local/bin:$HOME/protobuf/bin:$PATH - export MAKEFLAGS=-j$(grep processor /proc/cpuinfo | wc -l) - $CC --version + - $CXX --version - python --version - lsb_release -a -cache: - directories: - - $HOME/protobuf +# Seems to be issues with concurrent builds +#cache: +# directories: +# - $HOME/protobuf + install: - - pip install --user protobuf + - pip install protobuf - test \! -d $HOME/protobuf && curl -L https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.bz2 | tar xjf - && pushd protobuf-2.6.1 @@ -23,6 +51,6 @@ install: && popd || true # True if test is false as the cache exists -script: +script: - pushd generator/proto && make && popd - - pushd tests && scons && popd + - pushd tests && python2 $(which scons) CC=$CC CXX=$CXX && popd -- cgit 1.2.3-korg From 77dad2e4e271ccafa618619e23b81480542f9832 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Mon, 21 Sep 2015 09:04:11 -0700 Subject: travis-ci: Use protobuf v3 as it adds python3 * Still in testing currently. --- .travis.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6b8db516..2e77e7c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,13 +43,11 @@ before_install: # - $HOME/protobuf install: - - pip install protobuf - - test \! -d $HOME/protobuf - && curl -L https://github.com/google/protobuf/releases/download/v2.6.1/protobuf-2.6.1.tar.bz2 | tar xjf - - && pushd protobuf-2.6.1 + - curl -L https://github.com/google/protobuf/releases/download/v3.0.0-beta-1/protobuf-python-3.0.0-alpha-4.tar.gz | tar xzf - + && pushd protobuf-3.0.0-alpha-4 && ./configure --prefix=$HOME/protobuf && make && make install + && pushd python && python setup.py build && python setup.py install && popd && popd - || true # True if test is false as the cache exists script: - pushd generator/proto && make && popd -- cgit 1.2.3-korg From e3283e77b825ae9c0c468762fce9b5797b70cba4 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Mon, 21 Sep 2015 11:03:12 -0700 Subject: decode: Fix compiler issue with gcc-5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * gcc 5.0 and 5.1 appear to take issue with this line and generates the following error: /home/nitro/tmp/nanopb/pb_decode.c: In function ‘pb_decode_noinit’: /home/nitro/tmp/nanopb/pb_decode.c:889:60: error: conversion to ‘uint8_t {aka unsigned char}’ from ‘int’ may alter its value [-Werror=conversion] fields_seen[iter.required_field_index >> 3] |= (uint8_t)(1 << (iter.required_field_index & 7)); ^ * This seems like a compiler bug, but this workaround is harmless. --- pb_decode.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pb_decode.c b/pb_decode.c index b21bfe37..5cdcbcfb 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -886,7 +886,8 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ if (PB_HTYPE(iter.pos->type) == PB_HTYPE_REQUIRED && iter.required_field_index < PB_MAX_REQUIRED_FIELDS) { - fields_seen[iter.required_field_index >> 3] |= (uint8_t)(1 << (iter.required_field_index & 7)); + uint8_t tmp = (uint8_t)(1 << (iter.required_field_index & 7)); + fields_seen[iter.required_field_index >> 3] |= tmp; } if (!decode_field(stream, wire_type, &iter)) -- cgit 1.2.3-korg From 5e4356ef4b5be9995f2038ba294e4c538167ac01 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 24 Sep 2015 20:32:29 +0300 Subject: Add initialization to examples/simple --- examples/cmake_simple/simple.c | 9 ++++++--- examples/simple/simple.c | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/examples/cmake_simple/simple.c b/examples/cmake_simple/simple.c index 31272301..1f6b1373 100644 --- a/examples/cmake_simple/simple.c +++ b/examples/cmake_simple/simple.c @@ -15,8 +15,11 @@ int main() /* Allocate space on the stack to store the message data. * * Nanopb generates simple struct definitions for all the messages. - * - check out the contents of simple.pb.h! */ - SimpleMessage message; + * - check out the contents of simple.pb.h! + * It is a good idea to always initialize your structures + * so that you do not have garbage data from RAM in there. + */ + SimpleMessage message = SimpleMessage_init_zero; /* Create a stream that will write to our buffer. */ pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); @@ -44,7 +47,7 @@ int main() { /* Allocate space for the decoded message. */ - SimpleMessage message; + SimpleMessage message = SimpleMessage_init_zero; /* Create a stream that reads from the buffer. */ pb_istream_t stream = pb_istream_from_buffer(buffer, message_length); diff --git a/examples/simple/simple.c b/examples/simple/simple.c index 31272301..1f6b1373 100644 --- a/examples/simple/simple.c +++ b/examples/simple/simple.c @@ -15,8 +15,11 @@ int main() /* Allocate space on the stack to store the message data. * * Nanopb generates simple struct definitions for all the messages. - * - check out the contents of simple.pb.h! */ - SimpleMessage message; + * - check out the contents of simple.pb.h! + * It is a good idea to always initialize your structures + * so that you do not have garbage data from RAM in there. + */ + SimpleMessage message = SimpleMessage_init_zero; /* Create a stream that will write to our buffer. */ pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); @@ -44,7 +47,7 @@ int main() { /* Allocate space for the decoded message. */ - SimpleMessage message; + SimpleMessage message = SimpleMessage_init_zero; /* Create a stream that reads from the buffer. */ pb_istream_t stream = pb_istream_from_buffer(buffer, message_length); -- cgit 1.2.3-korg From 47fdbaddbb26a0bcea4570796baa1a04d5f38e9c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 26 Sep 2015 11:23:59 +0300 Subject: Publishing nanopb-0.3.4 --- CHANGELOG.txt | 3 ++- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9c574291..b81b847e 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,6 +1,7 @@ -nanopb-0.3.4 (2015-09-xx) +nanopb-0.3.4 (2015-09-26) Fix handling of unsigned 8- and 16-bit enums (issue 164) Fix generator on systems where python = python3. (issue 155) + Fix compiler warning on GCC 5.x (issue 171) Make the generator better handle imported .protos (issue 165) Add packed_enum option to generator. Add syntax= line to .proto files (issue 167) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 3a5fac5a..7f65d1f8 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.4-dev" +nanopb_version = "nanopb-0.3.4" import sys import re diff --git a/pb.h b/pb.h index ef2a166f..45f6b4e5 100644 --- a/pb.h +++ b/pb.h @@ -50,7 +50,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.4-dev +#define NANOPB_VERSION nanopb-0.3.4 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 6cd4f20b0601ce7a68d555087ce0a62f837026ed Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 26 Sep 2015 12:07:08 +0300 Subject: Setting version to 0.3.5-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 7f65d1f8..4fd3a4d4 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,7 +1,7 @@ #!/usr/bin/python '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.4" +nanopb_version = "nanopb-0.3.5-dev" import sys import re diff --git a/pb.h b/pb.h index 45f6b4e5..98613a06 100644 --- a/pb.h +++ b/pb.h @@ -50,7 +50,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.4 +#define NANOPB_VERSION nanopb-0.3.5-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 3cb3872a1adc7e8f098595cc64f3c7ac820928d7 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 26 Sep 2015 12:32:12 +0300 Subject: Clean up the python2/python3 string type handling --- generator/nanopb_generator.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 7fe0db95..37f7beb5 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -74,6 +74,12 @@ intsizes = { nanopb_pb2.IS_64: 'int64_t', } +# String types (for python 2 / python 3 compatibility) +try: + strtypes = (unicode, str) +except NameError: + strtypes = (str, ) + class Names: '''Keeps a set of nested names and formats them to C identifier.''' def __init__(self, parts = ()): @@ -85,14 +91,7 @@ class Names: return '_'.join(self.parts) def __add__(self, other): - # The fdesc names are unicode and need to be handled for - # python2 and python3 - try: - realstr = unicode - except NameError: - realstr = str - - if isinstance(other, realstr): + if isinstance(other, strtypes): return Names(self.parts + (other,)) elif isinstance(other, tuple): return Names(self.parts + other) @@ -126,7 +125,7 @@ class EncodedSize: '''Class used to represent the encoded size of a field or a message. Consists of a combination of symbolic sizes and integer sizes.''' def __init__(self, value = 0, symbols = []): - if isinstance(value, (str, Names)): + if isinstance(value, strtypes + (Names,)): symbols = [str(value)] value = 0 self.value = value @@ -135,7 +134,7 @@ class EncodedSize: def __add__(self, other): if isinstance(other, int): return EncodedSize(self.value + other, self.symbols) - elif isinstance(other, (str, Names)): + elif isinstance(other, strtypes + (Names,)): return EncodedSize(self.value, self.symbols + [str(other)]) elif isinstance(other, EncodedSize): return EncodedSize(self.value + other.value, self.symbols + other.symbols) -- cgit 1.2.3-korg From 8b86c5a319c2da36cdd2b672544f7b01f3ead7d0 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 26 Sep 2015 12:47:34 +0300 Subject: Fine-tune the readme --- README.md | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ README.txt | 65 --------------------------------------------------------- 2 files changed, 70 insertions(+), 65 deletions(-) create mode 100644 README.md delete mode 100644 README.txt diff --git a/README.md b/README.md new file mode 100644 index 00000000..fc42e766 --- /dev/null +++ b/README.md @@ -0,0 +1,70 @@ +Nanopb - Protocol Buffers for Embedded Systems +============================================== + +[![Build Status](https://travis-ci.org/nanopb/nanopb.svg?branch=master)](https://travis-ci.org/nanopb/nanopb) + +Nanopb is a small code-size Protocol Buffers implementation in ansi C. It is +especially suitable for use in microcontrollers, but fits any memory +restricted system. + +* **Homepage:** http://kapsi.fi/~jpa/nanopb/ +* **Downloads:** http://koti.kapsi.fi/~jpa/nanopb/download/ +* **Forum:** https://groups.google.com/forum/#!forum/nanopb + + + +Using the nanopb library +------------------------ +To use the nanopb library, you need to do two things: + +1) Compile your .proto files for nanopb, using protoc. +2) Include pb_encode.c and pb_decode.c in your project. + +The easiest way to get started is to study the project in "examples/simple". +It contains a Makefile, which should work directly under most Linux systems. +However, for any other kind of build system, see the manual steps in +README.txt in that folder. + + + +Using the Protocol Buffers compiler (protoc) +-------------------------------------------- +The nanopb generator is implemented as a plugin for the Google's own protoc +compiler. This has the advantage that there is no need to reimplement the +basic parsing of .proto files. However, it does mean that you need the +Google's protobuf library in order to run the generator. + +If you have downloaded a binary package for nanopb (either Windows, Linux or +Mac OS X version), the 'protoc' binary is included in the 'generator-bin' +folder. In this case, you are ready to go. Simply run this command: + + generator-bin/protoc --nanopb_out=. myprotocol.proto + +However, if you are using a git checkout or a plain source distribution, you +need to provide your own version of protoc and the Google's protobuf library. +On Linux, the necessary packages are protobuf-compiler and python-protobuf. +On Windows, you can either build Google's protobuf library from source or use +one of the binary distributions of it. In either case, if you use a separate +protoc, you need to manually give the path to nanopb generator: + + protoc --plugin=protoc-gen-nanopb=nanopb/generator/protoc-gen-nanopb ... + + + +Running the tests +----------------- +If you want to perform further development of the nanopb core, or to verify +its functionality using your compiler and platform, you'll want to run the +test suite. The build rules for the test suite are implemented using Scons, +so you need to have that installed. To run the tests: + + cd tests + scons + +This will show the progress of various test cases. If the output does not +end in an error, the test cases were successful. + +Note: Mac OS X by default aliases 'clang' as 'gcc', while not actually +supporting the same command line options as gcc does. To run tests on +Mac OS X, use: "scons CC=clang CXX=clang". Same way can be used to run +tests with different compilers on any platform. diff --git a/README.txt b/README.txt deleted file mode 100644 index e9f7a7d1..00000000 --- a/README.txt +++ /dev/null @@ -1,65 +0,0 @@ -Nanopb is a small code-size Protocol Buffers implementation in ansi C. It is -especially suitable for use in microcontrollers, but fits any memory -restricted system. - -Homepage: http://kapsi.fi/~jpa/nanopb/ -Downloads: http://koti.kapsi.fi/~jpa/nanopb/download/ -Forum: https://groups.google.com/forum/#!forum/nanopb - - - -Using the nanopb library -======================== -To use the nanopb library, you need to do two things: - -1) Compile your .proto files for nanopb, using protoc. -2) Include pb_encode.c and pb_decode.c in your project. - -The easiest way to get started is to study the project in "examples/simple". -It contains a Makefile, which should work directly under most Linux systems. -However, for any other kind of build system, see the manual steps in -README.txt in that folder. - - - -Using the Protocol Buffers compiler (protoc) -============================================ -The nanopb generator is implemented as a plugin for the Google's own protoc -compiler. This has the advantage that there is no need to reimplement the -basic parsing of .proto files. However, it does mean that you need the -Google's protobuf library in order to run the generator. - -If you have downloaded a binary package for nanopb (either Windows, Linux or -Mac OS X version), the 'protoc' binary is included in the 'generator-bin' -folder. In this case, you are ready to go. Simply run this command: - - generator-bin/protoc --nanopb_out=. myprotocol.proto - -However, if you are using a git checkout or a plain source distribution, you -need to provide your own version of protoc and the Google's protobuf library. -On Linux, the necessary packages are protobuf-compiler and python-protobuf. -On Windows, you can either build Google's protobuf library from source or use -one of the binary distributions of it. In either case, if you use a separate -protoc, you need to manually give the path to nanopb generator: - - protoc --plugin=protoc-gen-nanopb=nanopb/generator/protoc-gen-nanopb ... - - - -Running the tests -================= -If you want to perform further development of the nanopb core, or to verify -its functionality using your compiler and platform, you'll want to run the -test suite. The build rules for the test suite are implemented using Scons, -so you need to have that installed. To run the tests: - - cd tests - scons - -This will show the progress of various test cases. If the output does not -end in an error, the test cases were successful. - -Note: Mac OS X by default aliases 'clang' as 'gcc', while not actually -supporting the same command line options as gcc does. To run tests on -Mac OS X, use: "scons CC=clang CXX=clang". Same way can be used to run -tests with different compilers on any platform. -- cgit 1.2.3-korg From 87d01fed5bb51f77d4e3a0084d9c9a1ee8ba3b70 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 26 Sep 2015 12:48:52 +0300 Subject: Add documentation link to readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fc42e766..beda45fb 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ especially suitable for use in microcontrollers, but fits any memory restricted system. * **Homepage:** http://kapsi.fi/~jpa/nanopb/ -* **Downloads:** http://koti.kapsi.fi/~jpa/nanopb/download/ +* **Documentation:** http://kapsi.fi/~jpa/nanopb/docs/ +* **Downloads:** http://kapsi.fi/~jpa/nanopb/download/ * **Forum:** https://groups.google.com/forum/#!forum/nanopb -- cgit 1.2.3-korg From 6b63b26404064272394740f6b7354e75c5805cec Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 26 Sep 2015 12:53:26 +0300 Subject: Fix list formatting in readme --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index beda45fb..2d35e85d 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,8 @@ Using the nanopb library ------------------------ To use the nanopb library, you need to do two things: -1) Compile your .proto files for nanopb, using protoc. -2) Include pb_encode.c and pb_decode.c in your project. +1. Compile your .proto files for nanopb, using protoc. +2. Include pb_encode.c and pb_decode.c in your project. The easiest way to get started is to study the project in "examples/simple". It contains a Makefile, which should work directly under most Linux systems. -- cgit 1.2.3-korg From be9c9bb55cea9ddac4c47041a5875f824629d03e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 1 Oct 2015 17:38:10 +0300 Subject: Fix bug in fuzzstub. The fread call always returned a message length of 1 byte, making the fuzz stub used for external fuzzers useless. The normal fuzztest.c was unaffected. Bug found using afl-fuzz. --- tests/fuzztest/fuzzstub.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/fuzztest/fuzzstub.c b/tests/fuzztest/fuzzstub.c index ce14b9ba..ec9e2afe 100644 --- a/tests/fuzztest/fuzzstub.c +++ b/tests/fuzztest/fuzzstub.c @@ -165,7 +165,7 @@ static void run_iteration() size_t msglen; bool status; - msglen = fread(buffer, BUFSIZE, 1, stdin); + msglen = fread(buffer, 1, BUFSIZE, stdin); status = do_static_decode(buffer, msglen, false); -- cgit 1.2.3-korg From 6448f5d40d498a8de05b953f8318dba0cf4ff26f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 5 Oct 2015 15:36:24 +0300 Subject: Add random message generator to help with fuzz testing --- tests/fuzztest/SConscript | 4 ++ tests/fuzztest/generate_message.c | 101 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) create mode 100644 tests/fuzztest/generate_message.c diff --git a/tests/fuzztest/SConscript b/tests/fuzztest/SConscript index 973148c2..d2fb689c 100644 --- a/tests/fuzztest/SConscript +++ b/tests/fuzztest/SConscript @@ -36,4 +36,8 @@ fuzzstub = malloc_env.Program(["fuzzstub.c", "$COMMON/pb_common_with_malloc.o", "$COMMON/malloc_wrappers.o"]) +generate_message = malloc_env.Program(["generate_message.c", + "alltypes_static.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) diff --git a/tests/fuzztest/generate_message.c b/tests/fuzztest/generate_message.c new file mode 100644 index 00000000..6e492990 --- /dev/null +++ b/tests/fuzztest/generate_message.c @@ -0,0 +1,101 @@ +/* Generates a random, valid protobuf message. Useful to seed + * external fuzzers such as afl-fuzz. + */ + +#include +#include +#include +#include +#include +#include +#include +#include "alltypes_static.pb.h" + +static uint64_t random_seed; + +/* Uses xorshift64 here instead of rand() for both speed and + * reproducibility across platforms. */ +static uint32_t rand_word() +{ + random_seed ^= random_seed >> 12; + random_seed ^= random_seed << 25; + random_seed ^= random_seed >> 27; + return random_seed * 2685821657736338717ULL; +} + +/* Fills a buffer with random data. */ +static void rand_fill(uint8_t *buf, size_t count) +{ + while (count--) + { + *buf++ = rand_word() & 0xff; + } +} + +/* Check that size/count fields do not exceed their max size. + * Otherwise we would have to loop pretty long in generate_message(). + * Note that there may still be a few encoding errors from submessages. + */ +static void limit_sizes(alltypes_static_AllTypes *msg) +{ + pb_field_iter_t iter; + pb_field_iter_begin(&iter, alltypes_static_AllTypes_fields, msg); + while (pb_field_iter_next(&iter)) + { + if (PB_LTYPE(iter.pos->type) == PB_LTYPE_BYTES) + { + ((pb_bytes_array_t*)iter.pData)->size %= iter.pos->data_size - PB_BYTES_ARRAY_T_ALLOCSIZE(0); + } + + if (PB_HTYPE(iter.pos->type) == PB_HTYPE_REPEATED) + { + *((pb_size_t*)iter.pSize) %= iter.pos->array_size; + } + + if (PB_HTYPE(iter.pos->type) == PB_HTYPE_ONEOF) + { + /* Set the oneof to this message type with 50% chance. */ + if (rand_word() & 1) + { + *((pb_size_t*)iter.pSize) = iter.pos->tag; + } + } + } +} + +static void generate_message() +{ + alltypes_static_AllTypes msg; + uint8_t buf[8192]; + pb_ostream_t stream = {0}; + + do { + if (stream.errmsg) + fprintf(stderr, "Encoder error: %s\n", stream.errmsg); + + stream = pb_ostream_from_buffer(buf, sizeof(buf)); + rand_fill((void*)&msg, sizeof(msg)); + limit_sizes(&msg); + } while (!pb_encode(&stream, alltypes_static_AllTypes_fields, &msg)); + + fwrite(buf, 1, stream.bytes_written, stdout); +} + +int main(int argc, char **argv) +{ + if (argc > 1) + { + random_seed = atol(argv[1]); + } + else + { + random_seed = time(NULL); + } + + fprintf(stderr, "Random seed: %llu\n", (long long unsigned)random_seed); + + generate_message(); + + return 0; +} + -- cgit 1.2.3-korg From 7f1e0ec17987ff10c5ce17f347b66423fe8ac4bd Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 5 Oct 2015 16:01:53 +0300 Subject: Fix regression in generating message size defines (issue #172). This bug was triggered when: 1. A .proto file included another .proto from a different directory. 2. The another .proto has an associated .options file. Added regression test for the same. --- generator/nanopb_generator.py | 18 +++++++++--------- tests/regression/issue_172/SConscript | 16 ++++++++++++++++ tests/regression/issue_172/msg_size.c | 9 +++++++++ .../regression/issue_172/submessage/submessage.options | 1 + tests/regression/issue_172/submessage/submessage.proto | 4 ++++ tests/regression/issue_172/test.proto | 6 ++++++ 6 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 tests/regression/issue_172/SConscript create mode 100644 tests/regression/issue_172/msg_size.c create mode 100644 tests/regression/issue_172/submessage/submessage.options create mode 100644 tests/regression/issue_172/submessage/submessage.proto create mode 100644 tests/regression/issue_172/test.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 37f7beb5..bc388958 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -514,18 +514,18 @@ class Field: return None if self.pbtype == 'MESSAGE': + encsize = None if str(self.submsgname) in dependencies: submsg = dependencies[str(self.submsgname)] encsize = submsg.encoded_size(dependencies) - if encsize is None: - return None # Submessage size is indeterminate - - # Include submessage length prefix - encsize += varint_max_size(encsize.upperlimit()) - else: - # Submessage cannot be found, this currently occurs when - # the submessage type is defined in a different file and - # not using the protoc plugin. + if encsize is not None: + # Include submessage length prefix + encsize += varint_max_size(encsize.upperlimit()) + + if encsize is None: + # Submessage or its size cannot be found. + # This can occur if submessage is defined in different + # file, and it or its .options could not be found. # Instead of direct numeric value, reference the size that # has been #defined in the other file. encsize = EncodedSize(self.submsgname + 'size') diff --git a/tests/regression/issue_172/SConscript b/tests/regression/issue_172/SConscript new file mode 100644 index 00000000..49c919e8 --- /dev/null +++ b/tests/regression/issue_172/SConscript @@ -0,0 +1,16 @@ +# Verify that _size define is generated for messages that have +# includes from another directory. + +Import('env') + +incpath = env.Clone() +incpath.Append(PROTOCPATH="#regression/issue_172/submessage") +incpath.Append(CPPPATH="$BUILD/regression/issue_172/submessage") +incpath.NanopbProto('test') +incpath.NanopbProto(['submessage/submessage', 'submessage/submessage.options']) + +p = incpath.Program(["msg_size.c", + "test.pb.c", + "submessage/submessage.pb.c"]) + + diff --git a/tests/regression/issue_172/msg_size.c b/tests/regression/issue_172/msg_size.c new file mode 100644 index 00000000..be45acb4 --- /dev/null +++ b/tests/regression/issue_172/msg_size.c @@ -0,0 +1,9 @@ +#include "test.pb.h" + +PB_STATIC_ASSERT(testmessage_size >= 1+1+1+1+16, TESTMESSAGE_SIZE_IS_WRONG) + +int main() +{ + return 0; +} + diff --git a/tests/regression/issue_172/submessage/submessage.options b/tests/regression/issue_172/submessage/submessage.options new file mode 100644 index 00000000..12fb1984 --- /dev/null +++ b/tests/regression/issue_172/submessage/submessage.options @@ -0,0 +1 @@ +submessage.data max_size: 16 diff --git a/tests/regression/issue_172/submessage/submessage.proto b/tests/regression/issue_172/submessage/submessage.proto new file mode 100644 index 00000000..ce6804af --- /dev/null +++ b/tests/regression/issue_172/submessage/submessage.proto @@ -0,0 +1,4 @@ +syntax = "proto2"; +message submessage { + required bytes data = 1; +} diff --git a/tests/regression/issue_172/test.proto b/tests/regression/issue_172/test.proto new file mode 100644 index 00000000..fbd97be5 --- /dev/null +++ b/tests/regression/issue_172/test.proto @@ -0,0 +1,6 @@ +syntax = "proto2"; +import "submessage.proto"; + +message testmessage { + optional submessage sub = 1; +} -- cgit 1.2.3-korg From b0a320dea81b084436d2a324c5df25c5cc1d9079 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Wed, 7 Oct 2015 18:16:36 -0700 Subject: generator: Remove trailing whitespace * Remove trailing whitespace * No functional changes --- generator/nanopb_generator.py | 274 +++++++++++++++++++++--------------------- 1 file changed, 137 insertions(+), 137 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index bc388958..2dad4ec1 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -86,7 +86,7 @@ class Names: if isinstance(parts, Names): parts = parts.parts self.parts = tuple(parts) - + def __str__(self): return '_'.join(self.parts) @@ -97,10 +97,10 @@ class Names: return Names(self.parts + other) else: raise ValueError("Name parts should be of type str") - + def __eq__(self, other): return isinstance(other, Names) and self.parts == other.parts - + def names_from_type_name(type_name): '''Parse Names() from FieldDescriptorProto type_name''' if type_name[0] != '.': @@ -130,7 +130,7 @@ class EncodedSize: value = 0 self.value = value self.symbols = symbols - + def __add__(self, other): if isinstance(other, int): return EncodedSize(self.value + other, self.symbols) @@ -162,43 +162,43 @@ class EncodedSize: class Enum: def __init__(self, names, desc, enum_options): '''desc is EnumDescriptorProto''' - + self.options = enum_options self.names = names + desc.name - + if enum_options.long_names: - self.values = [(self.names + x.name, x.number) for x in desc.value] + self.values = [(self.names + x.name, x.number) for x in desc.value] else: - self.values = [(names + x.name, x.number) for x in desc.value] - + self.values = [(names + x.name, x.number) for x in desc.value] + self.value_longnames = [self.names + x.name for x in desc.value] self.packed = enum_options.packed_enum - + def has_negative(self): for n, v in self.values: if v < 0: return True return False - + def encoded_size(self): return max([varint_max_size(v) for n,v in self.values]) - + def __str__(self): result = 'typedef enum _%s {\n' % self.names result += ',\n'.join([" %s = %d" % x for x in self.values]) result += '\n}' - + if self.packed: result += ' pb_packed' - + result += ' %s;' % self.names - + if not self.options.long_names: # Define the long names always so that enum value references # from other files work properly. for i, x in enumerate(self.values): result += '\n#define %s %s' % (self.value_longnames[i], x[0]) - + return result class FieldMaxSize: @@ -232,17 +232,17 @@ class Field: self.array_decl = "" self.enc_size = None self.ctype = None - + # Parse field options if field_options.HasField("max_size"): self.max_size = field_options.max_size - + if field_options.HasField("max_count"): self.max_count = field_options.max_count - + if desc.HasField('default_value'): self.default = desc.default_value - + # Check field rules, i.e. required/optional/repeated. can_be_static = True if desc.label == FieldD.LABEL_REQUIRED: @@ -257,26 +257,26 @@ class Field: self.array_decl = '[%d]' % self.max_count else: raise NotImplementedError(desc.label) - + # Check if the field can be implemented with static allocation # i.e. whether the data size is known. if desc.type == FieldD.TYPE_STRING and self.max_size is None: can_be_static = False - + if desc.type == FieldD.TYPE_BYTES and self.max_size is None: can_be_static = False - + # Decide how the field data will be allocated if field_options.type == nanopb_pb2.FT_DEFAULT: if can_be_static: field_options.type = nanopb_pb2.FT_STATIC else: field_options.type = nanopb_pb2.FT_CALLBACK - + if field_options.type == nanopb_pb2.FT_STATIC and not can_be_static: raise Exception("Field %s is defined as static, but max_size or " "max_count is not given." % self.name) - + if field_options.type == nanopb_pb2.FT_STATIC: self.allocation = 'STATIC' elif field_options.type == nanopb_pb2.FT_POINTER: @@ -285,7 +285,7 @@ class Field: self.allocation = 'CALLBACK' else: raise NotImplementedError(field_options.type) - + # Decide the C data type to use in the struct. if desc.type in datatypes: self.ctype, self.pbtype, self.enc_size, isa = datatypes[desc.type] @@ -321,16 +321,16 @@ class Field: self.enc_size = None # Needs to be filled in after the message type is available else: raise NotImplementedError(desc.type) - + def __lt__(self, other): return self.tag < other.tag - + def __str__(self): result = '' if self.allocation == 'POINTER': if self.rules == 'REPEATED': result += ' pb_size_t ' + self.name + '_count;\n' - + if self.pbtype == 'MESSAGE': # Use struct definition, so recursive submessages are possible result += ' struct _%s *%s;' % (self.ctype, self.name) @@ -348,7 +348,7 @@ class Field: result += ' pb_size_t ' + self.name + '_count;\n' result += ' %s %s%s;' % (self.ctype, self.name, self.array_decl) return result - + def types(self): '''Return definitions for any special types this field might need.''' if self.pbtype == 'BYTES' and self.allocation == 'STATIC': @@ -356,7 +356,7 @@ class Field: else: result = '' return result - + def get_dependencies(self): '''Get list of type names used by this field.''' if self.allocation == 'STATIC': @@ -403,7 +403,7 @@ class Field: inner_init = str(self.default) + 'll' else: inner_init = str(self.default) - + if inner_init_only: return inner_init @@ -438,7 +438,7 @@ class Field: ctype = self.ctype default = self.get_initializer(False, True) array_decl = '' - + if self.pbtype == 'STRING': if self.allocation != 'STATIC': return None # Not implemented @@ -446,17 +446,17 @@ class Field: elif self.pbtype == 'BYTES': if self.allocation != 'STATIC': return None # Not implemented - + if declaration_only: return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl) else: return 'const %s %s_default%s = %s;' % (ctype, self.struct_name + self.name, array_decl, default) - + def tags(self): '''Return the #define for the tag number of this field.''' identifier = '%s_%s_tag' % (self.struct_name, self.name) return '#define %-40s %d\n' % (identifier, self.tag) - + def pb_field_t(self, prev_field_name): '''Return the pb_field_t initializer to use in the constant array. prev_field_name is the name of the previous field or None. @@ -475,7 +475,7 @@ class Field: result += '%s, ' % self.struct_name result += '%s, ' % self.name result += '%s, ' % (prev_field_name or self.name) - + if self.pbtype == 'MESSAGE': result += '&%s_fields)' % self.submsgname elif self.default is None: @@ -486,9 +486,9 @@ class Field: result += '0)' # Default value for extensions is not implemented else: result += '&%s_default)' % (self.struct_name + self.name) - + return result - + def largest_field_value(self): '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly. Returns numeric value or a C-expression for assert.''' @@ -509,10 +509,10 @@ class Field: '''Return the maximum size that this field can take when encoded, including the field tag. If the size cannot be determined, returns None.''' - + if self.allocation != 'STATIC': return None - + if self.pbtype == 'MESSAGE': encsize = None if str(self.submsgname) in dependencies: @@ -547,7 +547,7 @@ class Field: % (self.struct_name, self.name)) else: encsize = EncodedSize(self.enc_size) - + encsize += varint_max_size(self.tag << 3) # Tag + wire type if self.rules == 'REPEATED': @@ -555,7 +555,7 @@ class Field: # Therefore we have to reserve space for it, even though # we emit packed arrays ourselves. encsize *= self.max_count - + return encsize @@ -577,16 +577,16 @@ class ExtensionRange(Field): self.default = None self.max_size = 0 self.max_count = 0 - + def __str__(self): return ' pb_extension_t *extensions;' - + def types(self): return '' - + def tags(self): return '' - + def encoded_size(self, dependencies): # We exclude extensions from the count, because they cannot be known # until runtime. Other option would be to return None here, but this @@ -598,7 +598,7 @@ class ExtensionField(Field): self.fullname = struct_name + desc.name self.extendee_name = names_from_type_name(desc.extendee) Field.__init__(self, self.fullname + 'struct', desc, field_options) - + if self.rules != 'OPTIONAL': self.skip = True else: @@ -616,7 +616,7 @@ class ExtensionField(Field): msg = '/* Extension field %s was skipped because only "optional"\n' % self.fullname msg +=' type of extension fields is currently supported. */\n' return msg - + return ('extern const pb_extension_type_t %s; /* field type: %s */\n' % (self.fullname, str(self).strip())) @@ -758,13 +758,13 @@ class Message: self.oneofs[f.oneof_index].add_field(field) else: self.fields.append(field) - + if len(desc.extension_range) > 0: field_options = get_nanopb_suboptions(desc, message_options, self.name + 'extensions') range_start = min([r.start for r in desc.extension_range]) if field_options.type != nanopb_pb2.FT_IGNORE: self.fields.append(ExtensionRange(self.name, range_start, field_options)) - + self.packed = message_options.packed_struct self.ordered_fields = self.fields[:] self.ordered_fields.sort() @@ -775,7 +775,7 @@ class Message: for f in self.fields: deps += f.get_dependencies() return deps - + def __str__(self): result = 'typedef struct _%s {\n' % self.name @@ -786,30 +786,30 @@ class Message: result += '\n'.join([str(f) for f in self.ordered_fields]) result += '\n}' - + if self.packed: result += ' pb_packed' - + result += ' %s;' % self.name - + if self.packed: result = 'PB_PACKED_STRUCT_START\n' + result result += '\nPB_PACKED_STRUCT_END' - + return result - + def types(self): return ''.join([f.types() for f in self.fields]) def get_initializer(self, null_init): if not self.ordered_fields: return '{0}' - + parts = [] for field in self.ordered_fields: parts.append(field.get_initializer(null_init)) return '{' + ', '.join(parts) + '}' - + def default_decl(self, declaration_only = False): result = "" for field in self.fields: @@ -842,7 +842,7 @@ class Message: def fields_definition(self): result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, self.count_all_fields() + 1) - + prev = None for field in self.ordered_fields: result += field.pb_field_t(prev) @@ -851,7 +851,7 @@ class Message: prev = field.name + '.' + field.fields[-1].name else: prev = field.name - + result += ' PB_LAST_FIELD\n};' return result @@ -865,7 +865,7 @@ class Message: if fsize is None: return None size += fsize - + return size @@ -879,11 +879,11 @@ def iterate_messages(desc, names = Names()): submsgs = desc.message_type else: submsgs = desc.nested_type - + for submsg in submsgs: sub_names = names + submsg.name yield sub_names, submsg - + for x in iterate_messages(submsg, sub_names): yield x @@ -924,7 +924,7 @@ def sort_dependencies(messages): for message in messages: dependencies[str(message.name)] = set(message.get_dependencies()) message_by_name[str(message.name)] = message - + for msgname in toposort2(dependencies): if msgname in message_by_name: yield message_by_name[msgname] @@ -946,7 +946,7 @@ class ProtoFile: self.file_options = file_options self.dependencies = {} self.parse() - + # Some of types used in this file probably come from the file itself. # Thus it has implicit dependency on itself. self.add_dependency(self) @@ -955,39 +955,39 @@ class ProtoFile: self.enums = [] self.messages = [] self.extensions = [] - + if self.fdesc.package: base_name = Names(self.fdesc.package.split('.')) else: base_name = Names() - + for enum in self.fdesc.enum_type: enum_options = get_nanopb_suboptions(enum, self.file_options, base_name + enum.name) self.enums.append(Enum(base_name, enum, enum_options)) - + for names, message in iterate_messages(self.fdesc, base_name): message_options = get_nanopb_suboptions(message, self.file_options, names) - + if message_options.skip_message: continue - + self.messages.append(Message(names, message, message_options)) for enum in message.enum_type: enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name) self.enums.append(Enum(names, enum, enum_options)) - + for names, extension in iterate_extensions(self.fdesc, base_name): field_options = get_nanopb_suboptions(extension, self.file_options, names + extension.name) if field_options.type != nanopb_pb2.FT_IGNORE: self.extensions.append(ExtensionField(names, extension, field_options)) - + def add_dependency(self, other): for enum in other.enums: self.dependencies[str(enum.names)] = enum - + for msg in other.messages: self.dependencies[str(msg.name)] = msg - + # Fix field default values where enum short names are used. for enum in other.enums: if not enum.options.long_names: @@ -996,7 +996,7 @@ class ProtoFile: if field.default in enum.value_longnames: idx = enum.value_longnames.index(field.default) field.default = enum.values[idx][0] - + # Fix field data types where enums have negative values. for enum in other.enums: if not enum.has_negative(): @@ -1009,13 +1009,13 @@ class ProtoFile: '''Generate content for a header file. Generates strings, which should be concatenated and stored to file. ''' - + yield '/* Automatically generated nanopb header */\n' if options.notimestamp: yield '/* Generated by %s */\n\n' % (nanopb_version) else: yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) - + symbol = make_identifier(headername) yield '#ifndef PB_%s_INCLUDED\n' % symbol yield '#define PB_%s_INCLUDED\n' % symbol @@ -1025,7 +1025,7 @@ class ProtoFile: # no %s specified - use whatever was passed in as options.libformat yield options.libformat yield '\n' - + for incfile in includes: noext = os.path.splitext(incfile)[0] yield options.genformat % (noext + options.extension + '.h') @@ -1039,30 +1039,30 @@ class ProtoFile: yield '#ifdef __cplusplus\n' yield 'extern "C" {\n' yield '#endif\n\n' - + if self.enums: yield '/* Enum definitions */\n' for enum in self.enums: yield str(enum) + '\n\n' - + if self.messages: yield '/* Struct definitions */\n' for msg in sort_dependencies(self.messages): yield msg.types() yield str(msg) + '\n\n' - + if self.extensions: yield '/* Extensions */\n' for extension in self.extensions: yield extension.extension_decl() yield '\n' - + if self.messages: yield '/* Default values for struct fields */\n' for msg in self.messages: yield msg.default_decl(True) yield '\n' - + yield '/* Initializer values for message structs */\n' for msg in self.messages: identifier = '%s_init_default' % msg.name @@ -1071,7 +1071,7 @@ class ProtoFile: identifier = '%s_init_zero' % msg.name yield '#define %-40s %s\n' % (identifier, msg.get_initializer(True)) yield '\n' - + yield '/* Field tags (for use in manual encoding/decoding) */\n' for msg in sort_dependencies(self.messages): for field in msg.fields: @@ -1079,12 +1079,12 @@ class ProtoFile: for extension in self.extensions: yield extension.tags() yield '\n' - + yield '/* Struct field encoding specification for nanopb */\n' for msg in self.messages: yield msg.fields_declaration() + '\n' yield '\n' - + yield '/* Maximum encoded size of messages (where known) */\n' for msg in self.messages: msize = msg.encoded_size(self.dependencies) @@ -1094,7 +1094,7 @@ class ProtoFile: yield '\n' yield '/* Message IDs (where set with "msgid" option) */\n' - + yield '#ifdef PB_MSGID\n' for msg in self.messages: if hasattr(msg,'msgid'): @@ -1123,13 +1123,13 @@ class ProtoFile: yield '#ifdef __cplusplus\n' yield '} /* extern "C" */\n' yield '#endif\n' - + # End of header yield '\n#endif\n' def generate_source(self, headername, options): '''Generate content for a source file.''' - + yield '/* Automatically generated nanopb constant definitions */\n' if options.notimestamp: yield '/* Generated by %s */\n\n' % (nanopb_version) @@ -1137,23 +1137,23 @@ class ProtoFile: yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) yield options.genformat % (headername) yield '\n' - + yield '#if PB_PROTO_HEADER_VERSION != 30\n' yield '#error Regenerate this file with the current version of nanopb generator.\n' yield '#endif\n' yield '\n' - + for msg in self.messages: yield msg.default_decl(False) - + yield '\n\n' - + for msg in self.messages: yield msg.fields_definition() + '\n\n' - + for ext in self.extensions: yield ext.extension_def() + '\n' - + # Add checks for numeric limits if self.messages: largest_msg = max(self.messages, key = lambda m: m.count_required_fields()) @@ -1178,7 +1178,7 @@ class ProtoFile: if worst > 255 or checks: yield '\n/* Check that field information fits in pb_field_t */\n' - + if worst > 65535 or checks: yield '#if !defined(PB_FIELD_32BIT)\n' if worst > 65535: @@ -1195,7 +1195,7 @@ class ProtoFile: yield ' */\n' yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) yield '#endif\n\n' - + if worst < 65536: yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n' if worst > 255: @@ -1212,14 +1212,14 @@ class ProtoFile: yield ' */\n' yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) yield '#endif\n\n' - + # Add check for sizeof(double) has_double = False for msg in self.messages: for field in msg.fields: if field.ctype == 'double': has_double = True - + if has_double: yield '\n' yield '/* On some platforms (such as AVR), double is really float.\n' @@ -1227,7 +1227,7 @@ class ProtoFile: yield ' * To get rid of this error, remove any double fields from your .proto.\n' yield ' */\n' yield 'PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n' - + yield '\n' # --------------------------------------------------------------------------- @@ -1249,17 +1249,17 @@ def read_options_file(infile): line = line.strip() if not line: continue - + parts = line.split(None, 1) - + if len(parts) < 2: sys.stderr.write("%s:%d: " % (infile.name, i + 1) + "Option lines should have space between field name and options. " + "Skipping line: '%s'\n" % line) continue - + opts = nanopb_pb2.NanoPBOptions() - + try: text_format.Merge(parts[1], opts) except Exception as e: @@ -1281,14 +1281,14 @@ def get_nanopb_suboptions(subdesc, options, name): '''Get copy of options, and merge information from subdesc.''' new_options = nanopb_pb2.NanoPBOptions() new_options.CopyFrom(options) - + # Handle options defined in a separate file dotname = '.'.join(name.parts) for namemask, options in Globals.separate_options: if fnmatch(dotname, namemask): Globals.matched_namemasks.add(namemask) new_options.MergeFrom(options) - + # Handle options defined in .proto if isinstance(subdesc.options, descriptor.FieldOptions): ext_type = nanopb_pb2.nanopb @@ -1300,15 +1300,15 @@ def get_nanopb_suboptions(subdesc, options, name): ext_type = nanopb_pb2.nanopb_enumopt else: raise Exception("Unknown options type") - + if subdesc.options.HasExtension(ext_type): ext = subdesc.options.Extensions[ext_type] new_options.MergeFrom(ext) - + if Globals.verbose_options: sys.stderr.write("Options for " + dotname + ": ") sys.stderr.write(text_format.MessageToString(new_options) + "\n") - + return new_options @@ -1317,7 +1317,7 @@ def get_nanopb_suboptions(subdesc, options, name): # --------------------------------------------------------------------------- import sys -import os.path +import os.path from optparse import OptionParser optparser = OptionParser( @@ -1353,11 +1353,11 @@ def parse_file(filename, fdesc, options): toplevel_options = nanopb_pb2.NanoPBOptions() for s in options.settings: text_format.Merge(s, toplevel_options) - + if not fdesc: data = open(filename, 'rb').read() fdesc = descriptor.FileDescriptorSet.FromString(data).file[0] - + # Check if there is a separate .options file had_abspath = False try: @@ -1384,12 +1384,12 @@ def parse_file(filename, fdesc, options): Globals.separate_options = [] Globals.matched_namemasks = set() - + # Parse the file file_options = get_nanopb_suboptions(fdesc, toplevel_options, Names([filename])) f = ProtoFile(fdesc, file_options) f.optfilename = optfilename - + return f def process_file(filename, fdesc, options, other_files = {}): @@ -1397,7 +1397,7 @@ def process_file(filename, fdesc, options, other_files = {}): filename: The full path to the .proto or .pb source file, as string. fdesc: The loaded FileDescriptorSet, or None to read from the input file. options: Command line options as they come from OptionsParser. - + Returns a dict: {'headername': Name of header file, 'headerdata': Data for the .h header file, @@ -1417,12 +1417,12 @@ def process_file(filename, fdesc, options, other_files = {}): headername = noext + options.extension + '.h' sourcename = noext + options.extension + '.c' headerbasename = os.path.basename(headername) - + # List of .proto files that should not be included in the C header file # even if they are mentioned in the source .proto. excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude includes = [d for d in f.fdesc.dependency if d not in excludes] - + headerdata = ''.join(f.generate_header(includes, headerbasename, options)) sourcedata = ''.join(f.generate_source(headerbasename, options)) @@ -1436,30 +1436,30 @@ def process_file(filename, fdesc, options, other_files = {}): return {'headername': headername, 'headerdata': headerdata, 'sourcename': sourcename, 'sourcedata': sourcedata} - + def main_cli(): '''Main function when invoked directly from the command line.''' - + options, filenames = optparser.parse_args() - + if not filenames: optparser.print_help() sys.exit(1) - + if options.quiet: options.verbose = False Globals.verbose_options = options.verbose - + for filename in filenames: results = process_file(filename, None, options) - + if not options.quiet: sys.stderr.write("Writing to " + results['headername'] + " and " + results['sourcename'] + "\n") - + open(results['headername'], 'w').write(results['headerdata']) - open(results['sourcename'], 'w').write(results['sourcedata']) + open(results['sourcename'], 'w').write(results['sourcedata']) def main_plugin(): '''Main function when invoked as a protoc plugin.''' @@ -1470,51 +1470,51 @@ def main_plugin(): # Set stdin and stdout to binary mode msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) - + data = io.open(sys.stdin.fileno(), "rb").read() request = plugin_pb2.CodeGeneratorRequest.FromString(data) - + try: # Versions of Python prior to 2.7.3 do not support unicode # input to shlex.split(). Try to convert to str if possible. params = str(request.parameter) except UnicodeEncodeError: params = request.parameter - + import shlex args = shlex.split(params) options, dummy = optparser.parse_args(args) - + Globals.verbose_options = options.verbose - + response = plugin_pb2.CodeGeneratorResponse() - + # Google's protoc does not currently indicate the full path of proto files. # Instead always add the main file path to the search dirs, that works for # the common case. import os.path options.options_path.append(os.path.dirname(request.file_to_generate[0])) - + # Process any include files first, in order to have them # available as dependencies other_files = {} for fdesc in request.proto_file: other_files[fdesc.name] = parse_file(fdesc.name, fdesc, options) - + for filename in request.file_to_generate: for fdesc in request.proto_file: if fdesc.name == filename: results = process_file(filename, fdesc, options, other_files) - + f = response.file.add() f.name = results['headername'] f.content = results['headerdata'] f = response.file.add() f.name = results['sourcename'] - f.content = results['sourcedata'] - + f.content = results['sourcedata'] + io.open(sys.stdout.fileno(), "wb").write(response.SerializeToString()) if __name__ == '__main__': -- cgit 1.2.3-korg From a97617bb41d2e03c670467a0d053b4d2eaae8dad Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Thu, 15 Oct 2015 13:04:23 -0700 Subject: cmake: Build generator files in build directory Treat the source directory as immutable. Copy the generator directory which previously generated files in-tree to the build directory and then generate files. Many emerging continuous integration build systems test builds across multiple versions of dependencies protobuf and python versions in particular. The previous source tree builds resulted in stale files from the last build breaking the current build. By placing the build files in the build directory, the build system automatically removes stale files (removes output build directory) and regenerates them as necessary. --- extra/FindNanopb.cmake | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/extra/FindNanopb.cmake b/extra/FindNanopb.cmake index 1fd25334..7734a937 100644 --- a/extra/FindNanopb.cmake +++ b/extra/FindNanopb.cmake @@ -26,8 +26,8 @@ # NANOPB_INCLUDE_DIRS - Include directories for Google Protocol Buffers # # The following cache variables are also available to set or use: -# NANOPB_GENERATOR_EXECUTABLE - The nanopb generator # PROTOBUF_PROTOC_EXECUTABLE - The protoc compiler +# NANOPB_GENERATOR_SOURCE_DIR - The nanopb generator source # # ==================================================================== # @@ -127,12 +127,29 @@ function(NANOPB_GENERATE_CPP SRCS HDRS) set(${SRCS}) set(${HDRS}) - get_filename_component(GENERATOR_PATH ${NANOPB_GENERATOR_EXECUTABLE} PATH) + + set(GENERATOR_PATH ${CMAKE_BINARY_DIR}/nanopb/generator) + + set(NANOPB_GENERATOR_EXECUTABLE ${GENERATOR_PATH}/nanopb_generator.py) + set(GENERATOR_CORE_DIR ${GENERATOR_PATH}/proto) set(GENERATOR_CORE_SRC ${GENERATOR_CORE_DIR}/nanopb.proto ${GENERATOR_CORE_DIR}/plugin.proto) + # Treat the source diretory as immutable. + # + # Copy the generator directory to the build directory before + # compiling python and proto files. Fixes issues when using the + # same build directory with different python/protobuf versions + # as the binary build directory is discarded across builds. + # + add_custom_command( + OUTPUT ${NANOPB_GENERATOR_EXECUTABLE} ${GENERATOR_CORE_SRC} + COMMAND ${CMAKE_COMMAND} -E copy_directory + ARGS ${NANOPB_GENERATOR_SOURCE_DIR} ${GENERATOR_PATH} + VERBATIM) + set(GENERATOR_CORE_PYTHON_SRC) foreach(FIL ${GENERATOR_CORE_SRC}) get_filename_component(ABS_FIL ${FIL} ABSOLUTE) @@ -235,14 +252,14 @@ find_program(PROTOBUF_PROTOC_EXECUTABLE ) mark_as_advanced(PROTOBUF_PROTOC_EXECUTABLE) -# Find nanopb generator -find_file(NANOPB_GENERATOR_EXECUTABLE +# Find nanopb generator source dir +find_path(NANOPB_GENERATOR_SOURCE_DIR NAMES nanopb_generator.py - DOC "nanopb generator" + DOC "nanopb generator source" PATHS ${NANOPB_SRC_ROOT_FOLDER}/generator ) -mark_as_advanced(NANOPB_GENERATOR_EXECUTABLE) +mark_as_advanced(NANOPB_GENERATOR_SOURCE_DIR) find_package(PythonInterp REQUIRED) @@ -250,6 +267,6 @@ include(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(NANOPB DEFAULT_MSG NANOPB_INCLUDE_DIRS NANOPB_SRCS NANOPB_HDRS - NANOPB_GENERATOR_EXECUTABLE + NANOPB_GENERATOR_SOURCE_DIR PROTOBUF_PROTOC_EXECUTABLE ) -- cgit 1.2.3-korg From 863dddca69983485fdc3faf58c7dd439d0d5bf5d Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Thu, 15 Oct 2015 13:56:20 -0700 Subject: cmake: Auto discover NANOPB_SRC_ROOT_FOLDER * Use CMAKE_CURRENT_LIST_DIR to learn the path of the currently running file to determine location of NanoPB. * Simplifies use in other projects. --- extra/FindNanopb.cmake | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/extra/FindNanopb.cmake b/extra/FindNanopb.cmake index 7734a937..9afb21d0 100644 --- a/extra/FindNanopb.cmake +++ b/extra/FindNanopb.cmake @@ -1,10 +1,6 @@ # This is an example script for use with CMake projects for locating and configuring # the nanopb library. # -# The following varialbes have to be set: -# -# NANOPB_SRC_ROOT_FOLDER - Path to nanopb source folder -# # The following variables can be set and are optional: # # @@ -217,6 +213,12 @@ if(NOT DEFINED NANOPB_GENERATE_CPP_APPEND_PATH) set(NANOPB_GENERATE_CPP_APPEND_PATH TRUE) endif() +# Make a really good guess regarding location of NANOPB_SRC_ROOT_FOLDER +if(NOT DEFINED NANOPB_SRC_ROOT_FOLDER) + get_filename_component(NANOPB_SRC_ROOT_FOLDER + ${CMAKE_CURRENT_LIST_DIR}/.. ABSOLUTE) +endif() + # Find the include directory find_path(NANOPB_INCLUDE_DIRS pb.h -- cgit 1.2.3-korg From 6cedf928c81885ea9cc5f4e0bed2679384427435 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Thu, 15 Oct 2015 13:57:37 -0700 Subject: examples: cmake_simple: Use auto discovered path * No need to specify the src path and the module path. Let cmake do it. --- examples/cmake_simple/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/cmake_simple/CMakeLists.txt b/examples/cmake_simple/CMakeLists.txt index 5edfb61f..e5f33a02 100644 --- a/examples/cmake_simple/CMakeLists.txt +++ b/examples/cmake_simple/CMakeLists.txt @@ -1,8 +1,7 @@ cmake_minimum_required(VERSION 2.8) project(NANOPB_CMAKE_SIMPLE C) -set(NANOPB_SRC_ROOT_FOLDER ${CMAKE_CURRENT_SOURCE_DIR}/../..) -set(CMAKE_MODULE_PATH ${NANOPB_SRC_ROOT_FOLDER}/extra) +set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../extra) find_package(Nanopb REQUIRED) include_directories(${NANOPB_INCLUDE_DIRS}) -- cgit 1.2.3-korg From d9107adbc0c9c2ee9e899a78877a95d350a5a9aa Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 24 Oct 2015 22:20:39 +0300 Subject: Add contributing.md to provide issue templates on github --- CONTRIBUTING.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..4041bc3c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,32 @@ +Contributing to Nanopb development +================================== + +Reporting issues and requesting features +---------------------------------------- + +Feel free to report any issues you see or features you would like +to see in the future to the Github issue tracker. Using the templates +below is preferred: + +* [Report a bug](https://github.com/nanopb/nanopb/issues/new?body=**Steps%20to%20reproduce%20the%20issue**%0a%0a1.%0a2.%0a3.%0a%0a**What%20happens?**%0A%0A**What%20should%20happen?**&labels=Type-Defect) +* [Request a feature](https://github.com/nanopb/nanopb/issues/new?body=**What%20should%20the%20feature%20do?**%0A%0A**In%20what%20situation%20would%20the%20feature%20be%20useful?**&labels=Type-Enhancement) + +Requesting help +--------------- + +If there is something strange going on, but you do not know if +it is actually a bug in nanopb, try asking first on the +[discussion forum](https://groups.google.com/forum/#!forum/nanopb). + +Pull requests +------------- + +Pull requests are welcome! + +If it is not obvious from the commit message, please indicate the +same information as you would for an issue report: + +* What functionality it fixes/adds. +* How can the problem be reproduced / when would the feature be useful. + + -- cgit 1.2.3-korg From 6c136580a091a30e94aaebd66dd978117d999ad0 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 25 Oct 2015 18:07:23 +0200 Subject: Ignore null pointers in pb_release() (issue #183). --- pb_decode.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pb_decode.c b/pb_decode.c index 5cdcbcfb..50ada86a 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -1067,6 +1067,9 @@ void pb_release(const pb_field_t fields[], void *dest_struct) { pb_field_iter_t iter; + if (!dest_struct) + return; /* Ignore NULL pointers, similar to free() */ + if (!pb_field_iter_begin(&iter, fields, dest_struct)) return; /* Empty message type */ -- cgit 1.2.3-korg From 9a8cc59703b227ff418cc55b4dc24f63eda64b62 Mon Sep 17 00:00:00 2001 From: Benjamin Kamath Date: Wed, 4 Nov 2015 16:42:25 -0800 Subject: Add option to allow for anonymous unions --- generator/nanopb_generator.py | 19 ++++++++++++++++--- generator/proto/nanopb.proto | 3 +++ pb.h | 17 ++++++++++++++++- 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 2dad4ec1..a21257b5 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -463,7 +463,10 @@ class Field: ''' if self.rules == 'ONEOF': - result = ' PB_ONEOF_FIELD(%s, ' % self.union_name + if self.anonymous: + result = ' PB_ANONYMOUS_ONEOF_FIELD(%s, ' % self.union_name + else: + result = ' PB_ONEOF_FIELD(%s, ' % self.union_name else: result = ' PB_FIELD(' @@ -497,7 +500,10 @@ class Field: if self.rules == 'REPEATED' and self.allocation == 'STATIC': check.append('pb_membersize(%s, %s[0])' % (self.struct_name, self.name)) elif self.rules == 'ONEOF': - check.append('pb_membersize(%s, %s.%s)' % (self.struct_name, self.union_name, self.name)) + if self.anonymous: + check.append('pb_membersize(%s, %s)' % (self.struct_name, self.name)) + else: + check.append('pb_membersize(%s, %s.%s)' % (self.struct_name, self.union_name, self.name)) else: check.append('pb_membersize(%s, %s)' % (self.struct_name, self.name)) @@ -653,6 +659,7 @@ class OneOf(Field): self.allocation = 'ONEOF' self.default = None self.rules = 'ONEOF' + self.anonymous = False def add_field(self, field): if field.allocation == 'CALLBACK': @@ -661,6 +668,7 @@ class OneOf(Field): field.union_name = self.name field.rules = 'ONEOF' + field.anonymous = self.anonymous self.fields.append(field) self.fields.sort(key = lambda f: f.tag) @@ -674,7 +682,10 @@ class OneOf(Field): result += ' union {\n' for f in self.fields: result += ' ' + str(f).replace('\n', '\n ') + '\n' - result += ' } ' + self.name + ';' + if self.anonymous: + result += ' };' + else: + result += ' } ' + self.name + ';' return result def types(self): @@ -742,6 +753,8 @@ class Message: pass # No union and skip fields also else: oneof = OneOf(self.name, f) + if oneof_options.anonymous_oneof: + oneof.anonymous = True self.oneofs[i] = oneof self.fields.append(oneof) diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index b8671bbd..9b2f0fbc 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -62,6 +62,9 @@ message NanoPBOptions { // integer type tag for a message optional uint32 msgid = 9; + + // decode oneof as anonymous union + optional bool anonymous_oneof = 11 [default = false]; } // Extensions to protoc 'Descriptor' type in order to define options diff --git a/pb.h b/pb.h index 98613a06..0286b855 100644 --- a/pb.h +++ b/pb.h @@ -518,10 +518,25 @@ struct pb_extension_s { pb_membersize(st, u.m[0]), 0, ptr} #define PB_ONEOF_FIELD(union_name, tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ - PB_ ## rules ## _ ## allocation(union_name, tag, message, field, \ + PB_ ## ONEOF_ ## allocation(union_name, tag, message, field, \ PB_DATAOFFSET_ ## placement(message, union_name.field, prevfield), \ PB_LTYPE_MAP_ ## type, ptr) +#define PB_ANONYMOUS_ONEOF_STATIC(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, m), \ + pb_membersize(st, m), 0, ptr} + +#define PB_ANONYMOUS_ONEOF_POINTER(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, m), \ + pb_membersize(st, m[0]), 0, ptr} + +#define PB_ANONYMOUS_ONEOF_FIELD(union_name, tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ + PB_ ## ANONYMOUS_ONEOF_ ## allocation(union_name, tag, message, field, \ + PB_DATAOFFSET_ ## placement(message, field, prevfield), \ + PB_LTYPE_MAP_ ## type, ptr) + /* These macros are used for giving out error messages. * They are mostly a debugging aid; the main error information * is the true/false return value from functions. -- cgit 1.2.3-korg From c1aa8aa980daea732c4683c2593d130d6bee7a79 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 14 Nov 2015 22:23:48 +0200 Subject: Add testcase for anonymous unions + few fixes. Fixes compilation error with anonymous unions when it is not the last field in message. Also fixes extraneous newlines in header file. Cleanup the pb.h extraneous use of ##. --- generator/nanopb_generator.py | 16 +++++-- pb.h | 4 +- tests/anonymous_oneof/SConscript | 29 ++++++++++++ tests/anonymous_oneof/decode_oneof.c | 88 ++++++++++++++++++++++++++++++++++++ tests/anonymous_oneof/oneof.proto | 23 ++++++++++ 5 files changed, 153 insertions(+), 7 deletions(-) create mode 100644 tests/anonymous_oneof/SConscript create mode 100644 tests/anonymous_oneof/decode_oneof.c create mode 100644 tests/anonymous_oneof/oneof.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index a21257b5..13217462 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -492,6 +492,9 @@ class Field: return result + def get_last_field_name(self): + return self.name + def largest_field_value(self): '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly. Returns numeric value or a C-expression for assert.''' @@ -704,12 +707,18 @@ class OneOf(Field): return None def tags(self): - return '\n'.join([f.tags() for f in self.fields]) + return ''.join([f.tags() for f in self.fields]) def pb_field_t(self, prev_field_name): result = ',\n'.join([f.pb_field_t(prev_field_name) for f in self.fields]) return result + def get_last_field_name(self): + if self.anonymous: + return self.fields[-1].name + else: + return self.name + '.' + self.fields[-1].name + def largest_field_value(self): largest = FieldMaxSize() for f in self.fields: @@ -860,10 +869,7 @@ class Message: for field in self.ordered_fields: result += field.pb_field_t(prev) result += ',\n' - if isinstance(field, OneOf): - prev = field.name + '.' + field.fields[-1].name - else: - prev = field.name + prev = field.get_last_field_name() result += ' PB_LAST_FIELD\n};' return result diff --git a/pb.h b/pb.h index 0286b855..8b9338cf 100644 --- a/pb.h +++ b/pb.h @@ -518,7 +518,7 @@ struct pb_extension_s { pb_membersize(st, u.m[0]), 0, ptr} #define PB_ONEOF_FIELD(union_name, tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ - PB_ ## ONEOF_ ## allocation(union_name, tag, message, field, \ + PB_ONEOF_ ## allocation(union_name, tag, message, field, \ PB_DATAOFFSET_ ## placement(message, union_name.field, prevfield), \ PB_LTYPE_MAP_ ## type, ptr) @@ -533,7 +533,7 @@ struct pb_extension_s { pb_membersize(st, m[0]), 0, ptr} #define PB_ANONYMOUS_ONEOF_FIELD(union_name, tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ - PB_ ## ANONYMOUS_ONEOF_ ## allocation(union_name, tag, message, field, \ + PB_ANONYMOUS_ONEOF_ ## allocation(union_name, tag, message, field, \ PB_DATAOFFSET_ ## placement(message, field, prevfield), \ PB_LTYPE_MAP_ ## type, ptr) diff --git a/tests/anonymous_oneof/SConscript b/tests/anonymous_oneof/SConscript new file mode 100644 index 00000000..8f634bd7 --- /dev/null +++ b/tests/anonymous_oneof/SConscript @@ -0,0 +1,29 @@ +# Test anonymous_oneof generator option + +Import('env') + +import re + +match = None +if 'PROTOC_VERSION' in env: + match = re.search('([0-9]+).([0-9]+).([0-9]+)', env['PROTOC_VERSION']) + +if match: + version = map(int, match.groups()) + +# Oneof is supported by protoc >= 2.6.0 +if env.GetOption('clean') or (match and (version[0] > 2 or (version[0] == 2 and version[1] >= 6))): + # Anonymous oneofs are supported by clang and gcc + if 'clang' in env['CC'] or 'gcc' in env['CC']: + env2 = env.Clone() + env2.Append(CFLAGS = "-Wno-pedantic") + env2.NanopbProto('oneof') + + dec = env2.Program(['decode_oneof.c', + 'oneof.pb.c', + '$COMMON/pb_decode.o', + '$COMMON/pb_common.o']) + + env2.RunTest("message1.txt", [dec, '$BUILD/oneof/message1.pb'], ARGS = ['1']) + env2.RunTest("message2.txt", [dec, '$BUILD/oneof/message2.pb'], ARGS = ['2']) + env2.RunTest("message3.txt", [dec, '$BUILD/oneof/message3.pb'], ARGS = ['3']) diff --git a/tests/anonymous_oneof/decode_oneof.c b/tests/anonymous_oneof/decode_oneof.c new file mode 100644 index 00000000..0f774dbc --- /dev/null +++ b/tests/anonymous_oneof/decode_oneof.c @@ -0,0 +1,88 @@ +/* Decode a message using oneof fields */ + +#include +#include +#include +#include +#include "oneof.pb.h" +#include "test_helpers.h" +#include "unittests.h" + +/* Test the 'AnonymousOneOfMessage' */ +int test_oneof_1(pb_istream_t *stream, int option) +{ + AnonymousOneOfMessage msg; + int status = 0; + + /* To better catch initialization errors */ + memset(&msg, 0xAA, sizeof(msg)); + + if (!pb_decode(stream, AnonymousOneOfMessage_fields, &msg)) + { + printf("Decoding failed: %s\n", PB_GET_ERROR(stream)); + return 1; + } + + /* Check that the basic fields work normally */ + TEST(msg.prefix == 123); + TEST(msg.suffix == 321); + + /* Check that we got the right oneof according to command line */ + if (option == 1) + { + TEST(msg.which_values == AnonymousOneOfMessage_first_tag); + TEST(msg.first == 999); + } + else if (option == 2) + { + TEST(msg.which_values == AnonymousOneOfMessage_second_tag); + TEST(strcmp(msg.second, "abcd") == 0); + } + else if (option == 3) + { + TEST(msg.which_values == AnonymousOneOfMessage_third_tag); + TEST(msg.third.array[0] == 1); + TEST(msg.third.array[1] == 2); + TEST(msg.third.array[2] == 3); + TEST(msg.third.array[3] == 4); + TEST(msg.third.array[4] == 5); + } + + return status; +} + +int main(int argc, char **argv) +{ + uint8_t buffer[AnonymousOneOfMessage_size]; + size_t count; + int option; + + if (argc != 2) + { + fprintf(stderr, "Usage: decode_oneof [number]\n"); + return 1; + } + option = atoi(argv[1]); + + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + if (!feof(stdin)) + { + printf("Message does not fit in buffer\n"); + return 1; + } + + { + int status = 0; + pb_istream_t stream; + + stream = pb_istream_from_buffer(buffer, count); + status = test_oneof_1(&stream, option); + + if (status != 0) + return status; + } + + return 0; +} diff --git a/tests/anonymous_oneof/oneof.proto b/tests/anonymous_oneof/oneof.proto new file mode 100644 index 00000000..d56285c0 --- /dev/null +++ b/tests/anonymous_oneof/oneof.proto @@ -0,0 +1,23 @@ +syntax = "proto2"; + +import 'nanopb.proto'; + +message SubMessage +{ + repeated int32 array = 1 [(nanopb).max_count = 8]; +} + +/* Oneof in a message with other fields */ +message AnonymousOneOfMessage +{ + option (nanopb_msgopt).anonymous_oneof = true; + required int32 prefix = 1; + oneof values + { + int32 first = 5; + string second = 6 [(nanopb).max_size = 8]; + SubMessage third = 7; + } + required int32 suffix = 99; +} + -- cgit 1.2.3-korg From c201f368c592508e8f2289c3f0f19e681db153a3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 14 Nov 2015 22:37:54 +0200 Subject: Fix -Wno-pedantic on old GCC --- tests/anonymous_oneof/SConscript | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/anonymous_oneof/SConscript b/tests/anonymous_oneof/SConscript index 8f634bd7..051c772f 100644 --- a/tests/anonymous_oneof/SConscript +++ b/tests/anonymous_oneof/SConscript @@ -16,7 +16,7 @@ if env.GetOption('clean') or (match and (version[0] > 2 or (version[0] == 2 and # Anonymous oneofs are supported by clang and gcc if 'clang' in env['CC'] or 'gcc' in env['CC']: env2 = env.Clone() - env2.Append(CFLAGS = "-Wno-pedantic") + env2['CFLAGS'].remove('-pedantic') env2.NanopbProto('oneof') dec = env2.Program(['decode_oneof.c', -- cgit 1.2.3-korg From 990b7223ebfae4161ba4bf7ef81094f2317a665b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 15 Nov 2015 09:26:10 +0200 Subject: Second fix for -pedantic build issue in tests --- tests/anonymous_oneof/SConscript | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/anonymous_oneof/SConscript b/tests/anonymous_oneof/SConscript index 051c772f..10672287 100644 --- a/tests/anonymous_oneof/SConscript +++ b/tests/anonymous_oneof/SConscript @@ -16,7 +16,8 @@ if env.GetOption('clean') or (match and (version[0] > 2 or (version[0] == 2 and # Anonymous oneofs are supported by clang and gcc if 'clang' in env['CC'] or 'gcc' in env['CC']: env2 = env.Clone() - env2['CFLAGS'].remove('-pedantic') + if '-pedantic' in env2['CFLAGS']: + env2['CFLAGS'].remove('-pedantic') env2.NanopbProto('oneof') dec = env2.Program(['decode_oneof.c', -- cgit 1.2.3-korg From 7af48e550f084aed929bb7db99070cb8f5d08210 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 20 Nov 2015 23:57:37 +0200 Subject: Fix generator crash with Enum inside Oneof (issue #188). Add testcase for the same. --- generator/nanopb_generator.py | 16 ++++++++++------ tests/regression/issue_188/SConscript | 6 ++++++ tests/regression/issue_188/oneof.proto | 29 +++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 tests/regression/issue_188/SConscript create mode 100644 tests/regression/issue_188/oneof.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 13217462..62ee41bf 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -125,11 +125,15 @@ class EncodedSize: '''Class used to represent the encoded size of a field or a message. Consists of a combination of symbolic sizes and integer sizes.''' def __init__(self, value = 0, symbols = []): - if isinstance(value, strtypes + (Names,)): - symbols = [str(value)] - value = 0 - self.value = value - self.symbols = symbols + if isinstance(value, EncodedSize): + self.value = value.value + self.symbols = value.symbols + elif isinstance(value, strtypes + (Names,)): + self.symbols = [str(value)] + self.value = 0 + else: + self.value = value + self.symbols = symbols def __add__(self, other): if isinstance(other, int): @@ -728,7 +732,7 @@ class OneOf(Field): def encoded_size(self, dependencies): largest = EncodedSize(0) for f in self.fields: - size = f.encoded_size(dependencies) + size = EncodedSize(f.encoded_size(dependencies)) if size is None: return None elif size.symbols: diff --git a/tests/regression/issue_188/SConscript b/tests/regression/issue_188/SConscript new file mode 100644 index 00000000..6bc32712 --- /dev/null +++ b/tests/regression/issue_188/SConscript @@ -0,0 +1,6 @@ +# Regression test for issue with Enums inside OneOf. + +Import('env') + +env.NanopbProto('oneof') + diff --git a/tests/regression/issue_188/oneof.proto b/tests/regression/issue_188/oneof.proto new file mode 100644 index 00000000..e37f5c02 --- /dev/null +++ b/tests/regression/issue_188/oneof.proto @@ -0,0 +1,29 @@ +syntax = "proto2"; + +message MessageOne +{ + required uint32 one = 1; + required uint32 two = 2; + required uint32 three = 3; + required int32 four = 4; +} + +enum EnumTwo +{ + SOME_ENUM_1 = 1; + SOME_ENUM_2 = 5; + SOME_ENUM_3 = 6; + SOME_ENUM_4 = 9; + SOME_ENUM_5 = 10; + SOME_ENUM_6 = 12; + SOME_ENUM_7 = 39; + SOME_ENUM_8 = 401; +} + +message OneofMessage +{ + oneof payload { + MessageOne message = 1; + EnumTwo enum = 2; + } +} -- cgit 1.2.3-korg From 56f7c488df99ae655b47b5838055e48b886665a1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 21 Nov 2015 09:47:29 +0200 Subject: Fix mistake in previous commit --- generator/nanopb_generator.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 62ee41bf..0748e630 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -730,10 +730,11 @@ class OneOf(Field): return largest def encoded_size(self, dependencies): + '''Returns the size of the largest oneof field.''' largest = EncodedSize(0) for f in self.fields: size = EncodedSize(f.encoded_size(dependencies)) - if size is None: + if size.value is None: return None elif size.symbols: return None # Cannot resolve maximum of symbols -- cgit 1.2.3-korg From 3d36157949dd1e5220bcb58b89381f59c767f558 Mon Sep 17 00:00:00 2001 From: Andrew Ruder Date: Wed, 16 Dec 2015 08:13:55 -0600 Subject: pb_istream_from_buffer: add const to prototype This commit changes the prototype for pb_istream_from_buffer from: pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); to pb_istream_t pb_istream_from_buffer(const uint8_t *buf, size_t bufsize); This allows pb_istream_from_buffer users to point to const buffers without having to inspect code (to ensure practical const-ness) and then be forced to manually cast away const. In order to not break compatibility with existing programs (by introducing a const/non-const union in the pb_istream_t state) we simply cast away the const in pb_istream_from_buffer and re-apply it when possible in the callbacks. Unfortunately we lose any compiler help in the callbacks to ensure we are treating the buffer as const but manual inspection is easy enough. --- pb_decode.c | 18 +++++++++++++----- pb_decode.h | 2 +- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 50ada86a..0bf8befd 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -74,8 +74,8 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count) { - uint8_t *source = (uint8_t*)stream->state; - stream->state = source + count; + const uint8_t *source = (const uint8_t*)stream->state; + stream->state = (uint8_t*)stream->state + count; if (buf != NULL) { @@ -131,7 +131,7 @@ static bool checkreturn pb_readbyte(pb_istream_t *stream, uint8_t *buf) if (!stream->callback(stream, buf, 1)) PB_RETURN_ERROR(stream, "io error"); #else - *buf = *(uint8_t*)stream->state; + *buf = *(const uint8_t*)stream->state; stream->state = (uint8_t*)stream->state + 1; #endif @@ -140,15 +140,23 @@ static bool checkreturn pb_readbyte(pb_istream_t *stream, uint8_t *buf) return true; } -pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize) +pb_istream_t pb_istream_from_buffer(const uint8_t *buf, size_t bufsize) { pb_istream_t stream; + /* Cast away the const from buf without a compiler error. We are + * careful to use it only in a const manner in the callbacks. + */ + union { + void *state; + const void *c_state; + } state; #ifdef PB_BUFFER_ONLY stream.callback = NULL; #else stream.callback = &buf_read; #endif - stream.state = buf; + state.c_state = buf; + stream.state = state.state; stream.bytes_left = bufsize; #ifndef PB_NO_ERRMSG stream.errmsg = NULL; diff --git a/pb_decode.h b/pb_decode.h index 3d433155..16de3e04 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -103,7 +103,7 @@ void pb_release(const pb_field_t fields[], void *dest_struct); * Alternatively, you can use a custom stream that reads directly from e.g. * a file or a network socket. */ -pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); +pb_istream_t pb_istream_from_buffer(const uint8_t *buf, size_t bufsize); /* Function to read from a pb_istream_t. You can use this if you need to * read some custom header data, or to read data in field callbacks. -- cgit 1.2.3-korg From d79b15d8aa377116735c87e4d694fc2d20a5fef5 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 26 Jan 2016 22:10:09 +0200 Subject: Get rid of type punning in pb_encode_fixedXX(). This was never very clean code, but it was fast. Hopefully compilers are smart enough to optimize it away, or the speed difference is not very large. This should be checked. However working code is always more important than fast code, and the previous way couldn't really work for platforms that do not have byte-sized memory access. Related to PR #191. --- pb_decode.c | 47 +++++++++++++++++++---------------------------- pb_encode.c | 44 ++++++++++++++++++-------------------------- 2 files changed, 37 insertions(+), 54 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 0bf8befd..e7689806 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -1106,44 +1106,35 @@ bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest) bool pb_decode_fixed32(pb_istream_t *stream, void *dest) { - #ifdef __BIG_ENDIAN__ - uint8_t *bytes = (uint8_t*)dest; - uint8_t lebytes[4]; - - if (!pb_read(stream, lebytes, 4)) + uint8_t bytes[4]; + + if (!pb_read(stream, bytes, 4)) return false; - bytes[0] = lebytes[3]; - bytes[1] = lebytes[2]; - bytes[2] = lebytes[1]; - bytes[3] = lebytes[0]; + *(uint32_t*)dest = ((uint32_t)bytes[0] << 0) | + ((uint32_t)bytes[1] << 8) | + ((uint32_t)bytes[2] << 16) | + ((uint32_t)bytes[3] << 24); return true; - #else - return pb_read(stream, (uint8_t*)dest, 4); - #endif } bool pb_decode_fixed64(pb_istream_t *stream, void *dest) { - #ifdef __BIG_ENDIAN__ - uint8_t *bytes = (uint8_t*)dest; - uint8_t lebytes[8]; - - if (!pb_read(stream, lebytes, 8)) + uint8_t bytes[8]; + + if (!pb_read(stream, bytes, 8)) return false; - bytes[0] = lebytes[7]; - bytes[1] = lebytes[6]; - bytes[2] = lebytes[5]; - bytes[3] = lebytes[4]; - bytes[4] = lebytes[3]; - bytes[5] = lebytes[2]; - bytes[6] = lebytes[1]; - bytes[7] = lebytes[0]; + *(uint64_t*)dest = ((uint64_t)bytes[0] << 0) | + ((uint64_t)bytes[1] << 8) | + ((uint64_t)bytes[2] << 16) | + ((uint64_t)bytes[3] << 24) | + ((uint64_t)bytes[4] << 32) | + ((uint64_t)bytes[5] << 40) | + ((uint64_t)bytes[6] << 48) | + ((uint64_t)bytes[7] << 56); + return true; - #else - return pb_read(stream, (uint8_t*)dest, 8); - #endif } static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) diff --git a/pb_encode.c b/pb_encode.c index cc372b8f..14e657e9 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -443,36 +443,28 @@ bool checkreturn pb_encode_svarint(pb_ostream_t *stream, int64_t value) bool checkreturn pb_encode_fixed32(pb_ostream_t *stream, const void *value) { - #ifdef __BIG_ENDIAN__ - const uint8_t *bytes = value; - uint8_t lebytes[4]; - lebytes[0] = bytes[3]; - lebytes[1] = bytes[2]; - lebytes[2] = bytes[1]; - lebytes[3] = bytes[0]; - return pb_write(stream, lebytes, 4); - #else - return pb_write(stream, (const uint8_t*)value, 4); - #endif + uint32_t val = *(const uint32_t*)value; + uint8_t bytes[4]; + bytes[0] = (uint8_t)(val & 0xFF); + bytes[1] = (uint8_t)((val >> 8) & 0xFF); + bytes[2] = (uint8_t)((val >> 16) & 0xFF); + bytes[3] = (uint8_t)((val >> 24) & 0xFF); + return pb_write(stream, bytes, 4); } bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value) { - #ifdef __BIG_ENDIAN__ - const uint8_t *bytes = value; - uint8_t lebytes[8]; - lebytes[0] = bytes[7]; - lebytes[1] = bytes[6]; - lebytes[2] = bytes[5]; - lebytes[3] = bytes[4]; - lebytes[4] = bytes[3]; - lebytes[5] = bytes[2]; - lebytes[6] = bytes[1]; - lebytes[7] = bytes[0]; - return pb_write(stream, lebytes, 8); - #else - return pb_write(stream, (const uint8_t*)value, 8); - #endif + uint64_t val = *(const uint64_t*)value; + uint8_t bytes[8]; + bytes[0] = (uint8_t)(val & 0xFF); + bytes[1] = (uint8_t)((val >> 8) & 0xFF); + bytes[2] = (uint8_t)((val >> 16) & 0xFF); + bytes[3] = (uint8_t)((val >> 24) & 0xFF); + bytes[4] = (uint8_t)((val >> 32) & 0xFF); + bytes[5] = (uint8_t)((val >> 40) & 0xFF); + bytes[6] = (uint8_t)((val >> 48) & 0xFF); + bytes[7] = (uint8_t)((val >> 56) & 0xFF); + return pb_write(stream, bytes, 8); } bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, uint32_t field_number) -- cgit 1.2.3-korg From abdb59410091e295e99dac4b9d581d77273f9d71 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 27 Jan 2016 18:24:33 +0200 Subject: Modify the int size STATIC_ASSERTS to work with CHAR_BITS!=8. This will still catch the most common bug of long int vs. long long int. The uint8_t checks do not seem necessary, test for this will be added in later commit. --- pb.h | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/pb.h b/pb.h index 8b9338cf..b72e3196 100644 --- a/pb.h +++ b/pb.h @@ -240,19 +240,13 @@ struct pb_field_s { PB_PACKED_STRUCT_END /* Make sure that the standard integer types are of the expected sizes. - * All kinds of things may break otherwise.. atleast all fixed* types. + * Otherwise fixed32/fixed64 fields can break. * * If you get errors here, it probably means that your stdint.h is not * correct for your platform. */ -PB_STATIC_ASSERT(sizeof(int8_t) == 1, INT8_T_WRONG_SIZE) -PB_STATIC_ASSERT(sizeof(uint8_t) == 1, UINT8_T_WRONG_SIZE) -PB_STATIC_ASSERT(sizeof(int16_t) == 2, INT16_T_WRONG_SIZE) -PB_STATIC_ASSERT(sizeof(uint16_t) == 2, UINT16_T_WRONG_SIZE) -PB_STATIC_ASSERT(sizeof(int32_t) == 4, INT32_T_WRONG_SIZE) -PB_STATIC_ASSERT(sizeof(uint32_t) == 4, UINT32_T_WRONG_SIZE) -PB_STATIC_ASSERT(sizeof(int64_t) == 8, INT64_T_WRONG_SIZE) -PB_STATIC_ASSERT(sizeof(uint64_t) == 8, UINT64_T_WRONG_SIZE) +PB_STATIC_ASSERT(sizeof(int64_t) == 2 * sizeof(int32_t), INT64_T_WRONG_SIZE) +PB_STATIC_ASSERT(sizeof(uint64_t) == 2 * sizeof(uint32_t), UINT64_T_WRONG_SIZE) /* This structure is used for 'bytes' arrays. * It has the number of bytes in the beginning, and after that an array. -- cgit 1.2.3-korg From fa45589d5736d2ce7f37ddd03daad2c638e38848 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 27 Jan 2016 18:53:26 +0200 Subject: Replace uint8_t with a pb_byte_t typedef. This supports platforms where uint8_t does not exist. If you are using a custom pb_syshdr.h, this may require adding definitions for uint_least8_t etc. --- extra/pb_syshdr.h | 8 +++++ pb.h | 26 +++++++++------- pb_decode.c | 90 +++++++++++++++++++++++++++++-------------------------- pb_decode.h | 6 ++-- pb_encode.c | 65 +++++++++++++++++++++------------------- pb_encode.h | 8 ++--- 6 files changed, 111 insertions(+), 92 deletions(-) diff --git a/extra/pb_syshdr.h b/extra/pb_syshdr.h index 1ff48230..55d06a3a 100644 --- a/extra/pb_syshdr.h +++ b/extra/pb_syshdr.h @@ -24,6 +24,14 @@ typedef signed int int32_t; typedef unsigned int uint32_t; typedef signed long long int64_t; typedef unsigned long long uint64_t; + +/* These are ok for most platforms, unless uint8_t is actually not available, + * in which case you should give the smallest available type. */ +typedef int8_t int_least8_t; +typedef uint8_t uint_least8_t; +typedef uint8_t uint_fast8_t; +typedef int16_t int_least16_t; +typedef uint16_t uint_least16_t; #endif /* stddef.h subset */ diff --git a/pb.h b/pb.h index b72e3196..92c5a702 100644 --- a/pb.h +++ b/pb.h @@ -55,7 +55,7 @@ /* Include all the system headers needed by nanopb. You will need the * definitions of the following: * - strlen, memcpy, memset functions - * - [u]int8_t, [u]int16_t, [u]int32_t, [u]int64_t + * - [u]int_least8_t, uint_fast8_t, [u]int_least16_t, [u]int32_t, [u]int64_t * - size_t * - bool * @@ -144,7 +144,7 @@ * Most-significant 4 bits specify repeated/required/packed etc. */ -typedef uint8_t pb_type_t; +typedef uint_least8_t pb_type_t; /**** Field data types ****/ @@ -201,18 +201,22 @@ typedef uint8_t pb_type_t; * and array counts. */ #if defined(PB_FIELD_32BIT) -#define PB_SIZE_MAX ((uint32_t)-1) typedef uint32_t pb_size_t; typedef int32_t pb_ssize_t; #elif defined(PB_FIELD_16BIT) -#define PB_SIZE_MAX ((uint16_t)-1) - typedef uint16_t pb_size_t; - typedef int16_t pb_ssize_t; + typedef uint_least16_t pb_size_t; + typedef int_least16_t pb_ssize_t; #else -#define PB_SIZE_MAX ((uint8_t)-1) - typedef uint8_t pb_size_t; - typedef int8_t pb_ssize_t; + typedef uint_least8_t pb_size_t; + typedef int_least8_t pb_ssize_t; #endif +#define PB_SIZE_MAX ((pb_size_t)-1) + +/* Data type for storing encoded data and other byte streams. + * This typedef exists to support platforms where uint8_t does not exist. + * You can regard it as equivalent on uint8_t on other platforms. + */ +typedef uint_least8_t pb_byte_t; /* This structure is used in auto-generated constants * to specify struct fields. @@ -252,12 +256,12 @@ PB_STATIC_ASSERT(sizeof(uint64_t) == 2 * sizeof(uint32_t), UINT64_T_WRONG_SIZE) * It has the number of bytes in the beginning, and after that an array. * Note that actual structs used will have a different length of bytes array. */ -#define PB_BYTES_ARRAY_T(n) struct { pb_size_t size; uint8_t bytes[n]; } +#define PB_BYTES_ARRAY_T(n) struct { pb_size_t size; pb_byte_t bytes[n]; } #define PB_BYTES_ARRAY_T_ALLOCSIZE(n) ((size_t)n + offsetof(pb_bytes_array_t, bytes)) struct pb_bytes_array_s { pb_size_t size; - uint8_t bytes[1]; + pb_byte_t bytes[1]; }; typedef struct pb_bytes_array_s pb_bytes_array_t; diff --git a/pb_decode.c b/pb_decode.c index e7689806..b7a0093e 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -23,9 +23,9 @@ typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn; -static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count); +static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count); static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest); -static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, uint8_t *buf, size_t *size); +static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size); static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); @@ -72,10 +72,10 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { * pb_istream_t implementation * *******************************/ -static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count) +static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count) { - const uint8_t *source = (const uint8_t*)stream->state; - stream->state = (uint8_t*)stream->state + count; + const pb_byte_t *source = (const pb_byte_t*)stream->state; + stream->state = (pb_byte_t*)stream->state + count; if (buf != NULL) { @@ -86,13 +86,13 @@ static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t coun return true; } -bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) +bool checkreturn pb_read(pb_istream_t *stream, pb_byte_t *buf, size_t count) { #ifndef PB_BUFFER_ONLY if (buf == NULL && stream->callback != buf_read) { /* Skip input bytes */ - uint8_t tmp[16]; + pb_byte_t tmp[16]; while (count > 16) { if (!pb_read(stream, tmp, 16)) @@ -122,7 +122,7 @@ bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) /* Read a single byte from input stream. buf may not be NULL. * This is an optimization for the varint decoding. */ -static bool checkreturn pb_readbyte(pb_istream_t *stream, uint8_t *buf) +static bool checkreturn pb_readbyte(pb_istream_t *stream, pb_byte_t *buf) { if (stream->bytes_left == 0) PB_RETURN_ERROR(stream, "end-of-stream"); @@ -131,8 +131,8 @@ static bool checkreturn pb_readbyte(pb_istream_t *stream, uint8_t *buf) if (!stream->callback(stream, buf, 1)) PB_RETURN_ERROR(stream, "io error"); #else - *buf = *(const uint8_t*)stream->state; - stream->state = (uint8_t*)stream->state + 1; + *buf = *(const pb_byte_t*)stream->state; + stream->state = (pb_byte_t*)stream->state + 1; #endif stream->bytes_left--; @@ -140,7 +140,7 @@ static bool checkreturn pb_readbyte(pb_istream_t *stream, uint8_t *buf) return true; } -pb_istream_t pb_istream_from_buffer(const uint8_t *buf, size_t bufsize) +pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t bufsize) { pb_istream_t stream; /* Cast away the const from buf without a compiler error. We are @@ -170,7 +170,7 @@ pb_istream_t pb_istream_from_buffer(const uint8_t *buf, size_t bufsize) static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) { - uint8_t byte; + pb_byte_t byte; uint32_t result; if (!pb_readbyte(stream, &byte)) @@ -184,7 +184,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) else { /* Multibyte case */ - uint8_t bitpos = 7; + uint_fast8_t bitpos = 7; result = byte & 0x7F; do @@ -196,7 +196,7 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) return false; result |= (uint32_t)(byte & 0x7F) << bitpos; - bitpos = (uint8_t)(bitpos + 7); + bitpos = (uint_fast8_t)(bitpos + 7); } while (byte & 0x80); } @@ -206,8 +206,8 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) { - uint8_t byte; - uint8_t bitpos = 0; + pb_byte_t byte; + uint_fast8_t bitpos = 0; uint64_t result = 0; do @@ -219,7 +219,7 @@ bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) return false; result |= (uint64_t)(byte & 0x7F) << bitpos; - bitpos = (uint8_t)(bitpos + 7); + bitpos = (uint_fast8_t)(bitpos + 7); } while (byte & 0x80); *dest = result; @@ -228,7 +228,7 @@ bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) bool checkreturn pb_skip_varint(pb_istream_t *stream) { - uint8_t byte; + pb_byte_t byte; do { if (!pb_read(stream, &byte, 1)) @@ -287,7 +287,7 @@ bool checkreturn pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type) /* Read a raw value to buffer, for the purpose of passing it to callback as * a substream. Size is maximum size on call, and actual size on return. */ -static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, uint8_t *buf, size_t *size) +static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size) { size_t max_size = *size; switch (wire_type) @@ -375,7 +375,7 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t while (substream.bytes_left > 0 && *size < iter->pos->array_size) { - void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); + void *pItem = (char*)iter->pData + iter->pos->data_size * (*size); if (!func(&substream, iter->pos, pItem)) { status = false; @@ -394,7 +394,7 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t { /* Repeated field */ pb_size_t *size = (pb_size_t*)iter->pSize; - void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); + void *pItem = (char*)iter->pData + iter->pos->data_size * (*size); if (*size >= iter->pos->array_size) PB_RETURN_ERROR(stream, "array overflow"); @@ -548,7 +548,7 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ } /* Decode the array entry */ - pItem = *(uint8_t**)iter->pData + iter->pos->data_size * (*size); + pItem = *(char**)iter->pData + iter->pos->data_size * (*size); initialize_pointer_field(pItem, iter); if (!func(&substream, iter->pos, pItem)) { @@ -584,7 +584,7 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ if (!allocate_field(stream, iter->pData, iter->pos->data_size, *size)) return false; - pItem = *(uint8_t**)iter->pData + iter->pos->data_size * (*size - 1); + pItem = *(char**)iter->pData + iter->pos->data_size * (*size - 1); initialize_pointer_field(pItem, iter); return func(stream, iter->pos, pItem); } @@ -631,7 +631,7 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type * which in turn allows to use same callback for packed and * not-packed fields. */ pb_istream_t substream; - uint8_t buffer[10]; + pb_byte_t buffer[10]; size_t size = sizeof(buffer); if (!read_raw_value(stream, wire_type, buffer, &size)) @@ -838,7 +838,8 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { - uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {0, 0, 0, 0, 0, 0, 0, 0}; + uint32_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 31) / 32] = {0, 0}; + const uint32_t allbits = ~(uint32_t)0; uint32_t extension_range_start = 0; pb_field_iter_t iter; @@ -894,8 +895,8 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ if (PB_HTYPE(iter.pos->type) == PB_HTYPE_REQUIRED && iter.required_field_index < PB_MAX_REQUIRED_FIELDS) { - uint8_t tmp = (uint8_t)(1 << (iter.required_field_index & 7)); - fields_seen[iter.required_field_index >> 3] |= tmp; + uint32_t tmp = ((uint32_t)1 << (iter.required_field_index & 31)); + fields_seen[iter.required_field_index >> 5] |= tmp; } if (!decode_field(stream, wire_type, &iter)) @@ -920,16 +921,19 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.pos->tag != 0) req_field_count++; - /* Check the whole bytes */ - for (i = 0; i < (req_field_count >> 3); i++) + if (req_field_count > 0) { - if (fields_seen[i] != 0xFF) + /* Check the whole words */ + for (i = 0; i < (req_field_count >> 5); i++) + { + if (fields_seen[i] != allbits) + PB_RETURN_ERROR(stream, "missing required field"); + } + + /* Check the remaining bits */ + if (fields_seen[req_field_count >> 5] != (allbits >> (32 - (req_field_count & 31)))) PB_RETURN_ERROR(stream, "missing required field"); } - - /* Check the remaining bits */ - if (fields_seen[req_field_count >> 3] != (0xFF >> (8 - (req_field_count & 7)))) - PB_RETURN_ERROR(stream, "missing required field"); } return true; @@ -1038,7 +1042,7 @@ static void pb_release_single_field(const pb_field_iter_t *iter) while (count--) { pb_release((const pb_field_t*)iter->pos->ptr, pItem); - pItem = (uint8_t*)pItem + iter->pos->data_size; + pItem = (char*)pItem + iter->pos->data_size; } } } @@ -1106,7 +1110,7 @@ bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest) bool pb_decode_fixed32(pb_istream_t *stream, void *dest) { - uint8_t bytes[4]; + pb_byte_t bytes[4]; if (!pb_read(stream, bytes, 4)) return false; @@ -1120,7 +1124,7 @@ bool pb_decode_fixed32(pb_istream_t *stream, void *dest) bool pb_decode_fixed64(pb_istream_t *stream, void *dest) { - uint8_t bytes[8]; + pb_byte_t bytes[8]; if (!pb_read(stream, bytes, 8)) return false; @@ -1179,8 +1183,8 @@ static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *f switch (field->data_size) { - case 1: clamped = *(uint8_t*)dest = (uint8_t)value; break; - case 2: clamped = *(uint16_t*)dest = (uint16_t)value; break; + case 1: clamped = *(uint_least8_t*)dest = (uint_least8_t)value; break; + case 2: clamped = *(uint_least16_t*)dest = (uint_least16_t)value; break; case 4: clamped = *(uint32_t*)dest = (uint32_t)value; break; case 8: clamped = *(uint64_t*)dest = value; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); @@ -1200,8 +1204,8 @@ static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *f switch (field->data_size) { - case 1: clamped = *(int8_t*)dest = (int8_t)value; break; - case 2: clamped = *(int16_t*)dest = (int16_t)value; break; + case 1: clamped = *(int_least8_t*)dest = (int_least8_t)value; break; + case 2: clamped = *(int_least16_t*)dest = (int_least16_t)value; break; case 4: clamped = *(int32_t*)dest = (int32_t)value; break; case 8: clamped = *(int64_t*)dest = value; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); @@ -1292,8 +1296,8 @@ static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *fi PB_RETURN_ERROR(stream, "string overflow"); } - status = pb_read(stream, (uint8_t*)dest, size); - *((uint8_t*)dest + size) = 0; + status = pb_read(stream, (pb_byte_t*)dest, size); + *((pb_byte_t*)dest + size) = 0; return status; } diff --git a/pb_decode.h b/pb_decode.h index 16de3e04..1d9bb194 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -34,7 +34,7 @@ struct pb_istream_s */ int *callback; #else - bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count); + bool (*callback)(pb_istream_t *stream, pb_byte_t *buf, size_t count); #endif void *state; /* Free field for use by callback implementation */ @@ -103,12 +103,12 @@ void pb_release(const pb_field_t fields[], void *dest_struct); * Alternatively, you can use a custom stream that reads directly from e.g. * a file or a network socket. */ -pb_istream_t pb_istream_from_buffer(const uint8_t *buf, size_t bufsize); +pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t bufsize); /* Function to read from a pb_istream_t. You can use this if you need to * read some custom header data, or to read data in field callbacks. */ -bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); +bool pb_read(pb_istream_t *stream, pb_byte_t *buf, size_t count); /************************************************ diff --git a/pb_encode.c b/pb_encode.c index 14e657e9..95683751 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -22,7 +22,7 @@ **************************************/ typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, const void *src) checkreturn; -static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); +static bool checkreturn buf_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, const void *pData, size_t count, pb_encoder_t func); static bool checkreturn encode_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData); static bool checkreturn default_extension_encoder(pb_ostream_t *stream, const pb_extension_t *extension); @@ -56,9 +56,9 @@ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { * pb_ostream_t implementation * *******************************/ -static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) +static bool checkreturn buf_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count) { - uint8_t *dest = (uint8_t*)stream->state; + pb_byte_t *dest = (pb_byte_t*)stream->state; stream->state = dest + count; while (count--) @@ -67,7 +67,7 @@ static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size return true; } -pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize) +pb_ostream_t pb_ostream_from_buffer(pb_byte_t *buf, size_t bufsize) { pb_ostream_t stream; #ifdef PB_BUFFER_ONLY @@ -84,7 +84,7 @@ pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize) return stream; } -bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) +bool checkreturn pb_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count) { if (stream->callback != NULL) { @@ -413,15 +413,18 @@ bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], const void *sr ********************/ bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value) { - uint8_t buffer[10]; + pb_byte_t buffer[10]; size_t i = 0; - if (value == 0) - return pb_write(stream, (uint8_t*)&value, 1); + if (value <= 0x7F) + { + pb_byte_t v = (pb_byte_t)value; + return pb_write(stream, &v, 1); + } while (value) { - buffer[i] = (uint8_t)((value & 0x7F) | 0x80); + buffer[i] = (pb_byte_t)((value & 0x7F) | 0x80); value >>= 7; i++; } @@ -444,26 +447,26 @@ bool checkreturn pb_encode_svarint(pb_ostream_t *stream, int64_t value) bool checkreturn pb_encode_fixed32(pb_ostream_t *stream, const void *value) { uint32_t val = *(const uint32_t*)value; - uint8_t bytes[4]; - bytes[0] = (uint8_t)(val & 0xFF); - bytes[1] = (uint8_t)((val >> 8) & 0xFF); - bytes[2] = (uint8_t)((val >> 16) & 0xFF); - bytes[3] = (uint8_t)((val >> 24) & 0xFF); + pb_byte_t bytes[4]; + bytes[0] = (pb_byte_t)(val & 0xFF); + bytes[1] = (pb_byte_t)((val >> 8) & 0xFF); + bytes[2] = (pb_byte_t)((val >> 16) & 0xFF); + bytes[3] = (pb_byte_t)((val >> 24) & 0xFF); return pb_write(stream, bytes, 4); } bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value) { uint64_t val = *(const uint64_t*)value; - uint8_t bytes[8]; - bytes[0] = (uint8_t)(val & 0xFF); - bytes[1] = (uint8_t)((val >> 8) & 0xFF); - bytes[2] = (uint8_t)((val >> 16) & 0xFF); - bytes[3] = (uint8_t)((val >> 24) & 0xFF); - bytes[4] = (uint8_t)((val >> 32) & 0xFF); - bytes[5] = (uint8_t)((val >> 40) & 0xFF); - bytes[6] = (uint8_t)((val >> 48) & 0xFF); - bytes[7] = (uint8_t)((val >> 56) & 0xFF); + pb_byte_t bytes[8]; + bytes[0] = (pb_byte_t)(val & 0xFF); + bytes[1] = (pb_byte_t)((val >> 8) & 0xFF); + bytes[2] = (pb_byte_t)((val >> 16) & 0xFF); + bytes[3] = (pb_byte_t)((val >> 24) & 0xFF); + bytes[4] = (pb_byte_t)((val >> 32) & 0xFF); + bytes[5] = (pb_byte_t)((val >> 40) & 0xFF); + bytes[6] = (pb_byte_t)((val >> 48) & 0xFF); + bytes[7] = (pb_byte_t)((val >> 56) & 0xFF); return pb_write(stream, bytes, 8); } @@ -505,7 +508,7 @@ bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t return pb_encode_tag(stream, wiretype, field->tag); } -bool checkreturn pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size) +bool checkreturn pb_encode_string(pb_ostream_t *stream, const pb_byte_t *buffer, size_t size) { if (!pb_encode_varint(stream, (uint64_t)size)) return false; @@ -573,8 +576,8 @@ static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *fi * or enums, and for int_size option. */ switch (field->data_size) { - case 1: value = *(const int8_t*)src; break; - case 2: value = *(const int16_t*)src; break; + case 1: value = *(const int_least8_t*)src; break; + case 2: value = *(const int_least16_t*)src; break; case 4: value = *(const int32_t*)src; break; case 8: value = *(const int64_t*)src; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); @@ -589,8 +592,8 @@ static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *f switch (field->data_size) { - case 1: value = *(const uint8_t*)src; break; - case 2: value = *(const uint16_t*)src; break; + case 1: value = *(const uint_least8_t*)src; break; + case 2: value = *(const uint_least16_t*)src; break; case 4: value = *(const uint32_t*)src; break; case 8: value = *(const uint64_t*)src; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); @@ -605,8 +608,8 @@ static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *f switch (field->data_size) { - case 1: value = *(const int8_t*)src; break; - case 2: value = *(const int16_t*)src; break; + case 1: value = *(const int_least8_t*)src; break; + case 2: value = *(const int_least16_t*)src; break; case 4: value = *(const int32_t*)src; break; case 8: value = *(const int64_t*)src; break; default: PB_RETURN_ERROR(stream, "invalid data_size"); @@ -669,7 +672,7 @@ static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *fi } } - return pb_encode_string(stream, (const uint8_t*)src, size); + return pb_encode_string(stream, (const pb_byte_t*)src, size); } static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src) diff --git a/pb_encode.h b/pb_encode.h index e992c8dc..d9909fb0 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -35,7 +35,7 @@ struct pb_ostream_s */ int *callback; #else - bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); + bool (*callback)(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); #endif void *state; /* Free field for use by callback implementation. */ size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ @@ -86,7 +86,7 @@ bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], const void *sr * Alternatively, you can use a custom stream that writes directly to e.g. * a file or a network socket. */ -pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize); +pb_ostream_t pb_ostream_from_buffer(pb_byte_t *buf, size_t bufsize); /* Pseudo-stream for measuring the size of a message without actually storing * the encoded data. @@ -106,7 +106,7 @@ pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize); /* Function to write into a pb_ostream_t stream. You can use this if you need * to append or prepend some custom headers to the message. */ -bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); +bool pb_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); /************************************************ @@ -130,7 +130,7 @@ bool pb_encode_varint(pb_ostream_t *stream, uint64_t value); bool pb_encode_svarint(pb_ostream_t *stream, int64_t value); /* Encode a string or bytes type field. For strings, pass strlen(s) as size. */ -bool pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size); +bool pb_encode_string(pb_ostream_t *stream, const pb_byte_t *buffer, size_t size); /* Encode a fixed32, sfixed32 or float value. * You need to pass a pointer to a 4-byte wide C variable. */ -- cgit 1.2.3-korg From 2a851f039ad5252a32e3b82d15ce5a3bbb8a0b21 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 27 Jan 2016 18:59:02 +0200 Subject: Add note to migration doc --- docs/migration.rst | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/migration.rst b/docs/migration.rst index ac92db88..26bd25dd 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -11,6 +11,24 @@ are included, in order to make it easier to find this document. .. contents :: +Nanopb-0.3.5 (2016-xx-xx) +========================= + +Add support for platforms without uint8_t +----------------------------------------- +**Rationale:** Some platforms cannot access 8-bit sized values directly, and +do not define *uint8_t*. Nanopb previously didn't support these platforms. + +**Changes:** References to *uint8_t* were replaced with several alternatives, +one of them being a new *pb_byte_t* typedef. This in turn uses *uint_least8_t* +which means the smallest available type. + +**Required actions:** If your platform does not have a standards-compliant +*stdint.h*, it may lack the definition for *[u]int_least8_t*. This must be +added manually, example can be found in *extra/pb_syshdr.h*. + +**Error indications:** Compiler error: "unknown type name 'uint_least8_t'". + Nanopb-0.3.2 (2015-01-24) ========================= -- cgit 1.2.3-korg From 8d35488bcde1cc6d912d4c875bcf6d2745899d7d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 27 Jan 2016 18:59:54 +0200 Subject: Remove obsolete __BIG_ENDIAN__ compilation option. This is now handled automatically using shift operations. --- docs/reference.rst | 4 ---- pb.h | 4 ---- 2 files changed, 8 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 296bc78c..be6567ee 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -20,10 +20,6 @@ You must have the same settings for the nanopb library and all code that includes pb.h. ============================ ================================================ -__BIG_ENDIAN__ Set this if your platform stores integers and - floats in big-endian format. Mixed-endian - systems (different layout for ints and floats) - are currently not supported. PB_NO_PACKED_STRUCTS Disable packed structs. Increases RAM usage but is necessary on some platforms that do not support unaligned memory access. diff --git a/pb.h b/pb.h index 92c5a702..50f71e81 100644 --- a/pb.h +++ b/pb.h @@ -13,10 +13,6 @@ /* Enable support for dynamically allocated fields */ /* #define PB_ENABLE_MALLOC 1 */ -/* Define this if your CPU architecture is big endian, i.e. it - * stores the most-significant byte first. */ -/* #define __BIG_ENDIAN__ 1 */ - /* Define this if your CPU / compiler combination does not support * unaligned memory access to packed structures. */ /* #define PB_NO_PACKED_STRUCTS 1 */ -- cgit 1.2.3-korg From 3b6099faa2d829c74c9576f4aa1ce4b237c997db Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 27 Jan 2016 18:59:54 +0200 Subject: Fix a few remaining bugs related to CHAR_BIT!=8 platforms. --- generator/nanopb_generator.py | 2 +- pb_decode.c | 59 +++++++++++++++++++++++++------------------ pb_encode.c | 56 +++++++++++++++++++++------------------- 3 files changed, 65 insertions(+), 52 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 0748e630..d16fd828 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -809,7 +809,7 @@ class Message: if not self.ordered_fields: # Empty structs are not allowed in C standard. # Therefore add a dummy field if an empty message occurs. - result += ' uint8_t dummy_field;' + result += ' char dummy_field;' result += '\n'.join([str(f) for f in self.ordered_fields]) result += '\n}' diff --git a/pb_decode.c b/pb_decode.c index b7a0093e..1699091c 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -1155,19 +1155,22 @@ static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *fi * not break decoding of such messages, we cast <=32 bit fields to * int32_t first to get the sign correct. */ - if (field->data_size == 8) + if (field->data_size == sizeof(int64_t)) svalue = (int64_t)value; else svalue = (int32_t)value; - switch (field->data_size) - { - case 1: clamped = *(int8_t*)dest = (int8_t)svalue; break; - case 2: clamped = *(int16_t*)dest = (int16_t)svalue; break; - case 4: clamped = *(int32_t*)dest = (int32_t)svalue; break; - case 8: clamped = *(int64_t*)dest = svalue; break; - default: PB_RETURN_ERROR(stream, "invalid data_size"); - } + /* Cast to the proper field size, while checking for overflows */ + if (field->data_size == sizeof(int64_t)) + clamped = *(int64_t*)dest = svalue; + else if (field->data_size == sizeof(int32_t)) + clamped = *(int32_t*)dest = (int32_t)svalue; + else if (field->data_size == sizeof(int_least16_t)) + clamped = *(int_least16_t*)dest = (int_least16_t)svalue; + else if (field->data_size == sizeof(int_least8_t)) + clamped = *(int_least8_t*)dest = (int_least8_t)svalue; + else + PB_RETURN_ERROR(stream, "invalid data_size"); if (clamped != svalue) PB_RETURN_ERROR(stream, "integer too large"); @@ -1181,14 +1184,17 @@ static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *f if (!pb_decode_varint(stream, &value)) return false; - switch (field->data_size) - { - case 1: clamped = *(uint_least8_t*)dest = (uint_least8_t)value; break; - case 2: clamped = *(uint_least16_t*)dest = (uint_least16_t)value; break; - case 4: clamped = *(uint32_t*)dest = (uint32_t)value; break; - case 8: clamped = *(uint64_t*)dest = value; break; - default: PB_RETURN_ERROR(stream, "invalid data_size"); - } + /* Cast to the proper field size, while checking for overflows */ + if (field->data_size == sizeof(uint64_t)) + clamped = *(uint64_t*)dest = value; + else if (field->data_size == sizeof(uint32_t)) + clamped = *(uint32_t*)dest = (uint32_t)value; + else if (field->data_size == sizeof(uint_least16_t)) + clamped = *(uint_least16_t*)dest = (uint_least16_t)value; + else if (field->data_size == sizeof(uint_least8_t)) + clamped = *(uint_least8_t*)dest = (uint_least8_t)value; + else + PB_RETURN_ERROR(stream, "invalid data_size"); if (clamped != value) PB_RETURN_ERROR(stream, "integer too large"); @@ -1202,14 +1208,17 @@ static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *f if (!pb_decode_svarint(stream, &value)) return false; - switch (field->data_size) - { - case 1: clamped = *(int_least8_t*)dest = (int_least8_t)value; break; - case 2: clamped = *(int_least16_t*)dest = (int_least16_t)value; break; - case 4: clamped = *(int32_t*)dest = (int32_t)value; break; - case 8: clamped = *(int64_t*)dest = value; break; - default: PB_RETURN_ERROR(stream, "invalid data_size"); - } + /* Cast to the proper field size, while checking for overflows */ + if (field->data_size == sizeof(int64_t)) + clamped = *(int64_t*)dest = value; + else if (field->data_size == sizeof(int32_t)) + clamped = *(int32_t*)dest = (int32_t)value; + else if (field->data_size == sizeof(int_least16_t)) + clamped = *(int_least16_t*)dest = (int_least16_t)value; + else if (field->data_size == sizeof(int_least8_t)) + clamped = *(int_least8_t*)dest = (int_least8_t)value; + else + PB_RETURN_ERROR(stream, "invalid data_size"); if (clamped != value) PB_RETURN_ERROR(stream, "integer too large"); diff --git a/pb_encode.c b/pb_encode.c index 95683751..9f91c9d5 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -572,16 +572,16 @@ static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *fi { int64_t value = 0; - /* Cases 1 and 2 are for compilers that have smaller types for bool - * or enums, and for int_size option. */ - switch (field->data_size) - { - case 1: value = *(const int_least8_t*)src; break; - case 2: value = *(const int_least16_t*)src; break; - case 4: value = *(const int32_t*)src; break; - case 8: value = *(const int64_t*)src; break; - default: PB_RETURN_ERROR(stream, "invalid data_size"); - } + if (field->data_size == sizeof(int_least8_t)) + value = *(const int_least8_t*)src; + else if (field->data_size == sizeof(int_least16_t)) + value = *(const int_least16_t*)src; + else if (field->data_size == sizeof(int32_t)) + value = *(const int32_t*)src; + else if (field->data_size == sizeof(int64_t)) + value = *(const int64_t*)src; + else + PB_RETURN_ERROR(stream, "invalid data_size"); return pb_encode_varint(stream, (uint64_t)value); } @@ -590,14 +590,16 @@ static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *f { uint64_t value = 0; - switch (field->data_size) - { - case 1: value = *(const uint_least8_t*)src; break; - case 2: value = *(const uint_least16_t*)src; break; - case 4: value = *(const uint32_t*)src; break; - case 8: value = *(const uint64_t*)src; break; - default: PB_RETURN_ERROR(stream, "invalid data_size"); - } + if (field->data_size == sizeof(uint_least8_t)) + value = *(const uint_least8_t*)src; + else if (field->data_size == sizeof(uint_least16_t)) + value = *(const uint_least16_t*)src; + else if (field->data_size == sizeof(uint32_t)) + value = *(const uint32_t*)src; + else if (field->data_size == sizeof(uint64_t)) + value = *(const uint64_t*)src; + else + PB_RETURN_ERROR(stream, "invalid data_size"); return pb_encode_varint(stream, value); } @@ -606,14 +608,16 @@ static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *f { int64_t value = 0; - switch (field->data_size) - { - case 1: value = *(const int_least8_t*)src; break; - case 2: value = *(const int_least16_t*)src; break; - case 4: value = *(const int32_t*)src; break; - case 8: value = *(const int64_t*)src; break; - default: PB_RETURN_ERROR(stream, "invalid data_size"); - } + if (field->data_size == sizeof(int_least8_t)) + value = *(const int_least8_t*)src; + else if (field->data_size == sizeof(int_least16_t)) + value = *(const int_least16_t*)src; + else if (field->data_size == sizeof(int32_t)) + value = *(const int32_t*)src; + else if (field->data_size == sizeof(int64_t)) + value = *(const int64_t*)src; + else + PB_RETURN_ERROR(stream, "invalid data_size"); return pb_encode_svarint(stream, value); } -- cgit 1.2.3-korg From 4507b0066c1a75c4f85d8d595a2236d0c84cb312 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 6 Feb 2016 13:57:38 +0200 Subject: Add code generator insertion points to files (#178). --- generator/nanopb_generator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index d16fd828..3c1f4341 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -812,6 +812,7 @@ class Message: result += ' char dummy_field;' result += '\n'.join([str(f) for f in self.ordered_fields]) + result += '\n/* @@protoc_insertion_point(struct:%s) */' % self.name result += '\n}' if self.packed: @@ -1054,6 +1055,8 @@ class ProtoFile: noext = os.path.splitext(incfile)[0] yield options.genformat % (noext + options.extension + '.h') yield '\n' + + yield '/* @@protoc_insertion_point(includes) */\n' yield '#if PB_PROTO_HEADER_VERSION != 30\n' yield '#error Regenerate this file with the current version of nanopb generator.\n' @@ -1149,6 +1152,7 @@ class ProtoFile: yield '#endif\n' # End of header + yield '/* @@protoc_insertion_point(eof) */\n' yield '\n#endif\n' def generate_source(self, headername, options): @@ -1161,6 +1165,7 @@ class ProtoFile: yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) yield options.genformat % (headername) yield '\n' + yield '/* @@protoc_insertion_point(includes) */\n' yield '#if PB_PROTO_HEADER_VERSION != 30\n' yield '#error Regenerate this file with the current version of nanopb generator.\n' @@ -1253,6 +1258,7 @@ class ProtoFile: yield 'PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n' yield '\n' + yield '/* @@protoc_insertion_point(eof) */\n' # --------------------------------------------------------------------------- # Options parsing for the .proto files -- cgit 1.2.3-korg From 65e7760c9c16f7451974ebc7275c4eb572ccf04c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 6 Feb 2016 14:28:33 +0200 Subject: Update changelog --- CHANGELOG.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index b81b847e..80d2b826 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,15 @@ +nanopb-0.3.5 (2016-02-xx) + Fix generator crash with Enum inside Oneof (#188) + Fix some generator regressions related to .options file path (#172) + Add support for platforms without uint8_t (#191) + Allow const parameter to pb_istream_from_buffer (#152) + Ignore null pointers in pb_release() (#183) + Add support for anonymous unions (#184) + Add Python3 support to the generator (#169) + Add code generator insertion points to generated files (#178) + Improvements to CMake script (#181) + Improvements to test cases. + nanopb-0.3.4 (2015-09-26) Fix handling of unsigned 8- and 16-bit enums (issue 164) Fix generator on systems where python = python3. (issue 155) -- cgit 1.2.3-korg From 6f0dfe1b3fd52c2d141fff412b70b64ebcaf5ad8 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 7 Feb 2016 10:50:02 +0200 Subject: Update pb_release docs --- docs/reference.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index be6567ee..6968c81b 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -630,10 +630,10 @@ pb_release ---------- Releases any dynamically allocated fields. - void pb_release(const pb_field_t fields[], void *dest_struct); + void pb_release(const pb_field_t fields[], void \*dest_struct); :fields: A field description array. Usually autogenerated. -:dest_struct: Pointer to structure where data will be stored. +:dest_struct: Pointer to structure where data is stored. If NULL, function does nothing. This function is only available if *PB_ENABLE_MALLOC* is defined. It will release any pointer type fields in the structure and set the pointers to NULL. -- cgit 1.2.3-korg From a57831acfabef58dd00dad557cbb340f49c3f30a Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 13 Feb 2016 12:39:45 +0200 Subject: Publishing nanopb-0.3.5 --- CHANGELOG.txt | 5 ++++- docs/migration.rst | 2 +- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 80d2b826..fd983dfe 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,4 +1,7 @@ -nanopb-0.3.5 (2016-02-xx) +nanopb-0.3.5 (2016-02-13) + NOTE: If you are using pb_syshdr.h, you will need to add uint_least8_t + definition. See docs/migration.rst for details. + Fix generator crash with Enum inside Oneof (#188) Fix some generator regressions related to .options file path (#172) Add support for platforms without uint8_t (#191) diff --git a/docs/migration.rst b/docs/migration.rst index 26bd25dd..cd5911f5 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -11,7 +11,7 @@ are included, in order to make it easier to find this document. .. contents :: -Nanopb-0.3.5 (2016-xx-xx) +Nanopb-0.3.5 (2016-02-13) ========================= Add support for platforms without uint8_t diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 3c1f4341..95d6e0c3 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.5-dev" +nanopb_version = "nanopb-0.3.5" import sys import re diff --git a/pb.h b/pb.h index 50f71e81..d3f24138 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.5-dev +#define NANOPB_VERSION nanopb-0.3.5 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 4110ff293571767b4de0044d88da8f2e704d764e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 13 Feb 2016 12:51:44 +0200 Subject: Setting version to 0.3.6-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 95d6e0c3..bc9e9da8 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.5" +nanopb_version = "nanopb-0.3.6-dev" import sys import re diff --git a/pb.h b/pb.h index d3f24138..114c9c74 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.5 +#define NANOPB_VERSION nanopb-0.3.6-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 0812fa14d9718d8558a91be4fa692cc3a1041878 Mon Sep 17 00:00:00 2001 From: Kenshi Kawaguchi Date: Fri, 19 Feb 2016 14:28:55 -0800 Subject: Add -D option to specify output directory --- generator/nanopb_generator.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index bc9e9da8..60dc7e87 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1055,7 +1055,7 @@ class ProtoFile: noext = os.path.splitext(incfile)[0] yield options.genformat % (noext + options.extension + '.h') yield '\n' - + yield '/* @@protoc_insertion_point(includes) */\n' yield '#if PB_PROTO_HEADER_VERSION != 30\n' @@ -1363,6 +1363,9 @@ optparser.add_option("-f", "--options-file", dest="options_file", metavar="FILE" optparser.add_option("-I", "--options-path", dest="options_path", metavar="DIR", action="append", default = [], help="Search for .options files additionally in this path") +optparser.add_option("-D", "--output-dir", dest="output_dir", + metavar="OUTPUTDIR", default=None, + help="Output directory of .pb.h and .pb.c files") optparser.add_option("-Q", "--generated-include-format", dest="genformat", metavar="FORMAT", default='#include "%s"\n', help="Set format string to use for including other .pb.h files. [default: %default]") @@ -1479,17 +1482,29 @@ def main_cli(): if options.quiet: options.verbose = False - Globals.verbose_options = options.verbose + if options.output_dir and not os.path.exists(options.output_dir): + optparser.print_help() + sys.stderr.write("\noutput_dir does not exist: %s\n" % options.output_dir) + sys.exit(1) + + Globals.verbose_options = options.verbose for filename in filenames: results = process_file(filename, None, options) + base_dir = options.output_dir or '' + to_write = [ + (os.path.join(base_dir, results['headername']), results['headerdata']), + (os.path.join(base_dir, results['sourcename']), results['sourcedata']), + ] + if not options.quiet: - sys.stderr.write("Writing to " + results['headername'] + " and " - + results['sourcename'] + "\n") + paths = " and ".join([x[0] for x in to_write]) + sys.stderr.write("Writing to %s\n" % paths) - open(results['headername'], 'w').write(results['headerdata']) - open(results['sourcename'], 'w').write(results['sourcedata']) + for path, data in to_write: + with open(path, 'w') as f: + f.write(data) def main_plugin(): '''Main function when invoked as a protoc plugin.''' @@ -1553,4 +1568,3 @@ if __name__ == '__main__': main_plugin() else: main_cli() - -- cgit 1.2.3-korg From df3642b832dec73cf00e6a353ad134d3b132f4b1 Mon Sep 17 00:00:00 2001 From: isotes Date: Fri, 19 Feb 2016 23:51:00 +0100 Subject: Generate MIN/MAX/ARRAYSIZE for enums This generates #defines mirroring the following values from the generated C++ code of GPB * const Foo Foo_MIN: the smallest valid value of the enum (VALUE_A in the example). * const Foo Foo_MAX: the largest valid value of the enum (VALUE_C in the example). * const Foo Foo_ARRAYSIZE: always defined as Foo_MAX + 1. --- generator/nanopb_generator.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index bc9e9da8..0e9b018b 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -197,6 +197,10 @@ class Enum: result += ' %s;' % self.names + result += '\n#define _%s_MIN %s' % (self.names, self.values[0][0]) + result += '\n#define _%s_MAX %s' % (self.names, self.values[-1][0]) + result += '\n#define _%s_ARRAYSIZE ((%s)(%s+1))' % (self.names, self.names, self.values[-1][0]) + if not self.options.long_names: # Define the long names always so that enum value references # from other files work properly. -- cgit 1.2.3-korg From 11c073bf46204001e52388cdda8a814479e29f43 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 4 Mar 2016 17:38:47 +0200 Subject: Provide comments about uncalculable message sizes (issue #195) --- generator/nanopb_generator.py | 4 +++- tests/regression/issue_195/SConscript | 10 ++++++++++ tests/regression/issue_195/test.expected | 1 + tests/regression/issue_195/test.proto | 8 ++++++++ 4 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 tests/regression/issue_195/SConscript create mode 100644 tests/regression/issue_195/test.expected create mode 100644 tests/regression/issue_195/test.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 357350f0..501affa0 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1119,9 +1119,11 @@ class ProtoFile: yield '/* Maximum encoded size of messages (where known) */\n' for msg in self.messages: msize = msg.encoded_size(self.dependencies) + identifier = '%s_size' % msg.name if msize is not None: - identifier = '%s_size' % msg.name yield '#define %-40s %s\n' % (identifier, msize) + else: + yield '/* %s depends on runtime parameters */\n' % identifier yield '\n' yield '/* Message IDs (where set with "msgid" option) */\n' diff --git a/tests/regression/issue_195/SConscript b/tests/regression/issue_195/SConscript new file mode 100644 index 00000000..78326d32 --- /dev/null +++ b/tests/regression/issue_195/SConscript @@ -0,0 +1,10 @@ +# Regression test for Issue 195: Message size not calculated if a submessage includes +# bytes. Basically a non-working #define being generated. + +Import("env") + +env.NanopbProto(["test"]) +env.Object('test.pb.c') + +env.Match(['test.pb.h', 'test.expected']) + diff --git a/tests/regression/issue_195/test.expected b/tests/regression/issue_195/test.expected new file mode 100644 index 00000000..83ea7ab8 --- /dev/null +++ b/tests/regression/issue_195/test.expected @@ -0,0 +1 @@ +/\* TestMessage_size depends diff --git a/tests/regression/issue_195/test.proto b/tests/regression/issue_195/test.proto new file mode 100644 index 00000000..7a77d69d --- /dev/null +++ b/tests/regression/issue_195/test.proto @@ -0,0 +1,8 @@ +message TestMessage { + required uint32 id = 1; + required bytes payload = 2; +} +message EncapsulatedMessage { + required uint32 id = 1; + required TestMessage test = 2; +} -- cgit 1.2.3-korg From 420265d39bab7f1de051e108f7123b8c3b844f89 Mon Sep 17 00:00:00 2001 From: Maxim Khitrov Date: Mon, 7 Mar 2016 08:15:03 -0500 Subject: Update API reference to match headers Update typedefs and function prototypes in the API reference to match header files. Delete documentation for pb_skip_varint/pb_skip_string, which are superseded by pb_skip_field, and add pb_get_encoded_size. --- docs/reference.rst | 70 +++++++++++++++++++++++++----------------------------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/docs/reference.rst b/docs/reference.rst index 6968c81b..372450be 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -202,7 +202,7 @@ pb_type_t --------- Defines the encoder/decoder behaviour that should be used for a field. :: - typedef uint8_t pb_type_t; + typedef uint_least8_t pb_type_t; The low-order nibble of the enumeration values defines the function that can be used for encoding and decoding the field data: @@ -248,14 +248,14 @@ pb_field_t ---------- Describes a single structure field with memory position in relation to others. The descriptions are usually autogenerated. :: - typedef struct _pb_field_t pb_field_t; - struct _pb_field_t { - uint8_t tag; + typedef struct pb_field_s pb_field_t; + struct pb_field_s { + pb_size_t tag; pb_type_t type; - uint8_t data_offset; - int8_t size_offset; - uint8_t data_size; - uint8_t array_size; + pb_size_t data_offset; + pb_ssize_t size_offset; + pb_size_t data_size; + pb_size_t array_size; const void *ptr; } pb_packed; @@ -274,8 +274,8 @@ pb_bytes_array_t An byte array with a field for storing the length:: typedef struct { - size_t size; - uint8_t bytes[1]; + pb_size_t size; + pb_byte_t bytes[1]; } pb_bytes_array_t; In an actual array, the length of *bytes* may be different. @@ -339,12 +339,14 @@ Ties together the extension field type and the storage for the field value:: const pb_extension_type_t *type; void *dest; pb_extension_t *next; + bool found; } pb_extension_t; :type: Pointer to the structure that defines the callback functions. :dest: Pointer to the variable that stores the field value (as used by the default extension callback functions.) :next: Pointer to the next extension handler, or *NULL*. +:found: Decoder sets this to true if the extension was found. PB_GET_ERROR ------------ @@ -388,7 +390,7 @@ pb_ostream_from_buffer ---------------------- Constructs an output stream for writing into a memory buffer. This is just a helper function, it doesn't do anything you couldn't do yourself in a callback function. It uses an internal callback that stores the pointer in stream *state* field. :: - pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize); + pb_ostream_t pb_ostream_from_buffer(pb_byte_t *buf, size_t bufsize); :buf: Memory buffer to write into. :bufsize: Maximum number of bytes to write. @@ -400,7 +402,7 @@ pb_write -------- Writes data to an output stream. Always use this function, instead of trying to call stream callback manually. :: - bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); + bool pb_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); :stream: Output stream to write to. :buf: Pointer to buffer with the data to be written. @@ -441,11 +443,22 @@ This function does this, and it is compatible with *parseDelimitedFrom* in Googl Writing packed arrays is a little bit more involved: you need to use `pb_encode_tag` and specify `PB_WT_STRING` as the wire type. Then you need to know exactly how much data you are going to write, and use `pb_encode_varint`_ to write out the number of bytes before writing the actual data. Substreams can be used to determine the number of bytes beforehand; see `pb_encode_submessage`_ source code for an example. +pb_get_encoded_size +------------------- +Calculates the length of the encoded message. :: + + bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], const void *src_struct); + +:size: Calculated size of the encoded message. +:fields: A field description array, usually autogenerated. +:src_struct: Pointer to the data that will be serialized. +:returns: True on success, false on detectable errors in field description or if a field encoder returns false. + pb_encode_tag ------------- Starts a field in the Protocol Buffers binary format: encodes the field number and the wire type of the data. :: - bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number); + bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, uint32_t field_number); :stream: Output stream to write to. 1-5 bytes will be written. :wiretype: PB_WT_VARINT, PB_WT_64BIT, PB_WT_STRING or PB_WT_32BIT @@ -499,7 +512,7 @@ pb_encode_string ---------------- Writes the length of a string as varint and then contents of the string. Works for fields of type `bytes` and `string`:: - bool pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size); + bool pb_encode_string(pb_ostream_t *stream, const pb_byte_t *buffer, size_t size); :stream: Output stream to write to. :buffer: Pointer to string data. @@ -559,7 +572,7 @@ pb_istream_from_buffer ---------------------- Helper function for creating an input stream that reads data from a memory buffer. :: - pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); + pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t bufsize); :buf: Pointer to byte array to read from. :bufsize: Size of the byte array. @@ -569,7 +582,7 @@ pb_read ------- Read data from input stream. Always use this function, don't try to call the stream callback directly. :: - bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); + bool pb_read(pb_istream_t *stream, pb_byte_t *buf, size_t count); :stream: Input stream to read from. :buf: Buffer to store the data to, or NULL to just read data without storing it anywhere. @@ -630,7 +643,7 @@ pb_release ---------- Releases any dynamically allocated fields. - void pb_release(const pb_field_t fields[], void \*dest_struct); + void pb_release(const pb_field_t fields[], void *dest_struct); :fields: A field description array. Usually autogenerated. :dest_struct: Pointer to structure where data is stored. If NULL, function does nothing. @@ -638,29 +651,11 @@ Releases any dynamically allocated fields. This function is only available if *PB_ENABLE_MALLOC* is defined. It will release any pointer type fields in the structure and set the pointers to NULL. -pb_skip_varint --------------- -Skip a varint_ encoded integer without decoding it. :: - - bool pb_skip_varint(pb_istream_t *stream); - -:stream: Input stream to read from. Will read 1 byte at a time until the MSB is clear. -:returns: True on success, false on IO error. - -pb_skip_string --------------- -Skip a varint-length-prefixed string. This means skipping a value with wire type PB_WT_STRING. :: - - bool pb_skip_string(pb_istream_t *stream); - -:stream: Input stream to read from. -:returns: True on success, false on IO error or length exceeding uint32_t. - pb_decode_tag ------------- Decode the tag that comes before field in the protobuf encoding:: - bool pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, int *tag, bool *eof); + bool pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, uint32_t *tag, bool *eof); :stream: Input stream to read from. :wire_type: Pointer to variable where to store the wire type of the field. @@ -727,10 +722,9 @@ pb_decode_fixed64 ----------------- Decode a *fixed64*, *sfixed64* or *double* value. :: - bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest); + bool pb_decode_fixed64(pb_istream_t *stream, void *dest); :stream: Input stream to read from. 8 bytes will be read. -:field: Not used. :dest: Pointer to destination *int64_t*, *uint64_t* or *double*. :returns: True on success, false on IO errors. -- cgit 1.2.3-korg From 5fd9038ff812d10aa0d10b287dddb1e08dbfc47b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 7 Mar 2016 18:03:09 +0200 Subject: Some more docs updates --- docs/reference.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/reference.rst b/docs/reference.rst index 372450be..fad1b4c9 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -91,6 +91,7 @@ no_unions Generate 'oneof' fields as optional fields instead of C unions. msgid Specifies a unique id for this message type. Can be used by user code as an identifier. +anonymous_oneof Generate 'oneof' fields as anonymous unions. ============================ ================================================ These options can be defined for the .proto files before they are converted @@ -198,9 +199,18 @@ The options can be defined in file, message and field scopes:: pb.h ==== +pb_byte_t +--------- +Type used for storing byte-sized data, such as raw binary input and bytes-type fields. :: + + typedef uint_least8_t pb_byte_t; + +For most platforms this is equivalent to `uint8_t`. Some platforms however do not support +8-bit variables, and on those platforms 16 or 32 bits need to be used for each byte. + pb_type_t --------- -Defines the encoder/decoder behaviour that should be used for a field. :: +Type used to store the type of each field, to control the encoder/decoder behaviour. :: typedef uint_least8_t pb_type_t; -- cgit 1.2.3-korg From 516a5eab2c5904b2cd39d4958b0c101bafd8e060 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 10 Mar 2016 16:49:53 +0200 Subject: Fix formatting in docs --- docs/reference.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference.rst b/docs/reference.rst index fad1b4c9..7d27a515 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -651,7 +651,7 @@ This function is compatible with *writeDelimitedTo* in the Google's Protocol Buf pb_release ---------- -Releases any dynamically allocated fields. +Releases any dynamically allocated fields:: void pb_release(const pb_field_t fields[], void *dest_struct); -- cgit 1.2.3-korg From f4f8d778e7d0623a22987bad94b8d5249bb489c6 Mon Sep 17 00:00:00 2001 From: Yaniv Mordekhay Date: Thu, 5 May 2016 17:52:19 +0300 Subject: Added explanation of `oneof` section usage --- docs/concepts.rst | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/docs/concepts.rst b/docs/concepts.rst index b0fd43aa..b4f657e2 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -255,6 +255,61 @@ generates this field description array for the structure *Person_PhoneNumber*:: PB_LAST_FIELD }; +Oneof +===== +Protocol Buffers supports `oneof`_ sections. Here is an example of ``oneof`` usage:: + + message MsgType1 { + required int32 value = 1; + } + + message MsgType2 { + required bool value = 1; + } + + message MsgType3 { + required int32 value1 = 1; + required int32 value2 = 2; + } + + message MyMessage { + required uint32 uid = 1; + required uint32 pid = 2; + required uint32 utime = 3; + + oneof payload { + MsgType1 msg1 = 4; + MsgType2 msg2 = 5; + MsgType3 msg3 = 6; + } + } + +Nanopb will generate ``payload`` as a C union and add an additional field ``which_payload``:: + + typedef struct _MyMessage { + uint32_t uid; + uint32_t pid; + uint32_t utime; + pb_size_t which_payload; + union { + MsgType1 msg1; + MsgType2 msg2; + MsgType3 msg3; + } payload; + /* @@protoc_insertion_point(struct:MyMessage) */ + } MyMessage; + +``which_payload`` indicates which of the ``oneof`` fields is actually set. +The user is expected to set the filed manually using the correct field tag:: + + MyMessage msg = MyMessage_init_zero; + msg.payload.msg2.value = true; + msg.which_payload = MyMessage_msg2_tag; + +Notice that neither ``which_payload`` field nor the unused fileds in ``payload`` +will consume any space in the resulting encoded message. + +.. _`oneof`: https://developers.google.com/protocol-buffers/docs/reference/proto2-spec#oneof_and_oneof_field Extension fields ================ -- cgit 1.2.3-korg From 70699f3b0354fc717270c5a614ed65d9a60341f2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 25 May 2016 17:58:01 +0300 Subject: Add testcase for issue #203 --- tests/regression/issue_203/SConscript | 9 +++++++++ tests/regression/issue_203/file1.proto | 10 ++++++++++ tests/regression/issue_203/file2.proto | 10 ++++++++++ tests/site_scons/site_tools/nanopb.py | 2 +- 4 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 tests/regression/issue_203/SConscript create mode 100644 tests/regression/issue_203/file1.proto create mode 100644 tests/regression/issue_203/file2.proto diff --git a/tests/regression/issue_203/SConscript b/tests/regression/issue_203/SConscript new file mode 100644 index 00000000..8b4d6cc7 --- /dev/null +++ b/tests/regression/issue_203/SConscript @@ -0,0 +1,9 @@ +# Regression test for issue with multiple files generated at once + +Import('env') + +env.Command(['file1.pb.c', 'file1.pb.h', 'file2.pb.c', 'file2.pb.h'], ['file1.proto', 'file2.proto'], + env['NANOPB_PROTO_CMD']) + +env.Object('file1.pb.c') +env.Object('file2.pb.c') diff --git a/tests/regression/issue_203/file1.proto b/tests/regression/issue_203/file1.proto new file mode 100644 index 00000000..dae250b8 --- /dev/null +++ b/tests/regression/issue_203/file1.proto @@ -0,0 +1,10 @@ +syntax = "proto2"; + +message SubMessage1 { + required int32 foo = 1; +} + +message Message1 { + required SubMessage1 bar = 1; +} + diff --git a/tests/regression/issue_203/file2.proto b/tests/regression/issue_203/file2.proto new file mode 100644 index 00000000..513b0f0d --- /dev/null +++ b/tests/regression/issue_203/file2.proto @@ -0,0 +1,10 @@ +syntax = "proto2"; + +message SubMessage2 { + required int32 foo = 1; +} + +message Message2 { + required SubMessage2 bar = 1; +} + diff --git a/tests/site_scons/site_tools/nanopb.py b/tests/site_scons/site_tools/nanopb.py index b3e58fa1..c72a45d3 100644 --- a/tests/site_scons/site_tools/nanopb.py +++ b/tests/site_scons/site_tools/nanopb.py @@ -118,7 +118,7 @@ def generate(env): env.SetDefault(PROTOCPATH = ['.', os.path.join(env['NANOPB'], 'generator', 'proto')]) - env.SetDefault(NANOPB_PROTO_CMD = '$PROTOC $PROTOC_OPTS --nanopb_out=. $SOURCE') + env.SetDefault(NANOPB_PROTO_CMD = '$PROTOC $PROTOCFLAGS --nanopb_out=. $SOURCES') env['BUILDERS']['NanopbProto'] = _nanopb_proto_builder def exists(env): -- cgit 1.2.3-korg From f7e4f36b7c991cc976124d6b5c250ffdac19244f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 25 May 2016 18:03:33 +0300 Subject: Fix error in STATIC_ASSERT with multiple files (issue #203) The FieldMaxSize class was reusing the same list instance, causing problems when multiple files were specified on the protoc command line. --- generator/nanopb_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 501affa0..84f5d094 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -217,7 +217,7 @@ class FieldMaxSize: self.worst = worst self.worst_field = field_name - self.checks = checks + self.checks = list(checks) def extend(self, extend, field_name = None): self.worst = max(self.worst, extend.worst) -- cgit 1.2.3-korg From 1251fa1065afc0d62f635e0f63fec8276e14e13c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 25 May 2016 18:08:49 +0300 Subject: Small updates to docs --- docs/index.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index d49abc07..24328574 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -23,6 +23,7 @@ So a typical project might include these files: 1) Nanopb runtime library: - pb.h + - pb_common.h and pb_common.c (always needed) - pb_decode.h and pb_decode.c (needed for decoding messages) - pb_encode.h and pb_encode.c (needed for encoding messages) 2) Protocol description (you can have many): @@ -39,9 +40,9 @@ Features and limitations #) Small code size (2–10 kB depending on processor, plus any message definitions) #) Small ram usage (typically ~300 bytes, plus any message structs) #) Allows specifying maximum size for strings and arrays, so that they can be allocated statically. -#) No malloc needed: everything can be allocated statically or on the stack. +#) No malloc needed: everything can be allocated statically or on the stack. Optional malloc support available. #) You can use either encoder or decoder alone to cut the code size in half. -#) Support for most protobuf features, including: all data types, nested submessages, default values, repeated and optional fields, packed arrays, extension fields. +#) Support for most protobuf features, including: all data types, nested submessages, default values, repeated and optional fields, oneofs, packed arrays, extension fields. #) Callback mechanism for handling messages larger than can fit in available RAM. #) Extensive set of tests. @@ -53,8 +54,8 @@ Features and limitations #) Fields in the generated structs are ordered by the tag number, instead of the natural ordering in .proto file. #) Unknown fields are not preserved when decoding and re-encoding a message. #) Reflection (runtime introspection) is not supported. E.g. you can't request a field by giving its name in a string. -#) Numeric arrays are always encoded as packed, even if not marked as packed in .proto. This causes incompatibility with decoders that do not support packed format. -#) Cyclic references between messages are supported only in callback mode. +#) Numeric arrays are always encoded as packed, even if not marked as packed in .proto.. +#) Cyclic references between messages are supported only in callback and malloc mode. Getting started =============== -- cgit 1.2.3-korg From de938076da92b659a4bdac286f0daa0a2f393989 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 6 Jun 2016 19:33:05 +0300 Subject: Add static repeated submessage field to mem_release test case. Attempt at reproducing issue #204. --- tests/mem_release/mem_release.c | 2 ++ tests/mem_release/mem_release.proto | 1 + 2 files changed, 3 insertions(+) diff --git a/tests/mem_release/mem_release.c b/tests/mem_release/mem_release.c index 40fdc9e4..dc6f87de 100644 --- a/tests/mem_release/mem_release.c +++ b/tests/mem_release/mem_release.c @@ -25,6 +25,8 @@ static void fill_TestMessage(TestMessage *msg) msg->static_req_submsg.dynamic_submsg = test_msg_arr; msg->static_req_submsg.dynamic_submsg[1].dynamic_str = "abc"; msg->static_opt_submsg.dynamic_str = "abc"; + msg->static_rep_submsg_count = 2; + msg->static_rep_submsg[1].dynamic_str = "abc"; msg->has_static_opt_submsg = true; msg->dynamic_submsg = &msg->static_req_submsg; diff --git a/tests/mem_release/mem_release.proto b/tests/mem_release/mem_release.proto index c3b38c8b..0816dc22 100644 --- a/tests/mem_release/mem_release.proto +++ b/tests/mem_release/mem_release.proto @@ -13,6 +13,7 @@ message TestMessage required SubMessage static_req_submsg = 1 [(nanopb).type = FT_STATIC]; optional SubMessage dynamic_submsg = 2 [(nanopb).type = FT_POINTER]; optional SubMessage static_opt_submsg = 3 [(nanopb).type = FT_STATIC]; + repeated SubMessage static_rep_submsg = 4 [(nanopb).type = FT_STATIC, (nanopb).max_count=2]; extensions 100 to 200; } -- cgit 1.2.3-korg From 3af7d0910b9660a270aa3279716596c7d7711671 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 6 Jun 2016 21:00:59 +0300 Subject: Add testcase for issue #205 --- tests/fuzztest/fuzztest.c | 1 + tests/regression/issue_205/SConscript | 14 ++++++++++++++ tests/regression/issue_205/size_corruption.c | 12 ++++++++++++ tests/regression/issue_205/size_corruption.proto | 11 +++++++++++ 4 files changed, 38 insertions(+) create mode 100644 tests/regression/issue_205/SConscript create mode 100644 tests/regression/issue_205/size_corruption.c create mode 100644 tests/regression/issue_205/size_corruption.proto diff --git a/tests/fuzztest/fuzztest.c b/tests/fuzztest/fuzztest.c index d3701724..ee851ec0 100644 --- a/tests/fuzztest/fuzztest.c +++ b/tests/fuzztest/fuzztest.c @@ -192,6 +192,7 @@ static bool do_static_encode(uint8_t *buffer, size_t *msglen) assert(stream.bytes_written <= alltypes_static_AllTypes_size); *msglen = stream.bytes_written; + pb_release(alltypes_static_AllTypes_fields, msg); free_with_check(msg); return status; diff --git a/tests/regression/issue_205/SConscript b/tests/regression/issue_205/SConscript new file mode 100644 index 00000000..ed8899dd --- /dev/null +++ b/tests/regression/issue_205/SConscript @@ -0,0 +1,14 @@ +# Check that pb_release() correctly handles corrupted size fields of +# static arrays. + +Import('env', 'malloc_env') + +env.NanopbProto('size_corruption') + +p = malloc_env.Program(["size_corruption.c", + "size_corruption.pb.c", + "$COMMON/pb_decode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", + "$COMMON/malloc_wrappers.o"]) +env.RunTest(p) + diff --git a/tests/regression/issue_205/size_corruption.c b/tests/regression/issue_205/size_corruption.c new file mode 100644 index 00000000..08cef457 --- /dev/null +++ b/tests/regression/issue_205/size_corruption.c @@ -0,0 +1,12 @@ +#include "size_corruption.pb.h" +#include + +int main() +{ + MainMessage msg = MainMessage_init_zero; + msg.bar_count = (pb_size_t)-1; + pb_release(MainMessage_fields, &msg); + + return 0; +} + diff --git a/tests/regression/issue_205/size_corruption.proto b/tests/regression/issue_205/size_corruption.proto new file mode 100644 index 00000000..6c9c2453 --- /dev/null +++ b/tests/regression/issue_205/size_corruption.proto @@ -0,0 +1,11 @@ +syntax = "proto2"; +import 'nanopb.proto'; + +message SubMessage { + repeated int32 foo = 1 [(nanopb).type = FT_POINTER]; +} + +message MainMessage { + repeated SubMessage bar = 1 [(nanopb).max_count = 5]; +} + -- cgit 1.2.3-korg From bb52a7a3e1802a65e2347f3a7a48c6fb3bdc47e4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 6 Jun 2016 21:01:22 +0300 Subject: Protect against corrupted _count fields in pb_release(). Fixes a potential security issue (#205). Only relevant if the user code writes untrusted data to _count fields, but this is allowed as per the security model. --- pb_decode.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pb_decode.c b/pb_decode.c index 1699091c..78911e7b 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -1035,6 +1035,12 @@ static void pb_release_single_field(const pb_field_iter_t *iter) if (PB_HTYPE(type) == PB_HTYPE_REPEATED) { count = *(pb_size_t*)iter->pSize; + + if (PB_ATYPE(type) == PB_ATYPE_STATIC && count > iter->pos->array_size) + { + /* Protect against corrupted _count fields */ + count = iter->pos->array_size; + } } if (pItem) -- cgit 1.2.3-korg From ed8ac90bd162bdbe284f7679f7ba45fa13e22bd1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 6 Jun 2016 21:07:28 +0300 Subject: Clarify security model with regard to pointer _count fields. --- docs/security.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/security.rst b/docs/security.rst index 2d0affc5..d8546122 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -38,8 +38,11 @@ these will cause "garbage in, garbage out" behaviour. It will not cause buffer overflows, information disclosure or other security problems: 1. All data read from *pb_istream_t*. -2. All fields in message structures, except callbacks, pointers and extensions. - (Beginning with nanopb-0.2.4, in earlier versions the field sizes are partially unchecked.) +2. All fields in message structures, except: + + - callbacks (*pb_callback_t* structures) + - pointer fields (malloc support) and *_count* fields for pointers + - extensions (*pb_extension_t* structures) Invariants ========== -- cgit 1.2.3-korg From b9f552d25610acccde8622afd5d2b445756c05cc Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 14 Jun 2016 18:34:44 +0300 Subject: Update changelog --- CHANGELOG.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index fd983dfe..9e3ad69a 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,12 @@ +nanopb-0.3.6 (2016-06-xx) + Protect against corrupted _count fields in pb_release (#205) + Fix error in STATIC_ASSERT with multiple files (#203) + Add -D option to specify output directory (#193) + Generate MIN/MAX/ARRAYSIZE defines for enums (#194) + Generate comments about uncalculable message sizes (#195) + Documentation updates (#196, #201) + Improvements to test cases. + nanopb-0.3.5 (2016-02-13) NOTE: If you are using pb_syshdr.h, you will need to add uint_least8_t definition. See docs/migration.rst for details. -- cgit 1.2.3-korg From eb0e73ca53e09f2ba42cd9775168b683c98b84ea Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 19 Jun 2016 10:18:03 +0300 Subject: Publishing nanopb-0.3.6 --- CHANGELOG.txt | 2 +- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9e3ad69a..8437688f 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,4 +1,4 @@ -nanopb-0.3.6 (2016-06-xx) +nanopb-0.3.6 (2016-06-19) Protect against corrupted _count fields in pb_release (#205) Fix error in STATIC_ASSERT with multiple files (#203) Add -D option to specify output directory (#193) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 84f5d094..2f1d92f4 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.6-dev" +nanopb_version = "nanopb-0.3.6" import sys import re diff --git a/pb.h b/pb.h index 114c9c74..5e5d07bb 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.6-dev +#define NANOPB_VERSION nanopb-0.3.6 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 9c36b59603c8a52edb8d786d1d6ef4f429f56ece Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 19 Jun 2016 10:54:26 +0300 Subject: Setting version to 0.3.7-dev --- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 2f1d92f4..64514577 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.6" +nanopb_version = "nanopb-0.3.7-dev" import sys import re diff --git a/pb.h b/pb.h index 5e5d07bb..790f8861 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.6 +#define NANOPB_VERSION nanopb-0.3.7-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 298d00e8d73c23b8b2537489aee4319a43968e9e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 23 Jul 2016 20:24:54 +0300 Subject: Include package name in include guard (issue #207). Fix suggested by Ulenspiegel. Also added testcase for the same. --- generator/nanopb_generator.py | 5 ++++- tests/multiple_files/SConscript | 5 ++++- tests/multiple_files/subdir/multifile2.proto | 25 +++++++++++++++++++++++++ tests/multiple_files/test_multiple_files.c | 8 ++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 tests/multiple_files/subdir/multifile2.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 64514577..9cf2de5f 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1045,7 +1045,10 @@ class ProtoFile: else: yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) - symbol = make_identifier(headername) + if self.fdesc.package: + symbol = make_identifier(self.fdesc.package + '_' + headername) + else: + symbol = make_identifier(headername) yield '#ifndef PB_%s_INCLUDED\n' % symbol yield '#define PB_%s_INCLUDED\n' % symbol try: diff --git a/tests/multiple_files/SConscript b/tests/multiple_files/SConscript index 1689f482..b1281e17 100644 --- a/tests/multiple_files/SConscript +++ b/tests/multiple_files/SConscript @@ -4,10 +4,13 @@ Import("env") incpath = env.Clone() incpath.Append(PROTOCPATH = '#multiple_files') +incpath.Append(CPPPATH = '$BUILD/multiple_files') incpath.NanopbProto(["multifile1", "multifile1.options"]) incpath.NanopbProto("multifile2") -test = incpath.Program(["test_multiple_files.c", "multifile1.pb.c", "multifile2.pb.c"]) +incpath.NanopbProto("subdir/multifile2") +test = incpath.Program(["test_multiple_files.c", "multifile1.pb.c", + "multifile2.pb.c", "subdir/multifile2.pb.c"]) env.RunTest(test) diff --git a/tests/multiple_files/subdir/multifile2.proto b/tests/multiple_files/subdir/multifile2.proto new file mode 100644 index 00000000..847a9290 --- /dev/null +++ b/tests/multiple_files/subdir/multifile2.proto @@ -0,0 +1,25 @@ +syntax = "proto2"; + +package subdir; + +import "multifile1.proto"; + +message Callback2Message { + required TestMessage tstmsg = 1; + required SubMessage submsg = 2; +} + +message OneofMessage { + oneof msgs { + StaticMessage tstmsg = 1; + } +} + +message Enums { + required SignedEnum senum = 1; + required UnsignedEnum uenum = 2; +} + +message SubdirMessage { + required int32 foo = 1 [default = 15]; +} diff --git a/tests/multiple_files/test_multiple_files.c b/tests/multiple_files/test_multiple_files.c index 292b8d7c..70a3e596 100644 --- a/tests/multiple_files/test_multiple_files.c +++ b/tests/multiple_files/test_multiple_files.c @@ -6,6 +6,7 @@ #include #include "unittests.h" #include "multifile2.pb.h" +#include "subdir/multifile2.pb.h" int main() { @@ -18,5 +19,12 @@ int main() TEST(PB_LTYPE(Enums_fields[0].type) == PB_LTYPE_VARINT); TEST(PB_LTYPE(Enums_fields[1].type) == PB_LTYPE_UVARINT); + /* Test that subdir file is correctly included */ + { + subdir_SubdirMessage foo = subdir_SubdirMessage_init_default; + TEST(foo.foo == 15); + /* TEST(subdir_OneofMessage_size == 27); */ /* TODO: Issue #172 */ + } + return status; } -- cgit 1.2.3-korg From 6e47677acf98934b76fb5f6337867b276ce7bce8 Mon Sep 17 00:00:00 2001 From: Konstantin Podsvirov Date: Tue, 26 Jul 2016 23:33:33 +0300 Subject: Added CMake project --- CMakeLists.txt | 59 ++++++++++++++++++++++++++++++++++++ extra/nanopb-config-version.cmake.in | 11 +++++++ extra/nanopb-config.cmake | 1 + tools/set_version.sh | 1 + 4 files changed, 72 insertions(+) create mode 100644 CMakeLists.txt create mode 100644 extra/nanopb-config-version.cmake.in create mode 100644 extra/nanopb-config.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 00000000..7b19f287 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,59 @@ +cmake_minimum_required(VERSION 2.8) + +project(nanopb C) + +set(nanopb_VERSION_STRING nanopb-0.3.7-dev) + +string(REPLACE "nanopb-" "" nanopb_VERSION ${nanopb_VERSION_STRING}) + +option(nanopb_MSVC_STATIC_RUNTIME "Link static runtime libraries" ON) + +if(NOT DEFINED CMAKE_DEBUG_POSTFIX) + set(CMAKE_DEBUG_POSTFIX "d") +endif() + +if(MSVC AND nanopb_MSVC_STATIC_RUNTIME) + foreach(flag_var + CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE + CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO) + if(${flag_var} MATCHES "/MD") + string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}") + endif(${flag_var} MATCHES "/MD") + endforeach(flag_var) +endif() + +add_library(libprotobuf-nanopb STATIC + pb.h + pb_common.h + pb_common.c + pb_encode.h + pb_encode.c + pb_decode.h + pb_decode.c) + +target_include_directories(libprotobuf-nanopb INTERFACE + $ +) + +include(GNUInstallDirs) + +if(NOT DEFINED CMAKE_INSTALL_CMAKEDIR) + set(CMAKE_INSTALL_CMAKEDIR "lib/cmake/nanopb") +endif() + +install(TARGETS libprotobuf-nanopb EXPORT nanopb-targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) + +install(EXPORT nanopb-targets + DESTINATION ${CMAKE_INSTALL_CMAKEDIR} + NAMESPACE nanopb::) + +configure_file(extra/nanopb-config-version.cmake.in + nanopb-config-version.cmake @ONLY) + +install(FILES extra/nanopb-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/nanopb-config-version.cmake + DESTINATION ${CMAKE_INSTALL_CMAKEDIR}) + +install(FILES pb.h pb_common.h pb_encode.h pb_decode.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) diff --git a/extra/nanopb-config-version.cmake.in b/extra/nanopb-config-version.cmake.in new file mode 100644 index 00000000..f9292a83 --- /dev/null +++ b/extra/nanopb-config-version.cmake.in @@ -0,0 +1,11 @@ +set(PACKAGE_VERSION "@nanopb_VERSION@") + +# Check whether the requested PACKAGE_FIND_VERSION is compatible +if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") + set(PACKAGE_VERSION_COMPATIBLE FALSE) +else() + set(PACKAGE_VERSION_COMPATIBLE TRUE) + if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") + set(PACKAGE_VERSION_EXACT TRUE) + endif() +endif() diff --git a/extra/nanopb-config.cmake b/extra/nanopb-config.cmake new file mode 100644 index 00000000..4f726a68 --- /dev/null +++ b/extra/nanopb-config.cmake @@ -0,0 +1 @@ +include(${CMAKE_CURRENT_LIST_DIR}/nanopb-targets.cmake) diff --git a/tools/set_version.sh b/tools/set_version.sh index e15a859d..f989308c 100755 --- a/tools/set_version.sh +++ b/tools/set_version.sh @@ -6,5 +6,6 @@ sed -i -e 's/nanopb_version\s*=\s*"[^"]*"/nanopb_version = "'$1'"/' generator/nanopb_generator.py sed -i -e 's/#define\s*NANOPB_VERSION\s*.*/#define NANOPB_VERSION '$1'/' pb.h +sed -i -e 's/set(\s*nanopb_VERSION_STRING\s*[^)]*)/set(nanopb_VERSION_STRING '$1')/' CMakeLists.txt -- cgit 1.2.3-korg From 4e34042804959184d3a38426724e0ec38176fd0b Mon Sep 17 00:00:00 2001 From: Ming Zhao Date: Mon, 1 Aug 2016 12:26:37 -0700 Subject: Add bazel BUILD file for nanopb. --- BUILD | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 BUILD diff --git a/BUILD b/BUILD new file mode 100644 index 00000000..57098843 --- /dev/null +++ b/BUILD @@ -0,0 +1,18 @@ +licenses(["notice"]) +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "nanopb", + visibility = ["//visibility:public"], + hdrs = [ + "pb.h", + "pb_common.h", + "pb_decode.h", + "pb_encode.h", + ], + srcs = [ + "pb_common.c", + "pb_decode.c", + "pb_encode.c", + ], +) -- cgit 1.2.3-korg From 62afd54964528c1fbd5ab802134f7e9ad912d904 Mon Sep 17 00:00:00 2001 From: Tom Roeder Date: Tue, 2 Aug 2016 14:57:37 -0700 Subject: Add inline allocation of bytes fields This commit adds a new FT_INLINE allocation type that forces bytes fields to be inlined into the struct. E.g., pb_byte_t my_bytes[32]. This requires max_size for the bytes field. The FT_INLINE type is represented as a new LTYPE: FT_LTYPE_FIXED_LENGTH_BYTES. This commit also updates the documentation with FT_INLINE and FT_LTYPE_FIXED_LENGTH_BYTES. Added an AUTHORS file in apparent order of appearance in the git log history from $(git log --all). --- AUTHORS | 24 ++++++++++++++ docs/concepts.rst | 4 ++- docs/index.rst | 4 +-- docs/reference.rst | 49 ++++++++++++++------------- generator/nanopb_generator.py | 35 ++++++++++++++++---- generator/proto/nanopb.proto | 1 + pb.h | 27 +++++++++++++-- pb_decode.c | 9 ++++- pb_encode.c | 15 ++++++--- tests/inline/SConscript | 16 +++++++++ tests/inline/inline.expected | 3 ++ tests/inline/inline.proto | 17 ++++++++++ tests/inline/inline_unittests.c | 73 +++++++++++++++++++++++++++++++++++++++++ 13 files changed, 239 insertions(+), 38 deletions(-) create mode 100644 AUTHORS create mode 100644 tests/inline/SConscript create mode 100644 tests/inline/inline.expected create mode 100644 tests/inline/inline.proto create mode 100644 tests/inline/inline_unittests.c diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 00000000..a967d3ec --- /dev/null +++ b/AUTHORS @@ -0,0 +1,24 @@ +Petteri Aimonen +Michael Poole +Daniel Kan +Stan Hu +dch +Steffen Siering +Jens Steinhauser +Pavel Ilin +Kent Ryhorchuk +Martin Donath +Oliver Lee +Michael Haberler +Nicolas Colomer +Ivan Kravets +Kyle Manna +Benjamin Kamath +Andrew Ruder +Kenshi Kawaguchi +isotes +Maxim Khitrov +Yaniv Mordekhay +Ming Zhao +Google, Inc. +Tom Roeder diff --git a/docs/concepts.rst b/docs/concepts.rst index b4f657e2..c43d8299 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -148,6 +148,7 @@ Most Protocol Buffers datatypes have directly corresponding C datatypes, such as 1) Strings, bytes and repeated fields of any type map to callback functions by default. 2) If there is a special option *(nanopb).max_size* specified in the .proto file, string maps to null-terminated char array and bytes map to a structure containing a char array and a size field. +3) If *(nanopb).type* is set to *FT_INLINE* and *(nanopb).max_size* is also set, then bytes map to an inline byte array of fixed size. 3) If there is a special option *(nanopb).max_count* specified on a repeated field, it maps to an array of whatever type is being repeated. Another field will be created for the actual number of entries stored. =============================================================================== ======================= @@ -160,9 +161,10 @@ repeated string name = 1 [(nanopb).max_size = 40, (nanopb).max_count = 5]; | char name[5][40]; required bytes data = 1 [(nanopb).max_size = 40]; | typedef struct { | size_t size; - | uint8_t bytes[40]; + | pb_byte_t bytes[40]; | } Person_data_t; | Person_data_t data; +required bytes data = 1 [(nanopb).max_size = 40, (nanopb.type) = FT_INLINE]; | pb_byte_t data[40]; =============================================================================== ======================= The maximum lengths are checked in runtime. If string/bytes/array exceeds the allocated length, *pb_decode* will return false. diff --git a/docs/index.rst b/docs/index.rst index 24328574..afc7ee4f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ Nanopb: Protocol Buffers with small code size Nanopb is an ANSI-C library for encoding and decoding messages in Google's `Protocol Buffers`__ format with minimal requirements for RAM and code space. It is primarily suitable for 32-bit microcontrollers. -__ http://code.google.com/apis/protocolbuffers/ +__ https://developers.google.com/protocol-buffers/docs/reference/overview Overall structure ================= @@ -54,7 +54,7 @@ Features and limitations #) Fields in the generated structs are ordered by the tag number, instead of the natural ordering in .proto file. #) Unknown fields are not preserved when decoding and re-encoding a message. #) Reflection (runtime introspection) is not supported. E.g. you can't request a field by giving its name in a string. -#) Numeric arrays are always encoded as packed, even if not marked as packed in .proto.. +#) Numeric arrays are always encoded as packed, even if not marked as packed in .proto. #) Cyclic references between messages are supported only in callback and malloc mode. Getting started diff --git a/docs/reference.rst b/docs/reference.rst index 7d27a515..ef3867a1 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -77,9 +77,11 @@ int_size Override the integer type of a field. type Type of the generated field. Default value is *FT_DEFAULT*, which selects automatically. You can use *FT_CALLBACK*, *FT_POINTER*, - *FT_STATIC* or *FT_IGNORE* to force a callback - field, a dynamically allocated field, a static - field or to completely ignore the field. + *FT_STATIC*, *FT_IGNORE*, or *FT_INLINE* to + force a callback field, a dynamically + allocated field, a static field, to + completely ignore the field or to + generate an inline bytes field. long_names Prefix the enum name to the enum value in definitions, i.e. *EnumName_EnumValue*. Enabled by default. @@ -216,17 +218,20 @@ Type used to store the type of each field, to control the encoder/decoder behavi The low-order nibble of the enumeration values defines the function that can be used for encoding and decoding the field data: -==================== ===== ================================================ -LTYPE identifier Value Storage format -==================== ===== ================================================ -PB_LTYPE_VARINT 0x00 Integer. -PB_LTYPE_SVARINT 0x01 Integer, zigzag encoded. -PB_LTYPE_FIXED32 0x02 32-bit integer or floating point. -PB_LTYPE_FIXED64 0x03 64-bit integer or floating point. -PB_LTYPE_BYTES 0x04 Structure with *size_t* field and byte array. -PB_LTYPE_STRING 0x05 Null-terminated string. -PB_LTYPE_SUBMESSAGE 0x06 Submessage structure. -==================== ===== ================================================ +=========================== ===== ================================================ +LTYPE identifier Value Storage format +=========================== ===== ================================================ +PB_LTYPE_VARINT 0x00 Integer. +PB_LTYPE_UVARINT 0x01 Unsigned integer. +PB_LTYPE_SVARINT 0x02 Integer, zigzag encoded. +PB_LTYPE_FIXED32 0x03 32-bit integer or floating point. +PB_LTYPE_FIXED64 0x04 64-bit integer or floating point. +PB_LTYPE_BYTES 0x05 Structure with *size_t* field and byte array. +PB_LTYPE_STRING 0x06 Null-terminated string. +PB_LTYPE_SUBMESSAGE 0x07 Submessage structure. +PB_LTYPE_EXTENSION 0x08 Point to *pb_extension_t*. +PB_LTYPE_FIXED_LENGTH_BYTES 0x09 Inline *pb_byte_t* array of fixed size. +=========================== ===== ================================================ The bits 4-5 define whether the field is required, optional or repeated: @@ -489,14 +494,14 @@ This function only considers the LTYPE of the field. You can use it from your fi Wire type mapping is as follows: -========================= ============ -LTYPEs Wire type -========================= ============ -VARINT, SVARINT PB_WT_VARINT -FIXED64 PB_WT_64BIT -STRING, BYTES, SUBMESSAGE PB_WT_STRING -FIXED32 PB_WT_32BIT -========================= ============ +============================================= ============ +LTYPEs Wire type +============================================= ============ +VARINT, UVARINT, SVARINT PB_WT_VARINT +FIXED64 PB_WT_64BIT +STRING, BYTES, SUBMESSAGE, FIXED_LENGTH_BYTES PB_WT_STRING +FIXED32 PB_WT_32BIT +============================================= ============ pb_encode_varint ---------------- diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 9cf2de5f..973c7610 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -241,6 +241,11 @@ class Field: self.enc_size = None self.ctype = None + self.inline = None + if field_options.type == nanopb_pb2.FT_INLINE: + field_options.type = nanopb_pb2.FT_STATIC + self.inline = nanopb_pb2.FT_INLINE + # Parse field options if field_options.HasField("max_size"): self.max_size = field_options.max_size @@ -319,7 +324,12 @@ class Field: elif desc.type == FieldD.TYPE_BYTES: self.pbtype = 'BYTES' if self.allocation == 'STATIC': - self.ctype = self.struct_name + self.name + 't' + # Inline STATIC for BYTES is like STATIC for STRING. + if self.inline: + self.ctype = 'pb_byte_t' + self.array_decl += '[%d]' % self.max_size + else: + self.ctype = self.struct_name + self.name + 't' self.enc_size = varint_max_size(self.max_size) + self.max_size elif self.allocation == 'POINTER': self.ctype = 'pb_bytes_array_t' @@ -359,7 +369,7 @@ class Field: def types(self): '''Return definitions for any special types this field might need.''' - if self.pbtype == 'BYTES' and self.allocation == 'STATIC': + if self.pbtype == 'BYTES' and self.allocation == 'STATIC' and not self.inline: result = 'typedef PB_BYTES_ARRAY_T(%d) %s;\n' % (self.max_size, self.ctype) else: result = '' @@ -388,7 +398,10 @@ class Field: if self.pbtype == 'STRING': inner_init = '""' elif self.pbtype == 'BYTES': - inner_init = '{0, {0}}' + if self.inline: + inner_init = '{0}' + else: + inner_init = '{0, {0}}' elif self.pbtype in ('ENUM', 'UENUM'): inner_init = '(%s)0' % self.ctype else: @@ -400,9 +413,15 @@ class Field: elif self.pbtype == 'BYTES': data = ['0x%02x' % ord(c) for c in self.default] if len(data) == 0: - inner_init = '{0, {0}}' + if self.inline: + inner_init = '{0}' + else: + inner_init = '{0, {0}}' else: - inner_init = '{%d, {%s}}' % (len(data), ','.join(data)) + if self.inline: + inner_init = '{%s}' % ','.join(data) + else: + inner_init = '{%d, {%s}}' % (len(data), ','.join(data)) elif self.pbtype in ['FIXED32', 'UINT32']: inner_init = str(self.default) + 'u' elif self.pbtype in ['FIXED64', 'UINT64']: @@ -454,6 +473,8 @@ class Field: elif self.pbtype == 'BYTES': if self.allocation != 'STATIC': return None # Not implemented + if self.inline: + array_decl = '[%d]' % self.max_size if declaration_only: return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl) @@ -481,7 +502,7 @@ class Field: result += '%3d, ' % self.tag result += '%-8s, ' % self.pbtype result += '%s, ' % self.rules - result += '%-8s, ' % self.allocation + result += '%-8s, ' % (self.allocation if not self.inline else "INLINE") result += '%s, ' % ("FIRST" if not prev_field_name else "OTHER") result += '%s, ' % self.struct_name result += '%s, ' % self.name @@ -594,6 +615,7 @@ class ExtensionRange(Field): self.default = None self.max_size = 0 self.max_count = 0 + self.inline = None def __str__(self): return ' pb_extension_t *extensions;' @@ -671,6 +693,7 @@ class OneOf(Field): self.default = None self.rules = 'ONEOF' self.anonymous = False + self.inline = None def add_field(self, field): if field.allocation == 'CALLBACK': diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index 9b2f0fbc..8aab19a1 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -16,6 +16,7 @@ enum FieldType { FT_POINTER = 4; // Always generate a dynamically allocated field. FT_STATIC = 2; // Generate a static field or raise an exception if not possible. FT_IGNORE = 3; // Ignore the field completely. + FT_INLINE = 5; // Always generate an inline array of fixed size. } enum IntSize { diff --git a/pb.h b/pb.h index 790f8861..4576f79a 100644 --- a/pb.h +++ b/pb.h @@ -170,8 +170,14 @@ typedef uint_least8_t pb_type_t; * The field contains a pointer to pb_extension_t */ #define PB_LTYPE_EXTENSION 0x08 +/* Byte array with inline, pre-allocated byffer. + * data_size is the length of the inline, allocated buffer. + * This differs from PB_LTYPE_BYTES by defining the element as + * pb_byte_t[data_size] rather than pb_bytes_array_t. */ +#define PB_LTYPE_FIXED_LENGTH_BYTES 0x09 + /* Number of declared LTYPES */ -#define PB_LTYPES_COUNT 9 +#define PB_LTYPES_COUNT 0x0A #define PB_LTYPE_MASK 0x0F /**** Field repetition rules ****/ @@ -415,6 +421,19 @@ struct pb_extension_s { pb_membersize(st, m[0]), \ pb_arraysize(st, m), ptr} +#define PB_REQUIRED_INLINE(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REQUIRED | PB_LTYPE_FIXED_LENGTH_BYTES, \ + fd, 0, pb_membersize(st, m), 0, ptr} + +/* Optional fields add the delta to the has_ variable. */ +#define PB_OPTIONAL_INLINE(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED_LENGTH_BYTES, \ + fd, \ + pb_delta(st, has_ ## m, m), \ + pb_membersize(st, m), 0, ptr} + +/* INLINE does not support REPEATED fields. */ + /* Allocated fields carry the size of the actual data, not the pointer */ #define PB_REQUIRED_POINTER(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_POINTER | PB_HTYPE_REQUIRED | ltype, \ @@ -454,6 +473,8 @@ struct pb_extension_s { #define PB_OPTEXT_POINTER(tag, st, m, fd, ltype, ptr) \ PB_OPTIONAL_POINTER(tag, st, m, fd, ltype, ptr) +/* INLINE does not support OPTEXT. */ + #define PB_OPTEXT_CALLBACK(tag, st, m, fd, ltype, ptr) \ PB_OPTIONAL_CALLBACK(tag, st, m, fd, ltype, ptr) @@ -485,7 +506,7 @@ struct pb_extension_s { * FLOAT, INT32, INT64, MESSAGE, SFIXED32, SFIXED64 * SINT32, SINT64, STRING, UINT32, UINT64 or EXTENSION * - Field rules: REQUIRED, OPTIONAL or REPEATED - * - Allocation: STATIC or CALLBACK + * - Allocation: STATIC, INLINE, or CALLBACK * - Placement: FIRST or OTHER, depending on if this is the first field in structure. * - Message name * - Field name @@ -511,6 +532,8 @@ struct pb_extension_s { fd, pb_delta(st, which_ ## u, u.m), \ pb_membersize(st, u.m[0]), 0, ptr} +/* INLINE does not support ONEOF. */ + #define PB_ONEOF_FIELD(union_name, tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ PB_ONEOF_ ## allocation(union_name, tag, message, field, \ PB_DATAOFFSET_ ## placement(message, union_name.field, prevfield), \ diff --git a/pb_decode.c b/pb_decode.c index 78911e7b..7a4e29a8 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -65,7 +65,8 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { &pb_dec_bytes, &pb_dec_string, &pb_dec_submessage, - NULL /* extensions */ + NULL, /* extensions */ + &pb_dec_bytes /* PB_LTYPE_FIXED_LENGTH_BYTES */ }; /******************************* @@ -1272,6 +1273,12 @@ static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *fie } else { + if (PB_LTYPE(field->type) == PB_LTYPE_FIXED_LENGTH_BYTES) { + if (size != field->data_size) + PB_RETURN_ERROR(stream, "incorrect inline bytes size"); + return pb_read(stream, (pb_byte_t*)dest, field->data_size); + } + if (alloc_size > field->data_size) PB_RETURN_ERROR(stream, "bytes overflow"); bdest = (pb_bytes_array_t*)dest; diff --git a/pb_encode.c b/pb_encode.c index 9f91c9d5..4685614c 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -49,7 +49,8 @@ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { &pb_enc_bytes, &pb_enc_string, &pb_enc_submessage, - NULL /* extensions */ + NULL, /* extensions */ + &pb_enc_bytes /* PB_LTYPE_FIXED_LENGTH_BYTES */ }; /******************************* @@ -498,6 +499,7 @@ bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t case PB_LTYPE_BYTES: case PB_LTYPE_STRING: case PB_LTYPE_SUBMESSAGE: + case PB_LTYPE_FIXED_LENGTH_BYTES: wiretype = PB_WT_STRING; break; @@ -636,11 +638,16 @@ static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *f static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src; + const pb_bytes_array_t *bytes = NULL; + + if (PB_LTYPE(field->type) == PB_LTYPE_FIXED_LENGTH_BYTES) + return pb_encode_string(stream, (const pb_byte_t*)src, field->data_size); + + bytes = (const pb_bytes_array_t*)src; if (src == NULL) { - /* Threat null pointer as an empty bytes field */ + /* Treat null pointer as an empty bytes field */ return pb_encode_string(stream, NULL, 0); } @@ -664,7 +671,7 @@ static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *fi if (src == NULL) { - size = 0; /* Threat null pointer as an empty string */ + size = 0; /* Treat null pointer as an empty string */ } else { diff --git a/tests/inline/SConscript b/tests/inline/SConscript new file mode 100644 index 00000000..34371fda --- /dev/null +++ b/tests/inline/SConscript @@ -0,0 +1,16 @@ +# Test that inlined bytes fields work. + +Import("env") + +env.NanopbProto("inline") +env.Object("inline.pb.c") + +env.Match(["inline.pb.h", "inline.expected"]) + +p = env.Program(["inline_unittests.c", + "inline.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_decode.o", + "$COMMON/pb_common.o"]) + +env.RunTest(p) diff --git a/tests/inline/inline.expected b/tests/inline/inline.expected new file mode 100644 index 00000000..593e972b --- /dev/null +++ b/tests/inline/inline.expected @@ -0,0 +1,3 @@ +pb_byte_t data\[32\]; +bool has_data; +pb_byte_t data\[64\]; diff --git a/tests/inline/inline.proto b/tests/inline/inline.proto new file mode 100644 index 00000000..6e511f0a --- /dev/null +++ b/tests/inline/inline.proto @@ -0,0 +1,17 @@ +/* Test nanopb option parsing. + * options.expected lists the patterns that are searched for in the output. + */ + +syntax = "proto2"; + +import "nanopb.proto"; + +message Message1 +{ + required bytes data = 1 [(nanopb).type = FT_INLINE, (nanopb).max_size = 32]; +} + +message Message2 +{ + optional bytes data = 1 [(nanopb).type = FT_INLINE, (nanopb).max_size = 64]; +} diff --git a/tests/inline/inline_unittests.c b/tests/inline/inline_unittests.c new file mode 100644 index 00000000..b5834c7e --- /dev/null +++ b/tests/inline/inline_unittests.c @@ -0,0 +1,73 @@ +#include +#include +#include +#include +#include "unittests.h" +#include "inline.pb.h" + +int main() +{ + int status = 0; + int i = 0; + COMMENT("Test inline byte fields"); + + { + Message1 msg1 = Message1_init_zero; + TEST(sizeof(msg1.data) == 32); + } + + { + Message1 msg1 = Message1_init_zero; + pb_byte_t msg1_buffer[Message1_size]; + pb_ostream_t ostream = pb_ostream_from_buffer(msg1_buffer, Message1_size); + Message1 msg1_deserialized = Message1_init_zero; + pb_istream_t istream = pb_istream_from_buffer(msg1_buffer, Message1_size); + + for (i = 0; i < 32; i++) { + msg1.data[i] = i; + } + + TEST(pb_encode(&ostream, Message1_fields, &msg1)); + TEST(ostream.bytes_written == Message1_size); + + TEST(pb_decode(&istream, Message1_fields, &msg1_deserialized)); + + TEST(istream.bytes_left == 0); + TEST(memcmp(&msg1_deserialized, &msg1, sizeof(msg1)) == 0); + } + + { + Message2 msg2 = {true, {0}}; + Message2 msg2_no_data = {false, {1}}; + pb_byte_t msg2_buffer[Message2_size]; + pb_ostream_t ostream = pb_ostream_from_buffer(msg2_buffer, Message2_size); + Message2 msg2_deserialized = Message2_init_zero; + pb_istream_t istream = pb_istream_from_buffer(msg2_buffer, Message2_size); + + for (i = 0; i < 64; i++) { + msg2.data[i] = i; + } + + TEST(pb_encode(&ostream, Message2_fields, &msg2)); + TEST(ostream.bytes_written == Message2_size); + + TEST(pb_decode(&istream, Message2_fields, &msg2_deserialized)); + + TEST(istream.bytes_left == 0); + TEST(memcmp(&msg2_deserialized, &msg2, sizeof(msg2)) == 0); + TEST(msg2_deserialized.has_data); + + memset(msg2_buffer, 0, sizeof(msg2_buffer)); + ostream = pb_ostream_from_buffer(msg2_buffer, Message2_size); + TEST(pb_encode(&ostream, Message2_fields, &msg2_no_data)); + istream = pb_istream_from_buffer(msg2_buffer, Message2_size); + TEST(pb_decode(&istream, Message2_fields, &msg2_deserialized)); + TEST(!msg2_deserialized.has_data); + TEST(memcmp(&msg2_deserialized, &msg2, sizeof(msg2)) != 0); + } + + if (status != 0) + fprintf(stdout, "\n\nSome tests FAILED!\n"); + + return status; +} -- cgit 1.2.3-korg From bc2c47ef77a29e7efdaf088b9a0f7bf15a53c976 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 6 Aug 2016 10:42:04 +0300 Subject: Fix formatting error in docs --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index c43d8299..ef880cff 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -149,7 +149,7 @@ Most Protocol Buffers datatypes have directly corresponding C datatypes, such as 1) Strings, bytes and repeated fields of any type map to callback functions by default. 2) If there is a special option *(nanopb).max_size* specified in the .proto file, string maps to null-terminated char array and bytes map to a structure containing a char array and a size field. 3) If *(nanopb).type* is set to *FT_INLINE* and *(nanopb).max_size* is also set, then bytes map to an inline byte array of fixed size. -3) If there is a special option *(nanopb).max_count* specified on a repeated field, it maps to an array of whatever type is being repeated. Another field will be created for the actual number of entries stored. +4) If there is a special option *(nanopb).max_count* specified on a repeated field, it maps to an array of whatever type is being repeated. Another field will be created for the actual number of entries stored. =============================================================================== ======================= field in .proto autogenerated in .h -- cgit 1.2.3-korg From 4710f9cef63e71f39100183f757d78add0ed7855 Mon Sep 17 00:00:00 2001 From: Piotr Sikora Date: Wed, 10 Aug 2016 01:27:20 -0700 Subject: Bazel: export LICENSE.txt file. Signed-off-by: Piotr Sikora --- BUILD | 3 +++ 1 file changed, 3 insertions(+) diff --git a/BUILD b/BUILD index 57098843..f9fc57f5 100644 --- a/BUILD +++ b/BUILD @@ -1,4 +1,7 @@ licenses(["notice"]) + +exports_files(["LICENSE.txt"]) + package(default_visibility = ["//visibility:public"]) cc_library( -- cgit 1.2.3-korg From 91bb64a47b36b112c9b22391ef76fab29cf2cffc Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 2 Sep 2016 07:21:24 +0300 Subject: Fix typo in docs (thanks to @alecdavis) --- docs/concepts.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index ef880cff..ea33863c 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -164,7 +164,7 @@ required bytes data = 1 [(nanopb).max_size = 40]; | pb_byte_t bytes[40]; | } Person_data_t; | Person_data_t data; -required bytes data = 1 [(nanopb).max_size = 40, (nanopb.type) = FT_INLINE]; | pb_byte_t data[40]; +required bytes data = 1 [(nanopb).max_size = 40, (nanopb).type = FT_INLINE]; | pb_byte_t data[40]; =============================================================================== ======================= The maximum lengths are checked in runtime. If string/bytes/array exceeds the allocated length, *pb_decode* will return false. -- cgit 1.2.3-korg From ba97926cfd1e82bd294cf5e484633850d8e0b368 Mon Sep 17 00:00:00 2001 From: Bernhard Krämer Date: Sun, 9 Oct 2016 20:26:28 +0200 Subject: Add proto3 option to handle singular fields --- generator/nanopb_generator.py | 12 +++++++----- generator/proto/nanopb.proto | 3 +++ pb.h | 13 +++++++++++++ pb_decode.c | 3 ++- 4 files changed, 25 insertions(+), 6 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 973c7610..185a97bb 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -258,16 +258,18 @@ class Field: # Check field rules, i.e. required/optional/repeated. can_be_static = True - if desc.label == FieldD.LABEL_REQUIRED: - self.rules = 'REQUIRED' - elif desc.label == FieldD.LABEL_OPTIONAL: - self.rules = 'OPTIONAL' - elif desc.label == FieldD.LABEL_REPEATED: + if desc.label == FieldD.LABEL_REPEATED: self.rules = 'REPEATED' if self.max_count is None: can_be_static = False else: self.array_decl = '[%d]' % self.max_count + elif field_options.HasField("proto3"): + self.rules = 'SINGULAR' + elif desc.label == FieldD.LABEL_REQUIRED: + self.rules = 'REQUIRED' + elif desc.label == FieldD.LABEL_OPTIONAL: + self.rules = 'OPTIONAL' else: raise NotImplementedError(desc.label) diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index 8aab19a1..b9961c88 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -66,6 +66,9 @@ message NanoPBOptions { // decode oneof as anonymous union optional bool anonymous_oneof = 11 [default = false]; + + // Proto3 singular field does not generate a "has_" flag + optional bool proto3 = 12 [default = false]; } // Extensions to protoc 'Descriptor' type in order to define options diff --git a/pb.h b/pb.h index 4576f79a..0e1e92d5 100644 --- a/pb.h +++ b/pb.h @@ -413,6 +413,10 @@ struct pb_extension_s { pb_delta(st, has_ ## m, m), \ pb_membersize(st, m), 0, ptr} +#define PB_SINGULAR_STATIC(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + /* Repeated fields have a _count field and also the maximum number of entries. */ #define PB_REPEATED_STATIC(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_STATIC | PB_HTYPE_REPEATED | ltype, \ @@ -444,6 +448,11 @@ struct pb_extension_s { {tag, PB_ATYPE_POINTER | PB_HTYPE_OPTIONAL | ltype, \ fd, 0, pb_membersize(st, m[0]), 0, ptr} +/* Same as optional fields*/ +#define PB_SINGULAR_POINTER(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m[0]), 0, ptr} + /* Repeated fields have a _count field and a pointer to array of pointers */ #define PB_REPEATED_POINTER(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_POINTER | PB_HTYPE_REPEATED | ltype, \ @@ -458,6 +467,10 @@ struct pb_extension_s { #define PB_OPTIONAL_CALLBACK(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ fd, 0, pb_membersize(st, m), 0, ptr} + +#define PB_SINGULAR_CALLBACK(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} #define PB_REPEATED_CALLBACK(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REPEATED | ltype, \ diff --git a/pb_decode.c b/pb_decode.c index 7a4e29a8..1f6aeae0 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -360,7 +360,8 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t return func(stream, iter->pos, iter->pData); case PB_HTYPE_OPTIONAL: - *(bool*)iter->pSize = true; + if (iter->pSize != iter->pData) + *(bool*)iter->pSize = true; return func(stream, iter->pos, iter->pData); case PB_HTYPE_REPEATED: -- cgit 1.2.3-korg From ee44d0cee9fa87891fdc5371578f6ff3974a8d59 Mon Sep 17 00:00:00 2001 From: Bernhard Krämer Date: Sun, 9 Oct 2016 20:27:28 +0200 Subject: Prevent fields with default value from encoding when proto3 option is set --- pb_encode.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pb_encode.c b/pb_encode.c index 4685614c..4f57fa5f 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -210,6 +210,23 @@ static bool checkreturn encode_basic_field(pb_ostream_t *stream, if (field->size_offset) pSize = (const char*)pData + field->size_offset; + else if (!field->size_offset && PB_HTYPE(field->type) == PB_HTYPE_OPTIONAL) + { + /* In proto3 there are optional fields but no has_ flag, do not encode this fields + * when value is default or empty. */ + if(PB_LTYPE(field->type) == PB_LTYPE_BYTES){ + const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)pData; + if(bytes->size == 0) + implicit_has = false; + else if ((PB_LTYPE(field->type) == PB_LTYPE_STRING && *(const char*)pData == '\0') || + (field->data_size == sizeof(uint_least8_t) && *(const uint_least8_t*)pData == 0) || + (field->data_size == sizeof(uint_least16_t) && *(const uint_least16_t*)pData == 0) || + (field->data_size == sizeof(uint32_t) && *(const uint_least32_t*)pData == 0) || + (field->data_size == sizeof(uint64_t) && *(const uint_least64_t*)pData == 0)) + implicit_has = false; + } + pSize = &implicit_has; + } else pSize = &implicit_has; -- cgit 1.2.3-korg From be75cf4d6002937f7767054ac5e540e6640d7628 Mon Sep 17 00:00:00 2001 From: berni155 Date: Mon, 10 Oct 2016 21:53:53 +0200 Subject: Fix typo in encoding --- pb_encode.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pb_encode.c b/pb_encode.c index 4f57fa5f..b0a736a2 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -214,15 +214,18 @@ static bool checkreturn encode_basic_field(pb_ostream_t *stream, { /* In proto3 there are optional fields but no has_ flag, do not encode this fields * when value is default or empty. */ - if(PB_LTYPE(field->type) == PB_LTYPE_BYTES){ + if(PB_LTYPE(field->type) == PB_LTYPE_BYTES) + { const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)pData; if(bytes->size == 0) implicit_has = false; + } else if ((PB_LTYPE(field->type) == PB_LTYPE_STRING && *(const char*)pData == '\0') || (field->data_size == sizeof(uint_least8_t) && *(const uint_least8_t*)pData == 0) || (field->data_size == sizeof(uint_least16_t) && *(const uint_least16_t*)pData == 0) || (field->data_size == sizeof(uint32_t) && *(const uint_least32_t*)pData == 0) || (field->data_size == sizeof(uint64_t) && *(const uint_least64_t*)pData == 0)) + { implicit_has = false; } pSize = &implicit_has; -- cgit 1.2.3-korg From 6e22ecdebefa40d87723e46298a2d71329ab75da Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 23 Oct 2016 14:01:10 +0300 Subject: Fix missing warning with large bytes fields (issue #220) Need to generate compile time check if the bytes field + size field might exceed 255 bytes. Also eliminated spurious checks generated for some callback fields. --- generator/nanopb_generator.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 185a97bb..5c53a633 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -530,8 +530,8 @@ class Field: '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly. Returns numeric value or a C-expression for assert.''' check = [] - if self.pbtype == 'MESSAGE': - if self.rules == 'REPEATED' and self.allocation == 'STATIC': + if self.pbtype == 'MESSAGE' and self.allocation == 'STATIC': + if self.rules == 'REPEATED': check.append('pb_membersize(%s, %s[0])' % (self.struct_name, self.name)) elif self.rules == 'ONEOF': if self.anonymous: @@ -540,6 +540,9 @@ class Field: check.append('pb_membersize(%s, %s.%s)' % (self.struct_name, self.union_name, self.name)) else: check.append('pb_membersize(%s, %s)' % (self.struct_name, self.name)) + elif self.pbtype == 'BYTES' and self.allocation == 'STATIC': + if self.max_size > 251: + check.append('pb_membersize(%s, %s)' % (self.struct_name, self.name)) return FieldMaxSize([self.tag, self.max_size, self.max_count], check, -- cgit 1.2.3-korg From f866af7db7af0c42e5394a714a61513d189f2ba3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 23 Oct 2016 14:43:24 +0300 Subject: Set the proto3 option by default if file specifies proto3 syntax --- generator/nanopb_generator.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 5c53a633..3e543942 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1356,6 +1356,9 @@ def get_nanopb_suboptions(subdesc, options, name): Globals.matched_namemasks.add(namemask) new_options.MergeFrom(options) + if hasattr(subdesc, 'syntax') and subdesc.syntax == "proto3": + new_options.proto3 = True + # Handle options defined in .proto if isinstance(subdesc.options, descriptor.FieldOptions): ext_type = nanopb_pb2.nanopb -- cgit 1.2.3-korg From e9667c1856411ab9368fb440276463c4c8f2f4ff Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 23 Oct 2016 14:43:59 +0300 Subject: Add test cases for proto3 has_ field support. --- tests/alltypes_proto3/SConscript | 35 +++++++ tests/alltypes_proto3/alltypes.options | 3 + tests/alltypes_proto3/alltypes.proto | 98 ++++++++++++++++++++ tests/alltypes_proto3/decode_alltypes.c | 158 ++++++++++++++++++++++++++++++++ tests/alltypes_proto3/encode_alltypes.c | 107 +++++++++++++++++++++ tests/options/options.expected | 1 + tests/options/options.proto | 6 ++ 7 files changed, 408 insertions(+) create mode 100644 tests/alltypes_proto3/SConscript create mode 100644 tests/alltypes_proto3/alltypes.options create mode 100644 tests/alltypes_proto3/alltypes.proto create mode 100644 tests/alltypes_proto3/decode_alltypes.c create mode 100644 tests/alltypes_proto3/encode_alltypes.c diff --git a/tests/alltypes_proto3/SConscript b/tests/alltypes_proto3/SConscript new file mode 100644 index 00000000..6c6238c6 --- /dev/null +++ b/tests/alltypes_proto3/SConscript @@ -0,0 +1,35 @@ +# Build and run a test that encodes and decodes a message that contains +# all of the Protocol Buffers data types. + +Import("env") + +env.NanopbProto(["alltypes", "alltypes.options"]) +enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) + +# Test the round-trip from nanopb encoder to nanopb decoder +env.RunTest(enc) +env.RunTest([dec, "encode_alltypes.output"]) + +# Re-encode the data using protoc, and check that the results from nanopb +# match byte-per-byte to the protoc output. +env.Decode("encode_alltypes.output.decoded", + ["encode_alltypes.output", "alltypes.proto"], + MESSAGE='AllTypes') +env.Encode("encode_alltypes.output.recoded", + ["encode_alltypes.output.decoded", "alltypes.proto"], + MESSAGE='AllTypes') +env.Compare(["encode_alltypes.output", "encode_alltypes.output.recoded"]) + +# Do the same checks with the optional fields present. +env.RunTest("optionals.output", enc, ARGS = ['1']) +env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) +env.Decode("optionals.output.decoded", + ["optionals.output", "alltypes.proto"], + MESSAGE='AllTypes') +env.Encode("optionals.output.recoded", + ["optionals.output.decoded", "alltypes.proto"], + MESSAGE='AllTypes') +env.Compare(["optionals.output", "optionals.output.recoded"]) + + diff --git a/tests/alltypes_proto3/alltypes.options b/tests/alltypes_proto3/alltypes.options new file mode 100644 index 00000000..b31e3cf0 --- /dev/null +++ b/tests/alltypes_proto3/alltypes.options @@ -0,0 +1,3 @@ +* max_size:16 +* max_count:5 + diff --git a/tests/alltypes_proto3/alltypes.proto b/tests/alltypes_proto3/alltypes.proto new file mode 100644 index 00000000..10b48f25 --- /dev/null +++ b/tests/alltypes_proto3/alltypes.proto @@ -0,0 +1,98 @@ +syntax = "proto3"; +// package name placeholder + +message SubMessage { + string substuff1 = 1; + int32 substuff2 = 2; + fixed32 substuff3 = 3; +} + +message EmptyMessage { + +} + +enum HugeEnum { + HE_Zero = 0; + Negative = -2147483647; /* protoc doesn't accept -2147483648 here */ + Positive = 2147483647; +} + +message Limits { + int32 int32_min = 1; + int32 int32_max = 2; + uint32 uint32_min = 3; + uint32 uint32_max = 4; + int64 int64_min = 5; + int64 int64_max = 6; + uint64 uint64_min = 7; + uint64 uint64_max = 8; + HugeEnum enum_min = 9; + HugeEnum enum_max = 10; +} + +enum MyEnum { + Zero = 0; + First = 1; + Second = 2; + Truth = 42; +} + +message AllTypes { + int32 sng_int32 = 1; + int64 sng_int64 = 2; + uint32 sng_uint32 = 3; + uint64 sng_uint64 = 4; + sint32 sng_sint32 = 5; + sint64 sng_sint64 = 6; + bool sng_bool = 7; + + fixed32 sng_fixed32 = 8; + sfixed32 sng_sfixed32= 9; + float sng_float = 10; + + fixed64 sng_fixed64 = 11; + sfixed64 sng_sfixed64= 12; + double sng_double = 13; + + string sng_string = 14; + bytes sng_bytes = 15; + SubMessage sng_submsg = 16; + MyEnum sng_enum = 17; + EmptyMessage sng_emptymsg = 18; + + repeated int32 rep_int32 = 21 [packed = true]; + repeated int64 rep_int64 = 22 [packed = true]; + repeated uint32 rep_uint32 = 23 [packed = true]; + repeated uint64 rep_uint64 = 24 [packed = true]; + repeated sint32 rep_sint32 = 25 [packed = true]; + repeated sint64 rep_sint64 = 26 [packed = true]; + repeated bool rep_bool = 27 [packed = true]; + + repeated fixed32 rep_fixed32 = 28 [packed = true]; + repeated sfixed32 rep_sfixed32= 29 [packed = true]; + repeated float rep_float = 30 [packed = true]; + + repeated fixed64 rep_fixed64 = 31 [packed = true]; + repeated sfixed64 rep_sfixed64= 32 [packed = true]; + repeated double rep_double = 33 [packed = true]; + + repeated string rep_string = 34; + repeated bytes rep_bytes = 35; + repeated SubMessage rep_submsg = 36; + repeated MyEnum rep_enum = 37 [packed = true]; + repeated EmptyMessage rep_emptymsg = 38; + + oneof oneof + { + SubMessage oneof_msg1 = 59; + EmptyMessage oneof_msg2 = 60; + } + + // Check that extreme integer values are handled correctly + Limits req_limits = 98; + + // Just to make sure that the size of the fields has been calculated + // properly, i.e. otherwise a bug in last field might not be detected. + int32 end = 99; +} + diff --git a/tests/alltypes_proto3/decode_alltypes.c b/tests/alltypes_proto3/decode_alltypes.c new file mode 100644 index 00000000..c1b0d524 --- /dev/null +++ b/tests/alltypes_proto3/decode_alltypes.c @@ -0,0 +1,158 @@ +/* Tests the decoding of all types. + * This is the counterpart of test_encode3. + * Run e.g. ./test_encode3 | ./test_decode3 + */ + +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "test_helpers.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed.\n"); \ + return false; \ + } + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_alltypes(pb_istream_t *stream, int mode) +{ + AllTypes alltypes = AllTypes_init_zero; + + /* Fill with garbage to better detect initialization errors */ + memset(&alltypes, 0xAA, sizeof(alltypes)); + + if (!pb_decode(stream, AllTypes_fields, &alltypes)) + return false; + + TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); + TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); + TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); + TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); + TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); + TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0); + TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); + + TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); + TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); + TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); + + TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); + TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0); + TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); + + TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); + TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); + TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); + + TEST(alltypes.rep_submsg_count == 5); + TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); + TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); + TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 0); + + TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); + TEST(alltypes.rep_emptymsg_count == 5); + + if (mode == 0) + { + /* Expect default values */ + TEST(alltypes.sng_int32 == 0); + TEST(alltypes.sng_int64 == 0); + TEST(alltypes.sng_uint32 == 0); + TEST(alltypes.sng_uint64 == 0); + TEST(alltypes.sng_sint32 == 0); + TEST(alltypes.sng_sint64 == 0); + TEST(alltypes.sng_bool == false); + + TEST(alltypes.sng_fixed32 == 0); + TEST(alltypes.sng_sfixed32 == 0); + TEST(alltypes.sng_float == 0.0f); + + TEST(alltypes.sng_fixed64 == 0); + TEST(alltypes.sng_sfixed64 == 0); + TEST(alltypes.sng_double == 0.0); + + TEST(strcmp(alltypes.sng_string, "") == 0); + TEST(alltypes.sng_bytes.size == 0); + TEST(strcmp(alltypes.sng_submsg.substuff1, "") == 0); + TEST(alltypes.sng_submsg.substuff2 == 0); + TEST(alltypes.sng_submsg.substuff3 == 0); + TEST(alltypes.sng_enum == MyEnum_Zero); + + TEST(alltypes.which_oneof == 0); + } + else + { + /* Expect filled-in values */ + TEST(alltypes.sng_int32 == 3041); + TEST(alltypes.sng_int64 == 3042); + TEST(alltypes.sng_uint32 == 3043); + TEST(alltypes.sng_uint64 == 3044); + TEST(alltypes.sng_sint32 == 3045); + TEST(alltypes.sng_sint64 == 3046); + TEST(alltypes.sng_bool == true); + + TEST(alltypes.sng_fixed32 == 3048); + TEST(alltypes.sng_sfixed32 == 3049); + TEST(alltypes.sng_float == 3050.0f); + + TEST(alltypes.sng_fixed64 == 3051); + TEST(alltypes.sng_sfixed64 == 3052); + TEST(alltypes.sng_double == 3053.0); + + TEST(strcmp(alltypes.sng_string, "3054") == 0); + TEST(alltypes.sng_bytes.size == 4); + TEST(memcmp(alltypes.sng_bytes.bytes, "3055", 4) == 0); + TEST(strcmp(alltypes.sng_submsg.substuff1, "3056") == 0); + TEST(alltypes.sng_submsg.substuff2 == 3056); + TEST(alltypes.sng_submsg.substuff3 == 0); + TEST(alltypes.sng_enum == MyEnum_Truth); + + TEST(alltypes.which_oneof == AllTypes_oneof_msg1_tag); + TEST(strcmp(alltypes.oneof.oneof_msg1.substuff1, "4059") == 0); + TEST(alltypes.oneof.oneof_msg1.substuff2 == 4059); + } + + TEST(alltypes.req_limits.int32_min == INT32_MIN); + TEST(alltypes.req_limits.int32_max == INT32_MAX); + TEST(alltypes.req_limits.uint32_min == 0); + TEST(alltypes.req_limits.uint32_max == UINT32_MAX); + TEST(alltypes.req_limits.int64_min == INT64_MIN); + TEST(alltypes.req_limits.int64_max == INT64_MAX); + TEST(alltypes.req_limits.uint64_min == 0); + TEST(alltypes.req_limits.uint64_max == UINT64_MAX); + TEST(alltypes.req_limits.enum_min == HugeEnum_Negative); + TEST(alltypes.req_limits.enum_max == HugeEnum_Positive); + + TEST(alltypes.end == 1099); + + return true; +} + +int main(int argc, char **argv) +{ + uint8_t buffer[1024]; + size_t count; + pb_istream_t stream; + + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Read the data into buffer */ + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_alltypes(&stream, mode)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} diff --git a/tests/alltypes_proto3/encode_alltypes.c b/tests/alltypes_proto3/encode_alltypes.c new file mode 100644 index 00000000..e11acd5e --- /dev/null +++ b/tests/alltypes_proto3/encode_alltypes.c @@ -0,0 +1,107 @@ +/* Attempts to test all the datatypes supported by ProtoBuf3. + */ + +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "test_helpers.h" + +int main(int argc, char **argv) +{ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Initialize the structure with constants */ + AllTypes alltypes = AllTypes_init_zero; + + alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; + alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002; + alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; + alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004; + alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005; + alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006; + alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; + + alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; + alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009; + alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; + + alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011; + alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012; + alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0; + + alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); + alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4; + memcpy(alltypes.rep_bytes[4].bytes, "2015", 4); + + alltypes.rep_submsg_count = 5; + strcpy(alltypes.rep_submsg[4].substuff1, "2016"); + alltypes.rep_submsg[4].substuff2 = 2016; + alltypes.rep_submsg[4].substuff3 = 2016; + + alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; + alltypes.rep_emptymsg_count = 5; + + alltypes.req_limits.int32_min = INT32_MIN; + alltypes.req_limits.int32_max = INT32_MAX; + alltypes.req_limits.uint32_min = 0; + alltypes.req_limits.uint32_max = UINT32_MAX; + alltypes.req_limits.int64_min = INT64_MIN; + alltypes.req_limits.int64_max = INT64_MAX; + alltypes.req_limits.uint64_min = 0; + alltypes.req_limits.uint64_max = UINT64_MAX; + alltypes.req_limits.enum_min = HugeEnum_Negative; + alltypes.req_limits.enum_max = HugeEnum_Positive; + + if (mode != 0) + { + /* Fill in values for singular fields */ + alltypes.sng_int32 = 3041; + alltypes.sng_int64 = 3042; + alltypes.sng_uint32 = 3043; + alltypes.sng_uint64 = 3044; + alltypes.sng_sint32 = 3045; + alltypes.sng_sint64 = 3046; + alltypes.sng_bool = true; + + alltypes.sng_fixed32 = 3048; + alltypes.sng_sfixed32 = 3049; + alltypes.sng_float = 3050.0f; + + alltypes.sng_fixed64 = 3051; + alltypes.sng_sfixed64 = 3052; + alltypes.sng_double = 3053.0; + + strcpy(alltypes.sng_string, "3054"); + alltypes.sng_bytes.size = 4; + memcpy(alltypes.sng_bytes.bytes, "3055", 4); + strcpy(alltypes.sng_submsg.substuff1, "3056"); + alltypes.sng_submsg.substuff2 = 3056; + alltypes.sng_enum = MyEnum_Truth; + + alltypes.which_oneof = AllTypes_oneof_msg1_tag; + strcpy(alltypes.oneof.oneof_msg1.substuff1, "4059"); + alltypes.oneof.oneof_msg1.substuff2 = 4059; + } + + alltypes.end = 1099; + + { + uint8_t buffer[AllTypes_size]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } + } +} diff --git a/tests/options/options.expected b/tests/options/options.expected index 63ba0fd8..0769880a 100644 --- a/tests/options/options.expected +++ b/tests/options/options.expected @@ -15,4 +15,5 @@ Message5_EnumValue1 \s+PB_MSG\(104,-1,Message4\) \\ \s+PB_MSG\(105,[0-9]*,Message5\) \\ #define Message5_msgid 105 +! has_proto3field diff --git a/tests/options/options.proto b/tests/options/options.proto index aa722b52..89bb086f 100644 --- a/tests/options/options.proto +++ b/tests/options/options.proto @@ -89,3 +89,9 @@ message OneofMessage } } +// Proto3-style optional field in proto2 file +message Proto3Field +{ + optional int32 proto3field = 1 [(nanopb).proto3 = true]; +} + -- cgit 1.2.3-korg From fb8ced6b3412528442c20aca31e2a21b792eb07e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 23 Oct 2016 18:01:20 +0300 Subject: Update changelog and authors list --- AUTHORS | 4 ++++ CHANGELOG.txt | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/AUTHORS b/AUTHORS index a967d3ec..53d31d9e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -22,3 +22,7 @@ Yaniv Mordekhay Ming Zhao Google, Inc. Tom Roeder +Piotr Sikora +Bernhard Krämer +Konstantin Podsvirov + diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 8437688f..9156564b 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,14 @@ +nanopb-0.3.7 (2016-10-xx) + Add support for proto3-style singular fields (#182, #206, #216) + Add FT_INLINE allocation of bytes fields (#211) + Include package name in include guard (#207) + Fix missing warning with large bytes fields (issue #220) + Added CMake project (#208) + Add bazel BUILD file for nanopb (#209) + Added an AUTHORS file (#211) + Documentation updates + Improvements to test cases. + nanopb-0.3.6 (2016-06-19) Protect against corrupted _count fields in pb_release (#205) Fix error in STATIC_ASSERT with multiple files (#203) -- cgit 1.2.3-korg From b850cf9fb101e0fe7fd71150210c6ff9877e57c1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 23 Oct 2016 18:40:50 +0300 Subject: Only run alltypes_proto3 test case if protoc version is new enough --- tests/alltypes_proto3/SConscript | 70 +++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/tests/alltypes_proto3/SConscript b/tests/alltypes_proto3/SConscript index 6c6238c6..c0b2fc1b 100644 --- a/tests/alltypes_proto3/SConscript +++ b/tests/alltypes_proto3/SConscript @@ -1,35 +1,45 @@ -# Build and run a test that encodes and decodes a message that contains -# all of the Protocol Buffers data types. +# Version of AllTypes test case for protobuf 3 file format. Import("env") -env.NanopbProto(["alltypes", "alltypes.options"]) -enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) -dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) - -# Test the round-trip from nanopb encoder to nanopb decoder -env.RunTest(enc) -env.RunTest([dec, "encode_alltypes.output"]) - -# Re-encode the data using protoc, and check that the results from nanopb -# match byte-per-byte to the protoc output. -env.Decode("encode_alltypes.output.decoded", - ["encode_alltypes.output", "alltypes.proto"], - MESSAGE='AllTypes') -env.Encode("encode_alltypes.output.recoded", - ["encode_alltypes.output.decoded", "alltypes.proto"], - MESSAGE='AllTypes') -env.Compare(["encode_alltypes.output", "encode_alltypes.output.recoded"]) - -# Do the same checks with the optional fields present. -env.RunTest("optionals.output", enc, ARGS = ['1']) -env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) -env.Decode("optionals.output.decoded", - ["optionals.output", "alltypes.proto"], - MESSAGE='AllTypes') -env.Encode("optionals.output.recoded", - ["optionals.output.decoded", "alltypes.proto"], - MESSAGE='AllTypes') -env.Compare(["optionals.output", "optionals.output.recoded"]) +import re +match = None +if 'PROTOC_VERSION' in env: + match = re.search('([0-9]+).([0-9]+).([0-9]+)', env['PROTOC_VERSION']) + +if match: + version = map(int, match.groups()) + +# proto3 syntax is supported by protoc >= 3.0.0 +if env.GetOption('clean') or (match and version[0] >= 3): + + env.NanopbProto(["alltypes", "alltypes.options"]) + enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) + dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) + + # Test the round-trip from nanopb encoder to nanopb decoder + env.RunTest(enc) + env.RunTest([dec, "encode_alltypes.output"]) + + # Re-encode the data using protoc, and check that the results from nanopb + # match byte-per-byte to the protoc output. + env.Decode("encode_alltypes.output.decoded", + ["encode_alltypes.output", "alltypes.proto"], + MESSAGE='AllTypes') + env.Encode("encode_alltypes.output.recoded", + ["encode_alltypes.output.decoded", "alltypes.proto"], + MESSAGE='AllTypes') + env.Compare(["encode_alltypes.output", "encode_alltypes.output.recoded"]) + + # Do the same checks with the optional fields present. + env.RunTest("optionals.output", enc, ARGS = ['1']) + env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) + env.Decode("optionals.output.decoded", + ["optionals.output", "alltypes.proto"], + MESSAGE='AllTypes') + env.Encode("optionals.output.recoded", + ["optionals.output.decoded", "alltypes.proto"], + MESSAGE='AllTypes') + env.Compare(["optionals.output", "optionals.output.recoded"]) -- cgit 1.2.3-korg From 89aaa43cfbfa081a4c580d63b6d01e7dc017d1df Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 24 Oct 2016 12:26:55 +0300 Subject: Strip debug symbols from linux library files --- tools/make_linux_package.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/make_linux_package.sh b/tools/make_linux_package.sh index 6598936a..d45f9856 100755 --- a/tools/make_linux_package.sh +++ b/tools/make_linux_package.sh @@ -42,6 +42,9 @@ exec "\$SCRIPTDIR/protoc.bin" "\$@" EOF chmod +x $DEST/generator-bin/protoc +# Remove debugging symbols to reduce size of package +( cd $DEST; strip *.so ) + # Tar it all up ( cd dist; tar -czf $VERSION.tar.gz $VERSION ) -- cgit 1.2.3-korg From 8d55dcc56fac67bd1166f13a5bddf3c8b2acbe3c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 24 Oct 2016 12:30:21 +0300 Subject: Build fix for previous commit --- tools/make_linux_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/make_linux_package.sh b/tools/make_linux_package.sh index d45f9856..1402f3d3 100755 --- a/tools/make_linux_package.sh +++ b/tools/make_linux_package.sh @@ -43,7 +43,7 @@ EOF chmod +x $DEST/generator-bin/protoc # Remove debugging symbols to reduce size of package -( cd $DEST; strip *.so ) +( cd $DEST/generator-bin; strip *.so ) # Tar it all up ( cd dist; tar -czf $VERSION.tar.gz $VERSION ) -- cgit 1.2.3-korg From ba6d86bdcd017195fc5780fe6882569984d0957a Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 24 Oct 2016 12:35:59 +0300 Subject: Build fix for previous commit --- tools/make_linux_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/make_linux_package.sh b/tools/make_linux_package.sh index 1402f3d3..aea1c567 100755 --- a/tools/make_linux_package.sh +++ b/tools/make_linux_package.sh @@ -43,7 +43,7 @@ EOF chmod +x $DEST/generator-bin/protoc # Remove debugging symbols to reduce size of package -( cd $DEST/generator-bin; strip *.so ) +( cd $DEST/generator-bin; strip *.so *.so.* ) # Tar it all up ( cd dist; tar -czf $VERSION.tar.gz $VERSION ) -- cgit 1.2.3-korg From 5105b251305ea3f20d0bc36c915488ba9fb3c93f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 24 Oct 2016 13:12:52 +0300 Subject: Note about protoc update --- CHANGELOG.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 9156564b..3d54b7b6 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,5 +1,6 @@ nanopb-0.3.7 (2016-10-xx) Add support for proto3-style singular fields (#182, #206, #216) + Updated binary package protoc to version 3.1.0 Add FT_INLINE allocation of bytes fields (#211) Include package name in include guard (#207) Fix missing warning with large bytes fields (issue #220) -- cgit 1.2.3-korg From 60d8ba229b58f07fdefc31e2756402cc74cede5f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 30 Oct 2016 07:30:49 +0200 Subject: Publishing nanopb-0.3.7 --- CHANGELOG.txt | 2 +- CMakeLists.txt | 2 +- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 3d54b7b6..02b7e71b 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,4 +1,4 @@ -nanopb-0.3.7 (2016-10-xx) +nanopb-0.3.7 (2016-10-30) Add support for proto3-style singular fields (#182, #206, #216) Updated binary package protoc to version 3.1.0 Add FT_INLINE allocation of bytes fields (#211) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b19f287..a09e97c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8) project(nanopb C) -set(nanopb_VERSION_STRING nanopb-0.3.7-dev) +set(nanopb_VERSION_STRING nanopb-0.3.7) string(REPLACE "nanopb-" "" nanopb_VERSION ${nanopb_VERSION_STRING}) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 3e543942..351c37e3 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.7-dev" +nanopb_version = "nanopb-0.3.7" import sys import re diff --git a/pb.h b/pb.h index 0e1e92d5..9306a86c 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.7-dev +#define NANOPB_VERSION nanopb-0.3.7 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From af601e4e971075e1f53a26f6c1ab63320823d8c2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 30 Oct 2016 08:13:46 +0200 Subject: Setting version to 0.3.8-dev --- CMakeLists.txt | 2 +- generator/nanopb_generator.py | 2 +- pb.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a09e97c0..7889bf4a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8) project(nanopb C) -set(nanopb_VERSION_STRING nanopb-0.3.7) +set(nanopb_VERSION_STRING nanopb-0.3.8-dev) string(REPLACE "nanopb-" "" nanopb_VERSION ${nanopb_VERSION_STRING}) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 351c37e3..9ccb9b93 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.7" +nanopb_version = "nanopb-0.3.8-dev" import sys import re diff --git a/pb.h b/pb.h index 9306a86c..2cf5c4dc 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.7 +#define NANOPB_VERSION nanopb-0.3.8-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From dfa8652db405b08292f3238e05f7592be19fe392 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 30 Oct 2016 08:14:03 +0200 Subject: Change download links to https --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2d35e85d..a32d6ca2 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ Nanopb is a small code-size Protocol Buffers implementation in ansi C. It is especially suitable for use in microcontrollers, but fits any memory restricted system. -* **Homepage:** http://kapsi.fi/~jpa/nanopb/ -* **Documentation:** http://kapsi.fi/~jpa/nanopb/docs/ -* **Downloads:** http://kapsi.fi/~jpa/nanopb/download/ +* **Homepage:** https://koti.kapsi.fi/jpa/nanopb/ +* **Documentation:** https://koti.kapsi.fi/jpa/nanopb/docs/ +* **Downloads:** https://koti.kapsi.fi/jpa/nanopb/download/ * **Forum:** https://groups.google.com/forum/#!forum/nanopb -- cgit 1.2.3-korg From 82dd587c7b8d10150ab6683733863e2b7e34a428 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 19 Nov 2016 07:36:21 +0200 Subject: Include version number in PlatformIO library.json (issue 222) --- library.json | 1 + tools/set_version.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/library.json b/library.json index 30a56c94..1a0b07c1 100644 --- a/library.json +++ b/library.json @@ -1,5 +1,6 @@ { "name": "Nanopb", + "version": "0.3.8-dev", "keywords": "protocol buffers, protobuf, google", "description": "Nanopb is a plain-C implementation of Google's Protocol Buffers data format. It is targeted at 32 bit microcontrollers, but is also fit for other embedded systems with tight (2-10 kB ROM, <1 kB RAM) memory constraints.", "repository": { diff --git a/tools/set_version.sh b/tools/set_version.sh index f989308c..729ca0d7 100755 --- a/tools/set_version.sh +++ b/tools/set_version.sh @@ -8,4 +8,6 @@ sed -i -e 's/nanopb_version\s*=\s*"[^"]*"/nanopb_version = "'$1'"/' generator/na sed -i -e 's/#define\s*NANOPB_VERSION\s*.*/#define NANOPB_VERSION '$1'/' pb.h sed -i -e 's/set(\s*nanopb_VERSION_STRING\s*[^)]*)/set(nanopb_VERSION_STRING '$1')/' CMakeLists.txt +VERSION_ONLY=$(echo $1 | sed 's/nanopb-//') +sed -i -e 's/"version":\s*"[^"]*"/"version": "'$VERSION_ONLY'"/' library.json -- cgit 1.2.3-korg From ad6d66010ee072043c61a533730e0691def40061 Mon Sep 17 00:00:00 2001 From: "William A. Kennington III" Date: Wed, 26 Oct 2016 13:55:44 -0700 Subject: nanopb: update generator to emit optional enum->string mapping function Google-Bug-Id: 28000875 Signed-off-by: William A. Kennington III Change-Id: I1bffd39168abe04593588291b0ebbe5199a00138 --- generator/nanopb_generator.py | 24 ++++++++++++++++++++++++ generator/proto/nanopb.proto | 3 +++ 2 files changed, 27 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 9ccb9b93..36cbbdfa 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -207,6 +207,27 @@ class Enum: for i, x in enumerate(self.values): result += '\n#define %s %s' % (self.value_longnames[i], x[0]) + if self.options.enum_to_string: + result += '\nconst char *%s_Name(%s v);\n' % (self.names, self.names) + + return result + + def enum_definition(self): + if not self.options.enum_to_string: + return "" + + result = 'const char *%s_Name(%s v) {\n' % (self.names, self.names) + result += ' switch (v) {\n' + + for ((enumname, _), strname) in zip(self.values, self.value_longnames): + # Strip off the leading type name from the string value. + strval = str(strname)[len(str(self.names)) + 1:] + result += ' case %s: return "%s";\n' % (enumname, strval) + + result += ' }\n' + result += ' return "unknown";\n' + result += '}\n' + return result class FieldMaxSize: @@ -1220,6 +1241,9 @@ class ProtoFile: for ext in self.extensions: yield ext.extension_def() + '\n' + for enum in self.enums: + yield enum.enum_definition() + '\n' + # Add checks for numeric limits if self.messages: largest_msg = max(self.messages, key = lambda m: m.count_required_fields()) diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index b9961c88..f6fe4a20 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -69,6 +69,9 @@ message NanoPBOptions { // Proto3 singular field does not generate a "has_" flag optional bool proto3 = 12 [default = false]; + + // Generate an enum->string mapping function (can take up lots of space). + optional bool enum_to_string = 13 [default = false]; } // Extensions to protoc 'Descriptor' type in order to define options -- cgit 1.2.3-korg From 3a01eea8644576d47c2cc05ad3eac900d4b8e4f4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 22 Nov 2016 17:24:33 +0200 Subject: Small indentation and naming fixes to enum_to_string functionality --- generator/nanopb_generator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 36cbbdfa..066ef936 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -208,21 +208,21 @@ class Enum: result += '\n#define %s %s' % (self.value_longnames[i], x[0]) if self.options.enum_to_string: - result += '\nconst char *%s_Name(%s v);\n' % (self.names, self.names) + result += '\nconst char *%s_name(%s v);\n' % (self.names, self.names) return result - def enum_definition(self): + def enum_to_string_definition(self): if not self.options.enum_to_string: return "" - result = 'const char *%s_Name(%s v) {\n' % (self.names, self.names) + result = 'const char *%s_name(%s v) {\n' % (self.names, self.names) result += ' switch (v) {\n' for ((enumname, _), strname) in zip(self.values, self.value_longnames): # Strip off the leading type name from the string value. strval = str(strname)[len(str(self.names)) + 1:] - result += ' case %s: return "%s";\n' % (enumname, strval) + result += ' case %s: return "%s";\n' % (enumname, strval) result += ' }\n' result += ' return "unknown";\n' @@ -1242,7 +1242,7 @@ class ProtoFile: yield ext.extension_def() + '\n' for enum in self.enums: - yield enum.enum_definition() + '\n' + yield enum.enum_to_string_definition() + '\n' # Add checks for numeric limits if self.messages: -- cgit 1.2.3-korg From 928becdc52d7c8805b0e4259e0682df56e637da7 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 22 Nov 2016 17:25:00 +0200 Subject: Test case for enum_to_string --- tests/enum_to_string/SConscript | 7 +++++++ tests/enum_to_string/enum.proto | 19 +++++++++++++++++++ tests/enum_to_string/enum_to_string.c | 19 +++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 tests/enum_to_string/SConscript create mode 100644 tests/enum_to_string/enum.proto create mode 100644 tests/enum_to_string/enum_to_string.c diff --git a/tests/enum_to_string/SConscript b/tests/enum_to_string/SConscript new file mode 100644 index 00000000..e86fcca0 --- /dev/null +++ b/tests/enum_to_string/SConscript @@ -0,0 +1,7 @@ +# Test enum to string functionality + +Import('env') +env.NanopbProto("enum.proto") +p = env.Program(["enum_to_string.c", "enum.pb.c"]) +env.RunTest(p) + diff --git a/tests/enum_to_string/enum.proto b/tests/enum_to_string/enum.proto new file mode 100644 index 00000000..07c67363 --- /dev/null +++ b/tests/enum_to_string/enum.proto @@ -0,0 +1,19 @@ +/* Test enum to string function generation */ + +syntax = "proto2"; + +import "nanopb.proto"; + +option (nanopb_fileopt).enum_to_string = true; + +enum MyEnum { + VALUE1 = 1; + VALUE2 = 2; + VALUE15 = 15; +} + +enum MyShortNameEnum { + option (nanopb_enumopt).long_names = false; + MSNE_VALUE256 = 256; +} + diff --git a/tests/enum_to_string/enum_to_string.c b/tests/enum_to_string/enum_to_string.c new file mode 100644 index 00000000..c4fb31d4 --- /dev/null +++ b/tests/enum_to_string/enum_to_string.c @@ -0,0 +1,19 @@ +#include +#include "unittests.h" +#include "enum.pb.h" + +int main() +{ + int status = 0; + TEST(strcmp(MyEnum_name(MyEnum_VALUE1), "VALUE1") == 0); + TEST(strcmp(MyEnum_name(MyEnum_VALUE2), "VALUE2") == 0); + TEST(strcmp(MyEnum_name(MyEnum_VALUE15), "VALUE15") == 0); + TEST(strcmp(MyShortNameEnum_name(MSNE_VALUE256), "MSNE_VALUE256") == 0); + TEST(strcmp(MyShortNameEnum_name(9999), "unknown") == 0); + + if (status != 0) + fprintf(stdout, "\n\nSome tests FAILED!\n"); + + return status; +} + -- cgit 1.2.3-korg From 69aaa491fbe57e7dbb2eded28248a6e5b570e535 Mon Sep 17 00:00:00 2001 From: Guillaume Lager Date: Fri, 9 Dec 2016 10:02:08 +0100 Subject: Fix potential unaligned access If the type is string, do not try to deference it as int16, int32 or int64. This may lead to unalign memory access, which may cause trap on some architectures (ARM) --- pb_encode.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pb_encode.c b/pb_encode.c index b0a736a2..00c381c0 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -220,11 +220,15 @@ static bool checkreturn encode_basic_field(pb_ostream_t *stream, if(bytes->size == 0) implicit_has = false; } - else if ((PB_LTYPE(field->type) == PB_LTYPE_STRING && *(const char*)pData == '\0') || - (field->data_size == sizeof(uint_least8_t) && *(const uint_least8_t*)pData == 0) || - (field->data_size == sizeof(uint_least16_t) && *(const uint_least16_t*)pData == 0) || - (field->data_size == sizeof(uint32_t) && *(const uint_least32_t*)pData == 0) || - (field->data_size == sizeof(uint64_t) && *(const uint_least64_t*)pData == 0)) + else if (PB_LTYPE(field->type) == PB_LTYPE_STRING ) + { + if( *(const char*)pData == '\0') + implicit_has = false; + } + else if ((field->data_size == sizeof(uint_least8_t) && *(const uint_least8_t*)pData == 0) || + (field->data_size == sizeof(uint_least16_t) && *(const uint_least16_t*)pData == 0) || + (field->data_size == sizeof(uint32_t) && *(const uint_least32_t*)pData == 0) || + (field->data_size == sizeof(uint64_t) && *(const uint_least64_t*)pData == 0)) { implicit_has = false; } -- cgit 1.2.3-korg From bb78ba51642de29194ff70f6f7e602c393210e83 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 9 Dec 2016 18:37:55 +0200 Subject: Refactor proto3 logic into pb_check_proto3_default_value() Mainly to clean up the code. Also fixed handling of PB_LTYPE_FIXED_LENGTH_BYTES for proto3 files. --- pb_encode.c | 81 +++++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 26 deletions(-) diff --git a/pb_encode.c b/pb_encode.c index 00c381c0..13bda22c 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -197,45 +197,75 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie return true; } +/* In proto3, all fields are optional and are only encoded if their value is "non-zero". + * This function implements the check for the zero value. */ +static bool pb_check_proto3_default_value(const pb_field_t *field, const void *pData) +{ + if(PB_LTYPE(field->type) == PB_LTYPE_BYTES) + { + const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)pData; + return bytes->size == 0; + } + else if (PB_LTYPE(field->type) == PB_LTYPE_STRING) + { + return *(const char*)pData == '\0'; + } + else if (PB_LTYPE(field->type) == PB_LTYPE_FIXED_LENGTH_BYTES) + { + /* Fixed length bytes is only empty if its length is fixed + * as 0. Which would be pretty strange, but we can check + * it anyway. */ + return field->data_size == 0; + } + else + { + /* PB_LTYPE_VARINT, UVARINT, SVARINT, FIXED32, FIXED64, + * SUBMESSAGE, EXTENSION: These all have integer or pointer + * value which can be compared with 0. This does the check + * byte-by-byte to avoid the switch-cast logic used in + * pb_enc_varint(). (Casting to char* is safe with regards + * to C strict aliasing rules.) + */ + uint_fast8_t i; + const char *p = (const char*)pData; + for (i = 0; i < field->data_size; i++) + { + if (p[i] != 0) + { + return false; + } + } + + return true; + } +} + /* Encode a field with static or pointer allocation, i.e. one whose data * is available to the encoder directly. */ static bool checkreturn encode_basic_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData) { pb_encoder_t func; - const void *pSize; - bool implicit_has = true; + bool implicit_has; + const void *pSize = &implicit_has; func = PB_ENCODERS[PB_LTYPE(field->type)]; if (field->size_offset) + { + /* Static optional, repeated or oneof field */ pSize = (const char*)pData + field->size_offset; - else if (!field->size_offset && PB_HTYPE(field->type) == PB_HTYPE_OPTIONAL) + } + else if (PB_HTYPE(field->type) == PB_HTYPE_OPTIONAL) { - /* In proto3 there are optional fields but no has_ flag, do not encode this fields - * when value is default or empty. */ - if(PB_LTYPE(field->type) == PB_LTYPE_BYTES) - { - const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)pData; - if(bytes->size == 0) - implicit_has = false; - } - else if (PB_LTYPE(field->type) == PB_LTYPE_STRING ) - { - if( *(const char*)pData == '\0') - implicit_has = false; - } - else if ((field->data_size == sizeof(uint_least8_t) && *(const uint_least8_t*)pData == 0) || - (field->data_size == sizeof(uint_least16_t) && *(const uint_least16_t*)pData == 0) || - (field->data_size == sizeof(uint32_t) && *(const uint_least32_t*)pData == 0) || - (field->data_size == sizeof(uint64_t) && *(const uint_least64_t*)pData == 0)) - { - implicit_has = false; - } - pSize = &implicit_has; + /* Proto3 style field, optional but without explicit has_ field. */ + implicit_has = !pb_check_proto3_default_value(field, pData); } else - pSize = &implicit_has; + { + /* Required field, always present */ + implicit_has = true; + } if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { @@ -243,7 +273,6 @@ static bool checkreturn encode_basic_field(pb_ostream_t *stream, * the data. If the 2nd pointer is NULL, it is interpreted as if * the has_field was false. */ - pData = *(const void* const*)pData; implicit_has = (pData != NULL); } -- cgit 1.2.3-korg From d445c2b04d11aa949b5a8d870739aac534b57076 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 9 Dec 2016 18:50:20 +0200 Subject: Add regression test for issue #227 --- tests/SConstruct | 4 +++- tests/regression/issue_227/SConscript | 14 ++++++++++++++ tests/regression/issue_227/unaligned_uint64.c | 14 ++++++++++++++ tests/regression/issue_227/unaligned_uint64.proto | 8 ++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tests/regression/issue_227/SConscript create mode 100644 tests/regression/issue_227/unaligned_uint64.c create mode 100644 tests/regression/issue_227/unaligned_uint64.proto diff --git a/tests/SConstruct b/tests/SConstruct index d8ab9ab0..f2abe042 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -93,7 +93,9 @@ if not env.GetOption('clean'): conf.env.Append(CORECFLAGS = extra) # Check if we can use undefined behaviour sanitizer (only with clang) - extra = '-fsanitize=undefined ' + # TODO: Fuzz test triggers the bool sanitizer, figure out whether to + # modify the fuzz test or to keep ignoring the check. + extra = '-fsanitize=undefined -fno-sanitize-recover=undefined -fsanitize-recover=bool ' if 'clang' in env['CC']: if conf.CheckCCFLAGS(extra, linkflags = extra): conf.env.Append(CORECFLAGS = extra) diff --git a/tests/regression/issue_227/SConscript b/tests/regression/issue_227/SConscript new file mode 100644 index 00000000..10741240 --- /dev/null +++ b/tests/regression/issue_227/SConscript @@ -0,0 +1,14 @@ +# Regression test for Issue 227:Using proto3 type fields can cause unaligned access +# NOTE: This test will only detect problems when run with clang sanitizer (which +# is done regularly by a jenkins run). + +Import('env') + +env.NanopbProto('unaligned_uint64') + +p = env.Program(["unaligned_uint64.c", + "unaligned_uint64.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) + diff --git a/tests/regression/issue_227/unaligned_uint64.c b/tests/regression/issue_227/unaligned_uint64.c new file mode 100644 index 00000000..17c1d779 --- /dev/null +++ b/tests/regression/issue_227/unaligned_uint64.c @@ -0,0 +1,14 @@ +#include "unaligned_uint64.pb.h" +#include + +int main() +{ + uint8_t buf[128]; + pb_ostream_t stream = pb_ostream_from_buffer(buf, sizeof(buf)); + MainMessage msg = MainMessage_init_zero; + msg.bar[0] = 'A'; + pb_encode(&stream, MainMessage_fields, &msg); + + return 0; +} + diff --git a/tests/regression/issue_227/unaligned_uint64.proto b/tests/regression/issue_227/unaligned_uint64.proto new file mode 100644 index 00000000..f0269f60 --- /dev/null +++ b/tests/regression/issue_227/unaligned_uint64.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; +import 'nanopb.proto'; + +message MainMessage { + string foo = 1 [(nanopb).max_size = 3]; + string bar = 2 [(nanopb).max_size = 8]; +} + -- cgit 1.2.3-korg From 58af4d1fb733c5348b68dd3980f2a230d95400b4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 9 Dec 2016 18:57:08 +0200 Subject: Enable clang integer sanitizer and clean up a few warnings. Changed to use simple indexing instead of while (count--) in buf_read()/buf_write(), because the count overflowed from 0 to max on the last iteration. While the unsigned integer overflow is defined and behaviour was correct, making this simple change allowed enabling the sanitizer which might catch true errors elsewhere in the code. --- pb_decode.c | 5 +++-- pb_encode.c | 5 +++-- tests/SConstruct | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index 1f6aeae0..b2a3a310 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -75,13 +75,14 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count) { + size_t i; const pb_byte_t *source = (const pb_byte_t*)stream->state; stream->state = (pb_byte_t*)stream->state + count; if (buf != NULL) { - while (count--) - *buf++ = *source++; + for (i = 0; i < count; i++) + buf[i] = source[i]; } return true; diff --git a/pb_encode.c b/pb_encode.c index 13bda22c..cafe853c 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -59,11 +59,12 @@ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { static bool checkreturn buf_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count) { + size_t i; pb_byte_t *dest = (pb_byte_t*)stream->state; stream->state = dest + count; - while (count--) - *dest++ = *buf++; + for (i = 0; i < count; i++) + dest[i] = buf[i]; return true; } diff --git a/tests/SConstruct b/tests/SConstruct index f2abe042..ae79f710 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -95,7 +95,7 @@ if not env.GetOption('clean'): # Check if we can use undefined behaviour sanitizer (only with clang) # TODO: Fuzz test triggers the bool sanitizer, figure out whether to # modify the fuzz test or to keep ignoring the check. - extra = '-fsanitize=undefined -fno-sanitize-recover=undefined -fsanitize-recover=bool ' + extra = '-fsanitize=undefined,integer -fno-sanitize-recover=undefined,integer -fsanitize-recover=bool ' if 'clang' in env['CC']: if conf.CheckCCFLAGS(extra, linkflags = extra): conf.env.Append(CORECFLAGS = extra) -- cgit 1.2.3-korg From 283258dca59c470e8e209431e206a344839e0cf5 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 22 Dec 2016 17:37:14 +0200 Subject: Allow overriding proto3 mode (#228) --- generator/nanopb_generator.py | 8 ++++---- tests/options/SConscript | 5 ++++- tests/options/proto3_options.expected | 4 ++++ tests/options/proto3_options.proto | 11 +++++++++++ 4 files changed, 23 insertions(+), 5 deletions(-) create mode 100644 tests/options/proto3_options.expected create mode 100644 tests/options/proto3_options.proto diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 066ef936..a2ee22db 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -285,7 +285,7 @@ class Field: can_be_static = False else: self.array_decl = '[%d]' % self.max_count - elif field_options.HasField("proto3"): + elif field_options.proto3: self.rules = 'SINGULAR' elif desc.label == FieldD.LABEL_REQUIRED: self.rules = 'REQUIRED' @@ -1373,6 +1373,9 @@ def get_nanopb_suboptions(subdesc, options, name): new_options = nanopb_pb2.NanoPBOptions() new_options.CopyFrom(options) + if hasattr(subdesc, 'syntax') and subdesc.syntax == "proto3": + new_options.proto3 = True + # Handle options defined in a separate file dotname = '.'.join(name.parts) for namemask, options in Globals.separate_options: @@ -1380,9 +1383,6 @@ def get_nanopb_suboptions(subdesc, options, name): Globals.matched_namemasks.add(namemask) new_options.MergeFrom(options) - if hasattr(subdesc, 'syntax') and subdesc.syntax == "proto3": - new_options.proto3 = True - # Handle options defined in .proto if isinstance(subdesc.options, descriptor.FieldOptions): ext_type = nanopb_pb2.nanopb diff --git a/tests/options/SConscript b/tests/options/SConscript index 89a00fa5..215e3bd0 100644 --- a/tests/options/SConscript +++ b/tests/options/SConscript @@ -4,6 +4,9 @@ Import("env") env.NanopbProto("options") env.Object('options.pb.c') - env.Match(['options.pb.h', 'options.expected']) +env.NanopbProto("proto3_options") +env.Object('proto3_options.pb.c') +env.Match(['proto3_options.pb.h', 'proto3_options.expected']) + diff --git a/tests/options/proto3_options.expected b/tests/options/proto3_options.expected new file mode 100644 index 00000000..cc2f29c0 --- /dev/null +++ b/tests/options/proto3_options.expected @@ -0,0 +1,4 @@ +! bool has_proto3_default +bool has_proto3_off +! bool has_proto3_on + diff --git a/tests/options/proto3_options.proto b/tests/options/proto3_options.proto new file mode 100644 index 00000000..1017f046 --- /dev/null +++ b/tests/options/proto3_options.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +import "nanopb.proto"; + +message Message1 +{ + int32 proto3_default = 1; + int32 proto3_off = 2 [(nanopb).proto3 = false]; + int32 proto3_on = 3 [(nanopb).proto3 = true]; +} + -- cgit 1.2.3-korg From 1e4f46c92ab221df63a893467a2eec7ff9fa3004 Mon Sep 17 00:00:00 2001 From: Tobba Date: Wed, 21 Dec 2016 16:51:47 +0100 Subject: Make pb_decode_varint32 public API --- pb_decode.c | 3 +-- pb_decode.h | 4 ++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index b2a3a310..1fcc4e5a 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -24,7 +24,6 @@ typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn; static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count); -static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest); static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size); static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); @@ -170,7 +169,7 @@ pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t bufsize) * Helper functions * ********************/ -static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) +bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) { pb_byte_t byte; uint32_t result; diff --git a/pb_decode.h b/pb_decode.h index 1d9bb194..e7eb209c 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -126,6 +126,10 @@ bool pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type); * int64, uint32 and uint64 field types. */ bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest); +/* Decode an integer in the varint format. This works for bool, enum, int32, + * and uint32 field types. */ +bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest); + /* Decode an integer in the zig-zagged svarint format. This works for sint32 * and sint64. */ bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest); -- cgit 1.2.3-korg From 16f08f9fc8ff73e775cbf54b1024629ca7db0e58 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 23 Dec 2016 21:09:09 +0200 Subject: Update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a32d6ca2..459eaf84 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Using the nanopb library To use the nanopb library, you need to do two things: 1. Compile your .proto files for nanopb, using protoc. -2. Include pb_encode.c and pb_decode.c in your project. +2. Include pb_encode.c, pb_decode.c and pb_common.c in your project. The easiest way to get started is to study the project in "examples/simple". It contains a Makefile, which should work directly under most Linux systems. -- cgit 1.2.3-korg From 90a19bf0368035147a74b9b8edd65d672a03cd00 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 31 Dec 2016 10:33:19 +0200 Subject: Add testcase for issue #229 --- tests/regression/issue_229/SConscript | 13 +++++++++ tests/regression/issue_229/multiple_oneof.c | 35 +++++++++++++++++++++++++ tests/regression/issue_229/multiple_oneof.proto | 11 ++++++++ 3 files changed, 59 insertions(+) create mode 100644 tests/regression/issue_229/SConscript create mode 100644 tests/regression/issue_229/multiple_oneof.c create mode 100644 tests/regression/issue_229/multiple_oneof.proto diff --git a/tests/regression/issue_229/SConscript b/tests/regression/issue_229/SConscript new file mode 100644 index 00000000..b0f8376d --- /dev/null +++ b/tests/regression/issue_229/SConscript @@ -0,0 +1,13 @@ +# Regression test for Issue 229: problem encoding message that has +# multiple oneof fields +Import('env') + +env.NanopbProto('multiple_oneof') + +p = env.Program(["multiple_oneof.c", + "multiple_oneof.pb.c", + "$COMMON/pb_decode.o", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) + diff --git a/tests/regression/issue_229/multiple_oneof.c b/tests/regression/issue_229/multiple_oneof.c new file mode 100644 index 00000000..902248d0 --- /dev/null +++ b/tests/regression/issue_229/multiple_oneof.c @@ -0,0 +1,35 @@ +#include "multiple_oneof.pb.h" +#include +#include +#include + +int main() +{ + int status = 0; + uint8_t buf[128]; + size_t msglen; + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf, sizeof(buf)); + MainMessage msg = MainMessage_init_zero; + msg.which_oneof1 = MainMessage_oneof1_uint32_tag; + msg.oneof1.oneof1_uint32 = 1234; + msg.which_oneof2 = MainMessage_oneof2_uint32_tag; + msg.oneof2.oneof2_uint32 = 5678; + TEST(pb_encode(&stream, MainMessage_fields, &msg)); + msglen = stream.bytes_written; + } + + { + pb_istream_t stream = pb_istream_from_buffer(buf, msglen); + MainMessage msg = MainMessage_init_zero; + TEST(pb_decode(&stream, MainMessage_fields, &msg)); + TEST(msg.which_oneof1 == MainMessage_oneof1_uint32_tag); + TEST(msg.oneof1.oneof1_uint32 == 1234); + TEST(msg.which_oneof2 == MainMessage_oneof2_uint32_tag); + TEST(msg.oneof2.oneof2_uint32 == 5678); + } + + return status; +} + diff --git a/tests/regression/issue_229/multiple_oneof.proto b/tests/regression/issue_229/multiple_oneof.proto new file mode 100644 index 00000000..22373e1d --- /dev/null +++ b/tests/regression/issue_229/multiple_oneof.proto @@ -0,0 +1,11 @@ +syntax = "proto2"; + +message MainMessage { + oneof oneof1 { + uint32 oneof1_uint32 = 1; + } + oneof oneof2 { + uint32 oneof2_uint32 = 2; + } +} + -- cgit 1.2.3-korg From 48f5dd83530a46dd2423277f4488fd4ce9c93b72 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 31 Dec 2016 10:33:48 +0200 Subject: Fix multiple oneofs in same message (issue #229) Previously the field iterator logic didn't know whether two oneof fields were part of the same union, or separate. This caused wrong pointers to be calculated if multiple oneofs were inside a single message. This commit fixes this by using dataoffset of PB_SIZE_MAX to indicate union fields after the first field. Theoretically PB_SIZE_MAX is also a valid value for data offset, which could cause errors. Adding a compile-time assert for this is somewhat difficult. However I consider it extremely unlikely that there is any platform that could trigger this situation, as it would require 255 bytes of extra data/padding between two protobuf oneof fields. On 64-bit architectures the worst case is 16 bytes, and even esoteric platforms only align to 64 bytes or so. Manual modification of the generated .pb.h file could trigger this, but even then it would require pretty bad luck to happen. --- generator/nanopb_generator.py | 20 +++++++++++++++----- pb.h | 2 ++ pb_common.c | 6 +++--- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index a2ee22db..6e5ebaf9 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -509,9 +509,10 @@ class Field: identifier = '%s_%s_tag' % (self.struct_name, self.name) return '#define %-40s %d\n' % (identifier, self.tag) - def pb_field_t(self, prev_field_name): + def pb_field_t(self, prev_field_name, union_index = None): '''Return the pb_field_t initializer to use in the constant array. - prev_field_name is the name of the previous field or None. + prev_field_name is the name of the previous field or None. For OneOf + unions, union_index is the index of this field inside the OneOf. ''' if self.rules == 'ONEOF': @@ -526,7 +527,14 @@ class Field: result += '%-8s, ' % self.pbtype result += '%s, ' % self.rules result += '%-8s, ' % (self.allocation if not self.inline else "INLINE") - result += '%s, ' % ("FIRST" if not prev_field_name else "OTHER") + + if union_index is not None and union_index > 0: + result += 'UNION, ' + elif prev_field_name is None: + result += 'FIRST, ' + else: + result += 'OTHER, ' + result += '%s, ' % self.struct_name result += '%s, ' % self.name result += '%s, ' % (prev_field_name or self.name) @@ -767,8 +775,10 @@ class OneOf(Field): return ''.join([f.tags() for f in self.fields]) def pb_field_t(self, prev_field_name): - result = ',\n'.join([f.pb_field_t(prev_field_name) for f in self.fields]) - return result + parts = [] + for union_index, field in enumerate(self.fields): + parts.append(field.pb_field_t(prev_field_name, union_index)) + return ',\n'.join(parts) def get_last_field_name(self): if self.anonymous: diff --git a/pb.h b/pb.h index 2cf5c4dc..f68d1d65 100644 --- a/pb.h +++ b/pb.h @@ -393,6 +393,8 @@ struct pb_extension_s { #define PB_DATAOFFSET_FIRST(st, m1, m2) (offsetof(st, m1)) /* data_offset for subsequent fields */ #define PB_DATAOFFSET_OTHER(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) +/* data offset for subsequent fields inside an union (oneof) */ +#define PB_DATAOFFSET_UNION(st, m1, m2) (PB_SIZE_MAX) /* Choose first/other based on m1 == m2 (deprecated, remains for backwards compatibility) */ #define PB_DATAOFFSET_CHOOSE(st, m1, m2) (int)(offsetof(st, m1) == offsetof(st, m2) \ ? PB_DATAOFFSET_FIRST(st, m1, m2) \ diff --git a/pb_common.c b/pb_common.c index 385c0193..4fb7186b 100644 --- a/pb_common.c +++ b/pb_common.c @@ -42,11 +42,11 @@ bool pb_field_iter_next(pb_field_iter_t *iter) size_t prev_size = prev_field->data_size; if (PB_HTYPE(prev_field->type) == PB_HTYPE_ONEOF && - PB_HTYPE(iter->pos->type) == PB_HTYPE_ONEOF) + PB_HTYPE(iter->pos->type) == PB_HTYPE_ONEOF && + iter->pos->data_offset == PB_SIZE_MAX) { /* Don't advance pointers inside unions */ - prev_size = 0; - iter->pData = (char*)iter->pData - prev_field->data_offset; + return true; } else if (PB_ATYPE(prev_field->type) == PB_ATYPE_STATIC && PB_HTYPE(prev_field->type) == PB_HTYPE_REPEATED) -- cgit 1.2.3-korg From 31e09c6630c47d0aa1c1ec3909a936ba5674254f Mon Sep 17 00:00:00 2001 From: Tobba Date: Wed, 21 Dec 2016 21:02:53 +0100 Subject: Fix closing a non-empty substream resulting in an incorrect stream state --- docs/migration.rst | 13 +++++++++++++ pb_decode.c | 29 +++++++++++++++++++++-------- pb_decode.h | 2 +- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/docs/migration.rst b/docs/migration.rst index cd5911f5..2d9ce38c 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -11,6 +11,19 @@ are included, in order to make it easier to find this document. .. contents :: +Nanopb-0.3.8 (2017-xx-xx) +========================= +Fully drain substreams before closing + +**Rationale:** If the substream functions were called directly and the caller +did not completely empty the substring before closing it, the parent stream +would be put into an incorrect state. + +**Changes:** *pb_close_string_substream* can now error and returns a boolean. + +**Required actions:** Add error checking onto any call to +*pb_close_string_substream*. + Nanopb-0.3.5 (2016-02-13) ========================= diff --git a/pb_decode.c b/pb_decode.c index 1fcc4e5a..92d0175d 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -333,13 +333,19 @@ bool checkreturn pb_make_string_substream(pb_istream_t *stream, pb_istream_t *su return true; } -void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) +bool checkreturn pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) { + if (substream->bytes_left) { + if (!pb_read(substream, NULL, substream->bytes_left)) + return false; + } + stream->state = substream->state; #ifndef PB_NO_ERRMSG stream->errmsg = substream->errmsg; #endif + return true; } /************************* @@ -385,11 +391,12 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t } (*size)++; } - pb_close_string_substream(stream, &substream); - + if (substream.bytes_left != 0) PB_RETURN_ERROR(stream, "array overflow"); - + if (!pb_close_string_substream(stream, &substream)) + return false; + return status; } else @@ -569,7 +576,8 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ (*size)++; } - pb_close_string_substream(stream, &substream); + if (!pb_close_string_substream(stream, &substream)) + return false; return status; } @@ -623,7 +631,9 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type PB_RETURN_ERROR(stream, "callback failed"); } while (substream.bytes_left); - pb_close_string_substream(stream, &substream); + if (!pb_close_string_substream(stream, &substream)) + return false; + return true; } else @@ -964,7 +974,9 @@ bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void * return false; status = pb_decode(&substream, fields, dest_struct); - pb_close_string_substream(stream, &substream); + + if (!pb_close_string_substream(stream, &substream)) + return false; return status; } @@ -1343,6 +1355,7 @@ static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t else status = pb_decode_noinit(&substream, submsg_fields, dest); - pb_close_string_substream(stream, &substream); + if (!pb_close_string_substream(stream, &substream)) + return false; return status; } diff --git a/pb_decode.h b/pb_decode.h index e7eb209c..a426bdd7 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -144,7 +144,7 @@ bool pb_decode_fixed64(pb_istream_t *stream, void *dest); /* Make a limited-length substream for reading a PB_WT_STRING field. */ bool pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream); -void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream); +bool pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream); #ifdef __cplusplus } /* extern "C" */ -- cgit 1.2.3-korg From cff94b8f6669a16ee6a809932714d793b322e11b Mon Sep 17 00:00:00 2001 From: "William A. Kennington III" Date: Tue, 17 Jan 2017 12:58:48 -0800 Subject: cmake: Fix library name The produced static library should be `libprotobuf-nanopb.a` instead of the current `liblibprotobuf-nanopb.a`. Signed-off-by: William A. Kennington III --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7889bf4a..bb236a98 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -22,7 +22,7 @@ if(MSVC AND nanopb_MSVC_STATIC_RUNTIME) endforeach(flag_var) endif() -add_library(libprotobuf-nanopb STATIC +add_library(protobuf-nanopb STATIC pb.h pb_common.h pb_common.c @@ -31,7 +31,7 @@ add_library(libprotobuf-nanopb STATIC pb_decode.h pb_decode.c) -target_include_directories(libprotobuf-nanopb INTERFACE +target_include_directories(protobuf-nanopb INTERFACE $ ) @@ -41,7 +41,7 @@ if(NOT DEFINED CMAKE_INSTALL_CMAKEDIR) set(CMAKE_INSTALL_CMAKEDIR "lib/cmake/nanopb") endif() -install(TARGETS libprotobuf-nanopb EXPORT nanopb-targets +install(TARGETS protobuf-nanopb EXPORT nanopb-targets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) install(EXPORT nanopb-targets -- cgit 1.2.3-korg From 112b5b1908f6a7094386c7721ffa97c7108daf68 Mon Sep 17 00:00:00 2001 From: "William A. Kennington III" Date: Tue, 17 Jan 2017 17:22:06 -0800 Subject: cmake: Cleanup the build so that host and runtime tooling is installable Signed-off-by: William A. Kennington III --- CMakeLists.txt | 83 ++++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 57 insertions(+), 26 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bb236a98..f69386bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,12 +6,16 @@ set(nanopb_VERSION_STRING nanopb-0.3.8-dev) string(REPLACE "nanopb-" "" nanopb_VERSION ${nanopb_VERSION_STRING}) +option(nanopb_BUILD_RUNTIME "Build the headers and libraries needed at runtime" ON) +option(nanopb_BUILD_GENERATOR "Build the protoc plugin for code generation" ON) option(nanopb_MSVC_STATIC_RUNTIME "Link static runtime libraries" ON) if(NOT DEFINED CMAKE_DEBUG_POSTFIX) set(CMAKE_DEBUG_POSTFIX "d") endif() +include(GNUInstallDirs) + if(MSVC AND nanopb_MSVC_STATIC_RUNTIME) foreach(flag_var CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE @@ -22,38 +26,65 @@ if(MSVC AND nanopb_MSVC_STATIC_RUNTIME) endforeach(flag_var) endif() -add_library(protobuf-nanopb STATIC - pb.h - pb_common.h - pb_common.c - pb_encode.h - pb_encode.c - pb_decode.h - pb_decode.c) +if(NOT DEFINED CMAKE_INSTALL_CMAKEDIR) + set(CMAKE_INSTALL_CMAKEDIR "lib/cmake/nanopb") +endif() -target_include_directories(protobuf-nanopb INTERFACE - $ -) +if(nanopb_BUILD_GENERATOR) + set(generator_protos nanopb plugin) -include(GNUInstallDirs) + find_package(PythonInterp 2.7 REQUIRED) + execute_process( + COMMAND ${PYTHON_EXECUTABLE} -c + "from distutils import sysconfig; print(sysconfig.get_python_lib(prefix='${CMAKE_INSTALL_PREFIX}'))" + OUTPUT_VARIABLE PYTHON_INSTDIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) -if(NOT DEFINED CMAKE_INSTALL_CMAKEDIR) - set(CMAKE_INSTALL_CMAKEDIR "lib/cmake/nanopb") + foreach(generator_proto IN LISTS generator_protos) + string(REGEX REPLACE "([^;]+)" "generator/proto/\\1.proto" generator_proto_file "${generator_proto}") + string(REGEX REPLACE "([^;]+)" "\\1_pb2.py" generator_proto_py_file "${generator_proto}") + add_custom_command( + OUTPUT ${generator_proto_py_file} + COMMAND protoc --python_out=${CMAKE_CURRENT_BINARY_DIR} -Igenerator/proto ${generator_proto_file} + DEPENDS ${generator_proto_file} + ) + add_custom_target("generate_${generator_proto_py_file}" ALL DEPENDS ${generator_proto_py_file}) + install( + FILES ${generator_proto_py_file} + DESTINATION ${PYTHON_INSTDIR} + ) + endforeach() endif() -install(TARGETS protobuf-nanopb EXPORT nanopb-targets - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +if(nanopb_BUILD_RUNTIME) + add_library(protobuf-nanopb STATIC + pb.h + pb_common.h + pb_common.c + pb_encode.h + pb_encode.c + pb_decode.h + pb_decode.c) + + target_include_directories(protobuf-nanopb INTERFACE + $ + ) -install(EXPORT nanopb-targets - DESTINATION ${CMAKE_INSTALL_CMAKEDIR} - NAMESPACE nanopb::) + configure_file(extra/nanopb-config-version.cmake.in + nanopb-config-version.cmake @ONLY) -configure_file(extra/nanopb-config-version.cmake.in - nanopb-config-version.cmake @ONLY) + install(TARGETS protobuf-nanopb EXPORT nanopb-targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) -install(FILES extra/nanopb-config.cmake - ${CMAKE_CURRENT_BINARY_DIR}/nanopb-config-version.cmake - DESTINATION ${CMAKE_INSTALL_CMAKEDIR}) + install(EXPORT nanopb-targets + DESTINATION ${CMAKE_INSTALL_CMAKEDIR} + NAMESPACE nanopb::) -install(FILES pb.h pb_common.h pb_encode.h pb_decode.h - DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + install(FILES extra/nanopb-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/nanopb-config-version.cmake + DESTINATION ${CMAKE_INSTALL_CMAKEDIR}) + + install(FILES pb.h pb_common.h pb_encode.h pb_decode.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +endif() -- cgit 1.2.3-korg From a5a341f151e12f7048ea92c1bfb1beb11831a8ef Mon Sep 17 00:00:00 2001 From: Maxim Khitrov Date: Fri, 20 Jan 2017 18:25:10 -0500 Subject: Fix documentation for protoc --plugin argument --- generator/protoc-gen-nanopb | 2 +- generator/protoc-gen-nanopb.bat | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generator/protoc-gen-nanopb b/generator/protoc-gen-nanopb index 358f97cf..471a620b 100755 --- a/generator/protoc-gen-nanopb +++ b/generator/protoc-gen-nanopb @@ -3,7 +3,7 @@ # This file is used to invoke nanopb_generator.py as a plugin # to protoc on Linux and other *nix-style systems. # Use it like this: -# protoc --plugin=nanopb=..../protoc-gen-nanopb --nanopb_out=dir foo.proto +# protoc --plugin=protoc-gen-nanopb=..../protoc-gen-nanopb --nanopb_out=dir foo.proto # # Note that if you use the binary package of nanopb, the protoc # path is already set up properly and there is no need to give diff --git a/generator/protoc-gen-nanopb.bat b/generator/protoc-gen-nanopb.bat index 7624984e..e6cf187f 100644 --- a/generator/protoc-gen-nanopb.bat +++ b/generator/protoc-gen-nanopb.bat @@ -2,7 +2,7 @@ :: This file is used to invoke nanopb_generator.py as a plugin :: to protoc on Windows. :: Use it like this: -:: protoc --plugin=nanopb=..../protoc-gen-nanopb.bat --nanopb_out=dir foo.proto +:: protoc --plugin=protoc-gen-nanopb=..../protoc-gen-nanopb.bat --nanopb_out=dir foo.proto :: :: Note that if you use the binary package of nanopb, the protoc :: path is already set up properly and there is no need to give -- cgit 1.2.3-korg From 9dd81c51b3c64221117ad2c71daff0980baaf1c2 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 12 Feb 2017 10:45:07 +0200 Subject: Add regression test for issue 242 --- tests/regression/issue_242/SConscript | 13 ++++++++ tests/regression/issue_242/zero_value.c | 51 +++++++++++++++++++++++++++++ tests/regression/issue_242/zero_value.proto | 15 +++++++++ 3 files changed, 79 insertions(+) create mode 100644 tests/regression/issue_242/SConscript create mode 100644 tests/regression/issue_242/zero_value.c create mode 100644 tests/regression/issue_242/zero_value.proto diff --git a/tests/regression/issue_242/SConscript b/tests/regression/issue_242/SConscript new file mode 100644 index 00000000..000063ef --- /dev/null +++ b/tests/regression/issue_242/SConscript @@ -0,0 +1,13 @@ +# Regression test for Issue 242: pb_encode does not encode tag for +# extension fields that is all zeros +Import('env') + +env.NanopbProto('zero_value') + +p = env.Program(["zero_value.c", + "zero_value.pb.c", + "$COMMON/pb_decode.o", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) + diff --git a/tests/regression/issue_242/zero_value.c b/tests/regression/issue_242/zero_value.c new file mode 100644 index 00000000..b3d96b7a --- /dev/null +++ b/tests/regression/issue_242/zero_value.c @@ -0,0 +1,51 @@ +#include +#include +#include +#include +#include "zero_value.pb.h" + +int main() +{ + int status = 0; + + COMMENT("Test extension fields with zero values"); + { + uint8_t buffer[256] = {0}; + pb_ostream_t ostream; + int32_t value = 0; + Extendable source = {0}; + + pb_extension_t source_ext = {0}; + source_ext.type = &opt_int32; + source_ext.dest = &value; + source.extensions = &source_ext; + + ostream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + TEST(pb_encode(&ostream, Extendable_fields, &source)); + + TEST(ostream.bytes_written == 2); + TEST(memcmp(buffer, "\x58\x00", 2) == 0); + } + + /* Note: There never was a bug here, but this check is included + * in the regression test because the logic is closely related. + */ + COMMENT("Test pointer fields with zero values"); + { + uint8_t buffer[256] = {0}; + pb_ostream_t ostream; + int32_t value = 0; + PointerMessage source = {0}; + + source.opt_int32 = &value; + + ostream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + TEST(pb_encode(&ostream, PointerMessage_fields, &source)); + + TEST(ostream.bytes_written == 2); + TEST(memcmp(buffer, "\x58\x00", 2) == 0); + } + + return status; +} + diff --git a/tests/regression/issue_242/zero_value.proto b/tests/regression/issue_242/zero_value.proto new file mode 100644 index 00000000..020a39a5 --- /dev/null +++ b/tests/regression/issue_242/zero_value.proto @@ -0,0 +1,15 @@ +syntax = "proto2"; +import "nanopb.proto"; + +message Extendable { + extensions 10 to 100; +} + +extend Extendable { + optional int32 opt_int32 = 11; +} + +message PointerMessage { + optional int32 opt_int32 = 11 [(nanopb).type = FT_POINTER]; +} + -- cgit 1.2.3-korg From 2467922d47a579d447406eddcecdb1127d33213b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 12 Feb 2017 10:48:51 +0200 Subject: Zero-valued extension fields were mistakenly ignored by encoder. (issue #242) --- pb.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pb.h b/pb.h index f68d1d65..f68a1cc0 100644 --- a/pb.h +++ b/pb.h @@ -478,9 +478,14 @@ struct pb_extension_s { {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REPEATED | ltype, \ fd, 0, pb_membersize(st, m), 0, ptr} -/* Optional extensions don't have the has_ field, as that would be redundant. */ +/* Optional extensions don't have the has_ field, as that would be redundant. + * Furthermore, the combination of OPTIONAL without has_ field is used + * for indicating proto3 style fields. Extensions exist in proto2 mode only, + * so they should be encoded according to proto2 rules. To avoid the conflict, + * extensions are marked as REQUIRED instead. + */ #define PB_OPTEXT_STATIC(tag, st, m, fd, ltype, ptr) \ - {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REQUIRED | ltype, \ 0, \ 0, \ pb_membersize(st, m), 0, ptr} -- cgit 1.2.3-korg From 473816c66b559be6850fc1ef753a7141d1848300 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 13 Feb 2017 03:53:11 +0200 Subject: Update download links (changes in webhost system) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 459eaf84..07860f06 100644 --- a/README.md +++ b/README.md @@ -7,9 +7,9 @@ Nanopb is a small code-size Protocol Buffers implementation in ansi C. It is especially suitable for use in microcontrollers, but fits any memory restricted system. -* **Homepage:** https://koti.kapsi.fi/jpa/nanopb/ -* **Documentation:** https://koti.kapsi.fi/jpa/nanopb/docs/ -* **Downloads:** https://koti.kapsi.fi/jpa/nanopb/download/ +* **Homepage:** https://jpa.kapsi.fi/nanopb/ +* **Documentation:** https://jpa.kapsi.fi/nanopb/docs/ +* **Downloads:** https://jpa.kapsi.fi/nanopb/download/ * **Forum:** https://groups.google.com/forum/#!forum/nanopb -- cgit 1.2.3-korg From ca74746e23b5a9e7916e8fde6632d71d61603f50 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Mon, 20 Feb 2017 15:47:44 +0200 Subject: Add new option max_length for strings (issue #107) Max_size is the allocated size, so users had to add +1 for the null terminator. Max_length does the +1 automatically in the generator. --- generator/nanopb_generator.py | 4 ++++ generator/proto/nanopb.proto | 5 +++++ tests/options/options.expected | 1 + tests/options/options.proto | 1 + 4 files changed, 11 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 6e5ebaf9..ca60c03a 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -270,6 +270,10 @@ class Field: # Parse field options if field_options.HasField("max_size"): self.max_size = field_options.max_size + + if desc.type == FieldD.TYPE_STRING and field_options.HasField("max_length"): + # max_length overrides max_size for strings + self.max_size = field_options.max_length + 1 if field_options.HasField("max_count"): self.max_count = field_options.max_count diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index f6fe4a20..7d39e1c3 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -32,8 +32,13 @@ enum IntSize { // fields. message NanoPBOptions { // Allocated size for 'bytes' and 'string' fields. + // For string fields, this should include the space for null terminator. optional int32 max_size = 1; + // Maximum length for 'string' fields. Setting this is equivalent + // to setting max_size to a value of length+1. + optional int32 max_length = 14; + // Allocated number of entries in arrays ('repeated' fields) optional int32 max_count = 2; diff --git a/tests/options/options.expected b/tests/options/options.expected index 0769880a..9e47e6a4 100644 --- a/tests/options/options.expected +++ b/tests/options/options.expected @@ -1,6 +1,7 @@ char filesize\[20\]; char msgsize\[30\]; char fieldsize\[40\]; +char fieldlen\[41\]; pb_callback_t int32_callback; \sEnumValue1 = 1 Message5_EnumValue1 diff --git a/tests/options/options.proto b/tests/options/options.proto index 89bb086f..c6ca5e25 100644 --- a/tests/options/options.proto +++ b/tests/options/options.proto @@ -26,6 +26,7 @@ message Message3 { option (nanopb_msgopt).msgid = 103; required string fieldsize = 1 [(nanopb).max_size = 40]; + required string fieldlen = 2 [(nanopb).max_length = 40]; } // Forced callback field -- cgit 1.2.3-korg From 07375a126337916f3a34ea94f8085b8f89d789a1 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 22 Feb 2017 21:06:32 +0200 Subject: Extend inline / fixed length bytes array support (issue #244) Adds support for proto3 and POINTER field types to have fixed length bytes arrays. Also changed the .proto option to a separate fixed_length:true, while also supporting the old FT_INLINE option. Restructured the generator and decoder logic to threat the inline bytes fields more like "just another field type". --- docs/concepts.rst | 4 +-- docs/reference.rst | 8 +++--- generator/nanopb_generator.py | 64 +++++++++++++++++++++++-------------------- generator/proto/nanopb.proto | 5 +++- pb.h | 58 ++++++++++++++------------------------- pb_decode.c | 32 +++++++++++++++++----- pb_encode.c | 11 +++++--- 7 files changed, 97 insertions(+), 85 deletions(-) diff --git a/docs/concepts.rst b/docs/concepts.rst index ea33863c..2e0d3f9b 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -148,7 +148,7 @@ Most Protocol Buffers datatypes have directly corresponding C datatypes, such as 1) Strings, bytes and repeated fields of any type map to callback functions by default. 2) If there is a special option *(nanopb).max_size* specified in the .proto file, string maps to null-terminated char array and bytes map to a structure containing a char array and a size field. -3) If *(nanopb).type* is set to *FT_INLINE* and *(nanopb).max_size* is also set, then bytes map to an inline byte array of fixed size. +3) If *(nanopb).fixed_length* is set to *true* and *(nanopb).max_size* is also set, then bytes map to an inline byte array of fixed size. 4) If there is a special option *(nanopb).max_count* specified on a repeated field, it maps to an array of whatever type is being repeated. Another field will be created for the actual number of entries stored. =============================================================================== ======================= @@ -164,7 +164,7 @@ required bytes data = 1 [(nanopb).max_size = 40]; | pb_byte_t bytes[40]; | } Person_data_t; | Person_data_t data; -required bytes data = 1 [(nanopb).max_size = 40, (nanopb).type = FT_INLINE]; | pb_byte_t data[40]; +required bytes data = 1 [(nanopb).max_size = 40, (nanopb).fixed_length = true]; | pb_byte_t data[40]; =============================================================================== ======================= The maximum lengths are checked in runtime. If string/bytes/array exceeds the allocated length, *pb_decode* will return false. diff --git a/docs/reference.rst b/docs/reference.rst index ef3867a1..e59a0c94 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -77,11 +77,10 @@ int_size Override the integer type of a field. type Type of the generated field. Default value is *FT_DEFAULT*, which selects automatically. You can use *FT_CALLBACK*, *FT_POINTER*, - *FT_STATIC*, *FT_IGNORE*, or *FT_INLINE* to + *FT_STATIC* or *FT_IGNORE* to force a callback field, a dynamically - allocated field, a static field, to - completely ignore the field or to - generate an inline bytes field. + allocated field, a static field or to + completely ignore the field. long_names Prefix the enum name to the enum value in definitions, i.e. *EnumName_EnumValue*. Enabled by default. @@ -94,6 +93,7 @@ no_unions Generate 'oneof' fields as optional fields msgid Specifies a unique id for this message type. Can be used by user code as an identifier. anonymous_oneof Generate 'oneof' fields as anonymous unions. +fixed_length Generate 'bytes' fields with constant length. ============================ ================================================ These options can be defined for the .proto files before they are converted diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index ca60c03a..9cce6a5a 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -262,10 +262,12 @@ class Field: self.enc_size = None self.ctype = None - self.inline = None if field_options.type == nanopb_pb2.FT_INLINE: + # Before nanopb-0.3.8, fixed length bytes arrays were specified + # by setting type to FT_INLINE. But to handle pointer typed fields, + # it makes sense to have it as a separate option. field_options.type = nanopb_pb2.FT_STATIC - self.inline = nanopb_pb2.FT_INLINE + field_options.fixed_length = True # Parse field options if field_options.HasField("max_size"): @@ -349,17 +351,17 @@ class Field: self.array_decl += '[%d]' % self.max_size self.enc_size = varint_max_size(self.max_size) + self.max_size elif desc.type == FieldD.TYPE_BYTES: - self.pbtype = 'BYTES' - if self.allocation == 'STATIC': - # Inline STATIC for BYTES is like STATIC for STRING. - if self.inline: - self.ctype = 'pb_byte_t' - self.array_decl += '[%d]' % self.max_size - else: - self.ctype = self.struct_name + self.name + 't' + if field_options.fixed_length: + self.pbtype = 'FIXED_LENGTH_BYTES' self.enc_size = varint_max_size(self.max_size) + self.max_size - elif self.allocation == 'POINTER': + self.ctype = 'pb_byte_t' + self.array_decl += '[%d]' % self.max_size + else: + self.pbtype = 'BYTES' self.ctype = 'pb_bytes_array_t' + if self.allocation == 'STATIC': + self.ctype = self.struct_name + self.name + 't' + self.enc_size = varint_max_size(self.max_size) + self.max_size elif desc.type == FieldD.TYPE_MESSAGE: self.pbtype = 'MESSAGE' self.ctype = self.submsgname = names_from_type_name(desc.type_name) @@ -379,6 +381,9 @@ class Field: if self.pbtype == 'MESSAGE': # Use struct definition, so recursive submessages are possible result += ' struct _%s *%s;' % (self.ctype, self.name) + elif self.pbtype == 'FIXED_LENGTH_BYTES': + # Pointer to fixed size array + result += ' %s (*%s)%s;' % (self.ctype, self.name, self.array_decl) elif self.rules == 'REPEATED' and self.pbtype in ['STRING', 'BYTES']: # String/bytes arrays need to be defined as pointers to pointers result += ' %s **%s;' % (self.ctype, self.name) @@ -396,7 +401,7 @@ class Field: def types(self): '''Return definitions for any special types this field might need.''' - if self.pbtype == 'BYTES' and self.allocation == 'STATIC' and not self.inline: + if self.pbtype == 'BYTES' and self.allocation == 'STATIC': result = 'typedef PB_BYTES_ARRAY_T(%d) %s;\n' % (self.max_size, self.ctype) else: result = '' @@ -425,10 +430,9 @@ class Field: if self.pbtype == 'STRING': inner_init = '""' elif self.pbtype == 'BYTES': - if self.inline: - inner_init = '{0}' - else: - inner_init = '{0, {0}}' + inner_init = '{0, {0}}' + elif self.pbtype == 'FIXED_LENGTH_BYTES': + inner_init = '{0}' elif self.pbtype in ('ENUM', 'UENUM'): inner_init = '(%s)0' % self.ctype else: @@ -440,15 +444,15 @@ class Field: elif self.pbtype == 'BYTES': data = ['0x%02x' % ord(c) for c in self.default] if len(data) == 0: - if self.inline: - inner_init = '{0}' - else: - inner_init = '{0, {0}}' + inner_init = '{0, {0}}' + else: + inner_init = '{%d, {%s}}' % (len(data), ','.join(data)) + elif self.pbtype == 'FIXED_LENGTH_BYTES': + data = ['0x%02x' % ord(c) for c in self.default] + if len(data) == 0: + inner_init = '{0}' else: - if self.inline: - inner_init = '{%s}' % ','.join(data) - else: - inner_init = '{%d, {%s}}' % (len(data), ','.join(data)) + inner_init = '{%s}' % ','.join(data) elif self.pbtype in ['FIXED32', 'UINT32']: inner_init = str(self.default) + 'u' elif self.pbtype in ['FIXED64', 'UINT64']: @@ -500,8 +504,10 @@ class Field: elif self.pbtype == 'BYTES': if self.allocation != 'STATIC': return None # Not implemented - if self.inline: - array_decl = '[%d]' % self.max_size + elif self.pbtype == 'FIXED_LENGTH_BYTES': + if self.allocation != 'STATIC': + return None # Not implemented + array_decl = '[%d]' % self.max_size if declaration_only: return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl) @@ -530,7 +536,7 @@ class Field: result += '%3d, ' % self.tag result += '%-8s, ' % self.pbtype result += '%s, ' % self.rules - result += '%-8s, ' % (self.allocation if not self.inline else "INLINE") + result += '%-8s, ' % self.allocation if union_index is not None and union_index > 0: result += 'UNION, ' @@ -547,7 +553,7 @@ class Field: result += '&%s_fields)' % self.submsgname elif self.default is None: result += '0)' - elif self.pbtype in ['BYTES', 'STRING'] and self.allocation != 'STATIC': + elif self.pbtype in ['BYTES', 'STRING', 'FIXED_LENGTH_BYTES'] and self.allocation != 'STATIC': result += '0)' # Arbitrary size default values not implemented elif self.rules == 'OPTEXT': result += '0)' # Default value for extensions is not implemented @@ -653,7 +659,6 @@ class ExtensionRange(Field): self.default = None self.max_size = 0 self.max_count = 0 - self.inline = None def __str__(self): return ' pb_extension_t *extensions;' @@ -731,7 +736,6 @@ class OneOf(Field): self.default = None self.rules = 'ONEOF' self.anonymous = False - self.inline = None def add_field(self, field): if field.allocation == 'CALLBACK': diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index 7d39e1c3..e4c1da79 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -16,7 +16,7 @@ enum FieldType { FT_POINTER = 4; // Always generate a dynamically allocated field. FT_STATIC = 2; // Generate a static field or raise an exception if not possible. FT_IGNORE = 3; // Ignore the field completely. - FT_INLINE = 5; // Always generate an inline array of fixed size. + FT_INLINE = 5; // Legacy option, use the separate 'fixed_length' option instead } enum IntSize { @@ -77,6 +77,9 @@ message NanoPBOptions { // Generate an enum->string mapping function (can take up lots of space). optional bool enum_to_string = 13 [default = false]; + + // Generate bytes arrays with fixed length + optional bool fixed_length = 15 [default = false]; } // Extensions to protoc 'Descriptor' type in order to define options diff --git a/pb.h b/pb.h index f68a1cc0..a3001bad 100644 --- a/pb.h +++ b/pb.h @@ -427,19 +427,6 @@ struct pb_extension_s { pb_membersize(st, m[0]), \ pb_arraysize(st, m), ptr} -#define PB_REQUIRED_INLINE(tag, st, m, fd, ltype, ptr) \ - {tag, PB_ATYPE_STATIC | PB_HTYPE_REQUIRED | PB_LTYPE_FIXED_LENGTH_BYTES, \ - fd, 0, pb_membersize(st, m), 0, ptr} - -/* Optional fields add the delta to the has_ variable. */ -#define PB_OPTIONAL_INLINE(tag, st, m, fd, ltype, ptr) \ - {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | PB_LTYPE_FIXED_LENGTH_BYTES, \ - fd, \ - pb_delta(st, has_ ## m, m), \ - pb_membersize(st, m), 0, ptr} - -/* INLINE does not support REPEATED fields. */ - /* Allocated fields carry the size of the actual data, not the pointer */ #define PB_REQUIRED_POINTER(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_POINTER | PB_HTYPE_REQUIRED | ltype, \ @@ -493,31 +480,30 @@ struct pb_extension_s { #define PB_OPTEXT_POINTER(tag, st, m, fd, ltype, ptr) \ PB_OPTIONAL_POINTER(tag, st, m, fd, ltype, ptr) -/* INLINE does not support OPTEXT. */ - #define PB_OPTEXT_CALLBACK(tag, st, m, fd, ltype, ptr) \ PB_OPTIONAL_CALLBACK(tag, st, m, fd, ltype, ptr) /* The mapping from protobuf types to LTYPEs is done using these macros. */ -#define PB_LTYPE_MAP_BOOL PB_LTYPE_VARINT -#define PB_LTYPE_MAP_BYTES PB_LTYPE_BYTES -#define PB_LTYPE_MAP_DOUBLE PB_LTYPE_FIXED64 -#define PB_LTYPE_MAP_ENUM PB_LTYPE_VARINT -#define PB_LTYPE_MAP_UENUM PB_LTYPE_UVARINT -#define PB_LTYPE_MAP_FIXED32 PB_LTYPE_FIXED32 -#define PB_LTYPE_MAP_FIXED64 PB_LTYPE_FIXED64 -#define PB_LTYPE_MAP_FLOAT PB_LTYPE_FIXED32 -#define PB_LTYPE_MAP_INT32 PB_LTYPE_VARINT -#define PB_LTYPE_MAP_INT64 PB_LTYPE_VARINT -#define PB_LTYPE_MAP_MESSAGE PB_LTYPE_SUBMESSAGE -#define PB_LTYPE_MAP_SFIXED32 PB_LTYPE_FIXED32 -#define PB_LTYPE_MAP_SFIXED64 PB_LTYPE_FIXED64 -#define PB_LTYPE_MAP_SINT32 PB_LTYPE_SVARINT -#define PB_LTYPE_MAP_SINT64 PB_LTYPE_SVARINT -#define PB_LTYPE_MAP_STRING PB_LTYPE_STRING -#define PB_LTYPE_MAP_UINT32 PB_LTYPE_UVARINT -#define PB_LTYPE_MAP_UINT64 PB_LTYPE_UVARINT -#define PB_LTYPE_MAP_EXTENSION PB_LTYPE_EXTENSION +#define PB_LTYPE_MAP_BOOL PB_LTYPE_VARINT +#define PB_LTYPE_MAP_BYTES PB_LTYPE_BYTES +#define PB_LTYPE_MAP_DOUBLE PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_ENUM PB_LTYPE_VARINT +#define PB_LTYPE_MAP_UENUM PB_LTYPE_UVARINT +#define PB_LTYPE_MAP_FIXED32 PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_FIXED64 PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_FLOAT PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_INT32 PB_LTYPE_VARINT +#define PB_LTYPE_MAP_INT64 PB_LTYPE_VARINT +#define PB_LTYPE_MAP_MESSAGE PB_LTYPE_SUBMESSAGE +#define PB_LTYPE_MAP_SFIXED32 PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_SFIXED64 PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_SINT32 PB_LTYPE_SVARINT +#define PB_LTYPE_MAP_SINT64 PB_LTYPE_SVARINT +#define PB_LTYPE_MAP_STRING PB_LTYPE_STRING +#define PB_LTYPE_MAP_UINT32 PB_LTYPE_UVARINT +#define PB_LTYPE_MAP_UINT64 PB_LTYPE_UVARINT +#define PB_LTYPE_MAP_EXTENSION PB_LTYPE_EXTENSION +#define PB_LTYPE_MAP_FIXED_LENGTH_BYTES PB_LTYPE_FIXED_LENGTH_BYTES /* This is the actual macro used in field descriptions. * It takes these arguments: @@ -526,7 +512,7 @@ struct pb_extension_s { * FLOAT, INT32, INT64, MESSAGE, SFIXED32, SFIXED64 * SINT32, SINT64, STRING, UINT32, UINT64 or EXTENSION * - Field rules: REQUIRED, OPTIONAL or REPEATED - * - Allocation: STATIC, INLINE, or CALLBACK + * - Allocation: STATIC, CALLBACK or POINTER * - Placement: FIRST or OTHER, depending on if this is the first field in structure. * - Message name * - Field name @@ -552,8 +538,6 @@ struct pb_extension_s { fd, pb_delta(st, which_ ## u, u.m), \ pb_membersize(st, u.m[0]), 0, ptr} -/* INLINE does not support ONEOF. */ - #define PB_ONEOF_FIELD(union_name, tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ PB_ONEOF_ ## allocation(union_name, tag, message, field, \ PB_DATAOFFSET_ ## placement(message, union_name.field, prevfield), \ diff --git a/pb_decode.c b/pb_decode.c index 92d0175d..a8cd61a7 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -42,6 +42,7 @@ static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *f static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest); +static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); static bool checkreturn pb_skip_varint(pb_istream_t *stream); static bool checkreturn pb_skip_string(pb_istream_t *stream); @@ -65,7 +66,7 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { &pb_dec_string, &pb_dec_submessage, NULL, /* extensions */ - &pb_dec_bytes /* PB_LTYPE_FIXED_LENGTH_BYTES */ + &pb_dec_fixed_length_bytes }; /******************************* @@ -1286,12 +1287,6 @@ static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *fie } else { - if (PB_LTYPE(field->type) == PB_LTYPE_FIXED_LENGTH_BYTES) { - if (size != field->data_size) - PB_RETURN_ERROR(stream, "incorrect inline bytes size"); - return pb_read(stream, (pb_byte_t*)dest, field->data_size); - } - if (alloc_size > field->data_size) PB_RETURN_ERROR(stream, "bytes overflow"); bdest = (pb_bytes_array_t*)dest; @@ -1359,3 +1354,26 @@ static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t return false; return status; } + +static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + uint32_t size; + + if (!pb_decode_varint32(stream, &size)) + return false; + + if (size > PB_SIZE_MAX) + PB_RETURN_ERROR(stream, "bytes overflow"); + + if (size == 0) + { + /* As a special case, treat empty bytes string as all zeros for fixed_length_bytes. */ + memset(dest, 0, field->data_size); + return true; + } + + if (size != field->data_size) + PB_RETURN_ERROR(stream, "incorrect fixed length bytes size"); + + return pb_read(stream, (pb_byte_t*)dest, field->data_size); +} diff --git a/pb_encode.c b/pb_encode.c index cafe853c..cd731dcf 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -35,6 +35,7 @@ static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *f static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src); static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src); static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_fixed_length_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src); /* --- Function pointers to field encoders --- * Order in the array must match pb_action_t LTYPE numbering. @@ -50,7 +51,7 @@ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { &pb_enc_string, &pb_enc_submessage, NULL, /* extensions */ - &pb_enc_bytes /* PB_LTYPE_FIXED_LENGTH_BYTES */ + &pb_enc_fixed_length_bytes }; /******************************* @@ -694,9 +695,6 @@ static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *fie { const pb_bytes_array_t *bytes = NULL; - if (PB_LTYPE(field->type) == PB_LTYPE_FIXED_LENGTH_BYTES) - return pb_encode_string(stream, (const pb_byte_t*)src, field->data_size); - bytes = (const pb_bytes_array_t*)src; if (src == NULL) @@ -748,3 +746,8 @@ static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t return pb_encode_submessage(stream, (const pb_field_t*)field->ptr, src); } +static bool checkreturn pb_enc_fixed_length_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) +{ + return pb_encode_string(stream, (const pb_byte_t*)src, field->data_size); +} + -- cgit 1.2.3-korg From 91dcdf573707a823bc450d3913b8b87c7adba212 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Wed, 22 Feb 2017 21:11:01 +0200 Subject: Add fixed length bytes to alltypes test case (issue #244) --- tests/alltypes/alltypes.options | 2 +- tests/alltypes/alltypes.proto | 8 +++++--- tests/alltypes/decode_alltypes.c | 8 ++++++++ tests/alltypes/encode_alltypes.c | 6 ++++++ tests/alltypes_callback/alltypes.options | 4 ++++ tests/alltypes_callback/decode_alltypes_callback.c | 6 +----- tests/alltypes_callback/encode_alltypes_callback.c | 9 +++++++++ tests/alltypes_pointer/alltypes.options | 1 + tests/alltypes_pointer/decode_alltypes_pointer.c | 8 +++++++- tests/alltypes_pointer/encode_alltypes_pointer.c | 6 ++++++ tests/alltypes_proto3/alltypes.options | 1 + tests/alltypes_proto3/alltypes.proto | 2 ++ tests/alltypes_proto3/decode_alltypes.c | 9 +++++++++ tests/alltypes_proto3/encode_alltypes.c | 4 ++++ tests/field_size_16/alltypes.options | 1 + tests/field_size_16/alltypes.proto | 8 +++++--- tests/field_size_32/alltypes.options | 2 +- tests/field_size_32/alltypes.proto | 8 +++++--- tests/fuzztest/alltypes_pointer.options | 2 +- tests/fuzztest/alltypes_static.options | 1 + tests/io_errors/alltypes.options | 2 +- tests/io_errors_pointers/alltypes.options | 2 +- 22 files changed, 80 insertions(+), 20 deletions(-) diff --git a/tests/alltypes/alltypes.options b/tests/alltypes/alltypes.options index b31e3cf0..0d5ab12b 100644 --- a/tests/alltypes/alltypes.options +++ b/tests/alltypes/alltypes.options @@ -1,3 +1,3 @@ * max_size:16 * max_count:5 - +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/alltypes/alltypes.proto b/tests/alltypes/alltypes.proto index 3995c552..b2250c00 100644 --- a/tests/alltypes/alltypes.proto +++ b/tests/alltypes/alltypes.proto @@ -58,7 +58,7 @@ message AllTypes { required SubMessage req_submsg = 16; required MyEnum req_enum = 17; required EmptyMessage req_emptymsg = 18; - + required bytes req_fbytes = 19; repeated int32 rep_int32 = 21 [packed = true]; repeated int64 rep_int64 = 22 [packed = true]; @@ -81,6 +81,7 @@ message AllTypes { repeated SubMessage rep_submsg = 36; repeated MyEnum rep_enum = 37 [packed = true]; repeated EmptyMessage rep_emptymsg = 38; + repeated bytes rep_fbytes = 39; optional int32 opt_int32 = 41 [default = 4041]; optional int64 opt_int64 = 42 [default = 4042]; @@ -103,11 +104,12 @@ message AllTypes { optional SubMessage opt_submsg = 56; optional MyEnum opt_enum = 57 [default = Second]; optional EmptyMessage opt_emptymsg = 58; + optional bytes opt_fbytes = 59 [default = "4059"]; oneof oneof { - SubMessage oneof_msg1 = 59; - EmptyMessage oneof_msg2 = 60; + SubMessage oneof_msg1 = 60; + EmptyMessage oneof_msg2 = 61; } // Check that extreme integer values are handled correctly diff --git a/tests/alltypes/decode_alltypes.c b/tests/alltypes/decode_alltypes.c index 458e5112..2e609e56 100644 --- a/tests/alltypes/decode_alltypes.c +++ b/tests/alltypes/decode_alltypes.c @@ -52,6 +52,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.req_submsg.substuff2 == 1016); TEST(alltypes.req_submsg.substuff3 == 3); TEST(alltypes.req_enum == MyEnum_Truth); + TEST(memcmp(alltypes.req_fbytes, "1019", 4) == 0); TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); @@ -80,6 +81,9 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); TEST(alltypes.rep_emptymsg_count == 5); + TEST(alltypes.rep_fbytes_count == 5); + TEST(alltypes.rep_fbytes[0][0] == 0 && alltypes.rep_fbytes[0][3] == 0); + TEST(memcmp(alltypes.rep_fbytes[4], "2019", 4) == 0); if (mode == 0) { @@ -125,6 +129,8 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.has_opt_enum == false); TEST(alltypes.opt_enum == MyEnum_Second); TEST(alltypes.has_opt_emptymsg == false); + TEST(alltypes.has_opt_fbytes == false); + TEST(memcmp(alltypes.opt_fbytes, "4059", 4) == 0); TEST(alltypes.which_oneof == 0); } @@ -172,6 +178,8 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.has_opt_enum == true); TEST(alltypes.opt_enum == MyEnum_Truth); TEST(alltypes.has_opt_emptymsg == true); + TEST(alltypes.has_opt_fbytes == true); + TEST(memcmp(alltypes.opt_fbytes, "3059", 4) == 0); TEST(alltypes.which_oneof == AllTypes_oneof_msg1_tag); TEST(strcmp(alltypes.oneof.oneof_msg1.substuff1, "4059") == 0); diff --git a/tests/alltypes/encode_alltypes.c b/tests/alltypes/encode_alltypes.c index 16f4b298..1b863555 100644 --- a/tests/alltypes/encode_alltypes.c +++ b/tests/alltypes/encode_alltypes.c @@ -37,6 +37,7 @@ int main(int argc, char **argv) strcpy(alltypes.req_submsg.substuff1, "1016"); alltypes.req_submsg.substuff2 = 1016; alltypes.req_enum = MyEnum_Truth; + memcpy(alltypes.req_fbytes, "1019", 4); alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002; @@ -67,6 +68,9 @@ int main(int argc, char **argv) alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; alltypes.rep_emptymsg_count = 5; + alltypes.rep_fbytes_count = 5; + memcpy(alltypes.rep_fbytes[4], "2019", 4); + alltypes.req_limits.int32_min = INT32_MIN; alltypes.req_limits.int32_max = INT32_MAX; alltypes.req_limits.uint32_min = 0; @@ -121,6 +125,8 @@ int main(int argc, char **argv) alltypes.has_opt_enum = true; alltypes.opt_enum = MyEnum_Truth; alltypes.has_opt_emptymsg = true; + alltypes.has_opt_fbytes = true; + memcpy(alltypes.opt_fbytes, "3059", 4); alltypes.which_oneof = AllTypes_oneof_msg1_tag; strcpy(alltypes.oneof.oneof_msg1.substuff1, "4059"); diff --git a/tests/alltypes_callback/alltypes.options b/tests/alltypes_callback/alltypes.options index daee5224..74d7a9c0 100644 --- a/tests/alltypes_callback/alltypes.options +++ b/tests/alltypes_callback/alltypes.options @@ -2,3 +2,7 @@ AllTypes.* type:FT_CALLBACK SubMessage.substuff1 max_size:16 AllTypes.oneof no_unions:true + +# With FT_CALLBACK, these options should get ignored +*.*fbytes fixed_length:true max_size:4 + diff --git a/tests/alltypes_callback/decode_alltypes_callback.c b/tests/alltypes_callback/decode_alltypes_callback.c index c53ab6ed..4366cf3f 100644 --- a/tests/alltypes_callback/decode_alltypes_callback.c +++ b/tests/alltypes_callback/decode_alltypes_callback.c @@ -218,11 +218,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) SubMessage oneof_msg1 = {"4059", 4059}; /* Bind callbacks for required fields */ - AllTypes alltypes; - - /* Fill with garbage to better detect initialization errors */ - memset(&alltypes, 0xAA, sizeof(alltypes)); - alltypes.extensions = 0; + AllTypes alltypes = AllTypes_init_zero; alltypes.req_int32.funcs.decode = &read_varint; alltypes.req_int32.arg = (void*)-1001; diff --git a/tests/alltypes_callback/encode_alltypes_callback.c b/tests/alltypes_callback/encode_alltypes_callback.c index abc43f58..b206783b 100644 --- a/tests/alltypes_callback/encode_alltypes_callback.c +++ b/tests/alltypes_callback/encode_alltypes_callback.c @@ -263,6 +263,9 @@ int main(int argc, char **argv) alltypes.req_emptymsg.funcs.encode = &write_emptymsg; + alltypes.req_fbytes.funcs.encode = &write_string; + alltypes.req_fbytes.arg = "1019"; + /* Bind callbacks for repeated fields */ alltypes.rep_int32.funcs.encode = &write_repeated_varint; alltypes.rep_int32.arg = (void*)-2001; @@ -317,6 +320,9 @@ int main(int argc, char **argv) alltypes.rep_emptymsg.funcs.encode = &write_repeated_emptymsg; + alltypes.rep_fbytes.funcs.encode = &write_repeated_string; + alltypes.rep_fbytes.arg = "2019"; + alltypes.req_limits.funcs.encode = &write_limits; /* Bind callbacks for optional fields */ @@ -375,6 +381,9 @@ int main(int argc, char **argv) alltypes.opt_emptymsg.funcs.encode = &write_emptymsg; + alltypes.opt_fbytes.funcs.encode = &write_string; + alltypes.opt_fbytes.arg = "3059"; + alltypes.oneof_msg1.funcs.encode = &write_submsg; alltypes.oneof_msg1.arg = &oneof_msg1; } diff --git a/tests/alltypes_pointer/alltypes.options b/tests/alltypes_pointer/alltypes.options index 52abeb7f..8699fe27 100644 --- a/tests/alltypes_pointer/alltypes.options +++ b/tests/alltypes_pointer/alltypes.options @@ -1,3 +1,4 @@ # Generate all fields as pointers. * type:FT_POINTER +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/alltypes_pointer/decode_alltypes_pointer.c b/tests/alltypes_pointer/decode_alltypes_pointer.c index 1dbb6c55..4ee6f8bf 100644 --- a/tests/alltypes_pointer/decode_alltypes_pointer.c +++ b/tests/alltypes_pointer/decode_alltypes_pointer.c @@ -47,7 +47,8 @@ bool check_alltypes(pb_istream_t *stream, int mode) && strcmp(alltypes.req_submsg->substuff1, "1016") == 0); TEST(alltypes.req_submsg && alltypes.req_submsg->substuff2 && *alltypes.req_submsg->substuff2 == 1016); - TEST(*alltypes.req_enum == MyEnum_Truth); + TEST(alltypes.req_enum && *alltypes.req_enum == MyEnum_Truth); + TEST(alltypes.req_fbytes && memcmp(alltypes.req_fbytes, "1019", 4) == 0); TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); @@ -76,6 +77,9 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); TEST(alltypes.rep_emptymsg_count == 5); + TEST(alltypes.rep_fbytes_count == 5); + TEST(alltypes.rep_fbytes[0][0] == 0 && alltypes.rep_fbytes[0][3] == 0); + TEST(memcmp(alltypes.rep_fbytes[4], "2019", 4) == 0); if (mode == 0) { @@ -99,6 +103,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.opt_bytes == NULL); TEST(alltypes.opt_submsg == NULL); TEST(alltypes.opt_enum == NULL); + TEST(alltypes.opt_fbytes == NULL); TEST(alltypes.which_oneof == 0); } @@ -127,6 +132,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.opt_submsg && *alltypes.opt_submsg->substuff2 == 3056); TEST(alltypes.opt_enum && *alltypes.opt_enum == MyEnum_Truth); TEST(alltypes.opt_emptymsg); + TEST(alltypes.opt_fbytes && memcmp(alltypes.opt_fbytes, "3059", 4) == 0); TEST(alltypes.which_oneof == AllTypes_oneof_msg1_tag); TEST(alltypes.oneof.oneof_msg1 && strcmp(alltypes.oneof.oneof_msg1->substuff1, "4059") == 0); diff --git a/tests/alltypes_pointer/encode_alltypes_pointer.c b/tests/alltypes_pointer/encode_alltypes_pointer.c index 7b52662f..a39af6fa 100644 --- a/tests/alltypes_pointer/encode_alltypes_pointer.c +++ b/tests/alltypes_pointer/encode_alltypes_pointer.c @@ -32,6 +32,7 @@ int main(int argc, char **argv) SubMessage req_submsg = {"1016", &req_substuff}; MyEnum req_enum = MyEnum_Truth; EmptyMessage req_emptymsg = {0}; + pb_byte_t req_fbytes[4] = {'1', '0', '1', '9'}; int32_t end = 1099; @@ -62,6 +63,7 @@ int main(int argc, char **argv) {"2016", &rep_substuff2, &rep_substuff3}}; MyEnum rep_enum[5] = {0, 0, 0, 0, MyEnum_Truth}; EmptyMessage rep_emptymsg[5] = {{0}, {0}, {0}, {0}, {0}}; + pb_byte_t rep_fbytes[5][4] = {{0}, {0}, {0}, {0}, {'2', '0', '1', '9'}}; /* Values for optional fields */ int32_t opt_int32 = 3041; @@ -83,6 +85,7 @@ int main(int argc, char **argv) SubMessage opt_submsg = {"3056", &opt_substuff}; MyEnum opt_enum = MyEnum_Truth; EmptyMessage opt_emptymsg = {0}; + pb_byte_t opt_fbytes[4] = {'3', '0', '5', '9'}; static int32_t oneof_substuff = 4059; SubMessage oneof_msg1 = {"4059", &oneof_substuff}; @@ -125,6 +128,7 @@ int main(int argc, char **argv) alltypes.req_submsg = &req_submsg; alltypes.req_enum = &req_enum; alltypes.req_emptymsg = &req_emptymsg; + alltypes.req_fbytes = &req_fbytes; alltypes.req_limits = &req_limits; alltypes.rep_int32_count = 5; alltypes.rep_int32 = rep_int32; @@ -145,6 +149,7 @@ int main(int argc, char **argv) alltypes.rep_submsg_count = 5; alltypes.rep_submsg = rep_submsg; alltypes.rep_enum_count = 5; alltypes.rep_enum = rep_enum; alltypes.rep_emptymsg_count = 5; alltypes.rep_emptymsg = rep_emptymsg; + alltypes.rep_fbytes_count = 5; alltypes.rep_fbytes = rep_fbytes; if (mode != 0) { @@ -167,6 +172,7 @@ int main(int argc, char **argv) alltypes.opt_submsg = &opt_submsg; alltypes.opt_enum = &opt_enum; alltypes.opt_emptymsg = &opt_emptymsg; + alltypes.opt_fbytes = &opt_fbytes; alltypes.which_oneof = AllTypes_oneof_msg1_tag; alltypes.oneof.oneof_msg1 = &oneof_msg1; diff --git a/tests/alltypes_proto3/alltypes.options b/tests/alltypes_proto3/alltypes.options index b31e3cf0..78dd08d1 100644 --- a/tests/alltypes_proto3/alltypes.options +++ b/tests/alltypes_proto3/alltypes.options @@ -1,3 +1,4 @@ * max_size:16 * max_count:5 +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/alltypes_proto3/alltypes.proto b/tests/alltypes_proto3/alltypes.proto index 10b48f25..f66109ec 100644 --- a/tests/alltypes_proto3/alltypes.proto +++ b/tests/alltypes_proto3/alltypes.proto @@ -59,6 +59,7 @@ message AllTypes { SubMessage sng_submsg = 16; MyEnum sng_enum = 17; EmptyMessage sng_emptymsg = 18; + bytes sng_fbytes = 19; repeated int32 rep_int32 = 21 [packed = true]; repeated int64 rep_int64 = 22 [packed = true]; @@ -81,6 +82,7 @@ message AllTypes { repeated SubMessage rep_submsg = 36; repeated MyEnum rep_enum = 37 [packed = true]; repeated EmptyMessage rep_emptymsg = 38; + repeated bytes rep_fbytes = 39; oneof oneof { diff --git a/tests/alltypes_proto3/decode_alltypes.c b/tests/alltypes_proto3/decode_alltypes.c index c1b0d524..51c1c41e 100644 --- a/tests/alltypes_proto3/decode_alltypes.c +++ b/tests/alltypes_proto3/decode_alltypes.c @@ -55,6 +55,10 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); TEST(alltypes.rep_emptymsg_count == 5); + TEST(alltypes.rep_fbytes_count == 5); + TEST(alltypes.rep_fbytes[0][0] == 0 && alltypes.rep_fbytes[0][3] == 0); + TEST(memcmp(alltypes.rep_fbytes[4], "2019", 4) == 0); + if (mode == 0) { /* Expect default values */ @@ -80,6 +84,10 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.sng_submsg.substuff2 == 0); TEST(alltypes.sng_submsg.substuff3 == 0); TEST(alltypes.sng_enum == MyEnum_Zero); + TEST(alltypes.sng_fbytes[0] == 0 && + alltypes.sng_fbytes[1] == 0 && + alltypes.sng_fbytes[2] == 0 && + alltypes.sng_fbytes[3] == 0); TEST(alltypes.which_oneof == 0); } @@ -109,6 +117,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.sng_submsg.substuff2 == 3056); TEST(alltypes.sng_submsg.substuff3 == 0); TEST(alltypes.sng_enum == MyEnum_Truth); + TEST(memcmp(alltypes.sng_fbytes, "3059", 4) == 0); TEST(alltypes.which_oneof == AllTypes_oneof_msg1_tag); TEST(strcmp(alltypes.oneof.oneof_msg1.substuff1, "4059") == 0); diff --git a/tests/alltypes_proto3/encode_alltypes.c b/tests/alltypes_proto3/encode_alltypes.c index e11acd5e..1da06688 100644 --- a/tests/alltypes_proto3/encode_alltypes.c +++ b/tests/alltypes_proto3/encode_alltypes.c @@ -43,6 +43,9 @@ int main(int argc, char **argv) alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; alltypes.rep_emptymsg_count = 5; + alltypes.rep_fbytes_count = 5; + memcpy(alltypes.rep_fbytes[4], "2019", 4); + alltypes.req_limits.int32_min = INT32_MIN; alltypes.req_limits.int32_max = INT32_MAX; alltypes.req_limits.uint32_min = 0; @@ -79,6 +82,7 @@ int main(int argc, char **argv) strcpy(alltypes.sng_submsg.substuff1, "3056"); alltypes.sng_submsg.substuff2 = 3056; alltypes.sng_enum = MyEnum_Truth; + memcpy(alltypes.sng_fbytes, "3059", 4); alltypes.which_oneof = AllTypes_oneof_msg1_tag; strcpy(alltypes.oneof.oneof_msg1.substuff1, "4059"); diff --git a/tests/field_size_16/alltypes.options b/tests/field_size_16/alltypes.options index b31e3cf0..78dd08d1 100644 --- a/tests/field_size_16/alltypes.options +++ b/tests/field_size_16/alltypes.options @@ -1,3 +1,4 @@ * max_size:16 * max_count:5 +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/field_size_16/alltypes.proto b/tests/field_size_16/alltypes.proto index ba1ec383..46ac46af 100644 --- a/tests/field_size_16/alltypes.proto +++ b/tests/field_size_16/alltypes.proto @@ -57,7 +57,7 @@ message AllTypes { required SubMessage req_submsg = 16; required MyEnum req_enum = 17; required EmptyMessage req_emptymsg = 18; - + required bytes req_fbytes = 19; repeated int32 rep_int32 = 21; repeated int64 rep_int64 = 22; @@ -80,6 +80,7 @@ message AllTypes { repeated SubMessage rep_submsg = 10036; repeated MyEnum rep_enum = 10037; repeated EmptyMessage rep_emptymsg = 10038; + repeated bytes rep_fbytes = 10039; optional int32 opt_int32 = 10041 [default = 4041]; optional int64 opt_int64 = 10042 [default = 4042]; @@ -102,11 +103,12 @@ message AllTypes { optional SubMessage opt_submsg = 10056; optional MyEnum opt_enum = 10057 [default = Second]; optional EmptyMessage opt_emptymsg = 10058; + optional bytes opt_fbytes = 10059 [default = "4059"]; oneof oneof { - SubMessage oneof_msg1 = 10059; - EmptyMessage oneof_msg2 = 10060; + SubMessage oneof_msg1 = 10060; + EmptyMessage oneof_msg2 = 10061; } // Check that extreme integer values are handled correctly diff --git a/tests/field_size_32/alltypes.options b/tests/field_size_32/alltypes.options index b31e3cf0..0d5ab12b 100644 --- a/tests/field_size_32/alltypes.options +++ b/tests/field_size_32/alltypes.options @@ -1,3 +1,3 @@ * max_size:16 * max_count:5 - +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/field_size_32/alltypes.proto b/tests/field_size_32/alltypes.proto index 02ee1a6a..ac76c8ef 100644 --- a/tests/field_size_32/alltypes.proto +++ b/tests/field_size_32/alltypes.proto @@ -57,7 +57,7 @@ message AllTypes { required SubMessage req_submsg = 16; required MyEnum req_enum = 17; required EmptyMessage req_emptymsg = 18; - + required bytes req_fbytes = 19; repeated int32 rep_int32 = 21; repeated int64 rep_int64 = 22; @@ -80,6 +80,7 @@ message AllTypes { repeated SubMessage rep_submsg = 10036; repeated MyEnum rep_enum = 10037; repeated EmptyMessage rep_emptymsg = 10038; + repeated bytes rep_fbytes = 10039; optional int32 opt_int32 = 10041 [default = 4041]; optional int64 opt_int64 = 10042 [default = 4042]; @@ -102,11 +103,12 @@ message AllTypes { optional SubMessage opt_submsg = 10056; optional MyEnum opt_enum = 10057 [default = Second]; optional EmptyMessage opt_emptymsg = 10058; + optional bytes opt_fbytes = 10059 [default = "4059"]; oneof oneof { - SubMessage oneof_msg1 = 10059; - EmptyMessage oneof_msg2 = 10060; + SubMessage oneof_msg1 = 10060; + EmptyMessage oneof_msg2 = 10061; } // Check that extreme integer values are handled correctly diff --git a/tests/fuzztest/alltypes_pointer.options b/tests/fuzztest/alltypes_pointer.options index 52abeb7f..7e3ad1e5 100644 --- a/tests/fuzztest/alltypes_pointer.options +++ b/tests/fuzztest/alltypes_pointer.options @@ -1,3 +1,3 @@ # Generate all fields as pointers. * type:FT_POINTER - +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/fuzztest/alltypes_static.options b/tests/fuzztest/alltypes_static.options index 1c10637c..e197e1df 100644 --- a/tests/fuzztest/alltypes_static.options +++ b/tests/fuzztest/alltypes_static.options @@ -1,3 +1,4 @@ * max_size:32 * max_count:8 *.extensions type:FT_IGNORE +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/io_errors/alltypes.options b/tests/io_errors/alltypes.options index b31e3cf0..0d5ab12b 100644 --- a/tests/io_errors/alltypes.options +++ b/tests/io_errors/alltypes.options @@ -1,3 +1,3 @@ * max_size:16 * max_count:5 - +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/io_errors_pointers/alltypes.options b/tests/io_errors_pointers/alltypes.options index 52abeb7f..7e3ad1e5 100644 --- a/tests/io_errors_pointers/alltypes.options +++ b/tests/io_errors_pointers/alltypes.options @@ -1,3 +1,3 @@ # Generate all fields as pointers. * type:FT_POINTER - +*.*fbytes fixed_length:true max_size:4 -- cgit 1.2.3-korg From 599387d2446bc622d27fd13291ad12b365926c49 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 24 Feb 2017 20:49:36 +0200 Subject: Add test coverage for calling pb_release() on a message with callback fields. --- tests/alltypes_callback/SConscript | 6 +++++- tests/alltypes_callback/decode_alltypes_callback.c | 9 ++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/alltypes_callback/SConscript b/tests/alltypes_callback/SConscript index a241f24e..1cef3972 100644 --- a/tests/alltypes_callback/SConscript +++ b/tests/alltypes_callback/SConscript @@ -1,6 +1,6 @@ # Test the AllTypes encoding & decoding using callbacks for all fields. -Import("env") +Import("env", "malloc_env") c = Copy("$TARGET", "$SOURCE") env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) @@ -21,3 +21,7 @@ env.RunTest("optionals.output", enc, ARGS = ['1']) env.RunTest("optionals.refdecout", [refdec, "optionals.output"], ARGS = ['1']) env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) +# Try with malloc support also +mallocbin = malloc_env.Object("decode_with_malloc.o", "decode_alltypes_callback.c") +mallocdec = malloc_env.Program("decode_with_malloc", [mallocbin, "alltypes.pb.o", "$COMMON/pb_decode_with_malloc.o", "$COMMON/pb_common_with_malloc.o", "$COMMON/malloc_wrappers.o"]) +env.RunTest("decode_with_malloc.output", [mallocdec, "encode_alltypes_callback.output"]) diff --git a/tests/alltypes_callback/decode_alltypes_callback.c b/tests/alltypes_callback/decode_alltypes_callback.c index 4366cf3f..1d8c268c 100644 --- a/tests/alltypes_callback/decode_alltypes_callback.c +++ b/tests/alltypes_callback/decode_alltypes_callback.c @@ -395,7 +395,14 @@ bool check_alltypes(pb_istream_t *stream, int mode) alltypes.oneof_msg1.arg = &oneof_msg1; } - return pb_decode(stream, AllTypes_fields, &alltypes); + bool status = pb_decode(stream, AllTypes_fields, &alltypes); + +#ifdef PB_ENABLE_MALLOC + /* Just to check for any interference between pb_release() and callback fields */ + pb_release(AllTypes_fields, &alltypes); +#endif + + return status; } int main(int argc, char **argv) -- cgit 1.2.3-korg From 60299d9270c1e7114a5aec864f192269ba6731dd Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 24 Feb 2017 21:00:51 +0200 Subject: Fix build failure --- tests/alltypes_callback/decode_alltypes_callback.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/alltypes_callback/decode_alltypes_callback.c b/tests/alltypes_callback/decode_alltypes_callback.c index 1d8c268c..ec946909 100644 --- a/tests/alltypes_callback/decode_alltypes_callback.c +++ b/tests/alltypes_callback/decode_alltypes_callback.c @@ -177,6 +177,7 @@ static bool read_limits(pb_istream_t *stream, const pb_field_t *field, void **ar bool check_alltypes(pb_istream_t *stream, int mode) { /* Values for use from callbacks through pointers. */ + bool status; uint32_t req_fixed32 = 1008; int32_t req_sfixed32 = -1009; float req_float = 1010.0f; @@ -395,7 +396,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) alltypes.oneof_msg1.arg = &oneof_msg1; } - bool status = pb_decode(stream, AllTypes_fields, &alltypes); + status = pb_decode(stream, AllTypes_fields, &alltypes); #ifdef PB_ENABLE_MALLOC /* Just to check for any interference between pb_release() and callback fields */ -- cgit 1.2.3-korg From 119d0fd396a9cb3b6e7067f8102694f1485e2e94 Mon Sep 17 00:00:00 2001 From: Justin DeMartino Date: Fri, 24 Feb 2017 20:55:14 -0800 Subject: Fix infinite loop in pb_check_proto3_default_value - Occurs with proto3, PB_FIELD_16BIT and submessage > 255 bytes - Add test case for PB_FIELD_16BIT and proto3 --- pb_encode.c | 2 +- tests/field_size_16_proto3/SConscript | 34 ++++++ tests/field_size_16_proto3/alltypes.options | 4 + tests/field_size_16_proto3/alltypes.proto | 100 ++++++++++++++++ tests/field_size_16_proto3/decode_alltypes.c | 167 +++++++++++++++++++++++++++ tests/field_size_16_proto3/encode_alltypes.c | 111 ++++++++++++++++++ 6 files changed, 417 insertions(+), 1 deletion(-) create mode 100644 tests/field_size_16_proto3/SConscript create mode 100644 tests/field_size_16_proto3/alltypes.options create mode 100644 tests/field_size_16_proto3/alltypes.proto create mode 100644 tests/field_size_16_proto3/decode_alltypes.c create mode 100644 tests/field_size_16_proto3/encode_alltypes.c diff --git a/pb_encode.c b/pb_encode.c index cd731dcf..0d6e1e7e 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -228,7 +228,7 @@ static bool pb_check_proto3_default_value(const pb_field_t *field, const void *p * pb_enc_varint(). (Casting to char* is safe with regards * to C strict aliasing rules.) */ - uint_fast8_t i; + pb_size_t i; const char *p = (const char*)pData; for (i = 0; i < field->data_size; i++) { diff --git a/tests/field_size_16_proto3/SConscript b/tests/field_size_16_proto3/SConscript new file mode 100644 index 00000000..912c0389 --- /dev/null +++ b/tests/field_size_16_proto3/SConscript @@ -0,0 +1,34 @@ +# Version of AllTypes test case for protobuf 3 file format. + +Import("env") + +import re +match = None +if 'PROTOC_VERSION' in env: + match = re.search('([0-9]+).([0-9]+).([0-9]+)', env['PROTOC_VERSION']) + +if match: + version = map(int, match.groups()) + +# proto3 syntax is supported by protoc >= 3.0.0 +if env.GetOption('clean') or (match and version[0] >= 3): + + env.NanopbProto(["alltypes", "alltypes.options"]) + + # Define the compilation options + opts = env.Clone() + opts.Append(CPPDEFINES = {'PB_FIELD_16BIT': 1}) + + # Build new version of core + strict = opts.Clone() + strict.Append(CFLAGS = strict['CORECFLAGS']) + strict.Object("pb_decode_fields16.o", "$NANOPB/pb_decode.c") + strict.Object("pb_encode_fields16.o", "$NANOPB/pb_encode.c") + strict.Object("pb_common_fields16.o", "$NANOPB/pb_common.c") + + # Now build and run the test normally. + enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields16.o", "pb_common_fields16.o"]) + dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields16.o", "pb_common_fields16.o"]) + + env.RunTest(enc) + env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/field_size_16_proto3/alltypes.options b/tests/field_size_16_proto3/alltypes.options new file mode 100644 index 00000000..edfbe788 --- /dev/null +++ b/tests/field_size_16_proto3/alltypes.options @@ -0,0 +1,4 @@ +* max_size:16 +* max_count:5 +*.*fbytes fixed_length:true max_size:4 +SubMessage.substuff1 max_size:256 diff --git a/tests/field_size_16_proto3/alltypes.proto b/tests/field_size_16_proto3/alltypes.proto new file mode 100644 index 00000000..f66109ec --- /dev/null +++ b/tests/field_size_16_proto3/alltypes.proto @@ -0,0 +1,100 @@ +syntax = "proto3"; +// package name placeholder + +message SubMessage { + string substuff1 = 1; + int32 substuff2 = 2; + fixed32 substuff3 = 3; +} + +message EmptyMessage { + +} + +enum HugeEnum { + HE_Zero = 0; + Negative = -2147483647; /* protoc doesn't accept -2147483648 here */ + Positive = 2147483647; +} + +message Limits { + int32 int32_min = 1; + int32 int32_max = 2; + uint32 uint32_min = 3; + uint32 uint32_max = 4; + int64 int64_min = 5; + int64 int64_max = 6; + uint64 uint64_min = 7; + uint64 uint64_max = 8; + HugeEnum enum_min = 9; + HugeEnum enum_max = 10; +} + +enum MyEnum { + Zero = 0; + First = 1; + Second = 2; + Truth = 42; +} + +message AllTypes { + int32 sng_int32 = 1; + int64 sng_int64 = 2; + uint32 sng_uint32 = 3; + uint64 sng_uint64 = 4; + sint32 sng_sint32 = 5; + sint64 sng_sint64 = 6; + bool sng_bool = 7; + + fixed32 sng_fixed32 = 8; + sfixed32 sng_sfixed32= 9; + float sng_float = 10; + + fixed64 sng_fixed64 = 11; + sfixed64 sng_sfixed64= 12; + double sng_double = 13; + + string sng_string = 14; + bytes sng_bytes = 15; + SubMessage sng_submsg = 16; + MyEnum sng_enum = 17; + EmptyMessage sng_emptymsg = 18; + bytes sng_fbytes = 19; + + repeated int32 rep_int32 = 21 [packed = true]; + repeated int64 rep_int64 = 22 [packed = true]; + repeated uint32 rep_uint32 = 23 [packed = true]; + repeated uint64 rep_uint64 = 24 [packed = true]; + repeated sint32 rep_sint32 = 25 [packed = true]; + repeated sint64 rep_sint64 = 26 [packed = true]; + repeated bool rep_bool = 27 [packed = true]; + + repeated fixed32 rep_fixed32 = 28 [packed = true]; + repeated sfixed32 rep_sfixed32= 29 [packed = true]; + repeated float rep_float = 30 [packed = true]; + + repeated fixed64 rep_fixed64 = 31 [packed = true]; + repeated sfixed64 rep_sfixed64= 32 [packed = true]; + repeated double rep_double = 33 [packed = true]; + + repeated string rep_string = 34; + repeated bytes rep_bytes = 35; + repeated SubMessage rep_submsg = 36; + repeated MyEnum rep_enum = 37 [packed = true]; + repeated EmptyMessage rep_emptymsg = 38; + repeated bytes rep_fbytes = 39; + + oneof oneof + { + SubMessage oneof_msg1 = 59; + EmptyMessage oneof_msg2 = 60; + } + + // Check that extreme integer values are handled correctly + Limits req_limits = 98; + + // Just to make sure that the size of the fields has been calculated + // properly, i.e. otherwise a bug in last field might not be detected. + int32 end = 99; +} + diff --git a/tests/field_size_16_proto3/decode_alltypes.c b/tests/field_size_16_proto3/decode_alltypes.c new file mode 100644 index 00000000..6611f8cc --- /dev/null +++ b/tests/field_size_16_proto3/decode_alltypes.c @@ -0,0 +1,167 @@ +/* Tests the decoding of all types. + * This is the counterpart of test_encode3. + * Run e.g. ./test_encode3 | ./test_decode3 + */ + +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "test_helpers.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed.\n"); \ + return false; \ + } + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_alltypes(pb_istream_t *stream, int mode) +{ + AllTypes alltypes = AllTypes_init_zero; + + /* Fill with garbage to better detect initialization errors */ + memset(&alltypes, 0xAA, sizeof(alltypes)); + + if (!pb_decode(stream, AllTypes_fields, &alltypes)) + return false; + + TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); + TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); + TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); + TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); + TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); + TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0); + TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); + + TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); + TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); + TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); + + TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); + TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0); + TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); + + TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); + TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); + TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); + + TEST(alltypes.rep_submsg_count == 5); + TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); + TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); + TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 0); + + TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); + TEST(alltypes.rep_emptymsg_count == 5); + + TEST(alltypes.rep_fbytes_count == 5); + TEST(alltypes.rep_fbytes[0][0] == 0 && alltypes.rep_fbytes[0][3] == 0); + TEST(memcmp(alltypes.rep_fbytes[4], "2019", 4) == 0); + + if (mode == 0) + { + /* Expect default values */ + TEST(alltypes.sng_int32 == 0); + TEST(alltypes.sng_int64 == 0); + TEST(alltypes.sng_uint32 == 0); + TEST(alltypes.sng_uint64 == 0); + TEST(alltypes.sng_sint32 == 0); + TEST(alltypes.sng_sint64 == 0); + TEST(alltypes.sng_bool == false); + + TEST(alltypes.sng_fixed32 == 0); + TEST(alltypes.sng_sfixed32 == 0); + TEST(alltypes.sng_float == 0.0f); + + TEST(alltypes.sng_fixed64 == 0); + TEST(alltypes.sng_sfixed64 == 0); + TEST(alltypes.sng_double == 0.0); + + TEST(strcmp(alltypes.sng_string, "") == 0); + TEST(alltypes.sng_bytes.size == 0); + TEST(strcmp(alltypes.sng_submsg.substuff1, "") == 0); + TEST(alltypes.sng_submsg.substuff2 == 0); + TEST(alltypes.sng_submsg.substuff3 == 0); + TEST(alltypes.sng_enum == MyEnum_Zero); + TEST(alltypes.sng_fbytes[0] == 0 && + alltypes.sng_fbytes[1] == 0 && + alltypes.sng_fbytes[2] == 0 && + alltypes.sng_fbytes[3] == 0); + + TEST(alltypes.which_oneof == 0); + } + else + { + /* Expect filled-in values */ + TEST(alltypes.sng_int32 == 3041); + TEST(alltypes.sng_int64 == 3042); + TEST(alltypes.sng_uint32 == 3043); + TEST(alltypes.sng_uint64 == 3044); + TEST(alltypes.sng_sint32 == 3045); + TEST(alltypes.sng_sint64 == 3046); + TEST(alltypes.sng_bool == true); + + TEST(alltypes.sng_fixed32 == 3048); + TEST(alltypes.sng_sfixed32 == 3049); + TEST(alltypes.sng_float == 3050.0f); + + TEST(alltypes.sng_fixed64 == 3051); + TEST(alltypes.sng_sfixed64 == 3052); + TEST(alltypes.sng_double == 3053.0); + + TEST(strcmp(alltypes.sng_string, "3054") == 0); + TEST(alltypes.sng_bytes.size == 4); + TEST(memcmp(alltypes.sng_bytes.bytes, "3055", 4) == 0); + TEST(strcmp(alltypes.sng_submsg.substuff1, "3056") == 0); + TEST(alltypes.sng_submsg.substuff2 == 3056); + TEST(alltypes.sng_submsg.substuff3 == 0); + TEST(alltypes.sng_enum == MyEnum_Truth); + TEST(memcmp(alltypes.sng_fbytes, "3059", 4) == 0); + + TEST(alltypes.which_oneof == AllTypes_oneof_msg1_tag); + TEST(strcmp(alltypes.oneof.oneof_msg1.substuff1, "4059") == 0); + TEST(alltypes.oneof.oneof_msg1.substuff2 == 4059); + } + + TEST(alltypes.req_limits.int32_min == INT32_MIN); + TEST(alltypes.req_limits.int32_max == INT32_MAX); + TEST(alltypes.req_limits.uint32_min == 0); + TEST(alltypes.req_limits.uint32_max == UINT32_MAX); + TEST(alltypes.req_limits.int64_min == INT64_MIN); + TEST(alltypes.req_limits.int64_max == INT64_MAX); + TEST(alltypes.req_limits.uint64_min == 0); + TEST(alltypes.req_limits.uint64_max == UINT64_MAX); + TEST(alltypes.req_limits.enum_min == HugeEnum_Negative); + TEST(alltypes.req_limits.enum_max == HugeEnum_Positive); + + TEST(alltypes.end == 1099); + + return true; +} + +int main(int argc, char **argv) +{ + uint8_t buffer[2048]; + size_t count; + pb_istream_t stream; + + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Read the data into buffer */ + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_alltypes(&stream, mode)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} diff --git a/tests/field_size_16_proto3/encode_alltypes.c b/tests/field_size_16_proto3/encode_alltypes.c new file mode 100644 index 00000000..1da06688 --- /dev/null +++ b/tests/field_size_16_proto3/encode_alltypes.c @@ -0,0 +1,111 @@ +/* Attempts to test all the datatypes supported by ProtoBuf3. + */ + +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "test_helpers.h" + +int main(int argc, char **argv) +{ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Initialize the structure with constants */ + AllTypes alltypes = AllTypes_init_zero; + + alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; + alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002; + alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; + alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004; + alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005; + alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006; + alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; + + alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; + alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009; + alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; + + alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011; + alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012; + alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0; + + alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); + alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4; + memcpy(alltypes.rep_bytes[4].bytes, "2015", 4); + + alltypes.rep_submsg_count = 5; + strcpy(alltypes.rep_submsg[4].substuff1, "2016"); + alltypes.rep_submsg[4].substuff2 = 2016; + alltypes.rep_submsg[4].substuff3 = 2016; + + alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; + alltypes.rep_emptymsg_count = 5; + + alltypes.rep_fbytes_count = 5; + memcpy(alltypes.rep_fbytes[4], "2019", 4); + + alltypes.req_limits.int32_min = INT32_MIN; + alltypes.req_limits.int32_max = INT32_MAX; + alltypes.req_limits.uint32_min = 0; + alltypes.req_limits.uint32_max = UINT32_MAX; + alltypes.req_limits.int64_min = INT64_MIN; + alltypes.req_limits.int64_max = INT64_MAX; + alltypes.req_limits.uint64_min = 0; + alltypes.req_limits.uint64_max = UINT64_MAX; + alltypes.req_limits.enum_min = HugeEnum_Negative; + alltypes.req_limits.enum_max = HugeEnum_Positive; + + if (mode != 0) + { + /* Fill in values for singular fields */ + alltypes.sng_int32 = 3041; + alltypes.sng_int64 = 3042; + alltypes.sng_uint32 = 3043; + alltypes.sng_uint64 = 3044; + alltypes.sng_sint32 = 3045; + alltypes.sng_sint64 = 3046; + alltypes.sng_bool = true; + + alltypes.sng_fixed32 = 3048; + alltypes.sng_sfixed32 = 3049; + alltypes.sng_float = 3050.0f; + + alltypes.sng_fixed64 = 3051; + alltypes.sng_sfixed64 = 3052; + alltypes.sng_double = 3053.0; + + strcpy(alltypes.sng_string, "3054"); + alltypes.sng_bytes.size = 4; + memcpy(alltypes.sng_bytes.bytes, "3055", 4); + strcpy(alltypes.sng_submsg.substuff1, "3056"); + alltypes.sng_submsg.substuff2 = 3056; + alltypes.sng_enum = MyEnum_Truth; + memcpy(alltypes.sng_fbytes, "3059", 4); + + alltypes.which_oneof = AllTypes_oneof_msg1_tag; + strcpy(alltypes.oneof.oneof_msg1.substuff1, "4059"); + alltypes.oneof.oneof_msg1.substuff2 = 4059; + } + + alltypes.end = 1099; + + { + uint8_t buffer[AllTypes_size]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } + } +} -- cgit 1.2.3-korg From b5abefc25d033f7e12e369f5c664f1f0efaf398b Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 25 Feb 2017 17:14:39 +0200 Subject: Windows build fix --- tests/alltypes_callback/SConscript | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/alltypes_callback/SConscript b/tests/alltypes_callback/SConscript index 1cef3972..8be53908 100644 --- a/tests/alltypes_callback/SConscript +++ b/tests/alltypes_callback/SConscript @@ -22,6 +22,7 @@ env.RunTest("optionals.refdecout", [refdec, "optionals.output"], ARGS = ['1']) env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) # Try with malloc support also -mallocbin = malloc_env.Object("decode_with_malloc.o", "decode_alltypes_callback.c") -mallocdec = malloc_env.Program("decode_with_malloc", [mallocbin, "alltypes.pb.o", "$COMMON/pb_decode_with_malloc.o", "$COMMON/pb_common_with_malloc.o", "$COMMON/malloc_wrappers.o"]) +mallocbin1 = malloc_env.Object("decode_with_malloc.o", "decode_alltypes_callback.c") +mallocbin2 = malloc_env.Object("alltypes_malloc.pb.o", "alltypes.pb.c") +mallocdec = malloc_env.Program("decode_with_malloc", [mallocbin1, mallocbin2, "$COMMON/pb_decode_with_malloc.o", "$COMMON/pb_common_with_malloc.o", "$COMMON/malloc_wrappers.o"]) env.RunTest("decode_with_malloc.output", [mallocdec, "encode_alltypes_callback.output"]) -- cgit 1.2.3-korg From efb9a31c36b369e1b3cedfb72d58b634aae1d30c Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 25 Feb 2017 21:22:41 +0200 Subject: Add testcase for issue #247 --- tests/regression/issue_247/SConscript | 14 ++++++++++++++ tests/regression/issue_247/padding.c | 32 ++++++++++++++++++++++++++++++++ tests/regression/issue_247/padding.proto | 12 ++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 tests/regression/issue_247/SConscript create mode 100644 tests/regression/issue_247/padding.c create mode 100644 tests/regression/issue_247/padding.proto diff --git a/tests/regression/issue_247/SConscript b/tests/regression/issue_247/SConscript new file mode 100644 index 00000000..b41e9f29 --- /dev/null +++ b/tests/regression/issue_247/SConscript @@ -0,0 +1,14 @@ +# Test that pb_check_proto3_default_value() correctly skips padding +# bytes in submessage structures. + +Import("env") + +env.NanopbProto("padding") + +p = env.Program(["padding.c", + "padding.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) + +env.RunTest(p) + diff --git a/tests/regression/issue_247/padding.c b/tests/regression/issue_247/padding.c new file mode 100644 index 00000000..8860179d --- /dev/null +++ b/tests/regression/issue_247/padding.c @@ -0,0 +1,32 @@ +#include +#include +#include +#include "padding.pb.h" + +int main() +{ + int status = 0; + + TestMessage msg; + + /* Set padding bytes to garbage */ + memset(&msg, 0xAA, sizeof(msg)); + + /* Set all meaningful fields to 0 */ + msg.submsg.boolfield = false; + msg.submsg.intfield = 0; + + /* Test encoding */ + { + pb_byte_t buf[128] = {0}; + pb_ostream_t stream = pb_ostream_from_buffer(buf, sizeof(buf)); + TEST(pb_encode(&stream, TestMessage_fields, &msg)); + + /* Because all fields have zero values, proto3 encoder + * shouldn't write out anything. */ + TEST(stream.bytes_written == 0); + } + + return status; +} + diff --git a/tests/regression/issue_247/padding.proto b/tests/regression/issue_247/padding.proto new file mode 100644 index 00000000..20bddac3 --- /dev/null +++ b/tests/regression/issue_247/padding.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +import "nanopb.proto"; + +message SubMessage { + bool boolfield = 1; + int64 intfield = 2; +} + +message TestMessage { + SubMessage submsg = 1; +} + -- cgit 1.2.3-korg From 9fdea3a35e5b0153bf4e651a1a2a1b8f3f4bad50 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 25 Feb 2017 21:23:10 +0200 Subject: Make pb_check_proto3_default_value() recurse into submessages (issue #247) --- pb_encode.c | 70 +++++++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/pb_encode.c b/pb_encode.c index 0d6e1e7e..e1f37f6d 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -27,6 +27,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie static bool checkreturn encode_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData); static bool checkreturn default_extension_encoder(pb_ostream_t *stream, const pb_extension_t *extension); static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData); +static void *const_cast(const void *p); static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src); static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); @@ -203,30 +204,53 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie * This function implements the check for the zero value. */ static bool pb_check_proto3_default_value(const pb_field_t *field, const void *pData) { - if(PB_LTYPE(field->type) == PB_LTYPE_BYTES) + if (PB_ATYPE(field->type) == PB_ATYPE_STATIC) { - const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)pData; - return bytes->size == 0; - } - else if (PB_LTYPE(field->type) == PB_LTYPE_STRING) - { - return *(const char*)pData == '\0'; - } - else if (PB_LTYPE(field->type) == PB_LTYPE_FIXED_LENGTH_BYTES) - { - /* Fixed length bytes is only empty if its length is fixed - * as 0. Which would be pretty strange, but we can check - * it anyway. */ - return field->data_size == 0; + if (PB_LTYPE(field->type) == PB_LTYPE_BYTES) + { + const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)pData; + return bytes->size == 0; + } + else if (PB_LTYPE(field->type) == PB_LTYPE_STRING) + { + return *(const char*)pData == '\0'; + } + else if (PB_LTYPE(field->type) == PB_LTYPE_FIXED_LENGTH_BYTES) + { + /* Fixed length bytes is only empty if its length is fixed + * as 0. Which would be pretty strange, but we can check + * it anyway. */ + return field->data_size == 0; + } + else if (PB_LTYPE(field->type) == PB_LTYPE_SUBMESSAGE) + { + /* Check all fields in the submessage to find if any of them + * are non-zero. The comparison cannot be done byte-per-byte + * because the C struct may contain padding bytes that must + * be skipped. + */ + pb_field_iter_t iter; + if (pb_field_iter_begin(&iter, (const pb_field_t*)field->ptr, const_cast(pData))) + { + do + { + if (!pb_check_proto3_default_value(iter.pos, iter.pData)) + { + return false; + } + } while (pb_field_iter_next(&iter)); + } + return true; + } } - else + { - /* PB_LTYPE_VARINT, UVARINT, SVARINT, FIXED32, FIXED64, - * SUBMESSAGE, EXTENSION: These all have integer or pointer - * value which can be compared with 0. This does the check - * byte-by-byte to avoid the switch-cast logic used in - * pb_enc_varint(). (Casting to char* is safe with regards - * to C strict aliasing rules.) + /* Catch-all branch that does byte-per-byte comparison for zero value. + * + * This is for all pointer fields, and for static PB_LTYPE_VARINT, + * UVARINT, SVARINT, FIXED32, FIXED64, EXTENSION fields, and also + * callback fields. These all have integer or pointer value which + * can be compared with 0. */ pb_size_t i; const char *p = (const char*)pData; @@ -412,7 +436,7 @@ static bool checkreturn encode_extension_field(pb_ostream_t *stream, * Encode all fields * *********************/ -static void *remove_const(const void *p) +static void *const_cast(const void *p) { /* Note: this casts away const, in order to use the common field iterator * logic for both encoding and decoding. */ @@ -427,7 +451,7 @@ static void *remove_const(const void *p) bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) { pb_field_iter_t iter; - if (!pb_field_iter_begin(&iter, fields, remove_const(src_struct))) + if (!pb_field_iter_begin(&iter, fields, const_cast(src_struct))) return true; /* Empty message type */ do { -- cgit 1.2.3-korg From ab6017e195c9f0aa5209feafac54f0cb5f5315d4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 25 Feb 2017 21:31:06 +0200 Subject: Add better error message on Python library version imcompatibility (issue #240) --- generator/nanopb_generator.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 9cce6a5a..b5473174 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -32,6 +32,20 @@ except: try: import proto.nanopb_pb2 as nanopb_pb2 import proto.plugin_pb2 as plugin_pb2 +except TypeError: + sys.stderr.write(''' + **************************************************************************** + *** Got TypeError when importing the protocol definitions for generator. *** + *** This usually means that the protoc in your path doesn't match the *** + *** Python protobuf library version. *** + *** *** + *** Please check the output of the following commands: *** + *** which protoc *** + *** protoc --version *** + *** python -c 'import google.protobuf; print(google.protobuf.__file__)' *** + **************************************************************************** + ''' + '\n') + raise except: sys.stderr.write(''' ******************************************************************** -- cgit 1.2.3-korg From e5f420cf768becd1f7b2dd591151bb6f948023b3 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 25 Feb 2017 21:44:52 +0200 Subject: Add transitional options.proto file (#241) --- generator/nanopb/options.proto | 120 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 generator/nanopb/options.proto diff --git a/generator/nanopb/options.proto b/generator/nanopb/options.proto new file mode 100644 index 00000000..f08e53d1 --- /dev/null +++ b/generator/nanopb/options.proto @@ -0,0 +1,120 @@ +// This is a transitional file, to provide parallel support between the old +// nanopb.proto and new options.proto files. Eventually nanopb.proto will +// be left only for legacy code, but for now the generator is still also +// using it. However, your new code can start using this file already now. +// See pull request #241 for details: +// https://github.com/nanopb/nanopb/pull/241 + +// Custom options for defining: +// - Maximum size of string/bytes +// - Maximum number of elements in array +// +// These are used by nanopb to generate statically allocable structures +// for memory-limited environments. + +syntax = "proto2"; +import "google/protobuf/descriptor.proto"; + +package nanopb; +option java_package = "fi.kapsi.koti.jpa.nanopb"; + +enum FieldType { + FT_DEFAULT = 0; // Automatically decide field type, generate static field if possible. + FT_CALLBACK = 1; // Always generate a callback field. + FT_POINTER = 4; // Always generate a dynamically allocated field. + FT_STATIC = 2; // Generate a static field or raise an exception if not possible. + FT_IGNORE = 3; // Ignore the field completely. + FT_INLINE = 5; // Legacy option, use the separate 'fixed_length' option instead +} + +enum IntSize { + IS_DEFAULT = 0; // Default, 32/64bit based on type in .proto + IS_8 = 8; + IS_16 = 16; + IS_32 = 32; + IS_64 = 64; +} + +// This is the inner options message, which basically defines options for +// a field. When it is used in message or file scope, it applies to all +// fields. +message Options { + // Allocated size for 'bytes' and 'string' fields. + // For string fields, this should include the space for null terminator. + optional int32 max_size = 1; + + // Maximum length for 'string' fields. Setting this is equivalent + // to setting max_size to a value of length+1. + optional int32 max_length = 14; + + // Allocated number of entries in arrays ('repeated' fields) + optional int32 max_count = 2; + + // Size of integer fields. Can save some memory if you don't need + // full 32 bits for the value. + optional IntSize int_size = 7 [default = IS_DEFAULT]; + + // Force type of field (callback or static allocation) + optional FieldType type = 3 [default = FT_DEFAULT]; + + // Use long names for enums, i.e. EnumName_EnumValue. + optional bool long_names = 4 [default = true]; + + // Add 'packed' attribute to generated structs. + // Note: this cannot be used on CPUs that break on unaligned + // accesses to variables. + optional bool packed_struct = 5 [default = false]; + + // Add 'packed' attribute to generated enums. + optional bool packed_enum = 10 [default = false]; + + // Skip this message + optional bool skip_message = 6 [default = false]; + + // Generate oneof fields as normal optional fields instead of union. + optional bool no_unions = 8 [default = false]; + + // integer type tag for a message + optional uint32 msgid = 9; + + // decode oneof as anonymous union + optional bool anonymous_oneof = 11 [default = false]; + + // Proto3 singular field does not generate a "has_" flag + optional bool proto3 = 12 [default = false]; + + // Generate an enum->string mapping function (can take up lots of space). + optional bool enum_to_string = 13 [default = false]; + + // Generate bytes arrays with fixed length + optional bool fixed_length = 15 [default = false]; +} + +// Extensions to protoc 'Descriptor' type in order to define options +// inside a .proto file. +// +// Protocol Buffers extension number registry +// -------------------------------- +// Project: Nanopb +// Contact: Petteri Aimonen +// Web site: http://kapsi.fi/~jpa/nanopb +// Extensions: 1010 (all types) +// -------------------------------- + +extend google.protobuf.FileOptions { + optional Options fileopt = 1010; +} + +extend google.protobuf.MessageOptions { + optional Options msgopt = 1010; +} + +extend google.protobuf.EnumOptions { + optional Options enumopt = 1010; +} + +extend google.protobuf.FieldOptions { + optional Options fieldopt = 1010; +} + + -- cgit 1.2.3-korg From 019d7bf948fc0d14b310fa397948ac550858af77 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sat, 25 Feb 2017 21:47:24 +0200 Subject: Fix build failure (const_cast name conflict) Thought I was cleaning up the naming by calling the function const_cast(), because that's what it does. But of course it conflicts with the C++ builtin, so renaming it to pb_const_cast() now. --- pb_encode.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pb_encode.c b/pb_encode.c index e1f37f6d..30f60d83 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -27,7 +27,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie static bool checkreturn encode_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData); static bool checkreturn default_extension_encoder(pb_ostream_t *stream, const pb_extension_t *extension); static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData); -static void *const_cast(const void *p); +static void *pb_const_cast(const void *p); static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src); static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); @@ -230,7 +230,7 @@ static bool pb_check_proto3_default_value(const pb_field_t *field, const void *p * be skipped. */ pb_field_iter_t iter; - if (pb_field_iter_begin(&iter, (const pb_field_t*)field->ptr, const_cast(pData))) + if (pb_field_iter_begin(&iter, (const pb_field_t*)field->ptr, pb_const_cast(pData))) { do { @@ -436,7 +436,7 @@ static bool checkreturn encode_extension_field(pb_ostream_t *stream, * Encode all fields * *********************/ -static void *const_cast(const void *p) +static void *pb_const_cast(const void *p) { /* Note: this casts away const, in order to use the common field iterator * logic for both encoding and decoding. */ @@ -451,7 +451,7 @@ static void *const_cast(const void *p) bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) { pb_field_iter_t iter; - if (!pb_field_iter_begin(&iter, fields, const_cast(src_struct))) + if (!pb_field_iter_begin(&iter, fields, pb_const_cast(src_struct))) return true; /* Empty message type */ do { -- cgit 1.2.3-korg From ac6405b4b8c7b3f449bd19bc3f8b84e6bc9fff77 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 26 Feb 2017 10:13:38 +0200 Subject: Update changelog --- AUTHORS | 7 +++++-- CHANGELOG.txt | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/AUTHORS b/AUTHORS index 53d31d9e..7618ff77 100644 --- a/AUTHORS +++ b/AUTHORS @@ -2,7 +2,7 @@ Petteri Aimonen Michael Poole Daniel Kan Stan Hu -dch +David Hotham Steffen Siering Jens Steinhauser Pavel Ilin @@ -25,4 +25,7 @@ Tom Roeder Piotr Sikora Bernhard Krämer Konstantin Podsvirov - +William A. Kennington III +Guillaume Lager +Tobias Haegermarck +Justin DeMartino diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 02b7e71b..a5de21cc 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,22 @@ +nanopb-0.3.8 (2016-03-xx) + Fix problems with multiple oneofs in same message (#229) + Zero-valued extension fields were mistakenly ignored by encoder (#242) + Various fixes related to proto3 encoding (#242, #245, #247) + Fix potential unaligned access (#226, #227) + Fix documentation for protoc --plugin argument (#239) + Extend inline / fixed length bytes array support (#244) + Add new option max_length for strings (#107) + Make string substream API more robust (#230) + Make pb_decode_varint32 public API (#231) + Allow overriding proto3 mode (#228) + Add optional enum->string mapping function (#223) + Add transitional options.proto file (#241) + Add better error message on Python library version imcompatibility (#240) + Include version number in PlatformIO library.json (#222) + CMake build script changes (#236, #237) + Change download links to https + Improvements to test cases. + nanopb-0.3.7 (2016-10-30) Add support for proto3-style singular fields (#182, #206, #216) Updated binary package protoc to version 3.1.0 -- cgit 1.2.3-korg From fde887cdd96a1bf64980bad8ecb67c32bae1fbdd Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 2 Mar 2017 21:51:31 +0200 Subject: Fix bug in alltypes_callback test case --- tests/alltypes_callback/alltypes.options | 1 + tests/alltypes_callback/decode_alltypes_callback.c | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/alltypes_callback/alltypes.options b/tests/alltypes_callback/alltypes.options index 74d7a9c0..f01cc9d5 100644 --- a/tests/alltypes_callback/alltypes.options +++ b/tests/alltypes_callback/alltypes.options @@ -2,6 +2,7 @@ AllTypes.* type:FT_CALLBACK SubMessage.substuff1 max_size:16 AllTypes.oneof no_unions:true +* packed:true # With FT_CALLBACK, these options should get ignored *.*fbytes fixed_length:true max_size:4 diff --git a/tests/alltypes_callback/decode_alltypes_callback.c b/tests/alltypes_callback/decode_alltypes_callback.c index ec946909..2cdffafa 100644 --- a/tests/alltypes_callback/decode_alltypes_callback.c +++ b/tests/alltypes_callback/decode_alltypes_callback.c @@ -74,7 +74,7 @@ static bool read_submsg(pb_istream_t *stream, const pb_field_t *field, void **ar if (!pb_decode(stream, SubMessage_fields, &submsg)) return false; - TEST(memcmp(&submsg, *arg, sizeof(submsg))); + TEST(memcmp(&submsg, *arg, sizeof(submsg)) == 0); return true; } @@ -184,7 +184,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) uint64_t req_fixed64 = 1011; int64_t req_sfixed64 = -1012; double req_double = 1013.0; - SubMessage req_submsg = {"1016", 1016}; + SubMessage req_submsg = {"1016", 1016, false, 3}; int32_t rep_int32[5] = {0, 0, 0, 0, -2001}; int32_t rep_int64[5] = {0, 0, 0, 0, -2002}; @@ -214,9 +214,9 @@ bool check_alltypes(pb_istream_t *stream, int mode) uint64_t opt_fixed64 = 3051; int64_t opt_sfixed64 = 3052; double opt_double = 3053.0f; - SubMessage opt_submsg = {"3056", 3056}; + SubMessage opt_submsg = {"3056", 3056, false, 3}; - SubMessage oneof_msg1 = {"4059", 4059}; + SubMessage oneof_msg1 = {"4059", 4059, false, 3}; /* Bind callbacks for required fields */ AllTypes alltypes = AllTypes_init_zero; -- cgit 1.2.3-korg From 554886611824e56cf589aa5b7c5942a1fc7bf60a Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 2 Mar 2017 22:03:50 +0200 Subject: Add callback test for proto3 mode --- tests/alltypes_proto3_callback/SConscript | 23 ++ tests/alltypes_proto3_callback/alltypes.options | 9 + .../decode_alltypes_callback.c | 369 +++++++++++++++++++++ .../encode_alltypes_callback.c | 343 +++++++++++++++++++ 4 files changed, 744 insertions(+) create mode 100644 tests/alltypes_proto3_callback/SConscript create mode 100644 tests/alltypes_proto3_callback/alltypes.options create mode 100644 tests/alltypes_proto3_callback/decode_alltypes_callback.c create mode 100644 tests/alltypes_proto3_callback/encode_alltypes_callback.c diff --git a/tests/alltypes_proto3_callback/SConscript b/tests/alltypes_proto3_callback/SConscript new file mode 100644 index 00000000..183a138a --- /dev/null +++ b/tests/alltypes_proto3_callback/SConscript @@ -0,0 +1,23 @@ +# Test the AllTypes encoding & decoding using callbacks for all fields. + +Import("env", "malloc_env") + +c = Copy("$TARGET", "$SOURCE") +env.Command("alltypes.proto", "#alltypes_proto3/alltypes.proto", c) + +env.NanopbProto(["alltypes", "alltypes.options"]) +enc = env.Program(["encode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) + +refdec = "$BUILD/alltypes_proto3/decode_alltypes$PROGSUFFIX" + +# Encode and compare results +env.RunTest(enc) +env.RunTest("decode_alltypes.output", [refdec, "encode_alltypes_callback.output"]) +env.RunTest("decode_alltypes_callback.output", [dec, "encode_alltypes_callback.output"]) + +# Do the same thing with the optional fields present +env.RunTest("optionals.output", enc, ARGS = ['1']) +env.RunTest("optionals.refdecout", [refdec, "optionals.output"], ARGS = ['1']) +env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) + diff --git a/tests/alltypes_proto3_callback/alltypes.options b/tests/alltypes_proto3_callback/alltypes.options new file mode 100644 index 00000000..f01cc9d5 --- /dev/null +++ b/tests/alltypes_proto3_callback/alltypes.options @@ -0,0 +1,9 @@ +# Generate all fields as callbacks. +AllTypes.* type:FT_CALLBACK +SubMessage.substuff1 max_size:16 +AllTypes.oneof no_unions:true +* packed:true + +# With FT_CALLBACK, these options should get ignored +*.*fbytes fixed_length:true max_size:4 + diff --git a/tests/alltypes_proto3_callback/decode_alltypes_callback.c b/tests/alltypes_proto3_callback/decode_alltypes_callback.c new file mode 100644 index 00000000..17e7f122 --- /dev/null +++ b/tests/alltypes_proto3_callback/decode_alltypes_callback.c @@ -0,0 +1,369 @@ +/* Attempts to test all the datatypes supported by ProtoBuf when used as callback fields. + * Note that normally there would be no reason to use callback fields for this, + * because each encoder defined here only gives a single field. + */ + +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "test_helpers.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed (in field %d).\n", field->tag); \ + return false; \ + } + +static bool read_varint(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint64_t value; + if (!pb_decode_varint(stream, &value)) + return false; + + TEST((int64_t)value == (long)*arg); + return true; +} + +static bool read_svarint(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + int64_t value; + if (!pb_decode_svarint(stream, &value)) + return false; + + TEST(value == (long)*arg); + return true; +} + +static bool read_fixed32(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint32_t value; + if (!pb_decode_fixed32(stream, &value)) + return false; + + TEST(value == *(uint32_t*)*arg); + return true; +} + +static bool read_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint64_t value; + if (!pb_decode_fixed64(stream, &value)) + return false; + + TEST(value == *(uint64_t*)*arg); + return true; +} + +static bool read_string(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint8_t buf[16] = {0}; + size_t len = stream->bytes_left; + + if (len > sizeof(buf) - 1 || !pb_read(stream, buf, len)) + return false; + + TEST(strcmp((char*)buf, *arg) == 0); + return true; +} + +static bool read_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + SubMessage submsg = {""}; + + if (!pb_decode(stream, SubMessage_fields, &submsg)) + return false; + + TEST(memcmp(&submsg, *arg, sizeof(submsg)) == 0); + return true; +} + +static bool read_emptymsg(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + EmptyMessage emptymsg = {0}; + return pb_decode(stream, EmptyMessage_fields, &emptymsg); +} + +static bool read_repeated_varint(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + int32_t** expected = (int32_t**)arg; + uint64_t value; + if (!pb_decode_varint(stream, &value)) + return false; + + TEST(*(*expected)++ == value); + return true; +} + +static bool read_repeated_svarint(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + int32_t** expected = (int32_t**)arg; + int64_t value; + if (!pb_decode_svarint(stream, &value)) + return false; + + TEST(*(*expected)++ == value); + return true; +} + +static bool read_repeated_fixed32(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint32_t** expected = (uint32_t**)arg; + uint32_t value; + if (!pb_decode_fixed32(stream, &value)) + return false; + + TEST(*(*expected)++ == value); + return true; +} + +static bool read_repeated_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint64_t** expected = (uint64_t**)arg; + uint64_t value; + if (!pb_decode_fixed64(stream, &value)) + return false; + + TEST(*(*expected)++ == value); + return true; +} + +static bool read_repeated_string(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint8_t*** expected = (uint8_t***)arg; + uint8_t buf[16] = {0}; + size_t len = stream->bytes_left; + + if (len > sizeof(buf) - 1 || !pb_read(stream, buf, len)) + return false; + + TEST(strcmp((char*)*(*expected)++, (char*)buf) == 0); + return true; +} + +static bool read_repeated_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + SubMessage** expected = (SubMessage**)arg; + SubMessage decoded = {""}; + if (!pb_decode(stream, SubMessage_fields, &decoded)) + return false; + + TEST(memcmp((*expected)++, &decoded, sizeof(decoded)) == 0); + return true; +} + +static bool read_limits(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + Limits decoded = {0}; + if (!pb_decode(stream, Limits_fields, &decoded)) + return false; + + TEST(decoded.int32_min == INT32_MIN); + TEST(decoded.int32_max == INT32_MAX); + TEST(decoded.uint32_min == 0); + TEST(decoded.uint32_max == UINT32_MAX); + TEST(decoded.int64_min == INT64_MIN); + TEST(decoded.int64_max == INT64_MAX); + TEST(decoded.uint64_min == 0); + TEST(decoded.uint64_max == UINT64_MAX); + TEST(decoded.enum_min == HugeEnum_Negative); + TEST(decoded.enum_max == HugeEnum_Positive); + + return true; +} + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_alltypes(pb_istream_t *stream, int mode) +{ + /* Values for use from callbacks through pointers. */ + bool status; + + int32_t rep_int32[5] = {0, 0, 0, 0, -2001}; + int32_t rep_int64[5] = {0, 0, 0, 0, -2002}; + int32_t rep_uint32[5] = {0, 0, 0, 0, 2003}; + int32_t rep_uint64[5] = {0, 0, 0, 0, 2004}; + int32_t rep_sint32[5] = {0, 0, 0, 0, -2005}; + int32_t rep_sint64[5] = {0, 0, 0, 0, -2006}; + int32_t rep_bool[5] = {false, false, false, false, true}; + uint32_t rep_fixed32[5] = {0, 0, 0, 0, 2008}; + int32_t rep_sfixed32[5] = {0, 0, 0, 0, -2009}; + float rep_float[5] = {0, 0, 0, 0, 2010.0f}; + uint64_t rep_fixed64[5] = {0, 0, 0, 0, 2011}; + int64_t rep_sfixed64[5] = {0, 0, 0, 0, -2012}; + double rep_double[5] = {0, 0, 0, 0, 2013.0}; + char* rep_string[5] = {"", "", "", "", "2014"}; + char* rep_bytes[5] = {"", "", "", "", "2015"}; + SubMessage rep_submsg[5] = {{"", 0, 0}, + {"", 0, 0}, + {"", 0, 0}, + {"", 0, 0}, + {"2016", 2016, 2016}}; + int32_t rep_enum[5] = {0, 0, 0, 0, MyEnum_Truth}; + + uint32_t sng_fixed32 = 3048; + int32_t sng_sfixed32 = 3049; + float sng_float = 3050.0f; + uint64_t sng_fixed64 = 3051; + int64_t sng_sfixed64 = 3052; + double sng_double = 3053.0f; + SubMessage sng_submsg = {"3056", 3056}; + + SubMessage oneof_msg1 = {"4059", 4059}; + + AllTypes alltypes = AllTypes_init_zero; + + /* Bind callbacks for repeated fields */ + alltypes.rep_int32.funcs.decode = &read_repeated_varint; + alltypes.rep_int32.arg = rep_int32; + + alltypes.rep_int64.funcs.decode = &read_repeated_varint; + alltypes.rep_int64.arg = rep_int64; + + alltypes.rep_uint32.funcs.decode = &read_repeated_varint; + alltypes.rep_uint32.arg = rep_uint32; + + alltypes.rep_uint64.funcs.decode = &read_repeated_varint; + alltypes.rep_uint64.arg = rep_uint64; + + alltypes.rep_sint32.funcs.decode = &read_repeated_svarint; + alltypes.rep_sint32.arg = rep_sint32; + + alltypes.rep_sint64.funcs.decode = &read_repeated_svarint; + alltypes.rep_sint64.arg = rep_sint64; + + alltypes.rep_bool.funcs.decode = &read_repeated_varint; + alltypes.rep_bool.arg = rep_bool; + + alltypes.rep_fixed32.funcs.decode = &read_repeated_fixed32; + alltypes.rep_fixed32.arg = rep_fixed32; + + alltypes.rep_sfixed32.funcs.decode = &read_repeated_fixed32; + alltypes.rep_sfixed32.arg = rep_sfixed32; + + alltypes.rep_float.funcs.decode = &read_repeated_fixed32; + alltypes.rep_float.arg = rep_float; + + alltypes.rep_fixed64.funcs.decode = &read_repeated_fixed64; + alltypes.rep_fixed64.arg = rep_fixed64; + + alltypes.rep_sfixed64.funcs.decode = &read_repeated_fixed64; + alltypes.rep_sfixed64.arg = rep_sfixed64; + + alltypes.rep_double.funcs.decode = &read_repeated_fixed64; + alltypes.rep_double.arg = rep_double; + + alltypes.rep_string.funcs.decode = &read_repeated_string; + alltypes.rep_string.arg = rep_string; + + alltypes.rep_bytes.funcs.decode = &read_repeated_string; + alltypes.rep_bytes.arg = rep_bytes; + + alltypes.rep_submsg.funcs.decode = &read_repeated_submsg; + alltypes.rep_submsg.arg = rep_submsg; + + alltypes.rep_enum.funcs.decode = &read_repeated_varint; + alltypes.rep_enum.arg = rep_enum; + + alltypes.rep_emptymsg.funcs.decode = &read_emptymsg; + + alltypes.req_limits.funcs.decode = &read_limits; + + alltypes.end.funcs.decode = &read_varint; + alltypes.end.arg = (void*)1099; + + /* Bind callbacks for optional fields */ + if (mode == 1) + { + alltypes.sng_int32.funcs.decode = &read_varint; + alltypes.sng_int32.arg = (void*)3041; + + alltypes.sng_int64.funcs.decode = &read_varint; + alltypes.sng_int64.arg = (void*)3042; + + alltypes.sng_uint32.funcs.decode = &read_varint; + alltypes.sng_uint32.arg = (void*)3043; + + alltypes.sng_uint64.funcs.decode = &read_varint; + alltypes.sng_uint64.arg = (void*)3044; + + alltypes.sng_sint32.funcs.decode = &read_svarint; + alltypes.sng_sint32.arg = (void*)3045; + + alltypes.sng_sint64.funcs.decode = &read_svarint; + alltypes.sng_sint64.arg = (void*)3046; + + alltypes.sng_bool.funcs.decode = &read_varint; + alltypes.sng_bool.arg = (void*)true; + + alltypes.sng_fixed32.funcs.decode = &read_fixed32; + alltypes.sng_fixed32.arg = &sng_fixed32; + + alltypes.sng_sfixed32.funcs.decode = &read_fixed32; + alltypes.sng_sfixed32.arg = &sng_sfixed32; + + alltypes.sng_float.funcs.decode = &read_fixed32; + alltypes.sng_float.arg = &sng_float; + + alltypes.sng_fixed64.funcs.decode = &read_fixed64; + alltypes.sng_fixed64.arg = &sng_fixed64; + + alltypes.sng_sfixed64.funcs.decode = &read_fixed64; + alltypes.sng_sfixed64.arg = &sng_sfixed64; + + alltypes.sng_double.funcs.decode = &read_fixed64; + alltypes.sng_double.arg = &sng_double; + + alltypes.sng_string.funcs.decode = &read_string; + alltypes.sng_string.arg = "3054"; + + alltypes.sng_bytes.funcs.decode = &read_string; + alltypes.sng_bytes.arg = "3055"; + + alltypes.sng_submsg.funcs.decode = &read_submsg; + alltypes.sng_submsg.arg = &sng_submsg; + + alltypes.sng_enum.funcs.decode = &read_varint; + alltypes.sng_enum.arg = (void*)MyEnum_Truth; + + alltypes.sng_emptymsg.funcs.decode = &read_emptymsg; + + alltypes.oneof_msg1.funcs.decode = &read_submsg; + alltypes.oneof_msg1.arg = &oneof_msg1; + } + + status = pb_decode(stream, AllTypes_fields, &alltypes); + +#ifdef PB_ENABLE_MALLOC + /* Just to check for any interference between pb_release() and callback fields */ + pb_release(AllTypes_fields, &alltypes); +#endif + + return status; +} + +int main(int argc, char **argv) +{ + uint8_t buffer[1024]; + size_t count; + pb_istream_t stream; + + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Read the data into buffer */ + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_alltypes(&stream, mode)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} diff --git a/tests/alltypes_proto3_callback/encode_alltypes_callback.c b/tests/alltypes_proto3_callback/encode_alltypes_callback.c new file mode 100644 index 00000000..8c7bdd66 --- /dev/null +++ b/tests/alltypes_proto3_callback/encode_alltypes_callback.c @@ -0,0 +1,343 @@ +/* Attempts to test all the datatypes supported by ProtoBuf when used as callback fields. + * Note that normally there would be no reason to use callback fields for this, + * because each encoder defined here only gives a single field. + */ + +#include +#include +#include +#include +#include "alltypes.pb.h" +#include "test_helpers.h" + +static bool write_varint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, (long)*arg); +} + +static bool write_svarint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, (long)*arg); +} + +static bool write_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_fixed32(stream, *arg); +} + +static bool write_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_fixed64(stream, *arg); +} + +static bool write_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, *arg, strlen(*arg)); +} + +static bool write_submsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, *arg); +} + +static bool write_emptymsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + EmptyMessage emptymsg = {0}; + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg); +} + +static bool write_repeated_varint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, (long)*arg); +} + +static bool write_repeated_svarint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, (long)*arg); +} + +static bool write_repeated_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + uint32_t dummy = 0; + + /* Make it a packed field */ + return pb_encode_tag(stream, PB_WT_STRING, field->tag) && + pb_encode_varint(stream, 5 * 4) && /* Number of bytes */ + pb_encode_fixed32(stream, &dummy) && + pb_encode_fixed32(stream, &dummy) && + pb_encode_fixed32(stream, &dummy) && + pb_encode_fixed32(stream, &dummy) && + pb_encode_fixed32(stream, *arg); +} + +static bool write_repeated_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + uint64_t dummy = 0; + + /* Make it a packed field */ + return pb_encode_tag(stream, PB_WT_STRING, field->tag) && + pb_encode_varint(stream, 5 * 8) && /* Number of bytes */ + pb_encode_fixed64(stream, &dummy) && + pb_encode_fixed64(stream, &dummy) && + pb_encode_fixed64(stream, &dummy) && + pb_encode_fixed64(stream, &dummy) && + pb_encode_fixed64(stream, *arg); +} + +static bool write_repeated_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, 0, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, 0, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, 0, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, 0, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, *arg, strlen(*arg)); +} + +static bool write_repeated_submsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + SubMessage dummy = {""}; + + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, &dummy) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, &dummy) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, &dummy) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, &dummy) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, *arg); +} + +static bool write_limits(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + Limits limits = {0}; + limits.int32_min = INT32_MIN; + limits.int32_max = INT32_MAX; + limits.uint32_min = 0; + limits.uint32_max = UINT32_MAX; + limits.int64_min = INT64_MIN; + limits.int64_max = INT64_MAX; + limits.uint64_min = 0; + limits.uint64_max = UINT64_MAX; + limits.enum_min = HugeEnum_Negative; + limits.enum_max = HugeEnum_Positive; + + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, Limits_fields, &limits); +} + +static bool write_repeated_emptymsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + EmptyMessage emptymsg = {0}; + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg); +} + +int main(int argc, char **argv) +{ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Values for use from callbacks through pointers. */ + uint32_t rep_fixed32 = 2008; + int32_t rep_sfixed32 = -2009; + float rep_float = 2010.0f; + uint64_t rep_fixed64 = 2011; + int64_t rep_sfixed64 = -2012; + double rep_double = 2013.0; + SubMessage rep_submsg = {"2016", 2016, 2016}; + + uint32_t sng_fixed32 = 3048; + int32_t sng_sfixed32 = 3049; + float sng_float = 3050.0f; + uint64_t sng_fixed64 = 3051; + int64_t sng_sfixed64 = 3052; + double sng_double = 3053.0f; + SubMessage sng_submsg = {"3056", 3056}; + + SubMessage oneof_msg1 = {"4059", 4059}; + + AllTypes alltypes = AllTypes_init_zero; + + /* Bind callbacks for repeated fields */ + alltypes.rep_int32.funcs.encode = &write_repeated_varint; + alltypes.rep_int32.arg = (void*)-2001; + + alltypes.rep_int64.funcs.encode = &write_repeated_varint; + alltypes.rep_int64.arg = (void*)-2002; + + alltypes.rep_uint32.funcs.encode = &write_repeated_varint; + alltypes.rep_uint32.arg = (void*)2003; + + alltypes.rep_uint64.funcs.encode = &write_repeated_varint; + alltypes.rep_uint64.arg = (void*)2004; + + alltypes.rep_sint32.funcs.encode = &write_repeated_svarint; + alltypes.rep_sint32.arg = (void*)-2005; + + alltypes.rep_sint64.funcs.encode = &write_repeated_svarint; + alltypes.rep_sint64.arg = (void*)-2006; + + alltypes.rep_bool.funcs.encode = &write_repeated_varint; + alltypes.rep_bool.arg = (void*)true; + + alltypes.rep_fixed32.funcs.encode = &write_repeated_fixed32; + alltypes.rep_fixed32.arg = &rep_fixed32; + + alltypes.rep_sfixed32.funcs.encode = &write_repeated_fixed32; + alltypes.rep_sfixed32.arg = &rep_sfixed32; + + alltypes.rep_float.funcs.encode = &write_repeated_fixed32; + alltypes.rep_float.arg = &rep_float; + + alltypes.rep_fixed64.funcs.encode = &write_repeated_fixed64; + alltypes.rep_fixed64.arg = &rep_fixed64; + + alltypes.rep_sfixed64.funcs.encode = &write_repeated_fixed64; + alltypes.rep_sfixed64.arg = &rep_sfixed64; + + alltypes.rep_double.funcs.encode = &write_repeated_fixed64; + alltypes.rep_double.arg = &rep_double; + + alltypes.rep_string.funcs.encode = &write_repeated_string; + alltypes.rep_string.arg = "2014"; + + alltypes.rep_bytes.funcs.encode = &write_repeated_string; + alltypes.rep_bytes.arg = "2015"; + + alltypes.rep_submsg.funcs.encode = &write_repeated_submsg; + alltypes.rep_submsg.arg = &rep_submsg; + + alltypes.rep_enum.funcs.encode = &write_repeated_varint; + alltypes.rep_enum.arg = (void*)MyEnum_Truth; + + alltypes.rep_emptymsg.funcs.encode = &write_repeated_emptymsg; + + alltypes.rep_fbytes.funcs.encode = &write_repeated_string; + alltypes.rep_fbytes.arg = "2019"; + + alltypes.req_limits.funcs.encode = &write_limits; + + /* Bind callbacks for singular fields */ + if (mode != 0) + { + alltypes.sng_int32.funcs.encode = &write_varint; + alltypes.sng_int32.arg = (void*)3041; + + alltypes.sng_int64.funcs.encode = &write_varint; + alltypes.sng_int64.arg = (void*)3042; + + alltypes.sng_uint32.funcs.encode = &write_varint; + alltypes.sng_uint32.arg = (void*)3043; + + alltypes.sng_uint64.funcs.encode = &write_varint; + alltypes.sng_uint64.arg = (void*)3044; + + alltypes.sng_sint32.funcs.encode = &write_svarint; + alltypes.sng_sint32.arg = (void*)3045; + + alltypes.sng_sint64.funcs.encode = &write_svarint; + alltypes.sng_sint64.arg = (void*)3046; + + alltypes.sng_bool.funcs.encode = &write_varint; + alltypes.sng_bool.arg = (void*)true; + + alltypes.sng_fixed32.funcs.encode = &write_fixed32; + alltypes.sng_fixed32.arg = &sng_fixed32; + + alltypes.sng_sfixed32.funcs.encode = &write_fixed32; + alltypes.sng_sfixed32.arg = &sng_sfixed32; + + alltypes.sng_float.funcs.encode = &write_fixed32; + alltypes.sng_float.arg = &sng_float; + + alltypes.sng_fixed64.funcs.encode = &write_fixed64; + alltypes.sng_fixed64.arg = &sng_fixed64; + + alltypes.sng_sfixed64.funcs.encode = &write_fixed64; + alltypes.sng_sfixed64.arg = &sng_sfixed64; + + alltypes.sng_double.funcs.encode = &write_fixed64; + alltypes.sng_double.arg = &sng_double; + + alltypes.sng_string.funcs.encode = &write_string; + alltypes.sng_string.arg = "3054"; + + alltypes.sng_bytes.funcs.encode = &write_string; + alltypes.sng_bytes.arg = "3055"; + + alltypes.sng_submsg.funcs.encode = &write_submsg; + alltypes.sng_submsg.arg = &sng_submsg; + + alltypes.sng_enum.funcs.encode = &write_varint; + alltypes.sng_enum.arg = (void*)MyEnum_Truth; + + alltypes.sng_emptymsg.funcs.encode = &write_emptymsg; + + alltypes.sng_fbytes.funcs.encode = &write_string; + alltypes.sng_fbytes.arg = "3059"; + + alltypes.oneof_msg1.funcs.encode = &write_submsg; + alltypes.oneof_msg1.arg = &oneof_msg1; + } + + alltypes.end.funcs.encode = &write_varint; + alltypes.end.arg = (void*)1099; + + { + uint8_t buffer[2048]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } + } +} -- cgit 1.2.3-korg From 6fca8edc847bc70fd2e9d268af06015f8e64ec15 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 2 Mar 2017 22:04:02 +0200 Subject: Add regression test for issue #249 --- tests/regression/issue_249/SConscript | 12 +++++++ tests/regression/issue_249/test.c | 59 +++++++++++++++++++++++++++++++++++ tests/regression/issue_249/test.proto | 10 ++++++ 3 files changed, 81 insertions(+) create mode 100644 tests/regression/issue_249/SConscript create mode 100644 tests/regression/issue_249/test.c create mode 100644 tests/regression/issue_249/test.proto diff --git a/tests/regression/issue_249/SConscript b/tests/regression/issue_249/SConscript new file mode 100644 index 00000000..ba667129 --- /dev/null +++ b/tests/regression/issue_249/SConscript @@ -0,0 +1,12 @@ +# Regression test for Issue 249: proto3 mode pb_decode() corrupts callback fields +Import('env') + +env.NanopbProto('test') + +p = env.Program(["test.c", + "test.pb.c", + "$COMMON/pb_decode.o", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) + diff --git a/tests/regression/issue_249/test.c b/tests/regression/issue_249/test.c new file mode 100644 index 00000000..a37180fd --- /dev/null +++ b/tests/regression/issue_249/test.c @@ -0,0 +1,59 @@ +#include "test.pb.h" +#include +#include +#include + +static bool write_array(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + int i; + for (i = 0; i < 5; i++) + { + if (!pb_encode_tag_for_field(stream, field)) + return false; + if (!pb_encode_varint(stream, 1000 + i)) + return false; + } + + return true; +} + +static bool read_array(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint32_t i; + int *sum = *arg; + + if (!pb_decode_varint32(stream, &i)) + return false; + + *sum += i; + + return true; +} + +int main() +{ + int status = 0; + pb_byte_t buf[128] = {0}; + pb_size_t msglen; + + { + MainMessage msg = MainMessage_init_zero; + pb_ostream_t stream = pb_ostream_from_buffer(buf, sizeof(buf)); + msg.submsg.foo.funcs.encode = &write_array; + TEST(pb_encode(&stream, MainMessage_fields, &msg)); + msglen = stream.bytes_written; + } + + { + MainMessage msg = MainMessage_init_zero; + pb_istream_t stream = pb_istream_from_buffer(buf, msglen); + int sum = 0; + msg.submsg.foo.funcs.decode = &read_array; + msg.submsg.foo.arg = ∑ + TEST(pb_decode(&stream, MainMessage_fields, &msg)); + TEST(sum == 1000 + 1001 + 1002 + 1003 + 1004); + } + + return status; +} + diff --git a/tests/regression/issue_249/test.proto b/tests/regression/issue_249/test.proto new file mode 100644 index 00000000..eaa2abde --- /dev/null +++ b/tests/regression/issue_249/test.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +message SubMessage { + repeated int32 foo = 1; +} + +message MainMessage { + SubMessage submsg = 1; +} + -- cgit 1.2.3-korg From 6ea1047a7f868796621f1ec9f7324f101e135733 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 2 Mar 2017 22:14:00 +0200 Subject: Fix callback pointer corruption in proto3 mode (issue #249) --- pb_decode.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pb_decode.c b/pb_decode.c index a8cd61a7..e2e90caa 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -780,7 +780,7 @@ static void pb_field_set_to_default(pb_field_iter_t *iter) else if (PB_ATYPE(type) == PB_ATYPE_STATIC) { bool init_data = true; - if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL) + if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL && iter->pSize != iter->pData) { /* Set has_field to false. Still initialize the optional field * itself also. */ -- cgit 1.2.3-korg From 102a1aaa1474130e0921f9e69c9b3949eb0954ad Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Thu, 2 Mar 2017 22:40:15 +0200 Subject: Fix alltypes_callback testcase The test case was erroneously comparing whole submsg structures, which could result in false errors when padding bytes differed. --- tests/alltypes_callback/alltypes.options | 1 - tests/alltypes_callback/decode_alltypes_callback.c | 17 +++++++++++++---- tests/alltypes_proto3_callback/alltypes.options | 1 - .../alltypes_proto3_callback/decode_alltypes_callback.c | 15 +++++++++++---- 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/tests/alltypes_callback/alltypes.options b/tests/alltypes_callback/alltypes.options index f01cc9d5..74d7a9c0 100644 --- a/tests/alltypes_callback/alltypes.options +++ b/tests/alltypes_callback/alltypes.options @@ -2,7 +2,6 @@ AllTypes.* type:FT_CALLBACK SubMessage.substuff1 max_size:16 AllTypes.oneof no_unions:true -* packed:true # With FT_CALLBACK, these options should get ignored *.*fbytes fixed_length:true max_size:4 diff --git a/tests/alltypes_callback/decode_alltypes_callback.c b/tests/alltypes_callback/decode_alltypes_callback.c index 2cdffafa..576ce307 100644 --- a/tests/alltypes_callback/decode_alltypes_callback.c +++ b/tests/alltypes_callback/decode_alltypes_callback.c @@ -70,11 +70,15 @@ static bool read_string(pb_istream_t *stream, const pb_field_t *field, void **ar static bool read_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg) { SubMessage submsg = {""}; + SubMessage *ref = *arg; if (!pb_decode(stream, SubMessage_fields, &submsg)) return false; - TEST(memcmp(&submsg, *arg, sizeof(submsg)) == 0); + TEST(strcmp(submsg.substuff1, ref->substuff1) == 0); + TEST(submsg.substuff2 == ref->substuff2); + TEST(submsg.has_substuff3 == ref->has_substuff3); + TEST(submsg.substuff3 == ref->substuff3); return true; } @@ -144,11 +148,16 @@ static bool read_repeated_string(pb_istream_t *stream, const pb_field_t *field, static bool read_repeated_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg) { SubMessage** expected = (SubMessage**)arg; - SubMessage decoded = {""}; - if (!pb_decode(stream, SubMessage_fields, &decoded)) + SubMessage submsg = {""}; + if (!pb_decode(stream, SubMessage_fields, &submsg)) return false; - TEST(memcmp((*expected)++, &decoded, sizeof(decoded)) == 0); + TEST(strcmp(submsg.substuff1, (*expected)->substuff1) == 0); + TEST(submsg.substuff2 == (*expected)->substuff2); + TEST(submsg.has_substuff3 == (*expected)->has_substuff3); + TEST(submsg.substuff3 == (*expected)->substuff3); + (*expected)++; + return true; } diff --git a/tests/alltypes_proto3_callback/alltypes.options b/tests/alltypes_proto3_callback/alltypes.options index f01cc9d5..74d7a9c0 100644 --- a/tests/alltypes_proto3_callback/alltypes.options +++ b/tests/alltypes_proto3_callback/alltypes.options @@ -2,7 +2,6 @@ AllTypes.* type:FT_CALLBACK SubMessage.substuff1 max_size:16 AllTypes.oneof no_unions:true -* packed:true # With FT_CALLBACK, these options should get ignored *.*fbytes fixed_length:true max_size:4 diff --git a/tests/alltypes_proto3_callback/decode_alltypes_callback.c b/tests/alltypes_proto3_callback/decode_alltypes_callback.c index 17e7f122..2b3c2f32 100644 --- a/tests/alltypes_proto3_callback/decode_alltypes_callback.c +++ b/tests/alltypes_proto3_callback/decode_alltypes_callback.c @@ -70,11 +70,14 @@ static bool read_string(pb_istream_t *stream, const pb_field_t *field, void **ar static bool read_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg) { SubMessage submsg = {""}; + SubMessage *ref = *arg; if (!pb_decode(stream, SubMessage_fields, &submsg)) return false; - TEST(memcmp(&submsg, *arg, sizeof(submsg)) == 0); + TEST(strcmp(submsg.substuff1, ref->substuff1) == 0); + TEST(submsg.substuff2 == ref->substuff2); + TEST(submsg.substuff3 == ref->substuff3); return true; } @@ -144,11 +147,15 @@ static bool read_repeated_string(pb_istream_t *stream, const pb_field_t *field, static bool read_repeated_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg) { SubMessage** expected = (SubMessage**)arg; - SubMessage decoded = {""}; - if (!pb_decode(stream, SubMessage_fields, &decoded)) + SubMessage submsg = {""}; + if (!pb_decode(stream, SubMessage_fields, &submsg)) return false; - TEST(memcmp((*expected)++, &decoded, sizeof(decoded)) == 0); + TEST(strcmp(submsg.substuff1, (*expected)->substuff1) == 0); + TEST(submsg.substuff2 == (*expected)->substuff2); + TEST(submsg.substuff3 == (*expected)->substuff3); + (*expected)++; + return true; } -- cgit 1.2.3-korg From 6a24679f105d002c7f7ca1c2c063e07b23dca84f Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 5 Mar 2017 18:27:44 +0200 Subject: Update changelog --- CHANGELOG.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index a5de21cc..5006a387 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,7 +1,7 @@ nanopb-0.3.8 (2016-03-xx) Fix problems with multiple oneofs in same message (#229) Zero-valued extension fields were mistakenly ignored by encoder (#242) - Various fixes related to proto3 encoding (#242, #245, #247) + Multiple fixes related to proto3 mode (#242, #245, #247, #249) Fix potential unaligned access (#226, #227) Fix documentation for protoc --plugin argument (#239) Extend inline / fixed length bytes array support (#244) -- cgit 1.2.3-korg From cc74b9f200176edc5524aa00ba45fa90a5e87d27 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 5 Mar 2017 18:28:23 +0200 Subject: Publishing nanopb-0.3.8 --- CMakeLists.txt | 2 +- generator/nanopb_generator.py | 2 +- library.json | 2 +- pb.h | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f69386bf..b8a29463 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8) project(nanopb C) -set(nanopb_VERSION_STRING nanopb-0.3.8-dev) +set(nanopb_VERSION_STRING nanopb-0.3.8) string(REPLACE "nanopb-" "" nanopb_VERSION ${nanopb_VERSION_STRING}) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index b5473174..e4d3d27a 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.8-dev" +nanopb_version = "nanopb-0.3.8" import sys import re diff --git a/library.json b/library.json index 1a0b07c1..1acd14ef 100644 --- a/library.json +++ b/library.json @@ -1,6 +1,6 @@ { "name": "Nanopb", - "version": "0.3.8-dev", + "version": "0.3.8", "keywords": "protocol buffers, protobuf, google", "description": "Nanopb is a plain-C implementation of Google's Protocol Buffers data format. It is targeted at 32 bit microcontrollers, but is also fit for other embedded systems with tight (2-10 kB ROM, <1 kB RAM) memory constraints.", "repository": { diff --git a/pb.h b/pb.h index a3001bad..bf05a63c 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.8-dev +#define NANOPB_VERSION nanopb-0.3.8 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From a55a66e792b30a65606cce3e61f28c6c6e14d050 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 5 Mar 2017 18:39:40 +0200 Subject: Setting version to 0.3.9-dev --- CMakeLists.txt | 2 +- generator/nanopb_generator.py | 2 +- library.json | 2 +- pb.h | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b8a29463..c78f8c3c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required(VERSION 2.8) project(nanopb C) -set(nanopb_VERSION_STRING nanopb-0.3.8) +set(nanopb_VERSION_STRING nanopb-0.3.9-dev) string(REPLACE "nanopb-" "" nanopb_VERSION ${nanopb_VERSION_STRING}) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index e4d3d27a..359e1849 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.3.8" +nanopb_version = "nanopb-0.3.9-dev" import sys import re diff --git a/library.json b/library.json index 1acd14ef..8e963f34 100644 --- a/library.json +++ b/library.json @@ -1,6 +1,6 @@ { "name": "Nanopb", - "version": "0.3.8", + "version": "0.3.9-dev", "keywords": "protocol buffers, protobuf, google", "description": "Nanopb is a plain-C implementation of Google's Protocol Buffers data format. It is targeted at 32 bit microcontrollers, but is also fit for other embedded systems with tight (2-10 kB ROM, <1 kB RAM) memory constraints.", "repository": { diff --git a/pb.h b/pb.h index bf05a63c..c7e6bc67 100644 --- a/pb.h +++ b/pb.h @@ -46,7 +46,7 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.3.8 +#define NANOPB_VERSION nanopb-0.3.9-dev /* Include all the system headers needed by nanopb. You will need the * definitions of the following: -- cgit 1.2.3-korg From 2e6577dd120a26e0ab4cc985cffa2c2a82e8c59d Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 5 Mar 2017 18:40:43 +0200 Subject: Forgot to fill in the dates --- CHANGELOG.txt | 2 +- docs/migration.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 5006a387..194d6136 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,4 +1,4 @@ -nanopb-0.3.8 (2016-03-xx) +nanopb-0.3.8 (2017-03-05) Fix problems with multiple oneofs in same message (#229) Zero-valued extension fields were mistakenly ignored by encoder (#242) Multiple fixes related to proto3 mode (#242, #245, #247, #249) diff --git a/docs/migration.rst b/docs/migration.rst index 2d9ce38c..d6b32b53 100644 --- a/docs/migration.rst +++ b/docs/migration.rst @@ -11,7 +11,7 @@ are included, in order to make it easier to find this document. .. contents :: -Nanopb-0.3.8 (2017-xx-xx) +Nanopb-0.3.8 (2017-03-05) ========================= Fully drain substreams before closing -- cgit 1.2.3-korg From 0161193cf97f46aa491501e665a75aa0afd9698d Mon Sep 17 00:00:00 2001 From: Constantine Grantcharov Date: Sun, 5 Mar 2017 12:29:09 -0500 Subject: Remove use of relative paths When doing out of source builds: mkdir build cd build && cmake -G "Unix Makefiles" ../ && make The build script tripped up on relative paths of /generator/proto and the files under generator/proto/*.proto. By using ${PROJECT_SOURCE_DIR}, the paths become absolute and the issue disappears. --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c78f8c3c..22e6c70e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,11 +42,11 @@ if(nanopb_BUILD_GENERATOR) ) foreach(generator_proto IN LISTS generator_protos) - string(REGEX REPLACE "([^;]+)" "generator/proto/\\1.proto" generator_proto_file "${generator_proto}") + string(REGEX REPLACE "([^;]+)" "${PROJECT_SOURCE_DIR}/generator/proto/\\1.proto" generator_proto_file "${generator_proto}") string(REGEX REPLACE "([^;]+)" "\\1_pb2.py" generator_proto_py_file "${generator_proto}") add_custom_command( OUTPUT ${generator_proto_py_file} - COMMAND protoc --python_out=${CMAKE_CURRENT_BINARY_DIR} -Igenerator/proto ${generator_proto_file} + COMMAND protoc --python_out=${CMAKE_CURRENT_BINARY_DIR} -I${PROJECT_SOURCE_DIR}/generator/proto ${generator_proto_file} DEPENDS ${generator_proto_file} ) add_custom_target("generate_${generator_proto_py_file}" ALL DEPENDS ${generator_proto_py_file}) -- cgit 1.2.3-korg From 9deebe0f0f824ca9f2628fe7d927b2e8bf141be5 Mon Sep 17 00:00:00 2001 From: Constantine Grantcharov Date: Sun, 5 Mar 2017 13:02:43 -0500 Subject: Fixing install issue due to relative paths --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 22e6c70e..7fb7ad4a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,12 +46,12 @@ if(nanopb_BUILD_GENERATOR) string(REGEX REPLACE "([^;]+)" "\\1_pb2.py" generator_proto_py_file "${generator_proto}") add_custom_command( OUTPUT ${generator_proto_py_file} - COMMAND protoc --python_out=${CMAKE_CURRENT_BINARY_DIR} -I${PROJECT_SOURCE_DIR}/generator/proto ${generator_proto_file} + COMMAND protoc --python_out=${PROJECT_BINARY_DIR} -I${PROJECT_SOURCE_DIR}/generator/proto ${generator_proto_file} DEPENDS ${generator_proto_file} ) add_custom_target("generate_${generator_proto_py_file}" ALL DEPENDS ${generator_proto_py_file}) install( - FILES ${generator_proto_py_file} + FILES ${PROJECT_BINARY_DIR}/${generator_proto_py_file} DESTINATION ${PYTHON_INSTDIR} ) endforeach() -- cgit 1.2.3-korg From 459d9cf45c7a47e6fd034a134cc7653db82e8fe8 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 12 Mar 2017 11:44:24 +0200 Subject: Improve varint unittest coverage for error cases --- tests/decode_unittests/decode_unittests.c | 48 +++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/tests/decode_unittests/decode_unittests.c b/tests/decode_unittests/decode_unittests.c index 47f0fbdb..a6f5c17e 100644 --- a/tests/decode_unittests/decode_unittests.c +++ b/tests/decode_unittests/decode_unittests.c @@ -150,15 +150,59 @@ int main() { pb_istream_t s; pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 8, 0, 0}; - uint64_t d; + int64_t d; - COMMENT("Test pb_dec_svarint using uint64_t") + COMMENT("Test pb_dec_svarint using int64_t") TEST((s = S("\x01"), pb_dec_svarint(&s, &f, &d) && d == -1)) TEST((s = S("\x02"), pb_dec_svarint(&s, &f, &d) && d == 1)) TEST((s = S("\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_dec_svarint(&s, &f, &d) && d == INT64_MAX)) TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_dec_svarint(&s, &f, &d) && d == INT64_MIN)) } + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 4, 0, 0}; + int32_t d; + + COMMENT("Test pb_dec_svarint overflow detection using int32_t"); + TEST((s = S("\xfe\xff\xff\xff\x0f"), pb_dec_svarint(&s, &f, &d))); + TEST((s = S("\xfe\xff\xff\xff\x10"), !pb_dec_svarint(&s, &f, &d))); + TEST((s = S("\xff\xff\xff\xff\x0f"), pb_dec_svarint(&s, &f, &d))); + TEST((s = S("\xff\xff\xff\xff\x10"), !pb_dec_svarint(&s, &f, &d))); + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 4, 0, 0}; + uint32_t d; + + COMMENT("Test pb_dec_uvarint using uint32_t") + TEST((s = S("\x01"), pb_dec_uvarint(&s, &f, &d) && d == 1)) + TEST((s = S("\x02"), pb_dec_uvarint(&s, &f, &d) && d == 2)) + TEST((s = S("\xff\xff\xff\xff\x0f"), pb_dec_uvarint(&s, &f, &d) && d == UINT32_MAX)) + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 8, 0, 0}; + uint64_t d; + + COMMENT("Test pb_dec_uvarint using uint64_t") + TEST((s = S("\x01"), pb_dec_uvarint(&s, &f, &d) && d == 1)) + TEST((s = S("\x02"), pb_dec_uvarint(&s, &f, &d) && d == 2)) + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_dec_uvarint(&s, &f, &d) && d == UINT64_MAX)) + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 4, 0, 0}; + uint32_t d; + + COMMENT("Test pb_dec_uvarint overflow detection using int32_t"); + TEST((s = S("\xff\xff\xff\xff\x0f"), pb_dec_uvarint(&s, &f, &d))); + TEST((s = S("\xff\xff\xff\xff\x10"), !pb_dec_uvarint(&s, &f, &d))); + } + { pb_istream_t s; pb_field_t f = {1, PB_LTYPE_FIXED32, 0, 0, 4, 0, 0}; -- cgit 1.2.3-korg From 44e559d9ce74855bd48d8050ab6cf6391b980239 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Sun, 12 Mar 2017 12:18:32 +0200 Subject: Fix potential out-of-bounds read with more than 64 required fields --- pb_decode.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pb_decode.c b/pb_decode.c index e2e90caa..06d766af 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -934,6 +934,9 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.pos->tag != 0) req_field_count++; + if (req_field_count > PB_MAX_REQUIRED_FIELDS) + req_field_count = PB_MAX_REQUIRED_FIELDS; + if (req_field_count > 0) { /* Check the whole words */ @@ -943,9 +946,15 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ PB_RETURN_ERROR(stream, "missing required field"); } - /* Check the remaining bits */ - if (fields_seen[req_field_count >> 5] != (allbits >> (32 - (req_field_count & 31)))) - PB_RETURN_ERROR(stream, "missing required field"); + /* Check the remaining bits (if any) */ + if ((req_field_count & 31) != 0) + { + if (fields_seen[req_field_count >> 5] != + (allbits >> (32 - (req_field_count & 31)))) + { + PB_RETURN_ERROR(stream, "missing required field"); + } + } } } -- cgit 1.2.3-korg From d56c4cc79bdc85433e8cb592a67e51997d5d6f50 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 28 Mar 2017 23:20:16 +0300 Subject: Add regression test for issue #253 --- tests/regression/issue_253/SConscript | 15 +++++++++++++++ tests/regression/issue_253/short_array.c | 24 ++++++++++++++++++++++++ tests/regression/issue_253/short_array.proto | 7 +++++++ 3 files changed, 46 insertions(+) create mode 100644 tests/regression/issue_253/SConscript create mode 100644 tests/regression/issue_253/short_array.c create mode 100644 tests/regression/issue_253/short_array.proto diff --git a/tests/regression/issue_253/SConscript b/tests/regression/issue_253/SConscript new file mode 100644 index 00000000..5a16948c --- /dev/null +++ b/tests/regression/issue_253/SConscript @@ -0,0 +1,15 @@ +# Regression test for Issue 253: Wrong calculated message maximum size + +Import('env') + +env.NanopbProto('short_array') + +p = env.Program(['short_array.c', + 'short_array.pb.c', + "$COMMON/pb_decode.o", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) + +env.RunTest(p) + + diff --git a/tests/regression/issue_253/short_array.c b/tests/regression/issue_253/short_array.c new file mode 100644 index 00000000..5ed6c3f7 --- /dev/null +++ b/tests/regression/issue_253/short_array.c @@ -0,0 +1,24 @@ +#include +#include +#include "short_array.pb.h" + +int main() +{ + int status = 0; + + COMMENT("Test message length calculation for short arrays"); + { + uint8_t buffer[TestMessage_size] = {0}; + pb_ostream_t ostream = pb_ostream_from_buffer(buffer, TestMessage_size); + TestMessage msg = TestMessage_init_zero; + + msg.rep_uint32_count = 1; + msg.rep_uint32[0] = (1 << 31); + + TEST(pb_encode(&ostream, TestMessage_fields, &msg)); + TEST(ostream.bytes_written == TestMessage_size); + } + + return status; +} + diff --git a/tests/regression/issue_253/short_array.proto b/tests/regression/issue_253/short_array.proto new file mode 100644 index 00000000..5a5d8a3d --- /dev/null +++ b/tests/regression/issue_253/short_array.proto @@ -0,0 +1,7 @@ +syntax = "proto2"; +import "nanopb.proto"; + +message TestMessage { + repeated uint32 rep_uint32 = 1 [(nanopb).max_count = 1]; +} + -- cgit 1.2.3-korg From 25210ac1419567178502e99f005ddb49c4e63d21 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Tue, 28 Mar 2017 23:20:31 +0300 Subject: Fix message length calculation for arrays of size 1 (issue #253) --- generator/nanopb_generator.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 359e1849..42669ace 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -649,9 +649,14 @@ class Field: if self.rules == 'REPEATED': # Decoders must be always able to handle unpacked arrays. # Therefore we have to reserve space for it, even though - # we emit packed arrays ourselves. + # we emit packed arrays ourselves. For length of 1, packed + # arrays are larger however so we need to add allowance + # for the length byte. encsize *= self.max_count + if self.max_count == 1: + encsize += 1 + return encsize -- cgit 1.2.3-korg From e75d20b659aced782a0017244836e415164e2d0e Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 14 Apr 2017 11:45:57 +0300 Subject: Add testcase for issue #256 --- tests/regression/issue_256/SConscript | 16 +++++++++++ tests/regression/issue_256/submsg_array.c | 38 +++++++++++++++++++++++++++ tests/regression/issue_256/submsg_array.proto | 11 ++++++++ 3 files changed, 65 insertions(+) create mode 100644 tests/regression/issue_256/SConscript create mode 100644 tests/regression/issue_256/submsg_array.c create mode 100644 tests/regression/issue_256/submsg_array.proto diff --git a/tests/regression/issue_256/SConscript b/tests/regression/issue_256/SConscript new file mode 100644 index 00000000..b2c3e864 --- /dev/null +++ b/tests/regression/issue_256/SConscript @@ -0,0 +1,16 @@ +# Regression test for Issue 256: Proto3 mode skips submessages even when +# later array fields have non-zero value + +Import('env') + +env.NanopbProto('submsg_array') + +p = env.Program(['submsg_array.c', + 'submsg_array.pb.c', + "$COMMON/pb_decode.o", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) + +env.RunTest(p) + + diff --git a/tests/regression/issue_256/submsg_array.c b/tests/regression/issue_256/submsg_array.c new file mode 100644 index 00000000..c63bd30a --- /dev/null +++ b/tests/regression/issue_256/submsg_array.c @@ -0,0 +1,38 @@ +#include +#include +#include +#include "submsg_array.pb.h" + +int main() +{ + int status = 0; + + COMMENT("Test encoding for submessage with array"); + { + uint8_t buffer[TestMessage_size] = {0}; + pb_ostream_t ostream = pb_ostream_from_buffer(buffer, TestMessage_size); + TestMessage msg = TestMessage_init_zero; + + msg.submsg.rep_uint32_count = 3; + msg.submsg.rep_uint32[0] = 0; + msg.submsg.rep_uint32[1] = 1; + msg.submsg.rep_uint32[2] = 2; + + TEST(pb_encode(&ostream, TestMessage_fields, &msg)); + TEST(ostream.bytes_written > 0); + + { + pb_istream_t istream = pb_istream_from_buffer(buffer, ostream.bytes_written); + TestMessage msg2 = TestMessage_init_zero; + + TEST(pb_decode(&istream, TestMessage_fields, &msg2)); + TEST(msg2.submsg.rep_uint32_count == 3); + TEST(msg2.submsg.rep_uint32[0] == 0); + TEST(msg2.submsg.rep_uint32[1] == 1); + TEST(msg2.submsg.rep_uint32[2] == 2); + } + } + + return status; +} + diff --git a/tests/regression/issue_256/submsg_array.proto b/tests/regression/issue_256/submsg_array.proto new file mode 100644 index 00000000..4964a05f --- /dev/null +++ b/tests/regression/issue_256/submsg_array.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; +import "nanopb.proto"; + +message SubMessage { + repeated uint32 rep_uint32 = 1 [(nanopb).max_count = 3]; +} + +message TestMessage { + SubMessage submsg = 1; +} + -- cgit 1.2.3-korg From 278ffb890e3d8722e4c7d824baaf221a1e375fc4 Mon Sep 17 00:00:00 2001 From: Petteri Aimonen Date: Fri, 14 Apr 2017 11:46:18 +0300 Subject: Fix bugs in proto3 mode encoding of submessages (#256) pb_check_proto3_default_value() recurses into submessages, but it didn't handle other than singular fields correctly. This caused it to sometimes skip submessages with only repeated or oneof fields present. --- pb_encode.c | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/pb_encode.c b/pb_encode.c index 30f60d83..05d691df 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -204,25 +204,51 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie * This function implements the check for the zero value. */ static bool pb_check_proto3_default_value(const pb_field_t *field, const void *pData) { - if (PB_ATYPE(field->type) == PB_ATYPE_STATIC) + pb_type_t type = field->type; + const void *pSize = (const char*)pData + field->size_offset; + + if (PB_HTYPE(type) == PB_HTYPE_REQUIRED) + { + /* Required proto2 fields inside proto3 submessage, pretty rare case */ + return false; + } + else if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + { + /* Repeated fields inside proto3 submessage: present if count != 0 */ + return *(const pb_size_t*)pSize == 0; + } + else if (PB_HTYPE(type) == PB_HTYPE_ONEOF) + { + /* Oneof fields */ + return *(const pb_size_t*)pSize == 0; + } + else if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL && field->size_offset) + { + /* Proto2 optional fields inside proto3 submessage */ + return *(const bool*)pSize == false; + } + + /* Rest is proto3 singular fields */ + + if (PB_ATYPE(type) == PB_ATYPE_STATIC) { - if (PB_LTYPE(field->type) == PB_LTYPE_BYTES) + if (PB_LTYPE(type) == PB_LTYPE_BYTES) { const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)pData; return bytes->size == 0; } - else if (PB_LTYPE(field->type) == PB_LTYPE_STRING) + else if (PB_LTYPE(type) == PB_LTYPE_STRING) { return *(const char*)pData == '\0'; } - else if (PB_LTYPE(field->type) == PB_LTYPE_FIXED_LENGTH_BYTES) + else if (PB_LTYPE(type) == PB_LTYPE_FIXED_LENGTH_BYTES) { /* Fixed length bytes is only empty if its length is fixed * as 0. Which would be pretty strange, but we can check * it anyway. */ return field->data_size == 0; } - else if (PB_LTYPE(field->type) == PB_LTYPE_SUBMESSAGE) + else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) { /* Check all fields in the submessage to find if any of them * are non-zero. The comparison cannot be done byte-per-byte -- cgit 1.2.3-korg