diff options
| -rw-r--r-- | docs/concepts.rst | 4 | ||||
| -rw-r--r-- | docs/index.rst | 2 | ||||
| -rw-r--r-- | docs/reference.rst | 2 | ||||
| -rw-r--r-- | example/Makefile | 11 | ||||
| -rw-r--r-- | example/fileproto.proto | 26 | ||||
| -rw-r--r-- | example/server.c | 149 | ||||
| -rw-r--r-- | generator/nanopb_generator.py | 8 | ||||
| -rw-r--r-- | pb.h | 2 | ||||
| -rw-r--r-- | pb_decode.c | 33 | ||||
| -rw-r--r-- | pb_encode.c | 7 | ||||
| -rw-r--r-- | tests/Makefile | 18 | ||||
| -rw-r--r-- | tests/person.proto | 3 | ||||
| -rw-r--r-- | tests/test_decode1.c | 28 | ||||
| -rw-r--r-- | tests/test_encode1.c | 3 |
14 files changed, 263 insertions, 33 deletions
diff --git a/docs/concepts.rst b/docs/concepts.rst index 58620a85..fac9061f 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -164,7 +164,7 @@ Field callbacks =============== When a field has dynamic length, nanopb cannot statically allocate storage for it. Instead, it allows you to handle the field in whatever way you want, using a callback function. -The `pb_callback_t`_ structure contains a function pointer and a *void* pointer you can use for passing data to the callback. The actual behavior of the callback function is different in encoding and decoding modes. +The `pb_callback_t`_ structure contains a function pointer and a *void* pointer you can use for passing data to the callback. If the function pointer is NULL, the field will be skipped. The actual behavior of the callback function is different in encoding and decoding modes. .. _`pb_callback_t`: reference.html#pb-callback-t @@ -176,7 +176,7 @@ Encoding callbacks When encoding, the callback should write out complete fields, including the wire type and field number tag. It can write as many or as few fields as it likes. For example, if you want to write out an array as *repeated* field, you should do it all in a single call. -The callback may be called multiple times during a single call to `pb_encode`_. It must produce the same amount of data every time. +If the callback is used in a submessage, it will be called multiple times during a single call to `pb_encode`_. It must produce the same amount of data every time. If the callback is directly in the main message, it is called only once. .. _`pb_encode`: reference.html#pb-encode diff --git a/docs/index.rst b/docs/index.rst index 93f04c6c..ea891237 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -42,7 +42,7 @@ Features and limitations **Limitations** -#) User must provide callbacks when decoding arrays or strings without maximum size. +#) User must provide callbacks when decoding arrays or strings without maximum size. Malloc support could be added as a separate module. #) Some speed has been sacrificed for code size. For example varint calculations are always done in 64 bits. #) Encoding is focused on writing to streams. For memory buffers only it could be made more efficient. #) The deprecated Protocol Buffers feature called "groups" is not supported. diff --git a/docs/reference.rst b/docs/reference.rst index b8f9454e..2812958f 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -94,7 +94,7 @@ Part of a message structure, for fields with type PB_HTYPE_CALLBACK:: The *arg* is passed to the callback when calling. It can be used to store any information that the callback might need. -When calling `pb_encode`_, *funcs.encode* must be set, and similarly when calling `pb_decode`_, *funcs.decode* must be set. The function pointers are stored in the same memory location but are of incompatible types. +When calling `pb_encode`_, *funcs.encode* is used, and similarly when calling `pb_decode`_, *funcs.decode* is used. The function pointers are stored in the same memory location but are of incompatible types. You can set the function pointer to NULL to skip the field. pb_wire_type_t -------------- diff --git a/example/Makefile b/example/Makefile new file mode 100644 index 00000000..0f08f2d7 --- /dev/null +++ b/example/Makefile @@ -0,0 +1,11 @@ +CFLAGS=-ansi -Wall -Werror -I .. -g -O0 +DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h + +all: server + +%: %.c $(DEPS) fileproto.h + $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c fileproto.c + +fileproto.h: fileproto.proto ../generator/nanopb_generator.py + protoc -I. -I../generator -I/usr/include -ofileproto.pb $< + python ../generator/nanopb_generator.py fileproto.pb diff --git a/example/fileproto.proto b/example/fileproto.proto new file mode 100644 index 00000000..e2786b15 --- /dev/null +++ b/example/fileproto.proto @@ -0,0 +1,26 @@ +import "nanopb.proto"; + +// This defines protocol for a simple server that lists files. +// +// If you come from high-level programming background, the hardcoded +// maximum lengths may disgust you. However, if your microcontroller only +// has a few kB of ram to begin with, setting reasonable limits for +// filenames is ok. +// +// On the other hand, using the callback interface, it is not necessary +// to set a limit on the number of files in the response. + +message ListFilesRequest { + optional string path = 1 [default = "/", (nanopb).max_size = 128]; +} + +message FileInfo { + required uint64 inode = 1; + required string name = 2 [(nanopb).max_size = 128]; +} + +message ListFilesResponse { + optional bool path_error = 1 [default = false]; + repeated FileInfo file = 2; +} + diff --git a/example/server.c b/example/server.c new file mode 100644 index 00000000..a671f4cb --- /dev/null +++ b/example/server.c @@ -0,0 +1,149 @@ +/* This is a simple TCP server that listens on port 1234 and provides lists + * of files to clients, using a protocol defined in file_server.proto. + * + * It directly deserializes and serializes messages from network, minimizing + * memory use. + * + * For flexibility, this example is implemented using posix api. + * In a real embedded system you would typically use some other kind of + * a communication and filesystem layer. + */ + +#include <sys/socket.h> +#include <sys/types.h> +#include <netinet/in.h> +#include <unistd.h> +#include <dirent.h> +#include <stdio.h> +#include <string.h> + +#include <pb_encode.h> +#include <pb_decode.h> + +#include "fileproto.h" + +bool write_callback(pb_ostream_t *stream, const uint8_t *buf, size_t count) +{ + int fd = *(int*)stream->state; + return send(fd, buf, count, 0) == count; +} + +bool read_callback(pb_istream_t *stream, uint8_t *buf, size_t count) +{ + int fd = *(int*)stream->state; + + if (buf == NULL) + { + /* Well, this is a really inefficient way to skip input. */ + /* It is only used when there are unknown fields. */ + char dummy; + while (count-- && recv(fd, &dummy, 1, 0) == 1); + return count == 0; + } + + return recv(fd, buf, count, MSG_WAITALL) == count; +} + +bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, const void *arg) +{ + DIR *dir = (DIR*) arg; + struct dirent *file; + FileInfo fileinfo; + + while ((file = readdir(dir)) != NULL) + { + fileinfo.inode = file->d_ino; + strncpy(fileinfo.name, file->d_name, sizeof(fileinfo.name)); + fileinfo.name[sizeof(fileinfo.name) - 1] = '\0'; + + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!pb_enc_submessage(stream, field, &fileinfo)) + return false; + } + + return true; +} + +void handle_connection(int connfd) +{ + ListFilesRequest request; + ListFilesResponse response; + pb_istream_t input = {&read_callback, &connfd, SIZE_MAX}; + pb_ostream_t output = {&write_callback, &connfd, SIZE_MAX, 0}; + DIR *directory; + + if (!pb_decode(&input, ListFilesRequest_fields, &request)) + { + printf("Decoding failed.\n"); + return; + } + + directory = opendir(request.path); + + printf("Listing directory: %s\n", request.path); + + if (directory == NULL) + { + perror("opendir"); + + response.has_path_error = true; + response.path_error = true; + response.file.funcs.encode = NULL; + } + else + { + response.has_path_error = false; + response.file.funcs.encode = &listdir_callback; + response.file.arg = directory; + } + + if (!pb_encode(&output, ListFilesResponse_fields, &response)) + { + printf("Encoding failed.\n"); + } +} + +int main(int argc, char **argv) +{ + int listenfd, connfd; + struct sockaddr_in servaddr; + + listenfd = socket(AF_INET, SOCK_STREAM, 0); + + memset(&servaddr, 0, sizeof(servaddr)); + servaddr.sin_family = AF_INET; + servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + servaddr.sin_port = htons(1234); + if (bind(listenfd, (struct sockaddr*)&servaddr, sizeof(servaddr)) != 0) + { + perror("bind"); + return 1; + } + + if (listen(listenfd, 5) != 0) + { + perror("listen"); + return 1; + } + + for(;;) + { + connfd = accept(listenfd, NULL, NULL); + + if (connfd < 0) + { + perror("accept"); + return 1; + } + + printf("Got connection.\n"); + + handle_connection(connfd); + + printf("Closing connection.\n"); + + close(connfd); + } +} diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index e62d04f1..a6d38c0b 100644 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -124,7 +124,7 @@ class Field: self.ctype = self.struct_name + self.name + 't' elif desc.type == FieldD.TYPE_MESSAGE: self.ltype = 'PB_LTYPE_SUBMESSAGE' - self.ctype = names_from_type_name(desc.type_name) + self.ctype = self.submsgname = names_from_type_name(desc.type_name) else: raise NotImplementedError(desc.type) @@ -167,8 +167,8 @@ class Field: if self.max_size is None: return None # Not implemented else: - array_decl = '[%d]' % self.max_size - default = self.default.encode('string_escape') + array_decl = '[%d]' % (self.max_size + 1) + default = str(self.default).encode('string_escape') default = default.replace('"', '\\"') default = '"' + default + '"' elif self.ltype == 'PB_LTYPE_BYTES': @@ -223,7 +223,7 @@ class Field: result += ' 0,' if self.ltype == 'PB_LTYPE_SUBMESSAGE': - result += '\n &%s_fields}' % self.ctype + result += '\n &%s_fields}' % self.submsgname elif self.default is None or self.htype == 'PB_HTYPE_CALLBACK': result += ' 0}' else: @@ -126,6 +126,8 @@ typedef struct { * The encoding callback will receive the actual output stream. * It should write all the data in one call, including the field tag and * wire type. It can write multiple fields. + * + * The callback can be null if you want to skip a field. */ typedef struct _pb_istream_t pb_istream_t; typedef struct _pb_ostream_t pb_ostream_t; diff --git a/pb_decode.c b/pb_decode.c index 379d134c..2cde54c2 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -185,14 +185,19 @@ static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, v { iter->start = iter->current = fields; iter->field_index = 0; - iter->pData = dest_struct; + iter->pData = dest_struct + iter->current->data_offset; + iter->pSize = (char*)iter->pData + iter->current->size_offset; iter->dest_struct = dest_struct; } static bool pb_field_next(pb_field_iterator_t *iter) { bool notwrapped = true; - size_t prev_size = iter->current->data_size * iter->current->array_size; + size_t prev_size = iter->current->data_size; + + if (PB_HTYPE(iter->current->type) == PB_HTYPE_ARRAY) + prev_size *= iter->current->array_size; + iter->current++; iter->field_index++; if (iter->current->tag == 0) @@ -271,9 +276,14 @@ static bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_ } case PB_HTYPE_CALLBACK: + { + pb_callback_t *pCallback = (pb_callback_t*)iter->pData; + + if (pCallback->funcs.decode == NULL) + return skip(stream, wire_type); + if (wire_type == PB_WT_STRING) { - pb_callback_t *pCallback = (pb_callback_t*)iter->pData; pb_istream_t substream; if (!make_string_substream(stream, &substream)) @@ -293,7 +303,6 @@ static bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_ * which in turn allows to use same callback for packed and * not-packed fields. */ pb_istream_t substream; - pb_callback_t *pCallback = (pb_callback_t*)iter->pData; uint8_t buffer[10]; size_t size = sizeof(buffer); @@ -303,7 +312,8 @@ static bool decode_field(pb_istream_t *stream, int wire_type, pb_field_iterator_ return pCallback->funcs.decode(&substream, iter->current, pCallback->arg); } - + } + default: return false; } @@ -344,6 +354,10 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc { *(size_t*)iter.pSize = 0; } + else if (PB_HTYPE(iter.current->type) == PB_HTYPE_REQUIRED) + { + memset(iter.pData, 0, iter.current->data_size); + } } while (pb_field_next(&iter)); while (stream->bytes_left) @@ -351,10 +365,15 @@ bool pb_decode(pb_istream_t *stream, const pb_field_t fields[], void *dest_struc uint32_t temp; int tag, wire_type; if (!pb_decode_varint32(stream, &temp)) - return stream->bytes_left == 0; /* Was it EOF? */ + { + if (stream->bytes_left == 0) + break; /* It was EOF */ + else + return false; /* It was error */ + } if (temp == 0) - return true; /* Special feature: allow 0-terminated messages. */ + break; /* Special feature: allow 0-terminated messages. */ tag = temp >> 3; wire_type = temp & 7; diff --git a/pb_encode.c b/pb_encode.c index 188d768d..2e740347 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -130,7 +130,10 @@ bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_ { pData = (const char*)pData + prev_size + field->data_offset; pSize = (const char*)pData + field->size_offset; - prev_size = field->data_size * field->array_size; + + prev_size = field->data_size; + if (PB_HTYPE(field->type) == PB_HTYPE_ARRAY) + prev_size *= field->array_size; pb_encoder_t func = PB_ENCODERS[PB_LTYPE(field->type)]; @@ -339,7 +342,7 @@ bool pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void substream.max_size = size; substream.bytes_written = 0; - status = pb_encode(stream, (pb_field_t*)field->ptr, src); + status = pb_encode(&substream, (pb_field_t*)field->ptr, src); stream->bytes_written += substream.bytes_written; diff --git a/tests/Makefile b/tests/Makefile index 4993297e..243dbd17 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -2,21 +2,27 @@ CFLAGS=-ansi -Wall -Werror -I .. -g -O0 DEPS=../pb_decode.c ../pb_decode.h ../pb_encode.c ../pb_encode.h ../pb.h person.h unittests.h TESTS=test_decode1 test_encode1 decode_unittests encode_unittests -all: $(TESTS) run_unittests +all: $(TESTS) run_unittests breakpoints clean: rm -f $(TESTS) %: %.c $(DEPS) - $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c + $(CC) $(CFLAGS) -o $@ $< ../pb_decode.c ../pb_encode.c person.c -%.h: %.proto +person.h: person.proto protoc -I. -I../generator -I/usr/include -operson.pb $< python ../generator/nanopb_generator.py person.pb -run_unittests: decode_unittests encode_unittests - ./decode_unittests - ./encode_unittests +breakpoints: ../*.c *.c + grep -n 'return false;' $^ | cut -d: -f-2 | xargs -n 1 echo b > $@ + +run_unittests: decode_unittests encode_unittests test_encode1 test_decode1 + ./decode_unittests > /dev/null + ./encode_unittests > /dev/null + + [ "`./test_encode1 | ./test_decode1`" = \ + "`./test_encode1 | protoc --decode=Person -I. -I../generator -I/usr/include person.proto`" ] run_fuzztest: test_decode1 bash -c 'I=1; while cat /dev/urandom | ./test_decode1 > /dev/null; do I=$$(($$I+1)); echo -en "\r$$I"; done'
\ No newline at end of file diff --git a/tests/person.proto b/tests/person.proto index 5befb076..dafcf934 100644 --- a/tests/person.proto +++ b/tests/person.proto @@ -4,8 +4,7 @@ message Person { required string name = 1 [(nanopb).max_size = 40]; required int32 id = 2; optional string email = 3 [(nanopb).max_size = 40]; - optional bytes test = 5 [default="\x00\x01\x02", (nanopb).max_size = 20]; - + enum PhoneType { MOBILE = 0; HOME = 1; diff --git a/tests/test_decode1.c b/tests/test_decode1.c index a2c7f428..b7698efc 100644 --- a/tests/test_decode1.c +++ b/tests/test_decode1.c @@ -2,9 +2,6 @@ #include <pb_decode.h> #include "person.h" -/* This test has only one source file anyway.. */ -#include "person.c" - bool print_person(pb_istream_t *stream) { int i; @@ -13,12 +10,33 @@ bool print_person(pb_istream_t *stream) if (!pb_decode(stream, Person_fields, &person)) return false; - printf("Person: name '%s' id '%d' email '%s'\n", person.name, person.id, person.email); + printf("name: \"%s\"\n", person.name); + printf("id: %d\n", person.id); + + if (person.has_email) + printf("email: \"%s\"\n", person.email); for (i = 0; i < person.phone_count; i++) { Person_PhoneNumber *phone = &person.phone[i]; - printf("PhoneNumber: number '%s' type '%d'\n", phone->number, phone->type); + printf("phone {\n"); + printf(" number: \"%s\"\n", phone->number); + + switch (phone->type) + { + case Person_PhoneType_WORK: + printf(" type: WORK\n"); + break; + + case Person_PhoneType_HOME: + printf(" type: HOME\n"); + break; + + case Person_PhoneType_MOBILE: + printf(" type: MOBILE\n"); + break; + } + printf("}\n"); } return true; diff --git a/tests/test_encode1.c b/tests/test_encode1.c index 99be7cb5..ac13df35 100644 --- a/tests/test_encode1.c +++ b/tests/test_encode1.c @@ -2,9 +2,6 @@ #include <pb_encode.h> #include "person.h" -/* This test has only one source file anyway.. */ -#include "person.c" - bool streamcallback(pb_ostream_t *stream, const uint8_t *buf, size_t count) { FILE *file = (FILE*) stream->state; |