Implement generator support for extension fields (no encoder/decoder support yet)

6 files changed, 183 insertions, 16 deletions

diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py
index bd365dfd..61f4d7b3 100644
--- a/generator/nanopb_generator.py
+++ b/generator/nanopb_generator.py
@@ -275,8 +275,53 @@ class Field:
         return max(self.tag, self.max_size, self.max_count)        
 
 
+class ExtensionRange(Field):
+    def __init__(self, struct_name, desc, field_options):
+        '''desc is ExtensionRange'''
+        self.tag = desc.start
+        self.struct_name = struct_name
+        self.name = 'extensions'
+        self.pbtype = 'EXTENSION'
+        self.rules = 'OPTIONAL'
+        self.allocation = 'CALLBACK'
+        self.ctype = 'pb_extension_t'
+        self.array_decl = ''
+        self.default = None
+        self.max_size = 0
+        self.max_count = 0
+        
+    def __str__(self):
+        return '    pb_extension_t *extensions;'
+    
+    def types(self):
+        return None
+    
+    def tags(self):
+        return ''
 
-
+class ExtensionField(Field):
+    def __init__(self, struct_name, desc, field_options):
+        self.fullname = struct_name + desc.name
+        self.extendee_name = names_from_type_name(desc.extendee)
+        Field.__init__(self, self.fullname + 'struct', desc, field_options)
+
+    def extension_decl(self):
+        '''Declaration of the extension type in the .pb.h file'''
+        return 'extern const pb_extension_type_t %s;' % self.fullname
+
+    def extension_def(self):
+        '''Definition of the extension type in the .pb.c file'''
+        result  = 'typedef struct {\n'
+        result += str(self)
+        result += '} %s;\n' % self.struct_name
+        result += ('static const pb_field_t %s_field = %s;\n' %
+                    (self.fullname, self.pb_field_t(None)))
+        result += 'const pb_extension_type_t %s = {\n' % self.fullname
+        result += '    NULL,\n'
+        result += '    NULL,\n'
+        result += '    &%s_field\n' % self.fullname
+        result += '};\n'
+        return result
 
 
 # ---------------------------------------------------------------------------
@@ -294,6 +339,11 @@ class Message:
             if field_options.type != nanopb_pb2.FT_IGNORE:
                 self.fields.append(Field(self.name, f, field_options))
         
+        if len(desc.extension_range) > 0:
+            field_options = get_nanopb_suboptions(desc, message_options, self.name + 'extensions')
+            if field_options.type != nanopb_pb2.FT_IGNORE:
+                self.fields.append(ExtensionRange(self.name, desc.extension_range[0], field_options))
+        
         self.packed = message_options.packed_struct
         self.ordered_fields = self.fields[:]
         self.ordered_fields.sort()
@@ -358,9 +408,6 @@ class Message:
 
 
 
-
-
-
 # ---------------------------------------------------------------------------
 #                    Processing of entire .proto files
 # ---------------------------------------------------------------------------
@@ -380,11 +427,23 @@ def iterate_messages(desc, names = Names()):
         for x in iterate_messages(submsg, sub_names):
             yield x
 
+def iterate_extensions(desc, names = Names()):
+    '''Recursively find all extensions.
+    For each, yield name, FieldDescriptorProto.
+    '''
+    for extension in desc.extension:
+        yield names, extension
+
+    for subname, subdesc in iterate_messages(desc, names):
+        for extension in subdesc.extension:
+            yield subname, extension
+
 def parse_file(fdesc, file_options):
-    '''Takes a FileDescriptorProto and returns tuple (enum, messages).'''
+    '''Takes a FileDescriptorProto and returns tuple (enums, messages, extensions).'''
     
     enums = []
     messages = []
+    extensions = []
     
     if fdesc.package:
         base_name = Names(fdesc.package.split('.'))
@@ -402,6 +461,10 @@ def parse_file(fdesc, file_options):
             enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name)
             enums.append(Enum(names, enum, enum_options))
     
+    for names, extension in iterate_extensions(fdesc, base_name):
+        field_options = get_nanopb_suboptions(extension, file_options, names)
+        extensions.append(ExtensionField(names, extension, field_options))
+    
     # Fix field default values where enum short names are used.
     for enum in enums:
         if not enum.options.long_names:
@@ -411,7 +474,7 @@ def parse_file(fdesc, file_options):
                         idx = enum.value_longnames.index(field.default)
                         field.default = enum.values[idx][0]
     
-    return enums, messages
+    return enums, messages, extensions
 
 def toposort2(data):
     '''Topological sort.
@@ -454,7 +517,7 @@ def make_identifier(headername):
             result += '_'
     return result
 
-def generate_header(dependencies, headername, enums, messages, options):
+def generate_header(dependencies, headername, enums, messages, extensions, options):
     '''Generate content for a header file.
     Generates strings, which should be concatenated and stored to file.
     '''
@@ -489,6 +552,12 @@ def generate_header(dependencies, headername, enums, messages, options):
     for msg in sort_dependencies(messages):
         yield msg.types()
         yield str(msg) + '\n\n'
+    
+    if extensions:
+        yield '/* Extensions */\n'
+        for extension in extensions:
+            yield extension.extension_decl()
+        yield '\n'
         
     yield '/* Default values for struct fields */\n'
     for msg in messages:
@@ -512,7 +581,7 @@ def generate_header(dependencies, headername, enums, messages, options):
     # End of header
     yield '\n#endif\n'
 
-def generate_source(headername, enums, messages):
+def generate_source(headername, enums, messages, extensions):
     '''Generate content for a source file.'''
     
     yield '/* Automatically generated nanopb constant definitions */\n'
@@ -527,7 +596,11 @@ def generate_source(headername, enums, messages):
     
     for msg in messages:
         yield msg.fields_definition() + '\n\n'
+    
+    for ext in extensions:
+        yield ext.extension_def() + '\n'
         
+    # Add checks for numeric limits
     if messages:
         count_required_fields = lambda m: len([f for f in msg.fields if f.rules == 'REQUIRED'])
         largest_msg = max(messages, key = count_required_fields)
@@ -539,7 +612,6 @@ def generate_source(headername, enums, messages):
             yield '       setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count
             yield '#endif\n'
     
-    # Add checks for numeric limits
     worst = 0
     worst_field = ''
     checks = []
@@ -724,7 +796,7 @@ def process(filenames, options):
         
         # Parse the file
         file_options = get_nanopb_suboptions(fdesc.file[0], toplevel_options, Names([filename]))
-        enums, messages = parse_file(fdesc.file[0], file_options)
+        enums, messages, extensions = parse_file(fdesc.file[0], file_options)
         
         noext = os.path.splitext(filename)[0]
         headername = noext + '.' + options.extension + '.h'
@@ -740,11 +812,12 @@ def process(filenames, options):
         dependencies = [d for d in fdesc.file[0].dependency if d not in excludes]
         
         header = open(headername, 'w')
-        for part in generate_header(dependencies, headerbasename, enums, messages, options):
+        for part in generate_header(dependencies, headerbasename, enums,
+                                    messages, extensions, options):
             header.write(part)
 
         source = open(sourcename, 'w')
-        for part in generate_source(headerbasename, enums, messages):
+        for part in generate_source(headerbasename, enums, messages, extensions):
             source.write(part)
 
     return True
diff --git a/pb.h b/pb.h
index d5047ac5..15c346d8 100644
--- a/pb.h
+++ b/pb.h
@@ -148,8 +148,12 @@ typedef uint8_t pb_type_t;
  * submsg_fields is pointer to field descriptions */
 #define PB_LTYPE_SUBMESSAGE 0x06
 
+/* Extension pseudo-field
+ * The field contains a pointer to pb_extension_t */
+#define PB_LTYPE_EXTENSION 0x07
+
 /* Number of declared LTYPES */
-#define PB_LTYPES_COUNT 7
+#define PB_LTYPES_COUNT 8
 #define PB_LTYPE_MASK 0x0F
 
 /**** Field repetition rules ****/
@@ -270,6 +274,51 @@ typedef enum {
     PB_WT_32BIT  = 5
 } pb_wire_type_t;
 
+/* Structure for defining the handling of unknown/extension fields.
+ * Usually the pb_extension_type_t structure is automatically generated,
+ * while the pb_extension_t structure is created by the user. However,
+ * if you want to catch all unknown fields, you can also create a custom
+ * pb_extension_type_t with your own callback.
+ */
+typedef struct _pb_extension_type_t pb_extension_type_t;
+typedef struct _pb_extension_t pb_extension_t;
+struct _pb_extension_type_t {
+    /* Called for each unknown field in the message.
+     * If you handle the field, read off all of its data and return true.
+     * If you do not handle the field, do not read anything and return true.
+     * If you run into an error, return false.
+     * Set to NULL for default handler.
+     */
+    bool (*decode)(pb_istream_t *stream, pb_extension_t *extension,
+                   uint32_t tag, pb_wire_type_t wire_type);
+    
+    /* Called once after all regular fields have been encoded.
+     * If you have something to write, do so and return true.
+     * If you do not have anything to write, just return true.
+     * If you run into an error, return false.
+     * Set to NULL for default handler.
+     */
+    bool (*encode)(pb_ostream_t *stream, pb_extension_t *extension);
+    
+    /* Free field for use by the callback. */
+    const void *arg;
+};
+
+struct _pb_extension_t {
+    /* Type describing the extension field. Usually you'll initialize
+     * this to a pointer to the automatically generated structure. */
+    const pb_extension_type_t *type;
+    
+    /* Destination for the decoded data. This must match the datatype
+     * of the extension field. */
+    void *dest;
+    
+    /* Pointer to the next extension handler, or NULL.
+     * If this extension does not match a field, the next handler is
+     * automatically called. */
+    pb_extension_t *next;
+};
+
 /* These macros are used to declare pb_field_t's in the constant array. */
 #define pb_membersize(st, m) (sizeof ((st*)0)->m)
 #define pb_arraysize(st, m) (pb_membersize(st, m) / pb_membersize(st, m[0]))
@@ -333,13 +382,14 @@ typedef enum {
 #define PB_LTYPE_MAP_STRING     PB_LTYPE_STRING
 #define PB_LTYPE_MAP_UINT32     PB_LTYPE_VARINT
 #define PB_LTYPE_MAP_UINT64     PB_LTYPE_VARINT
+#define PB_LTYPE_MAP_EXTENSION  PB_LTYPE_EXTENSION
 
 /* This is the actual macro used in field descriptions.
  * It takes these arguments:
  * - Field tag number
  * - Field type:   BOOL, BYTES, DOUBLE, ENUM, FIXED32, FIXED64,
  *                 FLOAT, INT32, INT64, MESSAGE, SFIXED32, SFIXED64
- *                 SINT32, SINT64, STRING, UINT32 or UINT64
+ *                 SINT32, SINT64, STRING, UINT32, UINT64 or EXTENSION
  * - Field rules:  REQUIRED, OPTIONAL or REPEATED
  * - Allocation:   STATIC or CALLBACK
  * - Message name
@@ -352,6 +402,7 @@ typedef enum {
     PB_ ## rules ## _ ## allocation(tag, message, field, prevfield, \
                                     PB_LTYPE_MAP_ ## type, ptr)
 
+
 /* These macros are used for giving out error messages.
  * They are mostly a debugging aid; the main error information
  * is the true/false return value from functions.
diff --git a/tests/Makefile b/tests/Makefile
index 16f62e5c..dc564c28 100644
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -8,7 +8,7 @@ TESTS=  decode_unittests encode_unittests \
 	test_decode_callbacks test_encode_callbacks \
 	test_missing_fields test_no_messages test_funny_name \
 	test_multiple_files test_cxxcompile test_options \
-	bc_encode bc_decode
+	bc_encode bc_decode test_encode_extensions
 
 # More strict checks for the core part of nanopb
 CC_VERSION=$(shell $(CC) -v 2>&1)
@@ -84,6 +84,7 @@ test_no_messages: no_messages.pb.h no_messages.pb.c no_messages.pb.o
 test_funny_name: funny-proto+name.pb.h funny-proto+name.pb.o
 bc_encode: bc_alltypes.pb.o pb_encode.o bc_encode.o
 bc_decode: bc_alltypes.pb.o pb_decode.o bc_decode.o
+test_encode_extensions: test_encode_extensions.c pb_encode.o alltypes.pb.o extensions.pb.o
 
 %.pb: %.proto
 	protoc -I. -I../generator -I/usr/include -o$@ $<
@@ -125,7 +126,7 @@ run_unittests: $(TESTS)
 	./test_encode3_buf 1 | ./test_decode3_buf 1
 	./test_decode3 < alltypes_with_extra_fields.pb
 	./bc_encode | ./bc_decode
-	
+
 	./test_missing_fields
 
 test_options: options.pb.h options.expected options.pb.o
diff --git a/tests/alltypes.proto b/tests/alltypes.proto
index 6ccf57d7..a2cf8bbc 100644
--- a/tests/alltypes.proto
+++ b/tests/alltypes.proto
@@ -86,5 +86,8 @@ message AllTypes {
     // Just to make sure that the size of the fields has been calculated
     // properly, i.e. otherwise a bug in last field might not be detected.
     required int32      end = 99;
+
+
+    extensions 200 to 255;
 }
 
diff --git a/tests/extensions.proto b/tests/extensions.proto
new file mode 100644
index 00000000..12c7c9a5
--- /dev/null
+++ b/tests/extensions.proto
@@ -0,0 +1,5 @@
+import 'alltypes.proto';
+
+extend AllTypes {
+    optional int32 AllTypes_extensionfield1 = 255;
+}
diff --git a/tests/test_encode_extensions.c b/tests/test_encode_extensions.c
new file mode 100644
index 00000000..a3500d7d
--- /dev/null
+++ b/tests/test_encode_extensions.c
@@ -0,0 +1,34 @@
+/* Tests extension fields.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <pb_encode.h>
+#include "alltypes.pb.h"
+#include "extensions.pb.h"
+
+int main(int argc, char **argv)
+{
+    AllTypes alltypes = {0};
+    int32_t extensionfield1 = 12345;
+    pb_extension_t ext1 = {&AllTypes_extensionfield1, &extensionfield1, NULL};
+    
+    alltypes.extensions = &ext1;
+    
+    uint8_t buffer[1024];
+    pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer));
+    
+    /* Now encode it and check if we succeeded. */
+    if (pb_encode(&stream, AllTypes_fields, &alltypes))
+    {
+        fwrite(buffer, 1, stream.bytes_written, stdout);
+        return 0; /* Success */
+    }
+    else
+    {
+        fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream));
+        return 1; /* Failure */
+    }
+}
+