From b222209e1bf80b2e855817d8f9287c5e2dc1eb04 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 19 Sep 2015 14:42:03 -0700 Subject: generator: Don't force python2 Python2 is being phased out of the default python interpreter. Arch Linux has moved some time ago and upcoming Debian realeases will follow. My distro, Arch, doesn't have a python2-protobuf version, but does have a python3 version. With a python 2 & 3 compatible generator the exact interpreter can be ignored and can use the system default as most users expect. Update issue #155 --- generator/protoc-gen-nanopb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'generator') diff --git a/generator/protoc-gen-nanopb b/generator/protoc-gen-nanopb index 6dc468d..358f97c 100755 --- a/generator/protoc-gen-nanopb +++ b/generator/protoc-gen-nanopb @@ -10,5 +10,4 @@ # --plugin= on the command line. MYPATH=$(dirname "$0") -PYTHON=$(which python2 || which python) -exec $PYTHON "$MYPATH/nanopb_generator.py" --protoc-plugin +exec "$MYPATH/nanopb_generator.py" --protoc-plugin -- cgit 1.2.3-korg From 02367d6d77b20cb8bb1dcac4a70ae7d6849a2e1f Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 19 Sep 2015 14:12:36 -0700 Subject: generator: Run python's 2to3 converter * Invoked with `2to3 -w nanopb_generator.py` * No other changes. --- generator/nanopb_generator.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'generator') diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 3a5fac5..17cb674 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -5,6 +5,7 @@ nanopb_version = "nanopb-0.3.4-dev" import sys import re +from functools import reduce try: # Add some dummy imports to keep packaging tools happy. @@ -82,7 +83,7 @@ class Names: return '_'.join(self.parts) def __add__(self, other): - if isinstance(other, (str, unicode)): + if isinstance(other, str): return Names(self.parts + (other,)) elif isinstance(other, tuple): return Names(self.parts + other) @@ -123,7 +124,7 @@ class EncodedSize: self.symbols = symbols def __add__(self, other): - if isinstance(other, (int, long)): + if isinstance(other, int): return EncodedSize(self.value + other, self.symbols) elif isinstance(other, (str, Names)): return EncodedSize(self.value, self.symbols + [str(other)]) @@ -133,7 +134,7 @@ class EncodedSize: raise ValueError("Cannot add size: " + repr(other)) def __mul__(self, other): - if isinstance(other, (int, long)): + if isinstance(other, int): return EncodedSize(self.value * other, [str(other) + '*' + s for s in self.symbols]) else: raise ValueError("Cannot multiply size: " + repr(other)) @@ -260,7 +261,7 @@ class Field: raise NotImplementedError(field_options.type) # Decide the C data type to use in the struct. - if datatypes.has_key(desc.type): + if desc.type in datatypes: self.ctype, self.pbtype, self.enc_size, isa = datatypes[desc.type] # Override the field size if user wants to use smaller integers @@ -875,17 +876,17 @@ def toposort2(data): From http://code.activestate.com/recipes/577413-topological-sort/ This function is under the MIT license. ''' - for k, v in data.items(): + for k, v in list(data.items()): v.discard(k) # Ignore self dependencies - extra_items_in_deps = reduce(set.union, data.values(), set()) - set(data.keys()) + extra_items_in_deps = reduce(set.union, list(data.values()), set()) - set(data.keys()) data.update(dict([(item, set()) for item in extra_items_in_deps])) while True: - ordered = set(item for item,dep in data.items() if not dep) + ordered = set(item for item,dep in list(data.items()) if not dep) if not ordered: break for item in sorted(ordered): yield item - data = dict([(item, (dep - ordered)) for item,dep in data.items() + data = dict([(item, (dep - ordered)) for item,dep in list(data.items()) if item not in ordered]) assert not data, "A cyclic dependency exists amongst %r" % data @@ -1145,7 +1146,7 @@ class ProtoFile: checks_msgnames.append(msg.name) for field in msg.fields: status = field.largest_field_value() - if isinstance(status, (str, unicode)): + if isinstance(status, str): checks.append(status) elif status > worst: worst = status @@ -1237,7 +1238,7 @@ def read_options_file(infile): try: text_format.Merge(parts[1], opts) - except Exception, e: + except Exception as e: sys.stderr.write("%s:%d: " % (infile.name, i + 1) + "Unparseable option line: '%s'. " % line + "Error: %s\n" % str(e)) -- cgit 1.2.3-korg From 2731fe3e6a4ba6fd858947e343160c95c96042e2 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 19 Sep 2015 14:30:21 -0700 Subject: generator: Use python2/3 binary read/write method * This works with python2 and python3 --- generator/nanopb_generator.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'generator') diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 17cb674..c2acb06 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1440,14 +1440,15 @@ def main_cli(): def main_plugin(): '''Main function when invoked as a protoc plugin.''' - import sys + import io, sys if sys.platform == "win32": import os, msvcrt # Set stdin and stdout to binary mode msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) - data = sys.stdin.read() + data = io.open(sys.stdin.fileno(), "rb").read() + request = plugin_pb2.CodeGeneratorRequest.FromString(data) try: @@ -1490,7 +1491,7 @@ def main_plugin(): f.name = results['sourcename'] f.content = results['sourcedata'] - sys.stdout.write(response.SerializeToString()) + io.open(sys.stdout.fileno(), "wb").write(response.SerializeToString()) if __name__ == '__main__': # Check if we are running as a plugin under protoc -- cgit 1.2.3-korg From 56134e87657714b50d9837f846410fc30fa9fe7b Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 19 Sep 2015 14:36:55 -0700 Subject: generator: More exhaustive field size checking Create a FieldMaxSize class that: * Accumlates all C assertions * Handles the checking of the longest simple field * Also python3 doesn't support max(None) --- generator/nanopb_generator.py | 51 ++++++++++++++++++++++++++++++------------- 1 file changed, 36 insertions(+), 15 deletions(-) (limited to 'generator') diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index c2acb06..240fa15 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -193,6 +193,24 @@ class Enum: return result +class FieldMaxSize: + def __init__(self, worst = 0, checks = [], field_name = 'undefined'): + if isinstance(worst, list): + self.worst = max(i for i in worst if i is not None) + else: + self.worst = worst + + self.worst_field = field_name + self.checks = checks + + def extend(self, extend, field_name = None): + self.worst = max(self.worst, extend.worst) + + if self.worst == extend.worst: + self.worst_field = extend.worst_field + + self.checks.extend(extend.checks) + class Field: def __init__(self, struct_name, desc, field_options): '''desc is FieldDescriptorProto''' @@ -468,15 +486,18 @@ class Field: def largest_field_value(self): '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly. Returns numeric value or a C-expression for assert.''' + check = [] if self.pbtype == 'MESSAGE': if self.rules == 'REPEATED' and self.allocation == 'STATIC': - return 'pb_membersize(%s, %s[0])' % (self.struct_name, self.name) + check.append('pb_membersize(%s, %s[0])' % (self.struct_name, self.name)) elif self.rules == 'ONEOF': - return 'pb_membersize(%s, %s.%s)' % (self.struct_name, self.union_name, self.name) + check.append('pb_membersize(%s, %s.%s)' % (self.struct_name, self.union_name, self.name)) else: - return 'pb_membersize(%s, %s)' % (self.struct_name, self.name) + check.append('pb_membersize(%s, %s)' % (self.struct_name, self.name)) - return max(self.tag, self.max_size, self.max_count) + return FieldMaxSize([self.tag, self.max_size, self.max_count], + check, + ('%s.%s' % (self.struct_name, self.name))) def encoded_size(self, dependencies): '''Return the maximum size that this field can take when encoded, @@ -676,7 +697,10 @@ class OneOf(Field): return result def largest_field_value(self): - return max([f.largest_field_value() for f in self.fields]) + largest = FieldMaxSize() + for f in self.fields: + largest.extend(f.largest_field_value()) + return largest def encoded_size(self, dependencies): largest = EncodedSize(0) @@ -1137,20 +1161,17 @@ class ProtoFile: yield '#error Properly detecting missing required fields in %s requires \\\n' % largest_msg.name yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count yield '#endif\n' - - worst = 0 - worst_field = '' - checks = [] + + max_field = FieldMaxSize() checks_msgnames = [] for msg in self.messages: checks_msgnames.append(msg.name) for field in msg.fields: - status = field.largest_field_value() - if isinstance(status, str): - checks.append(status) - elif status > worst: - worst = status - worst_field = str(field.struct_name) + '.' + str(field.name) + max_field.extend(field.largest_field_value()) + + worst = max_field.worst + worst_field = max_field.worst_field + checks = max_field.checks if worst > 255 or checks: yield '\n/* Check that field information fits in pb_field_t */\n' -- cgit 1.2.3-korg From 0d7ef5f936afc32bfe0aaec8b92667d4c3a026a0 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 19 Sep 2015 15:03:13 -0700 Subject: generator: Remove cmp() to work with Python3 * Python3 ignores cmp() and __cmp__() and only needs __lt__() for sorting. Delete and update as appropriate. * Ref: https://docs.python.org/3.0/whatsnew/3.0.html#ordering-comparisons --- generator/nanopb_generator.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'generator') diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 240fa15..f7d4322 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -314,8 +314,8 @@ class Field: else: raise NotImplementedError(desc.type) - def __cmp__(self, other): - return cmp(self.tag, other.tag) + def __lt__(self, other): + return self.tag < other.tag def __str__(self): result = '' @@ -661,9 +661,6 @@ class OneOf(Field): # Sort by the lowest tag number inside union self.tag = min([f.tag for f in self.fields]) - def __cmp__(self, other): - return cmp(self.tag, other.tag) - def __str__(self): result = '' if self.fields: -- cgit 1.2.3-korg From 03e3af597f0adf64b21731b820c997116a16cdcd Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 19 Sep 2015 15:13:05 -0700 Subject: generator: Strings are utf-8 by default in python3 * Not sure how to handle this case in python2, seems to work * Python 3 doesn't need this since all strings are utf-8 --- generator/nanopb_generator.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'generator') diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index f7d4322..78140a4 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -379,12 +379,10 @@ class Field: inner_init = '0' else: if self.pbtype == 'STRING': - inner_init = self.default.encode('utf-8').encode('string_escape') - inner_init = inner_init.replace('"', '\\"') + inner_init = self.default.replace('"', '\\"') inner_init = '"' + inner_init + '"' elif self.pbtype == 'BYTES': - data = str(self.default).decode('string_escape') - data = ['0x%02x' % ord(c) for c in data] + data = ['0x%02x' % ord(c) for c in self.default] if len(data) == 0: inner_init = '{0, {0}}' else: -- cgit 1.2.3-korg From e02c65389493d195ee868bf927923108a753bf57 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sun, 20 Sep 2015 17:22:25 -0700 Subject: generator: Attempt to simplify the str/unicode madness * This is a shot in the dark. --- generator/nanopb_generator.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'generator') diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 78140a4..91a820f 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,5 +1,7 @@ #!/usr/bin/python +from __future__ import unicode_literals + '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' nanopb_version = "nanopb-0.3.4-dev" -- cgit 1.2.3-korg From 67cafac8f2f4797e195c5ee1e1fcf691129f8694 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sun, 20 Sep 2015 18:39:45 -0700 Subject: generator: Fix strange unicode/str issue in python2 * Work around this by checking the appropriate class for the given * environment. --- generator/nanopb_generator.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'generator') diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 91a820f..df97338 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -85,7 +85,14 @@ class Names: return '_'.join(self.parts) def __add__(self, other): - if isinstance(other, str): + # The fdesc names are unicode and need to be handled for + # python2 and python3 + try: + realstr = unicode + except NameError: + realstr = str + + if isinstance(other, realstr): return Names(self.parts + (other,)) elif isinstance(other, tuple): return Names(self.parts + other) -- cgit 1.2.3-korg From 9c9f7f14e71c3b49877cca8eccd448cabea6306d Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Mon, 21 Sep 2015 08:16:17 -0700 Subject: generator: Use search $PATH for python * Search $PATH for the python binary so that this works better with things like virtualenv as used on Travis CI --- generator/nanopb_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'generator') diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index df97338..82b7927 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python from __future__ import unicode_literals -- cgit 1.2.3-korg