From 40289ef48f4ec103aa491e488d15d670d1a38a72 Mon Sep 17 00:00:00 2001 From: Karl-Aksel Puulmann Date: Tue, 28 May 2013 23:13:08 +0300 Subject: [PATCH] Most tests pass, several unable to test (missing libraries). Waiting for lihaoyi to add the With interface before finishing pattern matching. --- macropy/core/__init__.py | 30 +++++++++++++++++------------- macropy/core/lift.py | 2 +- macropy/core/macros.py | 21 ++++++++++++--------- macropy/core/walkers.py | 2 +- macropy/macros/adt.py | 8 ++++++-- macropy/macros/pattern.py | 4 +++- macropy/macros/quicklambda.py | 3 ++- macropy/macros/quicklambda_test.py | 6 +++--- macropy/macros/string_interp.py | 2 +- macropy/macros2/linq_test.py | 2 +- macropy/macros2/peg.py | 2 +- macropy/macros2/peg_test.py | 5 ++--- macropy/macros2/tracing.py | 2 +- 13 files changed, 51 insertions(+), 38 deletions(-) diff --git a/macropy/core/__init__.py b/macropy/core/__init__.py index 204d016..5ae4e59 100644 --- a/macropy/core/__init__.py +++ b/macropy/core/__init__.py @@ -38,11 +38,12 @@ def ast_repr(x): if type(x) in (int, float): return ast.Num(n=x) elif type(x) is str: return ast.Str(s=x) elif type(x) is bytes: return ast.Bytes(s=x) - elif type(x) is list: return ast.List(elts=map(ast_repr, x)) - elif type(x) is dict: return ast.Dict(keys=map(ast_repr, x.keys()), values=map(ast_repr, x.values())) - elif type(x) is set: return ast.Set(elts=map(ast_repr, x)) + elif type(x) is list: return ast.List(elts=list(map(ast_repr, x))) + elif type(x) is dict: return ast.Dict(keys=list(map(ast_repr, x.keys())), values=list(map(ast_repr, x.values()))) + elif type(x) is set: return ast.Set(elts=list(map(ast_repr, x))) elif type(x) is Literal: return x.body - elif x is None: return ast.Name(id="None") + elif type(x) in (bool, type(None)): + return ast.NameConstant(value=x) elif isinstance(x, ast.AST): fields = [ast.keyword(a, ast_repr(b)) for a, b in ast.iter_fields(x)] return ast.Call( @@ -121,6 +122,10 @@ def jmap(s, f, *l): """Shorthand for the join+map operation""" return s.join(map(f, *l)) + def lmap(f, *l): + """Python 2 map, returns list""" + return list(map(f, *l)) + type_dict = { #Misc type(None): lambda: "", @@ -142,7 +147,7 @@ def jmap(s, f, *l): Delete: lambda: tabs + "del " + jmap(", ", rec, tree.targets), Assert: lambda: tabs + "assert " + rec(tree.test) + mix(", ", rec(tree.msg)), Global: lambda: tabs + "global " + jmap(", ", rec, tree.names), - NonLocal: lambda: tabs + "nonlocal " + jmap(", ", rec, tree.names), + Nonlocal: lambda: tabs + "nonlocal " + jmap(", ", rec, tree.names), Yield: lambda: "(yield " + rec(tree.value) + ")", YieldFrom: lambda: "(yield from " + rec(tree.value) + ")", Raise: lambda: tabs + "raise " + rec(tree.exc) + @@ -158,9 +163,9 @@ def jmap(s, f, *l): ClassDef: lambda: "\n" + "".join(tabs + "@" + rec(dec) for dec in tree.decorator_list) + tabs + "class " + tree.name + mix("(", ", ".join( - map(rec, tree.bases + tree.keywords) + - map(lambda e: "*" + rec(e), tree.starargs) + - map(lambda e: "**" + rec(e), tree.kwargs) + lmap(rec, tree.bases + tree.keywords) + + lmap(lambda e: "*" + rec(e), tree.starargs) + + lmap(lambda e: "**" + rec(e), tree.kwargs) ), ")") + ":" + irec(tree.body), FunctionDef: lambda: "\n" + "".join(tabs + "@" + rec(dec) for dec in tree.decorator_list) + tabs + "def " + tree.name + "(" + rec(tree.args) + ")" + @@ -181,7 +186,6 @@ def jmap(s, f, *l): Name: lambda: tree.id, NameConstant: lambda: str(tree.value), Starred: lambda: "*" + rec(tree.value), - Repr: lambda: "`" + rec(tree.value) + "`", Num: lambda: (lambda repr_n: "(" + repr_n.replace("inf", INFSTR) + ")" if repr_n.startswith("-") @@ -207,8 +211,8 @@ def jmap(s, f, *l): Attribute: lambda: rec(tree.value) + (" " if isinstance(tree.value, Num) and isinstance(tree.value.n, int) else "") + "." + tree.attr, Call: lambda: rec(tree.func) + "(" + ", ".join( - map(rec, tree.args) + - map(rec, tree.keywords) + + lmap(rec, tree.args) + + lmap(rec, tree.keywords) + box(mix("*", rec(tree.starargs))) + box(mix("**", rec(tree.kwargs))) ) + ")", @@ -218,12 +222,12 @@ def jmap(s, f, *l): Slice: lambda: rec(tree.lower) + ":" + rec(tree.upper) + mix(":", rec(tree.step)), ExtSlice: lambda: jmap(", ", rec, tree.dims), arguments: lambda: ", ".join( - map(lambda a, d: rec(a) + mix("=", rec(d)), + lmap(lambda a, d: rec(a) + mix("=", rec(d)), tree.arguments, [None] * (len(tree.args) - len(tree.defaults)) + tree.defaults ) + box(mix("*", tree.vararg)) + - map(lambda a, d: rec(a) + "=" + rec(d), + lmap(lambda a, d: rec(a) + "=" + rec(d), tree.kwonlyargs, tree.kw_defaults) + box(mix("**", tree.kwarg)) diff --git a/macropy/core/lift.py b/macropy/core/lift.py index d27c7f8..98b81b6 100644 --- a/macropy/core/lift.py +++ b/macropy/core/lift.py @@ -31,4 +31,4 @@ def q(tree): @macros.block() def q(tree): body = _unquote_search.recurse(tree.body) - return Assign([Name(id=tree.optional_vars.id)], ast_repr(body)) + return Assign([Name(id=tree.items[0].optional_vars.id)], ast_repr(body)) diff --git a/macropy/core/macros.py b/macropy/core/macros.py index 13016c1..8c5217f 100644 --- a/macropy/core/macros.py +++ b/macropy/core/macros.py @@ -28,19 +28,19 @@ def __init__(self): def expr(self, inside_out=False): def register(f): - self.expr_registry[f.func_name] = (f, inside_out) + self.expr_registry[f.__name__] = (f, inside_out) return f return register def decorator(self, inside_out=False): def register(f): - self.decorator_registry[f.func_name] = (f, inside_out) + self.decorator_registry[f.__name__] = (f, inside_out) return f return register def block(self, inside_out=False): def register(f): - self.block_registry[f.func_name] = (f, inside_out) + self.block_registry[f.__name__] = (f, inside_out) return f return register @@ -124,7 +124,7 @@ def load_module(self, fullname): mod.__package__ = fullname else: mod.__package__ = fullname.rpartition('.')[0] - exec(compile(tree, self.file_name, "exec") in mod.__dict__) + exec(compile(tree, self.file_name, "exec"), mod.__dict__) return mod def process_ast(tree, modules): @@ -165,7 +165,7 @@ def expand_if_in_registry(tree, args, registry): """check if `tree` is a macro in `registry`, and if so use it to expand `args`""" if isinstance(tree, Name) and tree.id in registry: the_macro, inside_out = registry[tree.id] - new_tree = safe_splat(the_macro, *args, gen_sym = lambda: symbols.next()) + new_tree = safe_splat(the_macro, *args, gen_sym = lambda: next(symbols)) return new_tree elif isinstance(tree, Call): args.extend(tree.args) @@ -190,7 +190,7 @@ def run(tree): def macro_expand(tree): """Tail Recursively expands all macros in a single AST node""" if isinstance(tree, With): - new_tree = expand_if_in_registry(tree.context_expr, [tree], block_registry) + new_tree = expand_if_in_registry(tree.items[0].context_expr, [tree], block_registry) if new_tree: return macro_expand(new_tree) @@ -220,12 +220,13 @@ def macro_searcher(tree): return tree -@sys.meta_path.append -@singleton +#@sys.meta_path.append +#@singleton class _MacroFinder(object): """Loads a module and looks for macros inside, only providing a loader if it finds some.""" def find_module(self, module_name, package_path): + #print("FInding module", module_name, package_path) try: (file, pathname, description) = imp.find_module( module_name.split('.')[-1], @@ -241,6 +242,8 @@ def find_module(self, module_name, package_path): except Exception as e: pass +sys.meta_path.insert(0, _MacroFinder()) + def gen_syms(tree): """Create a generator that creates symbols which are not used in the given `tree`. This means they will be hygienic, i.e. it guarantees that they will @@ -253,5 +256,5 @@ def name_finder(tree): tree, found_names = name_finder.recurse_real(tree) names = ("sym" + str(i) for i in itertools.count()) - return itertools.ifilter(lambda x: x not in found_names, names) + return filter(lambda x: x not in found_names, names) diff --git a/macropy/core/walkers.py b/macropy/core/walkers.py index 64152a5..e049b1b 100644 --- a/macropy/core/walkers.py +++ b/macropy/core/walkers.py @@ -69,7 +69,7 @@ def walk_children(self, tree, ctx=None): return aggregates elif isinstance(tree, list) and len(tree) > 0: - x = zip(*map(lambda x: self.recurse_real(x, ctx), tree)) + x = zip(*list(map(lambda x: self.recurse_real(x, ctx), tree))) [trees, aggregates] = x tree[:] = flatten(trees) return [x for y in aggregates for x in y] diff --git a/macropy/macros/adt.py b/macropy/macros/adt.py index 78f307b..6304ee9 100644 --- a/macropy/macros/adt.py +++ b/macropy/macros/adt.py @@ -5,6 +5,10 @@ NO_ARG = object() +def concat(*iters): + for iter in iters: + yield from iter + def link_children(cls): for child in cls._children: new_child = type(child.__name__, (cls, CaseClass), dict(**child.__dict__)) @@ -13,11 +17,11 @@ def link_children(cls): class CaseClass(object): def __init__(self, *args, **kwargs): - for k, v in zip(self.__class__._fields, args) + kwargs.items(): + for k, v in concat(zip(self.__class__._fields, args), kwargs.items()): setattr(self, k, v) def copy(self, **kwargs): - return self.__class__(**dict(self.__dict__.items() + kwargs.items())) + return self.__class__(**dict(concat(self.__dict__.items(), kwargs.items()))) def __str__(self): return self.__class__.__name__ + "(" + ", ".join(str(getattr(self, x)) for x in self.__class__._fields) + ")" diff --git a/macropy/macros/pattern.py b/macropy/macros/pattern.py index 12525ea..15e7625 100644 --- a/macropy/macros/pattern.py +++ b/macropy/macros/pattern.py @@ -227,6 +227,8 @@ def build_matcher(tree, modified): return q%(LiteralMatcher(u%(tree.n))) if isinstance(tree, Str): return q%(LiteralMatcher(u%(tree.s))) + if isinstance(tree, NameConstant): + return q%(LiteralMatcher(ast%(tree.value))) if isinstance(tree, Name): if tree.id in ['True', 'False', 'None']: return q%(LiteralMatcher(ast%(tree))) @@ -337,7 +339,7 @@ def switch(tree, arg, gen_sym): import string import random new_id = gen_sym() - for i in xrange(len(tree.body)): + for i in range(len(tree.body)): tree.body[i] = _maybe_rewrite_if(tree.body[i], new_id) tree.body = [Assign([Name(new_id, Store())], arg)] + tree.body return tree.body diff --git a/macropy/macros/quicklambda.py b/macropy/macros/quicklambda.py index d191bd0..64784bb 100644 --- a/macropy/macros/quicklambda.py +++ b/macropy/macros/quicklambda.py @@ -19,5 +19,6 @@ def underscore_search(tree): tree, used_names = underscore_search.recurse_real(tree) new_tree = q%(lambda: ast%tree) - new_tree.args.args = [Name(id = x) for x in used_names] + print(dump(new_tree), used_names) + new_tree.args.args = [arg(arg = x) for x in used_names] return new_tree diff --git a/macropy/macros/quicklambda_test.py b/macropy/macros/quicklambda_test.py index 23f2fbb..6802eae 100644 --- a/macropy/macros/quicklambda_test.py +++ b/macropy/macros/quicklambda_test.py @@ -4,15 +4,15 @@ class Tests(unittest.TestCase): def test_basic(self): - assert map(f%(_ - 1), [1, 2, 3]) == [0, 1, 2] - assert reduce(f%(_ + _), [1, 2, 3]) == 6 + assert list(map(f%(_ - 1), [1, 2, 3])) == [0, 1, 2] + #assert reduce(f%(_ + _), [1, 2, 3]) == 6 def test_partial(self): basetwo = f%int(_, base=2) assert basetwo('10010') == 18 def test_attribute(self): - assert map(f%_.split(' ')[0], ["i am cow", "hear me moo"]) == ["i", "hear"] + assert list(map(f%_.split(' ')[0], ["i am cow", "hear me moo"])) == ["i", "hear"] def test_no_args(self): from random import random diff --git a/macropy/macros/string_interp.py b/macropy/macros/string_interp.py index 22f3add..ebbec2f 100644 --- a/macropy/macros/string_interp.py +++ b/macropy/macros/string_interp.py @@ -17,6 +17,6 @@ def s(tree): new_string += "%s" captured += [chunks[i]] - result = q%((u%new_string) % tuple(ast_list%map(parse_expr, captured))) + result = q%((u%new_string) % tuple(ast_list%list(map(parse_expr, captured)))) return result diff --git a/macropy/macros2/linq_test.py b/macropy/macros2/linq_test.py index 50034ed..86c0750 100644 --- a/macropy/macros2/linq_test.py +++ b/macropy/macros2/linq_test.py @@ -18,7 +18,7 @@ def compare_queries(query1, query2, post_process=lambda x: x): res2 = engine.execute(query2).fetchall() try: assert post_process(res1) == post_process(res2) - except Exception, e: + except Exception as e: #print "FAILURE" #print e #print query1 diff --git a/macropy/macros2/peg.py b/macropy/macros2/peg.py index 5dbdcab..c4be86b 100644 --- a/macropy/macros2/peg.py +++ b/macropy/macros2/peg.py @@ -32,7 +32,7 @@ def _PegWalker(tree, ctx): if type(tree) is BinOp and type(tree.op) is RShift: tree.left, b_left = _PegWalker.recurse_real(tree.left) tree.right = q%(lambda bindings: ast%tree.right) - tree.right.args.args = map(f%Name(id = _), flatten(b_left)) + tree.right.args.args = list(map(f%arg(arg = _), flatten(b_left))) return tree, stop if type(tree) is BinOp and type(tree.op) is FloorDiv: diff --git a/macropy/macros2/peg_test.py b/macropy/macros2/peg_test.py index b70a56a..8209d9f 100644 --- a/macropy/macros2/peg_test.py +++ b/macropy/macros2/peg_test.py @@ -115,7 +115,7 @@ def reduce_chain(chain): "+": f%(_+_), "-": f%(_-_), "*": f%(_*_), - "/": f%(_/_), + "/": f%(_//_), } while len(chain) > 1: a, [o, b] = chain.pop(), chain.pop() @@ -136,14 +136,13 @@ def reduce_chain(chain): assert expr.parse_all("(((((((11)))))+22+33)*(4+5+((6))))/12*(17+5)") == [1804] - def test_bindings_json(self): def test(parser, string): import json try: assert parser.parse_all(string)[0] == json.loads(string) - except Exception, e: + except Exception as e: print(parser.parse_all(string)) print(json.loads(string)) raise e diff --git a/macropy/macros2/tracing.py b/macropy/macros2/tracing.py index 6d63516..a5a4774 100644 --- a/macropy/macros2/tracing.py +++ b/macropy/macros2/tracing.py @@ -77,4 +77,4 @@ def require(tree): return tree.body -def log(x): print x +def log(x): print(x)