First rewriter test case

8 files changed, 339 insertions, 90 deletions

diff --git a/mesonbuild/ast/__init__.py b/mesonbuild/ast/__init__.py
index 0d1a4d69d..a9370dc6e 100644
--- a/mesonbuild/ast/__init__.py
+++ b/mesonbuild/ast/__init__.py
@@ -22,10 +22,11 @@ __all__ = [
     'AstVisitor',
     'AstPrinter',
     'IntrospectionInterpreter',
+    'build_target_functions',
 ]
 
 from .interpreter import AstInterpreter
-from .introspection import IntrospectionInterpreter
+from .introspection import IntrospectionInterpreter, build_target_functions
 from .visitor import AstVisitor
 from .postprocess import AstIDGenerator, AstIndentationGenerator
 from .printer import AstPrinter
diff --git a/mesonbuild/ast/interpreter.py b/mesonbuild/ast/interpreter.py
index 8893e9b74..81f6d5838 100644
--- a/mesonbuild/ast/interpreter.py
+++ b/mesonbuild/ast/interpreter.py
@@ -191,7 +191,7 @@ class AstInterpreter(interpreterbase.InterpreterBase):
             else:
                 temp_args += [i]
         for i in temp_args:
-            if isinstance(i, mparser.ElementaryNode):
+            if isinstance(i, mparser.ElementaryNode) and not isinstance(i, mparser.IdNode):
                 flattend_args += [i.value]
             elif isinstance(i, (str, bool, int, float)) or include_unknown_args:
                 flattend_args += [i]
diff --git a/mesonbuild/ast/introspection.py b/mesonbuild/ast/introspection.py
index fde9cc176..11496db19 100644
--- a/mesonbuild/ast/introspection.py
+++ b/mesonbuild/ast/introspection.py
@@ -20,9 +20,11 @@ from .. import compilers, environment, mesonlib, mparser, optinterpreter
 from .. import coredata as cdata
 from ..interpreterbase import InvalidArguments
 from ..build import Executable, CustomTarget, Jar, RunTarget, SharedLibrary, SharedModule, StaticLibrary
-
+from pprint import pprint
 import sys, os
 
+build_target_functions = ['executable', 'jar', 'library', 'shared_library', 'shared_module', 'static_library', 'both_libraries']
+
 class IntrospectionHelper:
     # mimic an argparse namespace
     def __init__(self, cross_file):
@@ -127,12 +129,36 @@ class IntrospectionInterpreter(AstInterpreter):
     def build_target(self, node, args, kwargs, targetclass):
         if not args:
             return
-        args = self.flatten_args(args, True)
         kwargs = self.flatten_kwargs(kwargs, True)
-        name = args[0]
-        sources = args[1:]
+        name = self.flatten_args(args)[0]
+        srcqueue = [node]
         if 'sources' in kwargs:
-            sources += self.flatten_args(kwargs['sources'])
+            srcqueue += kwargs['sources']
+
+        source_nodes = []
+        while srcqueue:
+            curr = srcqueue.pop(0)
+            arg_node = None
+            if isinstance(curr, mparser.FunctionNode):
+                arg_node = curr.args
+            elif isinstance(curr, mparser.ArrayNode):
+                arg_node = curr.args
+            elif isinstance(curr, mparser.IdNode):
+                # Try to resolve the ID and append the node to the queue
+                id = curr.value
+                if id in self.assignments and self.assignments[id]:
+                    node = self.assignments[id][0]
+                    if isinstance(node, (mparser.ArrayNode, mparser.IdNode, mparser.FunctionNode)):
+                        srcqueue += [node]
+            if arg_node is None:
+                continue
+            elemetary_nodes = list(filter(lambda x: isinstance(x, (str, mparser.StringNode)), arg_node.arguments))
+            srcqueue += list(filter(lambda x: isinstance(x, (mparser.FunctionNode, mparser.ArrayNode, mparser.IdNode)), arg_node.arguments))
+            # Pop the first element if the function is a build target function
+            if isinstance(curr, mparser.FunctionNode) and curr.func_name in build_target_functions:
+                elemetary_nodes.pop(0)
+            if elemetary_nodes:
+                source_nodes += [curr]
 
         # Filter out kwargs from other target types. For example 'soversion'
         # passed to library() when default_library == 'static'.
@@ -140,7 +166,8 @@ class IntrospectionInterpreter(AstInterpreter):
 
         is_cross = False
         objects = []
-        target = targetclass(name, self.subdir, self.subproject, is_cross, sources, objects, self.environment, kwargs)
+        empty_sources = [] # Passing the unresolved sources list causes errors
+        target = targetclass(name, self.subdir, self.subproject, is_cross, empty_sources, objects, self.environment, kwargs)
 
         self.targets += [{
             'name': target.get_basename(),
@@ -149,7 +176,7 @@ class IntrospectionInterpreter(AstInterpreter):
             'defined_in': os.path.normpath(os.path.join(self.source_root, self.subdir, environment.build_filename)),
             'subdir': self.subdir,
             'build_by_default': target.build_by_default,
-            'sources': sources,
+            'sources': source_nodes,
             'kwargs': kwargs,
             'node': node,
         }]
diff --git a/mesonbuild/rewriter.py b/mesonbuild/rewriter.py
index cc5d2abef..37099cec5 100644
--- a/mesonbuild/rewriter.py
+++ b/mesonbuild/rewriter.py
@@ -23,36 +23,146 @@
 # - move targets
 # - reindent?
 
-from .ast import AstInterpreter, AstVisitor, AstIDGenerator, AstIndentationGenerator, AstPrinter
+from .ast import IntrospectionInterpreter, build_target_functions, AstVisitor, AstIDGenerator, AstIndentationGenerator, AstPrinter
 from mesonbuild.mesonlib import MesonException
-from mesonbuild import mlog
+from . import mlog, mparser, environment
 import traceback
+from functools import wraps
+from pprint import pprint
+import json, os
+
+class RewriterException(MesonException):
+    pass
 
 def add_arguments(parser):
     parser.add_argument('--sourcedir', default='.',
                         help='Path to source directory.')
     parser.add_argument('-p', '--print', action='store_true', default=False, dest='print',
                         help='Print the parsed AST.')
+    parser.add_argument('command', type=str)
+
+class RequiredKeys:
+    def __init__(self, keys):
+        self.keys = keys
+
+    def __call__(self, f):
+        @wraps(f)
+        def wrapped(*wrapped_args, **wrapped_kwargs):
+            assert(len(wrapped_args) >= 2)
+            cmd = wrapped_args[1]
+            for key, val in self.keys.items():
+                typ = val[0] # The type of the value
+                default = val[1] # The default value -- None is required
+                choices = val[2] # Valid choices -- None is for everything
+                if key not in cmd:
+                    if default is not None:
+                        cmd[key] = default
+                    else:
+                        raise RewriterException('Key "{}" is missing in object for {}'
+                                                .format(key, f.__name__))
+                if not isinstance(cmd[key], typ):
+                    raise RewriterException('Invalid type of "{}". Required is {} but provided was {}'
+                                            .format(key, typ.__name__, type(cmd[key]).__name__))
+                if choices is not None:
+                    assert(isinstance(choices, list))
+                    if cmd[key] not in choices:
+                        raise RewriterException('Invalid value of "{}": Possible values are {} but provided was "{}"'
+                                                .format(key, choices, cmd[key]))
+            return f(*wrapped_args, **wrapped_kwargs)
+
+        return wrapped
+
+rewriter_keys = {
+    'target': {
+        'target': (str, None, None),
+        'operation': (str, None, ['src_add', 'src_rm', 'test']),
+        'sources': (list, [], None),
+        'debug': (bool, False, None)
+    }
+}
+
+class Rewriter:
+    def __init__(self, sourcedir: str, generator: str = 'ninja'):
+        self.sourcedir = sourcedir
+        self.interpreter = IntrospectionInterpreter(sourcedir, '', generator)
+        self.id_generator = AstIDGenerator()
+        self.functions = {
+            'target': self.process_target,
+        }
+
+    def analyze_meson(self):
+        mlog.log('Analyzing meson file:', mlog.bold(os.path.join(self.sourcedir, environment.build_filename)))
+        self.interpreter.analyze()
+        mlog.log('  -- Project:', mlog.bold(self.interpreter.project_data['descriptive_name']))
+        mlog.log('  -- Version:', mlog.cyan(self.interpreter.project_data['version']))
+        self.interpreter.ast.accept(AstIndentationGenerator())
+        self.interpreter.ast.accept(self.id_generator)
+
+    def find_target(self, target: str):
+        for i in self.interpreter.targets:
+            if target == i['name'] or target == i['id']:
+                return i
+        return None
+
+    @RequiredKeys(rewriter_keys['target'])
+    def process_target(self, cmd):
+        mlog.log('Processing target', mlog.bold(cmd['target']), 'operation', mlog.cyan(cmd['operation']))
+        target = self.find_target(cmd['target'])
+        if target is None:
+            mlog.error('Unknown target "{}" --> skipping'.format(cmd['target']))
+            if cmd['debug']:
+                pprint(self.interpreter.targets)
+            return
+        if cmd['debug']:
+            pprint(target)
+
+        if cmd['operation'] == 'src_add':
+            mlog.warning('TODO')
+        elif cmd['operation'] == 'src_rm':
+            mlog.warning('TODO')
+        elif cmd['operation'] == 'test':
+            src_list = []
+            for i in target['sources']:
+                args = []
+                if isinstance(i, mparser.FunctionNode):
+                    args = list(i.args.arguments)
+                    if i.func_name in build_target_functions:
+                        args.pop(0)
+                elif isinstance(i, mparser.ArrayNode):
+                    args = i.args.arguments
+                elif isinstance(i, mparser.ArgumentNode):
+                    args = i.arguments
+                for j in args:
+                    if isinstance(j, mparser.StringNode):
+                        src_list += [j.value]
+            test_data = {
+                'name': target['name'],
+                'sources': src_list
+            }
+            mlog.log('  !! target {}={}'.format(target['id'], json.dumps(test_data)))
+
+    def process(self, cmd):
+        if 'type' not in cmd:
+            raise RewriterException('Command has no key "type"')
+        if cmd['type'] not in self.functions:
+            raise RewriterException('Unknown command "{}". Supported commands are: {}'
+                                    .format(cmd['type'], list(self.functions.keys())))
+        self.functions[cmd['type']](cmd)
 
 def run(options):
-    rewriter = AstInterpreter(options.sourcedir, '')
-    try:
-        rewriter.load_root_meson_file()
-        rewriter.sanity_check_ast()
-        rewriter.parse_project()
-        rewriter.run()
-
-        indentor = AstIndentationGenerator()
-        idgen = AstIDGenerator()
-        printer = AstPrinter()
-        rewriter.ast.accept(indentor)
-        rewriter.ast.accept(idgen)
-        rewriter.ast.accept(printer)
-        print(printer.result)
-    except Exception as e:
-        if isinstance(e, MesonException):
-            mlog.exception(e)
-        else:
-            traceback.print_exc()
-        return 1
+    rewriter = Rewriter(options.sourcedir)
+    rewriter.analyze_meson()
+    if os.path.exists(options.command):
+        with open(options.command, 'r') as fp:
+            commands = json.load(fp)
+    else:
+        commands = json.loads(options.command)
+
+    if not isinstance(commands, list):
+        raise TypeError('Command is not a list')
+
+    for i in commands:
+        if not isinstance(i, object):
+            raise TypeError('Command is not an object')
+        rewriter.process(i)
     return 0
diff --git a/run_unittests.py b/run_unittests.py
index 8c2ad12f2..f0ab40f4d 100755
--- a/run_unittests.py
+++ b/run_unittests.py
@@ -32,6 +32,7 @@ from unittest import mock
 from configparser import ConfigParser
 from glob import glob
 from pathlib import (PurePath, Path)
+from distutils.dir_util import copy_tree
 
 import mesonbuild.mlog
 import mesonbuild.compilers
@@ -995,6 +996,7 @@ class BasePlatformTests(unittest.TestCase):
         self.mconf_command = self.meson_command + ['configure']
         self.mintro_command = self.meson_command + ['introspect']
         self.wrap_command = self.meson_command + ['wrap']
+        self.rewrite_command = self.meson_command + ['rewrite']
         # Backend-specific build commands
         self.build_command, self.clean_command, self.test_command, self.install_command, \
             self.uninstall_command = get_backend_commands(self.backend)
@@ -1003,6 +1005,7 @@ class BasePlatformTests(unittest.TestCase):
         self.vala_test_dir = os.path.join(src_root, 'test cases/vala')
         self.framework_test_dir = os.path.join(src_root, 'test cases/frameworks')
         self.unit_test_dir = os.path.join(src_root, 'test cases/unit')
+        self.rewrite_test_dir = os.path.join(src_root, 'test cases/rewrite')
         # Misc stuff
         self.orig_env = os.environ.copy()
         if self.backend is Backend.ninja:
@@ -4914,67 +4917,53 @@ class PythonTests(BasePlatformTests):
 
 
 class RewriterTests(BasePlatformTests):
-
     def setUp(self):
         super().setUp()
-        src_root = os.path.dirname(__file__)
-        self.testroot = os.path.realpath(tempfile.mkdtemp())
-        self.rewrite_command = python_command + [os.path.join(src_root, 'mesonrewriter.py')]
-        self.tmpdir = os.path.realpath(tempfile.mkdtemp())
-        self.workdir = os.path.join(self.tmpdir, 'foo')
-        self.test_dir = os.path.join(src_root, 'test cases/rewrite')
-
-    def tearDown(self):
-        windows_proof_rmtree(self.tmpdir)
+        self.maxDiff = None
 
-    def read_contents(self, fname):
-        with open(os.path.join(self.workdir, fname)) as f:
-            return f.read()
+    def prime(self, dirname):
+        copy_tree(os.path.join(self.rewrite_test_dir, dirname), self.builddir)
 
-    def check_effectively_same(self, mainfile, truth):
-        mf = self.read_contents(mainfile)
-        t = self.read_contents(truth)
-        # Rewriting is not guaranteed to do a perfect job of
-        # maintaining whitespace.
-        self.assertEqual(mf.replace(' ', ''), t.replace(' ', ''))
+    def rewrite(self, directory, args):
+        if isinstance(args, str):
+            args = [args]
+        out = subprocess.check_output(self.rewrite_command + ['--sourcedir', directory] + args,
+                                      universal_newlines=True)
+        return out
 
-    def prime(self, dirname):
-        shutil.copytree(os.path.join(self.test_dir, dirname), self.workdir)
+    data_regex = re.compile(r'^\s*!!\s*(\w+)\s+([^=]+)=(.*)$')
+    def extract_test_data(self, out):
+        lines = out.split('\n')
+        result = {}
+        for i in lines:
+            match = RewriterTests.data_regex.match(i)
+            if match:
+                typ = match.group(1)
+                id = match.group(2)
+                data = json.loads(match.group(3))
+                if typ not in result:
+                    result[typ] = {}
+                result[typ][id] = data
+        return result
 
-    def test_basic(self):
+    def test_target_source_list(self):
         self.prime('1 basic')
-        subprocess.check_call(self.rewrite_command + ['remove',
-                                                      '--target=trivialprog',
-                                                      '--filename=notthere.c',
-                                                      '--sourcedir', self.workdir],
-                              universal_newlines=True)
-        self.check_effectively_same('meson.build', 'removed.txt')
-        subprocess.check_call(self.rewrite_command + ['add',
-                                                      '--target=trivialprog',
-                                                      '--filename=notthere.c',
-                                                      '--sourcedir', self.workdir],
-                              universal_newlines=True)
-        self.check_effectively_same('meson.build', 'added.txt')
-        subprocess.check_call(self.rewrite_command + ['remove',
-                                                      '--target=trivialprog',
-                                                      '--filename=notthere.c',
-                                                      '--sourcedir', self.workdir],
-                              universal_newlines=True)
-        self.check_effectively_same('meson.build', 'removed.txt')
-
-    def test_subdir(self):
-        self.prime('2 subdirs')
-        top = self.read_contents('meson.build')
-        s2 = self.read_contents('sub2/meson.build')
-        subprocess.check_call(self.rewrite_command + ['remove',
-                                                      '--target=something',
-                                                      '--filename=second.c',
-                                                      '--sourcedir', self.workdir],
-                              universal_newlines=True)
-        self.check_effectively_same('sub1/meson.build', 'sub1/after.txt')
-        self.assertEqual(top, self.read_contents('meson.build'))
-        self.assertEqual(s2, self.read_contents('sub2/meson.build'))
-
+        out = self.rewrite(self.builddir, os.path.join(self.builddir, 'info.json'))
+        out = self.extract_test_data(out)
+        expected = {
+            'target': {
+                'trivialprog1@exe': {'name': 'trivialprog1', 'sources': ['main.cpp', 'fileA.cpp']},
+                'trivialprog2@exe': {'name': 'trivialprog2', 'sources': ['fileB.cpp', 'fileC.cpp']},
+                'trivialprog3@exe': {'name': 'trivialprog3', 'sources': ['main.cpp', 'fileA.cpp']},
+                'trivialprog4@exe': {'name': 'trivialprog4', 'sources': ['main.cpp', 'fileA.cpp']},
+                'trivialprog5@exe': {'name': 'trivialprog5', 'sources': ['main.cpp', 'fileB.cpp', 'fileC.cpp']},
+                'trivialprog6@exe': {'name': 'trivialprog6', 'sources': ['main.cpp', 'fileA.cpp']},
+                'trivialprog7@exe': {'name': 'trivialprog7', 'sources': ['fileB.cpp', 'fileC.cpp', 'main.cpp', 'fileA.cpp']},
+                'trivialprog8@exe': {'name': 'trivialprog8', 'sources': ['main.cpp', 'fileA.cpp']},
+                'trivialprog9@exe': {'name': 'trivialprog9', 'sources': ['main.cpp', 'fileA.cpp']},
+            }
+        }
+        self.assertDictEqual(out, expected)
 
 class NativeFileTests(BasePlatformTests):
 
@@ -5267,7 +5256,7 @@ def should_run_cross_mingw_tests():
 def main():
     unset_envs()
     cases = ['InternalTests', 'DataTests', 'AllPlatformTests', 'FailureTests',
-             'PythonTests', 'NativeFileTests']
+             'PythonTests', 'NativeFileTests', 'RewriterTests']
     if not is_windows():
         cases += ['LinuxlikeTests']
         if should_run_cross_arm_tests():
diff --git a/test cases/rewrite/1 basic/addSrc.json b/test cases/rewrite/1 basic/addSrc.json
new file mode 100644
index 000000000..b609b0883
--- /dev/null
+++ b/test cases/rewrite/1 basic/addSrc.json
@@ -0,0 +1,65 @@
+[
+  {
+    "type": "target",
+    "target": "trivialprog1",
+    "operation": "src_add",
+    "sources": ["added1.cpp", "added2.cpp"],
+    "debug": true
+  },
+  {
+    "type": "target",
+    "target": "trivialprog2",
+    "operation": "src_add",
+    "sources": ["added1.cpp"],
+    "debug": true
+  },
+  {
+    "type": "target",
+    "target": "trivialprog3",
+    "operation": "src_add",
+    "sources": ["added1.cpp", "added2.cpp", "added3.cpp"],
+    "debug": true
+  },
+  {
+    "type": "target",
+    "target": "trivialprog4",
+    "operation": "src_add",
+    "sources": ["added1.cpp"],
+    "debug": true
+  },
+  {
+    "type": "target",
+    "target": "trivialprog5",
+    "operation": "src_add",
+    "sources": ["added1.cpp"],
+    "debug": true
+  },
+  {
+    "type": "target",
+    "target": "trivialprog6",
+    "operation": "src_add",
+    "sources": ["added1.cpp"],
+    "debug": true
+  },
+  {
+    "type": "target",
+    "target": "trivialprog7",
+    "operation": "src_add",
+    "sources": ["added1.cpp"],
+    "debug": true
+  },
+  {
+    "type": "target",
+    "target": "trivialprog8",
+    "operation": "src_add",
+    "sources": ["added1.cpp"],
+    "debug": true
+  },
+  {
+    "type": "target",
+    "target": "trivialprog9",
+    "operation": "src_add",
+    "sources": ["added1.cpp"],
+    "debug": true
+  }
+]
diff --git a/test cases/rewrite/1 basic/info.json b/test cases/rewrite/1 basic/info.json
new file mode 100644
index 000000000..be2a87384
--- /dev/null
+++ b/test cases/rewrite/1 basic/info.json
@@ -0,0 +1,47 @@
+[
+  {
+    "type": "target",
+    "target": "trivialprog1",
+    "operation": "test"
+  },
+  {
+    "type": "target",
+    "target": "trivialprog2",
+    "operation": "test"
+  },
+  {
+    "type": "target",
+    "target": "trivialprog3",
+    "operation": "test"
+  },
+  {
+    "type": "target",
+    "target": "trivialprog4",
+    "operation": "test"
+  },
+  {
+    "type": "target",
+    "target": "trivialprog5",
+    "operation": "test"
+  },
+  {
+    "type": "target",
+    "target": "trivialprog6",
+    "operation": "test"
+  },
+  {
+    "type": "target",
+    "target": "trivialprog7",
+    "operation": "test"
+  },
+  {
+    "type": "target",
+    "target": "trivialprog8",
+    "operation": "test"
+  },
+  {
+    "type": "target",
+    "target": "trivialprog9",
+    "operation": "test"
+  }
+]
diff --git a/test cases/rewrite/1 basic/meson.build b/test cases/rewrite/1 basic/meson.build
index b91767892..3e092ee60 100644
--- a/test cases/rewrite/1 basic/meson.build
+++ b/test cases/rewrite/1 basic/meson.build
@@ -1,6 +1,16 @@
-project('rewritetest', 'c')
+project('rewritetest', 'cpp')
 
-sources = ['trivial.c', 'notthere.c']
+src1 = ['main.cpp', 'fileA.cpp']
+src2 = files(['fileB.cpp', 'fileC.cpp'])
+src3 = src1
+src4 = [src3]
 
-exe1 = executable('trivialprog1', sources)
-exe2 = executable('trivialprog2', ['main.cpp', 'fileA.cpp'])
+exe1 = executable('trivialprog1', src1)
+exe2 = executable('trivialprog2', [src2])
+exe3 = executable('trivialprog3', ['main.cpp', 'fileA.cpp'])
+exe4 = executable('trivialprog4', ['main.cpp', ['fileA.cpp']])
+exe5 = executable('trivialprog5', [src2, 'main.cpp'])
+exe6 = executable('trivialprog6', 'main.cpp', 'fileA.cpp')
+exe7 = executable('trivialprog7', 'fileB.cpp', src1, 'fileC.cpp')
+exe8 = executable('trivialprog8', src3)
+exe9 = executable('trivialprog9', src4)