diff options
author | Angelos Mouzakitis <a.mouzakitis@virtualopensystems.com> | 2023-10-10 14:33:42 +0000 |
---|---|---|
committer | Angelos Mouzakitis <a.mouzakitis@virtualopensystems.com> | 2023-10-10 14:33:42 +0000 |
commit | af1a266670d040d2f4083ff309d732d648afba2a (patch) | |
tree | 2fc46203448ddcc6f81546d379abfaeb323575e9 /meson/mesonbuild/cmake | |
parent | e02cda008591317b1625707ff8e115a4841aa889 (diff) |
Change-Id: Iaf8d18082d3991dec7c0ebbea540f092188eb4ec
Diffstat (limited to 'meson/mesonbuild/cmake')
-rw-r--r-- | meson/mesonbuild/cmake/__init__.py | 46 | ||||
-rw-r--r-- | meson/mesonbuild/cmake/client.py | 373 | ||||
-rw-r--r-- | meson/mesonbuild/cmake/common.py | 334 | ||||
-rw-r--r-- | meson/mesonbuild/cmake/data/preload.cmake | 82 | ||||
-rw-r--r-- | meson/mesonbuild/cmake/executor.py | 246 | ||||
-rw-r--r-- | meson/mesonbuild/cmake/fileapi.py | 320 | ||||
-rw-r--r-- | meson/mesonbuild/cmake/generator.py | 134 | ||||
-rw-r--r-- | meson/mesonbuild/cmake/interpreter.py | 1369 | ||||
-rw-r--r-- | meson/mesonbuild/cmake/toolchain.py | 259 | ||||
-rw-r--r-- | meson/mesonbuild/cmake/traceparser.py | 756 |
10 files changed, 3919 insertions, 0 deletions
diff --git a/meson/mesonbuild/cmake/__init__.py b/meson/mesonbuild/cmake/__init__.py new file mode 100644 index 000000000..d39bf2424 --- /dev/null +++ b/meson/mesonbuild/cmake/__init__.py @@ -0,0 +1,46 @@ +# Copyright 2019 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This class contains the basic functionality needed to run any interpreter +# or an interpreter-based tool. + +__all__ = [ + 'CMakeClient', + 'CMakeExecutor', + 'CMakeExecScope', + 'CMakeException', + 'CMakeFileAPI', + 'CMakeInterpreter', + 'CMakeTarget', + 'CMakeToolchain', + 'CMakeTraceLine', + 'CMakeTraceParser', + 'SingleTargetOptions', + 'TargetOptions', + 'parse_generator_expressions', + 'language_map', + 'backend_generator_map', + 'cmake_get_generator_args', + 'cmake_defines_to_args', + 'check_cmake_args', +] + +from .common import CMakeException, SingleTargetOptions, TargetOptions, cmake_defines_to_args, language_map, backend_generator_map, cmake_get_generator_args, check_cmake_args +from .client import CMakeClient +from .executor import CMakeExecutor +from .fileapi import CMakeFileAPI +from .generator import parse_generator_expressions +from .interpreter import CMakeInterpreter +from .toolchain import CMakeToolchain, CMakeExecScope +from .traceparser import CMakeTarget, CMakeTraceLine, CMakeTraceParser diff --git a/meson/mesonbuild/cmake/client.py b/meson/mesonbuild/cmake/client.py new file mode 100644 index 000000000..bcbb52ef9 --- /dev/null +++ b/meson/mesonbuild/cmake/client.py @@ -0,0 +1,373 @@ +# Copyright 2019 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This class contains the basic functionality needed to run any interpreter +# or an interpreter-based tool. + +from .common import CMakeException, CMakeConfiguration, CMakeBuildFile +from .. import mlog +from contextlib import contextmanager +from subprocess import Popen, PIPE, TimeoutExpired +from pathlib import Path +import typing as T +import json + +if T.TYPE_CHECKING: + from ..environment import Environment + from .executor import CMakeExecutor + +CMAKE_SERVER_BEGIN_STR = '[== "CMake Server" ==[' +CMAKE_SERVER_END_STR = ']== "CMake Server" ==]' + +CMAKE_MESSAGE_TYPES = { + 'error': ['cookie', 'errorMessage'], + 'hello': ['supportedProtocolVersions'], + 'message': ['cookie', 'message'], + 'progress': ['cookie'], + 'reply': ['cookie', 'inReplyTo'], + 'signal': ['cookie', 'name'], +} # type: T.Dict[str, T.List[str]] + +CMAKE_REPLY_TYPES = { + 'handshake': [], + 'configure': [], + 'compute': [], + 'cmakeInputs': ['buildFiles', 'cmakeRootDirectory', 'sourceDirectory'], + 'codemodel': ['configurations'] +} # type: T.Dict[str, T.List[str]] + +# Base CMake server message classes + +class MessageBase: + def __init__(self, msg_type: str, cookie: str) -> None: + self.type = msg_type + self.cookie = cookie + + def to_dict(self) -> T.Dict[str, T.Union[str, T.List[str], T.Dict[str, int]]]: + return {'type': self.type, 'cookie': self.cookie} + + def log(self) -> None: + mlog.warning('CMake server message of type', mlog.bold(type(self).__name__), 'has no log function') + +class RequestBase(MessageBase): + cookie_counter = 0 + + def __init__(self, msg_type: str) -> None: + super().__init__(msg_type, self.gen_cookie()) + + @staticmethod + def gen_cookie() -> str: + RequestBase.cookie_counter += 1 + return f'meson_{RequestBase.cookie_counter}' + +class ReplyBase(MessageBase): + def __init__(self, cookie: str, in_reply_to: str) -> None: + super().__init__('reply', cookie) + self.in_reply_to = in_reply_to + +class SignalBase(MessageBase): + def __init__(self, cookie: str, signal_name: str) -> None: + super().__init__('signal', cookie) + self.signal_name = signal_name + + def log(self) -> None: + mlog.log(mlog.bold('CMake signal:'), mlog.yellow(self.signal_name)) + +# Special Message classes + +class Error(MessageBase): + def __init__(self, cookie: str, message: str) -> None: + super().__init__('error', cookie) + self.message = message + + def log(self) -> None: + mlog.error(mlog.bold('CMake server error:'), mlog.red(self.message)) + +class Message(MessageBase): + def __init__(self, cookie: str, message: str) -> None: + super().__init__('message', cookie) + self.message = message + + def log(self) -> None: + #mlog.log(mlog.bold('CMake:'), self.message) + pass + +class Progress(MessageBase): + def __init__(self, cookie: str) -> None: + super().__init__('progress', cookie) + + def log(self) -> None: + pass + +class MessageHello(MessageBase): + def __init__(self, supported_protocol_versions: T.List[T.Dict[str, int]]) -> None: + super().__init__('hello', '') + self.supported_protocol_versions = supported_protocol_versions + + def supports(self, major: int, minor: T.Optional[int] = None) -> bool: + for i in self.supported_protocol_versions: + assert 'major' in i + assert 'minor' in i + if major == i['major']: + if minor is None or minor == i['minor']: + return True + return False + +# Request classes + +class RequestHandShake(RequestBase): + def __init__(self, src_dir: Path, build_dir: Path, generator: str, vers_major: int, vers_minor: T.Optional[int] = None) -> None: + super().__init__('handshake') + self.src_dir = src_dir + self.build_dir = build_dir + self.generator = generator + self.vers_major = vers_major + self.vers_minor = vers_minor + + def to_dict(self) -> T.Dict[str, T.Union[str, T.List[str], T.Dict[str, int]]]: + vers = {'major': self.vers_major} + if self.vers_minor is not None: + vers['minor'] = self.vers_minor + + # Old CMake versions (3.7) want '/' even on Windows + self.src_dir = self.src_dir.resolve() + self.build_dir = self.build_dir.resolve() + + return { + **super().to_dict(), + 'sourceDirectory': self.src_dir.as_posix(), + 'buildDirectory': self.build_dir.as_posix(), + 'generator': self.generator, + 'protocolVersion': vers + } + +class RequestConfigure(RequestBase): + def __init__(self, args: T.Optional[T.List[str]] = None): + super().__init__('configure') + self.args = args + + def to_dict(self) -> T.Dict[str, T.Union[str, T.List[str], T.Dict[str, int]]]: + res = super().to_dict() + if self.args: + res['cacheArguments'] = self.args + return res + +class RequestCompute(RequestBase): + def __init__(self) -> None: + super().__init__('compute') + +class RequestCMakeInputs(RequestBase): + def __init__(self) -> None: + super().__init__('cmakeInputs') + +class RequestCodeModel(RequestBase): + def __init__(self) -> None: + super().__init__('codemodel') + +# Reply classes + +class ReplyHandShake(ReplyBase): + def __init__(self, cookie: str) -> None: + super().__init__(cookie, 'handshake') + +class ReplyConfigure(ReplyBase): + def __init__(self, cookie: str) -> None: + super().__init__(cookie, 'configure') + +class ReplyCompute(ReplyBase): + def __init__(self, cookie: str) -> None: + super().__init__(cookie, 'compute') + +class ReplyCMakeInputs(ReplyBase): + def __init__(self, cookie: str, cmake_root: Path, src_dir: Path, build_files: T.List[CMakeBuildFile]) -> None: + super().__init__(cookie, 'cmakeInputs') + self.cmake_root = cmake_root + self.src_dir = src_dir + self.build_files = build_files + + def log(self) -> None: + mlog.log('CMake root: ', mlog.bold(self.cmake_root.as_posix())) + mlog.log('Source dir: ', mlog.bold(self.src_dir.as_posix())) + mlog.log('Build files:', mlog.bold(str(len(self.build_files)))) + with mlog.nested(): + for i in self.build_files: + mlog.log(str(i)) + +class ReplyCodeModel(ReplyBase): + def __init__(self, data: T.Dict[str, T.Any]) -> None: + super().__init__(data['cookie'], 'codemodel') + self.configs = [] + for i in data['configurations']: + self.configs += [CMakeConfiguration(i)] + + def log(self) -> None: + mlog.log('CMake code mode:') + for idx, i in enumerate(self.configs): + mlog.log(f'Configuration {idx}:') + with mlog.nested(): + i.log() + +# Main client class + +class CMakeClient: + def __init__(self, env: 'Environment') -> None: + self.env = env + self.proc = None # type: T.Optional[Popen] + self.type_map = { + 'error': lambda data: Error(data['cookie'], data['errorMessage']), + 'hello': lambda data: MessageHello(data['supportedProtocolVersions']), + 'message': lambda data: Message(data['cookie'], data['message']), + 'progress': lambda data: Progress(data['cookie']), + 'reply': self.resolve_type_reply, + 'signal': lambda data: SignalBase(data['cookie'], data['name']) + } # type: T.Dict[str, T.Callable[[T.Dict[str, T.Any]], MessageBase]] + + self.reply_map = { + 'handshake': lambda data: ReplyHandShake(data['cookie']), + 'configure': lambda data: ReplyConfigure(data['cookie']), + 'compute': lambda data: ReplyCompute(data['cookie']), + 'cmakeInputs': self.resolve_reply_cmakeInputs, + 'codemodel': lambda data: ReplyCodeModel(data), + } # type: T.Dict[str, T.Callable[[T.Dict[str, T.Any]], ReplyBase]] + + def readMessageRaw(self) -> T.Dict[str, T.Any]: + assert self.proc is not None + rawData = [] + begin = False + while self.proc.poll() is None: + line = self.proc.stdout.readline() + if not line: + break + line = line.decode('utf-8') + line = line.strip() + + if begin and line == CMAKE_SERVER_END_STR: + break # End of the message + elif begin: + rawData += [line] + elif line == CMAKE_SERVER_BEGIN_STR: + begin = True # Begin of the message + + if rawData: + res = json.loads('\n'.join(rawData)) + assert isinstance(res, dict) + for i in res.keys(): + assert isinstance(i, str) + return res + raise CMakeException('Failed to read data from the CMake server') + + def readMessage(self) -> MessageBase: + raw_data = self.readMessageRaw() + if 'type' not in raw_data: + raise CMakeException('The "type" attribute is missing from the message') + msg_type = raw_data['type'] + func = self.type_map.get(msg_type, None) + if not func: + raise CMakeException(f'Recieved unknown message type "{msg_type}"') + for i in CMAKE_MESSAGE_TYPES[msg_type]: + if i not in raw_data: + raise CMakeException(f'Key "{i}" is missing from CMake server message type {msg_type}') + return func(raw_data) + + def writeMessage(self, msg: MessageBase) -> None: + raw_data = '\n{}\n{}\n{}\n'.format(CMAKE_SERVER_BEGIN_STR, json.dumps(msg.to_dict(), indent=2), CMAKE_SERVER_END_STR) + self.proc.stdin.write(raw_data.encode('ascii')) + self.proc.stdin.flush() + + def query(self, request: RequestBase) -> MessageBase: + self.writeMessage(request) + while True: + reply = self.readMessage() + if reply.cookie == request.cookie and reply.type in ['reply', 'error']: + return reply + + reply.log() + + def query_checked(self, request: RequestBase, message: str) -> MessageBase: + reply = self.query(request) + h = mlog.green('SUCCEEDED') if reply.type == 'reply' else mlog.red('FAILED') + mlog.log(message + ':', h) + if reply.type != 'reply': + reply.log() + raise CMakeException('CMake server query failed') + return reply + + def do_handshake(self, src_dir: Path, build_dir: Path, generator: str, vers_major: int, vers_minor: T.Optional[int] = None) -> None: + # CMake prints the hello message on startup + msg = self.readMessage() + if not isinstance(msg, MessageHello): + raise CMakeException('Recieved an unexpected message from the CMake server') + + request = RequestHandShake(src_dir, build_dir, generator, vers_major, vers_minor) + self.query_checked(request, 'CMake server handshake') + + def resolve_type_reply(self, data: T.Dict[str, T.Any]) -> ReplyBase: + reply_type = data['inReplyTo'] + func = self.reply_map.get(reply_type, None) + if not func: + raise CMakeException(f'Recieved unknown reply type "{reply_type}"') + for i in ['cookie'] + CMAKE_REPLY_TYPES[reply_type]: + if i not in data: + raise CMakeException(f'Key "{i}" is missing from CMake server message type {type}') + return func(data) + + def resolve_reply_cmakeInputs(self, data: T.Dict[str, T.Any]) -> ReplyCMakeInputs: + files = [] + for i in data['buildFiles']: + for j in i['sources']: + files += [CMakeBuildFile(Path(j), i['isCMake'], i['isTemporary'])] + return ReplyCMakeInputs(data['cookie'], Path(data['cmakeRootDirectory']), Path(data['sourceDirectory']), files) + + @contextmanager + def connect(self, cmake_exe: 'CMakeExecutor') -> T.Generator[None, None, None]: + self.startup(cmake_exe) + try: + yield + finally: + self.shutdown() + + def startup(self, cmake_exe: 'CMakeExecutor') -> None: + if self.proc is not None: + raise CMakeException('The CMake server was already started') + assert cmake_exe.found() + + mlog.debug('Starting CMake server with CMake', mlog.bold(' '.join(cmake_exe.get_command())), 'version', mlog.cyan(cmake_exe.version())) + self.proc = Popen(cmake_exe.get_command() + ['-E', 'server', '--experimental', '--debug'], stdin=PIPE, stdout=PIPE) + + def shutdown(self) -> None: + if self.proc is None: + return + + mlog.debug('Shutting down the CMake server') + + # Close the pipes to exit + self.proc.stdin.close() + self.proc.stdout.close() + + # Wait for CMake to finish + try: + self.proc.wait(timeout=2) + except TimeoutExpired: + # Terminate CMake if there is a timeout + # terminate() may throw a platform specific exception if the process has already + # terminated. This may be the case if there is a race condition (CMake exited after + # the timeout but before the terminate() call). Additionally, this behavior can + # also be triggered on cygwin if CMake crashes. + # See https://github.com/mesonbuild/meson/pull/4969#issuecomment-499413233 + try: + self.proc.terminate() + except Exception: + pass + + self.proc = None diff --git a/meson/mesonbuild/cmake/common.py b/meson/mesonbuild/cmake/common.py new file mode 100644 index 000000000..5cc154cb8 --- /dev/null +++ b/meson/mesonbuild/cmake/common.py @@ -0,0 +1,334 @@ +# Copyright 2019 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This class contains the basic functionality needed to run any interpreter +# or an interpreter-based tool. + +from ..mesonlib import MesonException, OptionKey +from .. import mlog +from pathlib import Path +import typing as T + +if T.TYPE_CHECKING: + from ..environment import Environment + +language_map = { + 'c': 'C', + 'cpp': 'CXX', + 'cuda': 'CUDA', + 'objc': 'OBJC', + 'objcpp': 'OBJCXX', + 'cs': 'CSharp', + 'java': 'Java', + 'fortran': 'Fortran', + 'swift': 'Swift', +} + +backend_generator_map = { + 'ninja': 'Ninja', + 'xcode': 'Xcode', + 'vs2010': 'Visual Studio 10 2010', + 'vs2012': 'Visual Studio 11 2012', + 'vs2013': 'Visual Studio 12 2013', + 'vs2015': 'Visual Studio 14 2015', + 'vs2017': 'Visual Studio 15 2017', + 'vs2019': 'Visual Studio 16 2019', +} + +blacklist_cmake_defs = [ + 'CMAKE_TOOLCHAIN_FILE', + 'CMAKE_PROJECT_INCLUDE', + 'MESON_PRELOAD_FILE', + 'MESON_PS_CMAKE_CURRENT_BINARY_DIR', + 'MESON_PS_CMAKE_CURRENT_SOURCE_DIR', + 'MESON_PS_DELAYED_CALLS', + 'MESON_PS_LOADED', + 'MESON_FIND_ROOT_PATH', + 'MESON_CMAKE_SYSROOT', + 'MESON_PATHS_LIST', + 'MESON_CMAKE_ROOT', +] + +class CMakeException(MesonException): + pass + +class CMakeBuildFile: + def __init__(self, file: Path, is_cmake: bool, is_temp: bool) -> None: + self.file = file + self.is_cmake = is_cmake + self.is_temp = is_temp + + def __repr__(self) -> str: + return f'<{self.__class__.__name__}: {self.file}; cmake={self.is_cmake}; temp={self.is_temp}>' + +def _flags_to_list(raw: str) -> T.List[str]: + # Convert a raw commandline string into a list of strings + res = [] + curr = '' + escape = False + in_string = False + for i in raw: + if escape: + # If the current char is not a quote, the '\' is probably important + if i not in ['"', "'"]: + curr += '\\' + curr += i + escape = False + elif i == '\\': + escape = True + elif i in ['"', "'"]: + in_string = not in_string + elif i in [' ', '\n']: + if in_string: + curr += i + else: + res += [curr] + curr = '' + else: + curr += i + res += [curr] + res = list(filter(lambda x: len(x) > 0, res)) + return res + +def cmake_get_generator_args(env: 'Environment') -> T.List[str]: + backend_name = env.coredata.get_option(OptionKey('backend')) + assert isinstance(backend_name, str) + assert backend_name in backend_generator_map + return ['-G', backend_generator_map[backend_name]] + +def cmake_defines_to_args(raw: T.Any, permissive: bool = False) -> T.List[str]: + res = [] # type: T.List[str] + if not isinstance(raw, list): + raw = [raw] + + for i in raw: + if not isinstance(i, dict): + raise MesonException('Invalid CMake defines. Expected a dict, but got a {}'.format(type(i).__name__)) + for key, val in i.items(): + assert isinstance(key, str) + if key in blacklist_cmake_defs: + mlog.warning('Setting', mlog.bold(key), 'is not supported. See the meson docs for cross compilation support:') + mlog.warning(' - URL: https://mesonbuild.com/CMake-module.html#cross-compilation') + mlog.warning(' --> Ignoring this option') + continue + if isinstance(val, (str, int, float)): + res += [f'-D{key}={val}'] + elif isinstance(val, bool): + val_str = 'ON' if val else 'OFF' + res += [f'-D{key}={val_str}'] + else: + raise MesonException('Type "{}" of "{}" is not supported as for a CMake define value'.format(type(val).__name__, key)) + + return res + +# TODO: this functuin will become obsolete once the `cmake_args` kwarg is dropped +def check_cmake_args(args: T.List[str]) -> T.List[str]: + res = [] # type: T.List[str] + dis = ['-D' + x for x in blacklist_cmake_defs] + assert dis # Ensure that dis is not empty. + for i in args: + if any([i.startswith(x) for x in dis]): + mlog.warning('Setting', mlog.bold(i), 'is not supported. See the meson docs for cross compilation support:') + mlog.warning(' - URL: https://mesonbuild.com/CMake-module.html#cross-compilation') + mlog.warning(' --> Ignoring this option') + continue + res += [i] + return res + +class CMakeInclude: + def __init__(self, path: Path, isSystem: bool = False): + self.path = path + self.isSystem = isSystem + + def __repr__(self) -> str: + return f'<CMakeInclude: {self.path} -- isSystem = {self.isSystem}>' + +class CMakeFileGroup: + def __init__(self, data: T.Dict[str, T.Any]) -> None: + self.defines = data.get('defines', '') # type: str + self.flags = _flags_to_list(data.get('compileFlags', '')) # type: T.List[str] + self.is_generated = data.get('isGenerated', False) # type: bool + self.language = data.get('language', 'C') # type: str + self.sources = [Path(x) for x in data.get('sources', [])] # type: T.List[Path] + + # Fix the include directories + self.includes = [] # type: T.List[CMakeInclude] + for i in data.get('includePath', []): + if isinstance(i, dict) and 'path' in i: + isSystem = i.get('isSystem', False) + assert isinstance(isSystem, bool) + assert isinstance(i['path'], str) + self.includes += [CMakeInclude(Path(i['path']), isSystem)] + elif isinstance(i, str): + self.includes += [CMakeInclude(Path(i))] + + def log(self) -> None: + mlog.log('flags =', mlog.bold(', '.join(self.flags))) + mlog.log('defines =', mlog.bold(', '.join(self.defines))) + mlog.log('includes =', mlog.bold(', '.join([str(x) for x in self.includes]))) + mlog.log('is_generated =', mlog.bold('true' if self.is_generated else 'false')) + mlog.log('language =', mlog.bold(self.language)) + mlog.log('sources:') + for i in self.sources: + with mlog.nested(): + mlog.log(i.as_posix()) + +class CMakeTarget: + def __init__(self, data: T.Dict[str, T.Any]) -> None: + self.artifacts = [Path(x) for x in data.get('artifacts', [])] # type: T.List[Path] + self.src_dir = Path(data.get('sourceDirectory', '')) # type: Path + self.build_dir = Path(data.get('buildDirectory', '')) # type: Path + self.name = data.get('name', '') # type: str + self.full_name = data.get('fullName', '') # type: str + self.install = data.get('hasInstallRule', False) # type: bool + self.install_paths = [Path(x) for x in set(data.get('installPaths', []))] # type: T.List[Path] + self.link_lang = data.get('linkerLanguage', '') # type: str + self.link_libraries = _flags_to_list(data.get('linkLibraries', '')) # type: T.List[str] + self.link_flags = _flags_to_list(data.get('linkFlags', '')) # type: T.List[str] + self.link_lang_flags = _flags_to_list(data.get('linkLanguageFlags', '')) # type: T.List[str] + # self.link_path = Path(data.get('linkPath', '')) # type: Path + self.type = data.get('type', 'EXECUTABLE') # type: str + # self.is_generator_provided = data.get('isGeneratorProvided', False) # type: bool + self.files = [] # type: T.List[CMakeFileGroup] + + for i in data.get('fileGroups', []): + self.files += [CMakeFileGroup(i)] + + def log(self) -> None: + mlog.log('artifacts =', mlog.bold(', '.join([x.as_posix() for x in self.artifacts]))) + mlog.log('src_dir =', mlog.bold(self.src_dir.as_posix())) + mlog.log('build_dir =', mlog.bold(self.build_dir.as_posix())) + mlog.log('name =', mlog.bold(self.name)) + mlog.log('full_name =', mlog.bold(self.full_name)) + mlog.log('install =', mlog.bold('true' if self.install else 'false')) + mlog.log('install_paths =', mlog.bold(', '.join([x.as_posix() for x in self.install_paths]))) + mlog.log('link_lang =', mlog.bold(self.link_lang)) + mlog.log('link_libraries =', mlog.bold(', '.join(self.link_libraries))) + mlog.log('link_flags =', mlog.bold(', '.join(self.link_flags))) + mlog.log('link_lang_flags =', mlog.bold(', '.join(self.link_lang_flags))) + # mlog.log('link_path =', mlog.bold(self.link_path)) + mlog.log('type =', mlog.bold(self.type)) + # mlog.log('is_generator_provided =', mlog.bold('true' if self.is_generator_provided else 'false')) + for idx, i in enumerate(self.files): + mlog.log(f'Files {idx}:') + with mlog.nested(): + i.log() + +class CMakeProject: + def __init__(self, data: T.Dict[str, T.Any]) -> None: + self.src_dir = Path(data.get('sourceDirectory', '')) # type: Path + self.build_dir = Path(data.get('buildDirectory', '')) # type: Path + self.name = data.get('name', '') # type: str + self.targets = [] # type: T.List[CMakeTarget] + + for i in data.get('targets', []): + self.targets += [CMakeTarget(i)] + + def log(self) -> None: + mlog.log('src_dir =', mlog.bold(self.src_dir.as_posix())) + mlog.log('build_dir =', mlog.bold(self.build_dir.as_posix())) + mlog.log('name =', mlog.bold(self.name)) + for idx, i in enumerate(self.targets): + mlog.log(f'Target {idx}:') + with mlog.nested(): + i.log() + +class CMakeConfiguration: + def __init__(self, data: T.Dict[str, T.Any]) -> None: + self.name = data.get('name', '') # type: str + self.projects = [] # type: T.List[CMakeProject] + for i in data.get('projects', []): + self.projects += [CMakeProject(i)] + + def log(self) -> None: + mlog.log('name =', mlog.bold(self.name)) + for idx, i in enumerate(self.projects): + mlog.log(f'Project {idx}:') + with mlog.nested(): + i.log() + +class SingleTargetOptions: + def __init__(self) -> None: + self.opts = {} # type: T.Dict[str, str] + self.lang_args = {} # type: T.Dict[str, T.List[str]] + self.link_args = [] # type: T.List[str] + self.install = 'preserve' + + def set_opt(self, opt: str, val: str) -> None: + self.opts[opt] = val + + def append_args(self, lang: str, args: T.List[str]) -> None: + if lang not in self.lang_args: + self.lang_args[lang] = [] + self.lang_args[lang] += args + + def append_link_args(self, args: T.List[str]) -> None: + self.link_args += args + + def set_install(self, install: bool) -> None: + self.install = 'true' if install else 'false' + + def get_override_options(self, initial: T.List[str]) -> T.List[str]: + res = [] # type: T.List[str] + for i in initial: + opt = i[:i.find('=')] + if opt not in self.opts: + res += [i] + res += [f'{k}={v}' for k, v in self.opts.items()] + return res + + def get_compile_args(self, lang: str, initial: T.List[str]) -> T.List[str]: + if lang in self.lang_args: + return initial + self.lang_args[lang] + return initial + + def get_link_args(self, initial: T.List[str]) -> T.List[str]: + return initial + self.link_args + + def get_install(self, initial: bool) -> bool: + return {'preserve': initial, 'true': True, 'false': False}[self.install] + +class TargetOptions: + def __init__(self) -> None: + self.global_options = SingleTargetOptions() + self.target_options = {} # type: T.Dict[str, SingleTargetOptions] + + def __getitem__(self, tgt: str) -> SingleTargetOptions: + if tgt not in self.target_options: + self.target_options[tgt] = SingleTargetOptions() + return self.target_options[tgt] + + def get_override_options(self, tgt: str, initial: T.List[str]) -> T.List[str]: + initial = self.global_options.get_override_options(initial) + if tgt in self.target_options: + initial = self.target_options[tgt].get_override_options(initial) + return initial + + def get_compile_args(self, tgt: str, lang: str, initial: T.List[str]) -> T.List[str]: + initial = self.global_options.get_compile_args(lang, initial) + if tgt in self.target_options: + initial = self.target_options[tgt].get_compile_args(lang, initial) + return initial + + def get_link_args(self, tgt: str, initial: T.List[str]) -> T.List[str]: + initial = self.global_options.get_link_args(initial) + if tgt in self.target_options: + initial = self.target_options[tgt].get_link_args(initial) + return initial + + def get_install(self, tgt: str, initial: bool) -> bool: + initial = self.global_options.get_install(initial) + if tgt in self.target_options: + initial = self.target_options[tgt].get_install(initial) + return initial diff --git a/meson/mesonbuild/cmake/data/preload.cmake b/meson/mesonbuild/cmake/data/preload.cmake new file mode 100644 index 000000000..234860b75 --- /dev/null +++ b/meson/mesonbuild/cmake/data/preload.cmake @@ -0,0 +1,82 @@ +if(MESON_PS_LOADED) + return() +endif() + +set(MESON_PS_LOADED ON) + +cmake_policy(PUSH) +cmake_policy(SET CMP0054 NEW) # https://cmake.org/cmake/help/latest/policy/CMP0054.html + +# Dummy macros that have a special meaning in the meson code +macro(meson_ps_execute_delayed_calls) +endmacro() + +macro(meson_ps_reload_vars) +endmacro() + +macro(meson_ps_disabled_function) + message(WARNING "The function '${ARGV0}' is disabled in the context of CMake subprojects.\n" + "This should not be an issue but may lead to compilation errors.") +endmacro() + +# Helper macro to inspect the current CMake state +macro(meson_ps_inspect_vars) + set(MESON_PS_CMAKE_CURRENT_BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}") + set(MESON_PS_CMAKE_CURRENT_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}") + meson_ps_execute_delayed_calls() +endmacro() + + +# Override some system functions with custom code and forward the args +# to the original function +macro(add_custom_command) + meson_ps_inspect_vars() + _add_custom_command(${ARGV}) +endmacro() + +macro(add_custom_target) + meson_ps_inspect_vars() + _add_custom_target(${ARGV}) +endmacro() + +macro(set_property) + meson_ps_inspect_vars() + _set_property(${ARGV}) +endmacro() + +function(set_source_files_properties) + set(FILES) + set(I 0) + set(PROPERTIES OFF) + + while(I LESS ARGC) + if(NOT PROPERTIES) + if("${ARGV${I}}" STREQUAL "PROPERTIES") + set(PROPERTIES ON) + else() + list(APPEND FILES "${ARGV${I}}") + endif() + + math(EXPR I "${I} + 1") + else() + set(ID_IDX ${I}) + math(EXPR PROP_IDX "${ID_IDX} + 1") + + set(ID "${ARGV${ID_IDX}}") + set(PROP "${ARGV${PROP_IDX}}") + + set_property(SOURCE ${FILES} PROPERTY "${ID}" "${PROP}") + math(EXPR I "${I} + 2") + endif() + endwhile() +endfunction() + +# Disable some functions that would mess up the CMake meson integration +macro(target_precompile_headers) + meson_ps_disabled_function(target_precompile_headers) +endmacro() + +set(MESON_PS_DELAYED_CALLS add_custom_command;add_custom_target;set_property) +meson_ps_reload_vars() + +cmake_policy(POP) diff --git a/meson/mesonbuild/cmake/executor.py b/meson/mesonbuild/cmake/executor.py new file mode 100644 index 000000000..7b06f2623 --- /dev/null +++ b/meson/mesonbuild/cmake/executor.py @@ -0,0 +1,246 @@ +# Copyright 2019 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This class contains the basic functionality needed to run any interpreter +# or an interpreter-based tool. + +import subprocess as S +from pathlib import Path +from threading import Thread +import typing as T +import re +import os + +from .. import mlog +from ..environment import Environment +from ..mesonlib import PerMachine, Popen_safe, version_compare, MachineChoice, is_windows, OptionKey +from ..programs import find_external_program, NonExistingExternalProgram + +if T.TYPE_CHECKING: + from ..environment import Environment + from ..programs import ExternalProgram + +TYPE_result = T.Tuple[int, T.Optional[str], T.Optional[str]] +TYPE_cache_key = T.Tuple[str, T.Tuple[str, ...], str, T.FrozenSet[T.Tuple[str, str]]] + +class CMakeExecutor: + # The class's copy of the CMake path. Avoids having to search for it + # multiple times in the same Meson invocation. + class_cmakebin = PerMachine(None, None) # type: PerMachine[T.Optional[ExternalProgram]] + class_cmakevers = PerMachine(None, None) # type: PerMachine[T.Optional[str]] + class_cmake_cache = {} # type: T.Dict[T.Any, TYPE_result] + + def __init__(self, environment: 'Environment', version: str, for_machine: MachineChoice, silent: bool = False): + self.min_version = version + self.environment = environment + self.for_machine = for_machine + self.cmakebin, self.cmakevers = self.find_cmake_binary(self.environment, silent=silent) + self.always_capture_stderr = True + self.print_cmout = False + self.prefix_paths = [] # type: T.List[str] + self.extra_cmake_args = [] # type: T.List[str] + + if self.cmakebin is None: + return + + if not version_compare(self.cmakevers, self.min_version): + mlog.warning( + 'The version of CMake', mlog.bold(self.cmakebin.get_path()), + 'is', mlog.bold(self.cmakevers), 'but version', mlog.bold(self.min_version), + 'is required') + self.cmakebin = None + return + + self.prefix_paths = self.environment.coredata.options[OptionKey('cmake_prefix_path', machine=self.for_machine)].value + if self.prefix_paths: + self.extra_cmake_args += ['-DCMAKE_PREFIX_PATH={}'.format(';'.join(self.prefix_paths))] + + def find_cmake_binary(self, environment: Environment, silent: bool = False) -> T.Tuple[T.Optional['ExternalProgram'], T.Optional[str]]: + # Only search for CMake the first time and store the result in the class + # definition + if isinstance(CMakeExecutor.class_cmakebin[self.for_machine], NonExistingExternalProgram): + mlog.debug(f'CMake binary for {self.for_machine} is cached as not found') + return None, None + elif CMakeExecutor.class_cmakebin[self.for_machine] is not None: + mlog.debug(f'CMake binary for {self.for_machine} is cached.') + else: + assert CMakeExecutor.class_cmakebin[self.for_machine] is None + + mlog.debug(f'CMake binary for {self.for_machine} is not cached') + for potential_cmakebin in find_external_program( + environment, self.for_machine, 'cmake', 'CMake', + environment.default_cmake, allow_default_for_cross=False): + version_if_ok = self.check_cmake(potential_cmakebin) + if not version_if_ok: + continue + if not silent: + mlog.log('Found CMake:', mlog.bold(potential_cmakebin.get_path()), + f'({version_if_ok})') + CMakeExecutor.class_cmakebin[self.for_machine] = potential_cmakebin + CMakeExecutor.class_cmakevers[self.for_machine] = version_if_ok + break + else: + if not silent: + mlog.log('Found CMake:', mlog.red('NO')) + # Set to False instead of None to signify that we've already + # searched for it and not found it + CMakeExecutor.class_cmakebin[self.for_machine] = NonExistingExternalProgram() + CMakeExecutor.class_cmakevers[self.for_machine] = None + return None, None + + return CMakeExecutor.class_cmakebin[self.for_machine], CMakeExecutor.class_cmakevers[self.for_machine] + + def check_cmake(self, cmakebin: 'ExternalProgram') -> T.Optional[str]: + if not cmakebin.found(): + mlog.log(f'Did not find CMake {cmakebin.name!r}') + return None + try: + p, out = Popen_safe(cmakebin.get_command() + ['--version'])[0:2] + if p.returncode != 0: + mlog.warning('Found CMake {!r} but couldn\'t run it' + ''.format(' '.join(cmakebin.get_command()))) + return None + except FileNotFoundError: + mlog.warning('We thought we found CMake {!r} but now it\'s not there. How odd!' + ''.format(' '.join(cmakebin.get_command()))) + return None + except PermissionError: + msg = 'Found CMake {!r} but didn\'t have permissions to run it.'.format(' '.join(cmakebin.get_command())) + if not is_windows(): + msg += '\n\nOn Unix-like systems this is often caused by scripts that are not executable.' + mlog.warning(msg) + return None + cmvers = re.search(r'(cmake|cmake3)\s*version\s*([\d.]+)', out).group(2) + return cmvers + + def set_exec_mode(self, print_cmout: T.Optional[bool] = None, always_capture_stderr: T.Optional[bool] = None) -> None: + if print_cmout is not None: + self.print_cmout = print_cmout + if always_capture_stderr is not None: + self.always_capture_stderr = always_capture_stderr + + def _cache_key(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_cache_key: + fenv = frozenset(env.items()) if env is not None else frozenset() + targs = tuple(args) + return (self.cmakebin.get_path(), targs, build_dir.as_posix(), fenv) + + def _call_cmout_stderr(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_result: + cmd = self.cmakebin.get_command() + args + proc = S.Popen(cmd, stdout=S.PIPE, stderr=S.PIPE, cwd=str(build_dir), env=env) # TODO [PYTHON_37]: drop Path conversion + + # stdout and stderr MUST be read at the same time to avoid pipe + # blocking issues. The easiest way to do this is with a separate + # thread for one of the pipes. + def print_stdout() -> None: + while True: + line = proc.stdout.readline() + if not line: + break + mlog.log(line.decode(errors='ignore').strip('\n')) + proc.stdout.close() + + t = Thread(target=print_stdout) + t.start() + + try: + # Read stderr line by line and log non trace lines + raw_trace = '' + tline_start_reg = re.compile(r'^\s*(.*\.(cmake|txt))\(([0-9]+)\):\s*(\w+)\(.*$') + inside_multiline_trace = False + while True: + line_raw = proc.stderr.readline() + if not line_raw: + break + line = line_raw.decode(errors='ignore') + if tline_start_reg.match(line): + raw_trace += line + inside_multiline_trace = not line.endswith(' )\n') + elif inside_multiline_trace: + raw_trace += line + else: + mlog.warning(line.strip('\n')) + + finally: + proc.stderr.close() + t.join() + proc.wait() + + return proc.returncode, None, raw_trace + + def _call_cmout(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_result: + cmd = self.cmakebin.get_command() + args + proc = S.Popen(cmd, stdout=S.PIPE, stderr=S.STDOUT, cwd=str(build_dir), env=env) # TODO [PYTHON_37]: drop Path conversion + while True: + line = proc.stdout.readline() + if not line: + break + mlog.log(line.decode(errors='ignore').strip('\n')) + proc.stdout.close() + proc.wait() + return proc.returncode, None, None + + def _call_quiet(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_result: + build_dir.mkdir(parents=True, exist_ok=True) + cmd = self.cmakebin.get_command() + args + ret = S.run(cmd, env=env, cwd=str(build_dir), close_fds=False, + stdout=S.PIPE, stderr=S.PIPE, universal_newlines=False) # TODO [PYTHON_37]: drop Path conversion + rc = ret.returncode + out = ret.stdout.decode(errors='ignore') + err = ret.stderr.decode(errors='ignore') + return rc, out, err + + def _call_impl(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]]) -> TYPE_result: + mlog.debug(f'Calling CMake ({self.cmakebin.get_command()}) in {build_dir} with:') + for i in args: + mlog.debug(f' - "{i}"') + if not self.print_cmout: + return self._call_quiet(args, build_dir, env) + else: + if self.always_capture_stderr: + return self._call_cmout_stderr(args, build_dir, env) + else: + return self._call_cmout(args, build_dir, env) + + def call(self, args: T.List[str], build_dir: Path, env: T.Optional[T.Dict[str, str]] = None, disable_cache: bool = False) -> TYPE_result: + if env is None: + env = os.environ.copy() + + args = args + self.extra_cmake_args + if disable_cache: + return self._call_impl(args, build_dir, env) + + # First check if cached, if not call the real cmake function + cache = CMakeExecutor.class_cmake_cache + key = self._cache_key(args, build_dir, env) + if key not in cache: + cache[key] = self._call_impl(args, build_dir, env) + return cache[key] + + def found(self) -> bool: + return self.cmakebin is not None + + def version(self) -> str: + return self.cmakevers + + def executable_path(self) -> str: + return self.cmakebin.get_path() + + def get_command(self) -> T.List[str]: + return self.cmakebin.get_command() + + def get_cmake_prefix_paths(self) -> T.List[str]: + return self.prefix_paths + + def machine_choice(self) -> MachineChoice: + return self.for_machine diff --git a/meson/mesonbuild/cmake/fileapi.py b/meson/mesonbuild/cmake/fileapi.py new file mode 100644 index 000000000..5d4d01a13 --- /dev/null +++ b/meson/mesonbuild/cmake/fileapi.py @@ -0,0 +1,320 @@ +# Copyright 2019 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .common import CMakeException, CMakeBuildFile, CMakeConfiguration +import typing as T +from .. import mlog +from pathlib import Path +import json +import re + +STRIP_KEYS = ['cmake', 'reply', 'backtrace', 'backtraceGraph', 'version'] + +class CMakeFileAPI: + def __init__(self, build_dir: Path): + self.build_dir = build_dir + self.api_base_dir = self.build_dir / '.cmake' / 'api' / 'v1' + self.request_dir = self.api_base_dir / 'query' / 'client-meson' + self.reply_dir = self.api_base_dir / 'reply' + self.cmake_sources = [] # type: T.List[CMakeBuildFile] + self.cmake_configurations = [] # type: T.List[CMakeConfiguration] + self.kind_resolver_map = { + 'codemodel': self._parse_codemodel, + 'cmakeFiles': self._parse_cmakeFiles, + } + + def get_cmake_sources(self) -> T.List[CMakeBuildFile]: + return self.cmake_sources + + def get_cmake_configurations(self) -> T.List[CMakeConfiguration]: + return self.cmake_configurations + + def setup_request(self) -> None: + self.request_dir.mkdir(parents=True, exist_ok=True) + + query = { + 'requests': [ + {'kind': 'codemodel', 'version': {'major': 2, 'minor': 0}}, + {'kind': 'cmakeFiles', 'version': {'major': 1, 'minor': 0}}, + ] + } + + query_file = self.request_dir / 'query.json' + query_file.write_text(json.dumps(query, indent=2), encoding='utf-8') + + def load_reply(self) -> None: + if not self.reply_dir.is_dir(): + raise CMakeException('No response from the CMake file API') + + root = None + reg_index = re.compile(r'^index-.*\.json$') + for i in self.reply_dir.iterdir(): + if reg_index.match(i.name): + root = i + break + + if not root: + raise CMakeException('Failed to find the CMake file API index') + + index = self._reply_file_content(root) # Load the root index + index = self._strip_data(index) # Avoid loading duplicate files + index = self._resolve_references(index) # Load everything + index = self._strip_data(index) # Strip unused data (again for loaded files) + + # Debug output + debug_json = self.build_dir / '..' / 'fileAPI.json' + debug_json = debug_json.resolve() + debug_json.write_text(json.dumps(index, indent=2), encoding='utf-8') + mlog.cmd_ci_include(debug_json.as_posix()) + + # parse the JSON + for i in index['objects']: + assert(isinstance(i, dict)) + assert('kind' in i) + assert(i['kind'] in self.kind_resolver_map) + + self.kind_resolver_map[i['kind']](i) + + def _parse_codemodel(self, data: T.Dict[str, T.Any]) -> None: + assert('configurations' in data) + assert('paths' in data) + + source_dir = data['paths']['source'] + build_dir = data['paths']['build'] + + # The file API output differs quite a bit from the server + # output. It is more flat than the server output and makes + # heavy use of references. Here these references are + # resolved and the resulting data structure is identical + # to the CMake serve output. + + def helper_parse_dir(dir_entry: T.Dict[str, T.Any]) -> T.Tuple[Path, Path]: + src_dir = Path(dir_entry.get('source', '.')) + bld_dir = Path(dir_entry.get('build', '.')) + src_dir = src_dir if src_dir.is_absolute() else source_dir / src_dir + bld_dir = bld_dir if bld_dir.is_absolute() else build_dir / bld_dir + src_dir = src_dir.resolve() + bld_dir = bld_dir.resolve() + + return src_dir, bld_dir + + def parse_sources(comp_group: T.Dict[str, T.Any], tgt: T.Dict[str, T.Any]) -> T.Tuple[T.List[Path], T.List[Path], T.List[int]]: + gen = [] + src = [] + idx = [] + + src_list_raw = tgt.get('sources', []) + for i in comp_group.get('sourceIndexes', []): + if i >= len(src_list_raw) or 'path' not in src_list_raw[i]: + continue + if src_list_raw[i].get('isGenerated', False): + gen += [Path(src_list_raw[i]['path'])] + else: + src += [Path(src_list_raw[i]['path'])] + idx += [i] + + return src, gen, idx + + def parse_target(tgt: T.Dict[str, T.Any]) -> T.Dict[str, T.Any]: + src_dir, bld_dir = helper_parse_dir(cnf.get('paths', {})) + + # Parse install paths (if present) + install_paths = [] + if 'install' in tgt: + prefix = Path(tgt['install']['prefix']['path']) + install_paths = [prefix / x['path'] for x in tgt['install']['destinations']] + install_paths = list(set(install_paths)) + + # On the first look, it looks really nice that the CMake devs have + # decided to use arrays for the linker flags. However, this feeling + # soon turns into despair when you realize that there only one entry + # per type in most cases, and we still have to do manual string splitting. + link_flags = [] + link_libs = [] + for i in tgt.get('link', {}).get('commandFragments', []): + if i['role'] == 'flags': + link_flags += [i['fragment']] + elif i['role'] == 'libraries': + link_libs += [i['fragment']] + elif i['role'] == 'libraryPath': + link_flags += ['-L{}'.format(i['fragment'])] + elif i['role'] == 'frameworkPath': + link_flags += ['-F{}'.format(i['fragment'])] + for i in tgt.get('archive', {}).get('commandFragments', []): + if i['role'] == 'flags': + link_flags += [i['fragment']] + + # TODO The `dependencies` entry is new in the file API. + # maybe we can make use of that in addition to the + # implicit dependency detection + tgt_data = { + 'artifacts': [Path(x.get('path', '')) for x in tgt.get('artifacts', [])], + 'sourceDirectory': src_dir, + 'buildDirectory': bld_dir, + 'name': tgt.get('name', ''), + 'fullName': tgt.get('nameOnDisk', ''), + 'hasInstallRule': 'install' in tgt, + 'installPaths': install_paths, + 'linkerLanguage': tgt.get('link', {}).get('language', 'CXX'), + 'linkLibraries': ' '.join(link_libs), # See previous comment block why we join the array + 'linkFlags': ' '.join(link_flags), # See previous comment block why we join the array + 'type': tgt.get('type', 'EXECUTABLE'), + 'fileGroups': [], + } + + processed_src_idx = [] + for cg in tgt.get('compileGroups', []): + # Again, why an array, when there is usually only one element + # and arguments are separated with spaces... + flags = [] + for i in cg.get('compileCommandFragments', []): + flags += [i['fragment']] + + cg_data = { + 'defines': [x.get('define', '') for x in cg.get('defines', [])], + 'compileFlags': ' '.join(flags), + 'language': cg.get('language', 'C'), + 'isGenerated': None, # Set later, flag is stored per source file + 'sources': [], + 'includePath': cg.get('includes', []), + } + + normal_src, generated_src, src_idx = parse_sources(cg, tgt) + if normal_src: + cg_data = dict(cg_data) + cg_data['isGenerated'] = False + cg_data['sources'] = normal_src + tgt_data['fileGroups'] += [cg_data] + if generated_src: + cg_data = dict(cg_data) + cg_data['isGenerated'] = True + cg_data['sources'] = generated_src + tgt_data['fileGroups'] += [cg_data] + processed_src_idx += src_idx + + # Object libraries have no compile groups, only source groups. + # So we add all the source files to a dummy source group that were + # not found in the previous loop + normal_src = [] + generated_src = [] + for idx, src in enumerate(tgt.get('sources', [])): + if idx in processed_src_idx: + continue + + if src.get('isGenerated', False): + generated_src += [src['path']] + else: + normal_src += [src['path']] + + if normal_src: + tgt_data['fileGroups'] += [{ + 'isGenerated': False, + 'sources': normal_src, + }] + if generated_src: + tgt_data['fileGroups'] += [{ + 'isGenerated': True, + 'sources': generated_src, + }] + return tgt_data + + def parse_project(pro: T.Dict[str, T.Any]) -> T.Dict[str, T.Any]: + # Only look at the first directory specified in directoryIndexes + # TODO Figure out what the other indexes are there for + p_src_dir = source_dir + p_bld_dir = build_dir + try: + p_src_dir, p_bld_dir = helper_parse_dir(cnf['directories'][pro['directoryIndexes'][0]]) + except (IndexError, KeyError): + pass + + pro_data = { + 'name': pro.get('name', ''), + 'sourceDirectory': p_src_dir, + 'buildDirectory': p_bld_dir, + 'targets': [], + } + + for ref in pro.get('targetIndexes', []): + tgt = {} + try: + tgt = cnf['targets'][ref] + except (IndexError, KeyError): + pass + pro_data['targets'] += [parse_target(tgt)] + + return pro_data + + for cnf in data.get('configurations', []): + cnf_data = { + 'name': cnf.get('name', ''), + 'projects': [], + } + + for pro in cnf.get('projects', []): + cnf_data['projects'] += [parse_project(pro)] + + self.cmake_configurations += [CMakeConfiguration(cnf_data)] + + def _parse_cmakeFiles(self, data: T.Dict[str, T.Any]) -> None: + assert 'inputs' in data + assert 'paths' in data + + src_dir = Path(data['paths']['source']) + + for i in data['inputs']: + path = Path(i['path']) + path = path if path.is_absolute() else src_dir / path + self.cmake_sources += [CMakeBuildFile(path, i.get('isCMake', False), i.get('isGenerated', False))] + + def _strip_data(self, data: T.Any) -> T.Any: + if isinstance(data, list): + for idx, i in enumerate(data): + data[idx] = self._strip_data(i) + + elif isinstance(data, dict): + new = {} + for key, val in data.items(): + if key not in STRIP_KEYS: + new[key] = self._strip_data(val) + data = new + + return data + + def _resolve_references(self, data: T.Any) -> T.Any: + if isinstance(data, list): + for idx, i in enumerate(data): + data[idx] = self._resolve_references(i) + + elif isinstance(data, dict): + # Check for the "magic" reference entry and insert + # it into the root data dict + if 'jsonFile' in data: + data.update(self._reply_file_content(data['jsonFile'])) + + for key, val in data.items(): + data[key] = self._resolve_references(val) + + return data + + def _reply_file_content(self, filename: Path) -> T.Dict[str, T.Any]: + real_path = self.reply_dir / filename + if not real_path.exists(): + raise CMakeException(f'File "{real_path}" does not exist') + + data = json.loads(real_path.read_text(encoding='utf-8')) + assert isinstance(data, dict) + for i in data.keys(): + assert isinstance(i, str) + return data diff --git a/meson/mesonbuild/cmake/generator.py b/meson/mesonbuild/cmake/generator.py new file mode 100644 index 000000000..848fdf944 --- /dev/null +++ b/meson/mesonbuild/cmake/generator.py @@ -0,0 +1,134 @@ +# Copyright 2019 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .. import mesonlib +import typing as T + +def parse_generator_expressions(raw: str) -> str: + '''Parse CMake generator expressions + + Most generator expressions are simply ignored for + simplicety, however some are required for some common + use cases. + ''' + + # Early abort if no generator expression present + if '$<' not in raw: + return raw + + out = '' # type: str + i = 0 # type: int + + def equal(arg: str) -> str: + col_pos = arg.find(',') + if col_pos < 0: + return '0' + else: + return '1' if arg[:col_pos] == arg[col_pos + 1:] else '0' + + def vers_comp(op: str, arg: str) -> str: + col_pos = arg.find(',') + if col_pos < 0: + return '0' + else: + return '1' if mesonlib.version_compare(arg[:col_pos], '{}{}'.format(op, arg[col_pos + 1:])) else '0' + + supported = { + # Boolean functions + 'BOOL': lambda x: '0' if x.upper() in ['0', 'FALSE', 'OFF', 'N', 'NO', 'IGNORE', 'NOTFOUND'] or x.endswith('-NOTFOUND') else '1', + 'AND': lambda x: '1' if all([y == '1' for y in x.split(',')]) else '0', + 'OR': lambda x: '1' if any([y == '1' for y in x.split(',')]) else '0', + 'NOT': lambda x: '0' if x == '1' else '1', + + '0': lambda x: '', + '1': lambda x: x, + + # String operations + 'STREQUAL': equal, + 'EQUAL': equal, + 'VERSION_LESS': lambda x: vers_comp('<', x), + 'VERSION_GREATER': lambda x: vers_comp('>', x), + 'VERSION_EQUAL': lambda x: vers_comp('=', x), + 'VERSION_LESS_EQUAL': lambda x: vers_comp('<=', x), + 'VERSION_GREATER_EQUAL': lambda x: vers_comp('>=', x), + + # String modification + 'LOWER_CASE': lambda x: x.lower(), + 'UPPER_CASE': lambda x: x.upper(), + + # Always assume the BUILD_INTERFACE is valid. + # INSTALL_INTERFACE is always invalid for subprojects and + # it should also never appear in CMake config files, used + # for dependencies + 'INSTALL_INTERFACE': lambda x: '', + 'BUILD_INTERFACE': lambda x: x, + + # Constants + 'ANGLE-R': lambda x: '>', + 'COMMA': lambda x: ',', + 'SEMICOLON': lambda x: ';', + } # type: T.Dict[str, T.Callable[[str], str]] + + # Recursively evaluate generator expressions + def eval_generator_expressions() -> str: + nonlocal i + i += 2 + + func = '' # type: str + args = '' # type: str + res = '' # type: str + exp = '' # type: str + + # Determine the body of the expression + while i < len(raw): + if raw[i] == '>': + # End of the generator expression + break + elif i < len(raw) - 1 and raw[i] == '$' and raw[i + 1] == '<': + # Nested generator expression + exp += eval_generator_expressions() + else: + # Generator expression body + exp += raw[i] + + i += 1 + + # Split the expression into a function and arguments part + col_pos = exp.find(':') + if col_pos < 0: + func = exp + else: + func = exp[:col_pos] + args = exp[col_pos + 1:] + + func = func.strip() + args = args.strip() + + # Evaluate the function + if func in supported: + res = supported[func](args) + + return res + + while i < len(raw): + if i < len(raw) - 1 and raw[i] == '$' and raw[i + 1] == '<': + # Generator expression detected --> try resolving it + out += eval_generator_expressions() + else: + # Normal string, leave unchanged + out += raw[i] + + i += 1 + + return out diff --git a/meson/mesonbuild/cmake/interpreter.py b/meson/mesonbuild/cmake/interpreter.py new file mode 100644 index 000000000..fe66becb9 --- /dev/null +++ b/meson/mesonbuild/cmake/interpreter.py @@ -0,0 +1,1369 @@ +# Copyright 2019 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This class contains the basic functionality needed to run any interpreter +# or an interpreter-based tool. + +from .common import CMakeException, CMakeTarget, TargetOptions, CMakeConfiguration, language_map, backend_generator_map, cmake_get_generator_args, check_cmake_args +from .client import CMakeClient, RequestCMakeInputs, RequestConfigure, RequestCompute, RequestCodeModel, ReplyCMakeInputs, ReplyCodeModel +from .fileapi import CMakeFileAPI +from .executor import CMakeExecutor +from .toolchain import CMakeToolchain, CMakeExecScope +from .traceparser import CMakeTraceParser, CMakeGeneratorTarget +from .. import mlog, mesonlib +from ..mesonlib import MachineChoice, OrderedSet, version_compare, path_is_in_root, relative_to_if_possible, OptionKey +from ..mesondata import mesondata +from ..compilers.compilers import assembler_suffixes, lang_suffixes, header_suffixes, obj_suffixes, lib_suffixes, is_header +from ..programs import ExternalProgram +from ..coredata import FORBIDDEN_TARGET_NAMES +from enum import Enum +from functools import lru_cache +from pathlib import Path +import typing as T +import re +from os import environ + +from ..mparser import ( + Token, + BaseNode, + CodeBlockNode, + FunctionNode, + ArrayNode, + ArgumentNode, + AssignmentNode, + BooleanNode, + StringNode, + IdNode, + IndexNode, + MethodNode, + NumberNode, +) + + +if T.TYPE_CHECKING: + from .._typing import ImmutableListProtocol + from ..build import Build + from ..backend.backends import Backend + from ..environment import Environment + +TYPE_mixed = T.Union[str, int, bool, Path, BaseNode] +TYPE_mixed_list = T.Union[TYPE_mixed, T.Sequence[TYPE_mixed]] +TYPE_mixed_kwargs = T.Dict[str, TYPE_mixed_list] + +# Disable all warnings automaticall enabled with --trace and friends +# See https://cmake.org/cmake/help/latest/variable/CMAKE_POLICY_WARNING_CMPNNNN.html +disable_policy_warnings = [ + 'CMP0025', + 'CMP0047', + 'CMP0056', + 'CMP0060', + 'CMP0065', + 'CMP0066', + 'CMP0067', + 'CMP0082', + 'CMP0089', + 'CMP0102', +] + +target_type_map = { + 'STATIC_LIBRARY': 'static_library', + 'MODULE_LIBRARY': 'shared_module', + 'SHARED_LIBRARY': 'shared_library', + 'EXECUTABLE': 'executable', + 'OBJECT_LIBRARY': 'static_library', + 'INTERFACE_LIBRARY': 'header_only' +} + +skip_targets = ['UTILITY'] + +blacklist_compiler_flags = [ + '-Wall', '-Wextra', '-Weverything', '-Werror', '-Wpedantic', '-pedantic', '-w', + '/W1', '/W2', '/W3', '/W4', '/Wall', '/WX', '/w', + '/O1', '/O2', '/Ob', '/Od', '/Og', '/Oi', '/Os', '/Ot', '/Ox', '/Oy', '/Ob0', + '/RTC1', '/RTCc', '/RTCs', '/RTCu', + '/Z7', '/Zi', '/ZI', +] + +blacklist_link_flags = [ + '/machine:x64', '/machine:x86', '/machine:arm', '/machine:ebc', + '/debug', '/debug:fastlink', '/debug:full', '/debug:none', + '/incremental', +] + +blacklist_clang_cl_link_flags = ['/GR', '/EHsc', '/MDd', '/Zi', '/RTC1'] + +blacklist_link_libs = [ + 'kernel32.lib', + 'user32.lib', + 'gdi32.lib', + 'winspool.lib', + 'shell32.lib', + 'ole32.lib', + 'oleaut32.lib', + 'uuid.lib', + 'comdlg32.lib', + 'advapi32.lib' +] + +transfer_dependencies_from = ['header_only'] + +_cmake_name_regex = re.compile(r'[^_a-zA-Z0-9]') +def _sanitize_cmake_name(name: str) -> str: + name = _cmake_name_regex.sub('_', name) + if name in FORBIDDEN_TARGET_NAMES or name.startswith('meson'): + name = 'cm_' + name + return name + +class OutputTargetMap: + rm_so_version = re.compile(r'(\.[0-9]+)+$') + + def __init__(self, build_dir: Path): + self.tgt_map = {} # type: T.Dict[str, T.Union['ConverterTarget', 'ConverterCustomTarget']] + self.build_dir = build_dir + + def add(self, tgt: T.Union['ConverterTarget', 'ConverterCustomTarget']) -> None: + def assign_keys(keys: T.List[str]) -> None: + for i in [x for x in keys if x]: + self.tgt_map[i] = tgt + keys = [self._target_key(tgt.cmake_name)] + if isinstance(tgt, ConverterTarget): + keys += [tgt.full_name] + keys += [self._rel_artifact_key(x) for x in tgt.artifacts] + keys += [self._base_artifact_key(x) for x in tgt.artifacts] + if isinstance(tgt, ConverterCustomTarget): + keys += [self._rel_generated_file_key(x) for x in tgt.original_outputs] + keys += [self._base_generated_file_key(x) for x in tgt.original_outputs] + assign_keys(keys) + + def _return_first_valid_key(self, keys: T.List[str]) -> T.Optional[T.Union['ConverterTarget', 'ConverterCustomTarget']]: + for i in keys: + if i and i in self.tgt_map: + return self.tgt_map[i] + return None + + def target(self, name: str) -> T.Optional[T.Union['ConverterTarget', 'ConverterCustomTarget']]: + return self._return_first_valid_key([self._target_key(name)]) + + def executable(self, name: str) -> T.Optional['ConverterTarget']: + tgt = self.target(name) + if tgt is None or not isinstance(tgt, ConverterTarget): + return None + if tgt.meson_func() != 'executable': + return None + return tgt + + def artifact(self, name: str) -> T.Optional[T.Union['ConverterTarget', 'ConverterCustomTarget']]: + keys = [] + candidates = [name, OutputTargetMap.rm_so_version.sub('', name)] + for i in lib_suffixes: + if not name.endswith('.' + i): + continue + new_name = name[:-len(i) - 1] + new_name = OutputTargetMap.rm_so_version.sub('', new_name) + candidates += [f'{new_name}.{i}'] + for i in candidates: + keys += [self._rel_artifact_key(Path(i)), Path(i).name, self._base_artifact_key(Path(i))] + return self._return_first_valid_key(keys) + + def generated(self, name: Path) -> T.Optional['ConverterCustomTarget']: + res = self._return_first_valid_key([self._rel_generated_file_key(name), self._base_generated_file_key(name)]) + assert res is None or isinstance(res, ConverterCustomTarget) + return res + + # Utility functions to generate local keys + def _rel_path(self, fname: Path) -> T.Optional[Path]: + try: + return fname.resolve().relative_to(self.build_dir) + except ValueError: + pass + return None + + def _target_key(self, tgt_name: str) -> str: + return f'__tgt_{tgt_name}__' + + def _rel_generated_file_key(self, fname: Path) -> T.Optional[str]: + path = self._rel_path(fname) + return f'__relgen_{path.as_posix()}__' if path else None + + def _base_generated_file_key(self, fname: Path) -> str: + return f'__gen_{fname.name}__' + + def _rel_artifact_key(self, fname: Path) -> T.Optional[str]: + path = self._rel_path(fname) + return f'__relart_{path.as_posix()}__' if path else None + + def _base_artifact_key(self, fname: Path) -> str: + return f'__art_{fname.name}__' + +class ConverterTarget: + def __init__(self, target: CMakeTarget, env: 'Environment', for_machine: MachineChoice) -> None: + self.env = env + self.for_machine = for_machine + self.artifacts = target.artifacts + self.src_dir = target.src_dir + self.build_dir = target.build_dir + self.name = target.name + self.cmake_name = target.name + self.full_name = target.full_name + self.type = target.type + self.install = target.install + self.install_dir = None # type: T.Optional[Path] + self.link_libraries = target.link_libraries + self.link_flags = target.link_flags + target.link_lang_flags + self.depends_raw = [] # type: T.List[str] + self.depends = [] # type: T.List[T.Union[ConverterTarget, ConverterCustomTarget]] + + if target.install_paths: + self.install_dir = target.install_paths[0] + + self.languages = set() # type: T.Set[str] + self.sources = [] # type: T.List[Path] + self.generated = [] # type: T.List[Path] + self.generated_ctgt = [] # type: T.List[CustomTargetReference] + self.includes = [] # type: T.List[Path] + self.sys_includes = [] # type: T.List[Path] + self.link_with = [] # type: T.List[T.Union[ConverterTarget, ConverterCustomTarget]] + self.object_libs = [] # type: T.List[ConverterTarget] + self.compile_opts = {} # type: T.Dict[str, T.List[str]] + self.public_compile_opts = [] # type: T.List[str] + self.pie = False + + # Project default override options (c_std, cpp_std, etc.) + self.override_options = [] # type: T.List[str] + + # Convert the target name to a valid meson target name + self.name = _sanitize_cmake_name(self.name) + + self.generated_raw = [] # type: T.List[Path] + + for i in target.files: + languages = set() # type: T.Set[str] + src_suffixes = set() # type: T.Set[str] + + # Insert suffixes + for j in i.sources: + if not j.suffix: + continue + src_suffixes.add(j.suffix[1:]) + + # Determine the meson language(s) + # Extract the default language from the explicit CMake field + lang_cmake_to_meson = {val.lower(): key for key, val in language_map.items()} + languages.add(lang_cmake_to_meson.get(i.language.lower(), 'c')) + + # Determine missing languages from the source suffixes + for sfx in src_suffixes: + for key, val in lang_suffixes.items(): + if sfx in val: + languages.add(key) + break + + # Register the new languages and initialize the compile opts array + for lang in languages: + self.languages.add(lang) + if lang not in self.compile_opts: + self.compile_opts[lang] = [] + + # Add arguments, but avoid duplicates + args = i.flags + args += [f'-D{x}' for x in i.defines] + for lang in languages: + self.compile_opts[lang] += [x for x in args if x not in self.compile_opts[lang]] + + # Handle include directories + self.includes += [x.path for x in i.includes if x.path not in self.includes and not x.isSystem] + self.sys_includes += [x.path for x in i.includes if x.path not in self.sys_includes and x.isSystem] + + # Add sources to the right array + if i.is_generated: + self.generated_raw += i.sources + else: + self.sources += i.sources + + def __repr__(self) -> str: + return f'<{self.__class__.__name__}: {self.name}>' + + std_regex = re.compile(r'([-]{1,2}std=|/std:v?|[-]{1,2}std:)(.*)') + + def postprocess(self, output_target_map: OutputTargetMap, root_src_dir: Path, subdir: Path, install_prefix: Path, trace: CMakeTraceParser) -> None: + # Detect setting the C and C++ standard and do additional compiler args manipulation + for i in ['c', 'cpp']: + if i not in self.compile_opts: + continue + + temp = [] + for j in self.compile_opts[i]: + m = ConverterTarget.std_regex.match(j) + ctgt = output_target_map.generated(Path(j)) + if m: + std = m.group(2) + supported = self._all_lang_stds(i) + if std not in supported: + mlog.warning( + 'Unknown {0}_std "{1}" -> Ignoring. Try setting the project-' + 'level {0}_std if build errors occur. Known ' + '{0}_stds are: {2}'.format(i, std, ' '.join(supported)), + once=True + ) + continue + self.override_options += [f'{i}_std={std}'] + elif j in ['-fPIC', '-fpic', '-fPIE', '-fpie']: + self.pie = True + elif isinstance(ctgt, ConverterCustomTarget): + # Sometimes projects pass generated source files as compiler + # flags. Add these as generated sources to ensure that the + # corresponding custom target is run.2 + self.generated_raw += [Path(j)] + temp += [j] + elif j in blacklist_compiler_flags: + pass + else: + temp += [j] + + self.compile_opts[i] = temp + + # Make sure to force enable -fPIC for OBJECT libraries + if self.type.upper() == 'OBJECT_LIBRARY': + self.pie = True + + # Use the CMake trace, if required + tgt = trace.targets.get(self.cmake_name) + if tgt: + self.depends_raw = trace.targets[self.cmake_name].depends + + # TODO refactor this copy paste from CMakeDependency for future releases + reg_is_lib = re.compile(r'^(-l[a-zA-Z0-9_]+|-l?pthread)$') + to_process = [self.cmake_name] + processed = [] + while len(to_process) > 0: + curr = to_process.pop(0) + + if curr in processed or curr not in trace.targets: + continue + + tgt = trace.targets[curr] + cfgs = [] + cfg = '' + otherDeps = [] + libraries = [] + mlog.debug(str(tgt)) + + if 'INTERFACE_INCLUDE_DIRECTORIES' in tgt.properties: + self.includes += [Path(x) for x in tgt.properties['INTERFACE_INCLUDE_DIRECTORIES'] if x] + + if 'INTERFACE_LINK_OPTIONS' in tgt.properties: + self.link_flags += [x for x in tgt.properties['INTERFACE_LINK_OPTIONS'] if x] + + if 'INTERFACE_COMPILE_DEFINITIONS' in tgt.properties: + self.public_compile_opts += ['-D' + re.sub('^-D', '', x) for x in tgt.properties['INTERFACE_COMPILE_DEFINITIONS'] if x] + + if 'INTERFACE_COMPILE_OPTIONS' in tgt.properties: + self.public_compile_opts += [x for x in tgt.properties['INTERFACE_COMPILE_OPTIONS'] if x] + + if 'IMPORTED_CONFIGURATIONS' in tgt.properties: + cfgs += [x for x in tgt.properties['IMPORTED_CONFIGURATIONS'] if x] + cfg = cfgs[0] + + if 'CONFIGURATIONS' in tgt.properties: + cfgs += [x for x in tgt.properties['CONFIGURATIONS'] if x] + cfg = cfgs[0] + + is_debug = self.env.coredata.get_option(OptionKey('debug')); + if is_debug: + if 'DEBUG' in cfgs: + cfg = 'DEBUG' + elif 'RELEASE' in cfgs: + cfg = 'RELEASE' + else: + if 'RELEASE' in cfgs: + cfg = 'RELEASE' + + if f'IMPORTED_IMPLIB_{cfg}' in tgt.properties: + libraries += [x for x in tgt.properties[f'IMPORTED_IMPLIB_{cfg}'] if x] + elif 'IMPORTED_IMPLIB' in tgt.properties: + libraries += [x for x in tgt.properties['IMPORTED_IMPLIB'] if x] + elif f'IMPORTED_LOCATION_{cfg}' in tgt.properties: + libraries += [x for x in tgt.properties[f'IMPORTED_LOCATION_{cfg}'] if x] + elif 'IMPORTED_LOCATION' in tgt.properties: + libraries += [x for x in tgt.properties['IMPORTED_LOCATION'] if x] + + if 'LINK_LIBRARIES' in tgt.properties: + otherDeps += [x for x in tgt.properties['LINK_LIBRARIES'] if x] + + if 'INTERFACE_LINK_LIBRARIES' in tgt.properties: + otherDeps += [x for x in tgt.properties['INTERFACE_LINK_LIBRARIES'] if x] + + if f'IMPORTED_LINK_DEPENDENT_LIBRARIES_{cfg}' in tgt.properties: + otherDeps += [x for x in tgt.properties[f'IMPORTED_LINK_DEPENDENT_LIBRARIES_{cfg}'] if x] + elif 'IMPORTED_LINK_DEPENDENT_LIBRARIES' in tgt.properties: + otherDeps += [x for x in tgt.properties['IMPORTED_LINK_DEPENDENT_LIBRARIES'] if x] + + for j in otherDeps: + if j in trace.targets: + to_process += [j] + elif reg_is_lib.match(j) or Path(j).exists(): + libraries += [j] + + for j in libraries: + if j not in self.link_libraries: + self.link_libraries += [j] + + processed += [curr] + elif self.type.upper() not in ['EXECUTABLE', 'OBJECT_LIBRARY']: + mlog.warning('CMake: Target', mlog.bold(self.cmake_name), 'not found in CMake trace. This can lead to build errors') + + temp = [] + for i in self.link_libraries: + # Let meson handle this arcane magic + if ',-rpath,' in i: + continue + if not Path(i).is_absolute(): + link_with = output_target_map.artifact(i) + if link_with: + self.link_with += [link_with] + continue + + temp += [i] + self.link_libraries = temp + + # Filter out files that are not supported by the language + supported = list(assembler_suffixes) + list(header_suffixes) + list(obj_suffixes) + for i in self.languages: + supported += list(lang_suffixes[i]) + supported = [f'.{x}' for x in supported] + self.sources = [x for x in self.sources if any([x.name.endswith(y) for y in supported])] + self.generated_raw = [x for x in self.generated_raw if any([x.name.endswith(y) for y in supported])] + + # Make paths relative + def rel_path(x: Path, is_header: bool, is_generated: bool) -> T.Optional[Path]: + if not x.is_absolute(): + x = self.src_dir / x + x = x.resolve() + assert x.is_absolute() + if not x.exists() and not any([x.name.endswith(y) for y in obj_suffixes]) and not is_generated: + if path_is_in_root(x, Path(self.env.get_build_dir()), resolve=True): + x.mkdir(parents=True, exist_ok=True) + return x.relative_to(Path(self.env.get_build_dir()) / subdir) + else: + mlog.warning('CMake: path', mlog.bold(x.as_posix()), 'does not exist.') + mlog.warning(' --> Ignoring. This can lead to build errors.') + return None + if x in trace.explicit_headers: + return None + if ( + path_is_in_root(x, Path(self.env.get_source_dir())) + and not ( + path_is_in_root(x, root_src_dir) or + path_is_in_root(x, Path(self.env.get_build_dir())) + ) + ): + mlog.warning('CMake: path', mlog.bold(x.as_posix()), 'is inside the root project but', mlog.bold('not'), 'inside the subproject.') + mlog.warning(' --> Ignoring. This can lead to build errors.') + return None + if path_is_in_root(x, Path(self.env.get_build_dir())) and is_header: + return x.relative_to(Path(self.env.get_build_dir()) / subdir) + if path_is_in_root(x, root_src_dir): + return x.relative_to(root_src_dir) + return x + + build_dir_rel = self.build_dir.relative_to(Path(self.env.get_build_dir()) / subdir) + self.generated_raw = [rel_path(x, False, True) for x in self.generated_raw] + self.includes = list(OrderedSet([rel_path(x, True, False) for x in OrderedSet(self.includes)] + [build_dir_rel])) + self.sys_includes = list(OrderedSet([rel_path(x, True, False) for x in OrderedSet(self.sys_includes)])) + self.sources = [rel_path(x, False, False) for x in self.sources] + + # Resolve custom targets + for gen_file in self.generated_raw: + ctgt = output_target_map.generated(gen_file) + if ctgt: + assert isinstance(ctgt, ConverterCustomTarget) + ref = ctgt.get_ref(gen_file) + assert isinstance(ref, CustomTargetReference) and ref.valid() + self.generated_ctgt += [ref] + elif gen_file is not None: + self.generated += [gen_file] + + # Remove delete entries + self.includes = [x for x in self.includes if x is not None] + self.sys_includes = [x for x in self.sys_includes if x is not None] + self.sources = [x for x in self.sources if x is not None] + + # Make sure '.' is always in the include directories + if Path('.') not in self.includes: + self.includes += [Path('.')] + + # make install dir relative to the install prefix + if self.install_dir and self.install_dir.is_absolute(): + if path_is_in_root(self.install_dir, install_prefix): + self.install_dir = self.install_dir.relative_to(install_prefix) + + # Remove blacklisted options and libs + def check_flag(flag: str) -> bool: + if flag.lower() in blacklist_link_flags or flag in blacklist_compiler_flags + blacklist_clang_cl_link_flags: + return False + if flag.startswith('/D'): + return False + return True + + self.link_libraries = [x for x in self.link_libraries if x.lower() not in blacklist_link_libs] + self.link_flags = [x for x in self.link_flags if check_flag(x)] + + # Handle OSX frameworks + def handle_frameworks(flags: T.List[str]) -> T.List[str]: + res: T.List[str] = [] + for i in flags: + p = Path(i) + if not p.exists() or not p.name.endswith('.framework'): + res += [i] + continue + res += ['-framework', p.stem] + return res + + self.link_libraries = handle_frameworks(self.link_libraries) + self.link_flags = handle_frameworks(self.link_flags) + + # Handle explicit CMake add_dependency() calls + for i in self.depends_raw: + dep_tgt = output_target_map.target(i) + if dep_tgt: + self.depends.append(dep_tgt) + + def process_object_libs(self, obj_target_list: T.List['ConverterTarget'], linker_workaround: bool) -> None: + # Try to detect the object library(s) from the generated input sources + temp = [x for x in self.generated if any([x.name.endswith('.' + y) for y in obj_suffixes])] + stem = [x.stem for x in temp] + exts = self._all_source_suffixes() + # Temp now stores the source filenames of the object files + for i in obj_target_list: + source_files = [x.name for x in i.sources + i.generated] + for j in stem: + # On some platforms (specifically looking at you Windows with vs20xy backend) CMake does + # not produce object files with the format `foo.cpp.obj`, instead it skipps the language + # suffix and just produces object files like `foo.obj`. Thus we have to do our best to + # undo this step and guess the correct language suffix of the object file. This is done + # by trying all language suffixes meson knows and checking if one of them fits. + candidates = [j] # type: T.List[str] + if not any([j.endswith('.' + x) for x in exts]): + mlog.warning('Object files do not contain source file extensions, thus falling back to guessing them.', once=True) + candidates += [f'{j}.{x}' for x in exts] + if any([x in source_files for x in candidates]): + if linker_workaround: + self._append_objlib_sources(i) + else: + self.includes += i.includes + self.includes = list(OrderedSet(self.includes)) + self.object_libs += [i] + break + + # Filter out object files from the sources + self.generated = [x for x in self.generated if not any([x.name.endswith('.' + y) for y in obj_suffixes])] + + def _append_objlib_sources(self, tgt: 'ConverterTarget') -> None: + self.includes += tgt.includes + self.sources += tgt.sources + self.generated += tgt.generated + self.generated_ctgt += tgt.generated_ctgt + self.includes = list(OrderedSet(self.includes)) + self.sources = list(OrderedSet(self.sources)) + self.generated = list(OrderedSet(self.generated)) + self.generated_ctgt = list(OrderedSet(self.generated_ctgt)) + + # Inherit compiler arguments since they may be required for building + for lang, opts in tgt.compile_opts.items(): + if lang not in self.compile_opts: + self.compile_opts[lang] = [] + self.compile_opts[lang] += [x for x in opts if x not in self.compile_opts[lang]] + + @lru_cache(maxsize=None) + def _all_source_suffixes(self) -> 'ImmutableListProtocol[str]': + suffixes = [] # type: T.List[str] + for exts in lang_suffixes.values(): + suffixes += [x for x in exts] + return suffixes + + @lru_cache(maxsize=None) + def _all_lang_stds(self, lang: str) -> 'ImmutableListProtocol[str]': + try: + res = self.env.coredata.options[OptionKey('std', machine=MachineChoice.BUILD, lang=lang)].choices + except KeyError: + return [] + + # TODO: Get rid of this once we have proper typing for options + assert isinstance(res, list) + for i in res: + assert isinstance(i, str) + + return res + + def process_inter_target_dependencies(self) -> None: + # Move the dependencies from all transfer_dependencies_from to the target + to_process = list(self.depends) + processed = [] + new_deps = [] + for i in to_process: + processed += [i] + if isinstance(i, ConverterTarget) and i.meson_func() in transfer_dependencies_from: + to_process += [x for x in i.depends if x not in processed] + else: + new_deps += [i] + self.depends = list(OrderedSet(new_deps)) + + def cleanup_dependencies(self) -> None: + # Clear the dependencies from targets that where moved from + if self.meson_func() in transfer_dependencies_from: + self.depends = [] + + def meson_func(self) -> str: + return target_type_map.get(self.type.upper()) + + def log(self) -> None: + mlog.log('Target', mlog.bold(self.name), f'({self.cmake_name})') + mlog.log(' -- artifacts: ', mlog.bold(str(self.artifacts))) + mlog.log(' -- full_name: ', mlog.bold(self.full_name)) + mlog.log(' -- type: ', mlog.bold(self.type)) + mlog.log(' -- install: ', mlog.bold('true' if self.install else 'false')) + mlog.log(' -- install_dir: ', mlog.bold(self.install_dir.as_posix() if self.install_dir else '')) + mlog.log(' -- link_libraries: ', mlog.bold(str(self.link_libraries))) + mlog.log(' -- link_with: ', mlog.bold(str(self.link_with))) + mlog.log(' -- object_libs: ', mlog.bold(str(self.object_libs))) + mlog.log(' -- link_flags: ', mlog.bold(str(self.link_flags))) + mlog.log(' -- languages: ', mlog.bold(str(self.languages))) + mlog.log(' -- includes: ', mlog.bold(str(self.includes))) + mlog.log(' -- sys_includes: ', mlog.bold(str(self.sys_includes))) + mlog.log(' -- sources: ', mlog.bold(str(self.sources))) + mlog.log(' -- generated: ', mlog.bold(str(self.generated))) + mlog.log(' -- generated_ctgt: ', mlog.bold(str(self.generated_ctgt))) + mlog.log(' -- pie: ', mlog.bold('true' if self.pie else 'false')) + mlog.log(' -- override_opts: ', mlog.bold(str(self.override_options))) + mlog.log(' -- depends: ', mlog.bold(str(self.depends))) + mlog.log(' -- options:') + for key, val in self.compile_opts.items(): + mlog.log(' -', key, '=', mlog.bold(str(val))) + +class CustomTargetReference: + def __init__(self, ctgt: 'ConverterCustomTarget', index: int) -> None: + self.ctgt = ctgt # type: ConverterCustomTarget + self.index = index # type: int + + def __repr__(self) -> str: + if self.valid(): + return '<{}: {} [{}]>'.format(self.__class__.__name__, self.ctgt.name, self.ctgt.outputs[self.index]) + else: + return f'<{self.__class__.__name__}: INVALID REFERENCE>' + + def valid(self) -> bool: + return self.ctgt is not None and self.index >= 0 + + def filename(self) -> str: + return self.ctgt.outputs[self.index] + +class ConverterCustomTarget: + tgt_counter = 0 # type: int + out_counter = 0 # type: int + + def __init__(self, target: CMakeGeneratorTarget, env: 'Environment', for_machine: MachineChoice) -> None: + assert target.current_bin_dir is not None + assert target.current_src_dir is not None + self.name = target.name + if not self.name: + self.name = f'custom_tgt_{ConverterCustomTarget.tgt_counter}' + ConverterCustomTarget.tgt_counter += 1 + self.cmake_name = str(self.name) + self.original_outputs = list(target.outputs) + self.outputs = [x.name for x in self.original_outputs] + self.conflict_map = {} # type: T.Dict[str, str] + self.command = [] # type: T.List[T.List[T.Union[str, ConverterTarget]]] + self.working_dir = target.working_dir + self.depends_raw = target.depends + self.inputs = [] # type: T.List[T.Union[str, CustomTargetReference]] + self.depends = [] # type: T.List[T.Union[ConverterTarget, ConverterCustomTarget]] + self.current_bin_dir = target.current_bin_dir # type: Path + self.current_src_dir = target.current_src_dir # type: Path + self.env = env + self.for_machine = for_machine + self._raw_target = target + + # Convert the target name to a valid meson target name + self.name = _sanitize_cmake_name(self.name) + + def __repr__(self) -> str: + return f'<{self.__class__.__name__}: {self.name} {self.outputs}>' + + def postprocess(self, output_target_map: OutputTargetMap, root_src_dir: Path, all_outputs: T.List[str], trace: CMakeTraceParser) -> None: + # Default the working directory to ${CMAKE_CURRENT_BINARY_DIR} + if self.working_dir is None: + self.working_dir = self.current_bin_dir + + # relative paths in the working directory are always relative + # to ${CMAKE_CURRENT_BINARY_DIR} + if not self.working_dir.is_absolute(): + self.working_dir = self.current_bin_dir / self.working_dir + + # Modify the original outputs if they are relative. Again, + # relative paths are relative to ${CMAKE_CURRENT_BINARY_DIR} + def ensure_absolute(x: Path) -> Path: + if x.is_absolute(): + return x + else: + return self.current_bin_dir / x + self.original_outputs = [ensure_absolute(x) for x in self.original_outputs] + + # Ensure that there is no duplicate output in the project so + # that meson can handle cases where the same filename is + # generated in multiple directories + temp_outputs = [] # type: T.List[str] + for i in self.outputs: + if i in all_outputs: + old = str(i) + i = f'c{ConverterCustomTarget.out_counter}_{i}' + ConverterCustomTarget.out_counter += 1 + self.conflict_map[old] = i + all_outputs += [i] + temp_outputs += [i] + self.outputs = temp_outputs + + # Check if the command is a build target + commands = [] # type: T.List[T.List[T.Union[str, ConverterTarget]]] + for curr_cmd in self._raw_target.command: + assert(isinstance(curr_cmd, list)) + cmd = [] # type: T.List[T.Union[str, ConverterTarget]] + + for j in curr_cmd: + if not j: + continue + target = output_target_map.executable(j) + if target: + # When cross compiling, binaries have to be executed with an exe_wrapper (for instance wine for mingw-w64) + if self.env.exe_wrapper is not None and self.env.properties[self.for_machine].get_cmake_use_exe_wrapper(): + assert isinstance(self.env.exe_wrapper, ExternalProgram) + cmd += self.env.exe_wrapper.get_command() + cmd += [target] + continue + elif j in trace.targets: + trace_tgt = trace.targets[j] + if trace_tgt.type == 'EXECUTABLE' and 'IMPORTED_LOCATION' in trace_tgt.properties: + cmd += trace_tgt.properties['IMPORTED_LOCATION'] + continue + mlog.debug(f'CMake: Found invalid CMake target "{j}" --> ignoring \n{trace_tgt}') + + # Fallthrough on error + cmd += [j] + + commands += [cmd] + self.command = commands + + # If the custom target does not declare any output, create a dummy + # one that can be used as dependency. + if not self.outputs: + self.outputs = [self.name + '.h'] + + # Check dependencies and input files + for i in self.depends_raw: + if not i: + continue + raw = Path(i) + art = output_target_map.artifact(i) + tgt = output_target_map.target(i) + gen = output_target_map.generated(raw) + + rel_to_root = None + try: + rel_to_root = raw.relative_to(root_src_dir) + except ValueError: + rel_to_root = None + + # First check for existing files. Only then check for existing + # targets, etc. This reduces the chance of misdetecting input files + # as outputs from other targets. + # See https://github.com/mesonbuild/meson/issues/6632 + if not raw.is_absolute() and (self.current_src_dir / raw).exists(): + self.inputs += [(self.current_src_dir / raw).relative_to(root_src_dir).as_posix()] + elif raw.is_absolute() and raw.exists() and rel_to_root is not None: + self.inputs += [rel_to_root.as_posix()] + elif art: + self.depends += [art] + elif tgt: + self.depends += [tgt] + elif gen: + ctgt_ref = gen.get_ref(raw) + assert ctgt_ref is not None + self.inputs += [ctgt_ref] + + def process_inter_target_dependencies(self) -> None: + # Move the dependencies from all transfer_dependencies_from to the target + to_process = list(self.depends) + processed = [] + new_deps = [] + for i in to_process: + processed += [i] + if isinstance(i, ConverterTarget) and i.meson_func() in transfer_dependencies_from: + to_process += [x for x in i.depends if x not in processed] + else: + new_deps += [i] + self.depends = list(OrderedSet(new_deps)) + + def get_ref(self, fname: Path) -> T.Optional[CustomTargetReference]: + name = fname.name + try: + if name in self.conflict_map: + name = self.conflict_map[name] + idx = self.outputs.index(name) + return CustomTargetReference(self, idx) + except ValueError: + return None + + def log(self) -> None: + mlog.log('Custom Target', mlog.bold(self.name), f'({self.cmake_name})') + mlog.log(' -- command: ', mlog.bold(str(self.command))) + mlog.log(' -- outputs: ', mlog.bold(str(self.outputs))) + mlog.log(' -- conflict_map: ', mlog.bold(str(self.conflict_map))) + mlog.log(' -- working_dir: ', mlog.bold(str(self.working_dir))) + mlog.log(' -- depends_raw: ', mlog.bold(str(self.depends_raw))) + mlog.log(' -- inputs: ', mlog.bold(str(self.inputs))) + mlog.log(' -- depends: ', mlog.bold(str(self.depends))) + +class CMakeAPI(Enum): + SERVER = 1 + FILE = 2 + +class CMakeInterpreter: + def __init__(self, build: 'Build', subdir: Path, src_dir: Path, install_prefix: Path, env: 'Environment', backend: 'Backend'): + self.build = build + self.subdir = subdir + self.src_dir = src_dir + self.build_dir_rel = subdir / '__CMake_build' + self.build_dir = Path(env.get_build_dir()) / self.build_dir_rel + self.install_prefix = install_prefix + self.env = env + self.for_machine = MachineChoice.HOST # TODO make parameter + self.backend_name = backend.name + self.linkers = set() # type: T.Set[str] + self.cmake_api = CMakeAPI.SERVER + self.client = CMakeClient(self.env) + self.fileapi = CMakeFileAPI(self.build_dir) + + # Raw CMake results + self.bs_files = [] # type: T.List[Path] + self.codemodel_configs = None # type: T.Optional[T.List[CMakeConfiguration]] + self.raw_trace = None # type: T.Optional[str] + + # Analysed data + self.project_name = '' + self.languages = [] # type: T.List[str] + self.targets = [] # type: T.List[ConverterTarget] + self.custom_targets = [] # type: T.List[ConverterCustomTarget] + self.trace = CMakeTraceParser('', Path('.')) # Will be replaced in analyse + self.output_target_map = OutputTargetMap(self.build_dir) + + # Generated meson data + self.generated_targets = {} # type: T.Dict[str, T.Dict[str, T.Optional[str]]] + self.internal_name_map = {} # type: T.Dict[str, str] + + # Do some special handling for object libraries for certain configurations + self._object_lib_workaround = False + if self.backend_name.startswith('vs'): + for comp in self.env.coredata.compilers[self.for_machine].values(): + if comp.get_linker_id() == 'link': + self._object_lib_workaround = True + break + + def configure(self, extra_cmake_options: T.List[str]) -> CMakeExecutor: + # Find CMake + # TODO: Using MachineChoice.BUILD should always be correct here, but also evaluate the use of self.for_machine + cmake_exe = CMakeExecutor(self.env, '>=3.7', MachineChoice.BUILD) + if not cmake_exe.found(): + raise CMakeException('Unable to find CMake') + self.trace = CMakeTraceParser(cmake_exe.version(), self.build_dir, permissive=True) + + preload_file = mesondata['cmake/data/preload.cmake'].write_to_private(self.env) + toolchain = CMakeToolchain(cmake_exe, self.env, self.for_machine, CMakeExecScope.SUBPROJECT, self.build_dir, preload_file) + toolchain_file = toolchain.write() + + # TODO: drop this check once the deprecated `cmake_args` kwarg is removed + extra_cmake_options = check_cmake_args(extra_cmake_options) + + cmake_args = [] + cmake_args += cmake_get_generator_args(self.env) + cmake_args += [f'-DCMAKE_INSTALL_PREFIX={self.install_prefix}'] + cmake_args += extra_cmake_options + trace_args = self.trace.trace_args() + cmcmp_args = [f'-DCMAKE_POLICY_WARNING_{x}=OFF' for x in disable_policy_warnings] + + if version_compare(cmake_exe.version(), '>=3.14'): + self.cmake_api = CMakeAPI.FILE + self.fileapi.setup_request() + + # Run CMake + mlog.log() + with mlog.nested(): + mlog.log('Configuring the build directory with', mlog.bold('CMake'), 'version', mlog.cyan(cmake_exe.version())) + mlog.log(mlog.bold('Running CMake with:'), ' '.join(cmake_args)) + mlog.log(mlog.bold(' - build directory: '), self.build_dir.as_posix()) + mlog.log(mlog.bold(' - source directory: '), self.src_dir.as_posix()) + mlog.log(mlog.bold(' - toolchain file: '), toolchain_file.as_posix()) + mlog.log(mlog.bold(' - preload file: '), preload_file.as_posix()) + mlog.log(mlog.bold(' - trace args: '), ' '.join(trace_args)) + mlog.log(mlog.bold(' - disabled policy warnings:'), '[{}]'.format(', '.join(disable_policy_warnings))) + mlog.log() + self.build_dir.mkdir(parents=True, exist_ok=True) + os_env = environ.copy() + os_env['LC_ALL'] = 'C' + final_args = cmake_args + trace_args + cmcmp_args + toolchain.get_cmake_args() + [self.src_dir.as_posix()] + + cmake_exe.set_exec_mode(print_cmout=True, always_capture_stderr=self.trace.requires_stderr()) + rc, _, self.raw_trace = cmake_exe.call(final_args, self.build_dir, env=os_env, disable_cache=True) + + mlog.log() + h = mlog.green('SUCCEEDED') if rc == 0 else mlog.red('FAILED') + mlog.log('CMake configuration:', h) + if rc != 0: + raise CMakeException('Failed to configure the CMake subproject') + + return cmake_exe + + def initialise(self, extra_cmake_options: T.List[str]) -> None: + # Run configure the old way because doing it + # with the server doesn't work for some reason + # Additionally, the File API requires a configure anyway + cmake_exe = self.configure(extra_cmake_options) + + # Continue with the file API If supported + if self.cmake_api is CMakeAPI.FILE: + # Parse the result + self.fileapi.load_reply() + + # Load the buildsystem file list + cmake_files = self.fileapi.get_cmake_sources() + self.bs_files = [x.file for x in cmake_files if not x.is_cmake and not x.is_temp] + self.bs_files = [relative_to_if_possible(x, Path(self.env.get_source_dir())) for x in self.bs_files] + self.bs_files = [x for x in self.bs_files if not path_is_in_root(x, Path(self.env.get_build_dir()), resolve=True)] + self.bs_files = list(OrderedSet(self.bs_files)) + + # Load the codemodel configurations + self.codemodel_configs = self.fileapi.get_cmake_configurations() + return + + with self.client.connect(cmake_exe): + generator = backend_generator_map[self.backend_name] + self.client.do_handshake(self.src_dir, self.build_dir, generator, 1) + + # Do a second configure to initialise the server + self.client.query_checked(RequestConfigure(), 'CMake server configure') + + # Generate the build system files + self.client.query_checked(RequestCompute(), 'Generating build system files') + + # Get CMake build system files + bs_reply = self.client.query_checked(RequestCMakeInputs(), 'Querying build system files') + assert isinstance(bs_reply, ReplyCMakeInputs) + + # Now get the CMake code model + cm_reply = self.client.query_checked(RequestCodeModel(), 'Querying the CMake code model') + assert isinstance(cm_reply, ReplyCodeModel) + + src_dir = bs_reply.src_dir + self.bs_files = [x.file for x in bs_reply.build_files if not x.is_cmake and not x.is_temp] + self.bs_files = [relative_to_if_possible(src_dir / x, Path(self.env.get_source_dir()), resolve=True) for x in self.bs_files] + self.bs_files = [x for x in self.bs_files if not path_is_in_root(x, Path(self.env.get_build_dir()), resolve=True)] + self.bs_files = list(OrderedSet(self.bs_files)) + self.codemodel_configs = cm_reply.configs + + def analyse(self) -> None: + if self.codemodel_configs is None: + raise CMakeException('CMakeInterpreter was not initialized') + + # Clear analyser data + self.project_name = '' + self.languages = [] + self.targets = [] + self.custom_targets = [] + + # Parse the trace + self.trace.parse(self.raw_trace) + + # Find all targets + added_target_names = [] # type: T.List[str] + for i_0 in self.codemodel_configs: + for j_0 in i_0.projects: + if not self.project_name: + self.project_name = j_0.name + for k_0 in j_0.targets: + # Avoid duplicate targets from different configurations and known + # dummy CMake internal target types + if k_0.type not in skip_targets and k_0.name not in added_target_names: + added_target_names += [k_0.name] + self.targets += [ConverterTarget(k_0, self.env, self.for_machine)] + + # Add interface targets from trace, if not already present. + # This step is required because interface targets were removed from + # the CMake file API output. + api_target_name_list = [x.name for x in self.targets] + for i_1 in self.trace.targets.values(): + if i_1.type != 'INTERFACE' or i_1.name in api_target_name_list or i_1.imported: + continue + dummy = CMakeTarget({ + 'name': i_1.name, + 'type': 'INTERFACE_LIBRARY', + 'sourceDirectory': self.src_dir, + 'buildDirectory': self.build_dir, + }) + self.targets += [ConverterTarget(dummy, self.env, self.for_machine)] + + for i_2 in self.trace.custom_targets: + self.custom_targets += [ConverterCustomTarget(i_2, self.env, self.for_machine)] + + # generate the output_target_map + for i_3 in [*self.targets, *self.custom_targets]: + assert isinstance(i_3, (ConverterTarget, ConverterCustomTarget)) + self.output_target_map.add(i_3) + + # First pass: Basic target cleanup + object_libs = [] + custom_target_outputs = [] # type: T.List[str] + for ctgt in self.custom_targets: + ctgt.postprocess(self.output_target_map, self.src_dir, custom_target_outputs, self.trace) + for tgt in self.targets: + tgt.postprocess(self.output_target_map, self.src_dir, self.subdir, self.install_prefix, self.trace) + if tgt.type == 'OBJECT_LIBRARY': + object_libs += [tgt] + self.languages += [x for x in tgt.languages if x not in self.languages] + + # Second pass: Detect object library dependencies + for tgt in self.targets: + tgt.process_object_libs(object_libs, self._object_lib_workaround) + + # Third pass: Reassign dependencies to avoid some loops + for tgt in self.targets: + tgt.process_inter_target_dependencies() + for ctgt in self.custom_targets: + ctgt.process_inter_target_dependencies() + + # Fourth pass: Remove rassigned dependencies + for tgt in self.targets: + tgt.cleanup_dependencies() + + mlog.log('CMake project', mlog.bold(self.project_name), 'has', mlog.bold(str(len(self.targets) + len(self.custom_targets))), 'build targets.') + + def pretend_to_be_meson(self, options: TargetOptions) -> CodeBlockNode: + if not self.project_name: + raise CMakeException('CMakeInterpreter was not analysed') + + def token(tid: str = 'string', val: TYPE_mixed = '') -> Token: + return Token(tid, self.subdir.as_posix(), 0, 0, 0, None, val) + + def string(value: str) -> StringNode: + return StringNode(token(val=value)) + + def id_node(value: str) -> IdNode: + return IdNode(token(val=value)) + + def number(value: int) -> NumberNode: + return NumberNode(token(val=value)) + + def nodeify(value: TYPE_mixed_list) -> BaseNode: + if isinstance(value, str): + return string(value) + if isinstance(value, Path): + return string(value.as_posix()) + elif isinstance(value, bool): + return BooleanNode(token(val=value)) + elif isinstance(value, int): + return number(value) + elif isinstance(value, list): + return array(value) + elif isinstance(value, BaseNode): + return value + raise RuntimeError('invalid type of value: {} ({})'.format(type(value).__name__, str(value))) + + def indexed(node: BaseNode, index: int) -> IndexNode: + return IndexNode(node, nodeify(index)) + + def array(elements: TYPE_mixed_list) -> ArrayNode: + args = ArgumentNode(token()) + if not isinstance(elements, list): + elements = [args] + args.arguments += [nodeify(x) for x in elements if x is not None] + return ArrayNode(args, 0, 0, 0, 0) + + def function(name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> FunctionNode: + args = [] if args is None else args + kwargs = {} if kwargs is None else kwargs + args_n = ArgumentNode(token()) + if not isinstance(args, list): + assert isinstance(args, (str, int, bool, Path, BaseNode)) + args = [args] + args_n.arguments = [nodeify(x) for x in args if x is not None] + args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} + func_n = FunctionNode(self.subdir.as_posix(), 0, 0, 0, 0, name, args_n) + return func_n + + def method(obj: BaseNode, name: str, args: T.Optional[TYPE_mixed_list] = None, kwargs: T.Optional[TYPE_mixed_kwargs] = None) -> MethodNode: + args = [] if args is None else args + kwargs = {} if kwargs is None else kwargs + args_n = ArgumentNode(token()) + if not isinstance(args, list): + assert isinstance(args, (str, int, bool, Path, BaseNode)) + args = [args] + args_n.arguments = [nodeify(x) for x in args if x is not None] + args_n.kwargs = {id_node(k): nodeify(v) for k, v in kwargs.items() if v is not None} + return MethodNode(self.subdir.as_posix(), 0, 0, obj, name, args_n) + + def assign(var_name: str, value: BaseNode) -> AssignmentNode: + return AssignmentNode(self.subdir.as_posix(), 0, 0, var_name, value) + + # Generate the root code block and the project function call + root_cb = CodeBlockNode(token()) + root_cb.lines += [function('project', [self.project_name] + self.languages)] + + # Add the run script for custom commands + + # Add the targets + processing = [] # type: T.List[str] + processed = {} # type: T.Dict[str, T.Dict[str, T.Optional[str]]] + name_map = {} # type: T.Dict[str, str] + + def extract_tgt(tgt: T.Union[ConverterTarget, ConverterCustomTarget, CustomTargetReference]) -> IdNode: + tgt_name = None + if isinstance(tgt, (ConverterTarget, ConverterCustomTarget)): + tgt_name = tgt.name + elif isinstance(tgt, CustomTargetReference): + tgt_name = tgt.ctgt.name + assert(tgt_name is not None and tgt_name in processed) + res_var = processed[tgt_name]['tgt'] + return id_node(res_var) if res_var else None + + def detect_cycle(tgt: T.Union[ConverterTarget, ConverterCustomTarget]) -> None: + if tgt.name in processing: + raise CMakeException('Cycle in CMake inputs/dependencies detected') + processing.append(tgt.name) + + def resolve_ctgt_ref(ref: CustomTargetReference) -> T.Union[IdNode, IndexNode]: + tgt_var = extract_tgt(ref) + if len(ref.ctgt.outputs) == 1: + return tgt_var + else: + return indexed(tgt_var, ref.index) + + def process_target(tgt: ConverterTarget) -> None: + detect_cycle(tgt) + + # First handle inter target dependencies + link_with = [] # type: T.List[IdNode] + objec_libs = [] # type: T.List[IdNode] + sources = [] # type: T.List[Path] + generated = [] # type: T.List[T.Union[IdNode, IndexNode]] + generated_filenames = [] # type: T.List[str] + custom_targets = [] # type: T.List[ConverterCustomTarget] + dependencies = [] # type: T.List[IdNode] + for i in tgt.link_with: + assert(isinstance(i, ConverterTarget)) + if i.name not in processed: + process_target(i) + link_with += [extract_tgt(i)] + for i in tgt.object_libs: + assert(isinstance(i, ConverterTarget)) + if i.name not in processed: + process_target(i) + objec_libs += [extract_tgt(i)] + for i in tgt.depends: + if not isinstance(i, ConverterCustomTarget): + continue + if i.name not in processed: + process_custom_target(i) + dependencies += [extract_tgt(i)] + + # Generate the source list and handle generated sources + sources += tgt.sources + sources += tgt.generated + + for ctgt_ref in tgt.generated_ctgt: + ctgt = ctgt_ref.ctgt + if ctgt.name not in processed: + process_custom_target(ctgt) + generated += [resolve_ctgt_ref(ctgt_ref)] + generated_filenames += [ctgt_ref.filename()] + if ctgt not in custom_targets: + custom_targets += [ctgt] + + # Add all header files from all used custom targets. This + # ensures that all custom targets are built before any + # sources of the current target are compiled and thus all + # header files are present. This step is necessary because + # CMake always ensures that a custom target is executed + # before another target if at least one output is used. + for ctgt in custom_targets: + for j in ctgt.outputs: + if not is_header(j) or j in generated_filenames: + continue + + generated += [resolve_ctgt_ref(ctgt.get_ref(Path(j)))] + generated_filenames += [j] + + # Determine the meson function to use for the build target + tgt_func = tgt.meson_func() + if not tgt_func: + raise CMakeException(f'Unknown target type "{tgt.type}"') + + # Determine the variable names + inc_var = f'{tgt.name}_inc' + dir_var = f'{tgt.name}_dir' + sys_var = f'{tgt.name}_sys' + src_var = f'{tgt.name}_src' + dep_var = f'{tgt.name}_dep' + tgt_var = tgt.name + + install_tgt = options.get_install(tgt.cmake_name, tgt.install) + + # Generate target kwargs + tgt_kwargs = { + 'build_by_default': install_tgt, + 'link_args': options.get_link_args(tgt.cmake_name, tgt.link_flags + tgt.link_libraries), + 'link_with': link_with, + 'include_directories': id_node(inc_var), + 'install': install_tgt, + 'override_options': options.get_override_options(tgt.cmake_name, tgt.override_options), + 'objects': [method(x, 'extract_all_objects') for x in objec_libs], + } # type: TYPE_mixed_kwargs + + # Only set if installed and only override if it is set + if install_tgt and tgt.install_dir: + tgt_kwargs['install_dir'] = tgt.install_dir + + # Handle compiler args + for key, val in tgt.compile_opts.items(): + tgt_kwargs[f'{key}_args'] = options.get_compile_args(tgt.cmake_name, key, val) + + # Handle -fPCI, etc + if tgt_func == 'executable': + tgt_kwargs['pie'] = tgt.pie + elif tgt_func == 'static_library': + tgt_kwargs['pic'] = tgt.pie + + # declare_dependency kwargs + dep_kwargs = { + 'link_args': tgt.link_flags + tgt.link_libraries, + 'link_with': id_node(tgt_var), + 'compile_args': tgt.public_compile_opts, + 'include_directories': id_node(inc_var), + } # type: TYPE_mixed_kwargs + + if dependencies: + generated += dependencies + + # Generate the function nodes + dir_node = assign(dir_var, function('include_directories', tgt.includes)) + sys_node = assign(sys_var, function('include_directories', tgt.sys_includes, {'is_system': True})) + inc_node = assign(inc_var, array([id_node(dir_var), id_node(sys_var)])) + node_list = [dir_node, sys_node, inc_node] + if tgt_func == 'header_only': + del dep_kwargs['link_with'] + dep_node = assign(dep_var, function('declare_dependency', kwargs=dep_kwargs)) + node_list += [dep_node] + src_var = None + tgt_var = None + else: + src_node = assign(src_var, function('files', sources)) + tgt_node = assign(tgt_var, function(tgt_func, [tgt_var, id_node(src_var), *generated], tgt_kwargs)) + node_list += [src_node, tgt_node] + if tgt_func in ['static_library', 'shared_library']: + dep_node = assign(dep_var, function('declare_dependency', kwargs=dep_kwargs)) + node_list += [dep_node] + elif tgt_func in ['shared_module']: + del dep_kwargs['link_with'] + dep_node = assign(dep_var, function('declare_dependency', kwargs=dep_kwargs)) + node_list += [dep_node] + else: + dep_var = None + + # Add the nodes to the ast + root_cb.lines += node_list + processed[tgt.name] = {'inc': inc_var, 'src': src_var, 'dep': dep_var, 'tgt': tgt_var, 'func': tgt_func} + name_map[tgt.cmake_name] = tgt.name + + def process_custom_target(tgt: ConverterCustomTarget) -> None: + # CMake allows to specify multiple commands in a custom target. + # To map this to meson, a helper script is used to execute all + # commands in order. This additionally allows setting the working + # directory. + + detect_cycle(tgt) + tgt_var = tgt.name # type: str + + def resolve_source(x: T.Union[str, ConverterTarget, ConverterCustomTarget, CustomTargetReference]) -> T.Union[str, IdNode, IndexNode]: + if isinstance(x, ConverterTarget): + if x.name not in processed: + process_target(x) + return extract_tgt(x) + if isinstance(x, ConverterCustomTarget): + if x.name not in processed: + process_custom_target(x) + return extract_tgt(x) + elif isinstance(x, CustomTargetReference): + if x.ctgt.name not in processed: + process_custom_target(x.ctgt) + return resolve_ctgt_ref(x) + else: + return x + + # Generate the command list + command = [] # type: T.List[T.Union[str, IdNode, IndexNode]] + command += mesonlib.get_meson_command() + command += ['--internal', 'cmake_run_ctgt'] + command += ['-o', '@OUTPUT@'] + if tgt.original_outputs: + command += ['-O'] + [x.as_posix() for x in tgt.original_outputs] + command += ['-d', tgt.working_dir.as_posix()] + + # Generate the commands. Subcommands are separated by ';;;' + for cmd in tgt.command: + command += [resolve_source(x) for x in cmd] + [';;;'] + + tgt_kwargs = { + 'input': [resolve_source(x) for x in tgt.inputs], + 'output': tgt.outputs, + 'command': command, + 'depends': [resolve_source(x) for x in tgt.depends], + } # type: TYPE_mixed_kwargs + + root_cb.lines += [assign(tgt_var, function('custom_target', [tgt.name], tgt_kwargs))] + processed[tgt.name] = {'inc': None, 'src': None, 'dep': None, 'tgt': tgt_var, 'func': 'custom_target'} + name_map[tgt.cmake_name] = tgt.name + + # Now generate the target function calls + for ctgt in self.custom_targets: + if ctgt.name not in processed: + process_custom_target(ctgt) + for tgt in self.targets: + if tgt.name not in processed: + process_target(tgt) + + self.generated_targets = processed + self.internal_name_map = name_map + return root_cb + + def target_info(self, target: str) -> T.Optional[T.Dict[str, str]]: + # Try resolving the target name + # start by checking if there is a 100% match (excluding the name prefix) + prx_tgt = _sanitize_cmake_name(target) + if prx_tgt in self.generated_targets: + return self.generated_targets[prx_tgt] + # check if there exists a name mapping + if target in self.internal_name_map: + target = self.internal_name_map[target] + assert(target in self.generated_targets) + return self.generated_targets[target] + return None + + def target_list(self) -> T.List[str]: + return list(self.internal_name_map.keys()) diff --git a/meson/mesonbuild/cmake/toolchain.py b/meson/mesonbuild/cmake/toolchain.py new file mode 100644 index 000000000..34b737c79 --- /dev/null +++ b/meson/mesonbuild/cmake/toolchain.py @@ -0,0 +1,259 @@ +# Copyright 2020 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +from .traceparser import CMakeTraceParser +from ..envconfig import CMakeSkipCompilerTest +from ..mesonlib import MachineChoice +from ..compilers import VisualStudioLikeCompiler +from .common import language_map, cmake_get_generator_args +from .. import mlog + +import shutil +import typing as T +from enum import Enum +from textwrap import dedent + +if T.TYPE_CHECKING: + from .executor import CMakeExecutor + from ..envconfig import MachineInfo, Properties, CMakeVariables + from ..environment import Environment + from ..compilers import Compiler + +class CMakeExecScope(Enum): + SUBPROJECT = 'subproject' + DEPENDENCY = 'dependency' + +class CMakeToolchain: + def __init__(self, cmakebin: 'CMakeExecutor', env: 'Environment', for_machine: MachineChoice, exec_scope: CMakeExecScope, build_dir: Path, preload_file: T.Optional[Path] = None) -> None: + self.env = env + self.cmakebin = cmakebin + self.for_machine = for_machine + self.exec_scope = exec_scope + self.preload_file = preload_file + self.build_dir = build_dir + self.build_dir = self.build_dir.resolve() + self.toolchain_file = build_dir / 'CMakeMesonToolchainFile.cmake' + self.cmcache_file = build_dir / 'CMakeCache.txt' + self.minfo = self.env.machines[self.for_machine] + self.properties = self.env.properties[self.for_machine] + self.compilers = self.env.coredata.compilers[self.for_machine] + self.cmakevars = self.env.cmakevars[self.for_machine] + self.cmakestate = self.env.coredata.cmake_cache[self.for_machine] + + self.variables = self.get_defaults() + self.variables.update(self.cmakevars.get_variables()) + + # Determine whether CMake the compiler test should be skipped + skip_status = self.properties.get_cmake_skip_compiler_test() + self.skip_check = skip_status == CMakeSkipCompilerTest.ALWAYS + if skip_status == CMakeSkipCompilerTest.DEP_ONLY and self.exec_scope == CMakeExecScope.DEPENDENCY: + self.skip_check = True + if not self.properties.get_cmake_defaults(): + self.skip_check = False + + assert self.toolchain_file.is_absolute() + + def write(self) -> Path: + if not self.toolchain_file.parent.exists(): + self.toolchain_file.parent.mkdir(parents=True) + self.toolchain_file.write_text(self.generate(), encoding='utf-8') + self.cmcache_file.write_text(self.generate_cache(), encoding='utf-8') + mlog.cmd_ci_include(self.toolchain_file.as_posix()) + return self.toolchain_file + + def get_cmake_args(self) -> T.List[str]: + args = ['-DCMAKE_TOOLCHAIN_FILE=' + self.toolchain_file.as_posix()] + if self.preload_file is not None: + args += ['-DMESON_PRELOAD_FILE=' + self.preload_file.as_posix()] + return args + + @staticmethod + def _print_vars(vars: T.Dict[str, T.List[str]]) -> str: + res = '' + for key, value in vars.items(): + res += 'set(' + key + for i in value: + res += f' "{i}"' + res += ')\n' + return res + + def generate(self) -> str: + res = dedent('''\ + ###################################### + ### AUTOMATICALLY GENERATED FILE ### + ###################################### + + # This file was generated from the configuration in the + # relevant meson machine file. See the meson documentation + # https://mesonbuild.com/Machine-files.html for more information + + if(DEFINED MESON_PRELOAD_FILE) + include("${MESON_PRELOAD_FILE}") + endif() + + ''') + + # Escape all \ in the values + for key, value in self.variables.items(): + self.variables[key] = [x.replace('\\', '/') for x in value] + + # Set compiler + if self.skip_check: + self.update_cmake_compiler_state() + res += '# CMake compiler state variables\n' + for lang, vars in self.cmakestate: + res += f'# -- Variables for language {lang}\n' + res += self._print_vars(vars) + res += '\n' + res += '\n' + + # Set variables from the current machine config + res += '# Variables from meson\n' + res += self._print_vars(self.variables) + res += '\n' + + # Add the user provided toolchain file + user_file = self.properties.get_cmake_toolchain_file() + if user_file is not None: + res += dedent(''' + # Load the CMake toolchain file specified by the user + include("{}") + + '''.format(user_file.as_posix())) + + return res + + def generate_cache(self) -> str: + if not self.skip_check: + return '' + + res = '' + for name, v in self.cmakestate.cmake_cache.items(): + res += f'{name}:{v.type}={";".join(v.value)}\n' + return res + + def get_defaults(self) -> T.Dict[str, T.List[str]]: + defaults = {} # type: T.Dict[str, T.List[str]] + + # Do nothing if the user does not want automatic defaults + if not self.properties.get_cmake_defaults(): + return defaults + + # Best effort to map the meson system name to CMAKE_SYSTEM_NAME, which + # is not trivial since CMake lacks a list of all supported + # CMAKE_SYSTEM_NAME values. + SYSTEM_MAP = { + 'android': 'Android', + 'linux': 'Linux', + 'windows': 'Windows', + 'freebsd': 'FreeBSD', + 'darwin': 'Darwin', + } # type: T.Dict[str, str] + + # Only set these in a cross build. Otherwise CMake will trip up in native + # builds and thing they are cross (which causes TRY_RUN() to break) + if self.env.is_cross_build(when_building_for=self.for_machine): + defaults['CMAKE_SYSTEM_NAME'] = [SYSTEM_MAP.get(self.minfo.system, self.minfo.system)] + defaults['CMAKE_SYSTEM_PROCESSOR'] = [self.minfo.cpu_family] + + defaults['CMAKE_SIZEOF_VOID_P'] = ['8' if self.minfo.is_64_bit else '4'] + + sys_root = self.properties.get_sys_root() + if sys_root: + defaults['CMAKE_SYSROOT'] = [sys_root] + + def make_abs(exe: str) -> str: + if Path(exe).is_absolute(): + return exe + + p = shutil.which(exe) + if p is None: + return exe + return p + + # Set the compiler variables + for lang, comp_obj in self.compilers.items(): + prefix = 'CMAKE_{}_'.format(language_map.get(lang, lang.upper())) + + exe_list = comp_obj.get_exelist() + if not exe_list: + continue + + if len(exe_list) >= 2 and not self.is_cmdline_option(comp_obj, exe_list[1]): + defaults[prefix + 'COMPILER_LAUNCHER'] = [make_abs(exe_list[0])] + exe_list = exe_list[1:] + + exe_list[0] = make_abs(exe_list[0]) + defaults[prefix + 'COMPILER'] = exe_list + if comp_obj.get_id() == 'clang-cl': + defaults['CMAKE_LINKER'] = comp_obj.get_linker_exelist() + + return defaults + + @staticmethod + def is_cmdline_option(compiler: 'Compiler', arg: str) -> bool: + if isinstance(compiler, VisualStudioLikeCompiler): + return arg.startswith('/') + else: + return arg.startswith('-') + + def update_cmake_compiler_state(self) -> None: + # Check if all variables are already cached + if self.cmakestate.languages.issuperset(self.compilers.keys()): + return + + # Generate the CMakeLists.txt + mlog.debug('CMake Toolchain: Calling CMake once to generate the compiler state') + languages = list(self.compilers.keys()) + lang_ids = [language_map.get(x, x.upper()) for x in languages] + cmake_content = dedent(f''' + cmake_minimum_required(VERSION 3.7) + project(CompInfo {' '.join(lang_ids)}) + ''') + + build_dir = Path(self.env.scratch_dir) / '__CMake_compiler_info__' + build_dir.mkdir(parents=True, exist_ok=True) + cmake_file = build_dir / 'CMakeLists.txt' + cmake_file.write_text(cmake_content, encoding='utf-8') + + # Generate the temporary toolchain file + temp_toolchain_file = build_dir / 'CMakeMesonTempToolchainFile.cmake' + temp_toolchain_file.write_text(CMakeToolchain._print_vars(self.variables), encoding='utf-8') + + # Configure + trace = CMakeTraceParser(self.cmakebin.version(), build_dir) + self.cmakebin.set_exec_mode(print_cmout=False, always_capture_stderr=trace.requires_stderr()) + cmake_args = [] + cmake_args += trace.trace_args() + cmake_args += cmake_get_generator_args(self.env) + cmake_args += [f'-DCMAKE_TOOLCHAIN_FILE={temp_toolchain_file.as_posix()}', '.'] + rc, _, raw_trace = self.cmakebin.call(cmake_args, build_dir=build_dir, disable_cache=True) + + if rc != 0: + mlog.warning('CMake Toolchain: Failed to determine CMake compilers state') + return + + # Parse output + trace.parse(raw_trace) + self.cmakestate.cmake_cache = {**trace.cache} + + vars_by_file = {k.name: v for (k, v) in trace.vars_by_file.items()} + + for lang in languages: + lang_cmake = language_map.get(lang, lang.upper()) + file_name = f'CMake{lang_cmake}Compiler.cmake' + vars = vars_by_file.setdefault(file_name, {}) + vars[f'CMAKE_{lang_cmake}_COMPILER_FORCED'] = ['1'] + self.cmakestate.update(lang, vars) diff --git a/meson/mesonbuild/cmake/traceparser.py b/meson/mesonbuild/cmake/traceparser.py new file mode 100644 index 000000000..4ddc91533 --- /dev/null +++ b/meson/mesonbuild/cmake/traceparser.py @@ -0,0 +1,756 @@ +# Copyright 2019 The Meson development team + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This class contains the basic functionality needed to run any interpreter +# or an interpreter-based tool. + +from .common import CMakeException +from .generator import parse_generator_expressions +from .. import mlog +from ..mesonlib import version_compare + +import typing as T +from pathlib import Path +from functools import lru_cache +import re +import json +import textwrap + +class CMakeTraceLine: + def __init__(self, file_str: str, line: int, func: str, args: T.List[str]) -> None: + self.file = CMakeTraceLine._to_path(file_str) + self.line = line + self.func = func.lower() + self.args = args + + @staticmethod + @lru_cache(maxsize=None) + def _to_path(file_str: str) -> Path: + return Path(file_str) + + def __repr__(self) -> str: + s = 'CMake TRACE: {0}:{1} {2}({3})' + return s.format(self.file, self.line, self.func, self.args) + +class CMakeCacheEntry(T.NamedTuple): + value: T.List[str] + type: str + +class CMakeTarget: + def __init__( + self, + name: str, + target_type: str, + properties: T.Optional[T.Dict[str, T.List[str]]] = None, + imported: bool = False, + tline: T.Optional[CMakeTraceLine] = None + ): + if properties is None: + properties = {} + self.name = name + self.type = target_type + self.properties = properties + self.imported = imported + self.tline = tline + self.depends = [] # type: T.List[str] + self.current_bin_dir = None # type: T.Optional[Path] + self.current_src_dir = None # type: T.Optional[Path] + + def __repr__(self) -> str: + s = 'CMake TARGET:\n -- name: {}\n -- type: {}\n -- imported: {}\n -- properties: {{\n{} }}\n -- tline: {}' + propSTR = '' + for i in self.properties: + propSTR += " '{}': {}\n".format(i, self.properties[i]) + return s.format(self.name, self.type, self.imported, propSTR, self.tline) + + def strip_properties(self) -> None: + # Strip the strings in the properties + if not self.properties: + return + for key, val in self.properties.items(): + self.properties[key] = [x.strip() for x in val] + assert all([';' not in x for x in self.properties[key]]) + +class CMakeGeneratorTarget(CMakeTarget): + def __init__(self, name: str) -> None: + super().__init__(name, 'CUSTOM', {}) + self.outputs = [] # type: T.List[Path] + self.command = [] # type: T.List[T.List[str]] + self.working_dir = None # type: T.Optional[Path] + +class CMakeTraceParser: + def __init__(self, cmake_version: str, build_dir: Path, permissive: bool = True) -> None: + self.vars: T.Dict[str, T.List[str]] = {} + self.vars_by_file: T.Dict[Path, T.Dict[str, T.List[str]]] = {} + self.targets: T.Dict[str, CMakeTarget] = {} + self.cache: T.Dict[str, CMakeCacheEntry] = {} + + self.explicit_headers = set() # type: T.Set[Path] + + # T.List of targes that were added with add_custom_command to generate files + self.custom_targets = [] # type: T.List[CMakeGeneratorTarget] + + self.permissive = permissive # type: bool + self.cmake_version = cmake_version # type: str + self.trace_file = 'cmake_trace.txt' + self.trace_file_path = build_dir / self.trace_file + self.trace_format = 'json-v1' if version_compare(cmake_version, '>=3.17') else 'human' + + # State for delayed command execution. Delayed command execution is realised + # with a custom CMake file that overrides some functions and adds some + # introspection information to the trace. + self.delayed_commands = [] # type: T.List[str] + self.stored_commands = [] # type: T.List[CMakeTraceLine] + + # All supported functions + self.functions = { + 'set': self._cmake_set, + 'unset': self._cmake_unset, + 'add_executable': self._cmake_add_executable, + 'add_library': self._cmake_add_library, + 'add_custom_command': self._cmake_add_custom_command, + 'add_custom_target': self._cmake_add_custom_target, + 'set_property': self._cmake_set_property, + 'set_target_properties': self._cmake_set_target_properties, + 'target_compile_definitions': self._cmake_target_compile_definitions, + 'target_compile_options': self._cmake_target_compile_options, + 'target_include_directories': self._cmake_target_include_directories, + 'target_link_libraries': self._cmake_target_link_libraries, + 'target_link_options': self._cmake_target_link_options, + 'add_dependencies': self._cmake_add_dependencies, + + # Special functions defined in the preload script. + # These functions do nothing in the CMake code, but have special + # meaning here in the trace parser. + 'meson_ps_execute_delayed_calls': self._meson_ps_execute_delayed_calls, + 'meson_ps_reload_vars': self._meson_ps_reload_vars, + 'meson_ps_disabled_function': self._meson_ps_disabled_function, + } # type: T.Dict[str, T.Callable[[CMakeTraceLine], None]] + + def trace_args(self) -> T.List[str]: + arg_map = { + 'human': ['--trace', '--trace-expand'], + 'json-v1': ['--trace-expand', '--trace-format=json-v1'], + } + + base_args = ['--no-warn-unused-cli'] + if not self.requires_stderr(): + base_args += [f'--trace-redirect={self.trace_file}'] + + return arg_map[self.trace_format] + base_args + + def requires_stderr(self) -> bool: + return version_compare(self.cmake_version, '<3.16') + + def parse(self, trace: T.Optional[str] = None) -> None: + # First load the trace (if required) + if not self.requires_stderr(): + if not self.trace_file_path.exists and not self.trace_file_path.is_file(): + raise CMakeException(f'CMake: Trace file "{self.trace_file_path!s}" not found') + trace = self.trace_file_path.read_text(errors='ignore', encoding='utf-8') + if not trace: + raise CMakeException('CMake: The CMake trace was not provided or is empty') + + # Second parse the trace + lexer1 = None + if self.trace_format == 'human': + lexer1 = self._lex_trace_human(trace) + elif self.trace_format == 'json-v1': + lexer1 = self._lex_trace_json(trace) + else: + raise CMakeException(f'CMake: Internal error: Invalid trace format {self.trace_format}. Expected [human, json-v1]') + + # Primary pass -- parse everything + for l in lexer1: + # store the function if its execution should be delayed + if l.func in self.delayed_commands: + self.stored_commands += [l] + continue + + # "Execute" the CMake function if supported + fn = self.functions.get(l.func, None) + if(fn): + fn(l) + + # Postprocess + for tgt in self.targets.values(): + tgt.strip_properties() + + def get_first_cmake_var_of(self, var_list: T.List[str]) -> T.List[str]: + # Return the first found CMake variable in list var_list + for i in var_list: + if i in self.vars: + return self.vars[i] + + return [] + + def get_cmake_var(self, var: str) -> T.List[str]: + # Return the value of the CMake variable var or an empty list if var does not exist + if var in self.vars: + return self.vars[var] + + return [] + + def var_to_str(self, var: str) -> T.Optional[str]: + if var in self.vars and self.vars[var]: + return self.vars[var][0] + + return None + + def _str_to_bool(self, expr: T.Union[str, T.List[str]]) -> bool: + if not expr: + return False + if isinstance(expr, list): + expr_str = expr[0] + else: + expr_str = expr + expr_str = expr_str.upper() + return expr_str not in ['0', 'OFF', 'NO', 'FALSE', 'N', 'IGNORE'] and not expr_str.endswith('NOTFOUND') + + def var_to_bool(self, var: str) -> bool: + return self._str_to_bool(self.vars.get(var, [])) + + def _gen_exception(self, function: str, error: str, tline: CMakeTraceLine) -> None: + # Generate an exception if the parser is not in permissive mode + + if self.permissive: + mlog.debug(f'CMake trace warning: {function}() {error}\n{tline}') + return None + raise CMakeException(f'CMake: {function}() {error}\n{tline}') + + def _cmake_set(self, tline: CMakeTraceLine) -> None: + """Handler for the CMake set() function in all variaties. + + comes in three flavors: + set(<var> <value> [PARENT_SCOPE]) + set(<var> <value> CACHE <type> <docstring> [FORCE]) + set(ENV{<var>} <value>) + + We don't support the ENV variant, and any uses of it will be ignored + silently. the other two variates are supported, with some caveats: + - we don't properly handle scoping, so calls to set() inside a + function without PARENT_SCOPE set could incorrectly shadow the + outer scope. + - We don't honor the type of CACHE arguments + """ + # DOC: https://cmake.org/cmake/help/latest/command/set.html + + cache_type = None + cache_force = 'FORCE' in tline.args + try: + cache_idx = tline.args.index('CACHE') + cache_type = tline.args[cache_idx + 1] + except (ValueError, IndexError): + pass + + # 1st remove PARENT_SCOPE and CACHE from args + args = [] + for i in tline.args: + if not i or i == 'PARENT_SCOPE': + continue + + # Discard everything after the CACHE keyword + if i == 'CACHE': + break + + args.append(i) + + if len(args) < 1: + return self._gen_exception('set', 'requires at least one argument', tline) + + # Now that we've removed extra arguments all that should be left is the + # variable identifier and the value, join the value back together to + # ensure spaces in the value are correctly handled. This assumes that + # variable names don't have spaces. Please don't do that... + identifier = args.pop(0) + value = ' '.join(args) + + # Write to the CMake cache instead + if cache_type: + # Honor how the CMake FORCE parameter works + if identifier not in self.cache or cache_force: + self.cache[identifier] = CMakeCacheEntry(value.split(';'), cache_type) + + if not value: + # Same as unset + if identifier in self.vars: + del self.vars[identifier] + else: + self.vars[identifier] = value.split(';') + self.vars_by_file.setdefault(tline.file, {})[identifier] = value.split(';') + + def _cmake_unset(self, tline: CMakeTraceLine) -> None: + # DOC: https://cmake.org/cmake/help/latest/command/unset.html + if len(tline.args) < 1: + return self._gen_exception('unset', 'requires at least one argument', tline) + + if tline.args[0] in self.vars: + del self.vars[tline.args[0]] + + def _cmake_add_executable(self, tline: CMakeTraceLine) -> None: + # DOC: https://cmake.org/cmake/help/latest/command/add_executable.html + args = list(tline.args) # Make a working copy + + # Make sure the exe is imported + is_imported = True + if 'IMPORTED' not in args: + return self._gen_exception('add_executable', 'non imported executables are not supported', tline) + + args.remove('IMPORTED') + + if len(args) < 1: + return self._gen_exception('add_executable', 'requires at least 1 argument', tline) + + self.targets[args[0]] = CMakeTarget(args[0], 'EXECUTABLE', {}, tline=tline, imported=is_imported) + + def _cmake_add_library(self, tline: CMakeTraceLine) -> None: + # DOC: https://cmake.org/cmake/help/latest/command/add_library.html + args = list(tline.args) # Make a working copy + + # Make sure the lib is imported + if 'INTERFACE' in args: + args.remove('INTERFACE') + + if len(args) < 1: + return self._gen_exception('add_library', 'interface library name not specified', tline) + + self.targets[args[0]] = CMakeTarget(args[0], 'INTERFACE', {}, tline=tline, imported='IMPORTED' in args) + elif 'IMPORTED' in args: + args.remove('IMPORTED') + + # Now, only look at the first two arguments (target_name and target_type) and ignore the rest + if len(args) < 2: + return self._gen_exception('add_library', 'requires at least 2 arguments', tline) + + self.targets[args[0]] = CMakeTarget(args[0], args[1], {}, tline=tline, imported=True) + elif 'ALIAS' in args: + args.remove('ALIAS') + + # Now, only look at the first two arguments (target_name and target_ref) and ignore the rest + if len(args) < 2: + return self._gen_exception('add_library', 'requires at least 2 arguments', tline) + + # Simulate the ALIAS with INTERFACE_LINK_LIBRARIES + self.targets[args[0]] = CMakeTarget(args[0], 'ALIAS', {'INTERFACE_LINK_LIBRARIES': [args[1]]}, tline=tline) + elif 'OBJECT' in args: + return self._gen_exception('add_library', 'OBJECT libraries are not supported', tline) + else: + self.targets[args[0]] = CMakeTarget(args[0], 'NORMAL', {}, tline=tline) + + def _cmake_add_custom_command(self, tline: CMakeTraceLine, name: T.Optional[str] = None) -> None: + # DOC: https://cmake.org/cmake/help/latest/command/add_custom_command.html + args = self._flatten_args(list(tline.args)) # Commands can be passed as ';' separated lists + + if not args: + return self._gen_exception('add_custom_command', 'requires at least 1 argument', tline) + + # Skip the second function signature + if args[0] == 'TARGET': + return self._gen_exception('add_custom_command', 'TARGET syntax is currently not supported', tline) + + magic_keys = ['OUTPUT', 'COMMAND', 'MAIN_DEPENDENCY', 'DEPENDS', 'BYPRODUCTS', + 'IMPLICIT_DEPENDS', 'WORKING_DIRECTORY', 'COMMENT', 'DEPFILE', + 'JOB_POOL', 'VERBATIM', 'APPEND', 'USES_TERMINAL', 'COMMAND_EXPAND_LISTS'] + + target = CMakeGeneratorTarget(name) + + def handle_output(key: str, target: CMakeGeneratorTarget) -> None: + target.outputs += [Path(key)] + + def handle_command(key: str, target: CMakeGeneratorTarget) -> None: + if key == 'ARGS': + return + target.command[-1] += [key] + + def handle_depends(key: str, target: CMakeGeneratorTarget) -> None: + target.depends += [key] + + working_dir = None + def handle_working_dir(key: str, target: CMakeGeneratorTarget) -> None: + nonlocal working_dir + if working_dir is None: + working_dir = key + else: + working_dir += ' ' + working_dir += key + + fn = None + + for i in args: + if i in magic_keys: + if i == 'OUTPUT': + fn = handle_output + elif i == 'DEPENDS': + fn = handle_depends + elif i == 'WORKING_DIRECTORY': + fn = handle_working_dir + elif i == 'COMMAND': + fn = handle_command + target.command += [[]] + else: + fn = None + continue + + if fn is not None: + fn(i, target) + + cbinary_dir = self.var_to_str('MESON_PS_CMAKE_CURRENT_BINARY_DIR') + csource_dir = self.var_to_str('MESON_PS_CMAKE_CURRENT_SOURCE_DIR') + + target.working_dir = Path(working_dir) if working_dir else None + target.current_bin_dir = Path(cbinary_dir) if cbinary_dir else None + target.current_src_dir = Path(csource_dir) if csource_dir else None + target.outputs = [Path(x) for x in self._guess_files([str(y) for y in target.outputs])] + target.depends = self._guess_files(target.depends) + target.command = [self._guess_files(x) for x in target.command] + + self.custom_targets += [target] + if name: + self.targets[name] = target + + def _cmake_add_custom_target(self, tline: CMakeTraceLine) -> None: + # DOC: https://cmake.org/cmake/help/latest/command/add_custom_target.html + # We only the first parameter (the target name) is interesting + if len(tline.args) < 1: + return self._gen_exception('add_custom_target', 'requires at least one argument', tline) + + # It's pretty much the same as a custom command + self._cmake_add_custom_command(tline, tline.args[0]) + + def _cmake_set_property(self, tline: CMakeTraceLine) -> None: + # DOC: https://cmake.org/cmake/help/latest/command/set_property.html + args = list(tline.args) + + scope = args.pop(0) + + append = False + targets = [] + while args: + curr = args.pop(0) + # XXX: APPEND_STRING is specifically *not* supposed to create a + # list, is treating them as aliases really okay? + if curr == 'APPEND' or curr == 'APPEND_STRING': + append = True + continue + + if curr == 'PROPERTY': + break + + targets += curr.split(';') + + if not args: + return self._gen_exception('set_property', 'faild to parse argument list', tline) + + if len(args) == 1: + # Tries to set property to nothing so nothing has to be done + return + + identifier = args.pop(0) + if self.trace_format == 'human': + value = ' '.join(args).split(';') + else: + value = [y for x in args for y in x.split(';')] + if not value: + return + + def do_target(t: str) -> None: + if t not in self.targets: + return self._gen_exception('set_property', f'TARGET {t} not found', tline) + + tgt = self.targets[t] + if identifier not in tgt.properties: + tgt.properties[identifier] = [] + + if append: + tgt.properties[identifier] += value + else: + tgt.properties[identifier] = value + + def do_source(src: str) -> None: + if identifier != 'HEADER_FILE_ONLY' or not self._str_to_bool(value): + return + + current_src_dir = self.var_to_str('MESON_PS_CMAKE_CURRENT_SOURCE_DIR') + if not current_src_dir: + mlog.warning(textwrap.dedent('''\ + CMake trace: set_property(SOURCE) called before the preload script was loaded. + Unable to determine CMAKE_CURRENT_SOURCE_DIR. This can lead to build errors. + ''')) + current_src_dir = '.' + + cur_p = Path(current_src_dir) + src_p = Path(src) + + if not src_p.is_absolute(): + src_p = cur_p / src_p + self.explicit_headers.add(src_p) + + if scope == 'TARGET': + for i in targets: + do_target(i) + elif scope == 'SOURCE': + files = self._guess_files(targets) + for i in files: + do_source(i) + + def _cmake_set_target_properties(self, tline: CMakeTraceLine) -> None: + # DOC: https://cmake.org/cmake/help/latest/command/set_target_properties.html + args = list(tline.args) + + targets = [] + while args: + curr = args.pop(0) + if curr == 'PROPERTIES': + break + + targets.append(curr) + + # Now we need to try to reconsitute the original quoted format of the + # arguments, as a property value could have spaces in it. Unlike + # set_property() this is not context free. There are two approaches I + # can think of, both have drawbacks: + # + # 1. Assume that the property will be capitalized ([A-Z_]), this is + # convention but cmake doesn't require it. + # 2. Maintain a copy of the list here: https://cmake.org/cmake/help/latest/manual/cmake-properties.7.html#target-properties + # + # Neither of these is awesome for obvious reasons. I'm going to try + # option 1 first and fall back to 2, as 1 requires less code and less + # synchroniztion for cmake changes. + # + # With the JSON output format, introduced in CMake 3.17, spaces are + # handled properly and we don't have to do either options + + arglist = [] # type: T.List[T.Tuple[str, T.List[str]]] + if self.trace_format == 'human': + name = args.pop(0) + values = [] # type: T.List[str] + prop_regex = re.compile(r'^[A-Z_]+$') + for a in args: + if prop_regex.match(a): + if values: + arglist.append((name, ' '.join(values).split(';'))) + name = a + values = [] + else: + values.append(a) + if values: + arglist.append((name, ' '.join(values).split(';'))) + else: + arglist = [(x[0], x[1].split(';')) for x in zip(args[::2], args[1::2])] + + for name, value in arglist: + for i in targets: + if i not in self.targets: + return self._gen_exception('set_target_properties', f'TARGET {i} not found', tline) + + self.targets[i].properties[name] = value + + def _cmake_add_dependencies(self, tline: CMakeTraceLine) -> None: + # DOC: https://cmake.org/cmake/help/latest/command/add_dependencies.html + args = list(tline.args) + + if len(args) < 2: + return self._gen_exception('add_dependencies', 'takes at least 2 arguments', tline) + + target = self.targets.get(args[0]) + if not target: + return self._gen_exception('add_dependencies', 'target not found', tline) + + for i in args[1:]: + target.depends += i.split(';') + + def _cmake_target_compile_definitions(self, tline: CMakeTraceLine) -> None: + # DOC: https://cmake.org/cmake/help/latest/command/target_compile_definitions.html + self._parse_common_target_options('target_compile_definitions', 'COMPILE_DEFINITIONS', 'INTERFACE_COMPILE_DEFINITIONS', tline) + + def _cmake_target_compile_options(self, tline: CMakeTraceLine) -> None: + # DOC: https://cmake.org/cmake/help/latest/command/target_compile_options.html + self._parse_common_target_options('target_compile_options', 'COMPILE_OPTIONS', 'INTERFACE_COMPILE_OPTIONS', tline) + + def _cmake_target_include_directories(self, tline: CMakeTraceLine) -> None: + # DOC: https://cmake.org/cmake/help/latest/command/target_include_directories.html + self._parse_common_target_options('target_include_directories', 'INCLUDE_DIRECTORIES', 'INTERFACE_INCLUDE_DIRECTORIES', tline, ignore=['SYSTEM', 'BEFORE'], paths=True) + + def _cmake_target_link_options(self, tline: CMakeTraceLine) -> None: + # DOC: https://cmake.org/cmake/help/latest/command/target_link_options.html + self._parse_common_target_options('target_link_options', 'LINK_OPTIONS', 'INTERFACE_LINK_OPTIONS', tline) + + def _cmake_target_link_libraries(self, tline: CMakeTraceLine) -> None: + # DOC: https://cmake.org/cmake/help/latest/command/target_link_libraries.html + self._parse_common_target_options('target_link_options', 'LINK_LIBRARIES', 'INTERFACE_LINK_LIBRARIES', tline) + + def _parse_common_target_options(self, func: str, private_prop: str, interface_prop: str, tline: CMakeTraceLine, ignore: T.Optional[T.List[str]] = None, paths: bool = False) -> None: + if ignore is None: + ignore = ['BEFORE'] + + args = list(tline.args) + + if len(args) < 1: + return self._gen_exception(func, 'requires at least one argument', tline) + + target = args[0] + if target not in self.targets: + return self._gen_exception(func, f'TARGET {target} not found', tline) + + interface = [] + private = [] + + mode = 'PUBLIC' + for i in args[1:]: + if i in ignore: + continue + + if i in ['INTERFACE', 'LINK_INTERFACE_LIBRARIES', 'PUBLIC', 'PRIVATE', 'LINK_PUBLIC', 'LINK_PRIVATE']: + mode = i + continue + + if mode in ['INTERFACE', 'LINK_INTERFACE_LIBRARIES', 'PUBLIC', 'LINK_PUBLIC']: + interface += i.split(';') + + if mode in ['PUBLIC', 'PRIVATE', 'LINK_PRIVATE']: + private += i.split(';') + + if paths: + interface = self._guess_files(interface) + private = self._guess_files(private) + + interface = [x for x in interface if x] + private = [x for x in private if x] + + for j in [(private_prop, private), (interface_prop, interface)]: + if not j[0] in self.targets[target].properties: + self.targets[target].properties[j[0]] = [] + + self.targets[target].properties[j[0]] += j[1] + + def _meson_ps_execute_delayed_calls(self, tline: CMakeTraceLine) -> None: + for l in self.stored_commands: + fn = self.functions.get(l.func, None) + if(fn): + fn(l) + + # clear the stored commands + self.stored_commands = [] + + def _meson_ps_reload_vars(self, tline: CMakeTraceLine) -> None: + self.delayed_commands = self.get_cmake_var('MESON_PS_DELAYED_CALLS') + + def _meson_ps_disabled_function(self, tline: CMakeTraceLine) -> None: + args = list(tline.args) + if not args: + mlog.error('Invalid preload.cmake script! At least one argument to `meson_ps_disabled_function` is expected') + return + mlog.warning(f'The CMake function "{args[0]}" was disabled to avoid compatibility issues with Meson.') + + def _lex_trace_human(self, trace: str) -> T.Generator[CMakeTraceLine, None, None]: + # The trace format is: '<file>(<line>): <func>(<args -- can contain \n> )\n' + reg_tline = re.compile(r'\s*(.*\.(cmake|txt))\(([0-9]+)\):\s*(\w+)\(([\s\S]*?) ?\)\s*\n', re.MULTILINE) + reg_other = re.compile(r'[^\n]*\n') + loc = 0 + while loc < len(trace): + mo_file_line = reg_tline.match(trace, loc) + if not mo_file_line: + skip_match = reg_other.match(trace, loc) + if not skip_match: + print(trace[loc:]) + raise CMakeException('Failed to parse CMake trace') + + loc = skip_match.end() + continue + + loc = mo_file_line.end() + + file = mo_file_line.group(1) + line = mo_file_line.group(3) + func = mo_file_line.group(4) + args = mo_file_line.group(5) + args = parse_generator_expressions(args) + argl = args.split(' ') + argl = list(map(lambda x: x.strip(), argl)) + + yield CMakeTraceLine(file, int(line), func, argl) + + def _lex_trace_json(self, trace: str) -> T.Generator[CMakeTraceLine, None, None]: + lines = trace.splitlines(keepends=False) + lines.pop(0) # The first line is the version + for i in lines: + data = json.loads(i) + assert isinstance(data['file'], str) + assert isinstance(data['line'], int) + assert isinstance(data['cmd'], str) + assert isinstance(data['args'], list) + args = data['args'] + for j in args: + assert isinstance(j, str) + args = [parse_generator_expressions(x) for x in args] + yield CMakeTraceLine(data['file'], data['line'], data['cmd'], args) + + def _flatten_args(self, args: T.List[str]) -> T.List[str]: + # Split lists in arguments + res = [] # type: T.List[str] + for i in args: + res += i.split(';') + return res + + def _guess_files(self, broken_list: T.List[str]) -> T.List[str]: + # Nothing has to be done for newer formats + if self.trace_format != 'human': + return broken_list + + # Try joining file paths that contain spaces + + reg_start = re.compile(r'^([A-Za-z]:)?/(.*/)*[^./]+$') + reg_end = re.compile(r'^.*\.[a-zA-Z]+$') + + fixed_list = [] # type: T.List[str] + curr_str = None # type: T.Optional[str] + path_found = False # type: bool + + for i in broken_list: + if curr_str is None: + curr_str = i + path_found = False + elif Path(curr_str).is_file(): + # Abort concatenation if curr_str is an existing file + fixed_list += [curr_str] + curr_str = i + path_found = False + elif not reg_start.match(curr_str): + # Abort concatenation if curr_str no longer matches the regex + fixed_list += [curr_str] + curr_str = i + path_found = False + elif reg_end.match(i): + # File detected + curr_str = f'{curr_str} {i}' + fixed_list += [curr_str] + curr_str = None + path_found = False + elif Path(f'{curr_str} {i}').exists(): + # Path detected + curr_str = f'{curr_str} {i}' + path_found = True + elif path_found: + # Add path to fixed_list after ensuring the whole path is in curr_str + fixed_list += [curr_str] + curr_str = i + path_found = False + else: + curr_str = f'{curr_str} {i}' + path_found = False + + if curr_str: + fixed_list += [curr_str] + return fixed_list |