From 19edba743309bda92c2cb0c7f00d12d126526e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc?= Date: Wed, 18 Jan 2023 10:45:57 +0100 Subject: [PATCH] vlcrs: add meson infra for building and linking Rust modules Despite meson officially "supporting" Rust it does support working with Cargo (Rust package manager and build system) so we need to have a bit of custom system for using it. `cargo-output.py` to build and copy the output files (static archive and depfile) to the expected location by meson. And `cargo-rustc-static-libs.py` to get the common linker args for a bar-bone std rust program (ie without dependencies). Those two scripts would potentially be removed if meson and/or cargo became more flexible and/or intelligent. --- buildsystem/cargo-output.py | 80 ++++++++++++ buildsystem/cargo-rustc-static-libs.py | 83 +++++++++++++ meson.build | 3 + meson_options.txt | 5 + modules/meson.build | 163 +++++++++++++++++++++++++ 5 files changed, 334 insertions(+) create mode 100644 buildsystem/cargo-output.py create mode 100644 buildsystem/cargo-rustc-static-libs.py diff --git a/buildsystem/cargo-output.py b/buildsystem/cargo-output.py new file mode 100644 index 0000000000..727f60cdb6 --- /dev/null +++ b/buildsystem/cargo-output.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: LGPL-2.1-or-later +# Copyright (C) 2022 Loïc Branstett +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA. + + +# This is a wrapper/interceptor script for Cargo (the Rust package manager) that adds +# the ability to specify an output directory for the final compiler artifact +# (archive lib, depfile). +# +# Usage: ./buildsystem/cargo-output.py --output=out/ --depfile NORMAL_CARGO_INVOCATION +# Usage: ./buildsystem/cargo-output.py --output=out/ --depfile /usr/bin/cargo +# --target=x86_64-unknown-linux-gnu rustc --crate-type=staticlib +# +# The resulting static library will be put in the output directory as well as its depfile +# if asked (--depfile) and available + +import os, sys, json, pathlib, shutil, subprocess, argparse + +def dir_path(string): + if os.path.isdir(string): + return string + else: + raise NotADirectoryError(string) + +parser = argparse.ArgumentParser() + +parser.add_argument("--depfile", action="store_true") +parser.add_argument("-o", "--output", type=dir_path) +parser.add_argument("cargo_cmds", nargs=argparse.REMAINDER) + +args = parser.parse_args() + +cargo_argv = args.cargo_cmds + +# Insert Cargo argument `--message-format=json-render-diagnostics` +# before a `--` (if there are any) to still be in Cargo arguments +# and not in the inner thing (rustc, ...) +cargo_argv.insert( + cargo_argv.index('--') if '--' in cargo_argv else len(cargo_argv), + "--message-format=json-render-diagnostics" +) + +# Execute the cargo build and redirect stdout (and not stderr) +cargo_r = subprocess.run(cargo_argv, stdout=subprocess.PIPE) + +# We don't use `check=True` in the above `run` method call because it +# raise an execption and outputing a python traceback isn't useful at all +if cargo_r.returncode != 0: + sys.exit(cargo_r.returncode) + +# Get the jsons output +cargo_stdout = cargo_r.stdout.decode('utf-8') +cargo_jsons = [json.loads(line) for line in cargo_stdout.splitlines()] + +# We are only interrested in the final artifact (ie the output, not it's deps) +last_compiler_artifact = next( + j for j in reversed(cargo_jsons) if j["reason"] == "compiler-artifact") + +# We only take the first one, because the other one are just aliases and thus +# are not relevant for us +first_compiler_artifact_filename = last_compiler_artifact["filenames"][0] + +for filename in [first_compiler_artifact_filename]: + shutil.copy2(filename, args.output) + if args.depfile: + shutil.copy2(pathlib.Path(filename).with_suffix(".d"), args.output) diff --git a/buildsystem/cargo-rustc-static-libs.py b/buildsystem/cargo-rustc-static-libs.py new file mode 100644 index 0000000000..2e2f380a04 --- /dev/null +++ b/buildsystem/cargo-rustc-static-libs.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: LGPL-2.1-or-later +# Copyright (C) 2022 Loïc Branstett +# +# This program is free software; you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA. + + +# This script is meant to find the required native libs of an empty Rust crate +# compiled as a static lib. +# +# Usage: ./buildsystem/cargo-rustc-static-libs.py /usr/bin/cargo +# Usage: ./buildsystem/cargo-rustc-static-libs.py /usr/bin/cargo --target=x86_64-unknown-linux-gnu +# +# The result is then printed to the standard output. + +import os, sys, json, subprocess, tempfile, argparse + +NATIVE_STATIC_LIBS="native-static-libs" + +parser = argparse.ArgumentParser() +parser.add_argument("cargo_cmds", nargs=argparse.REMAINDER) + +args = parser.parse_args() + +cargo_argv = args.cargo_cmds +cargo_argv.append("rustc") +cargo_argv.append("--message-format=json") +cargo_argv.append("--crate-type=staticlib") +cargo_argv.append("--quiet") +cargo_argv.append("--") +cargo_argv.append("--print=native-static-libs") + +with tempfile.TemporaryDirectory() as tmpdir: + os.chdir(tmpdir) + + with open("Cargo.toml", "w") as cargo_toml: + cargo_toml.write(""" +[package] +name = "native-static-libs" +version = "0.0.0" + +[lib] +path = "lib.rs" +""") + + with open("lib.rs", "w") as lib_rs: + lib_rs.write("#![allow(dead_code)] fn main(){}") + + # Execute the cargo build and redirect stdout (and not stderr) + cargo_r = subprocess.run(cargo_argv, stdout=subprocess.PIPE) + + # We don't use `check=True in run because it raise an execption + # and outputing a python traceback isn't useful at all. + # + # We also exit here so that the output o tmp dir is not cleared when + # there is an error. + if cargo_r.returncode != 0: + print("command: {cargo_argv}", file=sys.stderr) + print("cwd: {tmpdir}", file=sys.stderr) + sys.exit(cargo_r.returncode) + +# Get the jsons output +cargo_stdout = cargo_r.stdout.decode('utf-8') +cargo_jsons = [json.loads(line) for line in cargo_stdout.splitlines()] + +# Print the last message with a `NATIVE_STATIC_LIBS` message +for j in reversed(cargo_jsons): + if j["reason"] == "compiler-message": + msg = j["message"]["message"] + if msg.startswith(NATIVE_STATIC_LIBS): + print(msg[len(NATIVE_STATIC_LIBS + ": "):]) diff --git a/meson.build b/meson.build index 897e76ba9e..375d9df291 100644 --- a/meson.build +++ b/meson.build @@ -53,6 +53,9 @@ vlc_about = custom_target('vlc_about.h', '@INPUT2@', '@OUTPUT@']) +cargo_rustc_static_libs = find_program('buildsystem/cargo-rustc-static-libs.py') +cargo_output = find_program('buildsystem/cargo-output.py') + add_project_arguments('-DHAVE_CONFIG_H=1', language: ['c', 'cpp', 'objc']) # If building with contribs, read the relevant paths from the machine file diff --git a/meson_options.txt b/meson_options.txt index 847a90471e..8c131927e8 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -70,6 +70,11 @@ option('extra_rust_flags', value : [], description : 'Extra RUSTFLAGS to be passed to the compiler when compiling Rust VLC modules') +option('vendored_rust_deps', + type : 'string', + value : 'no', + description : 'Should use vendored sources: `no`, `yes` or PATH_TO_VENDORED_SOURCES') + # TODO: Missing pdb option, this should probably be solved in meson itself # TODO: Missing ssp option diff --git a/modules/meson.build b/modules/meson.build index 045d1c7a31..108e6d90ee 100644 --- a/modules/meson.build +++ b/modules/meson.build @@ -199,9 +199,12 @@ rsvg_dep = dependency('librsvg-2.0', version: '>= 2.9.0', required: get_option(' libgcrypt_dep = dependency('libgcrypt', version: '>= 1.6.0', required: get_option('libgcrypt')) +# Rust support +cargo_bin = find_program('cargo', required: get_option('rust')) # Array that holds all enabled VLC module dicts vlc_modules = [] +vlc_rust_modules = [] # video chroma modules subdir('video_chroma') @@ -376,3 +379,163 @@ foreach module : vlc_modules module['name']: module } endforeach + +# Rust/cargo common handling code +if get_option('rust') + rust_optimization_args = { + 'plain': [], + '0': [], + 'g': ['-C', 'opt-level=0'], + '1': ['-C', 'opt-level=1'], + '2': ['-C', 'opt-level=2'], + '3': ['-C', 'opt-level=3'], + 's': ['-C', 'opt-level=s'], + } + + print_static_libs_rustc = run_command([cargo_rustc_static_libs, cargo_bin], + check: true, capture: true) + rust_common_link_args = print_static_libs_rustc.stdout().split() + + # Do not try to reorder those two operations, they may triger a bug in + # meson where array += element is not equal to array += array + rust_flags = get_option('extra_rust_flags') + rust_flags += rust_optimization_args[get_option('optimization')] + + cargo_target_dir = join_paths(meson.current_build_dir(), 'cargo_target') + + extra_cargo_args = [] + module_cargo_depends = [] + if get_option('vendored_rust_deps') != 'no' + if get_option('vendored_rust_deps') == 'yes' + cargo_fetch_deps_tgt = custom_target('cargo_fetch_deps', + capture: false, + console: true, + build_by_default: true, + input: files('Cargo.lock'), + output: 'cargo_vendored_deps', + command: [cargo_bin, 'vendor', '--locked', '--versioned-dirs', '--manifest-path', + files('Cargo.toml'), '@OUTPUT@'], + ) + vendored_rust_deps_sources = cargo_fetch_deps_tgt.full_path() + module_cargo_depends = [cargo_fetch_deps_tgt] + else + vendored_rust_deps_sources = get_option('vendored_rust_deps') + endif + + extra_cargo_args += ['--offline', + '--config', 'source.crates-io.replace-with="vendored-sources"', + '--config', f'source.vendored-sources.directory="@vendored_rust_deps_sources@"'] + endif + + vlcrs_core = custom_target('vlcrs_core-cargo', + capture: false, + console: true, + build_by_default: true, + input: files( + 'vlcrs-core/Cargo.toml', + 'vlcrs-core/src/error.rs', + 'vlcrs-core/src/input_item.rs', + 'vlcrs-core/src/lib.rs', + 'vlcrs-core/src/messages.rs', + 'vlcrs-core/src/module/args.rs', + 'vlcrs-core/src/module/capi.rs', + 'vlcrs-core/src/module.rs', + 'vlcrs-core/src/object.rs', + 'vlcrs-core/src/tick.rs', + 'vlcrs-core/src/url.rs', + 'vlcrs-core/sys/build.rs', + 'vlcrs-core/sys/Cargo.toml', + 'vlcrs-core/sys/src/lib.rs', + 'vlcrs-core/sys/wrapper.h', + ), + output: 'libvlcrs_core.rlib', + depfile: 'libvlcrs_core.d', + depends: module_cargo_depends, + env: { + 'RUSTFLAGS': rust_flags, + 'CARGO_TARGET_DIR': cargo_target_dir + }, + command: [cargo_output, '--output', '@OUTDIR@', '--depfile', + cargo_bin, 'build', '--locked', extra_cargo_args, + '--manifest-path', files('Cargo.toml'), + '-p', 'vlcrs-core'] + ) + vlcrs_core_macros = custom_target('vlcrs_core_macros-cargo', + capture: false, + console: true, + build_by_default: true, + input: files( + 'vlcrs-core/macros/Cargo.toml', + 'vlcrs-core/macros/src/lib.rs', + 'vlcrs-core/macros/src/module.rs', + ), + depfile: 'libvlcrs_core_macros.d', + output: 'libvlcrs_core_macros.so', + depends: module_cargo_depends, + env: { + 'RUSTFLAGS': rust_flags, + 'CARGO_TARGET_DIR': cargo_target_dir + }, + command: [cargo_output, '--output', '@OUTDIR@', '--depfile', + cargo_bin, 'build', '--locked', extra_cargo_args, + '--manifest-path', files('Cargo.toml'), + '-p', 'vlcrs-core-macros'] + ) + + module_cargo_depends += [vlcrs_core, vlcrs_core_macros] + + foreach module : vlc_rust_modules + if not module.has_key('name') + error('Got invalid vlc_rust_modules entry without \'name\' key') + endif + if not module.has_key('sources') + error('The vlc_rust_modules entry for ' + module['name'] + ' has no sources') + endif + if not module.has_key('cargo_toml') + error('The vlc_rust_modules entry for ' + module['name'] + ' has no cargo_toml') + endif + + # This list MUST be kept in sync with the keys used below! + valid_dict_keys = [ + 'name', + 'sources', + 'cargo_toml', + 'link_with', + 'link_depends', + 'dependencies', + ] + foreach key : module.keys() + if key not in valid_dict_keys + error('Invalid key \'@0@\' found in vlc_rust_modules entry for \'@1@\'' + .format(key, module['name'])) + endif + endforeach + + module_cargo = custom_target(module['name'] + '_cargo', + build_by_default: true, + input: module['cargo_toml'], + depends: module_cargo_depends, + depfile: 'lib' + module['name'] + '.d', + output: 'lib' + module['name'] + '.a', + env: { + 'RUSTFLAGS': rust_flags, + 'CARGO_TARGET_DIR': cargo_target_dir, + }, + command: [cargo_output, '--output', '@OUTDIR@', '--depfile', + cargo_bin, 'rustc', '--quiet', '--color=always', '--crate-type=staticlib', + '--locked', extra_cargo_args, '--manifest-path', module['cargo_toml']] + ) + + library(module['name'] + '_plugin', + # FIXME(meson): `module_cargo` shouldn't be in `link_whole` because it prevents dead code + # elimination but meson doesn't yet have any way of setting the symbols to export and since + # `module_cargo` is an archive file everything is exported so nothing is really. + link_whole: module_cargo, + link_with: module.get('link_with', []), + link_args: [module.get('link_args', []), rust_common_link_args], + link_depends: module.get('link_depends', []), + dependencies: [module.get('dependencies', []), libvlccore_dep], + build_by_default: true, + ) + endforeach +endif