Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace cli flag --prebuilt-platform with --build-host #6859

Open
wants to merge 57 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
57 commits
Select commit Hold shift click to select a range
dc0902b
WIP
lukewilliamboswell Jul 2, 2024
753cfb4
fix: building valgrind surgical host
lukewilliamboswell Jul 9, 2024
85447ec
replace stray strings with impl on Target
lukewilliamboswell Jul 9, 2024
96c559e
minor fixes
lukewilliamboswell Jul 9, 2024
4ccbc41
replace gen-stub-lib in nix CI tests
lukewilliamboswell Jul 9, 2024
287d00d
remove gen_stub_lib
lukewilliamboswell Jul 9, 2024
f515311
clippy fix
lukewilliamboswell Jul 9, 2024
bd68787
add --build-host for benchmark tests
lukewilliamboswell Jul 9, 2024
61a5a0a
checkpoint: refactor prebuilt host artifacts
r-bar Jul 9, 2024
d747cc9
Merge remote-tracking branch 'remotes/luke/rebuild-platform' into reb…
r-bar Jul 9, 2024
dfa3254
checkpoint: it compiles
r-bar Jul 11, 2024
bc29782
checkpoint: Run struct refactor for helpers
r-bar Jul 22, 2024
89d63f2
Fix ownership issues with runners
smores56 Aug 14, 2024
3662c90
WIP get things compiling... hacks everywhere beware
lukewilliamboswell Aug 15, 2024
4ec213a
Get tests working
smores56 Aug 15, 2024
550bade
Merge branch 'main' into rebuild-platform
smores56 Aug 15, 2024
49f5ea8
Merge branch 'main' into rebuild-platform
smores56 Aug 15, 2024
28ceb0e
Fix formatting and clippy issues
smores56 Aug 15, 2024
2928969
restore one test
lukewilliamboswell Aug 15, 2024
6276cc7
WIP more cli_run tests passing
lukewilliamboswell Aug 15, 2024
4863234
Merge branch 'rebuild-platform' of github.com:roc-lang/roc into rebui…
lukewilliamboswell Aug 15, 2024
014514a
WIP more passing roc_cli tests
lukewilliamboswell Aug 15, 2024
64cc816
WIP all the roc_cli tests passing, woo
lukewilliamboswell Aug 15, 2024
9ecb209
add suppress warning flag, remove test test_roc_app_slim
lukewilliamboswell Aug 16, 2024
0767115
WIP refactor to build test platform once per test run
lukewilliamboswell Aug 16, 2024
804cb6e
WIP upgrade cli test packages fixture tests
lukewilliamboswell Aug 16, 2024
c8522c2
WIP upgrade cli tests transitive-deps
lukewilliamboswell Aug 16, 2024
d2fbf86
format test files, upgrade some tests
lukewilliamboswell Aug 16, 2024
7ce9ecd
WIP improve cli tests
lukewilliamboswell Aug 16, 2024
894e4d1
make clippy happy again
lukewilliamboswell Aug 16, 2024
5c8eb4f
rename cli to basic-cli
lukewilliamboswell Aug 16, 2024
6ab0de1
fix benchmark tests
lukewilliamboswell Aug 16, 2024
0256c4e
move false interpreter into cli tests
lukewilliamboswell Aug 16, 2024
8ad8198
move effects cli test
lukewilliamboswell Aug 16, 2024
fa40166
move tui example into cli tests
lukewilliamboswell Aug 16, 2024
c3785a2
fix more tests
lukewilliamboswell Aug 16, 2024
767ec7a
cli run tests passing on macos
lukewilliamboswell Aug 16, 2024
ac78d39
Merge remote-tracking branch 'remote/main' into rebuild-platform
lukewilliamboswell Aug 17, 2024
8612220
fix plumbing for roc_glue
lukewilliamboswell Aug 18, 2024
b0c1d88
Merge remote-tracking branch 'remote/main' into rebuild-platform
lukewilliamboswell Aug 18, 2024
e7a90f1
fix clippy
lukewilliamboswell Aug 18, 2024
cd955fd
remove check_compile_error, fix cli tests
lukewilliamboswell Aug 18, 2024
93a4c58
fix cli tests - mainly wasm and i386
lukewilliamboswell Aug 18, 2024
0e917be
add LEGACY LINKER conditional for test runs
lukewilliamboswell Aug 18, 2024
4e6178c
move valgrind configuration into Run builder
lukewilliamboswell Aug 18, 2024
f92974c
use legacy linker in tests
lukewilliamboswell Aug 18, 2024
012a2fc
Merge remote-tracking branch 'remote/main' into rebuild-platform
lukewilliamboswell Sep 3, 2024
a11f51a
fixup
lukewilliamboswell Sep 3, 2024
ec8f6e8
clippy fix
Anton-4 Sep 3, 2024
251e1ed
misc improvements
Anton-4 Sep 3, 2024
ef87653
migrate cli test combine-tasks.roc off basic-cli
lukewilliamboswell Sep 4, 2024
c6a67c9
remove basic-cli tests from roc_cli
lukewilliamboswell Sep 4, 2024
5dfcdae
migrate inspect-logging.roc into cli tests and use effects platform
lukewilliamboswell Sep 4, 2024
73846ed
add test for module_params pass_task.roc
lukewilliamboswell Sep 4, 2024
beb2dcb
remove unneeded basic-cli reference in formatting test
lukewilliamboswell Sep 4, 2024
bb57c4a
remove unneded basic-cli reference in test_reporting
lukewilliamboswell Sep 4, 2024
dc138b8
remove last usage of basic-cli platform from the tests
lukewilliamboswell Sep 4, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 9 additions & 16 deletions crates/cli/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 70,7 @@ pub const FLAG_TARGET: &str = "target";
pub const FLAG_TIME: &str = "time";
pub const FLAG_VERBOSE: &str = "verbose";
pub const FLAG_LINKER: &str = "linker";
pub const FLAG_PREBUILT: &str = "prebuilt-platform";
pub const FLAG_BUILD_HOST: &str = "build-host";
pub const FLAG_CHECK: &str = "check";
pub const FLAG_STDIN: &str = "stdin";
pub const FLAG_STDOUT: &str = "stdout";
Expand Down Expand Up @@ -140,9 140,9 @@ pub fn build_app() -> Command {
.value_parser(["surgical", "legacy"])
.required(false);

let flag_prebuilt = Arg::new(FLAG_PREBUILT)
.long(FLAG_PREBUILT)
.help("Assume the platform has been prebuilt and skip rebuilding the platform\n(This is enabled implicitly when using `roc build` with a --target other than `--target <current machine>`, unless the target is wasm.)")
let flag_prebuilt = Arg::new(FLAG_BUILD_HOST)
.long(FLAG_BUILD_HOST)
.help("WARNING: platforms are responsible for building hosts, this flag will be removed when internal test platforms have a build script")
.action(ArgAction::SetTrue)
.required(false);

Expand Down Expand Up @@ -855,17 855,10 @@ pub fn build(
LinkingStrategy::Surgical
};

let prebuilt = {
let cross_compile = target != Target::default();
let targeting_wasm = matches!(target.architecture(), Architecture::Wasm32);

matches.get_flag(FLAG_PREBUILT) ||
// When compiling for a different target, assume a prebuilt platform.
// Otherwise compilation would most likely fail because many toolchains
// assume you're compiling for the current machine. We make an exception
// for Wasm, because cross-compiling is the norm in that case.
(cross_compile && !targeting_wasm)
};
// TODO: remove once host rebuilding is no longer required
// all hosts should be prebuilt, this flag keeps the rebuilding behvaiour
// until no longer required for internal tests
let rebuild_host = matches.get_flag(FLAG_BUILD_HOST);

let fuzz = matches.get_flag(FLAG_FUZZ);
if fuzz && !matches!(code_gen_backend, CodeGenBackend::Llvm(_)) {
Expand Down Expand Up @@ -901,7 894,7 @@ pub fn build(
emit_timings,
link_type,
linking_strategy,
prebuilt,
rebuild_host,
wasm_dev_stack_bytes,
roc_cache_dir,
load_config,
Expand Down
6 changes: 5 additions & 1 deletion crates/cli/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -169,10 169,14 @@ fn main() -> io::Result<()> {

let verbose_and_time = matches.get_one::<bool>(roc_cli::FLAG_VERBOSE).unwrap();

let preprocessed_path = platform_path.with_file_name(target.prebuilt_surgical_host());
let metadata_path = platform_path.with_file_name(target.metadata_file_name());

roc_linker::preprocess_host(
target,
host_path,
platform_path,
metadata_path.as_path(),
preprocessed_path.as_path(),
dylib_path,
*verbose_and_time,
*verbose_and_time,
Expand Down
15 changes: 10 additions & 5 deletions crates/cli/tests/cli_run.rs
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are some big changes here. The primary thrust is to remove layers of abstraction, and reduce the number of ways we do things to just one. There were various places where we would do the same thing multiple different ways, and also calling layers of different functions abstracted away what was actually being done in each tests.

You can now see at a glance what is being done for each test, without digging through multiple function calls. The code is still fairly DRY as common behaviour is rolled up into the Run builder API for calling roc.

The other major change, is moving tests with a common platform into the same module, and using a Once variable to build the platform host once per test run.

Original file line number Diff line number Diff line change
Expand Up @@ -55,8 55,6 @@ mod cli_run {
const LINKER_FLAG: &str = concatcp!("--", roc_cli::FLAG_LINKER);
const CHECK_FLAG: &str = concatcp!("--", roc_cli::FLAG_CHECK);
#[allow(dead_code)]
const PREBUILT_PLATFORM: &str = concatcp!("--", roc_cli::FLAG_PREBUILT);
#[allow(dead_code)]
const TARGET_FLAG: &str = concatcp!("--", roc_cli::FLAG_TARGET);

#[derive(Debug)]
Expand Down Expand Up @@ -193,6 191,8 @@ mod cli_run {
let flags = {
let mut vec = flags.to_vec();

vec.push("--build-host");

// max-threads segfaults on windows right now
if !cfg!(windows) {
vec.push("--max-threads=1");
Expand Down Expand Up @@ -540,6 540,7 @@ mod cli_run {
)
}

#[ignore = "TODO move this to roc-lang/examples repository"]
#[test]
fn platform_switching_swift() {
test_roc_app_slim(
Expand Down Expand Up @@ -755,6 756,7 @@ mod cli_run {
);
}

#[ignore = "TODO move this to roc-lang/examples repository"]
#[test]
#[cfg_attr(
windows,
Expand Down Expand Up @@ -783,6 785,7 @@ mod cli_run {
)
}

#[ignore = "TODO move this to roc-lang/examples repository"]
#[test]
fn hello_gui() {
test_roc_app_slim("examples/gui", "hello-guiBROKEN.roc", "", UseValgrind::No)
Expand Down Expand Up @@ -941,6 944,7 @@ mod cli_run {
)
}

#[ignore = "TODO move this to roc-lang/examples repository"]
#[test]
fn swift_ui() {
test_roc_app_slim("examples/swiftui", "main.roc", "", UseValgrind::No)
Expand Down Expand Up @@ -1087,6 1091,7 @@ mod cli_run {
)
}

#[ignore = "TODO move this to roc-lang/examples repository"]
#[test]
fn inspect_gui() {
test_roc_app_slim("examples", "inspect-gui.roc", "", UseValgrind::No)
Expand All @@ -1100,7 1105,7 @@ mod cli_run {
use cli_utils::helpers::cli_testing_dir;

#[allow(unused_imports)]
use super::{check_output_with_stdin, OPTIMIZE_FLAG, PREBUILT_PLATFORM};
use super::{check_output_with_stdin, OPTIMIZE_FLAG};

#[allow(unused_imports)]
use std::{path::Path, sync::Once};
Expand Down Expand Up @@ -1178,7 1183,7 @@ mod cli_run {
check_output_with_stdin(
file_name,
stdin,
&[PREBUILT_PLATFORM],
&[],
&[],
&[],
expected_ending,
Expand All @@ -1190,7 1195,7 @@ mod cli_run {
check_output_with_stdin(
file_name,
stdin,
&[PREBUILT_PLATFORM, OPTIMIZE_FLAG],
&[OPTIMIZE_FLAG],
&[],
&[],
expected_ending,
Expand Down
2 changes: 1 addition & 1 deletion crates/cli_utils/src/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 50,7 @@ where

pub fn has_error(stderr: &str) -> bool {
let stderr_stripped = stderr
.replacen("🔨 Rebuilding platform...\n", "", 1)
.replacen("🔨 Building host ...\n", "", 1)
// for some reason, llvm prints out this warning when targeting windows
.replacen(
"warning: ignoring debug info with an invalid version (0) in app\r\n",
Expand Down
61 changes: 43 additions & 18 deletions crates/compiler/build/src/link.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,26 42,50 @@ pub fn link(
}
}

/// Same format as the precompiled host filename, except with a file extension like ".o" or ".obj"
pub fn legacy_host_file(target: Target, platform_main_roc: &Path) -> PathBuf {
let lib_ext = target.static_library_file_ext();
/// Search for a prebuilt surgical host in the platform main directory.
pub fn find_surgical_host(target: Target, platform_main_roc: &Path) -> Result<PathBuf, String> {
let surgical_host_path = platform_main_roc.with_file_name(target.prebuilt_surgical_host());

let file_name = roc_linker::preprocessed_host_filename(target)
.replace(roc_linker::PRECOMPILED_HOST_EXT, lib_ext);
let generic_host_path: PathBuf = platform_main_roc.with_file_name("host.rh");

let lib_path = platform_main_roc.with_file_name(file_name);
if generic_host_path.exists() {
Ok(generic_host_path)
} else if surgical_host_path.exists() {
Ok(surgical_host_path)
} else {
Err(format!(
"\n {}\n {}",
surgical_host_path.display(),
generic_host_path.display(),
)
.to_string())
}
}

let default_host_path: PathBuf = platform_main_roc
.with_file_name("libhost")
.with_extension(lib_ext);
/// Search for a prebuilt legacy host in the platform main directory.
pub fn find_legacy_host(target: Target, platform_main_roc: &Path) -> Result<PathBuf, String> {
let static_library_path = platform_main_roc.with_file_name(target.prebuilt_static_library());

let static_object_path = platform_main_roc.with_file_name(target.prebuilt_static_object());

if lib_path.exists() {
lib_path
} else if default_host_path.exists() {
default_host_path
let generic_host_path: PathBuf = platform_main_roc
.with_file_name("libhost")
.with_extension(target.static_library_file_ext());

if static_library_path.exists() {
Ok(static_library_path)
} else if generic_host_path.exists() {
Ok(generic_host_path)
} else if static_object_path.exists() {
Ok(static_object_path)
} else {
let obj_ext = target.object_file_ext();
lib_path.with_extension(obj_ext)
Err(format!(
"\n {}\n {}\n {}",
static_library_path.display(),
static_object_path.display(),
generic_host_path.display(),
)
.to_string())
}
}

Expand Down Expand Up @@ -457,6 481,7 @@ pub fn rebuild_host(
};

let host_dest = if matches!(target.architecture(), Architecture::Wasm32) {
// TODO verify this is corect, how do we do get here with OptLevel::Development
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TODO -- investigate

if matches!(opt_level, OptLevel::Development) {
platform_main_roc.with_extension("o")
} else {
Expand All @@ -467,7 492,7 @@ pub fn rebuild_host(
.with_file_name("dynhost")
.with_extension(executable_extension)
} else {
legacy_host_file(target, platform_main_roc)
platform_main_roc.with_file_name(target.prebuilt_static_object())
};

let env_path = env::var("PATH").unwrap_or_else(|_| "".to_string());
Expand Down Expand Up @@ -1330,9 1355,9 @@ pub fn llvm_module_to_dylib(
unsafe { Library::new(path) }
}

pub fn preprocess_host_wasm32(host_input_path: &Path, preprocessed_host_path: &Path) {
pub fn preprocess_host_wasm32(host_input_path: &Path, host_output_path: &Path) {
let host_input = host_input_path.to_str().unwrap();
let output_file = preprocessed_host_path.to_str().unwrap();
let output_file = host_output_path.to_str().unwrap();

/*
Notes:
Expand Down
Loading
Loading