diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ee9f065199..e950ea6377 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -108,7 +108,7 @@ To update the replica to a given $SHA from the dfinity repo, execute the followi ### Updating Motoko -To update Motoko to a given $VERSION from the motoko and motoko-base repos, run the [the GitHub Action](https://github.com/dfinity/sdk/actions/workflows/update-motoko.yml). +To update Motoko to a given $VERSION from the `motoko`, `motoko-base`, and `motoko-core` repos, run the [the GitHub Action](https://github.com/dfinity/sdk/actions/workflows/update-motoko.yml). You can also execute the following locally: diff --git a/e2e/assets/core/main.mo b/e2e/assets/core/main.mo new file mode 100644 index 0000000000..90b676e60a --- /dev/null +++ b/e2e/assets/core/main.mo @@ -0,0 +1,10 @@ +import Int "mo:core/Int"; + +persistent actor TestCore { + + public query func test_core() : async Bool { + // Test a simple function from the core package + Int.abs(-42) == 42; + }; + +} diff --git a/e2e/assets/core/patch.bash b/e2e/assets/core/patch.bash new file mode 100644 index 0000000000..da753f124a --- /dev/null +++ b/e2e/assets/core/patch.bash @@ -0,0 +1 @@ +jq '.canisters.e2e_project_backend.main="main.mo"' dfx.json | sponge dfx.json diff --git a/e2e/tests-dfx/core.bash b/e2e/tests-dfx/core.bash new file mode 100644 index 0000000000..114e47bdfb --- /dev/null +++ b/e2e/tests-dfx/core.bash @@ -0,0 +1,37 @@ +#!/usr/bin/env bats + +load ../utils/_ + +setup() { + standard_setup + + dfx_new +} + +teardown() { + dfx_stop + + standard_teardown +} + +@test "provides core package location by default" { + install_asset core + + dfx_start + dfx canister create --all + dfx build + dfx canister install e2e_project_backend + + assert_command dfx canister call --query e2e_project_backend test_core + assert_eq '(true)' +} + +@test "does not provide core package if there is a packtool" { + install_asset core + jq '.defaults.build.packtool="echo"' dfx.json | sponge dfx.json + + dfx_start + dfx canister create --all + assert_command_fail dfx build + assert_match 'import error \[M0010\], package "core" not defined' +} diff --git a/e2e/tests-dfx/dfx_install.bash b/e2e/tests-dfx/dfx_install.bash index d7fec8aaa3..498ba65c36 100644 --- a/e2e/tests-dfx/dfx_install.bash +++ b/e2e/tests-dfx/dfx_install.bash @@ -59,6 +59,15 @@ teardown() { done } +@test "Motoko core package files are not executable" { + assert_command dfx cache install + for file in "$(dfx cache show)"/core/*.mo; do + assert_command_fail test -x "$file" + assert_command_fail "$file" + assert_contains "Permission denied" + done +} + @test "forced install overwrites a cached version" { assert_command dfx cache install diff --git a/scripts/update-motoko.sh b/scripts/update-motoko.sh index 780d6116d0..3652d51c80 100755 --- a/scripts/update-motoko.sh +++ b/scripts/update-motoko.sh @@ -29,6 +29,11 @@ motoko_base_sha=$(curl --proto '=https' --tlsv1.2 -sSfL "$motoko_base_url" | sha jq '.common."motoko-base" = {url: $url, sha256: $sha256, version: $version}' --arg version "$version" \ --arg url "$motoko_base_url" --arg sha256 "$motoko_base_sha" "$sources" | sponge "$sources" +motoko_core_url=$(printf 'https://github.com/dfinity/motoko/releases/download/%s/motoko-core.tar.gz' "$(urlencode "$version")") +motoko_core_sha=$(curl --proto '=https' --tlsv1.2 -sSfL "$motoko_core_url" | sha256sum | head -c 64) +jq '.common."motoko-core" = {url: $url, sha256: $sha256, version: $version}' --arg version "$version" \ + --arg url "$motoko_core_url" --arg sha256 "$motoko_core_sha" "$sources" | sponge "$sources" + declare -A variants=([x86_64-darwin]=Darwin-x86_64 [x86_64-linux]=Linux-x86_64 [arm64-darwin]=Darwin-arm64 [arm64-linux]=Linux-aarch64) for platform in "${!variants[@]}"; do motoko_url=$(printf 'https://github.com/dfinity/motoko/releases/download/%s/motoko-%s-%s.tar.gz' \ diff --git a/src/dfx/assets/dfx-asset-sources.json b/src/dfx/assets/dfx-asset-sources.json index 5fc46f66d7..a920e2734a 100644 --- a/src/dfx/assets/dfx-asset-sources.json +++ b/src/dfx/assets/dfx-asset-sources.json @@ -58,6 +58,11 @@ "url": "https://github.com/dfinity/bitcoin-canister/releases/download/release%2F2024-08-30/ic-btc-canister.wasm.gz", "sha256": "d0ada614be09055f37d45d8ad92560432c86b426d9e66f42609ce64007a13c3e", "version": "release/2024-08-30" + }, + "motoko-core": { + "url": "https://github.com/dfinity/motoko/releases/download/0.16.0/motoko-core.tar.gz", + "sha256": "dacbeb1735306be8c6bfa3738167bcf372b0364dc5319e5a024d70d4e11231e0", + "version": "0.16.0" } } } diff --git a/src/dfx/assets/prepare_assets.rs b/src/dfx/assets/prepare_assets.rs index ec4a82d7c4..8bbc49f735 100644 --- a/src/dfx/assets/prepare_assets.rs +++ b/src/dfx/assets/prepare_assets.rs @@ -88,6 +88,7 @@ async fn make_binary_cache(out_dir: PathBuf, sources: HashMap) { .build() .unwrap(); let mo_base = spawn(download_mo_base(client.clone(), sources.clone())); + let mo_core = spawn(download_mo_core(client.clone(), sources.clone())); let bins = spawn(download_binaries(client.clone(), sources.clone())); let bin_tars = spawn(download_bin_tarballs(client.clone(), sources.clone())); let canisters = spawn(download_canisters( @@ -95,9 +96,9 @@ async fn make_binary_cache(out_dir: PathBuf, sources: HashMap) { sources.clone(), out_dir.clone(), )); - let (mo_base, bins, bin_tars, _) = - tokio::try_join!(mo_base, bins, bin_tars, canisters).unwrap(); - spawn_blocking(|| write_binary_cache(out_dir, mo_base, bins, bin_tars)) + let (mo_base, mo_core, bins, bin_tars, _) = + tokio::try_join!(mo_base, mo_core, bins, bin_tars, canisters).unwrap(); + spawn_blocking(|| write_binary_cache(out_dir, mo_base, mo_core, bins, bin_tars)) .await .unwrap(); } @@ -105,6 +106,7 @@ async fn make_binary_cache(out_dir: PathBuf, sources: HashMap) { fn write_binary_cache( out_dir: PathBuf, mo_base: HashMap, + mo_core: HashMap, bins: HashMap, mut bin_tars: HashMap, ) { @@ -140,6 +142,18 @@ fn write_binary_cache( tar.append_data(&mut header, Path::new("base").join(path), file.reader()) .unwrap(); } + let mut core_hdr = Header::new_gnu(); + core_hdr.set_entry_type(EntryType::dir()); + core_hdr.set_mode(0o755); + core_hdr.set_size(0); + tar.append_data(&mut core_hdr, "core", io::empty()).unwrap(); + for (path, file) in mo_core { + let mut header = Header::new_gnu(); + header.set_mode(0o644); + header.set_size(file.len() as u64); + tar.append_data(&mut header, Path::new("core").join(path), file.reader()) + .unwrap(); + } tar.finish().unwrap(); } @@ -251,6 +265,22 @@ async fn download_mo_base( map } +async fn download_mo_core( + client: Client, + sources: Arc>, +) -> HashMap { + let source = sources["motoko-core"].clone(); + let mo_core = download_and_check_sha(client, source).await; + let mut map = HashMap::new(); + tar_xzf(&mo_core, |path, content| { + let path = path.strip_prefix(".").unwrap_or(&path); // normalize ./x to x + if let Ok(file) = path.strip_prefix("src") { + map.insert(file.to_owned(), content); + } + }); + map +} + fn tar_xzf(gz: &[u8], mut each: impl FnMut(PathBuf, Bytes)) { let mut tar = Archive::new(GzDecoder::new(gz)); for entry in tar.entries().unwrap() { diff --git a/src/dfx/src/config/cache.rs b/src/dfx/src/config/cache.rs index e54c4a3962..591f25928d 100644 --- a/src/dfx/src/config/cache.rs +++ b/src/dfx/src/config/cache.rs @@ -126,7 +126,8 @@ pub fn install_version( #[cfg(unix)] { let archive_path = dfx_core::fs::get_archive_path(&file)?; - let mode = if archive_path.starts_with("base/") { + let mode = if archive_path.starts_with("base/") || archive_path.starts_with("core/") + { READ_USER_ONLY_PERMISSION } else { EXEC_READ_USER_ONLY_PERMISSION diff --git a/src/dfx/src/lib/package_arguments.rs b/src/dfx/src/lib/package_arguments.rs index c8e55034de..c04d02c6ed 100644 --- a/src/dfx/src/lib/package_arguments.rs +++ b/src/dfx/src/lib/package_arguments.rs @@ -8,7 +8,7 @@ use std::process::Command; /// Package arguments for moc as returned by a package tool like /// https://github.com/kritzcreek/vessel or, if there is no package -/// tool, the base library. +/// tool, the `base` and `core` packages. pub type PackageArguments = Vec; #[context("Failed to load package arguments.")] @@ -19,13 +19,20 @@ pub fn load( project_root: &Path, ) -> DfxResult { if packtool.is_none() { - let stdlib_path = cache - .get_binary_command_path(env, "base")? - .into_os_string() - .into_string() - .map_err(|_| anyhow!("Path contains invalid Unicode data."))?; - let base = vec![String::from("--package"), String::from("base"), stdlib_path]; - return Ok(base); + let mut flags = Vec::new(); + for package_name in ["base", "core"] { + let package_path = cache + .get_binary_command_path(env, package_name)? + .into_os_string() + .into_string() + .map_err(|_| anyhow!("Path contains invalid Unicode data."))?; + flags.extend_from_slice(&[ + String::from("--package"), + String::from(package_name), + package_path, + ]); + } + return Ok(flags); } let commandline: Vec = packtool