feat: add new Motoko standard library (core)

CONTRIBUTING.md

@@ -108,7 +108,7 @@ To update the replica to a given $SHA from the dfinity repo, execute the followi
 
 ### Updating Motoko
 
-To update Motoko to a given $VERSION from the motoko and motoko-base repos, run the [the GitHub Action](https://github.com/dfinity/sdk/actions/workflows/update-motoko.yml).
+To update Motoko to a given $VERSION from the `motoko`, `motoko-base`, and `motoko-core` repos, run the [the GitHub Action](https://github.com/dfinity/sdk/actions/workflows/update-motoko.yml).
 
 
 You can also execute the following locally:

e2e/assets/core/main.mo

@@ -0,0 +1,10 @@
+import Int "mo:core/Int";
+
+persistent actor TestCore {
+
+    public query func test_core() : async Bool {
+        // Test a simple function from the core package
+        Int.abs(-42) == 42;
+    };
+
+}

e2e/assets/core/patch.bash

@@ -0,0 +1 @@
+jq '.canisters.e2e_project_backend.main="main.mo"' dfx.json | sponge dfx.json

e2e/tests-dfx/core.bash

@@ -0,0 +1,37 @@
+#!/usr/bin/env bats
+
+load ../utils/_
+
+setup() {
+  standard_setup
+
+  dfx_new
+}
+
+teardown() {
+  dfx_stop
+
+  standard_teardown
+}
+
+@test "provides core package location by default" {
+  install_asset core
+
+  dfx_start
+  dfx canister create --all
+  dfx build
+  dfx canister install e2e_project_backend
+
+  assert_command dfx canister call --query e2e_project_backend test_core
+  assert_eq '(true)'
+}
+
+@test "does not provide core package if there is a packtool" {
+  install_asset core
+  jq '.defaults.build.packtool="echo"' dfx.json | sponge dfx.json
+
+  dfx_start
+  dfx canister create --all
+  assert_command_fail dfx build
+  assert_match 'import error \[M0010\], package "core" not defined'
+}

e2e/tests-dfx/dfx_install.bash

@@ -59,6 +59,15 @@ teardown() {
   done
 }
 
+@test "Motoko core package files are not executable" {
+  assert_command dfx cache install
+  for file in "$(dfx cache show)"/core/*.mo; do
+    assert_command_fail test -x "$file"
+    assert_command_fail "$file"
+    assert_contains "Permission denied"
+  done
+}
+
 
 @test "forced install overwrites a cached version" {
   assert_command dfx cache install

scripts/update-motoko.sh

@@ -29,6 +29,11 @@ motoko_base_sha=$(curl --proto '=https' --tlsv1.2 -sSfL "$motoko_base_url" | sha
 jq '.common."motoko-base" = {url: $url, sha256: $sha256, version: $version}' --arg version "$version" \
     --arg url "$motoko_base_url" --arg sha256 "$motoko_base_sha" "$sources" | sponge "$sources"
 
+motoko_core_url=$(printf 'https://github.com/dfinity/motoko/releases/download/%s/motoko-core.tar.gz' "$(urlencode "$version")")
+motoko_core_sha=$(curl --proto '=https' --tlsv1.2 -sSfL "$motoko_core_url" | sha256sum | head -c 64)
+jq '.common."motoko-core" = {url: $url, sha256: $sha256, version: $version}' --arg version "$version" \
+    --arg url "$motoko_core_url" --arg sha256 "$motoko_core_sha" "$sources" | sponge "$sources"
+
 declare -A variants=([x86_64-darwin]=Darwin-x86_64 [x86_64-linux]=Linux-x86_64 [arm64-darwin]=Darwin-arm64 [arm64-linux]=Linux-aarch64)
 for platform in "${!variants[@]}"; do
     motoko_url=$(printf 'https://github.com/dfinity/motoko/releases/download/%s/motoko-%s-%s.tar.gz' \

src/dfx/assets/dfx-asset-sources.json

@@ -58,6 +58,11 @@
       "url": "https://github.com/dfinity/bitcoin-canister/releases/download/release%2F2024-08-30/ic-btc-canister.wasm.gz",
       "sha256": "d0ada614be09055f37d45d8ad92560432c86b426d9e66f42609ce64007a13c3e",
       "version": "release/2024-08-30"
+    },
+    "motoko-core": {
+      "url": "https://github.com/dfinity/motoko/releases/download/0.16.0/motoko-core.tar.gz",
+      "sha256": "dacbeb1735306be8c6bfa3738167bcf372b0364dc5319e5a024d70d4e11231e0",
+      "version": "0.16.0"
     }
   }
 }

src/dfx/assets/prepare_assets.rs

@@ -88,23 +88,25 @@ async fn make_binary_cache(out_dir: PathBuf, sources: HashMap<String, Source>) {
         .build()
         .unwrap();
     let mo_base = spawn(download_mo_base(client.clone(), sources.clone()));
+    let mo_core = spawn(download_mo_core(client.clone(), sources.clone()));
     let bins = spawn(download_binaries(client.clone(), sources.clone()));
     let bin_tars = spawn(download_bin_tarballs(client.clone(), sources.clone()));
     let canisters = spawn(download_canisters(
         client.clone(),
         sources.clone(),
         out_dir.clone(),
     ));
-    let (mo_base, bins, bin_tars, _) =
-        tokio::try_join!(mo_base, bins, bin_tars, canisters).unwrap();
-    spawn_blocking(|| write_binary_cache(out_dir, mo_base, bins, bin_tars))
+    let (mo_base, mo_core, bins, bin_tars, _) =
+        tokio::try_join!(mo_base, mo_core, bins, bin_tars, canisters).unwrap();
+    spawn_blocking(|| write_binary_cache(out_dir, mo_base, mo_core, bins, bin_tars))
         .await
         .unwrap();
 }
 
 fn write_binary_cache(
     out_dir: PathBuf,
     mo_base: HashMap<PathBuf, Bytes>,
+    mo_core: HashMap<PathBuf, Bytes>,
     bins: HashMap<PathBuf, Bytes>,
     mut bin_tars: HashMap<PathBuf, Bytes>,
 ) {
@@ -140,6 +142,18 @@ fn write_binary_cache(
         tar.append_data(&mut header, Path::new("base").join(path), file.reader())
             .unwrap();
     }
+    let mut core_hdr = Header::new_gnu();
+    core_hdr.set_entry_type(EntryType::dir());
+    core_hdr.set_mode(0o755);
+    core_hdr.set_size(0);
+    tar.append_data(&mut core_hdr, "core", io::empty()).unwrap();
+    for (path, file) in mo_core {
+        let mut header = Header::new_gnu();
+        header.set_mode(0o644);
+        header.set_size(file.len() as u64);
+        tar.append_data(&mut header, Path::new("core").join(path), file.reader())
+            .unwrap();
+    }
     tar.finish().unwrap();
 }
 
@@ -251,6 +265,22 @@ async fn download_mo_base(
     map
 }
 
+async fn download_mo_core(
+    client: Client,
+    sources: Arc<HashMap<String, Source>>,
+) -> HashMap<PathBuf, Bytes> {
+    let source = sources["motoko-core"].clone();
+    let mo_core = download_and_check_sha(client, source).await;
+    let mut map = HashMap::new();
+    tar_xzf(&mo_core, |path, content| {
+        let path = path.strip_prefix(".").unwrap_or(&path); // normalize ./x to x
+        if let Ok(file) = path.strip_prefix("src") {
+            map.insert(file.to_owned(), content);
+        }
+    });
+    map
+}
+
 fn tar_xzf(gz: &[u8], mut each: impl FnMut(PathBuf, Bytes)) {
     let mut tar = Archive::new(GzDecoder::new(gz));
     for entry in tar.entries().unwrap() {

src/dfx/src/config/cache.rs

@@ -126,7 +126,8 @@ pub fn install_version(
             #[cfg(unix)]
             {
                 let archive_path = dfx_core::fs::get_archive_path(&file)?;
-                let mode = if archive_path.starts_with("base/") {
+                let mode = if archive_path.starts_with("base/") || archive_path.starts_with("core/")
+                {
                     READ_USER_ONLY_PERMISSION
                 } else {
                     EXEC_READ_USER_ONLY_PERMISSION

src/dfx/src/lib/package_arguments.rs

@@ -8,7 +8,7 @@ use std::process::Command;
 
 /// Package arguments for moc as returned by a package tool like
 /// https://github.com/kritzcreek/vessel or, if there is no package
-/// tool, the base library.
+/// tool, the `base` and `core` packages.
 pub type PackageArguments = Vec<String>;
 
 #[context("Failed to load package arguments.")]
@@ -19,13 +19,20 @@ pub fn load(
     project_root: &Path,
 ) -> DfxResult<PackageArguments> {
     if packtool.is_none() {
-        let stdlib_path = cache
-            .get_binary_command_path(env, "base")?
-            .into_os_string()
-            .into_string()
-            .map_err(|_| anyhow!("Path contains invalid Unicode data."))?;
-        let base = vec![String::from("--package"), String::from("base"), stdlib_path];
-        return Ok(base);
+        let mut flags = Vec::new();
+        for package_name in ["base", "core"] {
+            let package_path = cache
+                .get_binary_command_path(env, package_name)?
+                .into_os_string()
+                .into_string()
+                .map_err(|_| anyhow!("Path contains invalid Unicode data."))?;
+            flags.extend_from_slice(&[
+                String::from("--package"),
+                String::from(package_name),
+                package_path,
+            ]);
+        }
+        return Ok(flags);
     }
 
     let commandline: Vec<String> = packtool