From 790deec36511a5935818e29d8f46d443c7bff380 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kat=20March=C3=A1n?= Date: Sun, 7 Jul 2019 12:27:26 -0700 Subject: [PATCH 01/16] meta: Adding workspace config (#1) --- Cargo.lock | 4 ++++ Cargo.toml | 4 ++++ packages/entropic-client/Cargo.toml | 9 +++++++++ packages/entropic-client/src/lib.rs | 7 +++++++ 4 files changed, 24 insertions(+) create mode 100644 packages/entropic-client/Cargo.toml create mode 100644 packages/entropic-client/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 448c933..66d3f27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1229,6 +1229,10 @@ name = "smallvec" version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "something" +version = "0.1.0" + [[package]] name = "spin" version = "0.5.0" diff --git a/Cargo.toml b/Cargo.toml index 7d8ef1f..6dbac7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,3 +12,7 @@ cacache = "1.0.1" ssri = "3.0.0" reqwest = "0.9.18" log = "0.4.6" + +[workspace] +members = [".", "packages/*"] +default-members = [".", "packages/*"] diff --git a/packages/entropic-client/Cargo.toml b/packages/entropic-client/Cargo.toml new file mode 100644 index 0000000..655ddf2 --- /dev/null +++ b/packages/entropic-client/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "entropic-client" +version = "0.1.0" +authors = ["Kat Marchán "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/packages/entropic-client/src/lib.rs b/packages/entropic-client/src/lib.rs new file mode 100644 index 0000000..31e1bb2 --- /dev/null +++ b/packages/entropic-client/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} From 0d5a0c1bfb5113a747b6170f12b0a327efb1d98c Mon Sep 17 00:00:00 2001 From: Milan Loveless Date: Fri, 12 Jul 2019 15:59:51 -0700 Subject: [PATCH 02/16] working proof of concept --- Cargo.lock | 91 ++++++++++++++++++++++++++++++++++-- packages/scanner/Cargo.toml | 11 +++++ packages/scanner/src/main.rs | 36 ++++++++++++++ 3 files changed, 134 insertions(+), 4 deletions(-) create mode 100644 packages/scanner/Cargo.toml create mode 100644 packages/scanner/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 66d3f27..b28b166 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -312,6 +312,22 @@ dependencies = [ "cfg-if 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "entropic-client" +version = "0.1.0" + +[[package]] +name = "env_logger" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "atty 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 1.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "error-chain" version = "0.12.1" @@ -472,6 +488,14 @@ name = "httparse" version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "humantime" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "hyper" version = "0.12.31" @@ -858,6 +882,11 @@ dependencies = [ "url 1.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "quick-error" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "quote" version = "0.6.12" @@ -1081,6 +1110,31 @@ dependencies = [ "uuid 0.7.4 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "resast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "ress" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ressa" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "backtrace 0.3.32 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "resast 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ress 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rustc-demangle" version = "0.1.15" @@ -1107,6 +1161,14 @@ dependencies = [ "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "scanner" +version = "0.1.0" +dependencies = [ + "resast 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ressa 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "schannel" version = "0.1.15" @@ -1229,10 +1291,6 @@ name = "smallvec" version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -[[package]] -name = "something" -version = "0.1.0" - [[package]] name = "spin" version = "0.5.0" @@ -1305,6 +1363,14 @@ dependencies = [ "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "termcolor" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "termion" version = "1.5.3" @@ -1630,6 +1696,15 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "wincolor" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ws2_32-sys" version = "0.2.1" @@ -1675,6 +1750,7 @@ dependencies = [ "checksum dtoa 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ea57b42383d091c85abcc2706240b94ab2a8fa1fc81c10ff23c4de06e2a90b5e" "checksum either 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5527cfe0d098f36e3f8839852688e63c8fff1c90b2b405aef730615f9a7bcf7b" "checksum encoding_rs 0.8.17 (registry+https://github.com/rust-lang/crates.io-index)" = "4155785c79f2f6701f185eb2e6b4caf0555ec03477cb4c70db67b465311620ed" +"checksum env_logger 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" "checksum error-chain 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "3ab49e9dcb602294bc42f9a7dfc9bc6e936fca4418ea300dbfb84fe16de0b7d9" "checksum failure 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "795bd83d3abeb9220f257e597aa0080a508b27533824adf336529648f6abf7e2" "checksum failure_derive 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ea1063915fd7ef4309e222a5a07cf9c319fb9c7836b1f89b85458672dbb127e1" @@ -1695,6 +1771,7 @@ dependencies = [ "checksum http 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "eed324f0f0daf6ec10c474f150505af2c143f251722bf9dbd1261bd1f2ee2c1a" "checksum http-body 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6741c859c1b2463a423a1dbce98d418e6c3c3fc720fb0d45528657320920292d" "checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +"checksum humantime 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3ca7e5f2e110db35f93b837c81797f3714500b81d517bf20c431b16d3ca4f114" "checksum hyper 0.12.31 (registry+https://github.com/rust-lang/crates.io-index)" = "6481fff8269772d4463253ca83c788104a7305cb3fb9136bc651a6211e46e03f" "checksum hyper-tls 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "3a800d6aa50af4b5850b2b0f659625ce9504df908e9733b635720483be26174f" "checksum idna 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "38f09e0f0b1fb55fdee1f17470ad800da77af5186a1a76c026b679358b7e844e" @@ -1738,6 +1815,7 @@ dependencies = [ "checksum ppv-lite86 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e3cbf9f658cdb5000fcf6f362b8ea2ba154b9f146a61c7a20d647034c6b6561b" "checksum proc-macro2 0.4.30 (registry+https://github.com/rust-lang/crates.io-index)" = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" "checksum publicsuffix 1.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "5afecba86dcf1e4fd610246f89899d1924fe12e1e89f555eb7c7f710f3c5ad1d" +"checksum quick-error 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "9274b940887ce9addde99c4eee6b5c44cc494b182b97e73dc8ffdcb3397fd3f0" "checksum quote 0.6.12 (registry+https://github.com/rust-lang/crates.io-index)" = "faf4799c5d274f3868a4aae320a0a182cbd2baee377b378f080e16a23e9d80db" "checksum rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" "checksum rand 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d47eab0e83d9693d40f825f86948aa16eff6750ead4bdffc4ab95b8b3a7f052c" @@ -1760,6 +1838,9 @@ dependencies = [ "checksum regex-syntax 0.6.8 (registry+https://github.com/rust-lang/crates.io-index)" = "9b01330cce219c1c6b2e209e5ed64ccd587ae5c67bed91c0b49eecf02ae40e21" "checksum remove_dir_all 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" "checksum reqwest 0.9.18 (registry+https://github.com/rust-lang/crates.io-index)" = "00eb63f212df0e358b427f0f40aa13aaea010b470be642ad422bcbca2feff2e4" +"checksum resast 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5a836e0d4ce031353a8f6deb745043ec65b006b9b4912fbb66feb564d9f2b1e5" +"checksum ress 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "98b87fd8b15cd43ced87c2f465d821e63842b33953048f8d5473dc50c68b6b4e" +"checksum ressa 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "480e184fc4255ed1542485f0e2f8a7362e3b32e6404d1030abe187591900a8e7" "checksum rustc-demangle 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)" = "a7f4dccf6f4891ebcc0c39f9b6eb1a83b9bf5d747cb439ec6fba4f3b977038af" "checksum rustc_version 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" "checksum ryu 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c92464b447c0ee8c4fb3824ecc8383b81717b9f1e74ba2e72540aef7b9f82997" @@ -1787,6 +1868,7 @@ dependencies = [ "checksum syn 0.15.39 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d960b829a55e56db167e861ddb43602c003c7be0bee1d345021703fac2fb7c" "checksum synstructure 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "02353edf96d6e4dc81aea2d8490a7e9db177bf8acb0e951c24940bf866cb313f" "checksum tempfile 3.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +"checksum termcolor 1.0.5 (registry+https://github.com/rust-lang/crates.io-index)" = "96d6098003bde162e4277c70665bd87c326f5a0c3f3fbfb285787fa482d54e6e" "checksum termion 1.5.3 (registry+https://github.com/rust-lang/crates.io-index)" = "6a8fb22f7cde82c8220e5aeacb3258ed7ce996142c77cba193f203515e26c330" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" "checksum thread_local 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "c6b53e329000edc2b34dbe8545fd20e55a333362d0a321909685a19bd28c3f1b" @@ -1826,4 +1908,5 @@ dependencies = [ "checksum winapi-i686-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" "checksum winapi-util 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7168bab6e1daee33b4557efd0e95d5ca70a03706d39fa5f3fe7a236f584b03c9" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +"checksum wincolor 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "561ed901ae465d6185fa7864d63fbd5720d0ef718366c9a4dc83cf6170d7e9ba" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" diff --git a/packages/scanner/Cargo.toml b/packages/scanner/Cargo.toml new file mode 100644 index 0000000..0199726 --- /dev/null +++ b/packages/scanner/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "scanner" +version = "0.1.0" +authors = ["Milan Loveless "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +ressa = "0.6.0" +resast = "0.3.0" \ No newline at end of file diff --git a/packages/scanner/src/main.rs b/packages/scanner/src/main.rs new file mode 100644 index 0000000..c9818cf --- /dev/null +++ b/packages/scanner/src/main.rs @@ -0,0 +1,36 @@ +use ressa::Parser; +use resast; + +fn main() { + let js = "const _ = require('lodash'); import * as moment from 'moment';"; + let p = Parser::new(&js).unwrap(); + + for part in p { + if let Ok(the_part) = part { + match the_part { + resast::ref_tree::ProgramPart::Decl(some_part) => log_the_declaration(some_part), + _ => println!("You lose!"), + } + } + } +} + +fn log_the_declaration (declaration: resast::ref_tree::prelude::Decl) { + match declaration { + resast::ref_tree::prelude::Decl::Import(import) => println!("{:?}", import), + resast::ref_tree::prelude::Decl::Variable(_, variable_vec) => log_if_require(variable_vec), + _ => println!("You Lose!") + } + +} + +fn log_if_require (variable_vec: Vec) { + for v in variable_vec { + if let Some(variable_init) = v.init { + match variable_init { + resast::ref_tree::prelude::Expr::Call(call) => println!("{:?}", call), + _ => println!("You lose!") + } + } + } +} From 659bc01e1e89f88b416ddf04994ad1299d0f9aec Mon Sep 17 00:00:00 2001 From: Milan Loveless Date: Fri, 12 Jul 2019 17:24:28 -0700 Subject: [PATCH 03/16] added @matuzak to the authors array --- packages/scanner/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scanner/Cargo.toml b/packages/scanner/Cargo.toml index 0199726..db8f29c 100644 --- a/packages/scanner/Cargo.toml +++ b/packages/scanner/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "scanner" version = "0.1.0" -authors = ["Milan Loveless "] +authors = ["Milan Loveless ", "Michael Matuzak "] edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 130aeeef5ad4a07edf26917b5035141976fdc43b Mon Sep 17 00:00:00 2001 From: Milan Loveless Date: Fri, 12 Jul 2019 17:25:57 -0700 Subject: [PATCH 04/16] added @zkat to authors array --- packages/scanner/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scanner/Cargo.toml b/packages/scanner/Cargo.toml index db8f29c..050f49e 100644 --- a/packages/scanner/Cargo.toml +++ b/packages/scanner/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "scanner" version = "0.1.0" -authors = ["Milan Loveless ", "Michael Matuzak "] +authors = ["Kat Marchán ", "Milan Loveless ", "Michael Matuzak "] edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 600adcbd80ffc401d0276e1ff7d779833f961005 Mon Sep 17 00:00:00 2001 From: Milan Loveless Date: Mon, 15 Jul 2019 00:18:19 -0700 Subject: [PATCH 05/16] changed to lib, full top level require() support, and tests --- packages/scanner/src/lib.rs | 78 ++++++++++++++++++++++++++++++++++++ packages/scanner/src/main.rs | 36 ----------------- 2 files changed, 78 insertions(+), 36 deletions(-) create mode 100644 packages/scanner/src/lib.rs delete mode 100644 packages/scanner/src/main.rs diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs new file mode 100644 index 0000000..0e0970a --- /dev/null +++ b/packages/scanner/src/lib.rs @@ -0,0 +1,78 @@ +use resast::ref_tree::prelude::*; +use ressa::Parser; +use std::vec::Vec; + +pub fn parse_js<'a>(js: &'a str) -> Vec<&'a str> { + let p = Parser::new(js).unwrap(); + let mut pd = Vec::<&str>::new(); + for part in p { + if let Ok(the_part) = part { + match the_part { + resast::ref_tree::ProgramPart::Decl(some_part) => { + pd = match_declaration(some_part, pd) + } + the_thing => println!("Not a program part: {:?}", the_thing), + } + } + } + return pd; +} + +fn match_declaration<'a>(declaration: Decl<'a>, mut pd: Vec<&'a str>) -> Vec<&'a str> { + match declaration { + Decl::Import(import) => println!("{:?}", import), + Decl::Variable(_, variable_vec) => pd = match_variable(variable_vec, pd), + the_thing => println!("log_declaration: {:?}", the_thing), + } + pd +} + +fn match_variable<'a>( + variable_vec: Vec>, + mut pd: Vec<&'a str>, +) -> Vec<&'a str> { + for v in variable_vec { + if let Some(variable_init) = v.init { + match variable_init { + Expr::Call(call) => pd = match_expr(call, pd), + the_thing => println!("log_require: {:?}", the_thing), + } + } + } + pd +} + +fn match_expr<'a>( + expression: resast::ref_tree::expr::CallExpr<'a>, + mut pd: Vec<&'a str>, +) -> Vec<&'a str> { + if let Expr::Ident(callee) = *expression.callee { + if callee == "require" { + if let Some(argument) = expression.arguments.get(0) { + println!("{:?}", argument); + if let resast::ref_tree::expr::Expr::Literal(literal) = argument { + if let resast::ref_tree::expr::Literal::String(string_arg) = literal { + let quotes: &[_] = &['\'', '\"']; + pd.push(string_arg.trim_matches(quotes)); + } + } + } + } + } + pd +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn top_level_require_single_quotes() { + let js = "const _ = require('lodash');"; + assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) + } + #[test] + fn top_level_require_double_quotes() { + let js = "const _ = require(\"lodash\");"; + assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) + } +} diff --git a/packages/scanner/src/main.rs b/packages/scanner/src/main.rs deleted file mode 100644 index c9818cf..0000000 --- a/packages/scanner/src/main.rs +++ /dev/null @@ -1,36 +0,0 @@ -use ressa::Parser; -use resast; - -fn main() { - let js = "const _ = require('lodash'); import * as moment from 'moment';"; - let p = Parser::new(&js).unwrap(); - - for part in p { - if let Ok(the_part) = part { - match the_part { - resast::ref_tree::ProgramPart::Decl(some_part) => log_the_declaration(some_part), - _ => println!("You lose!"), - } - } - } -} - -fn log_the_declaration (declaration: resast::ref_tree::prelude::Decl) { - match declaration { - resast::ref_tree::prelude::Decl::Import(import) => println!("{:?}", import), - resast::ref_tree::prelude::Decl::Variable(_, variable_vec) => log_if_require(variable_vec), - _ => println!("You Lose!") - } - -} - -fn log_if_require (variable_vec: Vec) { - for v in variable_vec { - if let Some(variable_init) = v.init { - match variable_init { - resast::ref_tree::prelude::Expr::Call(call) => println!("{:?}", call), - _ => println!("You lose!") - } - } - } -} From 320bab41eca913fefe8f16834cb1bbe2c9cb541e Mon Sep 17 00:00:00 2001 From: Michael Matuzak Date: Tue, 16 Jul 2019 13:10:57 -0700 Subject: [PATCH 06/16] refactor: cleaning up some of the resast uses --- packages/scanner/src/lib.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 0e0970a..16051b8 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -1,4 +1,5 @@ use resast::ref_tree::prelude::*; +use resast::ref_tree::{ProgramPart, decl, expr}; use ressa::Parser; use std::vec::Vec; @@ -8,14 +9,14 @@ pub fn parse_js<'a>(js: &'a str) -> Vec<&'a str> { for part in p { if let Ok(the_part) = part { match the_part { - resast::ref_tree::ProgramPart::Decl(some_part) => { + ProgramPart::Decl(some_part) => { pd = match_declaration(some_part, pd) } the_thing => println!("Not a program part: {:?}", the_thing), } } } - return pd; + pd } fn match_declaration<'a>(declaration: Decl<'a>, mut pd: Vec<&'a str>) -> Vec<&'a str> { @@ -28,7 +29,7 @@ fn match_declaration<'a>(declaration: Decl<'a>, mut pd: Vec<&'a str>) -> Vec<&'a } fn match_variable<'a>( - variable_vec: Vec>, + variable_vec: Vec>, mut pd: Vec<&'a str>, ) -> Vec<&'a str> { for v in variable_vec { @@ -50,8 +51,8 @@ fn match_expr<'a>( if callee == "require" { if let Some(argument) = expression.arguments.get(0) { println!("{:?}", argument); - if let resast::ref_tree::expr::Expr::Literal(literal) = argument { - if let resast::ref_tree::expr::Literal::String(string_arg) = literal { + if let expr::Expr::Literal(literal) = argument { + if let expr::Literal::String(string_arg) = literal { let quotes: &[_] = &['\'', '\"']; pd.push(string_arg.trim_matches(quotes)); } From f812845f97ffa6e03a471f3f9ef25348b4fbac13 Mon Sep 17 00:00:00 2001 From: Michael Matuzak Date: Tue, 16 Jul 2019 14:26:51 -0700 Subject: [PATCH 07/16] feat: `import`s! This parses some various imports including dynamic imports. There is probably some broken stuff, but I think it works overall. --- packages/scanner/src/lib.rs | 50 +++++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 16051b8..ac9f8f6 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -11,7 +11,16 @@ pub fn parse_js<'a>(js: &'a str) -> Vec<&'a str> { match the_part { ProgramPart::Decl(some_part) => { pd = match_declaration(some_part, pd) - } + }, + + // Dynamic imports + ProgramPart::Stmt(some_part) => { + if let Stmt::Expr(expr) = some_part { + if let Expr::Call(call) = expr { + pd = match_expr(call, pd) + } + } + }, the_thing => println!("Not a program part: {:?}", the_thing), } } @@ -21,13 +30,22 @@ pub fn parse_js<'a>(js: &'a str) -> Vec<&'a str> { fn match_declaration<'a>(declaration: Decl<'a>, mut pd: Vec<&'a str>) -> Vec<&'a str> { match declaration { - Decl::Import(import) => println!("{:?}", import), + Decl::Import(import) => pd = match_import(import, pd), Decl::Variable(_, variable_vec) => pd = match_variable(variable_vec, pd), the_thing => println!("log_declaration: {:?}", the_thing), } pd } +fn match_import<'a>(import: Box>, mut pd: Vec<&'a str>) -> Vec<&'a str> { + let source = import.source; + if let expr::Literal::String(string_arg) = source { + let quotes: &[_] = &['\'', '\"']; + pd.push(string_arg.trim_matches(quotes)); + } + pd +} + fn match_variable<'a>( variable_vec: Vec>, mut pd: Vec<&'a str>, @@ -43,12 +61,10 @@ fn match_variable<'a>( pd } -fn match_expr<'a>( - expression: resast::ref_tree::expr::CallExpr<'a>, - mut pd: Vec<&'a str>, -) -> Vec<&'a str> { +fn match_expr<'a>(expression: expr::CallExpr<'a>, mut pd: Vec<&'a str>,) -> Vec<&'a str> { if let Expr::Ident(callee) = *expression.callee { - if callee == "require" { + println!("callee: {:?}", callee); + if callee == "require" || callee == "import" { if let Some(argument) = expression.arguments.get(0) { println!("{:?}", argument); if let expr::Expr::Literal(literal) = argument { @@ -76,4 +92,24 @@ mod tests { let js = "const _ = require(\"lodash\");"; assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) } + #[test] + fn top_level_default_import() { + let js = "import _ from 'lodash';"; + assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) + } + #[test] + fn top_level_import_alias() { + let js = "import * as lodash from \"lodash\";"; + assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) + } + #[test] + fn top_level_import_named() { + let js = "import { map } from \"lodash\";"; + assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) + } + #[test] + fn dynamic_import() { + let js = "import('lodash');"; + assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) + } } From 38cb66be1ed6328bb1c3ae22d3e1406472dcd72c Mon Sep 17 00:00:00 2001 From: Michael Matuzak Date: Wed, 17 Jul 2019 18:49:29 -0700 Subject: [PATCH 08/16] refactor: rm dynamic imports --- packages/scanner/src/lib.rs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index ac9f8f6..264ca17 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -12,15 +12,6 @@ pub fn parse_js<'a>(js: &'a str) -> Vec<&'a str> { ProgramPart::Decl(some_part) => { pd = match_declaration(some_part, pd) }, - - // Dynamic imports - ProgramPart::Stmt(some_part) => { - if let Stmt::Expr(expr) = some_part { - if let Expr::Call(call) = expr { - pd = match_expr(call, pd) - } - } - }, the_thing => println!("Not a program part: {:?}", the_thing), } } @@ -107,9 +98,4 @@ mod tests { let js = "import { map } from \"lodash\";"; assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) } - #[test] - fn dynamic_import() { - let js = "import('lodash');"; - assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) - } } From 391ece330023b9b80204eb2605423b18866f7caf Mon Sep 17 00:00:00 2001 From: Michael Matuzak Date: Wed, 24 Jul 2019 15:18:07 -0700 Subject: [PATCH 09/16] remove import and parse require statements --- packages/scanner/src/lib.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 264ca17..f30d630 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -12,7 +12,14 @@ pub fn parse_js<'a>(js: &'a str) -> Vec<&'a str> { ProgramPart::Decl(some_part) => { pd = match_declaration(some_part, pd) }, - the_thing => println!("Not a program part: {:?}", the_thing), + ProgramPart::Stmt(some_part) => { + if let Stmt::Expr(expr) = some_part { + if let Expr::Call(call) = expr { + pd = match_expr(call, pd) + } + } + }, + _ => (), } } } @@ -23,7 +30,7 @@ fn match_declaration<'a>(declaration: Decl<'a>, mut pd: Vec<&'a str>) -> Vec<&'a match declaration { Decl::Import(import) => pd = match_import(import, pd), Decl::Variable(_, variable_vec) => pd = match_variable(variable_vec, pd), - the_thing => println!("log_declaration: {:?}", the_thing), + _ => (), } pd } @@ -45,7 +52,7 @@ fn match_variable<'a>( if let Some(variable_init) = v.init { match variable_init { Expr::Call(call) => pd = match_expr(call, pd), - the_thing => println!("log_require: {:?}", the_thing), + _ => (), } } } @@ -54,10 +61,8 @@ fn match_variable<'a>( fn match_expr<'a>(expression: expr::CallExpr<'a>, mut pd: Vec<&'a str>,) -> Vec<&'a str> { if let Expr::Ident(callee) = *expression.callee { - println!("callee: {:?}", callee); - if callee == "require" || callee == "import" { + if callee == "require" { if let Some(argument) = expression.arguments.get(0) { - println!("{:?}", argument); if let expr::Expr::Literal(literal) = argument { if let expr::Literal::String(string_arg) = literal { let quotes: &[_] = &['\'', '\"']; @@ -84,6 +89,11 @@ mod tests { assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) } #[test] + fn top_level_require_statement() { + let js = "require(\"lodash\");"; + assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) + } + #[test] fn top_level_default_import() { let js = "import _ from 'lodash';"; assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) From 8f19037fac3dd266098be8318a90c6064bd5148a Mon Sep 17 00:00:00 2001 From: Michael Matuzak Date: Wed, 24 Jul 2019 15:22:24 -0700 Subject: [PATCH 10/16] parse_js returns an `Error` --- packages/scanner/src/lib.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index f30d630..5a91118 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -3,8 +3,8 @@ use resast::ref_tree::{ProgramPart, decl, expr}; use ressa::Parser; use std::vec::Vec; -pub fn parse_js<'a>(js: &'a str) -> Vec<&'a str> { - let p = Parser::new(js).unwrap(); +pub fn parse_js<'a>(js: &'a str) -> Result, ressa::Error> { + let p = Parser::new(js)?; let mut pd = Vec::<&str>::new(); for part in p { if let Ok(the_part) = part { @@ -23,7 +23,7 @@ pub fn parse_js<'a>(js: &'a str) -> Vec<&'a str> { } } } - pd + Ok(pd) } fn match_declaration<'a>(declaration: Decl<'a>, mut pd: Vec<&'a str>) -> Vec<&'a str> { @@ -81,31 +81,31 @@ mod tests { #[test] fn top_level_require_single_quotes() { let js = "const _ = require('lodash');"; - assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) + assert_eq!(&"lodash", parse_js(js).unwrap().get(0).unwrap()) } #[test] fn top_level_require_double_quotes() { let js = "const _ = require(\"lodash\");"; - assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) + assert_eq!(&"lodash", parse_js(js).unwrap().get(0).unwrap()) } #[test] fn top_level_require_statement() { let js = "require(\"lodash\");"; - assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) + assert_eq!(&"lodash", parse_js(js).unwrap().get(0).unwrap()) } #[test] fn top_level_default_import() { let js = "import _ from 'lodash';"; - assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) + assert_eq!(&"lodash", parse_js(js).unwrap().get(0).unwrap()) } #[test] fn top_level_import_alias() { let js = "import * as lodash from \"lodash\";"; - assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) + assert_eq!(&"lodash", parse_js(js).unwrap().get(0).unwrap()) } #[test] fn top_level_import_named() { let js = "import { map } from \"lodash\";"; - assert_eq!(&"lodash", parse_js(js).get(0).unwrap()) + assert_eq!(&"lodash", parse_js(js).unwrap().get(0).unwrap()) } } From 8ae3c1531e72a3641be6f04b09f1a7082a194874 Mon Sep 17 00:00:00 2001 From: Michael Matuzak Date: Wed, 24 Jul 2019 15:34:25 -0700 Subject: [PATCH 11/16] rust magic --- packages/scanner/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 5a91118..be8db55 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -60,8 +60,8 @@ fn match_variable<'a>( } fn match_expr<'a>(expression: expr::CallExpr<'a>, mut pd: Vec<&'a str>,) -> Vec<&'a str> { - if let Expr::Ident(callee) = *expression.callee { - if callee == "require" { + if let Expr::Ident(callee) = &*expression.callee { + if callee == &"require" { if let Some(argument) = expression.arguments.get(0) { if let expr::Expr::Literal(literal) = argument { if let expr::Literal::String(string_arg) = literal { From e67e2a3f026f5ed257b1a858f018834aa7b2061c Mon Sep 17 00:00:00 2001 From: Milan Loveless Date: Wed, 24 Jul 2019 15:54:46 -0700 Subject: [PATCH 12/16] added newline --- packages/scanner/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scanner/Cargo.toml b/packages/scanner/Cargo.toml index 050f49e..1657d9d 100644 --- a/packages/scanner/Cargo.toml +++ b/packages/scanner/Cargo.toml @@ -8,4 +8,4 @@ edition = "2018" [dependencies] ressa = "0.6.0" -resast = "0.3.0" \ No newline at end of file +resast = "0.3.0" From 0009a016cf780dc2d2fede018768c093fa263299 Mon Sep 17 00:00:00 2001 From: Milan Loveless Date: Sat, 27 Jul 2019 12:03:32 -0700 Subject: [PATCH 13/16] reorganized code and refactored fn trim_quotes --- packages/scanner/src/lib.rs | 61 +++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 30 deletions(-) diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index be8db55..386c308 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -1,24 +1,16 @@ use resast::ref_tree::prelude::*; -use resast::ref_tree::{ProgramPart, decl, expr}; +use resast::ref_tree::{decl, expr, ProgramPart}; use ressa::Parser; use std::vec::Vec; pub fn parse_js<'a>(js: &'a str) -> Result, ressa::Error> { let p = Parser::new(js)?; let mut pd = Vec::<&str>::new(); - for part in p { - if let Ok(the_part) = part { - match the_part { - ProgramPart::Decl(some_part) => { - pd = match_declaration(some_part, pd) - }, - ProgramPart::Stmt(some_part) => { - if let Stmt::Expr(expr) = some_part { - if let Expr::Call(call) = expr { - pd = match_expr(call, pd) - } - } - }, + for part_result in p { + if let Ok(part) = part_result { + match part { + ProgramPart::Decl(declaration) => pd = decl_to_impt_or_var(declaration, pd), + ProgramPart::Stmt(statement) => pd = stmt_to_call(statement, pd), _ => (), } } @@ -26,47 +18,49 @@ pub fn parse_js<'a>(js: &'a str) -> Result, ressa::Error> { Ok(pd) } -fn match_declaration<'a>(declaration: Decl<'a>, mut pd: Vec<&'a str>) -> Vec<&'a str> { +fn decl_to_impt_or_var<'a>(declaration: Decl<'a>, mut pd: Vec<&'a str>) -> Vec<&'a str> { match declaration { - Decl::Import(import) => pd = match_import(import, pd), - Decl::Variable(_, variable_vec) => pd = match_variable(variable_vec, pd), + Decl::Import(import) => pd = impt_to_str(import, pd), + Decl::Variable(_, variable_vec) => pd = var_init_to_call(variable_vec, pd), _ => (), } pd } -fn match_import<'a>(import: Box>, mut pd: Vec<&'a str>) -> Vec<&'a str> { - let source = import.source; - if let expr::Literal::String(string_arg) = source { - let quotes: &[_] = &['\'', '\"']; - pd.push(string_arg.trim_matches(quotes)); +fn stmt_to_call<'a>(statement: Stmt<'a>, mut pd: Vec<&'a str>) -> Vec<&'a str> { + if let Stmt::Expr(expression) = statement { + if let Expr::Call(call) = expression { + pd = call_to_str(call, pd) + } } pd } -fn match_variable<'a>( +fn var_init_to_call<'a>( variable_vec: Vec>, mut pd: Vec<&'a str>, ) -> Vec<&'a str> { for v in variable_vec { if let Some(variable_init) = v.init { - match variable_init { - Expr::Call(call) => pd = match_expr(call, pd), - _ => (), + if let Expr::Call(call) = variable_init { + pd = call_to_str(call, pd); } } } pd } -fn match_expr<'a>(expression: expr::CallExpr<'a>, mut pd: Vec<&'a str>,) -> Vec<&'a str> { +fn trim_quotes<'a>(string: &'a str) -> &'a str { + string.trim_matches(|c| c == '\'' || c == '\"') +} + +fn call_to_str<'a>(expression: expr::CallExpr<'a>, mut pd: Vec<&'a str>) -> Vec<&'a str> { if let Expr::Ident(callee) = &*expression.callee { if callee == &"require" { if let Some(argument) = expression.arguments.get(0) { if let expr::Expr::Literal(literal) = argument { - if let expr::Literal::String(string_arg) = literal { - let quotes: &[_] = &['\'', '\"']; - pd.push(string_arg.trim_matches(quotes)); + if let expr::Literal::String(string) = literal { + pd.push(trim_quotes(string)); } } } @@ -75,6 +69,13 @@ fn match_expr<'a>(expression: expr::CallExpr<'a>, mut pd: Vec<&'a str>,) -> Vec< pd } +fn impt_to_str<'a>(import: Box>, mut pd: Vec<&'a str>) -> Vec<&'a str> { + if let expr::Literal::String(string) = import.source { + pd.push(trim_quotes(string)); + } + pd +} + #[cfg(test)] mod tests { use super::*; From f0062cf77bc9cee013909ea0966ed30a384cbd95 Mon Sep 17 00:00:00 2001 From: Milan Loveless Date: Sat, 27 Jul 2019 12:31:58 -0700 Subject: [PATCH 14/16] added more tests --- packages/scanner/src/lib.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 386c308..3cbd213 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -109,4 +109,32 @@ mod tests { let js = "import { map } from \"lodash\";"; assert_eq!(&"lodash", parse_js(js).unwrap().get(0).unwrap()) } + #[test] + fn no_valid_calls() { + let js = "const five = 5;"; + assert_eq!(None, parse_js(js).unwrap().get(0)); + } + #[test] + fn multi_line_one_valid_call() { + let js = &[ + "const five = 5;", + "const moment = require('moment');", + "start('now');", + ] + .join(&"\n"); + assert_eq!(&"moment", parse_js(js).unwrap().get(0).unwrap()); + } + #[test] + fn multi_line_two_valid_calls() { + let js = &[ + "const five = 5;", + "const moment = require('moment');", + "import _ from 'lodash';", + "start();", + ] + .join(&"\n"); + let packages = parse_js(js).unwrap(); + assert_eq!(&"moment", packages.get(0).unwrap()); + assert_eq!(&"lodash", packages.get(1).unwrap()); + } } From 4b04a2666ad910f25984fa0b5282f934aef6b479 Mon Sep 17 00:00:00 2001 From: Milan Loveless Date: Sun, 28 Jul 2019 16:48:37 -0700 Subject: [PATCH 15/16] starting documentation --- packages/scanner/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 3cbd213..43ae5be 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -3,6 +3,7 @@ use resast::ref_tree::{decl, expr, ProgramPart}; use ressa::Parser; use std::vec::Vec; +#[doc = "Parses a `&a' str` containing the contents of a JavaScript file and returns a `Result>` containing a list of packages/files that are statically imported/required in root level scope of the file."] pub fn parse_js<'a>(js: &'a str) -> Result, ressa::Error> { let p = Parser::new(js)?; let mut pd = Vec::<&str>::new(); From 6dd4d9871a3ec7b404835aa3cd68d3d4ceab27df Mon Sep 17 00:00:00 2001 From: Milan Loveless Date: Thu, 15 Aug 2019 17:07:01 -0700 Subject: [PATCH 16/16] cleanup lifetime --- packages/scanner/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/scanner/src/lib.rs b/packages/scanner/src/lib.rs index 43ae5be..777e7d1 100644 --- a/packages/scanner/src/lib.rs +++ b/packages/scanner/src/lib.rs @@ -51,7 +51,7 @@ fn var_init_to_call<'a>( pd } -fn trim_quotes<'a>(string: &'a str) -> &'a str { +fn trim_quotes(string: &str) -> &str { string.trim_matches(|c| c == '\'' || c == '\"') }