From ff8bb64fd052c14b17fdd96214386bcf516f2954 Mon Sep 17 00:00:00 2001 From: Louis-Marie Givel Date: Sat, 19 Mar 2022 11:19:59 +0100 Subject: [PATCH] Add include filter It can be inserted in workspace files, to allow for composition Add tests --- src/filter/mod.rs | 136 ++++++++++++++++++ src/filter/parse.rs | 16 ++- tests/filter/include.t | 212 +++++++++++++++++++++++++++++ tests/filter/include_example.t | 140 +++++++++++++++++++ tests/filter/include_same_folder.t | 70 ++++++++++ tests/filter/workspace_unique.t | 22 ++- tests/proxy/clone_invalid_filter.t | 4 +- 7 files changed, 590 insertions(+), 10 deletions(-) create mode 100644 tests/filter/include.t create mode 100644 tests/filter/include_example.t create mode 100644 tests/filter/include_same_folder.t diff --git a/src/filter/mod.rs b/src/filter/mod.rs index 873238fd7..0ceaea638 100644 --- a/src/filter/mod.rs +++ b/src/filter/mod.rs @@ -72,6 +72,7 @@ enum Op { Prefix(std::path::PathBuf), Subdir(std::path::PathBuf), Workspace(std::path::PathBuf), + Include(std::path::PathBuf), Glob(String), @@ -169,6 +170,9 @@ fn spec2(op: &Op) -> String { Op::Workspace(path) => { format!(":workspace={}", path.to_string_lossy()) } + Op::Include(path) => { + format!(":include={}", path.to_string_lossy()) + } Op::Chain(a, b) => match (to_op(*a), to_op(*b)) { (Op::Subdir(p1), Op::Prefix(p2)) if p1 == p2 => { @@ -354,6 +358,54 @@ fn apply_to_commit2( )) .transpose(); } + Op::Include(include_path) => { + let normal_parents = commit + .parent_ids() + .map(|parent| transaction.get(filter, parent)) + .collect::>>(); + + let normal_parents = some_or!(normal_parents, { return Ok(None) }); + + let cw = parse::parse(&tree::get_blob(repo, &commit.tree()?, &include_path)) + .unwrap_or(to_filter(Op::Empty)); + + let extra_parents = commit + .parents() + .map(|parent| { + rs_tracing::trace_scoped!("parent", "id": parent.id().to_string()); + let pcw = parse::parse(&tree::get_blob( + repo, + &parent.tree().unwrap_or(tree::empty(repo)), + &include_path, + )) + .unwrap_or(to_filter(Op::Empty)); + + apply_to_commit2( + &to_op(opt::optimize(to_filter(Op::Subtract(cw, pcw)))), + &parent, + transaction, + ) + }) + .collect::>>>()?; + + let extra_parents = some_or!(extra_parents, { return Ok(None) }); + + let filtered_parent_ids = normal_parents + .into_iter() + .chain(extra_parents.into_iter()) + .collect(); + + let filtered_tree = apply(transaction, filter, commit.tree()?)?; + + return Some(history::create_filtered_commit( + commit, + filtered_parent_ids, + filtered_tree, + transaction, + filter, + )) + .transpose(); + } Op::Fold => { let filtered_parent_ids = commit .parents() @@ -521,6 +573,15 @@ fn apply2<'a>( } } + Op::Include(path) => { + let file = to_filter(Op::File(path.to_owned())); + if let Ok(cw) = parse::parse(&tree::get_blob(repo, &tree, &path)) { + apply(transaction, compose(file, cw), tree) + } else { + apply(transaction, file, tree) + } + } + Op::Compose(filters) => { let filtered: Vec<_> = filters .iter() @@ -626,6 +687,71 @@ fn unapply2<'a>( return Ok(r); } + Op::Include(path) => { + let root = to_filter(Op::File(path.to_owned())); + let mapped = &tree::get_blob(transaction.repo(), &tree, path); + let parsed = parse(mapped)?; + + let mut blob = String::new(); + if let Ok(c) = get_comments(mapped) { + if !c.is_empty() { + blob = c; + } + } + let blob = &format!("{}{}\n", &blob, pretty(parsed, 0)); + + // TODO: is this still necessary? + // Remove filters file from the tree to prevent it from being parsed again + // further down the callstack leading to endless recursion. + let tree = tree::insert( + transaction.repo(), + &tree, + path, + git2::Oid::zero(), + 0o0100644, + )?; + + // Insert a dummy file to prevent the directory from dissappearing through becoming + // empty. + let tree = tree::insert( + transaction.repo(), + &tree, + Path::new("DUMMY-df97a89d-b11f-4e1c-8400-345f895f0d40"), + transaction.repo().blob("".as_bytes())?, + 0o0100644, + )?; + + let r = unapply( + transaction, + compose(root, parsed), + tree.clone(), + parent_tree, + )?; + + // Remove the dummy file inserted above + let r = tree::insert( + transaction.repo(), + &r, + &path.join("DUMMY-df97a89d-b11f-4e1c-8400-345f895f0d40"), + git2::Oid::zero(), + 0o0100644, + )?; + + // Put the filters file back to it's target location. + let r = if !mapped.is_empty() { + tree::insert( + transaction.repo(), + &r, + &path, + transaction.repo().blob(blob.as_bytes())?, + 0o0100644, // Should this handle filemode? + )? + } else { + r + }; + + return Ok(r); + } Op::Compose(filters) => { let mut remaining = tree.clone(); let mut result = parent_tree.clone(); @@ -748,6 +874,16 @@ pub fn compute_warnings<'a>( } } + if let Op::Include(path) = to_op(filter) { + let full_filter = &tree::get_blob(transaction.repo(), &tree, &path); + if let Ok(res) = parse(full_filter) { + filter = res; + } else { + warnings.push("couldn't parse include file\n".to_string()); + return warnings; + } + } + let filter = opt::flatten(filter); if let Op::Compose(filters) = to_op(filter) { for f in filters { diff --git a/src/filter/parse.rs b/src/filter/parse.rs index 33306aba2..a68eee39c 100644 --- a/src/filter/parse.rs +++ b/src/filter/parse.rs @@ -7,6 +7,7 @@ fn make_op(args: &[&str]) -> JoshResult { ["empty"] => Ok(Op::Empty), ["prefix", arg] => Ok(Op::Prefix(Path::new(arg).to_owned())), ["workspace", arg] => Ok(Op::Workspace(Path::new(arg).to_owned())), + ["include", arg] => Ok(Op::Include(Path::new(arg).to_owned())), ["prefix"] => Err(josh_error(indoc!( r#" Filter ":prefix" requires an argument. @@ -15,7 +16,7 @@ fn make_op(args: &[&str]) -> JoshResult { :prefix=path - Where `path` is path to be used as a prefix + Where `path` is the path to be used as a prefix "# ))), ["workspace"] => Err(josh_error(indoc!( @@ -26,7 +27,18 @@ fn make_op(args: &[&str]) -> JoshResult { :workspace=path - Where `path` is path to the directory where workspace.josh file is located + Where `path` is the path to the directory where workspace.josh file is located + "# + ))), + ["include"] => Err(josh_error(indoc!( + r#" + Filter ":include" requires an argument. + + Note: use "=" to provide the argument value: + + :include=path/to/include.josh + + Where `path/to/include.josh` is the path to the include file "# ))), ["SQUASH"] => Ok(Op::Squash), diff --git a/tests/filter/include.t b/tests/filter/include.t new file mode 100644 index 000000000..a6354f845 --- /dev/null +++ b/tests/filter/include.t @@ -0,0 +1,212 @@ + $ export TERM=dumb + $ export RUST_LOG_STYLE=never + +-------- +1) Setup +-------- + + $ git init real_repo 1> /dev/null + $ cd real_repo + + $ mkdir sub1 + $ echo contents1 > sub1/file1 + $ echo contents4 > sub1/file4 + $ git add sub1 + $ git commit -m "add file1" 1> /dev/null + + $ echo contents3 > file3 + $ git add file3 + $ git commit -m "add file3" 1> /dev/null + + $ mkdir -p sub2/subsub + $ echo contents1 > sub2/subsub/file2 + $ git add sub2 + $ git commit -m "add file2" 1> /dev/null + +--------------- +2) include file +--------------- + + $ mkdir f + $ cat > f/include.josh < :/sub1::file1 + > ::sub2/subsub/ + > a = :/sub1 + > EOF + $ git add f/include.josh + $ git commit -m "add include file" 1> /dev/null + + $ josh-filter -s :include=f/include.josh master --update refs/josh/master + [1] :/sub1 + [1] :/subsub + [1] ::file1 + [1] :[ + ::file1 + :prefix=a + ] + [1] :prefix=a + [1] :prefix=sub2 + [1] :prefix=subsub + [2] :/sub2 + [2] :[ + :/sub1:[ + ::file1 + :prefix=a + ] + ::sub2/subsub/ + ] + [2] :include=f/include.josh + + $ git log --graph --pretty=%s refs/josh/master + * add include file + * add file2 + * add file1 + + $ git checkout refs/josh/master 2> /dev/null + $ tree + . + |-- a + | `-- file4 + |-- f + | `-- include.josh + |-- file1 + `-- sub2 + `-- subsub + `-- file2 + + 4 directories, 4 files + +---------------------------------------- +3) workspace file including include file +---------------------------------------- + + $ git checkout master 2> /dev/null + + $ mkdir ws + $ cat > ws/workspace.josh << EOF + > :include=f/include.josh + > ::file3 + > EOF + + $ git add ws/workspace.josh + + $ git commit -m "add ws" + [master e1ff4ff] add ws + 1 file changed, 2 insertions(+) + create mode 100644 ws/workspace.josh + + $ josh-filter -s :workspace=ws master --update refs/josh/master + [1] :/sub1 + [1] :/subsub + [1] ::file1 + [1] :[ + ::file1 + :prefix=a + ] + [1] :prefix=a + [1] :prefix=sub2 + [1] :prefix=subsub + [2] :/sub2 + [2] ::file3 + [2] :[ + :/sub1:[ + ::file1 + :prefix=a + ] + ::sub2/subsub/ + ] + [2] :include=f/include.josh + [2] :workspace=ws + [3] :[ + :include=f/include.josh + ::file3 + ] + + $ git checkout refs/josh/master 2> /dev/null + + $ tree + . + |-- a + | `-- file4 + |-- f + | `-- include.josh + |-- file1 + |-- file3 + |-- sub2 + | `-- subsub + | `-- file2 + `-- workspace.josh + + 4 directories, 6 files + + $ git log --graph --oneline + * 3b918e7 add ws + * b562c24 add include file + * 7c30b7a add file3 + +---------------------------------------- +4) Edit include file +---------------------------------------- + + $ git checkout master 2>/dev/null + +Remove :/a from include file + + $ cat > f/include.josh < :/sub1::file1 + > ::sub2/subsub/ + > EOF + + $ git add f/include.josh + + $ git commit -m "Edit include file" + [master 57f1865] Edit include file + 1 file changed, 1 deletion(-) + + $ josh-filter -s :workspace=ws master --update refs/josh/master + [1] :/sub1 + [1] :/subsub + [1] ::file1 + [1] :[ + ::file1 + :prefix=a + ] + [1] :prefix=a + [1] :prefix=sub2 + [1] :prefix=subsub + [2] :/sub2 + [2] ::file3 + [2] :[ + :/sub1:[ + ::file1 + :prefix=a + ] + ::sub2/subsub/ + ] + [2] :include=f/include.josh + [3] :[ + :include=f/include.josh + ::file3 + ] + [3] :workspace=ws + + $ git checkout refs/josh/master 2>/dev/null + + $ tree + . + |-- f + | `-- include.josh + |-- file1 + |-- file3 + |-- sub2 + | `-- subsub + | `-- file2 + `-- workspace.josh + + 3 directories, 5 files + + $ git log --graph --oneline + * 586aad7 Edit include file + * 3b918e7 add ws + * b562c24 add include file + * 7c30b7a add file3 diff --git a/tests/filter/include_example.t b/tests/filter/include_example.t new file mode 100644 index 000000000..c9aa2e9d2 --- /dev/null +++ b/tests/filter/include_example.t @@ -0,0 +1,140 @@ + $ export TERM=dumb + $ export RUST_LOG_STYLE=never + +----------------------------------------------- +1) Setup a module with doc and src +----------------------------------------------- + + $ git init real_repo 1> /dev/null + $ cd real_repo + + $ mkdir module1 + $ mkdir module1/doc + $ echo contents1 > module1/doc/file1 + $ git add module1/doc/file1 + $ git commit -m "add file1" 1> /dev/null + + $ mkdir module1/src + $ echo contents2 > module1/src/file2 + $ git add module1/src/file2 + $ git commit -m "add file2" 1> /dev/null + +------------------------------------- +2) include files for doc, and for src +------------------------------------- + + $ cat > module1/include_doc.josh < ::module1/doc/ + > EOF + $ git add module1/include_doc.josh + $ git commit -m "add include doc file" 1> /dev/null + + $ cat > module1/include_src.josh < ::module1/src/ + > EOF + $ git add module1/include_src.josh + $ git commit -m "add include src file" 1> /dev/null + +---------------------------------------------- +3) workspace file including src +---------------------------------------------- + + $ git checkout master 2> /dev/null + + $ mkdir ws + $ cat > ws/workspace.josh << EOF + > deps = :[ + > :include=module1/include_src.josh + > ] + > EOF + + $ git add ws/workspace.josh + + $ git commit -m "add ws" + [master 5c76da1] add ws + 1 file changed, 3 insertions(+) + create mode 100644 ws/workspace.josh + + $ josh-filter -s :workspace=ws master --update refs/josh/master + [1] :prefix=module1 + [1] :prefix=src + [2] :/src + [2] :include=module1/include_src.josh + [2] :prefix=deps + [2] :workspace=ws + [3] :/module1 + + $ git checkout refs/josh/master 2> /dev/null + + $ tree + . + |-- deps + | `-- module1 + | |-- include_src.josh + | `-- src + | `-- file2 + `-- workspace.josh + + 3 directories, 3 files + + $ git log --graph --oneline + * e255d4c add ws + * d1da88e add include src file + * 4581710 add file2 + +---------------------------------------------- +4) workspace file now needs doc +---------------------------------------------- + + $ git checkout master 2> /dev/null + + $ cat > ws/workspace.josh << EOF + > deps = :[ + > :include=module1/include_src.josh + > :include=module1/include_doc.josh + > ] + > EOF + + $ git add ws/workspace.josh + + $ git commit -m "update ws" + [master 873b6d0] update ws + 1 file changed, 1 insertion(+) + + $ josh-filter -s :workspace=ws master --update refs/josh/master + [1] :/doc + [1] :prefix=doc + [1] :prefix=src + [2] :/src + [2] :include=module1/include_doc.josh + [2] :include=module1/include_src.josh + [2] :prefix=module1 + [3] :/module1 + [3] :workspace=ws + [4] :prefix=deps + + $ git checkout refs/josh/master 2> /dev/null + + $ tree + . + |-- deps + | `-- module1 + | |-- doc + | | `-- file1 + | |-- include_doc.josh + | |-- include_src.josh + | `-- src + | `-- file2 + `-- workspace.josh + + 4 directories, 5 files + + $ git log --graph --oneline + * c567916 update ws + |\ + | * fcce2eb add include doc file + | * f15f408 add file1 + * e255d4c add ws + * d1da88e add include src file + * 4581710 add file2 + diff --git a/tests/filter/include_same_folder.t b/tests/filter/include_same_folder.t new file mode 100644 index 000000000..2818e9515 --- /dev/null +++ b/tests/filter/include_same_folder.t @@ -0,0 +1,70 @@ + $ export TERM=dumb + $ export RUST_LOG_STYLE=never + + $ git init real_repo 1> /dev/null + $ cd real_repo + + $ mkdir ws + $ echo contents1 > ws/file1 + $ git add ws/file1 + $ git commit -m "add file1" + [master (root-commit) 26a7c70] add file1 + 1 file changed, 1 insertion(+) + create mode 100644 ws/file1 + + $ mkdir sub1 + $ echo contents2 > sub1/file2 + $ git add sub1/file2 + $ git commit -m "add file2" + [master 7b0b2f8] add file2 + 1 file changed, 1 insertion(+) + create mode 100644 sub1/file2 + + $ mkdir sub2 + $ echo contents3 > sub2/file3 + $ git add sub2/file3 + $ git commit -m "add file3" + [master 184c02d] add file3 + 1 file changed, 1 insertion(+) + create mode 100644 sub2/file3 + + $ cat > ws/deps.josh << EOF + > ::sub1/ + > EOF + $ git add ws/deps.josh + $ git commit -m "add deps.josh" + [master 27e392b] add deps.josh + 1 file changed, 1 insertion(+) + create mode 100644 ws/deps.josh + + $ cat > ws/workspace.josh << EOF + > :include=ws/deps.josh + > EOF + $ git add ws/workspace.josh + $ git commit -m "add workspace.josh" + [master 24bc83e] add workspace.josh + 1 file changed, 1 insertion(+) + create mode 100644 ws/workspace.josh + + $ josh-filter -s :workspace=ws master --update refs/josh/master + [1] :prefix=sub1 + [2] :/sub1 + [2] :include=ws/deps.josh + [3] :workspace=ws + $ git checkout refs/josh/master 2>/dev/null + $ tree + . + |-- deps.josh + |-- file1 + |-- sub1 + | `-- file2 + `-- workspace.josh + + 1 directory, 4 files + $ git log --oneline --graph + * ab5e400 add workspace.josh + |\ + | * b70ec25 add deps.josh + | * fccda12 add file2 + * a11175e add deps.josh + * 0b4cf6c add file1 diff --git a/tests/filter/workspace_unique.t b/tests/filter/workspace_unique.t index a385d33bc..ab36ec035 100644 --- a/tests/filter/workspace_unique.t +++ b/tests/filter/workspace_unique.t @@ -16,7 +16,11 @@ $ git commit -m "add file2" 1> /dev/null $ mkdir ws + $ echo contents5 > ws/file5 + $ git add ws + $ git commit -m "add file5" 1> /dev/null $ cat > ws/workspace.josh < ::ws/file5 > :/sub1::file1 > ::sub2/subsub/ > a = :/sub1 @@ -36,19 +40,24 @@ [1] :prefix=sub2 [1] :prefix=subsub [2] :/sub2 - [2] :[ + [2] ::ws/file5 + [3] :[ :/sub1:[ ::file1 :prefix=a ] ::sub2/subsub/ + ::ws/file5 ] - [2] :workspace=ws + [3] :workspace=ws $ git log --graph --pretty=%s refs/josh/master - * add ws - * add file2 - * add file1 + * add ws + |\ + | * add file5 + | * add file2 + | * add file1 + * add file5 $ git checkout refs/josh/master 2> /dev/null $ tree @@ -56,9 +65,10 @@ |-- a | `-- file4 |-- file1 + |-- file5 |-- sub2 | `-- subsub | `-- file2 `-- workspace.josh - 3 directories, 4 files + 3 directories, 5 files diff --git a/tests/proxy/clone_invalid_filter.t b/tests/proxy/clone_invalid_filter.t index efdfcf1dc..e5d09ade2 100644 --- a/tests/proxy/clone_invalid_filter.t +++ b/tests/proxy/clone_invalid_filter.t @@ -69,7 +69,7 @@ remote: remote: :workspace=path remote: - remote: Where `path` is path to the directory where workspace.josh file is located + remote: Where `path` is the path to the directory where workspace.josh file is located fatal: unable to access 'http://localhost:8002/real_repo.git:workspace.git/': The requested URL returned error: 500 [128] @@ -80,7 +80,7 @@ remote: remote: :prefix=path remote: - remote: Where `path` is path to be used as a prefix + remote: Where `path` is the path to be used as a prefix fatal: unable to access 'http://localhost:8002/real_repo.git:prefix.git/': The requested URL returned error: 500 [128]