Implement stored filter

Author: christian-schilling

docs/src/reference/filters.md

@@ -66,10 +66,22 @@ Remove all paths present in the *output* of ``:filter`` from the input tree.
 It should generally be avoided to use any filters that change paths and instead only
 use filters that select paths without altering them.
 
+### Stored **`:+path/to/file`**
+Looks for a file with a ``.josh`` extension at the specified path and applies the filter defined in that file.
+The path argument should be provided *without* the ``.josh`` extension, as it will be automatically appended.
+
+For example, ``:+st/config`` will look for a file at ``st/config.josh`` and apply the filter defined in that file.
+The resulting tree will contain the contents specified by the filter in the ``.josh`` file.
+
+Stored filters apply from the root of the repository, making them useful for configuration files that define
+filters to be applied at the repository root level.
+
 ### Workspace **`:workspace=a`**
-Similar to ``:/a`` but also looks for a ``workspace.josh`` file inside the
-specified directory (called the "workspace root").
-The resulting tree will contain the contents of the
+Like stored filters, but with an additional "base" component that first selects the specified directory
+(called the "workspace root") before applying the filter from the ``workspace.josh`` file inside it.
+
+The workspace filter is equivalent to ``:/a`` combined with applying a stored filter, where the filter is read
+from ``workspace.josh`` within the selected directory. The resulting tree will contain the contents of the
 workspace root as well as additional files specified in the ``workspace.josh`` file.
 (see [Workspaces](./workspace.md))
 

josh-core/src/filter/grammar.pest

@@ -29,13 +29,15 @@ filter_spec = { (
   | filter_squash
   | filter_presub
   | filter_subdir
+  | filter_stored
   | filter_nop
   | filter
   | filter_noarg
 )+ }
 
 filter_group = { CMD_START ~ cmd? ~ GROUP_START ~ compose ~ GROUP_END }
 filter_subdir = { CMD_START ~ "/" ~ argument }
+filter_stored = { CMD_START ~ "+" ~ argument }
 filter_nop = { CMD_START ~ "/" }
 filter_presub = { CMD_START ~ ":" ~ argument ~ ("=" ~ argument)? }
 filter = { CMD_START ~ cmd ~ "=" ~ (argument ~ (";" ~ argument)*)? }

josh-core/src/filter/mod.rs

@@ -312,6 +312,7 @@ enum Op {
     Prefix(std::path::PathBuf),
     Subdir(std::path::PathBuf),
     Workspace(std::path::PathBuf),
+    Stored(std::path::PathBuf),
 
     Pattern(String),
     Message(String, regex::Regex),
@@ -589,6 +590,9 @@ fn spec2(op: &Op) -> String {
         Op::Workspace(path) => {
             format!(":workspace={}", parse::quote_if(&path.to_string_lossy()))
         }
+        Op::Stored(path) => {
+            format!(":+{}", parse::quote_if(&path.to_string_lossy()))
+        }
         Op::RegexReplace(replacements) => {
             let v = replacements
                 .iter()
@@ -737,6 +741,31 @@ fn get_workspace<'a>(repo: &'a git2::Repository, tree: &'a git2::Tree<'a>, path:
     )
 }
 
+// Handle stored.josh files that contain ":stored=..." as their only filter as
+// a "redirect" to that other stored. We chain an exclude of the redirecting stored
+// in front to prevent infinite recursion.
+fn resolve_stored_redirect<'a>(
+    repo: &'a git2::Repository,
+    tree: &'a git2::Tree<'a>,
+    path: &Path,
+) -> Option<(Filter, std::path::PathBuf)> {
+    let stored_path = path.with_extension("josh");
+    let f = parse::parse(&tree::get_blob(repo, tree, &stored_path))
+        .unwrap_or_else(|_| to_filter(Op::Empty));
+
+    if let Op::Stored(p) = to_op(f) {
+        Some((chain(to_filter(Op::Exclude(file(stored_path))), f), p))
+    } else {
+        None
+    }
+}
+
+fn get_stored<'a>(repo: &'a git2::Repository, tree: &'a git2::Tree<'a>, path: &Path) -> Filter {
+    let stored_path = path.with_extension("josh");
+    let sj_file = file(stored_path.clone());
+    compose(sj_file, get_filter(repo, tree, &stored_path))
+}
+
 fn get_filter<'a>(repo: &'a git2::Repository, tree: &'a git2::Tree<'a>, path: &Path) -> Filter {
     let ws_path = normalize_path(path);
     let ws_id = ok_or!(tree.get_path(&ws_path), {
@@ -1022,6 +1051,33 @@ fn apply_to_commit2(
 
             return per_rev_filter(transaction, commit, filter, commit_filter, parent_filters);
         }
+        Op::Stored(s_path) => {
+            if let Some((redirect, _)) = resolve_stored_redirect(repo, &commit.tree()?, s_path) {
+                if let Some(r) = apply_to_commit2(&to_op(redirect), commit, transaction)? {
+                    transaction.insert(filter, commit.id(), r, true);
+                    return Ok(Some(r));
+                } else {
+                    return Ok(None);
+                }
+            }
+
+            let commit_filter = get_stored(repo, &commit.tree()?, &s_path);
+
+            let parent_filters = commit
+                .parents()
+                .map(|parent| {
+                    rs_tracing::trace_scoped!("parent", "id": parent.id().to_string());
+                    let pcs = get_stored(
+                        repo,
+                        &parent.tree().unwrap_or_else(|_| tree::empty(repo)),
+                        &s_path,
+                    );
+                    Ok((parent, pcs))
+                })
+                .collect::<JoshResult<Vec<_>>>()?;
+
+            return per_rev_filter(transaction, commit, filter, commit_filter, parent_filters);
+        }
         Op::Fold => {
             let filtered_parent_ids = commit
                 .parents()
@@ -1207,6 +1263,7 @@ fn apply2<'a>(transaction: &'a cache::Transaction, op: &Op, x: Apply<'a>) -> Jos
         }
 
         Op::Workspace(path) => apply(transaction, get_workspace(repo, &x.tree(), &path), x),
+        Op::Stored(path) => apply(transaction, get_stored(repo, &x.tree(), &path), x),
 
         Op::Compose(filters) => {
             let filtered: Vec<_> = filters
@@ -1329,6 +1386,31 @@ fn unapply_workspace<'a>(
 
             Ok(Some(result))
         }
+        Op::Stored(path) => {
+            let stored_path = path.with_extension("josh");
+            let stored = get_filter(transaction.repo(), &tree, &stored_path);
+            let original_stored = get_filter(transaction.repo(), &parent_tree, &stored_path);
+
+            let sj_file = file(stored_path.clone());
+            let filter = compose(sj_file, stored);
+            let original_filter = compose(sj_file, original_stored);
+            let filtered = apply(
+                transaction,
+                original_filter,
+                Apply::from_tree(parent_tree.clone()),
+            )?;
+            let matching = apply(transaction, invert(original_filter)?, filtered.clone())?;
+            let stripped = tree::subtract(transaction, parent_tree.id(), matching.tree().id())?;
+            let x = apply(transaction, invert(filter)?, Apply::from_tree(tree))?;
+
+            let result = transaction.repo().find_tree(tree::overlay(
+                transaction,
+                x.tree().id(),
+                stripped,
+            )?)?;
+
+            Ok(Some(result))
+        }
         _ => Ok(None),
     }
 }
@@ -1397,6 +1479,17 @@ pub fn compute_warnings<'a>(
         }
     }
 
+    if let Op::Stored(path) = to_op(filter) {
+        let stored_path = path.with_extension("josh");
+        let stored_filter = &tree::get_blob(transaction.repo(), &tree, &stored_path);
+        if let Ok(res) = parse(stored_filter) {
+            filter = res;
+        } else {
+            warnings.push("couldn't parse stored\n".to_string());
+            return warnings;
+        }
+    }
+
     let filter = opt::flatten(filter);
     if let Op::Compose(filters) = to_op(filter) {
         for f in filters {

josh-core/src/filter/parse.rs

@@ -84,6 +84,9 @@ fn parse_item(pair: pest::iterators::Pair<Rule>) -> JoshResult<Op> {
         Rule::filter_subdir => Ok(Op::Subdir(
             Path::new(&unquote(pair.into_inner().next().unwrap().as_str())).to_owned(),
         )),
+        Rule::filter_stored => Ok(Op::Stored(
+            Path::new(&unquote(pair.into_inner().next().unwrap().as_str())).to_owned(),
+        )),
         Rule::filter_presub => {
             let mut inner = pair.into_inner();
             let arg = &unquote(inner.next().unwrap().as_str());

josh-core/src/filter/persist.rs

@@ -228,6 +228,10 @@ impl InMemoryBuilder {
                 let blob = self.write_blob(path.to_string_lossy().as_bytes());
                 push_blob_entries(&mut entries, [("workspace", blob)]);
             }
+            Op::Stored(path) => {
+                let blob = self.write_blob(path.to_string_lossy().as_bytes());
+                push_blob_entries(&mut entries, [("stored", blob)]);
+            }
             Op::Nop => {
                 let blob = self.write_blob(b"");
                 push_blob_entries(&mut entries, [("nop", blob)]);
@@ -507,6 +511,11 @@ fn from_tree2(repo: &git2::Repository, tree_oid: git2::Oid) -> JoshResult<Op> {
             let path = std::str::from_utf8(blob.content())?;
             Ok(Op::Workspace(std::path::PathBuf::from(path)))
         }
+        "stored" => {
+            let blob = repo.find_blob(entry.id())?;
+            let path = std::str::from_utf8(blob.content())?;
+            Ok(Op::Stored(std::path::PathBuf::from(path)))
+        }
         "compose" => {
             let compose_tree = repo.find_tree(entry.id())?;
             let mut filters = Vec::new();

tests/filter/stored_combine_filter.t

@@ -0,0 +1,121 @@
+  $ export TERM=dumb
+  $ export RUST_LOG_STYLE=never
+
+  $ git init -q real_repo 1> /dev/null
+  $ cd real_repo
+
+  $ mkdir sub1
+  $ echo contents1 > sub1/file1
+  $ git add sub1
+  $ git commit -m "add file1" 1> /dev/null
+
+  $ mkdir -p sub2/subsub
+  $ echo contents1 > sub2/subsub/file2
+  $ git add sub2
+  $ git commit -m "add file2" 1> /dev/null
+
+  $ mkdir -p sub3
+  $ echo contents1 > sub3/sub_file
+  $ git add .
+  $ git commit -m "add sub_file" 1> /dev/null
+
+  $ mkdir st
+  $ cat > st/config.josh <<EOF
+  > x = :[::sub2/subsub/,::sub1/]
+  > EOF
+  $ mkdir st2
+  $ cat > st2/config.josh <<EOF
+  > :[
+  >   a = :[::sub2/subsub/,::sub3/]
+  >   :/sub1:prefix=blub
+  > ]:prefix=xyz
+  > EOF
+  $ git add .
+  $ git commit -m "add st" 1> /dev/null
+
+  $ tree
+  .
+  |-- st
+  |   `-- config.josh
+  |-- st2
+  |   `-- config.josh
+  |-- sub1
+  |   `-- file1
+  |-- sub2
+  |   `-- subsub
+  |       `-- file2
+  `-- sub3
+      `-- sub_file
+  
+  7 directories, 5 files
+
+  $ josh-filter -s :+st/config
+  [2] :+st/config
+  [2] :[
+      ::sub1/
+      ::sub2/subsub/
+  ]
+  [2] :prefix=x
+  [4] sequence_number
+
+  $ git log --graph --pretty=%s FILTERED_HEAD
+  * add st
+  * add file2
+  * add file1
+
+  $ git checkout FILTERED_HEAD 2> /dev/null
+  $ tree
+  .
+  |-- st
+  |   `-- config.josh
+  `-- x
+      |-- sub1
+      |   `-- file1
+      `-- sub2
+          `-- subsub
+              `-- file2
+  
+  6 directories, 3 files
+
+  $ git checkout master 2> /dev/null
+  $ josh-filter -s :+st2/config
+  [2] :+st/config
+  [2] :+st2/config
+  [2] :[
+      ::sub1/
+      ::sub2/subsub/
+  ]
+  [2] :prefix=x
+  [3] :[
+      a = :[
+          ::sub2/subsub/
+          ::sub3/
+      ]
+      blub = :/sub1
+  ]
+  [3] :prefix=xyz
+  [7] sequence_number
+
+  $ git log --graph --pretty=%s FILTERED_HEAD
+  * add st
+  * add sub_file
+  * add file2
+  * add file1
+
+  $ git checkout FILTERED_HEAD 2> /dev/null
+  $ tree
+  .
+  |-- st2
+  |   `-- config.josh
+  `-- xyz
+      |-- a
+      |   |-- sub2
+      |   |   `-- subsub
+      |   |       `-- file2
+      |   `-- sub3
+      |       `-- sub_file
+      `-- blub
+          `-- file1
+  
+  8 directories, 4 files
+

tests/filter/stored_implicit_filter.t

@@ -0,0 +1,50 @@
+  $ export TERM=dumb
+  $ export RUST_LOG_STYLE=never
+
+  $ git init -q real_repo 1> /dev/null
+  $ cd real_repo
+
+  $ mkdir sub1
+  $ echo contents1 > sub1/file1
+  $ git add sub1
+  $ git commit -m "add file1" 1> /dev/null
+
+  $ mkdir -p sub2/subsub
+  $ echo contents1 > sub2/subsub/file2
+  $ git add sub2
+  $ git commit -m "add file2" 1> /dev/null
+
+  $ mkdir st
+  $ cat > st/config.josh <<EOF
+  > ::sub2/subsub/
+  > ::sub1/
+  > EOF
+  $ git add st
+  $ git commit -m "add st" 1> /dev/null
+
+  $ josh-filter -s :+st/config master --update refs/josh/master
+  [2] :+st/config
+  [2] :[
+      ::sub1/
+      ::sub2/subsub/
+  ]
+  [3] sequence_number
+
+  $ git log --graph --pretty=%s refs/josh/master
+  * add st
+  * add file2
+  * add file1
+
+  $ git checkout refs/josh/master 2> /dev/null
+  $ tree
+  .
+  |-- st
+  |   `-- config.josh
+  |-- sub1
+  |   `-- file1
+  `-- sub2
+      `-- subsub
+          `-- file2
+  
+  5 directories, 3 files
+