diff --git a/dir.c b/dir.c index f1703356ee5502..d22c88274ac5c7 100644 --- a/dir.c +++ b/dir.c @@ -1004,8 +1004,26 @@ static int read_skip_worktree_file_from_index(struct index_state *istate, struct oid_stat *oid_stat) { int pos, len; + struct cache_entry *sparse_dir; len = strlen(path); + + /* + * Everything inside a sparse-directory entry has the skip-worktree + * bit, so read the blob from the tree of that entry. Compare with + * case, as index_name_pos() does. + */ + sparse_dir = vfs_sparse_dir_containing(istate, path, len, 0); + if (sparse_dir) { + struct object_id oid; + unsigned short mode; + + if (sparse_dir_get_tree_entry(istate, sparse_dir, path, len, 0, + &oid, &mode)) + return -1; + return do_read_blob(&oid, oid_stat, size_out, data_out); + } + pos = index_name_pos(istate, path, len); if (pos < 0) return -1; @@ -1252,6 +1270,8 @@ static int add_patterns(const char *fname, const char *base, int baselen, !match_stat_data_racy(istate, &oid_stat->stat, &st)) ; /* no content change, oid_stat->oid still good */ else if (istate && + !vfs_sparse_dir_containing(istate, fname, + strlen(fname), 0) && (pos = index_name_pos(istate, fname, strlen(fname))) >= 0 && !ce_stage(istate->cache[pos]) && ce_uptodate(istate->cache[pos]) && @@ -1994,6 +2014,29 @@ static enum exist_status directory_exists_in_index(struct index_state *istate, const char *dirname, int len) { int pos; + struct cache_entry *sparse_dir; + + /* + * The name hash does not contain the directories inside a + * sparse-directory entry, so look in its tree before the + * case-insensitive lookup below. + */ + sparse_dir = vfs_sparse_dir_containing( + istate, dirname, len, repo_ignore_case(the_repository)); + if (sparse_dir) { + struct object_id oid; + unsigned short mode; + + if (sparse_dir_get_tree_entry(istate, sparse_dir, dirname, len, + repo_ignore_case(the_repository), + &oid, &mode)) + return index_nonexistent; + if (S_ISDIR(mode)) + return index_directory; + if (S_ISGITLINK(mode)) + return index_gitdir; + return index_nonexistent; + } if (repo_ignore_case(the_repository)) return directory_exists_in_index_icase(istate, dirname, len); @@ -2400,6 +2443,14 @@ static int get_index_dtype(struct index_state *istate, return DT_REG; } + /* + * A sparse-directory entry contains this path, so the index has no + * entries below it to scan. Return DT_UNKNOWN and let the caller + * use lstat(), instead of an expansion of the whole index. + */ + if (vfs_sparse_dir_containing(istate, path, len, 0)) + return DT_UNKNOWN; + /* Try to look it up as a directory */ pos = index_name_pos(istate, path, len); if (pos >= 0) diff --git a/read-cache.c b/read-cache.c index 2657161e54a8ed..babb844991c7f3 100644 --- a/read-cache.c +++ b/read-cache.c @@ -3501,8 +3501,31 @@ int index_name_is_other(struct index_state *istate, const char *name, int namelen) { int pos; + struct cache_entry *sparse_dir; + if (namelen && name[namelen - 1] == '/') namelen--; + + /* + * An index has entries for files only, so a directory in the tree + * of a sparse-directory entry is still "other". Compare without + * case if core.ignoreCase is set: with a full index, the walk + * already drops such tracked paths with index_file_exists(). + */ + sparse_dir = vfs_sparse_dir_containing( + istate, name, namelen, repo_ignore_case(the_repository)); + if (sparse_dir) { + struct object_id oid; + unsigned short mode; + + if (sparse_dir_get_tree_entry(istate, sparse_dir, name, namelen, + repo_ignore_case(the_repository), + &oid, &mode) || + S_ISDIR(mode)) + return 1; + return 0; + } + pos = index_name_pos(istate, name, namelen); if (0 <= pos) return 0; /* exact match */ diff --git a/sparse-index.c b/sparse-index.c index e21a91c47649c3..adbef7b370f80b 100644 --- a/sparse-index.c +++ b/sparse-index.c @@ -5,11 +5,14 @@ #include "environment.h" #include "ewah/ewok.h" #include "gettext.h" +#include "hex.h" #include "name-hash.h" +#include "odb.h" #include "read-cache-ll.h" #include "repository.h" #include "sparse-index.h" #include "tree.h" +#include "tree-walk.h" #include "pathspec.h" #include "trace2.h" #include "cache-tree.h" @@ -725,6 +728,130 @@ void clear_skip_worktree_from_present_files(struct index_state *istate) } } +/* + * The index sorts names with case, so a name with a different case does + * not sort next to its sparse-directory entry. The dir_hash of the name + * hash compares without case, and it contains the directory of each + * sparse-directory entry. Look up each leading directory of 'path' there, + * and check if the name that the index uses for it is a sparse-directory + * entry. A sparse-directory entry contains no other entries, so the first + * match is the only one. + */ +static struct cache_entry *sparse_dir_containing_icase( + struct index_state *istate, const char *path, int pathlen) +{ + struct strbuf dir = STRBUF_INIT; + struct cache_entry *found = NULL; + const char *end = path + pathlen; + const char *slash = path; + + while ((slash = memchr(slash, '/', end - slash)) && + slash + 1 < end) { + int pos; + + if (!index_dir_find(istate, path, slash - path, &dir)) + break; + strbuf_addch(&dir, '/'); + pos = index_name_pos_sparse(istate, dir.buf, dir.len); + if (pos >= 0 && S_ISSPARSEDIR(istate->cache[pos]->ce_mode)) { + found = istate->cache[pos]; + break; + } + slash++; + } + strbuf_release(&dir); + return found; +} + +struct cache_entry *vfs_sparse_dir_containing(struct index_state *istate, + const char *path, int pathlen, + int icase) +{ + struct cache_entry *ce; + int pos; + + if (!core_virtualfilesystem || !istate->sparse_index || pathlen <= 0) + return NULL; + + pos = index_name_pos_sparse(istate, path, pathlen); + if (pos >= 0) + return NULL; + pos = -pos - 1; + + /* + * Only the entry immediately before the insertion point can be a + * sparse-directory entry that contains 'path' with the same case. + * This is the same test that index_name_pos() uses to decide to + * expand the index. + */ + if (pos) { + ce = istate->cache[pos - 1]; + if (S_ISSPARSEDIR(ce->ce_mode) && + ce_namelen(ce) < pathlen && + !strncmp(path, ce->name, ce_namelen(ce))) + return ce; + } + + if (!icase) + return NULL; + return sparse_dir_containing_icase(istate, path, pathlen); +} + +int sparse_dir_get_tree_entry(struct index_state *istate, + const struct cache_entry *sparse_dir, + const char *path, int pathlen, int icase, + struct object_id *oid, unsigned short *mode) +{ + const char *name = path + ce_namelen(sparse_dir); + const char *end = path + pathlen; + struct object_id tree_oid; + + oidcpy(&tree_oid, &sparse_dir->oid); + for (;;) { + const char *slash = memchr(name, '/', end - name); + size_t len = (slash ? slash : end) - name; + struct tree_desc desc; + struct name_entry entry; + enum object_type type; + size_t size; + void *buf; + int found = 0; + + /* + * Do not report an unreadable tree as "not in the tree". The + * callers would then call tracked files untracked, and + * "git clean" could delete them. + */ + buf = odb_read_object(istate->repo->objects, &tree_oid, + &type, &size); + if (!buf || type != OBJ_TREE) + die(_("unable to read tree %s for '%.*s'"), + oid_to_hex(&tree_oid), pathlen, path); + + init_tree_desc(&desc, &tree_oid, buf, size); + while (tree_entry(&desc, &entry)) { + if (tree_entry_len(&entry) == len && + !(icase ? strncasecmp(entry.path, name, len) : + strncmp(entry.path, name, len))) { + oidcpy(oid, &entry.oid); + *mode = entry.mode; + found = 1; + break; + } + } + free(buf); + + if (!found) + return -1; + if (!slash) + return 0; + if (!S_ISDIR(*mode)) + return -1; + oidcpy(&tree_oid, oid); + name = slash + 1; + } +} + /* * This static global helps avoid infinite recursion between * expand_to_path() and index_file_exists(). diff --git a/sparse-index.h b/sparse-index.h index 15180b02ea6599..47935b3ae227e5 100644 --- a/sparse-index.h +++ b/sparse-index.h @@ -11,6 +11,8 @@ extern int give_advice_on_expansion; struct index_state; +struct cache_entry; +struct object_id; #define SPARSE_INDEX_MEMORY_ONLY (1 << 0) int is_sparse_index_allowed(struct index_state *istate, int flags); int convert_to_sparse(struct index_state *istate, int flags); @@ -30,6 +32,34 @@ void clear_skip_worktree_from_present_files(struct index_state *istate); void expand_to_path(struct index_state *istate, const char *path, size_t pathlen, int icase); +/* + * Under core.virtualFileSystem, the working tree can contain paths inside + * directories that a sparse index collapses into sparse-directory entries. + * A lookup of such a path with index_name_pos() expands the whole index. + * + * If core.virtualFileSystem is set, the index is sparse, and a + * sparse-directory entry contains 'path', return that entry. Otherwise + * return NULL. If 'icase' is set, the name of the sparse-directory entry + * can differ in case from the start of 'path'. This function never expands + * the index. If core.virtualFileSystem is set and this function returns + * NULL, then index_name_pos() for 'path' does not expand the index either. + */ +struct cache_entry *vfs_sparse_dir_containing(struct index_state *istate, + const char *path, int pathlen, + int icase); + +/* + * Find the part of 'path' below the sparse-directory entry 'sparse_dir' + * in the tree of that entry. 'sparse_dir' must contain 'path'. If 'icase' + * is set, compare names without case. Return 0 and set 'oid' and 'mode' + * if the tree contains the path. Return -1 if it does not. Die if a tree + * cannot be read. + */ +int sparse_dir_get_tree_entry(struct index_state *istate, + const struct cache_entry *sparse_dir, + const char *path, int pathlen, int icase, + struct object_id *oid, unsigned short *mode); + struct repository; int set_sparse_index_config(struct repository *repo, int enable); diff --git a/t/meson.build b/t/meson.build index 139269d6ebd95a..6a9f4363c8f8fd 100644 --- a/t/meson.build +++ b/t/meson.build @@ -188,6 +188,7 @@ integration_tests = [ 't1091-sparse-checkout-builtin.sh', 't1092-sparse-checkout-compatibility.sh', 't1093-virtualfilesystem.sh', + 't1094-virtualfilesystem-sparse-walk.sh', 't1100-commit-tree-options.sh', 't1300-config.sh', 't1301-shared-repo.sh', diff --git a/t/t1094-virtualfilesystem-sparse-walk.sh b/t/t1094-virtualfilesystem-sparse-walk.sh new file mode 100755 index 00000000000000..749173203ad5b6 --- /dev/null +++ b/t/t1094-virtualfilesystem-sparse-walk.sh @@ -0,0 +1,298 @@ +#!/bin/sh + +test_description='sparse index and virtual file system: working-tree walk + +A virtual file system projects files on disk inside directories that a +sparse index collapses into sparse-directory entries. The working-tree walk +must answer its index lookups for those paths without an expansion of the +index, and give the same results as an expanded index. +' + +# A split index cannot be sparse. +GIT_TEST_SPLIT_INDEX=0 +GIT_TEST_SPARSE_INDEX= + +. ./test-lib.sh + +# Run a command in "repo" with the virtual file system and record trace2 +# events in trace-vfs.txt. +run_vfs () { + rm -f trace-vfs.txt && + GIT_TRACE2_EVENT="$(pwd)/trace-vfs.txt" GIT_TRACE2_EVENT_NESTING=10 \ + git -C repo "$@" +} + +# Run a command in "full", which has the same working tree and sparse-checkout +# definition as "repo" but a full index and no virtual file system. +run_full () { + git -C full -c sparse.expectFilesOutsideOfPatterns=true "$@" +} + +# Run a command in "repo" without the virtual file system. The out-of-cone +# files stay on disk, so the walk visits them, and its index lookups use +# the code that expands a sparse index. +run_no_vfs () { + rm -f trace-no-vfs.txt && + GIT_TRACE2_EVENT="$(pwd)/trace-no-vfs.txt" \ + GIT_TRACE2_EVENT_NESTING=10 \ + git -C repo -c core.virtualfilesystem= \ + -c sparse.expectFilesOutsideOfPatterns=true "$@" +} + +ensure_not_expanded () { + test_region ! index ensure_full_index "$1" && + test_grep ! "expansion-reason" "$1" +} + +ensure_index_is_sparse () { + git -C repo -c core.virtualfilesystem= \ + -c sparse.expectFilesOutsideOfPatterns=true \ + ls-files --sparse >sparse-entries && + test_grep "^out/$" sparse-entries +} + +# Run a command with the virtual file system over a sparse index, and in +# the full-index repository, then compare. The virtual file system run +# must not expand the index. +test_vfs_matches_full () { + run_vfs "$@" >actual-vfs && + ensure_not_expanded trace-vfs.txt && + ensure_index_is_sparse && + run_full "$@" >actual-full && + test_cmp actual-full actual-vfs +} + +# Rename the directory $1 to $2, a name that differs only in case, in each +# repository that follows them. +rename_case () { + from=$1 && + to=$2 && + shift 2 && + for r in "$@" + do + mv "$r/$from" "$r/$from.tmp" && + mv "$r/$from.tmp" "$r/$to" || return 1 + done +} + +# Put the out-of-cone files back on disk, as a virtual file system does, +# and add untracked and ignored files inside "out/". Leave +# out/deep/.gitignore absent, so that the walk reads it from the index. +populate_out_of_cone () { + mkdir -p $1/out/deep/sub $1/out/other && + for f in out/a out/deep/a out/deep/sub/a \ + out/other/a out/other/.gitignore + do + git -C $1 show HEAD:$f >$1/$f || return 1 + done && + echo new >$1/out/new.txt && + echo new >$1/out/deep/new.txt && + echo new >$1/out/deep/sub/new.txt && + echo new >$1/out/deep/x.ignored && + mkdir -p $1/out/deep/ignored-dir $1/out/untracked-dir && + echo new >$1/out/deep/ignored-dir/y.ignored && + echo new >$1/out/untracked-dir/new.txt && + echo new >$1/out/other/z.skip +} + +test_expect_success 'setup' ' + git init repo && + mkdir -p repo/in repo/out/deep/sub repo/out/other && + echo a >repo/a && + echo a >repo/in/a && + echo a >repo/out/a && + echo a >repo/out/deep/a && + echo a >repo/out/deep/sub/a && + echo a >repo/out/other/a && + echo "*.ignored" >repo/out/deep/.gitignore && + echo "*.skip" >repo/out/other/.gitignore && + git -C repo add . && + git -C repo commit -m initial && + git clone repo full && + + git -C repo sparse-checkout set --cone --sparse-index in && + test_path_is_missing repo/out && + populate_out_of_cone repo && + + git -C full sparse-checkout set --cone --no-sparse-index in && + test_path_is_missing full/out && + populate_out_of_cone full && + + # List "out/" as a whole, so the walk visits it. This name is the + # exact name of the sparse-directory entry, so the hook does not make + # git expand the index. + write_script repo/.git/hooks/virtualfilesystem <<-\EOF && + printf "a\0in/\0out/\0" + EOF + git -C repo config core.virtualfilesystem \ + .git/hooks/virtualfilesystem && + ensure_index_is_sparse +' + +test_expect_success 'status does not expand the index' ' + run_vfs status --porcelain=v1 >actual && + ensure_not_expanded trace-vfs.txt && + ensure_index_is_sparse && + cat >expect <<-\EOF && + ?? out/deep/new.txt + ?? out/deep/sub/new.txt + ?? out/new.txt + ?? out/untracked-dir/ + EOF + test_cmp expect actual +' + +test_expect_success 'status -uall does not expand the index' ' + run_vfs status --porcelain=v1 -uall >actual && + ensure_not_expanded trace-vfs.txt && + cat >expect <<-\EOF && + ?? out/deep/new.txt + ?? out/deep/sub/new.txt + ?? out/new.txt + ?? out/untracked-dir/new.txt + EOF + test_cmp expect actual +' + +for ignorecase in false true +do + for args in "" "-uall" "--ignored" "--ignored -uall" \ + "--ignored=matching" + do + test_expect_success "status $args (ignoreCase=$ignorecase)" " + test_vfs_matches_full -c core.ignoreCase=$ignorecase \ + status --porcelain=v1 $args + " + done + + for args in "" "-d" "-d -x" "-d -X" + do + test_expect_success "clean -n $args (ignoreCase=$ignorecase)" " + test_vfs_matches_full -c core.ignoreCase=$ignorecase \ + clean -n $args + " + done +done + +test_expect_success 'clean -n -d -x lists only untracked paths' ' + run_vfs clean -n -d -x >actual && + ensure_not_expanded trace-vfs.txt && + cat >expect <<-\EOF && + Would remove out/deep/ignored-dir/ + Would remove out/deep/new.txt + Would remove out/deep/sub/new.txt + Would remove out/deep/x.ignored + Would remove out/new.txt + Would remove out/other/z.skip + Would remove out/untracked-dir/ + EOF + test_cmp expect actual +' + +# Spell a tracked directory with a different case on disk. With +# core.ignoreCase, a full index treats it as tracked, so "git clean" +# must not offer to remove it. +test_expect_success 'directory with a different case on disk (ignoreCase)' ' + rename_case out/other out/OTHER repo full && + test_when_finished "rename_case out/OTHER out/other repo full" && + test_vfs_matches_full -c core.ignoreCase=true \ + status --porcelain=v1 -uall && + test_vfs_matches_full -c core.ignoreCase=true clean -n -d && + test_grep ! OTHER actual-vfs +' + +test_expect_success 'sparse directory with a different case on disk (ignoreCase)' ' + rename_case out OUT repo full && + test_when_finished "rename_case OUT out repo full" && + test_vfs_matches_full -c core.ignoreCase=true \ + status --porcelain=v1 -uall && + test_grep ! "OUT/a$" actual-vfs && + test_vfs_matches_full -c core.ignoreCase=true clean -n -d && + test_grep "OUT/new.txt" actual-vfs && + test_grep ! "OUT/a$" actual-vfs && + test_grep ! "OUT/deep/$" actual-vfs +' + +# Here "out/" is in the cone as a parent directory, and "out/deep/" is the +# sparse-directory entry. Both have a different case on disk. +test_expect_success 'nested sparse directory with a different case on disk' ' + test_when_finished "rm -rf repo2 full2" && + git clone full repo2 && + git clone full full2 && + git -C repo2 sparse-checkout set --cone --sparse-index in out/other && + git -C full2 sparse-checkout set --cone --no-sparse-index in out/other && + populate_out_of_cone repo2 && + populate_out_of_cone full2 && + write_script repo2/.git/hooks/virtualfilesystem <<-\EOF && + printf "a\0in/\0out/a\0out/new.txt\0out/deep/\0out/other/\0" + printf "out/untracked-dir/\0" + EOF + git -C repo2 config core.virtualfilesystem \ + .git/hooks/virtualfilesystem && + rename_case out OUT repo2 full2 && + rename_case OUT/deep OUT/Deep repo2 full2 && + + for args in "status --porcelain=v1 -uall" "clean -n -d" + do + rm -f trace-vfs.txt && + GIT_TRACE2_EVENT="$(pwd)/trace-vfs.txt" \ + GIT_TRACE2_EVENT_NESTING=10 \ + git -C repo2 -c core.ignoreCase=true $args >actual-vfs && + ensure_not_expanded trace-vfs.txt && + git -C full2 -c sparse.expectFilesOutsideOfPatterns=true \ + -c core.ignoreCase=true $args >actual-full && + test_cmp actual-full actual-vfs || return 1 + done && + test_grep "OUT/Deep/new.txt" actual-vfs && + test_grep ! "OUT/Deep/a$" actual-vfs && + test_grep ! "OUT/Deep/$" actual-vfs && + git -C repo2 -c core.virtualfilesystem= \ + -c sparse.expectFilesOutsideOfPatterns=true \ + ls-files --sparse >sparse-entries && + test_grep "^out/deep/$" sparse-entries +' + +test_expect_success 'status with the untracked cache' ' + test_vfs_matches_full -c core.untrackedCache=true \ + status --porcelain=v1 && + test_vfs_matches_full -c core.untrackedCache=true \ + status --porcelain=v1 -uall +' + +# With a glob pathspec, "git add --ignore-missing" asks is_excluded() for +# the type of the path, which looks up the path in the index. +test_expect_success 'add --ignore-missing of an ignored path' ' + rm -f trace-vfs.txt && + test_must_fail env GIT_TRACE2_EVENT="$(pwd)/trace-vfs.txt" \ + GIT_TRACE2_EVENT_NESTING=10 \ + git -C repo add --dry-run --ignore-missing \ + ":(glob)out/deep/x.ignored" >actual-vfs 2>err-vfs && + ensure_not_expanded trace-vfs.txt && + ensure_index_is_sparse && + test_must_fail git -C full -c sparse.expectFilesOutsideOfPatterns=true \ + add --dry-run --ignore-missing \ + ":(glob)out/deep/x.ignored" >actual-full 2>err-full && + test_cmp actual-full actual-vfs && + test_cmp err-full err-vfs && + test_grep "ignored by one of your .gitignore files" err-vfs && + test_grep "out/deep/x.ignored" err-vfs +' + +test_expect_success 'an unreadable tree in a sparse directory is an error' ' + test_when_finished "rm -rf broken" && + cp -R repo broken && + tree=$(git -C broken rev-parse HEAD:out/deep) && + rm broken/.git/objects/$(test_oid_to_path $tree) && + test_must_fail git -C broken clean -n -d >out 2>err && + test_grep "unable to read tree $tree" err && + test_grep ! "Would remove" out +' + +test_expect_success 'without a virtual file system, the walk still expands' ' + run_no_vfs status --porcelain=v1 >actual && + test_grep "found parent dir .out/." trace-no-vfs.txt && + run_full status --porcelain=v1 >expect && + test_cmp expect actual +' + +test_done