sparse-index: do not expand the index during the working-tree walk under a virtual filesystem

dir.c

@@ -1004,8 +1004,26 @@ static int read_skip_worktree_file_from_index(struct index_state *istate,
 					      struct oid_stat *oid_stat)
 {
 	int pos, len;
+	struct cache_entry *sparse_dir;
 
 	len = strlen(path);
+
+	/*
+	 * Everything inside a sparse-directory entry has the skip-worktree
+	 * bit, so read the blob from the tree of that entry. Compare with
+	 * case, as index_name_pos() does.
+	 */
+	sparse_dir = vfs_sparse_dir_containing(istate, path, len, 0);
+	if (sparse_dir) {
+		struct object_id oid;
+		unsigned short mode;
+
+		if (sparse_dir_get_tree_entry(istate, sparse_dir, path, len, 0,
+					      &oid, &mode))
+			return -1;
+		return do_read_blob(&oid, oid_stat, size_out, data_out);
+	}
+
 	pos = index_name_pos(istate, path, len);
 	if (pos < 0)
 		return -1;
@@ -1252,6 +1270,8 @@ static int add_patterns(const char *fname, const char *base, int baselen,
 			    !match_stat_data_racy(istate, &oid_stat->stat, &st))
 				; /* no content change, oid_stat->oid still good */
 			else if (istate &&
+				 !vfs_sparse_dir_containing(istate, fname,
+							    strlen(fname), 0) &&
 				 (pos = index_name_pos(istate, fname, strlen(fname))) >= 0 &&
 				 !ce_stage(istate->cache[pos]) &&
 				 ce_uptodate(istate->cache[pos]) &&
@@ -1994,6 +2014,29 @@ static enum exist_status directory_exists_in_index(struct index_state *istate,
 						   const char *dirname, int len)
 {
 	int pos;
+	struct cache_entry *sparse_dir;
+
+	/*
+	 * The name hash does not contain the directories inside a
+	 * sparse-directory entry, so look in its tree before the
+	 * case-insensitive lookup below.
+	 */
+	sparse_dir = vfs_sparse_dir_containing(
+		istate, dirname, len, repo_ignore_case(the_repository));
+	if (sparse_dir) {
+		struct object_id oid;
+		unsigned short mode;
+
+		if (sparse_dir_get_tree_entry(istate, sparse_dir, dirname, len,
+					      repo_ignore_case(the_repository),
+					      &oid, &mode))
+			return index_nonexistent;
+		if (S_ISDIR(mode))
+			return index_directory;
+		if (S_ISGITLINK(mode))
+			return index_gitdir;
+		return index_nonexistent;
+	}
 
 	if (repo_ignore_case(the_repository))
 		return directory_exists_in_index_icase(istate, dirname, len);
@@ -2400,6 +2443,14 @@ static int get_index_dtype(struct index_state *istate,
 		return DT_REG;
 	}
 
+	/*
+	 * A sparse-directory entry contains this path, so the index has no
+	 * entries below it to scan. Return DT_UNKNOWN and let the caller
+	 * use lstat(), instead of an expansion of the whole index.
+	 */
+	if (vfs_sparse_dir_containing(istate, path, len, 0))
+		return DT_UNKNOWN;
+
 	/* Try to look it up as a directory */
 	pos = index_name_pos(istate, path, len);
 	if (pos >= 0)

read-cache.c

@@ -3501,8 +3501,31 @@ int index_name_is_other(struct index_state *istate, const char *name,
 			int namelen)
 {
 	int pos;
+	struct cache_entry *sparse_dir;
+
 	if (namelen && name[namelen - 1] == '/')
 		namelen--;
+
+	/*
+	 * An index has entries for files only, so a directory in the tree
+	 * of a sparse-directory entry is still "other". Compare without
+	 * case if core.ignoreCase is set: with a full index, the walk
+	 * already drops such tracked paths with index_file_exists().
+	 */
+	sparse_dir = vfs_sparse_dir_containing(
+		istate, name, namelen, repo_ignore_case(the_repository));
+	if (sparse_dir) {
+		struct object_id oid;
+		unsigned short mode;
+
+		if (sparse_dir_get_tree_entry(istate, sparse_dir, name, namelen,
+					      repo_ignore_case(the_repository),
+					      &oid, &mode) ||
+		    S_ISDIR(mode))
+			return 1;
+		return 0;
+	}
+
 	pos = index_name_pos(istate, name, namelen);
 	if (0 <= pos)
 		return 0;	/* exact match */

sparse-index.c

@@ -5,11 +5,14 @@
 #include "environment.h"
 #include "ewah/ewok.h"
 #include "gettext.h"
+#include "hex.h"
 #include "name-hash.h"
+#include "odb.h"
 #include "read-cache-ll.h"
 #include "repository.h"
 #include "sparse-index.h"
 #include "tree.h"
+#include "tree-walk.h"
 #include "pathspec.h"
 #include "trace2.h"
 #include "cache-tree.h"
@@ -725,6 +728,130 @@ void clear_skip_worktree_from_present_files(struct index_state *istate)
 	}
 }
 
+/*
+ * The index sorts names with case, so a name with a different case does
+ * not sort next to its sparse-directory entry. The dir_hash of the name
+ * hash compares without case, and it contains the directory of each
+ * sparse-directory entry. Look up each leading directory of 'path' there,
+ * and check if the name that the index uses for it is a sparse-directory
+ * entry. A sparse-directory entry contains no other entries, so the first
+ * match is the only one.
+ */
+static struct cache_entry *sparse_dir_containing_icase(
+	struct index_state *istate, const char *path, int pathlen)
+{
+	struct strbuf dir = STRBUF_INIT;
+	struct cache_entry *found = NULL;
+	const char *end = path + pathlen;
+	const char *slash = path;
+
+	while ((slash = memchr(slash, '/', end - slash)) &&
+	       slash + 1 < end) {
+		int pos;
+
+		if (!index_dir_find(istate, path, slash - path, &dir))
+			break;
+		strbuf_addch(&dir, '/');
+		pos = index_name_pos_sparse(istate, dir.buf, dir.len);
+		if (pos >= 0 && S_ISSPARSEDIR(istate->cache[pos]->ce_mode)) {
+			found = istate->cache[pos];
+			break;
+		}
+		slash++;
+	}
+	strbuf_release(&dir);
+	return found;
+}
+
+struct cache_entry *vfs_sparse_dir_containing(struct index_state *istate,
+					      const char *path, int pathlen,
+					      int icase)
+{
+	struct cache_entry *ce;
+	int pos;
+
+	if (!core_virtualfilesystem || !istate->sparse_index || pathlen <= 0)
+		return NULL;
+
+	pos = index_name_pos_sparse(istate, path, pathlen);
+	if (pos >= 0)
+		return NULL;
+	pos = -pos - 1;
+
+	/*
+	 * Only the entry immediately before the insertion point can be a
+	 * sparse-directory entry that contains 'path' with the same case.
+	 * This is the same test that index_name_pos() uses to decide to
+	 * expand the index.
+	 */
+	if (pos) {
+		ce = istate->cache[pos - 1];
+		if (S_ISSPARSEDIR(ce->ce_mode) &&
+		    ce_namelen(ce) < pathlen &&
+		    !strncmp(path, ce->name, ce_namelen(ce)))
+			return ce;
+	}
+
+	if (!icase)
+		return NULL;
+	return sparse_dir_containing_icase(istate, path, pathlen);
+}
+
+int sparse_dir_get_tree_entry(struct index_state *istate,
+			      const struct cache_entry *sparse_dir,
+			      const char *path, int pathlen, int icase,
+			      struct object_id *oid, unsigned short *mode)
+{
+	const char *name = path + ce_namelen(sparse_dir);
+	const char *end = path + pathlen;
+	struct object_id tree_oid;
+
+	oidcpy(&tree_oid, &sparse_dir->oid);
+	for (;;) {
+		const char *slash = memchr(name, '/', end - name);
+		size_t len = (slash ? slash : end) - name;
+		struct tree_desc desc;
+		struct name_entry entry;
+		enum object_type type;
+		size_t size;
+		void *buf;
+		int found = 0;
+
+		/*
+		 * Do not report an unreadable tree as "not in the tree". The
+		 * callers would then call tracked files untracked, and
+		 * "git clean" could delete them.
+		 */
+		buf = odb_read_object(istate->repo->objects, &tree_oid,
+				      &type, &size);
+		if (!buf || type != OBJ_TREE)
+			die(_("unable to read tree %s for '%.*s'"),
+			    oid_to_hex(&tree_oid), pathlen, path);
+
+		init_tree_desc(&desc, &tree_oid, buf, size);
+		while (tree_entry(&desc, &entry)) {
+			if (tree_entry_len(&entry) == len &&
+			    !(icase ? strncasecmp(entry.path, name, len) :
+				      strncmp(entry.path, name, len))) {
+				oidcpy(oid, &entry.oid);
+				*mode = entry.mode;
+				found = 1;
+				break;
+			}
+		}
+		free(buf);
+
+		if (!found)
+			return -1;
+		if (!slash)
+			return 0;
+		if (!S_ISDIR(*mode))
+			return -1;
+		oidcpy(&tree_oid, oid);
+		name = slash + 1;
+	}
+}
+
 /*
  * This static global helps avoid infinite recursion between
  * expand_to_path() and index_file_exists().

sparse-index.h

@@ -11,6 +11,8 @@
 extern int give_advice_on_expansion;
 
 struct index_state;
+struct cache_entry;
+struct object_id;
 #define SPARSE_INDEX_MEMORY_ONLY (1 << 0)
 int is_sparse_index_allowed(struct index_state *istate, int flags);
 int convert_to_sparse(struct index_state *istate, int flags);
@@ -30,6 +32,34 @@ void clear_skip_worktree_from_present_files(struct index_state *istate);
 void expand_to_path(struct index_state *istate,
 		    const char *path, size_t pathlen, int icase);
 
+/*
+ * Under core.virtualFileSystem, the working tree can contain paths inside
+ * directories that a sparse index collapses into sparse-directory entries.
+ * A lookup of such a path with index_name_pos() expands the whole index.
+ *
+ * If core.virtualFileSystem is set, the index is sparse, and a
+ * sparse-directory entry contains 'path', return that entry. Otherwise
+ * return NULL. If 'icase' is set, the name of the sparse-directory entry
+ * can differ in case from the start of 'path'. This function never expands
+ * the index. If core.virtualFileSystem is set and this function returns
+ * NULL, then index_name_pos() for 'path' does not expand the index either.
+ */
+struct cache_entry *vfs_sparse_dir_containing(struct index_state *istate,
+					      const char *path, int pathlen,
+					      int icase);
+
+/*
+ * Find the part of 'path' below the sparse-directory entry 'sparse_dir'
+ * in the tree of that entry. 'sparse_dir' must contain 'path'. If 'icase'
+ * is set, compare names without case. Return 0 and set 'oid' and 'mode'
+ * if the tree contains the path. Return -1 if it does not. Die if a tree
+ * cannot be read.
+ */
+int sparse_dir_get_tree_entry(struct index_state *istate,
+			      const struct cache_entry *sparse_dir,
+			      const char *path, int pathlen, int icase,
+			      struct object_id *oid, unsigned short *mode);
+
 struct repository;
 int set_sparse_index_config(struct repository *repo, int enable);
 

t/meson.build

@@ -188,6 +188,7 @@ integration_tests = [
   't1091-sparse-checkout-builtin.sh',
   't1092-sparse-checkout-compatibility.sh',
   't1093-virtualfilesystem.sh',
+  't1094-virtualfilesystem-sparse-walk.sh',
   't1100-commit-tree-options.sh',
   't1300-config.sh',
   't1301-shared-repo.sh',