Watchman

Documentation/technical/api-builtin.txt

@@ -23,10 +23,15 @@ where options is the bitwise-or of:
 
 `RUN_SETUP`::
 
-	Make sure there is a Git directory to work on, and if there is a
-	work tree, chdir to the top of it if the command was invoked
-	in a subdirectory.  If there is no work tree, no chdir() is
-	done.
+	If there is not a Git directory to work on, abort.  If there
+	is a work tree, chdir to the top of it if the command was
+	invoked in a subdirectory.  If there is no work tree, no
+	chdir() is done.
+
+`RUN_SETUP_GENTLY`::
+
+	If there is a Git directory, chdir as per RUN_SETUP, otherwise,
+	don't chdir anywhere.
 
 `USE_PAGER`::
 

Documentation/technical/index-format.txt

@@ -32,8 +32,10 @@ Git index format
 
      Extension data
 
-   - 160-bit SHA-1 over the content of the index file before this
-     checksum.
+   - For versions before 5, a 160-bit SHA-1 over the content of the
+     index file before this checksum.  For version 5, a 128-bit VMAC
+     hash (keyed with "abcdefghijklmnop") over same, followed by 4
+     bytes of zeros.
 
 == Index entry
 

Documentation/watchman.txt

@@ -0,0 +1,27 @@
+How git uses watchman
+---------------------
+
+Git status (and some other commands) have to determine which files
+have changed between the working copy and the index.  Ordinarily, this
+requires checking every file in the working directory.  But if you
+have watchman (https://github.com/facebook/watchman) installed, git
+can cache the state of the working directory and use watchman to track
+file changes, making commands like git status faster.
+
+set core.usewatchman = true to use watchman.
+You can also set
+core.watchmansynctimeout = [number of milliseconds]
+to change watchman's sync timeout; see the watchman docs for details
+on this.  You should only change this if you see watchman timeout
+error messages.
+
+Internals
+---------
+
+The filesystem cache stores information about every file in the
+working tree.  In almost every case where git calls lstat or
+opendir/readdir, the modified file cache can be consulted instead.
+
+The file system cache is stored on disk in .git/fs_cache.  It is
+stored very similarly to the index, except without path prefix
+compression.

Makefile

@@ -405,6 +405,7 @@ TCLTK_PATH = wish
 XGETTEXT = xgettext
 MSGFMT = msgfmt
 PTHREAD_LIBS = -lpthread
+WATCHMAN_LIBS = -lwatchman
 PTHREAD_CFLAGS =
 GCOV = gcov
 
@@ -667,12 +668,14 @@ LIB_H += ewah/ewok.h
 LIB_H += ewah/ewok_rlw.h
 LIB_H += fetch-pack.h
 LIB_H += fmt-merge-msg.h
+LIB_H += fs_cache.h
 LIB_H += fsck.h
 LIB_H += gettext.h
 LIB_H += git-compat-util.h
 LIB_H += gpg-interface.h
 LIB_H += graph.h
 LIB_H += grep.h
+LIB_H += hash-io.h
 LIB_H += hashmap.h
 LIB_H += help.h
 LIB_H += http.h
@@ -711,6 +714,7 @@ LIB_H += remote.h
 LIB_H += rerere.h
 LIB_H += resolve-undo.h
 LIB_H += revision.h
+LIB_H += rijndael-alg-fst.h
 LIB_H += run-command.h
 LIB_H += send-pack.h
 LIB_H += sequencer.h
@@ -741,6 +745,7 @@ LIB_H += vcs-svn/repo_tree.h
 LIB_H += vcs-svn/sliding_window.h
 LIB_H += vcs-svn/svndiff.h
 LIB_H += vcs-svn/svndump.h
+LIB_H += vmac.h
 LIB_H += walker.h
 LIB_H += wildmatch.h
 LIB_H += wt-status.h
@@ -805,11 +810,13 @@ LIB_OBJS += ewah/ewah_io.o
 LIB_OBJS += ewah/ewah_rlw.o
 LIB_OBJS += exec_cmd.o
 LIB_OBJS += fetch-pack.o
+LIB_OBJS += fs_cache.o
 LIB_OBJS += fsck.o
 LIB_OBJS += gettext.o
 LIB_OBJS += gpg-interface.o
 LIB_OBJS += graph.o
 LIB_OBJS += grep.o
+LIB_OBJS += hash-io.o
 LIB_OBJS += hashmap.o
 LIB_OBJS += help.o
 LIB_OBJS += hex.o
@@ -863,6 +870,7 @@ LIB_OBJS += replace_object.o
 LIB_OBJS += rerere.o
 LIB_OBJS += resolve-undo.o
 LIB_OBJS += revision.o
+LIB_OBJS += rijndael-alg-fst.o
 LIB_OBJS += run-command.o
 LIB_OBJS += send-pack.o
 LIB_OBJS += sequencer.o
@@ -896,6 +904,7 @@ LIB_OBJS += utf8.o
 LIB_OBJS += varint.o
 LIB_OBJS += version.o
 LIB_OBJS += versioncmp.o
+LIB_OBJS += vmac.o
 LIB_OBJS += walker.o
 LIB_OBJS += wildmatch.o
 LIB_OBJS += wrapper.o
@@ -1326,6 +1335,11 @@ else
 		COMPAT_OBJS += compat/win32mmap.o
 	endif
 endif
+ifdef NO_SSE
+	BASIC_CFLAGS += -DNO_SSE
+else
+	BASIC_CFLAGS += -msse4
+endif
 ifdef OBJECT_CREATION_USES_RENAMES
 	COMPAT_CFLAGS += -DOBJECT_CREATION_MODE=1
 endif
@@ -1439,6 +1453,13 @@ ifdef RUNTIME_PREFIX
 	COMPAT_CFLAGS += -DRUNTIME_PREFIX
 endif
 
+ifdef USE_WATCHMAN
+	LIB_H += watchman-support.h
+	LIB_OBJS += watchman-support.o
+	EXTLIBS += $(WATCHMAN_LIBS)
+	BASIC_CFLAGS += -DUSE_WATCHMAN
+endif
+
 ifdef NO_PTHREADS
 	BASIC_CFLAGS += -DNO_PTHREADS
 else
@@ -2199,6 +2220,8 @@ GIT-BUILD-OPTIONS: FORCE
 	@echo NO_PERL=\''$(subst ','\'',$(subst ','\'',$(NO_PERL)))'\' >>$@
 	@echo NO_PYTHON=\''$(subst ','\'',$(subst ','\'',$(NO_PYTHON)))'\' >>$@
 	@echo NO_UNIX_SOCKETS=\''$(subst ','\'',$(subst ','\'',$(NO_UNIX_SOCKETS)))'\' >>$@
+	@echo USE_WATCHMAN=\''$(subst ','\'',$(subst ','\'',$(USE_WATCHMAN)))'\' >>$@
+	@echo NO_SSE=\''$(subst ','\'',$(subst ','\'',$(NO_SSE)))'\' >>$@
 ifdef TEST_OUTPUT_DIRECTORY
 	@echo TEST_OUTPUT_DIRECTORY=\''$(subst ','\'',$(subst ','\'',$(TEST_OUTPUT_DIRECTORY)))'\' >>$@
 endif

block-sha1/sha1.h

@@ -1,3 +1,6 @@
+#ifndef BLOCK_SHA1_H
+#define BLOCK_SHA1_H
+
 /*
  * SHA1 routine optimized to do word accesses rather than byte accesses,
  * and to avoid unnecessary copies into the context array.
@@ -20,3 +23,5 @@ void blk_SHA1_Final(unsigned char hashout[20], blk_SHA_CTX *ctx);
 #define git_SHA1_Init	blk_SHA1_Init
 #define git_SHA1_Update	blk_SHA1_Update
 #define git_SHA1_Final	blk_SHA1_Final
+
+#endif

cache.h

@@ -107,8 +107,8 @@ struct cache_header {
 };
 
 #define INDEX_FORMAT_LB 2
-#define INDEX_FORMAT_UB 4
-
+#define INDEX_FORMAT_UB 5
+#define INDEX_FORMAT_VMAC_LB 5
 /*
  * The "cache_time" is just the low 32 bits of the
  * time. It doesn't matter if it overflows - we only
@@ -272,6 +272,7 @@ struct index_state {
 	struct cache_entry **cache;
 	unsigned int version;
 	unsigned int cache_nr, cache_alloc, cache_changed;
+	struct fs_cache *fs_cache;
 	struct string_list *resolve_undo;
 	struct cache_tree *cache_tree;
 	struct cache_time timestamp;
@@ -352,6 +353,7 @@ static inline enum object_type object_type(unsigned int mode)
 #define DEFAULT_GIT_DIR_ENVIRONMENT ".git"
 #define DB_ENVIRONMENT "GIT_OBJECT_DIRECTORY"
 #define INDEX_ENVIRONMENT "GIT_INDEX_FILE"
+#define FS_CACHE_ENVIRONMENT "GIT_FS_CACHE_FILE"
 #define GRAFT_ENVIRONMENT "GIT_GRAFT_FILE"
 #define GIT_SHALLOW_FILE_ENVIRONMENT "GIT_SHALLOW_FILE"
 #define TEMPLATE_DIR_ENVIRONMENT "GIT_TEMPLATE_DIR"
@@ -594,6 +596,8 @@ extern int check_replace_refs;
 
 extern int fsync_object_files;
 extern int core_preload_index;
+extern int core_use_watchman;
+extern int core_watchman_sync_timeout;
 extern int core_apply_sparse_checkout;
 extern int precomposed_unicode;
 

config.c

@@ -854,6 +854,16 @@ static int git_default_core_config(const char *var, const char *value)
 		return 0;
 	}
 
+	if (!strcmp(var, "core.usewatchman")) {
+		core_use_watchman = git_config_bool(var, value);
+		return 0;
+	}
+
+	if (!strcmp(var, "core.watchmansynctimeout")) {
+		core_watchman_sync_timeout = git_config_int(var, value);
+		return 0;
+	}
+
 	if (!strcmp(var, "core.createobject")) {
 		if (!strcmp(value, "rename"))
 			object_creation_mode = OBJECT_CREATION_USES_RENAMES;

configure.ac

@@ -382,6 +382,11 @@ AS_HELP_STRING([],[Tcl/Tk interpreter will be found in a system.]),
 GIT_PARSE_WITH(tcltk))
 #
 
+# Declare the with-sse/without-sse options.
+AC_ARG_WITH(sse,
+AS_HELP_STRING([--with-sse],[use SSE instructions (default is YES)]),
+GIT_PARSE_WITH(sse))
+
 
 ## Checks for programs.
 AC_MSG_NOTICE([CHECKS for programs])
@@ -449,6 +454,7 @@ else
   fi
 fi
 GIT_CONF_SUBST([TCLTK_PATH])
+GIT_CONF_SUBST([NO_SSE])
 AC_CHECK_PROGS(ASCIIDOC, [asciidoc])
 if test -n "$ASCIIDOC"; then
 	AC_MSG_CHECKING([for asciidoc version])
@@ -962,6 +968,12 @@ GIT_CONF_SUBST([NO_INITGROUPS])
 #
 # Define NO_ICONV if your libc does not properly support iconv.
 
+# Check for watchman client library
+
+AC_CHECK_LIB([watchman], [watchman_connect],
+       [USE_WATCHMAN=YesPlease],
+       [USE_WATCHMAN=])
+GIT_CONF_SUBST([USE_WATCHMAN])
 
 ## Other checks.
 # Define USE_PIC if you need the main git objects to be built with -fPIC

diff-lib.c

@@ -2,6 +2,7 @@
  * Copyright (C) 2005 Junio C Hamano
  */
 #include "cache.h"
+#include "fs_cache.h"
 #include "quote.h"
 #include "commit.h"
 #include "diff.h"
@@ -17,24 +18,8 @@
  * diff-files
  */
 
-/*
- * Has the work tree entity been removed?
- *
- * Return 1 if it was removed from the work tree, 0 if an entity to be
- * compared with the cache entry ce still exists (the latter includes
- * the case where a directory that is not a submodule repository
- * exists for ce that is a submodule -- it is a submodule that is not
- * checked out).  Return negative for an error.
- */
-static int check_removed(const struct cache_entry *ce, struct stat *st)
+static int check_gitlink(const struct cache_entry *ce, struct stat *st)
 {
-	if (lstat(ce->name, st) < 0) {
-		if (errno != ENOENT && errno != ENOTDIR)
-			return -1;
-		return 1;
-	}
-	if (has_symlink_leading_path(ce->name, ce_namelen(ce)))
-		return 1;
 	if (S_ISDIR(st->st_mode)) {
 		unsigned char sub[20];
 
@@ -56,6 +41,52 @@ static int check_removed(const struct cache_entry *ce, struct stat *st)
 	return 0;
 }
 
+static int fs_cache_check_removed(const struct fs_cache *fs_cache, const struct cache_entry *ce, struct stat *st)
+{
+	struct fsc_entry *fe;
+
+	fe = fs_cache_file_exists(fs_cache, ce->name, ce_namelen(ce));
+	if (!fe) {
+		return 1;
+	}
+	if (fe_deleted(fe)) {
+		return 1;
+	}
+
+	fe_to_stat(fe, st);
+
+	if (check_gitlink(ce, st))
+		return 1;
+
+	return 0;
+}
+
+/*
+ * Has the work tree entity been removed?
+ *
+ * Return 1 if it was removed from the work tree, 0 if an entity to be
+ * compared with the cache entry ce still exists (the latter includes
+ * the case where a directory that is not a submodule repository
+ * exists for ce that is a submodule -- it is a submodule that is not
+ * checked out).  Return negative for an error.
+ */
+static int check_removed(const struct cache_entry *ce, struct stat *st)
+{
+	if (the_index.fs_cache)
+		return fs_cache_check_removed(the_index.fs_cache, ce, st);
+
+	if (lstat(ce->name, st) < 0) {
+		if (errno != ENOENT && errno != ENOTDIR)
+			return -1;
+		return 1;
+	}
+	if (has_symlink_leading_path(ce->name, ce_namelen(ce)))
+		return 1;
+	if (check_gitlink(ce, st))
+		return 1;
+	return 0;
+}
+
 /*
  * Has a file changed or has a submodule new commits or a dirty work tree?
  *