[PATCH 00/18] Gitweb caching v8

@@ -57,6 +57,7 @@ PERL_PATH  ?= /usr/bin/perl
 bindir_SQ = $(subst ','\'',$(bindir))#'
 gitwebdir_SQ = $(subst ','\'',$(gitwebdir))#'
 gitwebstaticdir_SQ = $(subst ','\'',$(gitwebdir)/static)#'
+gitweblibdir_SQ = $(subst ','\'',$(gitwebdir)/lib)#'
 SHELL_PATH_SQ = $(subst ','\'',$(SHELL_PATH))#'
 PERL_PATH_SQ  = $(subst ','\'',$(PERL_PATH))#'
 DESTDIR_SQ    = $(subst ','\'',$(DESTDIR))#'

@@ -153,20 +154,32 @@ test:
 
 test-installed:
 	GITWEB_TEST_INSTALLED='$(DESTDIR_SQ)$(gitwebdir_SQ)' \
+	GITWEBLIBDIR='$(DESTDIR_SQ)$(gitweblibdir_SQ)' \
 		$(MAKE) -C ../t gitweb-test
 
 ### Installation rules
 
-install: all
+install: all install-modules
 	$(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(gitwebdir_SQ)'
 	$(INSTALL) -m 755 $(GITWEB_PROGRAMS) '$(DESTDIR_SQ)$(gitwebdir_SQ)'
 	$(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(gitwebstaticdir_SQ)'
 	$(INSTALL) -m 644 $(GITWEB_FILES) '$(DESTDIR_SQ)$(gitwebstaticdir_SQ)'
 
+install-modules:
+	install_dirs="$(sort $(dir $(GITWEB_MODULES)))" && \
+	for dir in $$install_dirs; do \
+		test -d '$(DESTDIR_SQ)$(gitweblibdir_SQ)/$$dir' || \
+		$(INSTALL) -d -m 755 '$(DESTDIR_SQ)$(gitweblibdir_SQ)/$$dir'; \
+	done
+	gitweb_modules="$(GITWEB_MODULES)" && \
+	for mod in $$gitweb_modules; do \
+		$(INSTALL) -m 644 lib/$$mod '$(DESTDIR_SQ)$(gitweblibdir_SQ)/$$(dirname $$mod)'; \
+	done
+
 ### Cleaning rules
 
 clean:
 	$(RM) gitweb.cgi static/gitweb.min.js static/gitweb.min.css GITWEB-BUILD-OPTIONS
 
-.PHONY: all clean install test test-installed .FORCE-GIT-VERSION-FILE FORCE
+.PHONY: all clean install install-modules test test-installed .FORCE-GIT-VERSION-FILE FORCE
 

@@ -10,6 +10,14 @@
 use 5.008;
 use strict;
 use warnings;
+
+use File::Spec;
+# __DIR__ is taken from Dir::Self __DIR__ fragment
+sub __DIR__ () {
+	File::Spec->rel2abs(join '', (File::Spec->splitpath(__FILE__))[0, 1]);
+}
+use lib __DIR__ . '/lib';
+
 use CGI qw(:standard :escapeHTML -nosticky);
 use CGI::Util qw(unescape);
 use CGI::Carp qw(fatalsToBrowser set_message);

@@ -52,6 +52,21 @@ EOF
 	export SCRIPT_NAME
 }
 
+gitweb_enable_caching () {
+	test_expect_success 'enable caching' '
+		cat >>gitweb_config.perl <<-\EOF &&
+		our $caching_enabled = 1;
+		our $minCacheTime = 60*60*24*7*30;     # very long expiration time for tests (a month)
+		our $maxCacheTime = 60*60*24*7*30*365; # upper bound for dynamic (adaptive) caching
+		our $cachedir = "cache";               # for testsuite to clear the right thing
+		# required, because otherwise some tests might intermittently not pass
+		our $backgroundCache = 0; # should turn off cacheWaitForUpdate() / "Generating..."
+		#our $cacheDoFork = 0;
+		EOF
+		rm -rf cache/
+	'
+}
+
 gitweb_run () {
 	GATEWAY_INTERFACE='CGI/1.1'
 	HTTP_ACCEPT='*/*'

@@ -677,4 +677,24 @@ test_expect_success HIGHLIGHT \
 	 gitweb_run "p=.git;a=blob;f=test.sh"'
 test_debug 'cat gitweb.log'
 
+# ----------------------------------------------------------------------
+# caching
+
+gitweb_enable_caching
+
+test_expect_success \
+	'caching enabled (project summary, first run, generating cache)' \
+	'gitweb_run "p=.git;a=summary"'
+test_debug 'cat gitweb.log'
+
+test_expect_success \
+	'caching enabled (project summary, second run, cached version)' \
+	'gitweb_run "p=.git;a=summary"'
+test_debug 'cat gitweb.log'
+
+test_expect_success \
+	'caching enabled (non-existent commit, non-cache error page)' \
+	'gitweb_run "p=.git;a=commit;h=non-existent"'
+test_debug 'cat gitweb.log'
+
 test_done

@@ -134,5 +134,18 @@ cat >>gitweb_config.perl <<\EOF
 our $maxload = undef;
 EOF
 
+# ----------------------------------------------------------------------
+# output caching
+
+gitweb_enable_caching
+
+test_expect_success 'caching enabled (non-existent commit, 404 error)' '
+	gitweb_run "p=.git;a=commit;h=non-existent" &&
+	grep "Status: 404 Not Found" gitweb.headers &&
+	grep "404 - Unknown commit object" gitweb.body
+'
+test_debug 'echo "log"     && cat gitweb.log'
+test_debug 'echo "headers" && cat gitweb.headers'
+test_debug 'echo "body"    && cat gitweb.body'
 
 test_done

@@ -112,4 +112,37 @@ test_expect_success 'snapshot: hierarchical branch name (xx/test)' '
 '
 test_debug 'cat gitweb.headers'
 
+
+# ----------------------------------------------------------------------
+# whether gitweb with caching enabled produces the same output
+
+test_expect_success 'setup for caching tests (utf8 patch, binary file)' '
+	. "$TEST_DIRECTORY"/t3901-utf8.txt &&
+	cp "$TEST_DIRECTORY"/test9200a.png image.png &&
+	git add image.png &&
+	git commit -F "$TEST_DIRECTORY"/t3900/1-UTF-8.txt &&
+	gitweb_run "p=.git;a=patch" &&
+	mv gitweb.body no_cache.txt &&
+	gitweb_run "p=.git;a=blob_plain;f=image.png" &&
+	mv gitweb.body no_cache.png
+'
+
+gitweb_enable_caching
+
+for desc in 'generating cache' 'cached version'; do
+	test_expect_success "caching enabled, plain text (utf8) output, $desc" '
+		gitweb_run "p=.git;a=patch" &&
+		mv gitweb.body cache.txt &&
+		test_cmp no_cache.txt cache.txt
+	'
+done
+
+for desc in 'generating cache' 'cached version'; do
+	test_expect_success "caching enabled, binary output (raw), $desc" '
+		gitweb_run "p=.git;a=blob_plain;f=image.png" &&
+		mv gitweb.body cache.png &&
+		cmp no_cache.png cache.png
+	'
+done
+
 test_done

@@ -1225,7 +1225,7 @@ sub change_output {
 
 sub reset_output {
 	# This basically takes STDOUT_REAL and puts it back as STDOUT
-	open STDOUT, ">&", \*STDOUT_REAL;
+	open(STDOUT,">&STDOUT_REAL");
 }
 
 sub run {

@@ -258,7 +258,7 @@ our $maxload = 300;
 # that the cache directory be periodically completely deleted, and this is safe to perform.
 # Suggested mechanism
 # mv $cacheidr $cachedir.flush;mkdir $cachedir;rm -rf $cachedir.flush
-our $caching_enabled = 0;
+our $cache_enable = 0;
 
 # Used to set the minimum cache timeout for the dynamic caching algorithm.  Basically
 # if we calculate the cache to be under this number of seconds we set the cache timeout

@@ -1138,7 +1138,7 @@ sub dispatch {
 	    !$project) {
 		die_error(400, "Project needed");
 	}
-	if ($caching_enabled && is_cacheable($action)) {
+	if ($cache_enable && is_cacheable($action)) {
 		cache_fetch($action);
 	} else {
 		$actions{$action}->();

@@ -3516,7 +3516,7 @@ sub git_header_html {
 	if (defined $cgi->http('HTTP_ACCEPT') &&
 	    $cgi->http('HTTP_ACCEPT') =~ m/(,|;|\s|^)application\/xhtml\+xml(,|;|\s|$)/ &&
 	    $cgi->Accept('application/xhtml+xml') != 0 &&
-	    !$caching_enabled) {
+	    !$cache_enable) {
 		$content_type = 'application/xhtml+xml';
 	} else {
 		$content_type = 'text/html';

@@ -3757,7 +3757,7 @@ sub die_error {
 	#
 	# Reset the output so that we are actually going to STDOUT as opposed
 	# to buffering the output.
-	reset_output() if ($caching_enabled);
+	reset_output() if ($cache_enable && ! $cacheErrorCache);
 
 	git_header_html($http_responses{$status}, undef, %opts);
 	print <<EOF;

@@ -5667,7 +5667,7 @@ sub git_blob_plain {
 			($sandbox ? 'attachment' : 'inline')
 			. '; filename="' . $save_as . '"');
 	local $/ = undef;
-	if ($caching_enabled) {
+	if ($cache_enable) {
 		open BINOUT, '>', $fullhashbinpath or die_error(500, "Could not open bin dump file");
 	}else{
 		open BINOUT, '>&', \$fullhashbinpath or die_error(500, "Could not open bin dump file");

@@ -5960,7 +5960,7 @@ sub git_snapshot {
 
 	open my $fd, "-|", $cmd
 		or die_error(500, "Execute git-archive failed");
-	if ($caching_enabled) {
+	if ($cache_enable) {
 		open BINOUT, '>', $fullhashbinpath or die_error(500, "Could not open bin dump file");
 	}else{
 		open BINOUT, '>&', \$fullhashbinpath or die_error(500, "Could not open bin dump file");

@@ -56,7 +56,7 @@ EOF
 gitweb_enable_caching () {
 	test_expect_success 'enable caching' '
 		cat >>gitweb_config.perl <<-\EOF &&
-		our $caching_enabled = 1;
+		our $cache_enable = 1;
 		our $minCacheTime = 60*60*24*7*30;     # very long expiration time for tests (a month)
 		our $maxCacheTime = 60*60*24*7*30*365; # upper bound for dynamic (adaptive) caching
 		our $cachedir = "cache";               # for testsuite to clear the right thing

@@ -798,7 +798,8 @@ our %actions = (
 );
 sub is_cacheable {
 	my $action = shift;
-	return !($action eq 'blame_data' || $action eq 'blame_incremental');
+	# There are no known actions that do no involve a unique URL that shouldn't be cached.
+	return 1;
 }
 
 # finally, we have the hash of allowed extra_options for the commands that

@@ -5664,7 +5664,7 @@ sub git_blob_plain {
 	if ($caching_enabled) {
 		open BINOUT, '>', $fullhashbinpath or die_error(500, "Could not open bin dump file");
 	}else{
-		open BINOUT, '>', \$fullhashbinpath or die_error(500, "Could not open bin dump file");
+		open BINOUT, '>&', \$fullhashbinpath or die_error(500, "Could not open bin dump file");
 	}
 	binmode BINOUT, ':raw';
 	print BINOUT <$fd>;

@@ -5957,7 +5957,7 @@ sub git_snapshot {
 	if ($caching_enabled) {
 		open BINOUT, '>', $fullhashbinpath or die_error(500, "Could not open bin dump file");
 	}else{
-		open BINOUT, '>', \$fullhashbinpath or die_error(500, "Could not open bin dump file");
+		open BINOUT, '>&', \$fullhashbinpath or die_error(500, "Could not open bin dump file");
 	}
 	binmode BINOUT, ':raw';
 	print BINOUT <$fd>;

@@ -94,6 +94,31 @@ sub cache_fetch {
 	#$actions{$action}->();
 }
 
+sub print_transient_header {
+	print $::cgi->header(
+				-type=>'text/html',
+				-charset => 'utf-8',
+				-status=> 200,
+				-expires => 'now',
+				# HTTP/1.0
+				-Pragma => 'no-cache',
+				# HTTP/1.1
+				-Cache_Control => join(
+							', ',
+							qw(
+								private
+								no-cache
+								no-store
+								must-revalidate
+								max-age=0
+								pre-check=0
+								post-check=0
+							)
+						)
+				);
+	return;
+}
+
 sub isBinaryAction {
 	my ($action) = @_;
 

@@ -292,27 +317,7 @@ sub cacheWaitForUpdate {
 
 	$| = 1;
 
-	print $::cgi->header(
-				-type=>'text/html',
-				-charset => 'utf-8',
-				-status=> 200,
-				-expires => 'now',
-				# HTTP/1.0
-				-Pragma => 'no-cache',
-				# HTTP/1.1
-				-Cache_Control => join(
-							', ',
-							qw(
-								private
-								no-cache
-								no-store
-								must-revalidate
-								max-age=0
-								pre-check=0
-								post-check=0
-							)
-						)
-				);
+	print_transient_header();
 
 	print <<EOF;
 <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www/w3.porg/TR/html4/strict.dtd">

@@ -1214,6 +1214,10 @@ sub evaluate_argv {
 sub change_output {
 	our $output;
 
+	#
+	# STDOUT
+	#
+
 	# Trap the 'proper' STDOUT to STDOUT_REAL for things like error messages and such
 	open(STDOUT_REAL,">&STDOUT") or die "Unable to capture STDOUT $!\n";
 	print STDOUT_REAL "";

@@ -1223,12 +1227,28 @@ sub change_output {
 
 	# Trap STDOUT to the $output variable, which is what I was using in the original
 	# patch anyway.
-	open(STDOUT,">", \$output) || die "Unable to open STDOUT: $!"; #open STDOUT handle to use $var
+	open(STDOUT,">", \$output) || die "Unable to open STDOUT: $!"; #open STDOUT handle to use $output
+
+	#
+	# STDERR
+	#
+
+	# Trap the 'proper' STDOUT to STDOUT_REAL for things like error messages and such
+	open(STDERR_REAL,">&STDERR") or die "Unable to capture STDERR $!\n";
+	print STDERR_REAL "";
+
+	# Close STDOUT, so that it isn't being used anymore.
+	close STDERR;
+
+	# Trap STDOUT to the $output variable, which is what I was using in the original
+	# patch anyway.
+	open(STDERR,">", \$output_err) || die "Unable to open STDERR: $!"; #open STDERR handle to use $output_err
 }
 
 sub reset_output {
 	# This basically takes STDOUT_REAL and puts it back as STDOUT
 	open(STDOUT,">&STDOUT_REAL");
+	open(STDERR,">&STDERR_REAL");
 }
 
 sub run {

@@ -380,28 +380,6 @@ EOF
 	return;
 }
 
-sub cacheDisplayErr {
-
-	return if ( ! -e "$fullhashpath.err" );
-
-	open($cacheFileErr, '<:utf8', "$fullhashpath.err");
-	$lockStatus = flock($cacheFileErr,LOCK_SH|LOCK_NB);
-
-	if (! $lockStatus ){
-		show_warning(
-				"<p>".
-				"<strong>*** Warning ***:</strong> Locking error when trying to lock error cache page, file $fullhashpath.err<br/>/\n".
-				"This is about as screwed up as it gets folks - see your systems administrator for more help with this.".
-				"<p>"
-				);
-	}
-
-	while( <$cacheFileErr> ){
-		print $_;
-	}
-	exit(0);
-}
-
 sub cacheDisplay {
 	local $/ = undef;
 	$|++;

@@ -14,6 +14,12 @@ use Digest::MD5 qw(md5 md5_hex md5_base64);
 use Fcntl ':flock';
 use File::Copy;
 
+# Global declarations
+our $cacheFile;
+our $cacheFileBG;
+our $cacheFileBinWT;
+our $cacheFileBin;
+
 sub cache_fetch {
 	my ($action) = @_;
 	my $cacheTime = 0;

@@ -49,9 +55,9 @@ sub cache_fetch {
 	}else{
 		#if cache is out dated, update
 		#else displayCache();
-		open(cacheFile, '<', "$fullhashpath");
-		stat(cacheFile);
-		close(cacheFile);
+		open($cacheFile, '<', "$fullhashpath");
+		stat($cacheFile);
+		close($cacheFile);
 		my $stat_time = (stat(_))[9];
 		my $stat_size = (stat(_))[7];
 

@@ -150,13 +156,13 @@ sub cacheUpdate {
 	my $fileData = "";
 
 	if($backgroundCache){
-		open(cacheFileBG, '>:utf8', "$fullhashpath.bg");
-		my $lockStatBG = flock(cacheFileBG,LOCK_EX|LOCK_NB);
+		open($cacheFileBG, '>:utf8', "$fullhashpath.bg");
+		my $lockStatBG = flock($cacheFileBG,LOCK_EX|LOCK_NB);
 
 		$lockStatus = $lockStatBG;
 	}else{
-		open(cacheFile, '>:utf8', \$fullhashpath);
-		my $lockStat = flock(cacheFile,LOCK_EX|LOCK_NB);
+		open($cacheFile, '>:utf8', \$fullhashpath);
+		my $lockStat = flock($cacheFile,LOCK_EX|LOCK_NB);
 
 		$lockStatus = $lockStat;
 	}

@@ -172,8 +178,8 @@ sub cacheUpdate {
 	}
 
 	if( isBinaryAction($action) ){
-		my $openstat = open(cacheFileBinWT, '>>:utf8', "$fullhashbinpath");
-		my $lockStatBin = flock(cacheFileBinWT,LOCK_EX|LOCK_NB);
+		my $openstat = open($cacheFileBinWT, '>>:utf8', "$fullhashbinpath");
+		my $lockStatBin = flock($cacheFileBinWT,LOCK_EX|LOCK_NB);
 	}
 
 	# Trap all output from the action

@@ -186,8 +192,8 @@ sub cacheUpdate {
 
 
 	if($backgroundCache){
-		open(cacheFile, '>:utf8', "$fullhashpath");
-		$lockStat = flock(cacheFile,LOCK_EX);
+		open($cacheFile, '>:utf8', "$fullhashpath");
+		$lockStat = flock($cacheFile,LOCK_EX);
 
 		if (! $lockStat ){
 			if ( $areForked ){

@@ -199,8 +205,8 @@ sub cacheUpdate {
 	}
 
 	if( isBinaryAction($action) ){
-		my $openstat = open(cacheFileBinFINAL, '>:utf8', "$fullhashbinpathfinal");
-		$lockStatBIN = flock(cacheFileBinFINAL,LOCK_EX);
+		my $openstat = open($cacheFileBinFINAL, '>:utf8', "$fullhashbinpathfinal");
+		$lockStatBIN = flock($cacheFileBinFINAL,LOCK_EX);
 
 		if (! $lockStatBIN ){
 			if ( $areForked ){

@@ -214,24 +220,24 @@ sub cacheUpdate {
 	# Actually dump the output to the proper file handler
 	local $/ = undef;
 	$|++;
-	print cacheFile "$output";
+	print $cacheFile "$output";
 	$|--;
 	if( isBinaryAction($action) ){
 		move("$fullhashbinpath", "$fullhashbinpathfinal") or die "Binary Cache file could not be updated: $!";
 
-		flock(cacheFileBinFINAL,LOCK_UN);
-		close(cacheFileBinFINAL);
+		flock($cacheFileBinFINAL,LOCK_UN);
+		close($cacheFileBinFINAL);
 
-		flock(cacheFileBinWT,LOCK_UN);
-		close(cacheFileBinWT);
+		flock($cacheFileBinWT,LOCK_UN);
+		close($cacheFileBinWT);
 	}
 
-	flock(cacheFile,LOCK_UN);
-	close(cacheFile);
+	flock($cacheFile,LOCK_UN);
+	close($cacheFile);
 
 	if($backgroundCache){
-		flock(cacheFileBG,LOCK_UN);
-		close(cacheFileBG);
+		flock($cacheFileBG,LOCK_UN);
+		close($cacheFileBG);
 	}
 
 	if ( $areForked ){

@@ -250,10 +256,10 @@ sub cacheWaitForUpdate {
 
 	if( $backgroundCache ){
 		if( -e "$fullhashpath" ){
-			open(cacheFile, '<:utf8', "$fullhashpath");
-			$lockStat = flock(cacheFile,LOCK_SH|LOCK_NB);
-			stat(cacheFile);
-			close(cacheFile);
+			open($cacheFile, '<:utf8', "$fullhashpath");
+			$lockStat = flock($cacheFile,LOCK_SH|LOCK_NB);
+			stat($cacheFile);
+			close($cacheFile);
 
 			if( $lockStat && ( (stat(_))[9] > (time - $maxCacheLife) ) ){
 				cacheDisplay($action);

@@ -271,9 +277,9 @@ sub cacheWaitForUpdate {
 	  ){
 		do {
 			sleep 2 if $x > 0;
-			open(cacheFile, '<:utf8', "$fullhashpath");
-			$lockStat = flock(cacheFile,LOCK_SH|LOCK_NB);
-			close(cacheFile);
+			open($cacheFile, '<:utf8', "$fullhashpath");
+			$lockStat = flock($cacheFile,LOCK_SH|LOCK_NB);
+			close($cacheFile);
 			$x++;
 			$combinedLockStat = $lockStat;
 		} while ((! $combinedLockStat) && ($x < $max));

@@ -326,9 +332,9 @@ EOF
 	do {
 		print ".";
 		sleep 2 if $x > 0;
-		open(cacheFile, '<:utf8', "$fullhashpath");
-		$lockStat = flock(cacheFile,LOCK_SH|LOCK_NB);
-		close(cacheFile);
+		open($cacheFile, '<:utf8', "$fullhashpath");
+		$lockStat = flock($cacheFile,LOCK_SH|LOCK_NB);
+		close($cacheFile);
 		$x++;
 		$combinedLockStat = $lockStat;
 	} while ((! $combinedLockStat) && ($x < $max));

@@ -339,41 +345,63 @@ EOF
 	return;
 }
 
+sub cacheDisplayErr {
+
+	return if ( ! -e "$fullhashpath.err" );
+
+	open($cacheFileErr, '<:utf8', "$fullhashpath.err");
+	$lockStatus = flock($cacheFileErr,LOCK_SH|LOCK_NB);
+
+	if (! $lockStatus ){
+		show_warning(
+				"<p>".
+				"<strong>*** Warning ***:</strong> Locking error when trying to lock error cache page, file $fullhashpath.err<br/>/\n".
+				"This is about as screwed up as it gets folks - see your systems administrator for more help with this.".
+				"<p>"
+				);
+	}
+
+	while( <$cacheFileErr> ){
+		print $_;
+	}
+	exit(0);
+}
+
 sub cacheDisplay {
 	local $/ = undef;
 	$|++;
 
 	my ($action) = @_;
-	open(cacheFile, '<:utf8', "$fullhashpath");
-	$lockStat = flock(cacheFile,LOCK_SH|LOCK_NB);
+	open($cacheFile, '<:utf8', "$fullhashpath");
+	$lockStat = flock($cacheFile,LOCK_SH|LOCK_NB);
 
 	if (! $lockStat ){
-		close(cacheFile);
+		close($cacheFile);
 		cacheWaitForUpdate($action);
 	}
 
 	if( isBinaryAction($action) ){
-		my $openstat = open(cacheFileBin, '<', "$fullhashbinpathfinal");
-		$lockStatBIN = flock(cacheFileBin,LOCK_SH|LOCK_NB);
+		my $openstat = open($cacheFileBin, '<', "$fullhashbinpathfinal");
+		$lockStatBIN = flock($cacheFileBin,LOCK_SH|LOCK_NB);
 		if (! $lockStatBIN ){
-			close(cacheFile);
-			close(cacheFileBin);
+			close($cacheFile);
+			close($cacheFileBin);
 			cacheWaitForUpdate($action);
 		}
 
 		my $binfilesize = -s "$fullhashbinpathfinal";
 		print "Content-Length: $binfilesize";
 	}
-	while( <cacheFile> ){
+	while( <$cacheFile> ){
 		print $_;
 	}
 	if( isBinaryAction($action) ){
 		binmode STDOUT, ':raw';
-		print <cacheFileBin>;
+		print <$cacheFileBin>;
 		binmode STDOUT, ':utf8'; # as set at the beginning of gitweb.cgi
-		close(cacheFileBin);
+		close($cacheFileBin);
 	}
-	close(cacheFile);
+	close($cacheFile);
 	$|--;
 }
 

@@ -303,6 +303,9 @@ our $fullhashpath = *STDOUT;
 our $fullhashbinpath = *STDOUT;
 our $fullhashbinpathfinal = *STDOUT;
 
+our $full_url;
+our $urlhash;
+
 # configuration for 'highlight' (http://www.andre-simon.de/)
 # match by basename
 our %highlight_basename = (

@@ -3663,6 +3666,10 @@ sub git_footer_html {
 
 	print "<div class=\"page_footer\">\n";
 	print "<div class=\"cachetime\">Cache Last Updated: ". gmtime( time ) ." GMT</div>\n";
+	print	"<!--\n".
+		"	Full URL: |$full_url|\n".
+		"	URL Hash: |$urlhash|\n".
+		"-->\n" if ($cache_enable);
 	if (defined $project) {
 		my $descr = git_get_project_description($project);
 		if (defined $descr) {

@@ -30,8 +30,8 @@ sub cache_fetch {
 		print "Cache directory created successfully\n";
 	}
 
-	our $full_url = "$my_url?". $ENV{'QUERY_STRING'};
-	our $urlhash = md5_hex($full_url);
+	$full_url = "$my_url?". $ENV{'QUERY_STRING'};
+	$urlhash = md5_hex($full_url);
 	our $fullhashdir = "$cachedir/". substr( $urlhash, 0, 2) ."/";
 
 	eval { mkpath( $fullhashdir, 0, 0777 ) };

@@ -88,6 +88,34 @@ sub cache_fetch {
 	#$actions{$action}->();
 }
 
+sub isBinaryAction {
+	my ($action) = @_;
+
+	if(
+		$action eq "snapshot"
+		||
+		$action eq "blob_plain"
+	){
+		return 1;	# True
+	}
+
+	return 0;		# False
+}
+
+sub isFeedAction {
+	if(
+		$action eq "atom"
+		||
+		$action eq "rss"
+		||
+		$action eq "opml"
+	){
+		return 1;	# True
+	}
+
+	return 0;		# False
+}
+
 sub cacheUpdate {
 	my ($action,$areForked) = @_;
 	my $lockingStatus;

@@ -115,11 +143,7 @@ sub cacheUpdate {
 		}
 	}
 
-	if(
-		$action eq "snapshot"
-		||
-		$action eq "blob_plain"
-	){
+	if( isBinaryAction($action) ){
 		my $openstat = open(cacheFileBinWT, '>>:utf8', "$fullhashbinpath");
 		my $lockStatBin = flock(cacheFileBinWT,LOCK_EX|LOCK_NB);
 	}

@@ -146,11 +170,7 @@ sub cacheUpdate {
 		}
 	}
 
-	if(
-		$action eq "snapshot"
-		||
-		$action eq "blob_plain"
-	){
+	if( isBinaryAction($action) ){
 		my $openstat = open(cacheFileBinFINAL, '>:utf8', "$fullhashbinpathfinal");
 		$lockStatBIN = flock(cacheFileBinFINAL,LOCK_EX);
 

@@ -168,11 +188,7 @@ sub cacheUpdate {
 	$|++;
 	print cacheFile "$output";
 	$|--;
-	if(
-		$action eq "snapshot"
-		||
-		$action eq "blob_plain"
-	){
+	if( isBinaryAction($action) ){
 		move("$fullhashbinpath", "$fullhashbinpathfinal") or die "Binary Cache file could not be updated: $!";
 
 		flock(cacheFileBinFINAL,LOCK_UN);

@@ -219,14 +235,10 @@ sub cacheWaitForUpdate {
 	}
 
 	if(
-		$action eq "atom"
-		||
-		$action eq "rss"
-		||
-		$action eq "opml"
+		isFeedAction($action)
 		||
 		! $cacheGenStatus
-	){
+	  ){
 		do {
 			sleep 2 if $x > 0;
 			open(cacheFile, '<:utf8', "$fullhashpath");

@@ -310,17 +322,10 @@ sub cacheDisplay {
 		cacheWaitForUpdate($action);
 	}
 
-	if(
-		(
-			$action eq "snapshot"
-			||
-			$action eq "blob_plain"
-		)
-	){
+	if( isBinaryAction($action) ){
 		my $openstat = open(cacheFileBin, '<', "$fullhashbinpathfinal");
 		$lockStatBIN = flock(cacheFileBin,LOCK_SH|LOCK_NB);
 		if (! $lockStatBIN ){
-			system ("echo 'cacheDisplay - bailing due to binary lock failure' >> /tmp/gitweb.log");
 			close(cacheFile);
 			close(cacheFileBin);
 			cacheWaitForUpdate($action);

@@ -332,11 +337,7 @@ sub cacheDisplay {
 	while( <cacheFile> ){
 		print $_;
 	}
-	if(
-		$action eq "snapshot"
-		||
-		$action eq "blob_plain"
-	){
+	if( isBinaryAction($action) ){
 		binmode STDOUT, ':raw';
 		print <cacheFileBin>;
 		binmode STDOUT, ':utf8'; # as set at the beginning of gitweb.cgi

@@ -116,6 +116,34 @@ sub isFeedAction {
 	return 0;		# False
 }
 
+# There have been a number of requests that things like "dumb" clients, I.E. wget
+# lynx, links, etc (things that just download, but don't parse the html) actually
+# work without getting the wonkiness that is the "Generating..." page.
+#
+# There's only one good way to deal with this, and that's to read the browser User
+# Agent string and do matching based on that.  This has a whole slew of error cases
+# and mess, but there's no other way to determine if the "Generating..." page
+# will break things.
+#
+# This assumes the client is not dumb, thus the default behavior is to return
+# "false" (0) (and eventually the "Generating..." page).  If it is a dumb client
+# return "true" (1)
+sub isDumbClient {
+	my($user_agent) = $ENV{'HTTP_USER_AGENT'};
+	
+	if(
+		# wget case
+		$user_agent =~ /^Wget/i
+		||
+		# curl should be excluded I think, probably better safe than sorry
+		$user_agent =~ /^curl/i
+	  ){
+		return 1;	# True
+	}
+
+	return 0;
+}
+
 sub cacheUpdate {
 	my ($action,$areForked) = @_;
 	my $lockingStatus;

@@ -237,6 +265,8 @@ sub cacheWaitForUpdate {
 	if(
 		isFeedAction($action)
 		||
+		isDumbClient()
+		||
 		! $cacheGenStatus
 	  ){
 		do {

@@ -41,6 +41,7 @@ our $version = "++GIT_VERSION++";
 
 # Output buffer variable
 our $output = "";
+our $output_err = "";
 
 our ($my_url, $my_uri, $base_url, $path_info, $home_link);
 sub evaluate_uri {

@@ -303,6 +304,9 @@ our $fullhashpath = *STDOUT;
 our $fullhashbinpath = *STDOUT;
 our $fullhashbinpathfinal = *STDOUT;
 
+our $cacheErrorCache = 0; # false
+our $cacheErrorCount = 0;
+
 our $full_url;
 our $urlhash;
 

@@ -3786,6 +3790,7 @@ sub die_error {
 	# Reset the output so that we are actually going to STDOUT as opposed
 	# to buffering the output.
 	reset_output() if ($cache_enable && ! $cacheErrorCache);
+	$cacheErrorCount++ if( $cacheErrorCache );
 
 	git_header_html($http_responses{$status}, undef, %opts);
 	print <<EOF;

@@ -3801,6 +3806,9 @@ EOF
 	print "</div>\n";
 
 	git_footer_html();
+
+	die_error_cache($output) if ( $cacheErrorCache );
+
 	goto DONE_GITWEB
 		unless ($opts{'-error_handler'});
 }

@@ -240,8 +240,14 @@ sub cacheUpdate {
 	# Trap all output from the action
 	change_output();
 
+	# Set the error handler so we cache
+	$cacheErrorCache = 1; # true
+
 	$actions{$action}->();
 
+	# Reset Error Handler to not cache
+	$cacheErrorCache = 0; # false
+
 	# Reset the outputs as we should be fine now
 	reset_output();
 

@@ -295,6 +301,8 @@ sub cacheUpdate {
 		close($cacheFileBG);
 	}
 
+	unlink("$fullhashpath.err") if (-e "$fullhashpath.err");
+
 	if ( $areForked ){
 		exit(0);
 	} else {

@@ -339,6 +347,9 @@ sub cacheWaitForUpdate {
 	my $max = 10;
 	my $lockStat = 0;
 
+	# Call cacheDisplayErr - if an error exists it will display and die.  If not it will just return
+	cacheDisplayErr($action);
+
 	if( $backgroundCache ){
 		if( -e "$fullhashpath" ){
 			open($cacheFile, '<:utf8', "$fullhashpath");

@@ -402,6 +413,9 @@ EOF
 		close($cacheFile);
 		$x++;
 		$combinedLockStat = $lockStat;
+
+		# Call cacheDisplayErr - if an error exists it will display and die.  If not it will just return
+		cacheDisplayErr($action);
 	} while ((! $combinedLockStat) && ($x < $max));
 	print <<EOF;
 </body>

@@ -302,6 +302,36 @@ sub cacheUpdate {
 	}
 }
 
+sub die_error_cache {
+	my ($output) = @_;
+
+	open(my $cacheFileErr, '>:utf8', "$fullhashpath.err");
+	my $lockStatus = flock($cacheFileErr,LOCK_EX|LOCK_NB);
+
+	if (! $lockStatus ){
+		if ( $areForked ){
+			exit(0);
+		}else{
+			return;
+		}
+	}
+
+	# Actually dump the output to the proper file handler
+	local $/ = undef;
+	$|++;
+	print $cacheFileErr "$output";
+	$|--;
+
+	flock($cacheFileErr,LOCK_UN);
+	close($cacheFileErr);
+
+	if ( $areForked ){
+		exit(0);
+	}else{
+		return;
+	}
+}
+
 
 sub cacheWaitForUpdate {
 	my ($action) = @_;

@@ -380,6 +410,28 @@ EOF
 	return;
 }
 
+sub cacheDisplayErr {
+
+	return if ( ! -e "$fullhashpath.err" );
+
+	open($cacheFileErr, '<:utf8', "$fullhashpath.err");
+	$lockStatus = flock($cacheFileErr,LOCK_SH|LOCK_NB);
+
+	if (! $lockStatus ){
+		show_warning(
+				"<p>".
+				"<strong>*** Warning ***:</strong> Locking error when trying to lock error cache page, file $fullhashpath.err<br/>/\n".
+				"This is about as screwed up as it gets folks - see your systems administrator for more help with this.".
+				"<p>"
+				);
+	}
+
+	while( <$cacheFileErr> ){
+		print $_;
+	}
+	exit(0);
+}
+
 sub cacheDisplay {
 	local $/ = undef;
 	$|++;

@@ -25,9 +25,13 @@ sub cache_fetch {
 	my $cacheTime = 0;
 
 	if(! -d $cachedir){
-		print "*** Warning ***: Caching enabled but cache directory does not exsist.  ($cachedir)\n";
-		mkdir ("cache", 0755) || die "Cannot create cache dir - you will need to manually create";
-		print "Cache directory created successfully\n";
+		mkdir ("cache", 0755) || die_error(500, "Internal Server Error", "Cannot create cache dir () - you will need to manually create");
+		show_warning(
+				"<p>".
+				"<strong>*** Warning ***:</strong> Caching enabled but cache directory did not exsist.  ($cachedir)<br/>/\n".
+				"Cache directory created successfully\n".
+				"<p>"
+				);
 	}
 
 	$full_url = "$my_url?". $ENV{'QUERY_STRING'};

@@ -119,6 +123,32 @@ sub print_transient_header {
 	return;
 }
 
+sub show_warning {
+	$| = 1;
+
+	my $warning = esc_html(shift) || "Unknown Warning";
+
+	print_transient_header();
+
+	print <<EOF;
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www/w3.porg/TR/html4/strict.dtd">
+<!-- git web w/caching interface version $version, (C) 2006-2010, John 'Warthog9' Hawley <warthog9\@kernel.org> -->
+<!-- git core binaries version $git_version -->
+<head>
+<meta http-equiv="content-type" content="$content_type; charset=utf-8"/>
+<meta name="generator" content="gitweb/$version git/$git_version"/>
+<meta name="robots" content="index, nofollow"/>
+<meta http-equiv="refresh" content="10"/>
+<title>$title</title>
+</head>
+<body>
+$warning
+</body>
+</html>
+EOF
+	exit(0);
+}
+
 sub isBinaryAction {
 	my ($action) = @_;
 

@@ -39,6 +39,9 @@ BEGIN {
 
 our $version = "++GIT_VERSION++";
 
+# Output buffer variable
+our $output = "";
+
 our ($my_url, $my_uri, $base_url, $path_info, $home_link);
 sub evaluate_uri {
 	our $cgi;

@@ -1134,6 +1137,25 @@ sub evaluate_argv {
 	);
 }
 
+sub change_output {
+	our $output;
+
+	# Trap the 'proper' STDOUT to STDOUT_REAL for things like error messages and such
+	open(STDOUT_REAL,">&STDOUT") or die "Unable to capture STDOUT $!\n";
+
+	# Close STDOUT, so that it isn't being used anymore.
+	close STDOUT;
+
+	# Trap STDOUT to the $output variable, which is what I was using in the original
+	# patch anyway.
+	open(STDOUT,">", \$output) || die "Unable to open STDOUT: $!"; #open STDOUT handle to use $var
+}
+
+sub reset_output {
+	# This basically takes STDOUT_REAL and puts it back as STDOUT
+	open STDOUT, ">&", \*STDOUT_REAL;
+}
+
 sub run {
 	evaluate_argv();
 

@@ -1145,7 +1167,10 @@ sub run {
 		$pre_dispatch_hook->()
 			if $pre_dispatch_hook;
 
+		change_output();
 		run_request();
+		reset_output();
+		print $output;
 
 		$post_dispatch_hook->()
 			if $post_dispatch_hook;

@@ -3655,6 +3680,10 @@ sub die_error {
 		500 => '500 Internal Server Error',
 		503 => '503 Service Unavailable',
 	);
+	# Reset the output so that we are actually going to STDOUT as opposed
+	# to buffering the output.
+	reset_output();
+
 	git_header_html($http_responses{$status}, undef, %opts);
 	print <<EOF;
 <div class="page_body">

@@ -293,6 +293,12 @@ our $maxCacheLife = 18000;
 # Used to enable or disable background forking of the gitweb caching.  Mainly here for debugging purposes
 our $cacheDoFork = 1;
 
+# Used to enable or disable the foreground "Generating..." page.  This is here to be more explicit should
+# people want to disable it.
+# Default: 1 (True - Enabled)
+# To disable set to 0
+our $cacheGenStatus = 1;
+
 our $fullhashpath = *STDOUT;
 our $fullhashbinpath = *STDOUT;
 our $fullhashbinpathfinal = *STDOUT;

@@ -224,6 +224,8 @@ sub cacheWaitForUpdate {
 		$action eq "rss"
 		||
 		$action eq "opml"
+		||
+		! $cacheGenStatus
 	){
 		do {
 			sleep 2 if $x > 0;

@@ -26,6 +26,7 @@ our \$projects_list = '';
 our \$export_ok = '';
 our \$strict_export = '';
 our \$maxload = undef;
+our \$cacheGenStatus = 0;
 
 EOF
 

@@ -113,6 +113,9 @@ endif
 
 GITWEB_FILES += static/git-logo.png static/git-favicon.png
 
+# Gitweb caching
+GITWEB_MODULES += cache.pl
+
 GITWEB_REPLACE = \
 	-e 's|++GIT_VERSION++|$(GIT_VERSION)|g' \
 	-e 's|++GIT_BINDIR++|$(bindir)|g' \

@@ -250,6 +250,53 @@ our %avatar_size = (
 # Leave it undefined (or set to 'undef') to turn off load checking.
 our $maxload = 300;
 
+# This enables/disables the caching layer in gitweb.  This currently only supports the
+# 'dumb' file based caching layer, primarily used on git.kernel.org.  this is reasonably
+# effective but it has the downside of requiring a huge amount of disk space if there
+# are a number of repositories involved.  It is not uncommon for git.kernel.org to have
+# on the order of 80G - 120G accumulate over the course of a few months.  It is recommended
+# that the cache directory be periodically completely deleted, and this is safe to perform.
+# Suggested mechanism
+# mv $cacheidr $cachedir.flush;mkdir $cachedir;rm -rf $cachedir.flush
+our $caching_enabled = 0;
+
+# Used to set the minimum cache timeout for the dynamic caching algorithm.  Basically
+# if we calculate the cache to be under this number of seconds we set the cache timeout
+# to this minimum.
+# Value is in seconds.  1 = 1 seconds, 60 = 1 minute, 600 = 10 minutes, 3600 = 1 hour
+our $minCacheTime = 20;
+
+# Used to set the maximum cache timeout for the dynamic caching algorithm.  Basically
+# if we calculate the cache to exceed this number of seconds we set the cache timeout
+# to this maximum.
+# Value is in seconds.  1 = 1 seconds, 60 = 1 minute, 600 = 10 minutes, 3600 = 1 hour
+our $maxCacheTime = 1200;
+
+# If you need to change the location of the caching directory, override this
+# otherwise this will probably do fine for you
+our $cachedir = 'cache';
+
+# If this is set (to 1) cache will do it's best to always display something instead
+# of making someone wait for the cache to update.  This will launch the cacheUpdate
+# into the background and it will lock a <file>.bg file and will only lock the
+# actual cache file when it needs to write into it.  In theory this will make
+# gitweb seem more responsive at the price of possibly stale data.
+our $backgroundCache = 1;
+
+# Used to set the maximum cache file life.  If a cache files last modify time exceeds
+# this value, it will assume that the data is just too old, and HAS to be regenerated
+# instead of trying to display the existing cache data.
+# Value is in seconds.  1 = 1 seconds, 60 = 1 minute, 600 = 10 minutes, 3600 = 1 hour
+# 18000 = 5 hours
+our $maxCacheLife = 18000;
+
+# Used to enable or disable background forking of the gitweb caching.  Mainly here for debugging purposes
+our $cacheDoFork = 1;
+
+our $fullhashpath = *STDOUT;
+our $fullhashbinpath = *STDOUT;
+our $fullhashbinpathfinal = *STDOUT;
+
 # configuration for 'highlight' (http://www.andre-simon.de/)
 # match by basename
 our %highlight_basename = (

@@ -506,6 +553,15 @@ our %feature = (
 		'default' => [0]},
 );
 
+#
+# Includes
+#
+if (!exists $INC{'cache.pl'}) {
+	my $return = do 'cache.pl';
+	die $@ if $@;
+	die "Couldn't read 'cache.pl': $!" if (!defined $return);
+}
+
 sub gitweb_get_feature {
 	my ($name) = @_;
 	return unless exists $feature{$name};

@@ -734,6 +790,10 @@ our %actions = (
 	"project_list" => \&git_project_list,
 	"project_index" => \&git_project_index,
 );
+sub is_cacheable {
+	my $action = shift;
+	return !($action eq 'blame_data' || $action eq 'blame_incremental');
+}
 
 # finally, we have the hash of allowed extra_options for the commands that
 # allow them

@@ -1072,7 +1132,11 @@ sub dispatch {
 	    !$project) {
 		die_error(400, "Project needed");
 	}
-	$actions{$action}->();
+	if ($caching_enabled && is_cacheable($action)) {
+		cache_fetch($action);
+	} else {
+		$actions{$action}->();
+	}
 }
 
 sub reset_timer {

@@ -1142,6 +1206,7 @@ sub change_output {
 
 	# Trap the 'proper' STDOUT to STDOUT_REAL for things like error messages and such
 	open(STDOUT_REAL,">&STDOUT") or die "Unable to capture STDOUT $!\n";
+	print STDOUT_REAL "";
 
 	# Close STDOUT, so that it isn't being used anymore.
 	close STDOUT;

@@ -1167,10 +1232,7 @@ sub run {
 		$pre_dispatch_hook->()
 			if $pre_dispatch_hook;
 
-		change_output();
 		run_request();
-		reset_output();
-		print $output;
 
 		$post_dispatch_hook->()
 			if $post_dispatch_hook;

@@ -3447,7 +3509,8 @@ sub git_header_html {
 	# support xhtml+xml but choking when it gets what it asked for.
 	if (defined $cgi->http('HTTP_ACCEPT') &&
 	    $cgi->http('HTTP_ACCEPT') =~ m/(,|;|\s|^)application\/xhtml\+xml(,|;|\s|$)/ &&
-	    $cgi->Accept('application/xhtml+xml') != 0) {
+	    $cgi->Accept('application/xhtml+xml') != 0 &&
+	    !$caching_enabled) {
 		$content_type = 'application/xhtml+xml';
 	} else {
 		$content_type = 'text/html';

@@ -3592,6 +3655,7 @@ sub git_footer_html {
 	my $feed_class = 'rss_logo';
 
 	print "<div class=\"page_footer\">\n";
+	print "<div class=\"cachetime\">Cache Last Updated: ". gmtime( time ) ." GMT</div>\n";
 	if (defined $project) {
 		my $descr = git_get_project_description($project);
 		if (defined $descr) {

@@ -3680,9 +3744,14 @@ sub die_error {
 		500 => '500 Internal Server Error',
 		503 => '503 Service Unavailable',
 	);
+	# The output handlers for die_error need to be reset to STDOUT
+	# so that half the message isn't being output to random and
+	# half to STDOUT as expected.  This is mainly for the benefit
+	# of using git_header_html() and git_footer_html() since
+	#
 	# Reset the output so that we are actually going to STDOUT as opposed
 	# to buffering the output.
-	reset_output();
+	reset_output() if ($caching_enabled);
 
 	git_header_html($http_responses{$status}, undef, %opts);
 	print <<EOF;

@@ -5592,9 +5661,15 @@ sub git_blob_plain {
 			($sandbox ? 'attachment' : 'inline')
 			. '; filename="' . $save_as . '"');
 	local $/ = undef;
-	binmode STDOUT, ':raw';
-	print <$fd>;
-	binmode STDOUT, ':utf8'; # as set at the beginning of gitweb.cgi
+	if ($caching_enabled) {
+		open BINOUT, '>', $fullhashbinpath or die_error(500, "Could not open bin dump file");
+	}else{
+		open BINOUT, '>', \$fullhashbinpath or die_error(500, "Could not open bin dump file");
+	}
+	binmode BINOUT, ':raw';
+	print BINOUT <$fd>;
+	binmode BINOUT, ':utf8'; # as set at the beginning of gitweb.cgi
+	close BINOUT;
 	close $fd;
 }