[PATCHv3 1/2] Add a remote helper to interact with mediawiki, pull & clone handled

@@ -0,0 +1,322 @@
+#! /usr/bin/perl
+
+use strict;
+use MediaWiki::API;
+use DateTime::Format::ISO8601;
+use encoding 'utf8';
+use URI::Escape;
+use warnings;
+
+# Mediawiki filenames can contain forward slashes. This variable decides by which pattern they should be replaced
+my $slash_replacement = "%2F";
+
+my $remotename = $ARGV[0];
+# Current syntax to fetch only a set of pages mediawiki::http://mediawikiurl##A_Page##Another_Page
+my @pages_titles = split(/##/,$ARGV[1]);
+my $url = shift (@pages_titles);
+
+
+# commands parser
+my $entry;
+my @cmd;
+while (1) {
+	$| = 1; #flush STDOUT
+	$entry = <STDIN>;
+	chomp($entry);
+	@cmd = split(/ /,$entry);
+	if (defined($cmd[0])) {
+		if ($cmd[0] eq "capabilities") {
+			last unless (!defined($cmd[1]));
+			mw_capabilities();
+		} elsif ($cmd[0] eq "list") {
+			last unless (!defined($cmd[2]));
+			mw_list($cmd[1]);
+		} elsif ($cmd[0] eq "import") {
+			last unless ($cmd[1] ne "" && !defined($cmd[2]));
+			mw_import($cmd[1]);
+		} elsif ($cmd[0] eq "option") {
+			last unless ($cmd[1] ne "" && $cmd[2] ne "" && !defined($cmd[3]));
+			mw_option($cmd[1],$cmd[2]);
+		} elsif ($cmd[0] eq "push") {
+			# Check the pattern <src>:<dst>
+			my @pushargs = split(/:/,$cmd[1]);
+			last unless ($pushargs[1] ne "" && !defined($pushargs[2]));
+			mw_push($pushargs[0],$pushargs[1]);
+		} else {
+			print STDERR "Unknown capability. Aborting...\n";
+			last;
+		}
+	} else {
+		# End of input
+		last;
+	}
+
+}
+
+########################## Functions ##############################
+sub get_pages{
+	my $mediawiki = MediaWiki::API->new;
+	$mediawiki->{config}->{api_url} = "$url/api.php";
+
+	my $pages;
+	if (!@pages_titles){
+		$pages = $mediawiki->list({
+			action => 'query',
+			list => 'allpages',
+			aplimit => 500,
+		});
+		if (!defined($pages)) {
+			print STDERR "fatal: '$url' does not appear to be a mediawiki\n";
+			print STDERR "fatal: make sure '$url/api.php' is a valid page\n";
+			exit 1;
+		}
+		return @$pages;
+	} else {
+		#the list of titles should follow the pattern 'page1|page2|...'
+		my $titles = "";
+		foreach my $title (@pages_titles){
+			$titles.="$title|";
+		}
+		#supress the last | that is add in the foreach
+		chop($titles);
+
+		$pages = $mediawiki->api({
+			action => 'query',
+			titles => $titles,
+		});
+		if (!defined($pages)) {
+			print STDERR "fatal: None of the pages exist \n";
+			exit 1;
+		}
+		return values (%{$pages->{query}->{pages}});
+	}
+}
+
+sub run_git {
+	open(my $git, "-|:encoding(UTF-8)", "git " . $_[0]);
+	my $res = do { local $/; <$git> };
+	close($git);
+
+	return $res;
+}
+
+
+sub get_last_local_revision {
+	# Get note regarding last mediawiki revision
+	my $note = run_git("notes --ref=mediawiki show refs/mediawiki/$remotename/master 2>/dev/null");
+	my @note_info = split(/ /, $note);
+
+	my $lastrevision_number;
+	if (!(defined($note_info[0]) && $note_info[0] eq "mediawiki_revision:")) {
+		print STDERR "No previous mediawiki revision found";
+		$lastrevision_number = 0;
+	} else {
+		# Notes are formatted : mediawiki_revision: #number
+		$lastrevision_number = $note_info[1];
+		chomp($lastrevision_number);
+		print STDERR "Last local mediawiki revision found is $lastrevision_number ";
+	}
+	return $lastrevision_number;
+}
+
+sub get_last_remote_revision {
+	my $mediawiki = MediaWiki::API->new;
+	$mediawiki->{config}->{api_url} = "$url/api.php";
+
+	my @pages = get_pages();
+
+	my $max_rev_num = 0;
+
+	foreach my $page (@pages) {
+		my $id = $page->{pageid};
+
+		my $query = {
+			action => 'query',
+			prop => 'revisions',
+			rvprop => 'ids',
+			pageids => $id,
+		};
+
+		my $result = $mediawiki->api($query);
+		
+		my $lastrev = pop(@{$result->{query}->{pages}->{$id}->{revisions}});
+		
+		$max_rev_num = ($lastrev->{revid} > $max_rev_num ? $lastrev->{revid} : $max_rev_num);
+	}
+
+	print STDERR "Last remote revision found is $max_rev_num\n";
+	return $max_rev_num;
+}
+
+sub literal_data {
+	my ($content) = @_;
+	print STDOUT "data ", bytes::length($content), "\n", $content;
+}
+
+sub mw_capabilities {
+	# Revisions are imported to the private namespace
+	# refs/mediawiki/$remotename/ by the helper and fetched into
+	# refs/remotes/$remotename later by fetch.
+	print STDOUT "refspec refs/heads/*:refs/mediawiki/$remotename/*\n";
+	print STDOUT "import\n";
+	print STDOUT "list\n";
+	print STDOUT "option\n";
+	print STDOUT "push\n";
+	print STDOUT "\n";
+}
+
+sub mw_list {
+	# MediaWiki do not have branches, we consider one branch arbitrarily
+	# called master
+	print STDOUT "? refs/heads/master\n";
+	print STDOUT '@'."refs/heads/master HEAD\n";
+	print STDOUT "\n";
+
+}
+
+sub mw_option {
+	print STDOUT "unsupported\n";
+}
+
+sub mw_import {
+	my @wiki_name = split(/:\/\//,$url);
+	my $wiki_name = $wiki_name[1];
+
+	my $mediawiki = MediaWiki::API->new;
+	$mediawiki->{config}->{api_url} = "$url/api.php";
+
+	my @pages = get_pages();
+
+	my @revisions;
+	print STDERR "Searching revisions...\n";
+	my $fetch_from = get_last_local_revision() + 1;
+	if ($fetch_from == 1) {
+		print STDERR ", fetching from beginning\n";
+	} else {
+		print STDERR ", fetching from here\n";
+	}
+	my $n = 1;
+	foreach my $page (@pages) {
+		my $id = $page->{pageid};
+
+		print STDERR "$n/", scalar(@pages), ": ". $page->{title}."\n";
+		$n++;
+
+		my $query = {
+			action => 'query',
+			prop => 'revisions',
+			rvprop => 'ids',
+			rvdir => 'newer',
+			rvstartid => $fetch_from,
+			rvlimit => 500,
+			pageids => $id,
+		};
+
+		my $revnum = 0;
+		# Get 500 revisions at a time due to the mediawiki api limit
+		while (1) {
+			my $result = $mediawiki->api($query);
+
+			# Parse each of those 500 revisions
+			foreach my $revision (@{$result->{query}->{pages}->{$id}->{revisions}}) {
+				my $page_rev_ids;
+				$page_rev_ids->{pageid} = $page->{pageid};
+				$page_rev_ids->{revid} = $revision->{revid};
+				push (@revisions, $page_rev_ids);
+				$revnum++;
+			}
+			last unless $result->{'query-continue'};
+			$query->{rvstartid} = $result->{'query-continue'}->{revisions}->{rvstartid};
+		}
+		print STDERR "  Found ", $revnum, " revision(s).\n";
+	}
+
+	# Creation of the fast-import stream
+	print STDERR "Fetching & writing export data...\n";
+	
+	$n = 0;
+	my $last_timestamp = 0; #Placeholer in case $rev->timestamp is undefined
+
+	foreach my $pagerevids (sort {$a->{revid} <=> $b->{revid}} @revisions) {
+		#fetch the content of the pages
+		my $query = {
+			action => 'query',
+			prop => 'revisions',
+			rvprop => 'content|timestamp|comment|user|ids',
+			revids => $pagerevids->{revid},
+		};
+
+		my $result = $mediawiki->api($query);
+
+		my $rev = pop(@{$result->{query}->{pages}->{$pagerevids->{pageid}}->{revisions}});
+
+		$n++;
+		my $user = $rev->{user} || 'Anonymous';
+
+		if (!defined($rev->{timestamp})) {
+			$last_timestamp++;
+		} else {
+			$last_timestamp = $rev->{timestamp};
+		}
+		my $dt = DateTime::Format::ISO8601->parse_datetime($last_timestamp);
+
+		my $comment = defined $rev->{comment} ? $rev->{comment} : '*Empty MediaWiki Message*';
+		
+		my $title = $result->{query}->{pages}->{$pagerevids->{pageid}}->{title};
+		$title =~ y/ /_/;
+		#$title = uri_escape($title); #It would be nice to use uri_escape to be cross compatible
+		#on different file systems handling accentuated characters differently
+		$title =~ s/\//$slash_replacement/g;
+		
+		my $content = $rev->{'*'};
+		# This \n is important. This is due to mediawiki's way to handle end of files.
+		$content .= "\n";
+
+		print STDERR "$n/", scalar(@revisions), ": Revision #$pagerevids->{revid} of $title\n";
+
+		print STDOUT "commit refs/mediawiki/$remotename/master\n";
+		print STDOUT "mark :$n\n";
+		print STDOUT "committer $user <$user\@$wiki_name> ", $dt->epoch, " +0000\n";
+		literal_data($comment);
+		# If it's not a clone, needs to know where to start from
+		if ($fetch_from != 1 && $n == 1) {
+			print STDOUT "from refs/mediawiki/$remotename/master^0\n";
+		}
+		print STDOUT "M 644 inline $title.mw\n";
+		literal_data($content);
+		print STDOUT "\n\n";
+
+
+
+
+		# mediawiki revision number in the git note
+		if ($fetch_from == 1 && $n == 1) {
+			print STDOUT "reset refs/notes/mediawiki\n";
+		}
+		print STDOUT "commit refs/notes/mediawiki\n";
+		print STDOUT "committer $user <$user\@$wiki_name> ", $dt->epoch, " +0000\n";
+		literal_data("note added by git-mediawiki");
+		if ($fetch_from != 1 && $n == 1) {
+			print STDOUT "from refs/notes/mediawiki^0\n";
+		}
+		print STDOUT "N inline :$n\n";
+		literal_data("mediawiki_revision: " . $pagerevids->{revid});
+		print STDOUT "\n\n";
+	}
+
+	if ($fetch_from == 1) {
+		if ($n != 0) {
+			print STDOUT "reset $_[0]\n"; #$_[0] contains refs/heads/master
+			print STDOUT "from :$n\n";
+		} else {
+			print STDERR "You appear to have cloned an empty mediawiki\n";
+			#Something has to be done remote-helper side. If nothing is done, an error is 
+			#thrown saying that HEAD is refering to unknown object 0000000000000000000
+		}
+	}
+
+}
+
+sub mw_push {
+	print STDERR "Push not yet implemented\n";
+}

@@ -0,0 +1,7 @@
+Git-Mediawiki is a project which aims the creation of a gate
+between git and mediawiki, allowing git users to push and pull
+objects from mediawiki just as one would do with a classic git
+repository thanks to remote-helpers.
+
+For more information, visit the wiki at
+https://github.com/Bibzball/Git-Mediawiki/wiki

@@ -148,6 +148,14 @@ sub get_last_remote_revision {
 	return $max_rev_num;
 }
 
+sub mediawiki_filter($) {
+	# Mediawiki does not allow blank space at the end of a page and ends with a single \n.
+	# This function right trims a string and adds a \n at the end to follow this rule
+	my $string = shift;
+	$string =~ s/\s+$//;
+	return $string."\n";
+}
+
 sub literal_data {
 	my ($content) = @_;
 	print STDOUT "data ", bytes::length($content), "\n", $content;

@@ -175,6 +183,7 @@ sub mw_list {
 }
 
 sub mw_option {
+	print STDERR "remote-helper capability 'option' not yet implemented \n";
 	print STDOUT "unsupported\n";
 }
 

@@ -318,5 +327,87 @@ sub mw_import {
 }
 
 sub mw_push {
-	print STDERR "Push not yet implemented\n";
+
+	sub push_file {
+		#$_[0] contains a string in this format :
+		#100644 100644 <sha1_of_blob_before_commit> <sha1_of_blob_now> <status>\0<filename.mw>\0
+		#$_[1] contains the title of the commit message (the only phrase kept in the revision message)
+		my @blob_info_split = split(/ |\t|\0/, $_[0]);
+
+		my $sha1 = $blob_info_split[3];
+		my $complete_file_name = $blob_info_split[5];
+		# complete_file_name = uri_unescape($complete_file_name); # If we use the uri escape before
+		# we should unescape here, before anything
+		
+		if (substr($complete_file_name,-3) eq ".mw"){
+			my $title = substr($complete_file_name,0,-3);
+			$title =~ s/$slash_replacement/\//g;
+
+			my $file_content = run_git("cat-file -p $sha1");
+			
+			my $mw = MediaWiki::API->new();
+			$mw->{config}->{api_url} = "$url/api.php";
+
+			# log in to the wiki : here should be added a way to push changes with an identity
+			#$mw->login( { lgname => 'login', lgpassword => 'passwd' })
+			#|| die $mw->{error}->{code} . ': ' . $mw->{error}->{details};
+			
+			$mw->edit( {
+				action => 'edit',
+				summary => $_[1],
+				title => $title,
+				text => mediawiki_filter($file_content),
+			}, {
+				skip_encoding => 1 # Helps with names with accentuated characters
+			}) || die 'Fatal: Error ' . $mw->{error}->{code} . ' from mediwiki: ' . $mw->{error}->{details};
+
+			print STDERR "Pushed file : $sha1 - $title\n";
+		} else {
+			print STDERR "$complete_file_name not a mediawiki file. '(Not pushable on this version)\n"
+		}
+	}
+	
+	my $last_local_revid = get_last_local_revision();
+	my $last_remote_revid = get_last_remote_revision();
+
+	# Get sha1 of commit pointed by local HEAD
+	my $HEAD_sha1 = run_git("rev-parse $_[0] 2>/dev/null"); chomp($HEAD_sha1);
+	# Get sha1 of commit pointed by remotes/origin/master
+	my $remoteorigin_sha1 = run_git("rev-parse refs/remotes/origin/master 2>/dev/null"); chomp($remoteorigin_sha1);
+
+	if ($last_local_revid < $last_remote_revid){
+		my $message = "\"To prevent you from losing history, non-fast-forward updates were rejected \\n";
+		$message .= "Merge the remote changes (e.g. 'git pull') before pushing again. See the";
+		$message .= " 'Note about fast-forwards' section of 'git push --help' for details.\"";
+		print STDOUT "error $_[0] $message\n";
+		print STDOUT "\n";
+	} elsif ($HEAD_sha1 ne $remoteorigin_sha1) {
+		# Get every commit in between HEAD and refs/remotes/origin/master,
+		# including HEAD and refs/remotes/origin/master
+		my $parsed_sha1 = $remoteorigin_sha1;
+		while ($parsed_sha1 ne $HEAD_sha1) {
+			my @commit_info =  grep(/^$parsed_sha1/, `git rev-list --children $_[0]`);
+			my @commit_info_split = split(/ |\n/, $commit_info[0]);
+			# $commit_info_split[0] is the sha1 of the commit itself
+			# $commit_info_split[1] is the sha1 of its direct child
+			my $blob_infos = run_git("diff --raw --abbrev=40 -z $commit_info_split[0] $commit_info_split[1]");
+			my @blob_info_list = split(/\n/, $blob_infos);
+			# Keep the first line of the commit message as mediawiki comment for the revision
+			my $commit_msg = (split(/\n/, run_git("show --pretty=format:\"%s\" $commit_info_split[1]")))[0];
+			chomp($commit_msg);
+			foreach my $blob_info (@blob_info_list) {
+				# Push every blob
+				push_file($blob_info, $commit_msg);
+			}
+			$parsed_sha1 = $commit_info_split[1];
+		}
+
+		print STDOUT "ok $_[1]\n";
+		print STDOUT "\n";
+		
+		# Pulling from mediawiki after pushing in order to keep things synchronized
+		exec("git pull --rebase >/dev/null");
+	} else {
+		print STDOUT "\n";
+	}
 }

@@ -11,10 +11,9 @@ use warnings;
 my $slash_replacement = "%2F";
 
 my $remotename = $ARGV[0];
-# Current syntax to fetch only a set of pages mediawiki::http://mediawikiurl##A_Page##Another_Page
-my @pages_titles = split(/##/,$ARGV[1]);
-my $url = shift (@pages_titles);
-
+my $url = $ARGV[1];
+my @pages_titles = `git config --get-all mediawiki.page`;
+chomp @pages_titles;
 
 # commands parser
 my $entry;