git-remote-mediawiki: remove import of unused open2
[git/git.git] / contrib / mw-to-git / git-remote-mediawiki.perl
CommitLineData
428c995c
JN
1#! /usr/bin/perl
2
3# Copyright (C) 2011
4# Jérémie Nikaes <jeremie.nikaes@ensimag.imag.fr>
5# Arnaud Lacurie <arnaud.lacurie@ensimag.imag.fr>
6# Claire Fousse <claire.fousse@ensimag.imag.fr>
7# David Amouyal <david.amouyal@ensimag.imag.fr>
8# Matthieu Moy <matthieu.moy@grenoble-inp.fr>
9# License: GPL v2 or later
10
11# Gateway between Git and MediaWiki.
9a7365cf 12# Documentation & bugtracker: https://github.com/moy/Git-Mediawiki/
428c995c
JN
13
14use strict;
15use MediaWiki::API;
da608b12 16use Git;
428c995c 17use DateTime::Format::ISO8601;
668eec6f 18use warnings;
428c995c 19
721a533f 20# By default, use UTF-8 to communicate with Git and the user
6a504a3f
CM
21binmode STDERR, ":encoding(UTF-8)";
22binmode STDOUT, ":encoding(UTF-8)";
428c995c
JN
23
24use URI::Escape;
a7271ad1 25
428c995c
JN
26# Mediawiki filenames can contain forward slashes. This variable decides by which pattern they should be replaced
27use constant SLASH_REPLACEMENT => "%2F";
28
29# It's not always possible to delete pages (may require some
2582ab18 30# privileges). Deleted pages are replaced with this content.
428c995c
JN
31use constant DELETED_CONTENT => "[[Category:Deleted]]\n";
32
33# It's not possible to create empty pages. New empty files in Git are
34# sent with this content instead.
35use constant EMPTY_CONTENT => "<!-- empty page -->\n";
36
37# used to reflect file creation or deletion in diff.
38use constant NULL_SHA1 => "0000000000000000000000000000000000000000";
39
dcb1ea62
MM
40# Used on Git's side to reflect empty edit messages on the wiki
41use constant EMPTY_MESSAGE => '*Empty MediaWiki Message*';
42
428c995c
JN
43my $remotename = $ARGV[0];
44my $url = $ARGV[1];
45
46# Accept both space-separated and multiple keys in config file.
47# Spaces should be written as _ anyway because we'll use chomp.
48my @tracked_pages = split(/[ \n]/, run_git("config --get-all remote.". $remotename .".pages"));
49chomp(@tracked_pages);
50
51# Just like @tracked_pages, but for MediaWiki categories.
52my @tracked_categories = split(/[ \n]/, run_git("config --get-all remote.". $remotename .".categories"));
53chomp(@tracked_categories);
54
2045e293 55# Import media files on pull
6a9e55b0
PV
56my $import_media = run_git("config --get --bool remote.". $remotename .".mediaimport");
57chomp($import_media);
58$import_media = ($import_media eq "true");
59
2045e293
MM
60# Export media files on push
61my $export_media = run_git("config --get --bool remote.". $remotename .".mediaexport");
62chomp($export_media);
63$export_media = !($export_media eq "false");
64
428c995c 65my $wiki_login = run_git("config --get remote.". $remotename .".mwLogin");
2da7830d 66# Note: mwPassword is discourraged. Use the credential system instead.
428c995c 67my $wiki_passwd = run_git("config --get remote.". $remotename .".mwPassword");
1d6abac9 68my $wiki_domain = run_git("config --get remote.". $remotename .".mwDomain");
428c995c
JN
69chomp($wiki_login);
70chomp($wiki_passwd);
1d6abac9 71chomp($wiki_domain);
428c995c
JN
72
73# Import only last revisions (both for clone and fetch)
74my $shallow_import = run_git("config --get --bool remote.". $remotename .".shallow");
75chomp($shallow_import);
76$shallow_import = ($shallow_import eq "true");
77
5a29217d
MM
78# Fetch (clone and pull) by revisions instead of by pages. This behavior
79# is more efficient when we have a wiki with lots of pages and we fetch
80# the revisions quite often so that they concern only few pages.
81# Possible values:
82# - by_rev: perform one query per new revision on the remote wiki
83# - by_page: query each tracked page for new revision
84my $fetch_strategy = run_git("config --get remote.$remotename.fetchStrategy");
85unless ($fetch_strategy) {
86 $fetch_strategy = run_git("config --get mediawiki.fetchStrategy");
87}
88chomp($fetch_strategy);
89unless ($fetch_strategy) {
90 $fetch_strategy = "by_page";
91}
92
0afd29e2
CM
93# Remember the timestamp corresponding to a revision id.
94my %basetimestamps;
95
93f0d338
MM
96# Dumb push: don't update notes and mediawiki ref to reflect the last push.
97#
98# Configurable with mediawiki.dumbPush, or per-remote with
99# remote.<remotename>.dumbPush.
100#
101# This means the user will have to re-import the just-pushed
102# revisions. On the other hand, this means that the Git revisions
103# corresponding to MediaWiki revisions are all imported from the wiki,
104# regardless of whether they were initially created in Git or from the
105# web interface, hence all users will get the same history (i.e. if
106# the push from Git to MediaWiki loses some information, everybody
107# will get the history with information lost). If the import is
108# deterministic, this means everybody gets the same sha1 for each
109# MediaWiki revision.
110my $dumb_push = run_git("config --get --bool remote.$remotename.dumbPush");
111unless ($dumb_push) {
112 $dumb_push = run_git("config --get --bool mediawiki.dumbPush");
113}
114chomp($dumb_push);
115$dumb_push = ($dumb_push eq "true");
116
428c995c 117my $wiki_name = $url;
857f21a3 118$wiki_name =~ s{[^/]*://}{};
9fb79503
MM
119# If URL is like http://user:password@example.com/, we clearly don't
120# want the password in $wiki_name. While we're there, also remove user
121# and '@' sign, to avoid author like MWUser@HTTPUser@host.com
122$wiki_name =~ s/^.*@//;
428c995c
JN
123
124# Commands parser
428c995c
JN
125my @cmd;
126while (<STDIN>) {
127 chomp;
128 @cmd = split(/ /);
129 if (defined($cmd[0])) {
130 # Line not blank
131 if ($cmd[0] eq "capabilities") {
267055f8 132 die("Too many arguments for capabilities\n") if (defined($cmd[1]));
428c995c
JN
133 mw_capabilities();
134 } elsif ($cmd[0] eq "list") {
267055f8 135 die("Too many arguments for list\n") if (defined($cmd[2]));
428c995c
JN
136 mw_list($cmd[1]);
137 } elsif ($cmd[0] eq "import") {
267055f8 138 die("Invalid arguments for import\n") if ($cmd[1] eq "" || defined($cmd[2]));
428c995c
JN
139 mw_import($cmd[1]);
140 } elsif ($cmd[0] eq "option") {
267055f8 141 die("Too many arguments for option\n") if ($cmd[1] eq "" || $cmd[2] eq "" || defined($cmd[3]));
428c995c
JN
142 mw_option($cmd[1],$cmd[2]);
143 } elsif ($cmd[0] eq "push") {
144 mw_push($cmd[1]);
145 } else {
146 print STDERR "Unknown command. Aborting...\n";
147 last;
148 }
149 } else {
150 # blank line: we should terminate
151 last;
152 }
153
154 BEGIN { $| = 1 } # flush STDOUT, to make sure the previous
155 # command is fully processed.
156}
157
158########################## Functions ##############################
159
160# MediaWiki API instance, created lazily.
161my $mediawiki;
162
163sub mw_connect_maybe {
164 if ($mediawiki) {
a7271ad1 165 return;
428c995c
JN
166 }
167 $mediawiki = MediaWiki::API->new;
168 $mediawiki->{config}->{api_url} = "$url/api.php";
169 if ($wiki_login) {
da608b12
BP
170 my %credential = (
171 'url' => $url,
172 'username' => $wiki_login,
173 'password' => $wiki_passwd
174 );
175 Git::credential(\%credential);
a7271ad1
MM
176 my $request = {lgname => $credential{username},
177 lgpassword => $credential{password},
178 lgdomain => $wiki_domain};
179 if ($mediawiki->login($request)) {
da608b12 180 Git::credential(\%credential, 'approve');
a7271ad1 181 print STDERR "Logged in mediawiki user \"$credential{username}\".\n";
428c995c 182 } else {
a7271ad1
MM
183 print STDERR "Failed to log in mediawiki user \"$credential{username}\" on $url\n";
184 print STDERR " (error " .
185 $mediawiki->{error}->{code} . ': ' .
186 $mediawiki->{error}->{details} . ")\n";
da608b12 187 Git::credential(\%credential, 'reject');
a7271ad1 188 exit 1;
428c995c
JN
189 }
190 }
1aff8c62 191 return;
428c995c
JN
192}
193
0aa03212
MM
194sub fatal_mw_error {
195 my $action = shift;
196 print STDERR "fatal: could not $action.\n";
197 print STDERR "fatal: '$url' does not appear to be a mediawiki\n";
198 if ($url =~ /^https/) {
199 print STDERR "fatal: make sure '$url/api.php' is a valid page\n";
200 print STDERR "fatal: and the SSL certificate is correct.\n";
201 } else {
202 print STDERR "fatal: make sure '$url/api.php' is a valid page.\n";
203 }
204 print STDERR "fatal: (error " .
205 $mediawiki->{error}->{code} . ': ' .
206 $mediawiki->{error}->{details} . ")\n";
207 exit 1;
208}
209
9cb74f36
MM
210## Functions for listing pages on the remote wiki
211sub get_mw_tracked_pages {
212 my $pages = shift;
6df7e0df 213 get_mw_page_list(\@tracked_pages, $pages);
1aff8c62 214 return;
6df7e0df
MM
215}
216
217sub get_mw_page_list {
218 my $page_list = shift;
219 my $pages = shift;
220 my @some_pages = @$page_list;
9cb74f36 221 while (@some_pages) {
b835baf6
CM
222 my $last_page = 50;
223 if ($#some_pages < $last_page) {
224 $last_page = $#some_pages;
9cb74f36 225 }
b835baf6 226 my @slice = @some_pages[0..$last_page];
9cb74f36
MM
227 get_mw_first_pages(\@slice, $pages);
228 @some_pages = @some_pages[51..$#some_pages];
229 }
1aff8c62 230 return;
9cb74f36
MM
231}
232
233sub get_mw_tracked_categories {
234 my $pages = shift;
235 foreach my $category (@tracked_categories) {
236 if (index($category, ':') < 0) {
237 # Mediawiki requires the Category
238 # prefix, but let's not force the user
239 # to specify it.
240 $category = "Category:" . $category;
241 }
242 my $mw_pages = $mediawiki->list( {
243 action => 'query',
244 list => 'categorymembers',
245 cmtitle => $category,
246 cmlimit => 'max' } )
247 || die $mediawiki->{error}->{code} . ': '
8a43b36a 248 . $mediawiki->{error}->{details} . "\n";
9cb74f36
MM
249 foreach my $page (@{$mw_pages}) {
250 $pages->{$page->{title}} = $page;
251 }
252 }
1aff8c62 253 return;
9cb74f36
MM
254}
255
256sub get_mw_all_pages {
257 my $pages = shift;
258 # No user-provided list, get the list of pages from the API.
259 my $mw_pages = $mediawiki->list({
260 action => 'query',
261 list => 'allpages',
262 aplimit => 'max'
263 });
264 if (!defined($mw_pages)) {
0aa03212 265 fatal_mw_error("get the list of wiki pages");
9cb74f36
MM
266 }
267 foreach my $page (@{$mw_pages}) {
268 $pages->{$page->{title}} = $page;
269 }
1aff8c62 270 return;
9cb74f36
MM
271}
272
273# queries the wiki for a set of pages. Meant to be used within a loop
274# querying the wiki for slices of page list.
428c995c
JN
275sub get_mw_first_pages {
276 my $some_pages = shift;
277 my @some_pages = @{$some_pages};
278
279 my $pages = shift;
280
281 # pattern 'page1|page2|...' required by the API
282 my $titles = join('|', @some_pages);
283
284 my $mw_pages = $mediawiki->api({
285 action => 'query',
286 titles => $titles,
287 });
288 if (!defined($mw_pages)) {
0aa03212 289 fatal_mw_error("query the list of wiki pages");
428c995c
JN
290 }
291 while (my ($id, $page) = each(%{$mw_pages->{query}->{pages}})) {
292 if ($id < 0) {
293 print STDERR "Warning: page $page->{title} not found on wiki\n";
294 } else {
295 $pages->{$page->{title}} = $page;
296 }
297 }
1aff8c62 298 return;
428c995c
JN
299}
300
9cb74f36 301# Get the list of pages to be fetched according to configuration.
428c995c
JN
302sub get_mw_pages {
303 mw_connect_maybe();
304
f690ddfd
MM
305 print STDERR "Listing pages on remote wiki...\n";
306
428c995c
JN
307 my %pages; # hash on page titles to avoid duplicates
308 my $user_defined;
309 if (@tracked_pages) {
310 $user_defined = 1;
311 # The user provided a list of pages titles, but we
312 # still need to query the API to get the page IDs.
9cb74f36 313 get_mw_tracked_pages(\%pages);
428c995c
JN
314 }
315 if (@tracked_categories) {
316 $user_defined = 1;
9cb74f36 317 get_mw_tracked_categories(\%pages);
428c995c
JN
318 }
319 if (!$user_defined) {
9cb74f36 320 get_mw_all_pages(\%pages);
428c995c 321 }
6a9e55b0
PV
322 if ($import_media) {
323 print STDERR "Getting media files for selected pages...\n";
324 if ($user_defined) {
325 get_linked_mediafiles(\%pages);
326 } else {
327 get_all_mediafiles(\%pages);
428c995c
JN
328 }
329 }
f690ddfd 330 print STDERR (scalar keys %pages) . " pages found.\n";
b1ede9a9 331 return %pages;
428c995c
JN
332}
333
b3d98595
NKT
334# usage: $out = run_git("command args");
335# $out = run_git("command args", "raw"); # don't interpret output as UTF-8.
428c995c 336sub run_git {
b3d98595
NKT
337 my $args = shift;
338 my $encoding = (shift || "encoding(UTF-8)");
ee25ff2c
CM
339 open(my $git, "-|:$encoding", "git " . $args)
340 or die "Unable to open: $!\n";
4f1b7883
CM
341 my $res = do {
342 local $/ = undef;
343 <$git>
344 };
428c995c
JN
345 close($git);
346
347 return $res;
348}
349
350
6a9e55b0
PV
351sub get_all_mediafiles {
352 my $pages = shift;
353 # Attach list of all pages for media files from the API,
354 # they are in a different namespace, only one namespace
355 # can be queried at the same moment
356 my $mw_pages = $mediawiki->list({
357 action => 'query',
358 list => 'allpages',
359 apnamespace => get_mw_namespace_id("File"),
360 aplimit => 'max'
361 });
362 if (!defined($mw_pages)) {
363 print STDERR "fatal: could not get the list of pages for media files.\n";
364 print STDERR "fatal: '$url' does not appear to be a mediawiki\n";
365 print STDERR "fatal: make sure '$url/api.php' is a valid page.\n";
366 exit 1;
367 }
368 foreach my $page (@{$mw_pages}) {
369 $pages->{$page->{title}} = $page;
370 }
1aff8c62 371 return;
6a9e55b0
PV
372}
373
374sub get_linked_mediafiles {
375 my $pages = shift;
81f6a7a4 376 my @titles = map { $_->{title} } values(%{$pages});
6a9e55b0
PV
377
378 # The query is split in small batches because of the MW API limit of
379 # the number of links to be returned (500 links max).
380 my $batch = 10;
381 while (@titles) {
382 if ($#titles < $batch) {
383 $batch = $#titles;
384 }
385 my @slice = @titles[0..$batch];
386
387 # pattern 'page1|page2|...' required by the API
388 my $mw_titles = join('|', @slice);
389
390 # Media files could be included or linked from
391 # a page, get all related
392 my $query = {
393 action => 'query',
394 prop => 'links|images',
395 titles => $mw_titles,
396 plnamespace => get_mw_namespace_id("File"),
397 pllimit => 'max'
398 };
399 my $result = $mediawiki->api($query);
400
401 while (my ($id, $page) = each(%{$result->{query}->{pages}})) {
6df7e0df 402 my @media_titles;
6a9e55b0 403 if (defined($page->{links})) {
81f6a7a4
CM
404 my @link_titles
405 = map { $_->{title} } @{$page->{links}};
6df7e0df 406 push(@media_titles, @link_titles);
6a9e55b0
PV
407 }
408 if (defined($page->{images})) {
81f6a7a4
CM
409 my @image_titles
410 = map { $_->{title} } @{$page->{images}};
6df7e0df 411 push(@media_titles, @image_titles);
6a9e55b0 412 }
6df7e0df
MM
413 if (@media_titles) {
414 get_mw_page_list(\@media_titles, $pages);
6a9e55b0
PV
415 }
416 }
417
418 @titles = @titles[($batch+1)..$#titles];
419 }
1aff8c62 420 return;
6a9e55b0
PV
421}
422
423sub get_mw_mediafile_for_page_revision {
424 # Name of the file on Wiki, with the prefix.
6df7e0df 425 my $filename = shift;
6a9e55b0
PV
426 my $timestamp = shift;
427 my %mediafile;
428
6df7e0df
MM
429 # Search if on a media file with given timestamp exists on
430 # MediaWiki. In that case download the file.
6a9e55b0
PV
431 my $query = {
432 action => 'query',
433 prop => 'imageinfo',
6df7e0df 434 titles => "File:" . $filename,
6a9e55b0
PV
435 iistart => $timestamp,
436 iiend => $timestamp,
437 iiprop => 'timestamp|archivename|url',
438 iilimit => 1
439 };
440 my $result = $mediawiki->api($query);
441
6df7e0df 442 my ($fileid, $file) = each( %{$result->{query}->{pages}} );
6a9e55b0
PV
443 # If not defined it means there is no revision of the file for
444 # given timestamp.
445 if (defined($file->{imageinfo})) {
6a9e55b0
PV
446 $mediafile{title} = $filename;
447
448 my $fileinfo = pop(@{$file->{imageinfo}});
449 $mediafile{timestamp} = $fileinfo->{timestamp};
6df7e0df
MM
450 # Mediawiki::API's download function doesn't support https URLs
451 # and can't download old versions of files.
452 print STDERR "\tDownloading file $mediafile{title}, version $mediafile{timestamp}\n";
453 $mediafile{content} = download_mw_mediafile($fileinfo->{url});
6a9e55b0
PV
454 }
455 return %mediafile;
456}
457
6df7e0df 458sub download_mw_mediafile {
8f04f7dd 459 my $download_url = shift;
6a9e55b0 460
8f04f7dd 461 my $response = $mediawiki->{ua}->get($download_url);
6df7e0df
MM
462 if ($response->code == 200) {
463 return $response->decoded_content;
6a9e55b0 464 } else {
6df7e0df 465 print STDERR "Error downloading mediafile from :\n";
8f04f7dd 466 print STDERR "URL: $download_url\n";
6df7e0df 467 print STDERR "Server response: " . $response->code . " " . $response->message . "\n";
6a9e55b0 468 exit 1;
6a9e55b0
PV
469 }
470}
471
428c995c
JN
472sub get_last_local_revision {
473 # Get note regarding last mediawiki revision
474 my $note = run_git("notes --ref=$remotename/mediawiki show refs/mediawiki/$remotename/master 2>/dev/null");
475 my @note_info = split(/ /, $note);
476
477 my $lastrevision_number;
478 if (!(defined($note_info[0]) && $note_info[0] eq "mediawiki_revision:")) {
479 print STDERR "No previous mediawiki revision found";
480 $lastrevision_number = 0;
481 } else {
482 # Notes are formatted : mediawiki_revision: #number
483 $lastrevision_number = $note_info[1];
484 chomp($lastrevision_number);
485 print STDERR "Last local mediawiki revision found is $lastrevision_number";
486 }
487 return $lastrevision_number;
488}
489
5a29217d
MM
490# Get the last remote revision without taking in account which pages are
491# tracked or not. This function makes a single request to the wiki thus
492# avoid a loop onto all tracked pages. This is useful for the fetch-by-rev
493# option.
494sub get_last_global_remote_rev {
495 mw_connect_maybe();
496
497 my $query = {
498 action => 'query',
499 list => 'recentchanges',
500 prop => 'revisions',
501 rclimit => '1',
502 rcdir => 'older',
503 };
504 my $result = $mediawiki->api($query);
505 return $result->{query}->{recentchanges}[0]->{revid};
506}
507
508# Get the last remote revision concerning the tracked pages and the tracked
509# categories.
428c995c
JN
510sub get_last_remote_revision {
511 mw_connect_maybe();
512
b1ede9a9
MM
513 my %pages_hash = get_mw_pages();
514 my @pages = values(%pages_hash);
428c995c
JN
515
516 my $max_rev_num = 0;
517
a393f488
MM
518 print STDERR "Getting last revision id on tracked pages...\n";
519
428c995c
JN
520 foreach my $page (@pages) {
521 my $id = $page->{pageid};
522
523 my $query = {
524 action => 'query',
525 prop => 'revisions',
3c1ed90e 526 rvprop => 'ids|timestamp',
428c995c
JN
527 pageids => $id,
528 };
529
530 my $result = $mediawiki->api($query);
531
532 my $lastrev = pop(@{$result->{query}->{pages}->{$id}->{revisions}});
533
3c1ed90e
MM
534 $basetimestamps{$lastrev->{revid}} = $lastrev->{timestamp};
535
428c995c
JN
536 $max_rev_num = ($lastrev->{revid} > $max_rev_num ? $lastrev->{revid} : $max_rev_num);
537 }
538
539 print STDERR "Last remote revision found is $max_rev_num.\n";
540 return $max_rev_num;
541}
542
543# Clean content before sending it to MediaWiki
544sub mediawiki_clean {
545 my $string = shift;
546 my $page_created = shift;
547 # Mediawiki does not allow blank space at the end of a page and ends with a single \n.
548 # This function right trims a string and adds a \n at the end to follow this rule
549 $string =~ s/\s+$//;
550 if ($string eq "" && $page_created) {
551 # Creating empty pages is forbidden.
552 $string = EMPTY_CONTENT;
553 }
554 return $string."\n";
555}
556
557# Filter applied on MediaWiki data before adding them to Git
558sub mediawiki_smudge {
559 my $string = shift;
560 if ($string eq EMPTY_CONTENT) {
561 $string = "";
562 }
563 # This \n is important. This is due to mediawiki's way to handle end of files.
564 return $string."\n";
565}
566
567sub mediawiki_clean_filename {
568 my $filename = shift;
857f21a3 569 $filename =~ s{@{[SLASH_REPLACEMENT]}}{/}g;
428c995c
JN
570 # [, ], |, {, and } are forbidden by MediaWiki, even URL-encoded.
571 # Do a variant of URL-encoding, i.e. looks like URL-encoding,
572 # but with _ added to prevent MediaWiki from thinking this is
573 # an actual special character.
574 $filename =~ s/[\[\]\{\}\|]/sprintf("_%%_%x", ord($&))/ge;
575 # If we use the uri escape before
576 # we should unescape here, before anything
577
578 return $filename;
579}
580
581sub mediawiki_smudge_filename {
582 my $filename = shift;
857f21a3 583 $filename =~ s{/}{@{[SLASH_REPLACEMENT]}}g;
428c995c
JN
584 $filename =~ s/ /_/g;
585 # Decode forbidden characters encoded in mediawiki_clean_filename
586 $filename =~ s/_%_([0-9a-fA-F][0-9a-fA-F])/sprintf("%c", hex($1))/ge;
587 return $filename;
588}
589
590sub literal_data {
591 my ($content) = @_;
592 print STDOUT "data ", bytes::length($content), "\n", $content;
1aff8c62 593 return;
428c995c
JN
594}
595
6a9e55b0
PV
596sub literal_data_raw {
597 # Output possibly binary content.
598 my ($content) = @_;
599 # Avoid confusion between size in bytes and in characters
600 utf8::downgrade($content);
601 binmode STDOUT, ":raw";
602 print STDOUT "data ", bytes::length($content), "\n", $content;
6a504a3f 603 binmode STDOUT, ":encoding(UTF-8)";
1aff8c62 604 return;
6a9e55b0
PV
605}
606
428c995c
JN
607sub mw_capabilities {
608 # Revisions are imported to the private namespace
609 # refs/mediawiki/$remotename/ by the helper and fetched into
610 # refs/remotes/$remotename later by fetch.
611 print STDOUT "refspec refs/heads/*:refs/mediawiki/$remotename/*\n";
612 print STDOUT "import\n";
613 print STDOUT "list\n";
614 print STDOUT "push\n";
615 print STDOUT "\n";
1aff8c62 616 return;
428c995c
JN
617}
618
619sub mw_list {
620 # MediaWiki do not have branches, we consider one branch arbitrarily
621 # called master, and HEAD pointing to it.
622 print STDOUT "? refs/heads/master\n";
623 print STDOUT "\@refs/heads/master HEAD\n";
624 print STDOUT "\n";
1aff8c62 625 return;
428c995c
JN
626}
627
628sub mw_option {
629 print STDERR "remote-helper command 'option $_[0]' not yet implemented\n";
630 print STDOUT "unsupported\n";
1aff8c62 631 return;
428c995c
JN
632}
633
634sub fetch_mw_revisions_for_page {
635 my $page = shift;
636 my $id = shift;
637 my $fetch_from = shift;
638 my @page_revs = ();
639 my $query = {
640 action => 'query',
641 prop => 'revisions',
642 rvprop => 'ids',
643 rvdir => 'newer',
644 rvstartid => $fetch_from,
645 rvlimit => 500,
646 pageids => $id,
647 };
648
649 my $revnum = 0;
650 # Get 500 revisions at a time due to the mediawiki api limit
651 while (1) {
652 my $result = $mediawiki->api($query);
653
654 # Parse each of those 500 revisions
655 foreach my $revision (@{$result->{query}->{pages}->{$id}->{revisions}}) {
656 my $page_rev_ids;
657 $page_rev_ids->{pageid} = $page->{pageid};
658 $page_rev_ids->{revid} = $revision->{revid};
659 push(@page_revs, $page_rev_ids);
660 $revnum++;
661 }
662 last unless $result->{'query-continue'};
663 $query->{rvstartid} = $result->{'query-continue'}->{revisions}->{rvstartid};
664 }
665 if ($shallow_import && @page_revs) {
666 print STDERR " Found 1 revision (shallow import).\n";
667 @page_revs = sort {$b->{revid} <=> $a->{revid}} (@page_revs);
668 return $page_revs[0];
669 }
670 print STDERR " Found ", $revnum, " revision(s).\n";
671 return @page_revs;
672}
673
674sub fetch_mw_revisions {
675 my $pages = shift; my @pages = @{$pages};
676 my $fetch_from = shift;
677
678 my @revisions = ();
679 my $n = 1;
680 foreach my $page (@pages) {
681 my $id = $page->{pageid};
682
683 print STDERR "page $n/", scalar(@pages), ": ". $page->{title} ."\n";
684 $n++;
685 my @page_revs = fetch_mw_revisions_for_page($page, $id, $fetch_from);
686 @revisions = (@page_revs, @revisions);
687 }
688
689 return ($n, @revisions);
690}
691
462d97da
MM
692sub fe_escape_path {
693 my $path = shift;
694 $path =~ s/\\/\\\\/g;
695 $path =~ s/"/\\"/g;
696 $path =~ s/\n/\\n/g;
697 return '"' . $path . '"';
698}
699
428c995c
JN
700sub import_file_revision {
701 my $commit = shift;
702 my %commit = %{$commit};
703 my $full_import = shift;
704 my $n = shift;
6a9e55b0
PV
705 my $mediafile = shift;
706 my %mediafile;
707 if ($mediafile) {
708 %mediafile = %{$mediafile};
709 }
428c995c
JN
710
711 my $title = $commit{title};
712 my $comment = $commit{comment};
713 my $content = $commit{content};
714 my $author = $commit{author};
715 my $date = $commit{date};
716
717 print STDOUT "commit refs/mediawiki/$remotename/master\n";
718 print STDOUT "mark :$n\n";
719 print STDOUT "committer $author <$author\@$wiki_name> ", $date->epoch, " +0000\n";
720 literal_data($comment);
721
722 # If it's not a clone, we need to know where to start from
723 if (!$full_import && $n == 1) {
724 print STDOUT "from refs/mediawiki/$remotename/master^0\n";
725 }
726 if ($content ne DELETED_CONTENT) {
462d97da
MM
727 print STDOUT "M 644 inline " .
728 fe_escape_path($title . ".mw") . "\n";
428c995c 729 literal_data($content);
6a9e55b0 730 if (%mediafile) {
462d97da
MM
731 print STDOUT "M 644 inline "
732 . fe_escape_path($mediafile{title}) . "\n";
6a9e55b0
PV
733 literal_data_raw($mediafile{content});
734 }
428c995c
JN
735 print STDOUT "\n\n";
736 } else {
462d97da 737 print STDOUT "D " . fe_escape_path($title . ".mw") . "\n";
428c995c
JN
738 }
739
740 # mediawiki revision number in the git note
741 if ($full_import && $n == 1) {
742 print STDOUT "reset refs/notes/$remotename/mediawiki\n";
743 }
744 print STDOUT "commit refs/notes/$remotename/mediawiki\n";
745 print STDOUT "committer $author <$author\@$wiki_name> ", $date->epoch, " +0000\n";
746 literal_data("Note added by git-mediawiki during import");
747 if (!$full_import && $n == 1) {
748 print STDOUT "from refs/notes/$remotename/mediawiki^0\n";
749 }
750 print STDOUT "N inline :$n\n";
751 literal_data("mediawiki_revision: " . $commit{mw_revision});
752 print STDOUT "\n\n";
1aff8c62 753 return;
428c995c
JN
754}
755
756# parse a sequence of
757# <cmd> <arg1>
758# <cmd> <arg2>
759# \n
760# (like batch sequence of import and sequence of push statements)
761sub get_more_refs {
762 my $cmd = shift;
763 my @refs;
764 while (1) {
765 my $line = <STDIN>;
eb96b750 766 if ($line =~ /^$cmd (.*)$/) {
428c995c
JN
767 push(@refs, $1);
768 } elsif ($line eq "\n") {
769 return @refs;
770 } else {
8a43b36a 771 die("Invalid command in a '$cmd' batch: $_\n");
428c995c
JN
772 }
773 }
1aff8c62 774 return;
428c995c
JN
775}
776
777sub mw_import {
778 # multiple import commands can follow each other.
779 my @refs = (shift, get_more_refs("import"));
780 foreach my $ref (@refs) {
781 mw_import_ref($ref);
782 }
783 print STDOUT "done\n";
1aff8c62 784 return;
428c995c
JN
785}
786
787sub mw_import_ref {
788 my $ref = shift;
789 # The remote helper will call "import HEAD" and
790 # "import refs/heads/master".
791 # Since HEAD is a symbolic ref to master (by convention,
792 # followed by the output of the command "list" that we gave),
793 # we don't need to do anything in this case.
794 if ($ref eq "HEAD") {
795 return;
796 }
797
798 mw_connect_maybe();
799
428c995c
JN
800 print STDERR "Searching revisions...\n";
801 my $last_local = get_last_local_revision();
802 my $fetch_from = $last_local + 1;
803 if ($fetch_from == 1) {
804 print STDERR ", fetching from beginning.\n";
805 } else {
806 print STDERR ", fetching from here.\n";
807 }
428c995c 808
5a29217d
MM
809 my $n = 0;
810 if ($fetch_strategy eq "by_rev") {
811 print STDERR "Fetching & writing export data by revs...\n";
812 $n = mw_import_ref_by_revs($fetch_from);
813 } elsif ($fetch_strategy eq "by_page") {
814 print STDERR "Fetching & writing export data by pages...\n";
815 $n = mw_import_ref_by_pages($fetch_from);
816 } else {
817 print STDERR "fatal: invalid fetch strategy \"$fetch_strategy\".\n";
818 print STDERR "Check your configuration variables remote.$remotename.fetchStrategy and mediawiki.fetchStrategy\n";
819 exit 1;
820 }
821
822 if ($fetch_from == 1 && $n == 0) {
823 print STDERR "You appear to have cloned an empty MediaWiki.\n";
824 # Something has to be done remote-helper side. If nothing is done, an error is
2582ab18 825 # thrown saying that HEAD is referring to unknown object 0000000000000000000
5a29217d
MM
826 # and the clone fails.
827 }
1aff8c62 828 return;
5a29217d
MM
829}
830
831sub mw_import_ref_by_pages {
832
833 my $fetch_from = shift;
834 my %pages_hash = get_mw_pages();
835 my @pages = values(%pages_hash);
836
428c995c
JN
837 my ($n, @revisions) = fetch_mw_revisions(\@pages, $fetch_from);
838
4465b6d6 839 @revisions = sort {$a->{revid} <=> $b->{revid}} @revisions;
81f6a7a4 840 my @revision_ids = map { $_->{revid} } @revisions;
4465b6d6 841
312fa9aa
MM
842 return mw_import_revids($fetch_from, \@revision_ids, \%pages_hash);
843}
844
5a29217d
MM
845sub mw_import_ref_by_revs {
846
847 my $fetch_from = shift;
848 my %pages_hash = get_mw_pages();
428c995c 849
5a29217d
MM
850 my $last_remote = get_last_global_remote_rev();
851 my @revision_ids = $fetch_from..$last_remote;
852 return mw_import_revids($fetch_from, \@revision_ids, \%pages_hash);
853}
854
855# Import revisions given in second argument (array of integers).
856# Only pages appearing in the third argument (hash indexed by page titles)
857# will be imported.
312fa9aa
MM
858sub mw_import_revids {
859 my $fetch_from = shift;
860 my $revision_ids = shift;
861 my $pages = shift;
862
863 my $n = 0;
5a29217d 864 my $n_actual = 0;
428c995c
JN
865 my $last_timestamp = 0; # Placeholer in case $rev->timestamp is undefined
866
312fa9aa 867 foreach my $pagerevid (@$revision_ids) {
ebd5fe1c
MM
868 # Count page even if we skip it, since we display
869 # $n/$total and $total includes skipped pages.
870 $n++;
871
428c995c
JN
872 # fetch the content of the pages
873 my $query = {
874 action => 'query',
875 prop => 'revisions',
876 rvprop => 'content|timestamp|comment|user|ids',
4465b6d6 877 revids => $pagerevid,
428c995c
JN
878 };
879
880 my $result = $mediawiki->api($query);
881
ff0bfd75 882 if (!$result) {
8a43b36a 883 die "Failed to retrieve modified page for revision $pagerevid\n";
ff0bfd75
MM
884 }
885
ebd5fe1c
MM
886 if (defined($result->{query}->{badrevids}->{$pagerevid})) {
887 # The revision id does not exist on the remote wiki.
888 next;
889 }
890
ff0bfd75 891 if (!defined($result->{query}->{pages})) {
8a43b36a 892 die "Invalid revision $pagerevid.\n";
ff0bfd75
MM
893 }
894
4465b6d6
MM
895 my @result_pages = values(%{$result->{query}->{pages}});
896 my $result_page = $result_pages[0];
897 my $rev = $result_pages[0]->{revisions}->[0];
428c995c 898
4465b6d6 899 my $page_title = $result_page->{title};
5a29217d
MM
900
901 if (!exists($pages->{$page_title})) {
902 print STDERR "$n/", scalar(@$revision_ids),
903 ": Skipping revision #$rev->{revid} of $page_title\n";
904 next;
905 }
906
907 $n_actual++;
908
428c995c
JN
909 my %commit;
910 $commit{author} = $rev->{user} || 'Anonymous';
dcb1ea62 911 $commit{comment} = $rev->{comment} || EMPTY_MESSAGE;
6a9e55b0 912 $commit{title} = mediawiki_smudge_filename($page_title);
4465b6d6 913 $commit{mw_revision} = $rev->{revid};
428c995c
JN
914 $commit{content} = mediawiki_smudge($rev->{'*'});
915
916 if (!defined($rev->{timestamp})) {
917 $last_timestamp++;
918 } else {
919 $last_timestamp = $rev->{timestamp};
920 }
921 $commit{date} = DateTime::Format::ISO8601->parse_datetime($last_timestamp);
922
6a9e55b0 923 # Differentiates classic pages and media files.
6df7e0df 924 my ($namespace, $filename) = $page_title =~ /^([^:]*):(.*)$/;
6a9e55b0 925 my %mediafile;
2272400c
MM
926 if ($namespace) {
927 my $id = get_mw_namespace_id($namespace);
928 if ($id && $id == get_mw_namespace_id("File")) {
929 %mediafile = get_mw_mediafile_for_page_revision($filename, $rev->{timestamp});
930 }
6a9e55b0
PV
931 }
932 # If this is a revision of the media page for new version
933 # of a file do one common commit for both file and media page.
934 # Else do commit only for that page.
312fa9aa 935 print STDERR "$n/", scalar(@$revision_ids), ": Revision #$rev->{revid} of $commit{title}\n";
5a29217d 936 import_file_revision(\%commit, ($fetch_from == 1), $n_actual, \%mediafile);
428c995c
JN
937 }
938
5a29217d 939 return $n_actual;
428c995c
JN
940}
941
942sub error_non_fast_forward {
fd47d7b9
MM
943 my $advice = run_git("config --bool advice.pushNonFastForward");
944 chomp($advice);
945 if ($advice ne "false") {
946 # Native git-push would show this after the summary.
947 # We can't ask it to display it cleanly, so print it
948 # ourselves before.
949 print STDERR "To prevent you from losing history, non-fast-forward updates were rejected\n";
950 print STDERR "Merge the remote changes (e.g. 'git pull') before pushing again. See the\n";
951 print STDERR "'Note about fast-forwards' section of 'git push --help' for details.\n";
952 }
428c995c
JN
953 print STDOUT "error $_[0] \"non-fast-forward\"\n";
954 return 0;
955}
956
b3d98595
NKT
957sub mw_upload_file {
958 my $complete_file_name = shift;
959 my $new_sha1 = shift;
960 my $extension = shift;
961 my $file_deleted = shift;
962 my $summary = shift;
963 my $newrevid;
964 my $path = "File:" . $complete_file_name;
965 my %hashFiles = get_allowed_file_extensions();
966 if (!exists($hashFiles{$extension})) {
967 print STDERR "$complete_file_name is not a permitted file on this wiki.\n";
968 print STDERR "Check the configuration of file uploads in your mediawiki.\n";
969 return $newrevid;
970 }
971 # Deleting and uploading a file requires a priviledged user
972 if ($file_deleted) {
973 mw_connect_maybe();
974 my $query = {
975 action => 'delete',
976 title => $path,
977 reason => $summary
978 };
979 if (!$mediawiki->edit($query)) {
980 print STDERR "Failed to delete file on remote wiki\n";
981 print STDERR "Check your permissions on the remote site. Error code:\n";
982 print STDERR $mediawiki->{error}->{code} . ':' . $mediawiki->{error}->{details};
983 exit 1;
984 }
985 } else {
986 # Don't let perl try to interpret file content as UTF-8 => use "raw"
987 my $content = run_git("cat-file blob $new_sha1", "raw");
988 if ($content ne "") {
989 mw_connect_maybe();
990 $mediawiki->{config}->{upload_url} =
991 "$url/index.php/Special:Upload";
992 $mediawiki->edit({
993 action => 'upload',
994 filename => $complete_file_name,
995 comment => $summary,
996 file => [undef,
997 $complete_file_name,
998 Content => $content],
999 ignorewarnings => 1,
1000 }, {
1001 skip_encoding => 1
1002 } ) || die $mediawiki->{error}->{code} . ':'
8a43b36a 1003 . $mediawiki->{error}->{details} . "\n";
b3d98595
NKT
1004 my $last_file_page = $mediawiki->get_page({title => $path});
1005 $newrevid = $last_file_page->{revid};
1006 print STDERR "Pushed file: $new_sha1 - $complete_file_name.\n";
1007 } else {
1008 print STDERR "Empty file $complete_file_name not pushed.\n";
1009 }
1010 }
1011 return $newrevid;
1012}
1013
428c995c
JN
1014sub mw_push_file {
1015 my $diff_info = shift;
1016 # $diff_info contains a string in this format:
1017 # 100644 100644 <sha1_of_blob_before_commit> <sha1_of_blob_now> <status>
1018 my @diff_info_split = split(/[ \t]/, $diff_info);
1019
1020 # Filename, including .mw extension
1021 my $complete_file_name = shift;
1022 # Commit message
1023 my $summary = shift;
93f0d338
MM
1024 # MediaWiki revision number. Keep the previous one by default,
1025 # in case there's no edit to perform.
b3d98595
NKT
1026 my $oldrevid = shift;
1027 my $newrevid;
428c995c 1028
dcb1ea62
MM
1029 if ($summary eq EMPTY_MESSAGE) {
1030 $summary = '';
1031 }
1032
428c995c
JN
1033 my $new_sha1 = $diff_info_split[3];
1034 my $old_sha1 = $diff_info_split[2];
1035 my $page_created = ($old_sha1 eq NULL_SHA1);
1036 my $page_deleted = ($new_sha1 eq NULL_SHA1);
1037 $complete_file_name = mediawiki_clean_filename($complete_file_name);
1038
b3d98595
NKT
1039 my ($title, $extension) = $complete_file_name =~ /^(.*)\.([^\.]*)$/;
1040 if (!defined($extension)) {
1041 $extension = "";
1042 }
1043 if ($extension eq "mw") {
2045e293
MM
1044 my $ns = get_mw_namespace_id_for_page($complete_file_name);
1045 if ($ns && $ns == get_mw_namespace_id("File") && (!$export_media)) {
1046 print STDERR "Ignoring media file related page: $complete_file_name\n";
1047 return ($oldrevid, "ok");
1048 }
428c995c
JN
1049 my $file_content;
1050 if ($page_deleted) {
1051 # Deleting a page usually requires
2582ab18 1052 # special privileges. A common
428c995c
JN
1053 # convention is to replace the page
1054 # with this content instead:
1055 $file_content = DELETED_CONTENT;
1056 } else {
1057 $file_content = run_git("cat-file blob $new_sha1");
1058 }
1059
1060 mw_connect_maybe();
1061
1062 my $result = $mediawiki->edit( {
1063 action => 'edit',
1064 summary => $summary,
1065 title => $title,
b3d98595 1066 basetimestamp => $basetimestamps{$oldrevid},
428c995c
JN
1067 text => mediawiki_clean($file_content, $page_created),
1068 }, {
1069 skip_encoding => 1 # Helps with names with accentuated characters
3c1ed90e
MM
1070 });
1071 if (!$result) {
1072 if ($mediawiki->{error}->{code} == 3) {
1073 # edit conflicts, considered as non-fast-forward
1074 print STDERR 'Warning: Error ' .
1075 $mediawiki->{error}->{code} .
1076 ' from mediwiki: ' . $mediawiki->{error}->{details} .
1077 ".\n";
b3d98595 1078 return ($oldrevid, "non-fast-forward");
3c1ed90e
MM
1079 } else {
1080 # Other errors. Shouldn't happen => just die()
1081 die 'Fatal: Error ' .
1082 $mediawiki->{error}->{code} .
8a43b36a 1083 ' from mediwiki: ' . $mediawiki->{error}->{details} . "\n";
3c1ed90e
MM
1084 }
1085 }
93f0d338 1086 $newrevid = $result->{edit}->{newrevid};
ac86ec0f 1087 print STDERR "Pushed file: $new_sha1 - $title\n";
2045e293 1088 } elsif ($export_media) {
b3d98595
NKT
1089 $newrevid = mw_upload_file($complete_file_name, $new_sha1,
1090 $extension, $page_deleted,
1091 $summary);
2045e293
MM
1092 } else {
1093 print STDERR "Ignoring media file $title\n";
428c995c 1094 }
b3d98595 1095 $newrevid = ($newrevid or $oldrevid);
3c1ed90e 1096 return ($newrevid, "ok");
428c995c
JN
1097}
1098
1099sub mw_push {
1100 # multiple push statements can follow each other
1101 my @refsspecs = (shift, get_more_refs("push"));
428c995c
JN
1102 my $pushed;
1103 for my $refspec (@refsspecs) {
1104 my ($force, $local, $remote) = $refspec =~ /^(\+)?([^:]*):([^:]*)$/
8a43b36a 1105 or die("Invalid refspec for push. Expected <src>:<dst> or +<src>:<dst>\n");
428c995c
JN
1106 if ($force) {
1107 print STDERR "Warning: forced push not allowed on a MediaWiki.\n";
1108 }
1109 if ($local eq "") {
1110 print STDERR "Cannot delete remote branch on a MediaWiki\n";
1111 print STDOUT "error $remote cannot delete\n";
1112 next;
1113 }
1114 if ($remote ne "refs/heads/master") {
1115 print STDERR "Only push to the branch 'master' is supported on a MediaWiki\n";
1116 print STDOUT "error $remote only master allowed\n";
1117 next;
1118 }
1119 if (mw_push_revision($local, $remote)) {
1120 $pushed = 1;
1121 }
1122 }
1123
1124 # Notify Git that the push is done
1125 print STDOUT "\n";
1126
93f0d338 1127 if ($pushed && $dumb_push) {
428c995c
JN
1128 print STDERR "Just pushed some revisions to MediaWiki.\n";
1129 print STDERR "The pushed revisions now have to be re-imported, and your current branch\n";
1130 print STDERR "needs to be updated with these re-imported commits. You can do this with\n";
1131 print STDERR "\n";
1132 print STDERR " git pull --rebase\n";
1133 print STDERR "\n";
1134 }
1aff8c62 1135 return;
428c995c
JN
1136}
1137
1138sub mw_push_revision {
1139 my $local = shift;
1140 my $remote = shift; # actually, this has to be "refs/heads/master" at this point.
1141 my $last_local_revid = get_last_local_revision();
1142 print STDERR ".\n"; # Finish sentence started by get_last_local_revision()
1143 my $last_remote_revid = get_last_remote_revision();
93f0d338 1144 my $mw_revision = $last_remote_revid;
428c995c
JN
1145
1146 # Get sha1 of commit pointed by local HEAD
1147 my $HEAD_sha1 = run_git("rev-parse $local 2>/dev/null"); chomp($HEAD_sha1);
1148 # Get sha1 of commit pointed by remotes/$remotename/master
1149 my $remoteorigin_sha1 = run_git("rev-parse refs/remotes/$remotename/master 2>/dev/null");
1150 chomp($remoteorigin_sha1);
1151
1152 if ($last_local_revid > 0 &&
1153 $last_local_revid < $last_remote_revid) {
1154 return error_non_fast_forward($remote);
1155 }
1156
1157 if ($HEAD_sha1 eq $remoteorigin_sha1) {
1158 # nothing to push
1159 return 0;
1160 }
1161
1162 # Get every commit in between HEAD and refs/remotes/origin/master,
1163 # including HEAD and refs/remotes/origin/master
1164 my @commit_pairs = ();
1165 if ($last_local_revid > 0) {
1166 my $parsed_sha1 = $remoteorigin_sha1;
1167 # Find a path from last MediaWiki commit to pushed commit
93e92d4d
MM
1168 print STDERR "Computing path from local to remote ...\n";
1169 my @local_ancestry = split(/\n/, run_git("rev-list --boundary --parents $local ^$parsed_sha1"));
1170 my %local_ancestry;
1171 foreach my $line (@local_ancestry) {
eb96b750 1172 if (my ($child, $parents) = $line =~ /^-?([a-f0-9]+) ([a-f0-9 ]+)/) {
11499573 1173 foreach my $parent (split(/ /, $parents)) {
93e92d4d
MM
1174 $local_ancestry{$parent} = $child;
1175 }
eb96b750 1176 } elsif (!$line =~ /^([a-f0-9]+)/) {
8a43b36a 1177 die "Unexpected output from git rev-list: $line\n";
93e92d4d
MM
1178 }
1179 }
428c995c 1180 while ($parsed_sha1 ne $HEAD_sha1) {
93e92d4d
MM
1181 my $child = $local_ancestry{$parsed_sha1};
1182 if (!$child) {
1183 printf STDERR "Cannot find a path in history from remote commit to last commit\n";
428c995c
JN
1184 return error_non_fast_forward($remote);
1185 }
93e92d4d
MM
1186 push(@commit_pairs, [$parsed_sha1, $child]);
1187 $parsed_sha1 = $child;
428c995c
JN
1188 }
1189 } else {
1190 # No remote mediawiki revision. Export the whole
1191 # history (linearized with --first-parent)
1192 print STDERR "Warning: no common ancestor, pushing complete history\n";
1193 my $history = run_git("rev-list --first-parent --children $local");
05d4c7b1 1194 my @history = split(/\n/, $history);
428c995c
JN
1195 @history = @history[1..$#history];
1196 foreach my $line (reverse @history) {
6b825a46 1197 my @commit_info_split = split(/[ \n]/, $line);
428c995c
JN
1198 push(@commit_pairs, \@commit_info_split);
1199 }
1200 }
1201
1202 foreach my $commit_info_split (@commit_pairs) {
1203 my $sha1_child = @{$commit_info_split}[0];
1204 my $sha1_commit = @{$commit_info_split}[1];
1205 my $diff_infos = run_git("diff-tree -r --raw -z $sha1_child $sha1_commit");
1206 # TODO: we could detect rename, and encode them with a #redirect on the wiki.
1207 # TODO: for now, it's just a delete+add
1208 my @diff_info_list = split(/\0/, $diff_infos);
28c24bd7
MM
1209 # Keep the subject line of the commit message as mediawiki comment for the revision
1210 my $commit_msg = run_git("log --no-walk --format=\"%s\" $sha1_commit");
428c995c
JN
1211 chomp($commit_msg);
1212 # Push every blob
1213 while (@diff_info_list) {
3c1ed90e 1214 my $status;
428c995c
JN
1215 # git diff-tree -z gives an output like
1216 # <metadata>\0<filename1>\0
1217 # <metadata>\0<filename2>\0
1218 # and we've split on \0.
1219 my $info = shift(@diff_info_list);
1220 my $file = shift(@diff_info_list);
3c1ed90e
MM
1221 ($mw_revision, $status) = mw_push_file($info, $file, $commit_msg, $mw_revision);
1222 if ($status eq "non-fast-forward") {
1223 # we may already have sent part of the
1224 # commit to MediaWiki, but it's too
1225 # late to cancel it. Stop the push in
1226 # the middle, but still give an
1227 # accurate error message.
1228 return error_non_fast_forward($remote);
1229 }
1230 if ($status ne "ok") {
8a43b36a 1231 die("Unknown error from mw_push_file()\n");
3c1ed90e 1232 }
93f0d338
MM
1233 }
1234 unless ($dumb_push) {
5c481745 1235 run_git("notes --ref=$remotename/mediawiki add -f -m \"mediawiki_revision: $mw_revision\" $sha1_commit");
93f0d338 1236 run_git("update-ref -m \"Git-MediaWiki push\" refs/mediawiki/$remotename/master $sha1_commit $sha1_child");
428c995c
JN
1237 }
1238 }
1239
1240 print STDOUT "ok $remote\n";
1241 return 1;
1242}
b3d98595
NKT
1243
1244sub get_allowed_file_extensions {
1245 mw_connect_maybe();
1246
1247 my $query = {
1248 action => 'query',
1249 meta => 'siteinfo',
1250 siprop => 'fileextensions'
1251 };
1252 my $result = $mediawiki->api($query);
81f6a7a4
CM
1253 my @file_extensions = map { $_->{ext}} @{$result->{query}->{fileextensions}};
1254 my %hashFile = map { $_ => 1 } @file_extensions;
b3d98595
NKT
1255
1256 return %hashFile;
1257}
6a9e55b0 1258
6df7e0df
MM
1259# In memory cache for MediaWiki namespace ids.
1260my %namespace_id;
1261
1262# Namespaces whose id is cached in the configuration file
1263# (to avoid duplicates)
1264my %cached_mw_namespace_id;
1265
6a9e55b0
PV
1266# Return MediaWiki id for a canonical namespace name.
1267# Ex.: "File", "Project".
6a9e55b0
PV
1268sub get_mw_namespace_id {
1269 mw_connect_maybe();
1270 my $name = shift;
1271
1272 if (!exists $namespace_id{$name}) {
1273 # Look at configuration file, if the record for that namespace is
6df7e0df 1274 # already cached. Namespaces are stored in form:
6a9e55b0 1275 # "Name_of_namespace:Id_namespace", ex.: "File:6".
477d4d42 1276 my @temp = split(/\n/, run_git("config --get-all remote."
6df7e0df 1277 . $remotename .".namespaceCache"));
6a9e55b0
PV
1278 chomp(@temp);
1279 foreach my $ns (@temp) {
6df7e0df 1280 my ($n, $id) = split(/:/, $ns);
2272400c
MM
1281 if ($id eq 'notANameSpace') {
1282 $namespace_id{$n} = {is_namespace => 0};
1283 } else {
1284 $namespace_id{$n} = {is_namespace => 1, id => $id};
1285 }
6df7e0df 1286 $cached_mw_namespace_id{$n} = 1;
6a9e55b0
PV
1287 }
1288 }
1289
1290 if (!exists $namespace_id{$name}) {
6df7e0df 1291 print STDERR "Namespace $name not found in cache, querying the wiki ...\n";
6a9e55b0
PV
1292 # NS not found => get namespace id from MW and store it in
1293 # configuration file.
1294 my $query = {
1295 action => 'query',
1296 meta => 'siteinfo',
1297 siprop => 'namespaces'
1298 };
1299 my $result = $mediawiki->api($query);
1300
1301 while (my ($id, $ns) = each(%{$result->{query}->{namespaces}})) {
6df7e0df 1302 if (defined($ns->{id}) && defined($ns->{canonical})) {
2272400c 1303 $namespace_id{$ns->{canonical}} = {is_namespace => 1, id => $ns->{id}};
6df7e0df
MM
1304 if ($ns->{'*'}) {
1305 # alias (e.g. french Fichier: as alias for canonical File:)
2272400c 1306 $namespace_id{$ns->{'*'}} = {is_namespace => 1, id => $ns->{id}};
6df7e0df
MM
1307 }
1308 }
6a9e55b0
PV
1309 }
1310 }
1311
2272400c
MM
1312 my $ns = $namespace_id{$name};
1313 my $id;
6df7e0df 1314
2272400c
MM
1315 unless (defined $ns) {
1316 print STDERR "No such namespace $name on MediaWiki.\n";
1317 $ns = {is_namespace => 0};
1318 $namespace_id{$name} = $ns;
1319 }
1320
1321 if ($ns->{is_namespace}) {
1322 $id = $ns->{id};
1323 }
1324
1325 # Store "notANameSpace" as special value for inexisting namespaces
1326 my $store_id = ($id || 'notANameSpace');
1327
1328 # Store explicitely requested namespaces on disk
1329 if (!exists $cached_mw_namespace_id{$name}) {
1330 run_git("config --add remote.". $remotename
1331 .".namespaceCache \"". $name .":". $store_id ."\"");
1332 $cached_mw_namespace_id{$name} = 1;
6a9e55b0 1333 }
2272400c 1334 return $id;
6a9e55b0 1335}
2045e293
MM
1336
1337sub get_mw_namespace_id_for_page {
6c2fbe25
CM
1338 my $namespace = shift;
1339 if ($namespace =~ /^([^:]*):/) {
2045e293
MM
1340 return get_mw_namespace_id($namespace);
1341 } else {
1342 return;
1343 }
1344}