update_responses 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656
  1. #!/usr/bin/perl -w
  2. use strict;
  3. use feature "state";
  4. use English;
  5. use FindBin;
  6. use YAML qw(LoadFile);
  7. use File::Slurp;
  8. use Digest::SHA qw(sha256_hex);
  9. use XML::Writer;
  10. use Cwd;
  11. use File::Copy;
  12. use File::Temp;
  13. use File::Find;
  14. use POSIX qw(setlocale LC_ALL);
  15. use IO::CaptureOutput qw(capture_exec);
  16. use Parallel::ForkManager;
  17. use File::Basename;
  18. use XML::LibXML '1.70';
  19. use LWP::Simple;
  20. use JSON;
  21. # Set umask and locale to provide a consistent environment for MAR file
  22. # generation, etc.
  23. umask(0022);
  24. $ENV{"LC_ALL"} = "C";
  25. setlocale(LC_ALL, "C");
  26. my $htdocsdir = "$FindBin::Bin/htdocs";
  27. my $config = LoadFile("$FindBin::Bin/config.yml");
  28. my %htdocsfiles;
  29. my $releases_dir = $config->{releases_dir};
  30. $releases_dir = "$FindBin::Bin/$releases_dir" unless $releases_dir =~ m/^\//;
  31. my @check_errors;
  32. my $initPATH = $ENV{PATH};
  33. my $initLD_LIBRARY_PATH = $ENV{LD_LIBRARY_PATH};
  34. sub exit_error {
  35. print STDERR "Error: ", $_[0], "\n";
  36. chdir '/';
  37. exit (exists $_[1] ? $_[1] : 1);
  38. }
  39. sub get_tmpdir {
  40. my ($config) = @_;
  41. return File::Temp->newdir($config->{tmp_dir} ?
  42. (DIR => $config->{tmp_dir})
  43. : ());
  44. }
  45. sub build_targets_by_os {
  46. return ($_[0]) unless $config->{build_targets}{$_[0]};
  47. my $r = $config->{build_targets}{$_[0]};
  48. return ref $r eq 'ARRAY' ? @$r : ($r);
  49. }
  50. sub get_nbprocs {
  51. return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS};
  52. if (-f '/proc/cpuinfo') {
  53. return scalar grep { m/^processor\s+:\s/ } read_file '/proc/cpuinfo';
  54. }
  55. return 4;
  56. }
  57. sub write_htdocs {
  58. my ($channel, $file, $content) = @_;
  59. mkdir $htdocsdir unless -d $htdocsdir;
  60. mkdir "$htdocsdir/$channel" unless -d "$htdocsdir/$channel";
  61. write_file("$htdocsdir/$channel/$file", $content);
  62. $htdocsfiles{$channel}->{$file} = 1;
  63. }
  64. sub clean_htdocs {
  65. my (@channels) = @_;
  66. foreach my $channel (@channels) {
  67. opendir(my $d, "$htdocsdir/$channel");
  68. my @files = grep { ! $htdocsfiles{$channel}->{$_} } readdir $d;
  69. closedir $d;
  70. unlink map { "$htdocsdir/$channel/$_" } @files;
  71. }
  72. }
  73. sub get_sha512_hex_of_file {
  74. my ($file) = @_;
  75. my $sha = Digest::SHA->new("512");
  76. $sha->addfile($file);
  77. return $sha->hexdigest;
  78. }
  79. sub get_version_files {
  80. my ($config, $version) = @_;
  81. return if $config->{versions}{$version}{files};
  82. my $appname = $config->{appname_marfile};
  83. my $files = {};
  84. my $vdir = version_dir($config, $version);
  85. my $download_url = "$config->{download}{mars_url}/$version";
  86. opendir(my $d, $vdir) or exit_error "Error opening directory $vdir";
  87. foreach my $file (readdir $d) {
  88. next unless -f "$vdir/$file";
  89. if ($file =~ m/^$appname-([^-]+)-${version}_(.+)\.mar$/) {
  90. my ($os, $lang) = ($1, $2);
  91. $files->{$os}{$lang}{complete} = {
  92. type => 'complete',
  93. URL => "$download_url/$file",
  94. size => -s "$vdir/$file",
  95. hashFunction => 'SHA512',
  96. hashValue => get_sha512_hex_of_file("$vdir/$file"),
  97. };
  98. next;
  99. }
  100. if ($file =~ m/^$appname-([^-]+)-(.+)-${version}_(.+)\.incremental\.mar$/) {
  101. my ($os, $from_version, $lang) = ($1, $2, $3);
  102. $files->{$os}{$lang}{partial}{$from_version} = {
  103. type => 'partial',
  104. URL => "$download_url/$file",
  105. size => -s "$vdir/$file",
  106. hashFunction => 'SHA512',
  107. hashValue => get_sha512_hex_of_file("$vdir/$file"),
  108. }
  109. }
  110. }
  111. closedir $d;
  112. $config->{versions}{$version}{files} = $files;
  113. }
  114. sub get_version_downloads {
  115. my ($config, $version) = @_;
  116. my $downloads = {};
  117. my $vdir = version_dir($config, $version);
  118. my $download_url = "$config->{download}{bundles_url}/$version";
  119. opendir(my $d, $vdir) or exit_error "Error opening directory $vdir";
  120. foreach my $file (readdir $d) {
  121. next unless -f "$vdir/$file";
  122. my ($os, $lang);
  123. if ($file =~ m/^$config->{appname_bundle_osx}-$version-osx64_(.+).dmg$/) {
  124. ($os, $lang) = ('osx64', $1);
  125. } elsif ($file =~ m/^$config->{appname_bundle_linux}-(linux32|linux64)-${version}_(.+).tar.xz$/) {
  126. ($os, $lang) = ($1, $2);
  127. } elsif ($file =~ m/^$config->{appname_bundle_win64}-${version}_(.+).exe$/) {
  128. ($os, $lang) = ('win64', $1);
  129. } elsif ($file =~ m/^$config->{appname_bundle_win32}-${version}_(.+).exe$/) {
  130. ($os, $lang) = ('win32', $1);
  131. } else {
  132. next;
  133. }
  134. $downloads->{$os}{$lang} = {
  135. binary => "$download_url/$file",
  136. sig => "$download_url/$file.asc",
  137. };
  138. }
  139. closedir $d;
  140. $config->{versions}{$version}{downloads} = $downloads;
  141. }
  142. sub extract_mar {
  143. my ($mar_file, $dest_dir) = @_;
  144. my $old_cwd = getcwd;
  145. mkdir $dest_dir;
  146. chdir $dest_dir or exit_error "Cannot enter $dest_dir";
  147. my $res = system('mar', '-x', $mar_file);
  148. exit_error "Error extracting $mar_file" if $res;
  149. my $bunzip_file = sub {
  150. return unless -f $File::Find::name;
  151. rename $File::Find::name, "$File::Find::name.bz2";
  152. system('bunzip2', "$File::Find::name.bz2") == 0
  153. || exit_error "Error decompressing $File::Find::name";
  154. };
  155. find($bunzip_file, $dest_dir);
  156. my $manifest = -f 'updatev3.manifest' ? 'updatev3.manifest'
  157. : 'updatev2.manifest';
  158. my @lines = read_file($manifest) if -f $manifest;
  159. foreach my $line (@lines) {
  160. if ($line =~ m/^addsymlink "(.+)" "(.+)"$/) {
  161. exit_error "$mar_file: Could not create symlink $1 -> $2"
  162. unless symlink $2, $1;
  163. }
  164. }
  165. chdir $old_cwd;
  166. }
  167. sub mar_filename {
  168. my ($config, $appname, $version, $os, $lang) = @_;
  169. version_dir($config, $version) . "/$appname-$os-${version}_$lang.mar";
  170. }
  171. sub create_incremental_mar {
  172. my ($config, $pm, $from_version, $new_version, $os, $lang) = @_;
  173. my $appname = $config->{appname_marfile};
  174. my $mar_file = "$appname-$os-${from_version}-${new_version}_$lang.incremental.mar";
  175. my $mar_file_path = version_dir($config, $new_version) . '/' . $mar_file;
  176. if ($ENV{MAR_SKIP_EXISTING} && -f $mar_file_path) {
  177. print "Skipping $mar_file\n";
  178. return;
  179. }
  180. print "Starting $mar_file\n";
  181. my $download_url = "$config->{download}{mars_url}/$new_version";
  182. my $finished_file = sub {
  183. exit_error "Error creating $mar_file" unless $_[1] == 0;
  184. print "Finished $mar_file\n";
  185. $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = {
  186. type => 'partial',
  187. URL => "$download_url/$mar_file",
  188. size => -s $mar_file_path,
  189. hashFunction => 'SHA512',
  190. hashValue => get_sha512_hex_of_file($mar_file_path),
  191. };
  192. };
  193. return if $pm->start($finished_file);
  194. my $tmpdir = get_tmpdir($config);
  195. extract_mar(mar_filename($config, $appname, $from_version, $os, $lang), "$tmpdir/A");
  196. extract_mar(mar_filename($config, $appname, $new_version, $os, $lang), "$tmpdir/B");
  197. # bug 26054: make sure previous macOS version is code signed
  198. if (($os eq 'osx64') && ! -f "$tmpdir/A/Contents/_CodeSignature/CodeResources") {
  199. exit_error "Missing code signature in $from_version while creating $mar_file";
  200. }
  201. if ($ENV{CHECK_CODESIGNATURE_EXISTS}) {
  202. unless (-f "$tmpdir/A/Contents/_CodeSignature/CodeResources"
  203. && -f "$tmpdir/B/Contents/_CodeSignature/CodeResources") {
  204. exit_error "Missing code signature while creating $mar_file";
  205. }
  206. }
  207. my ($out, $err, $success) = capture_exec('make_incremental_update.sh',
  208. $mar_file_path, "$tmpdir/A", "$tmpdir/B");
  209. if (!$success) {
  210. unlink $mar_file_path if -f $mar_file_path;
  211. exit_error "making incremental mar:\n" . $err;
  212. }
  213. $pm->finish;
  214. }
  215. sub create_incremental_mars_for_version {
  216. my ($config, $version) = @_;
  217. my $pm = Parallel::ForkManager->new(get_nbprocs);
  218. $pm->run_on_finish(sub { $_[2]->(@_) });
  219. my $v = $config->{versions}{$version};
  220. foreach my $from_version (@{$v->{incremental_from}}) {
  221. $config->{versions}{$from_version} //= {};
  222. get_version_files($config, $from_version);
  223. my $from_v = $config->{versions}{$from_version};
  224. foreach my $os (keys %{$v->{files}}) {
  225. foreach my $lang (keys %{$v->{files}{$os}}) {
  226. next unless defined $from_v->{files}{$os}{$lang}{complete};
  227. create_incremental_mar($config, $pm, $from_version, $version, $os, $lang);
  228. }
  229. }
  230. }
  231. $pm->wait_all_children;
  232. }
  233. sub get_config {
  234. my ($config, $version, $os, $name) = @_;
  235. return $config->{versions}{$version}{$os}{$name}
  236. // $config->{versions}{$version}{$name}
  237. // $config->{$name};
  238. }
  239. sub version_dir {
  240. my ($config, $version) = @_;
  241. return get_config($config, $version, 'any', 'releases_dir') . "/$version";
  242. }
  243. sub channel_to_version {
  244. my ($config, @channels) = @_;
  245. return values %{$config->{channels}} unless @channels;
  246. foreach my $channel (@channels) {
  247. exit_error "Unknown channel $channel"
  248. unless $config->{channels}{$channel};
  249. }
  250. return map { $config->{channels}{$_} } @channels;
  251. }
  252. sub get_buildinfos {
  253. my ($config, $version) = @_;
  254. return if exists $config->{versions}{$version}{buildID};
  255. extract_martools($config, $version);
  256. my $files = $config->{versions}{$version}{files};
  257. foreach my $os (keys %$files) {
  258. foreach my $lang (keys %{$files->{$os}}) {
  259. next unless $files->{$os}{$lang}{complete};
  260. my $tmpdir = get_tmpdir($config);
  261. extract_mar(
  262. mar_filename($config, $config->{appname_marfile}, $version, $os, $lang),
  263. "$tmpdir");
  264. my $appfile = "$tmpdir/application.ini" if -f "$tmpdir/application.ini";
  265. $appfile = "$tmpdir/Contents/Resources/application.ini"
  266. if -f "$tmpdir/Contents/Resources/application.ini";
  267. exit_error "Could not find application.ini" unless $appfile;
  268. foreach my $line (read_file($appfile)) {
  269. if ($line =~ m/^BuildID=(.*)$/) {
  270. $config->{versions}{$version}{buildID} = $1;
  271. return;
  272. }
  273. }
  274. exit_error "Could not extract buildID from application.ini";
  275. }
  276. }
  277. }
  278. sub get_response {
  279. my ($config, $version, $os, @patches) = @_;
  280. my $res;
  281. my $writer = XML::Writer->new(OUTPUT => \$res, ENCODING => 'UTF-8');
  282. $writer->xmlDecl;
  283. $writer->startTag('updates');
  284. if (get_config($config, $version, $os, 'unsupported')) {
  285. $writer->startTag('update',
  286. unsupported => 'true',
  287. detailsURL => get_config($config, $version, $os, 'detailsURL'),
  288. );
  289. goto CLOSETAGS;
  290. }
  291. my $minversion = get_config($config, $version, $os, 'minSupportedOSVersion');
  292. my $mininstruc = get_config($config, $version, $os, 'minSupportedInstructionSet');
  293. $writer->startTag('update',
  294. type => 'minor',
  295. displayVersion => $version,
  296. appVersion => $version,
  297. platformVersion => get_config($config, $version, $os, 'platformVersion'),
  298. buildID => get_config($config, $version, $os, 'buildID'),
  299. detailsURL => get_config($config, $version, $os, 'detailsURL'),
  300. actions => 'showURL',
  301. openURL => get_config($config, $version, $os, 'detailsURL'),
  302. defined $minversion ? ( minSupportedOSVersion => $minversion ) : (),
  303. defined $mininstruc ? ( minSupportedInstructionSet => $mininstruc ) : (),
  304. );
  305. foreach my $patch (@patches) {
  306. my @sorted_patch = map { $_ => $patch->{$_} } sort keys %$patch;
  307. $writer->startTag('patch', @sorted_patch);
  308. $writer->endTag('patch');
  309. }
  310. CLOSETAGS:
  311. $writer->endTag('update');
  312. $writer->endTag('updates');
  313. $writer->end;
  314. return $res;
  315. }
  316. sub write_responses {
  317. my ($config, @channels) = @_;
  318. @channels = keys %{$config->{channels}} unless @channels;
  319. foreach my $channel (@channels) {
  320. my $version = $config->{channels}{$channel};
  321. get_version_files($config, $version);
  322. get_buildinfos($config, $version);
  323. my $files = $config->{versions}{$version}{files};
  324. my $migrate_archs = $config->{versions}{$version}{migrate_archs} // {};
  325. foreach my $old_os (keys %$migrate_archs) {
  326. my $new_os = $migrate_archs->{$old_os};
  327. foreach my $lang (keys %{$files->{$new_os}}) {
  328. $files->{$old_os}{$lang}{complete} =
  329. $files->{$new_os}{$lang}{complete};
  330. }
  331. }
  332. foreach my $os (keys %$files) {
  333. foreach my $lang (keys %{$files->{$os}}) {
  334. my $resp = get_response($config, $version, $os,
  335. $files->{$os}{$lang}{complete});
  336. write_htdocs($channel, "$version-$os-$lang.xml", $resp);
  337. foreach my $from_version (keys %{$files->{$os}{$lang}{partial}}) {
  338. $resp = get_response($config, $version, $os,
  339. $files->{$os}{$lang}{complete},
  340. $files->{$os}{$lang}{partial}{$from_version});
  341. write_htdocs($channel, "$from_version-$version-$os-$lang.xml", $resp);
  342. }
  343. }
  344. }
  345. write_htdocs($channel, 'no-update.xml',
  346. '<?xml version="1.0" encoding="UTF-8"?>'
  347. . "\n<updates></updates>\n");
  348. }
  349. }
  350. sub write_htaccess {
  351. my ($config, @channels) = @_;
  352. @channels = keys %{$config->{channels}} unless @channels;
  353. my $flags = "[last]";
  354. foreach my $channel (@channels) {
  355. my $htaccess = "RewriteEngine On\n";
  356. my $version = $config->{channels}{$channel};
  357. my $migrate_langs = $config->{versions}{$version}{migrate_langs} // {};
  358. my $files = $config->{versions}{$version}{files};
  359. $htaccess .= "RewriteRule ^[^\/]+/$version/ no-update.xml $flags\n";
  360. foreach my $os (sort keys %$files) {
  361. foreach my $bt (build_targets_by_os($os)) {
  362. foreach my $lang (sort keys %{$files->{$os}}) {
  363. foreach my $from_version (sort keys %{$files->{$os}{$lang}{partial}}) {
  364. $htaccess .= "RewriteRule ^$bt/$from_version/$lang "
  365. . "$from_version-$version-$os-$lang.xml $flags\n";
  366. }
  367. $htaccess .= "RewriteRule ^$bt/[^\/]+/$lang "
  368. . "$version-$os-$lang.xml $flags\n";
  369. }
  370. foreach my $lang (sort keys %$migrate_langs) {
  371. $htaccess .= "RewriteRule ^$bt/[^\/]+/$lang "
  372. . "$version-$os-$migrate_langs->{$lang}.xml $flags\n";
  373. }
  374. $htaccess .= "RewriteRule ^$bt/ $version-$os-en-US.xml $flags\n";
  375. }
  376. }
  377. write_htdocs($channel, '.htaccess', $htaccess);
  378. }
  379. }
  380. sub write_downloads_json {
  381. my ($config, @channels) = @_;
  382. @channels = keys %{$config->{channels}} unless @channels;
  383. foreach my $channel (@channels) {
  384. my $version = $config->{channels}{$channel};
  385. my $data = {
  386. version => $version,
  387. downloads => get_version_downloads($config, $version),
  388. };
  389. write_htdocs($channel, 'downloads.json',
  390. JSON->new->utf8->canonical->encode($data));
  391. }
  392. }
  393. sub osname {
  394. my ($osname) = capture_exec('uname', '-s');
  395. my ($arch) = capture_exec('uname', '-m');
  396. chomp($osname, $arch);
  397. if ($osname eq 'Linux' && $arch eq 'x86_64') {
  398. return 'linux64';
  399. }
  400. if ($osname eq 'Linux' && $arch =~ m/^i.86$/) {
  401. return 'linux32';
  402. }
  403. exit_error 'Unknown OS';
  404. }
  405. my $martools_tmpdir;
  406. sub extract_martools {
  407. my ($config, $version) = @_;
  408. my $osname = osname;
  409. my $marzip = version_dir($config, $version) . "/mar-tools-$osname.zip";
  410. $martools_tmpdir = get_tmpdir($config);
  411. my $old_cwd = getcwd;
  412. chdir $martools_tmpdir;
  413. my (undef, undef, $success) = capture_exec('unzip', $marzip);
  414. chdir $old_cwd;
  415. exit_error "Error extracting $marzip" unless $success;
  416. $ENV{PATH} = "$martools_tmpdir/mar-tools:$initPATH";
  417. if ($initLD_LIBRARY_PATH) {
  418. $ENV{LD_LIBRARY_PATH} = "$initLD_LIBRARY_PATH:$martools_tmpdir/mar-tools";
  419. } else {
  420. $ENV{LD_LIBRARY_PATH} = "$martools_tmpdir/mar-tools";
  421. }
  422. }
  423. sub log_step {
  424. my ($url, $step, $status, $details) = @_;
  425. state $u;
  426. if (!defined $u || $url ne $u) {
  427. print "\n" if $u;
  428. print "$url\n";
  429. $u = $url;
  430. }
  431. print ' ', $step, $status ? ': OK' : ': ERROR',
  432. $details ? " - $details\n" : "\n";
  433. return if $status;
  434. push @check_errors, { url => $url, step => $step, details => $details };
  435. }
  436. sub get_remote_xml {
  437. my ($url) = @_;
  438. my $content = get $url;
  439. log_step($url, 'get', defined $content);
  440. return undef unless defined $content;
  441. my $dom = eval { XML::LibXML->load_xml(string => $content) };
  442. log_step($url, 'parse_xml', defined $dom, $@);
  443. return $dom;
  444. }
  445. sub check_get_version {
  446. my ($dom) = @_;
  447. my @updates = $dom->documentElement()->getChildrenByLocalName('update');
  448. return undef unless @updates;
  449. return $updates[0]->getAttribute('appVersion');
  450. }
  451. sub check_no_update {
  452. my ($dom) = @_;
  453. my @updates = $dom->documentElement()->getChildrenByLocalName('update');
  454. return @updates == 0;
  455. }
  456. sub check_has_incremental {
  457. my ($dom) = @_;
  458. my @updates = $dom->documentElement()->getChildrenByLocalName('update');
  459. return undef unless @updates;
  460. my @patches = $updates[0]->getChildrenByLocalName('patch');
  461. foreach my $patch (@patches) {
  462. return 1 if $patch->getAttribute('type') eq 'partial';
  463. }
  464. return undef;
  465. }
  466. sub build_targets_list {
  467. map { ref $_ eq 'ARRAY' ? @$_ : $_ } values %{$config->{build_targets}};
  468. }
  469. sub check_update_responses_channel {
  470. my ($config, $base_url, $channel) = @_;
  471. my $channel_version = $config->{channels}{$channel};
  472. foreach my $build_target (build_targets_list()) {
  473. foreach my $lang (qw(en-US de)) {
  474. my $url = "$base_url/$channel/$build_target/1.0/$lang";
  475. my $dom = get_remote_xml($url);
  476. if ($dom) {
  477. my $version = check_get_version($dom);
  478. log_step($url, 'version', $version eq $channel_version,
  479. "expected: $channel_version received: $version");
  480. }
  481. $url = "$base_url/$channel/$build_target/$channel_version/$lang";
  482. $dom = get_remote_xml($url);
  483. log_step($url, 'no_update', check_no_update($dom)) if $dom;
  484. my @inc = @{$config->{versions}{$channel_version}{incremental_from}}
  485. if $config->{versions}{$channel_version}{incremental_from};
  486. foreach my $inc_from (@inc) {
  487. my $url = "$base_url/$channel/$build_target/$inc_from/$lang";
  488. $dom = get_remote_xml($url);
  489. next unless $dom;
  490. my $version = check_get_version($dom);
  491. log_step($url, 'version', $version eq $channel_version,
  492. "expected: $channel_version received: $version");
  493. log_step($url, 'has_incremental', check_has_incremental($dom));
  494. }
  495. }
  496. }
  497. }
  498. sub download_version {
  499. my ($config, $version) = @_;
  500. my $tmpdir = get_tmpdir($config);
  501. my $destdir = version_dir($config, $version);
  502. my $urldir = "$config->{download}{archive_url}/$version";
  503. print "Downloading version $version\n";
  504. foreach my $file (qw(sha256sums-signed-build.txt sha256sums-signed-build.txt.asc)) {
  505. if (getstore("$urldir/$file", "$tmpdir/$file") != 200) {
  506. exit_error "Error downloading $urldir/$file";
  507. }
  508. }
  509. if (system('gpg', '--no-default-keyring', '--keyring',
  510. "$FindBin::Bin/$config->{download}{gpg_keyring}", '--verify',
  511. "$tmpdir/sha256sums-signed-build.txt.asc",
  512. "$tmpdir/sha256sums-signed-build.txt")) {
  513. exit_error "Error checking gpg signature for version $version";
  514. }
  515. mkdir $destdir;
  516. move "$tmpdir/sha256sums-signed-build.txt.asc", "$destdir/sha256sums-signed-build.txt.asc";
  517. move "$tmpdir/sha256sums-signed-build.txt", "$destdir/sha256sums-signed-build.txt";
  518. my %sums = map { chomp; reverse split ' ', $_ }
  519. read_file "$destdir/sha256sums-signed-build.txt";
  520. my $martools = 'mar-tools-' . osname . '.zip';
  521. exit_error "Error downloading $urldir/$martools\n"
  522. unless getstore("$urldir/$martools", "$tmpdir/$martools") == 200;
  523. exit_error "Error downloading $urldir/$martools.asc\n"
  524. unless getstore("$urldir/$martools.asc", "$tmpdir/$martools.asc") == 200;
  525. if (system('gpg', '--no-default-keyring', '--keyring',
  526. "$FindBin::Bin/$config->{download}{gpg_keyring}", '--verify',
  527. "$tmpdir/$martools.asc", "$tmpdir/$martools")) {
  528. exit_error "Error checking gpg signature for $version/$martools";
  529. }
  530. exit_error "Wrong checksum for $version/$martools"
  531. unless $sums{$martools} eq sha256_hex(read_file("$tmpdir/$martools"));
  532. move "$tmpdir/$martools", "$destdir/$martools";
  533. move "$tmpdir/$martools.asc", "$destdir/$martools.asc";
  534. foreach my $file (sort grep { $_ =~ m/\.mar$/ } keys %sums) {
  535. print "Downloading $file\n";
  536. exit_error "Error downloading $urldir/$file\n"
  537. unless getstore("$urldir/$file", "$tmpdir/$file") == 200;
  538. exit_error "Wrong checksum for $file"
  539. unless $sums{$file} eq sha256_hex(read_file("$tmpdir/$file"));
  540. move "$tmpdir/$file", "$destdir/$file";
  541. }
  542. }
  543. sub download_missing_versions {
  544. my ($config, @channels) = @_;
  545. foreach my $channel (@channels) {
  546. exit_error "Unknown channel $channel"
  547. unless $config->{channels}{$channel};
  548. my $cversion = $config->{channels}{$channel};
  549. next unless $config->{versions}{$cversion}{incremental_from};
  550. foreach my $version (@{$config->{versions}{$cversion}{incremental_from}}) {
  551. next if -d version_dir($config, $version);
  552. download_version($config, $version);
  553. }
  554. }
  555. }
  556. sub check_update_responses {
  557. my ($config) = @_;
  558. exit_error "usage: $PROGRAM_NAME <base_url> [channels...]" unless @ARGV;
  559. my ($base_url, @channels) = @ARGV;
  560. foreach my $channel (@channels ? @channels : keys %{$config->{channels}}) {
  561. check_update_responses_channel($config, $base_url, $channel);
  562. }
  563. if (!@check_errors) {
  564. print "\n\nNo errors\n";
  565. return;
  566. }
  567. print "\n\nErrors list:\n";
  568. my $url = '';
  569. foreach my $error (@check_errors) {
  570. if ($url ne $error->{url}) {
  571. $url = $error->{url};
  572. print "$url\n";
  573. }
  574. print " $error->{step}",
  575. $error->{details} ? " - $error->{details}\n" : "\n";
  576. }
  577. }
  578. my %actions = (
  579. update_responses => sub {
  580. my ($config) = @_;
  581. my @channels = @ARGV ? @ARGV : keys %{$config->{channels}};
  582. foreach my $channel (@channels) {
  583. exit_error "Unknown channel $channel"
  584. unless $config->{channels}{$channel};
  585. $htdocsfiles{$channel} = { '.' => 1, '..' => 1 };
  586. }
  587. write_responses($config, @channels);
  588. write_htaccess($config, @channels);
  589. write_downloads_json($config, @channels);
  590. clean_htdocs(@channels);
  591. },
  592. gen_incrementals => sub {
  593. my ($config) = @_;
  594. foreach my $version (channel_to_version($config, @ARGV)) {
  595. extract_martools($config, $version);
  596. get_version_files($config, $version);
  597. create_incremental_mars_for_version($config, $version);
  598. }
  599. },
  600. download_missing_versions => sub {
  601. my ($config) = @_;
  602. my @channels = @ARGV ? @ARGV : keys %{$config->{channels}};
  603. download_missing_versions($config, @channels);
  604. },
  605. check_update_responses_deployement => \&check_update_responses,
  606. get_channel_version => sub {
  607. my ($config) = @_;
  608. exit_error "Wrong arguments" unless @ARGV == 1;
  609. exit_error "Unknown channel" unless $config->{channels}{$ARGV[0]};
  610. print $config->{channels}{$ARGV[0]}, "\n";
  611. },
  612. );
  613. my $action = fileparse($PROGRAM_NAME);
  614. exit_error "Unknown action $action" unless $actions{$action};
  615. $actions{$action}->($config);