update_responses 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655
  1. #!/usr/bin/perl -w
  2. use strict;
  3. use feature "state";
  4. use English;
  5. use FindBin;
  6. use YAML::XS qw(LoadFile);
  7. use File::Slurp;
  8. use File::Path qw(make_path);
  9. use Digest::SHA qw(sha256_hex);
  10. use XML::Writer;
  11. use Cwd;
  12. use File::Copy;
  13. use File::Temp;
  14. use File::Find;
  15. use POSIX qw(setlocale LC_ALL);
  16. use IO::CaptureOutput qw(capture_exec);
  17. use Parallel::ForkManager;
  18. use File::Basename;
  19. use XML::LibXML '1.70';
  20. use LWP::Simple;
  21. use JSON;
  22. # Set umask and locale to provide a consistent environment for MAR file
  23. # generation, etc.
  24. umask(0022);
  25. $ENV{"LC_ALL"} = "C";
  26. setlocale(LC_ALL, "C");
  27. my $htdocsdir = "$FindBin::Bin/htdocs";
  28. my $config = LoadFile("$FindBin::Bin/config.yml");
  29. my %htdocsfiles;
  30. my $releases_dir = $config->{releases_dir};
  31. $releases_dir = "$FindBin::Bin/$releases_dir" unless $releases_dir =~ m/^\//;
  32. my @check_errors;
  33. my $initPATH = $ENV{PATH};
  34. my $initLD_LIBRARY_PATH = $ENV{LD_LIBRARY_PATH};
  35. sub exit_error {
  36. print STDERR "Error: ", $_[0], "\n";
  37. chdir '/';
  38. exit (exists $_[1] ? $_[1] : 1);
  39. }
  40. sub get_tmpdir {
  41. my ($config) = @_;
  42. return File::Temp->newdir($config->{tmp_dir} ?
  43. (DIR => $config->{tmp_dir})
  44. : ());
  45. }
  46. sub build_targets_by_os {
  47. return ($_[0]) unless $config->{build_targets}{$_[0]};
  48. my $r = $config->{build_targets}{$_[0]};
  49. return ref $r eq 'ARRAY' ? @$r : ($r);
  50. }
  51. sub get_nbprocs {
  52. return $ENV{NUM_PROCS} if defined $ENV{NUM_PROCS};
  53. if (-f '/proc/cpuinfo') {
  54. return scalar grep { m/^processor\s+:\s/ } read_file '/proc/cpuinfo';
  55. }
  56. return 4;
  57. }
  58. sub write_htdocs {
  59. my ($channel, $file, $content) = @_;
  60. mkdir $htdocsdir unless -d $htdocsdir;
  61. mkdir "$htdocsdir/$channel" unless -d "$htdocsdir/$channel";
  62. write_file("$htdocsdir/$channel/$file", $content);
  63. $htdocsfiles{$channel}->{$file} = 1;
  64. }
  65. sub clean_htdocs {
  66. my (@channels) = @_;
  67. foreach my $channel (@channels) {
  68. opendir(my $d, "$htdocsdir/$channel");
  69. my @files = grep { ! $htdocsfiles{$channel}->{$_} } readdir $d;
  70. closedir $d;
  71. unlink map { "$htdocsdir/$channel/$_" } @files;
  72. }
  73. }
  74. sub get_sha512_hex_of_file {
  75. my ($file) = @_;
  76. my $sha = Digest::SHA->new("512");
  77. $sha->addfile($file);
  78. return $sha->hexdigest;
  79. }
  80. sub get_version_files {
  81. my ($config, $version) = @_;
  82. return if $config->{versions}{$version}{files};
  83. my $appname = $config->{appname_marfile};
  84. my $files = {};
  85. my $vdir = version_dir($config, $version);
  86. my $download_url = "$config->{download}{mars_url}/$version";
  87. opendir(my $d, $vdir) or exit_error "Error opening directory $vdir";
  88. foreach my $file (readdir $d) {
  89. next unless -f "$vdir/$file";
  90. if ($file =~ m/^$appname-([^-]+)-${version}_(.+)\.mar$/) {
  91. my ($os, $lang) = ($1, $2);
  92. $files->{$os}{$lang}{complete} = {
  93. type => 'complete',
  94. URL => "$download_url/$file",
  95. size => -s "$vdir/$file",
  96. hashFunction => 'SHA512',
  97. hashValue => get_sha512_hex_of_file("$vdir/$file"),
  98. };
  99. next;
  100. }
  101. if ($file =~ m/^$appname-([^-]+)-(.+)-${version}_(.+)\.incremental\.mar$/) {
  102. my ($os, $from_version, $lang) = ($1, $2, $3);
  103. $files->{$os}{$lang}{partial}{$from_version} = {
  104. type => 'partial',
  105. URL => "$download_url/$file",
  106. size => -s "$vdir/$file",
  107. hashFunction => 'SHA512',
  108. hashValue => get_sha512_hex_of_file("$vdir/$file"),
  109. }
  110. }
  111. }
  112. closedir $d;
  113. $config->{versions}{$version}{files} = $files;
  114. }
  115. sub get_version_downloads {
  116. my ($config, $version) = @_;
  117. my $downloads = {};
  118. my $vdir = version_dir($config, $version);
  119. my $download_url = "$config->{download}{bundles_url}/$version";
  120. opendir(my $d, $vdir) or exit_error "Error opening directory $vdir";
  121. foreach my $file (readdir $d) {
  122. next unless -f "$vdir/$file";
  123. my ($os, $lang);
  124. if ($file =~ m/^$config->{appname_bundle_osx}-$version-osx64_(.+).dmg$/) {
  125. ($os, $lang) = ('osx64', $1);
  126. } elsif ($file =~ m/^$config->{appname_bundle_linux}-(linux32|linux64)-${version}_(.+).tar.xz$/) {
  127. ($os, $lang) = ($1, $2);
  128. } elsif ($file =~ m/^$config->{appname_bundle_win64}-${version}_(.+).exe$/) {
  129. ($os, $lang) = ('win64', $1);
  130. } elsif ($file =~ m/^$config->{appname_bundle_win32}-${version}_(.+).exe$/) {
  131. ($os, $lang) = ('win32', $1);
  132. } else {
  133. next;
  134. }
  135. $downloads->{$os}{$lang} = {
  136. binary => "$download_url/$file",
  137. sig => "$download_url/$file.asc",
  138. };
  139. }
  140. closedir $d;
  141. $config->{versions}{$version}{downloads} = $downloads;
  142. }
  143. sub extract_mar {
  144. my ($mar_file, $dest_dir, $compression) = @_;
  145. my $old_cwd = getcwd;
  146. mkdir $dest_dir;
  147. chdir $dest_dir or exit_error "Cannot enter $dest_dir";
  148. my $res = system('mar', '-x', $mar_file);
  149. exit_error "Error extracting $mar_file" if $res;
  150. if ($compression ne 'bzip2' && $compression ne 'xz') {
  151. exit_error "Unknown compression format $compression";
  152. }
  153. my $compr_ext = $compression eq 'bzip2' ? 'bz2' : 'xz';
  154. my $compr_cmd = $compression eq 'bzip2' ? 'bunzip2' : 'unxz';
  155. my $uncompress_file = sub {
  156. return unless -f $File::Find::name;
  157. rename $File::Find::name, "$File::Find::name.$compr_ext";
  158. system($compr_cmd, "$File::Find::name.$compr_ext") == 0
  159. || exit_error "Error decompressing $File::Find::name";
  160. };
  161. find($uncompress_file, $dest_dir);
  162. my $manifest = -f 'updatev3.manifest' ? 'updatev3.manifest'
  163. : 'updatev2.manifest';
  164. my @lines = read_file($manifest) if -f $manifest;
  165. foreach my $line (@lines) {
  166. if ($line =~ m/^addsymlink "(.+)" "(.+)"$/) {
  167. exit_error "$mar_file: Could not create symlink $1 -> $2"
  168. unless symlink $2, $1;
  169. }
  170. }
  171. chdir $old_cwd;
  172. }
  173. sub mar_filename {
  174. my ($config, $appname, $version, $os, $lang) = @_;
  175. version_dir($config, $version) . "/$appname-$os-${version}_$lang.mar";
  176. }
  177. sub create_incremental_mar {
  178. my ($config, $pm, $from_version, $new_version, $os, $lang, $channel) = @_;
  179. my $appname = $config->{appname_marfile};
  180. my $mar_file = "$appname-$os-${from_version}-${new_version}_$lang.incremental.mar";
  181. my $mar_file_path = version_dir($config, $new_version) . '/' . $mar_file;
  182. if ($ENV{MAR_SKIP_EXISTING} && -f $mar_file_path) {
  183. print "Skipping $mar_file\n";
  184. return;
  185. }
  186. print "Starting $mar_file\n";
  187. my $download_url = "$config->{download}{mars_url}/$new_version";
  188. my $finished_file = sub {
  189. exit_error "Error creating $mar_file" unless $_[1] == 0;
  190. print "Finished $mar_file\n";
  191. $config->{versions}{$new_version}{files}{$os}{$lang}{partial}{$from_version} = {
  192. type => 'partial',
  193. URL => "$download_url/$mar_file",
  194. size => -s $mar_file_path,
  195. hashFunction => 'SHA512',
  196. hashValue => get_sha512_hex_of_file($mar_file_path),
  197. };
  198. };
  199. return if $pm->start($finished_file);
  200. my $tmpdir = get_tmpdir($config);
  201. my $mar_c_from = get_config($config, $from_version, $os, 'mar_compression');
  202. my $mar_c_new = get_config($config, $new_version, $os, 'mar_compression');
  203. extract_mar(mar_filename($config, $appname, $from_version, $os, $lang),
  204. "$tmpdir/A", $mar_c_from);
  205. extract_mar(mar_filename($config, $appname, $new_version, $os, $lang),
  206. "$tmpdir/B", $mar_c_new);
  207. # bug 26054: make sure previous macOS version is code signed
  208. if (!$ENV{NO_CODESIGNATURE} && ($os eq 'osx64')
  209. && ! -f "$tmpdir/A/Contents/_CodeSignature/CodeResources") {
  210. exit_error "Missing code signature in $from_version while creating $mar_file";
  211. }
  212. if ($ENV{CHECK_CODESIGNATURE_EXISTS}) {
  213. unless (-f "$tmpdir/A/Contents/_CodeSignature/CodeResources"
  214. && -f "$tmpdir/B/Contents/_CodeSignature/CodeResources") {
  215. exit_error "Missing code signature while creating $mar_file";
  216. }
  217. }
  218. local $ENV{MOZ_PRODUCT_VERSION} = $new_version;
  219. local $ENV{MAR_CHANNEL_ID} = "torbrowser-torproject-$channel";
  220. local $ENV{TMPDIR} = $tmpdir;
  221. my ($out, $err, $success) = capture_exec('make_incremental_update.sh',
  222. $mar_file_path, "$tmpdir/A", "$tmpdir/B");
  223. if (!$success) {
  224. unlink $mar_file_path if -f $mar_file_path;
  225. exit_error "making incremental mar:\n" . $err;
  226. }
  227. $pm->finish;
  228. }
  229. sub create_incremental_mars_for_version {
  230. my ($config, $version, $channel) = @_;
  231. my $pm = Parallel::ForkManager->new(get_nbprocs);
  232. $pm->run_on_finish(sub { $_[2]->(@_) });
  233. my $v = $config->{versions}{$version};
  234. foreach my $from_version (@{$v->{incremental_from}}) {
  235. $config->{versions}{$from_version} //= {};
  236. get_version_files($config, $from_version);
  237. my $from_v = $config->{versions}{$from_version};
  238. foreach my $os (keys %{$v->{files}}) {
  239. foreach my $lang (keys %{$v->{files}{$os}}) {
  240. next unless defined $from_v->{files}{$os}{$lang}{complete};
  241. create_incremental_mar($config, $pm, $from_version, $version, $os, $lang, $channel);
  242. }
  243. }
  244. }
  245. $pm->wait_all_children;
  246. }
  247. sub get_config {
  248. my ($config, $version, $os, $name) = @_;
  249. return $config->{versions}{$version}{$os}{$name}
  250. // $config->{versions}{$version}{$name}
  251. // $config->{$name};
  252. }
  253. sub version_dir {
  254. my ($config, $version) = @_;
  255. return get_config($config, $version, 'any', 'releases_dir') . "/$version";
  256. }
  257. sub channel_to_version {
  258. my ($config, @channels) = @_;
  259. return values %{$config->{channels}} unless @channels;
  260. foreach my $channel (@channels) {
  261. exit_error "Unknown channel $channel"
  262. unless $config->{channels}{$channel};
  263. }
  264. return map { $config->{channels}{$_} } @channels;
  265. }
  266. sub get_buildinfos {
  267. my ($config, $version) = @_;
  268. return if exists $config->{versions}{$version}{buildID};
  269. extract_martools($config, $version);
  270. my $files = $config->{versions}{$version}{files};
  271. foreach my $os (keys %$files) {
  272. foreach my $lang (keys %{$files->{$os}}) {
  273. next unless $files->{$os}{$lang}{complete};
  274. my $tmpdir = get_tmpdir($config);
  275. my $mar_compression = get_config($config, $version, $os, 'mar_compression');
  276. extract_mar(
  277. mar_filename($config, $config->{appname_marfile}, $version, $os, $lang),
  278. "$tmpdir",
  279. $mar_compression);
  280. my $appfile = "$tmpdir/application.ini" if -f "$tmpdir/application.ini";
  281. $appfile = "$tmpdir/Contents/Resources/application.ini"
  282. if -f "$tmpdir/Contents/Resources/application.ini";
  283. exit_error "Could not find application.ini" unless $appfile;
  284. foreach my $line (read_file($appfile)) {
  285. if ($line =~ m/^BuildID=(.*)$/) {
  286. $config->{versions}{$version}{buildID} = $1;
  287. return;
  288. }
  289. }
  290. exit_error "Could not extract buildID from application.ini";
  291. }
  292. }
  293. }
  294. sub get_response {
  295. my ($config, $version, $os, @patches) = @_;
  296. my $res;
  297. my $writer = XML::Writer->new(OUTPUT => \$res, ENCODING => 'UTF-8');
  298. $writer->xmlDecl;
  299. $writer->startTag('updates');
  300. if (get_config($config, $version, $os, 'unsupported')) {
  301. $writer->startTag('update',
  302. unsupported => 'true',
  303. detailsURL => get_config($config, $version, $os, 'detailsURL'),
  304. );
  305. goto CLOSETAGS;
  306. }
  307. my $minversion = get_config($config, $version, $os, 'minSupportedOSVersion');
  308. my $mininstruc = get_config($config, $version, $os, 'minSupportedInstructionSet');
  309. $writer->startTag('update',
  310. type => 'minor',
  311. displayVersion => $version,
  312. appVersion => $version,
  313. platformVersion => get_config($config, $version, $os, 'platformVersion'),
  314. buildID => get_config($config, $version, $os, 'buildID'),
  315. detailsURL => get_config($config, $version, $os, 'detailsURL'),
  316. actions => 'showURL',
  317. openURL => get_config($config, $version, $os, 'detailsURL'),
  318. defined $minversion ? ( minSupportedOSVersion => $minversion ) : (),
  319. defined $mininstruc ? ( minSupportedInstructionSet => $mininstruc ) : (),
  320. );
  321. foreach my $patch (@patches) {
  322. my @sorted_patch = map { $_ => $patch->{$_} } sort keys %$patch;
  323. $writer->startTag('patch', @sorted_patch);
  324. $writer->endTag('patch');
  325. }
  326. CLOSETAGS:
  327. $writer->endTag('update');
  328. $writer->endTag('updates');
  329. $writer->end;
  330. return $res;
  331. }
  332. sub write_responses {
  333. my ($config, @channels) = @_;
  334. @channels = keys %{$config->{channels}} unless @channels;
  335. foreach my $channel (@channels) {
  336. my $version = $config->{channels}{$channel};
  337. get_version_files($config, $version);
  338. get_buildinfos($config, $version);
  339. my $files = $config->{versions}{$version}{files};
  340. my $migrate_archs = $config->{versions}{$version}{migrate_archs} // {};
  341. foreach my $old_os (keys %$migrate_archs) {
  342. my $new_os = $migrate_archs->{$old_os};
  343. foreach my $lang (keys %{$files->{$new_os}}) {
  344. $files->{$old_os}{$lang}{complete} =
  345. $files->{$new_os}{$lang}{complete};
  346. }
  347. }
  348. foreach my $os (keys %$files) {
  349. foreach my $lang (keys %{$files->{$os}}) {
  350. my $resp = get_response($config, $version, $os,
  351. $files->{$os}{$lang}{complete});
  352. write_htdocs($channel, "$version-$os-$lang.xml", $resp);
  353. foreach my $from_version (keys %{$files->{$os}{$lang}{partial}}) {
  354. $resp = get_response($config, $version, $os,
  355. $files->{$os}{$lang}{complete},
  356. $files->{$os}{$lang}{partial}{$from_version});
  357. write_htdocs($channel, "$from_version-$version-$os-$lang.xml", $resp);
  358. }
  359. }
  360. }
  361. write_htdocs($channel, 'no-update.xml',
  362. '<?xml version="1.0" encoding="UTF-8"?>'
  363. . "\n<updates></updates>\n");
  364. }
  365. }
  366. sub write_htaccess {
  367. my ($config, @channels) = @_;
  368. @channels = keys %{$config->{channels}} unless @channels;
  369. my $flags = "[last]";
  370. foreach my $channel (@channels) {
  371. my $htaccess = "RewriteEngine On\n";
  372. $htaccess .= $config->{htaccess_rewrite_rules}{$channel} // '';
  373. my $version = $config->{channels}{$channel};
  374. my $migrate_langs = $config->{versions}{$version}{migrate_langs} // {};
  375. my $files = $config->{versions}{$version}{files};
  376. $htaccess .= "RewriteRule ^[^\/]+/$version/ no-update.xml $flags\n";
  377. foreach my $os (sort keys %$files) {
  378. foreach my $bt (build_targets_by_os($os)) {
  379. foreach my $lang (sort keys %{$files->{$os}}) {
  380. foreach my $from_version (sort keys %{$files->{$os}{$lang}{partial}}) {
  381. $htaccess .= "RewriteRule ^$bt/$from_version/$lang "
  382. . "$from_version-$version-$os-$lang.xml $flags\n";
  383. }
  384. $htaccess .= "RewriteRule ^$bt/[^\/]+/$lang "
  385. . "$version-$os-$lang.xml $flags\n";
  386. }
  387. foreach my $lang (sort keys %$migrate_langs) {
  388. $htaccess .= "RewriteRule ^$bt/[^\/]+/$lang "
  389. . "$version-$os-$migrate_langs->{$lang}.xml $flags\n";
  390. }
  391. $htaccess .= "RewriteRule ^$bt/ $version-$os-en-US.xml $flags\n";
  392. }
  393. }
  394. write_htdocs($channel, '.htaccess', $htaccess);
  395. }
  396. }
  397. sub write_downloads_json {
  398. my ($config, @channels) = @_;
  399. return unless $config->{create_downloads_json};
  400. @channels = keys %{$config->{channels}} unless @channels;
  401. foreach my $channel (@channels) {
  402. my $version = $config->{channels}{$channel};
  403. my $data = {
  404. version => $version,
  405. downloads => get_version_downloads($config, $version),
  406. };
  407. write_htdocs($channel, 'downloads.json',
  408. JSON->new->utf8->canonical->encode($data));
  409. }
  410. }
  411. sub marzip_path {
  412. my ($config, $version) = @_;
  413. for my $osname (qw/linux64 linux32 mac64 win64 win32/) {
  414. my $marzip = version_dir($config, $version) . "/mar-tools-$osname.zip";
  415. return $marzip if -f $marzip;
  416. }
  417. exit_error 'Could not find mar-tools';
  418. }
  419. my $martools_tmpdir;
  420. sub extract_martools {
  421. my ($config, $version) = @_;
  422. my $marzip = marzip_path($config, $version);
  423. $martools_tmpdir = get_tmpdir($config);
  424. my $old_cwd = getcwd;
  425. chdir $martools_tmpdir;
  426. my (undef, undef, $success) = capture_exec('unzip', $marzip);
  427. chdir $old_cwd;
  428. exit_error "Error extracting $marzip" unless $success;
  429. $ENV{PATH} = "$martools_tmpdir/mar-tools:$initPATH";
  430. if ($initLD_LIBRARY_PATH) {
  431. $ENV{LD_LIBRARY_PATH} = "$initLD_LIBRARY_PATH:$martools_tmpdir/mar-tools";
  432. } else {
  433. $ENV{LD_LIBRARY_PATH} = "$martools_tmpdir/mar-tools";
  434. }
  435. }
  436. sub log_step {
  437. my ($url, $step, $status, $details) = @_;
  438. state $u;
  439. if (!defined $u || $url ne $u) {
  440. print "\n" if $u;
  441. print "$url\n";
  442. $u = $url;
  443. }
  444. print ' ', $step, $status ? ': OK' : ': ERROR',
  445. $details ? " - $details\n" : "\n";
  446. return if $status;
  447. push @check_errors, { url => $url, step => $step, details => $details };
  448. }
  449. sub get_remote_xml {
  450. my ($url) = @_;
  451. my $content = get $url;
  452. log_step($url, 'get', defined $content);
  453. return undef unless defined $content;
  454. my $dom = eval { XML::LibXML->load_xml(string => $content) };
  455. log_step($url, 'parse_xml', defined $dom, $@);
  456. return $dom;
  457. }
  458. sub check_get_version {
  459. my ($dom) = @_;
  460. my @updates = $dom->documentElement()->getChildrenByLocalName('update');
  461. return undef unless @updates;
  462. return $updates[0]->getAttribute('appVersion');
  463. }
  464. sub check_no_update {
  465. my ($dom) = @_;
  466. my @updates = $dom->documentElement()->getChildrenByLocalName('update');
  467. return @updates == 0;
  468. }
  469. sub check_has_incremental {
  470. my ($dom) = @_;
  471. my @updates = $dom->documentElement()->getChildrenByLocalName('update');
  472. return undef unless @updates;
  473. my @patches = $updates[0]->getChildrenByLocalName('patch');
  474. foreach my $patch (@patches) {
  475. return 1 if $patch->getAttribute('type') eq 'partial';
  476. }
  477. return undef;
  478. }
  479. sub build_targets_list {
  480. map { ref $_ eq 'ARRAY' ? @$_ : $_ } values %{$config->{build_targets}};
  481. }
  482. sub check_update_responses_channel {
  483. my ($config, $base_url, $channel) = @_;
  484. my $channel_version = $config->{channels}{$channel};
  485. foreach my $build_target (build_targets_list()) {
  486. foreach my $lang (qw(en-US de)) {
  487. my $url = "$base_url/$channel/$build_target/1.0/$lang";
  488. my $dom = get_remote_xml($url);
  489. if ($dom) {
  490. my $version = check_get_version($dom);
  491. log_step($url, 'version', $version eq $channel_version,
  492. "expected: $channel_version received: $version");
  493. }
  494. $url = "$base_url/$channel/$build_target/$channel_version/$lang";
  495. $dom = get_remote_xml($url);
  496. log_step($url, 'no_update', check_no_update($dom)) if $dom;
  497. my @inc = @{$config->{versions}{$channel_version}{incremental_from}}
  498. if $config->{versions}{$channel_version}{incremental_from};
  499. foreach my $inc_from (@inc) {
  500. my $url = "$base_url/$channel/$build_target/$inc_from/$lang";
  501. $dom = get_remote_xml($url);
  502. next unless $dom;
  503. my $version = check_get_version($dom);
  504. log_step($url, 'version', $version eq $channel_version,
  505. "expected: $channel_version received: $version");
  506. log_step($url, 'has_incremental', check_has_incremental($dom));
  507. }
  508. }
  509. }
  510. }
  511. sub download_version {
  512. my ($config, $version) = @_;
  513. my $tmpdir = get_tmpdir($config);
  514. my $destdir = version_dir($config, $version);
  515. my $urldir = "$config->{download}{archive_url}/$version";
  516. print "Downloading version $version\n";
  517. foreach my $file (qw(sha256sums-signed-build.txt sha256sums-signed-build.txt.asc)) {
  518. if (getstore("$urldir/$file", "$tmpdir/$file") != 200) {
  519. exit_error "Error downloading $urldir/$file";
  520. }
  521. }
  522. if (system('gpg', '--no-default-keyring', '--keyring',
  523. "$FindBin::Bin/$config->{download}{gpg_keyring}", '--verify',
  524. "$tmpdir/sha256sums-signed-build.txt.asc",
  525. "$tmpdir/sha256sums-signed-build.txt")) {
  526. exit_error "Error checking gpg signature for version $version";
  527. }
  528. make_path $destdir;
  529. move "$tmpdir/sha256sums-signed-build.txt.asc", "$destdir/sha256sums-signed-build.txt.asc";
  530. move "$tmpdir/sha256sums-signed-build.txt", "$destdir/sha256sums-signed-build.txt";
  531. my %sums = map { chomp; reverse split ' ', $_ }
  532. read_file "$destdir/sha256sums-signed-build.txt";
  533. foreach my $file (sort grep { $_ =~ m/\.mar$/ } keys %sums) {
  534. print "Downloading $file\n";
  535. exit_error "Error downloading $urldir/$file\n"
  536. unless getstore("$urldir/$file", "$tmpdir/$file") == 200;
  537. exit_error "Wrong checksum for $file"
  538. unless $sums{$file} eq sha256_hex(read_file("$tmpdir/$file"));
  539. move "$tmpdir/$file", "$destdir/$file";
  540. }
  541. }
  542. sub download_missing_versions {
  543. my ($config, @channels) = @_;
  544. foreach my $channel (@channels) {
  545. exit_error "Unknown channel $channel"
  546. unless $config->{channels}{$channel};
  547. my $cversion = $config->{channels}{$channel};
  548. next unless $config->{versions}{$cversion}{incremental_from};
  549. foreach my $version (@{$config->{versions}{$cversion}{incremental_from}}) {
  550. next if -d version_dir($config, $version);
  551. download_version($config, $version);
  552. }
  553. }
  554. }
  555. sub check_update_responses {
  556. my ($config) = @_;
  557. exit_error "usage: $PROGRAM_NAME <base_url> [channels...]" unless @ARGV;
  558. my ($base_url, @channels) = @ARGV;
  559. foreach my $channel (@channels ? @channels : keys %{$config->{channels}}) {
  560. check_update_responses_channel($config, $base_url, $channel);
  561. }
  562. if (!@check_errors) {
  563. print "\n\nNo errors\n";
  564. return;
  565. }
  566. print "\n\nErrors list:\n";
  567. my $url = '';
  568. foreach my $error (@check_errors) {
  569. if ($url ne $error->{url}) {
  570. $url = $error->{url};
  571. print "$url\n";
  572. }
  573. print " $error->{step}",
  574. $error->{details} ? " - $error->{details}\n" : "\n";
  575. }
  576. }
  577. my %actions = (
  578. update_responses => sub {
  579. my ($config) = @_;
  580. my @channels = @ARGV ? @ARGV : keys %{$config->{channels}};
  581. foreach my $channel (@channels) {
  582. exit_error "Unknown channel $channel"
  583. unless $config->{channels}{$channel};
  584. $htdocsfiles{$channel} = { '.' => 1, '..' => 1 };
  585. }
  586. write_responses($config, @channels);
  587. write_htaccess($config, @channels);
  588. write_downloads_json($config, @channels);
  589. clean_htdocs(@channels);
  590. },
  591. gen_incrementals => sub {
  592. my ($config) = @_;
  593. foreach my $channel (@ARGV) {
  594. my ($version) = channel_to_version($config, $channel);
  595. extract_martools($config, $version);
  596. get_version_files($config, $version);
  597. create_incremental_mars_for_version($config, $version, $channel);
  598. }
  599. },
  600. download_missing_versions => sub {
  601. my ($config) = @_;
  602. my @channels = @ARGV ? @ARGV : keys %{$config->{channels}};
  603. download_missing_versions($config, @channels);
  604. },
  605. check_update_responses_deployement => \&check_update_responses,
  606. get_channel_version => sub {
  607. my ($config) = @_;
  608. exit_error "Wrong arguments" unless @ARGV == 1;
  609. exit_error "Unknown channel" unless $config->{channels}{$ARGV[0]};
  610. print $config->{channels}{$ARGV[0]}, "\n";
  611. },
  612. );
  613. my $action = fileparse($PROGRAM_NAME);
  614. exit_error "Unknown action $action" unless $actions{$action};
  615. $actions{$action}->($config);