Rakefile 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415
  1. # -*- coding: utf-8 -*-
  2. # -*- mode: ruby -*-
  3. # vi: set ft=ruby :
  4. #
  5. # Tails: The Amnesic Incognito Live System
  6. # Copyright © 2012 Tails developers <tails@boum.org>
  7. #
  8. # This program is free software: you can redistribute it and/or modify
  9. # it under the terms of the GNU General Public License as published by
  10. # the Free Software Foundation, either version 3 of the License, or
  11. # (at your option) any later version.
  12. #
  13. # This program is distributed in the hope that it will be useful,
  14. # but WITHOUT ANY WARRANTY; without even the implied warranty of
  15. # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16. # GNU General Public License for more details.
  17. #
  18. # You should have received a copy of the GNU General Public License
  19. # along with this program. If not, see <http://www.gnu.org/licenses/>.
  20. require 'open3'
  21. require 'rbconfig'
  22. require 'uri'
  23. require_relative 'vagrant/lib/tails_build_settings'
  24. # Path to the directory which holds our Vagrantfile
  25. VAGRANT_PATH = File.expand_path('../vagrant', __FILE__)
  26. # Branches that are considered 'stable' (used to select SquashFS compression)
  27. STABLE_BRANCH_NAMES = ['stable', 'testing']
  28. # Environment variables that will be exported to the build script
  29. EXPORTED_VARIABLES = ['http_proxy', 'MKSQUASHFS_OPTIONS', 'TAILS_RAM_BUILD', 'TAILS_CLEAN_BUILD']
  30. # Let's save the http_proxy set before playing with it
  31. EXTERNAL_HTTP_PROXY = ENV['http_proxy']
  32. # In-VM proxy URL
  33. INTERNAL_HTTP_PROXY = "http://#{VIRTUAL_MACHINE_HOSTNAME}:3142"
  34. class VagrantCommandError < StandardError
  35. end
  36. # Runs the vagrant command, letting stdout/stderr through. Throws an
  37. # exception unless the vagrant command succeeds.
  38. def run_vagrant(*args)
  39. Process.wait Kernel.spawn('vagrant', *args, :chdir => './vagrant')
  40. if $?.exitstatus != 0
  41. raise(VagrantCommandError, "'vagrant #{args}' command failed: " +
  42. "#{$?.exitstatus}")
  43. end
  44. end
  45. # Runs the vagrant command, not letting stdout/stderr through, and
  46. # returns [stdout, stderr, Preocess:Status].
  47. def capture_vagrant(*args)
  48. stdout, stderr, proc_status =
  49. Open3.capture3('vagrant', *args, :chdir => './vagrant')
  50. if proc_status.exitstatus != 0
  51. raise(VagrantCommandError, "'vagrant #{args}' command failed: " +
  52. "#{proc_status.exitstatus}")
  53. end
  54. return stdout, stderr
  55. end
  56. def vagrant_ssh_config(key)
  57. # Cache results
  58. if $vagrant_ssh_config.nil?
  59. $vagrant_ssh_config = capture_vagrant('ssh-config').first.split("\n") \
  60. .map { |line| line.strip.split(/\s+/, 2) } .to_h
  61. # The path in the ssh-config output is quoted, which is not what
  62. # is expected outside of a shell, so let's get rid of the quotes.
  63. $vagrant_ssh_config['IdentityFile'].gsub!(/^"|"$/, '')
  64. end
  65. $vagrant_ssh_config[key]
  66. end
  67. def current_vm_cpus
  68. capture_vagrant('ssh', '-c', 'grep -c "^processor\s*:" /proc/cpuinfo').first.chomp.to_i
  69. end
  70. def vm_state
  71. out, _ = capture_vagrant('status')
  72. status_line = out.split("\n")[2]
  73. if status_line['not created']
  74. return :not_created
  75. elsif status_line['shutoff']
  76. return :poweroff
  77. elsif status_line['running']
  78. return :running
  79. else
  80. raise "could not determine VM state"
  81. end
  82. end
  83. def enough_free_host_memory_for_ram_build?
  84. return false unless RbConfig::CONFIG['host_os'] =~ /linux/i
  85. begin
  86. usable_free_mem = `free`.split[16].to_i
  87. usable_free_mem > VM_MEMORY_FOR_RAM_BUILDS * 1024
  88. rescue
  89. false
  90. end
  91. end
  92. def free_vm_memory
  93. capture_vagrant('ssh', '-c', 'free').first.chomp.split[16].to_i
  94. end
  95. def enough_free_vm_memory_for_ram_build?
  96. free_vm_memory > BUILD_SPACE_REQUIREMENT * 1024
  97. end
  98. def enough_free_memory_for_ram_build?
  99. if vm_state == :running
  100. enough_free_vm_memory_for_ram_build?
  101. else
  102. enough_free_host_memory_for_ram_build?
  103. end
  104. end
  105. def is_release?
  106. detached_head = `git symbolic-ref HEAD` == ""
  107. `git describe --tags --exact-match HEAD 2>/dev/null`
  108. is_tag = $?.success?
  109. detached_head && is_tag
  110. end
  111. def system_cpus
  112. return nil unless RbConfig::CONFIG['host_os'] =~ /linux/i
  113. begin
  114. File.read('/proc/cpuinfo').scan(/^processor\s+:/).count
  115. rescue
  116. nil
  117. end
  118. end
  119. task :parse_build_options do
  120. options = []
  121. # Default to in-memory builds if there is enough RAM available
  122. options << 'ram' if enough_free_memory_for_ram_build?
  123. # Default to build using the in-VM proxy
  124. options << 'vmproxy'
  125. # Default to fast compression on development branches
  126. options << 'gzipcomp' unless is_release?
  127. # Default to the number of system CPUs when we can figure it out
  128. cpus = system_cpus
  129. options << "cpus=#{cpus}" if cpus
  130. options += ENV['TAILS_BUILD_OPTIONS'].split if ENV['TAILS_BUILD_OPTIONS']
  131. # Make sure release builds are clean
  132. options << 'cleanall' if is_release?
  133. options.uniq.each do |opt|
  134. case opt
  135. # Memory build settings
  136. when 'ram'
  137. ENV['TAILS_RAM_BUILD'] = '1'
  138. when 'noram'
  139. ENV['TAILS_RAM_BUILD'] = nil
  140. # Bootstrap cache settings
  141. # HTTP proxy settings
  142. when 'extproxy'
  143. abort "No HTTP proxy set, but one is required by TAILS_BUILD_OPTIONS. Aborting." unless EXTERNAL_HTTP_PROXY
  144. ENV['http_proxy'] = EXTERNAL_HTTP_PROXY
  145. when 'vmproxy'
  146. ENV['http_proxy'] = INTERNAL_HTTP_PROXY
  147. when 'noproxy'
  148. ENV['http_proxy'] = nil
  149. # SquashFS compression settings
  150. when 'gzipcomp'
  151. ENV['MKSQUASHFS_OPTIONS'] = '-comp gzip'
  152. when 'defaultcomp'
  153. ENV['MKSQUASHFS_OPTIONS'] = nil
  154. # Clean-up settings
  155. when 'cleanall'
  156. ENV['TAILS_CLEAN_BUILD'] = '1'
  157. # Virtual CPUs settings
  158. when /cpus=(\d+)/
  159. ENV['TAILS_BUILD_CPUS'] = $1
  160. # Git settings
  161. when 'ignorechanges'
  162. ENV['TAILS_BUILD_IGNORE_CHANGES'] = '1'
  163. when 'noprovision'
  164. ENV['TAILS_NO_AUTO_PROVISION'] = '1'
  165. else
  166. raise "Unknown Tails build option '#{opt}'"
  167. end
  168. end
  169. end
  170. task :ensure_clean_repository do
  171. git_status = `git status --porcelain`
  172. unless git_status.empty?
  173. if ENV['TAILS_BUILD_IGNORE_CHANGES']
  174. $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
  175. You have uncommitted changes in the Git repository. They will
  176. be ignored for the upcoming build:
  177. #{git_status}
  178. END_OF_MESSAGE
  179. else
  180. $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
  181. You have uncommitted changes in the Git repository. Due to limitations
  182. of the build system, you need to commit them before building Tails:
  183. #{git_status}
  184. If you don't care about those changes and want to build Tails nonetheless,
  185. please add `ignorechanges` to the TAILS_BUILD_OPTIONS environment
  186. variable.
  187. END_OF_MESSAGE
  188. abort 'Uncommitted changes. Aborting.'
  189. end
  190. end
  191. end
  192. def list_artifacts
  193. user = vagrant_ssh_config('User')
  194. stdout = capture_vagrant('ssh', '-c', "find '/home/#{user}/' -maxdepth 1 " +
  195. "-name 'tails-*.iso*'").first
  196. stdout.split("\n")
  197. rescue VagrantCommandError
  198. return Array.new
  199. end
  200. def remove_artifacts
  201. list_artifacts.each do |artifact|
  202. run_vagrant('ssh', '-c', "sudo rm -f '#{artifact}'")
  203. end
  204. end
  205. desc "Make sure the vagrant user's home directory has no undesired artifacts"
  206. task :ensure_clean_home_directory => ['vm:up'] do
  207. remove_artifacts
  208. end
  209. task :validate_http_proxy do
  210. if ENV['http_proxy']
  211. proxy_host = URI.parse(ENV['http_proxy']).host
  212. if proxy_host.nil?
  213. ENV['http_proxy'] = nil
  214. $stderr.puts "Ignoring invalid HTTP proxy."
  215. return
  216. end
  217. if ['localhost', '[::1]'].include?(proxy_host) || proxy_host.start_with?('127.0.0.')
  218. abort 'Using an HTTP proxy listening on the loopback is doomed to fail. Aborting.'
  219. end
  220. $stderr.puts "Using HTTP proxy: #{ENV['http_proxy']}"
  221. else
  222. $stderr.puts "No HTTP proxy set."
  223. end
  224. end
  225. desc 'Build Tails'
  226. task :build => ['parse_build_options', 'ensure_clean_repository', 'ensure_clean_home_directory', 'validate_http_proxy', 'vm:up'] do
  227. if ENV['TAILS_RAM_BUILD'] && not(enough_free_memory_for_ram_build?)
  228. $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
  229. The virtual machine is not currently set with enough memory to
  230. perform an in-memory build. Either remove the `ram` option from
  231. the TAILS_BUILD_OPTIONS environment variable, or shut the
  232. virtual machine down using `rake vm:halt` before trying again.
  233. END_OF_MESSAGE
  234. abort 'Not enough memory for the virtual machine to run an in-memory build. Aborting.'
  235. end
  236. if ENV['TAILS_BUILD_CPUS'] && current_vm_cpus != ENV['TAILS_BUILD_CPUS'].to_i
  237. $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
  238. The virtual machine is currently running with #{current_vm_cpus}
  239. virtual CPU(s). In order to change that number, you need to
  240. stop the VM first, using `rake vm:halt`. Otherwise, please
  241. adjust the `cpus` options accordingly.
  242. END_OF_MESSAGE
  243. abort 'The virtual machine needs to be reloaded to change the number of CPUs. Aborting.'
  244. end
  245. # Let's make sure that, unless you know what you are doing and
  246. # explicitly disable this, we always provision in order to ensure
  247. # a valid, up-to-date build system.
  248. run_vagrant('provision') unless ENV['TAILS_NO_AUTO_PROVISION']
  249. exported_env = EXPORTED_VARIABLES.select { |k| ENV[k] }.
  250. collect { |k| "#{k}='#{ENV[k]}'" }.join(' ')
  251. run_vagrant('ssh', '-c', "#{exported_env} build-tails")
  252. artifacts = list_artifacts
  253. raise 'No build artifacts was found!' if artifacts.empty?
  254. user = vagrant_ssh_config('User')
  255. hostname = vagrant_ssh_config('HostName')
  256. key_file = vagrant_ssh_config('IdentityFile')
  257. $stderr.puts "Retrieving artifacts from Vagrant build box."
  258. artifacts.each do |artifact|
  259. run_vagrant('ssh', '-c', "sudo chown #{user} '#{artifact}'")
  260. Process.wait(
  261. Kernel.spawn(
  262. 'scp',
  263. '-i', key_file,
  264. # We need this since the user will not necessarily have a
  265. # known_hosts entry. It is safe since an attacker must
  266. # compromise libvirt's network config or the user running the
  267. # command to modify the #{hostname} below.
  268. '-o', 'StrictHostKeyChecking=no',
  269. "#{user}@#{hostname}:#{artifact}", '.'
  270. )
  271. )
  272. raise "Failed to fetch artifact '#{artifact}'" unless $?.success?
  273. end
  274. remove_artifacts
  275. end
  276. namespace :vm do
  277. desc 'Start the build virtual machine'
  278. task :up => ['parse_build_options', 'validate_http_proxy'] do
  279. case vm_state
  280. when :not_created
  281. # Do not use non-existant in-VM proxy to download the basebox
  282. if ENV['http_proxy'] == INTERNAL_HTTP_PROXY
  283. ENV['http_proxy'] = nil
  284. restore_internal_proxy = true
  285. end
  286. $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
  287. This is the first time that the Tails builder virtual machine is
  288. started. The virtual machine template is about 300 MB to download,
  289. so the process might take some time.
  290. Please remember to shut the virtual machine down once your work on
  291. Tails is done:
  292. $ rake vm:halt
  293. END_OF_MESSAGE
  294. when :poweroff
  295. $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
  296. Starting Tails builder virtual machine. This might take a short while.
  297. Please remember to shut it down once your work on Tails is done:
  298. $ rake vm:halt
  299. END_OF_MESSAGE
  300. end
  301. run_vagrant('up')
  302. ENV['http_proxy'] = INTERNAL_HTTP_PROXY if restore_internal_proxy
  303. end
  304. desc 'SSH into the builder VM'
  305. task :ssh do
  306. run_vagrant('ssh')
  307. end
  308. desc 'Stop the build virtual machine'
  309. task :halt do
  310. run_vagrant('halt')
  311. end
  312. desc 'Re-run virtual machine setup'
  313. task :provision => ['parse_build_options', 'validate_http_proxy'] do
  314. run_vagrant('provision')
  315. end
  316. desc 'Destroy build virtual machine (clean up all files)'
  317. task :destroy do
  318. run_vagrant('destroy', '--force')
  319. end
  320. end
  321. namespace :basebox do
  322. desc 'Generate a new base box'
  323. task :create do
  324. box_dir = VAGRANT_PATH + '/definitions/tails-builder'
  325. Dir.chdir(box_dir) do
  326. `./generate-tails-builder-box.sh`
  327. raise 'Base box generation failed!' unless $?.success?
  328. end
  329. box = Dir.glob("#{box_dir}/*.box").sort_by {|f| File.mtime(f) } .last
  330. $stderr.puts <<-END_OF_MESSAGE.gsub(/^ /, '')
  331. You have successfully generated a new Vagrant base box:
  332. #{box}
  333. To install the new base box, please run:
  334. $ vagrant box add #{box}
  335. To actually make Tails build using this base box, the `config.vm.box` key
  336. in `vagrant/Vagrantfile` has to be updated. Please check the documentation
  337. for details.
  338. END_OF_MESSAGE
  339. end
  340. end