parallel.scm 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468
  1. ;;; GNU Guix --- Functional package management for GNU
  2. ;;; Copyright © 2013, 2014, 2020 Eric Bavier <bavier@posteo.net>
  3. ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org>
  4. ;;; Copyright © 2015, 2016, 2017, 2018, 2020, 2021, 2022 Efraim Flashner <efraim@flashner.co.il>
  5. ;;; Copyright © 2016 Pjotr Prins <pjotr.guix@thebird.nl>
  6. ;;; Copyright © 2016 Andreas Enge <andreas@enge.fr>
  7. ;;; Copyright © 2016, 2020, 2021 Ricardo Wurmus <rekado@elephly.net>
  8. ;;; Copyright © 2016 Ben Woodcroft <donttrustben@gmail.com>
  9. ;;; Copyright © 2017, 2018 Rutger Helling <rhelling@mykolab.com>
  10. ;;; Copyright © 2018–2021 Tobias Geerinckx-Rice <me@tobias.gr>
  11. ;;; Copyright © 2018 Clément Lassieur <clement@lassieur.org>
  12. ;;; Copyright © 2019-2022 Ludovic Courtès <ludo@gnu.org>
  13. ;;; Copyright © 2020 Roel Janssen <roel@gnu.org>
  14. ;;; Copyright © 2021 Stefan Reichör <stefan@xsteve.at>
  15. ;;;
  16. ;;; This file is part of GNU Guix.
  17. ;;;
  18. ;;; GNU Guix is free software; you can redistribute it and/or modify it
  19. ;;; under the terms of the GNU General Public License as published by
  20. ;;; the Free Software Foundation; either version 3 of the License, or (at
  21. ;;; your option) any later version.
  22. ;;;
  23. ;;; GNU Guix is distributed in the hope that it will be useful, but
  24. ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  25. ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  26. ;;; GNU General Public License for more details.
  27. ;;;
  28. ;;; You should have received a copy of the GNU General Public License
  29. ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
  30. (define-module (gnu packages parallel)
  31. #:use-module (guix download)
  32. #:use-module (guix git-download)
  33. #:use-module (guix build-system cmake)
  34. #:use-module (guix build-system gnu)
  35. #:use-module (guix build-system python)
  36. #:use-module ((guix licenses) #:prefix license:)
  37. #:use-module ((guix utils) #:select (target-64bit?))
  38. #:use-module (guix packages)
  39. #:use-module (guix utils)
  40. #:use-module (guix gexp)
  41. #:use-module (gnu packages)
  42. #:use-module (gnu packages admin)
  43. #:use-module (gnu packages autotools)
  44. #:use-module (gnu packages base)
  45. #:use-module (gnu packages check)
  46. #:use-module (gnu packages flex)
  47. #:use-module (gnu packages freeipmi)
  48. #:use-module (gnu packages linux)
  49. #:use-module (gnu packages maths)
  50. #:use-module (gnu packages mpi)
  51. #:use-module (gnu packages perl)
  52. #:use-module (gnu packages pkg-config)
  53. #:use-module (gnu packages python)
  54. #:use-module (gnu packages python-science)
  55. #:use-module (gnu packages python-xyz)
  56. #:use-module (gnu packages readline)
  57. #:use-module (gnu packages tcl)
  58. #:use-module (gnu packages tls)
  59. #:use-module (gnu packages web))
  60. (define-public parallel
  61. (package
  62. (name "parallel")
  63. (version "20220222")
  64. (source
  65. (origin
  66. (method url-fetch)
  67. (uri (string-append "mirror://gnu/parallel/parallel-"
  68. version ".tar.bz2"))
  69. (sha256
  70. (base32 "0id4lr3q0fh0r4vcz8sp19am9yc6j8g00m2726dgpmzacfw845pq"))))
  71. (build-system gnu-build-system)
  72. (arguments
  73. `(#:phases
  74. (modify-phases %standard-phases
  75. (add-after 'unpack 'patch-bin-sh
  76. (lambda _
  77. (for-each
  78. (lambda (file)
  79. (substitute* file
  80. ;; Patch hard coded '/bin/sh' in the line ending in:
  81. ;; $Global::shell = $ENV{'PARALLEL_SHELL'} ||
  82. ;; parent_shell($$) || $ENV{'SHELL'} || "/bin/sh";
  83. (("/bin/sh\\\";\n$") (string-append (which "sh") "\";\n"))))
  84. (list "src/parallel" "src/sem"))))
  85. (add-after 'install 'wrap-program
  86. (lambda* (#:key inputs outputs #:allow-other-keys)
  87. (let ((out (assoc-ref outputs "out")))
  88. (wrap-program (string-append out "/bin/parallel")
  89. `("PATH" ":" prefix
  90. ,(map (lambda (input)
  91. (string-append (assoc-ref inputs input) "/bin"))
  92. '("perl"
  93. "procps")))))))
  94. (add-after 'wrap-program 'post-install-test
  95. (lambda* (#:key outputs #:allow-other-keys)
  96. (invoke (string-append
  97. (assoc-ref outputs "out") "/bin/parallel")
  98. "echo"
  99. ":::" "1" "2" "3"))))))
  100. (inputs
  101. (list perl procps))
  102. (home-page "https://www.gnu.org/software/parallel/")
  103. (synopsis "Build and execute command lines in parallel")
  104. (description
  105. "GNU Parallel is a tool for executing shell jobs in parallel using one
  106. or more computers. Jobs can consist of single commands or of scripts
  107. and they are executed on lists of files, hosts, users or other items.")
  108. (license license:gpl3+)))
  109. (define-public xe
  110. (package
  111. (name "xe")
  112. (version "0.11")
  113. (source
  114. (origin
  115. (method git-fetch)
  116. (uri (git-reference
  117. (url "https://github.com/leahneukirchen/xe")
  118. (commit (string-append "v" version))))
  119. (file-name (git-file-name name version))
  120. (sha256
  121. (base32 "04jr8f6jcijr0bsmn8ajm0aj35qh9my3xjsaq64h8lwg5bpyn29x"))))
  122. (build-system gnu-build-system)
  123. (arguments
  124. `(#:tests? #f
  125. #:make-flags (list (string-append "CC=" ,(cc-for-target))
  126. (string-append "PREFIX=" %output))
  127. #:phases (modify-phases %standard-phases
  128. (delete 'configure))))
  129. (synopsis "Execute a command for every argument")
  130. (description
  131. "The xe utility constructs command lines from specified arguments,
  132. combining some of the best features of xargs(1) and apply(1). Parallel
  133. execution is also possible.")
  134. (home-page "https://github.com/leahneukirchen/xe")
  135. (license license:public-domain)))
  136. (define-public xjobs
  137. (package
  138. (name "xjobs")
  139. (version "20200726")
  140. (source (origin
  141. (method url-fetch)
  142. (uri (string-append
  143. "http://www.maier-komor.de/xjobs/xjobs-"
  144. version ".tgz"))
  145. (sha256
  146. (base32
  147. "0ay6gn43pnm7r1jamwgpycl67bjg5n87ncl27jb01w2x6x70z0i3"))))
  148. (build-system gnu-build-system)
  149. (arguments `(#:tests? #f)) ;; No tests
  150. (native-inputs
  151. (list flex which))
  152. (home-page "http://www.maier-komor.de/xjobs.html")
  153. (properties `((release-monitoring-url . ,home-page)))
  154. (synopsis
  155. "Parallel execution of jobs with several useful options")
  156. (description
  157. "xjobs reads job descriptions line by line and executes them in
  158. parallel. It limits the number of parallel executing jobs and starts new jobs
  159. when jobs finish.")
  160. (license license:gpl2+)))
  161. (define-public slurm
  162. (package
  163. (name "slurm")
  164. (version "20.11.7")
  165. (source (origin
  166. (method url-fetch)
  167. (uri (string-append
  168. "https://download.schedmd.com/slurm/slurm-"
  169. version ".tar.bz2"))
  170. (sha256
  171. (base32
  172. "1fdjihg1x7ks5l77yjv14a4mg6r0v8c3zk1dcxkhrhq3n4dc9nbs"))
  173. (modules '((guix build utils)))
  174. (snippet
  175. '(begin
  176. ;; According to
  177. ;; <https://lists.gnu.org/archive/html/guix-devel/2016-02/msg00534.html>
  178. ;; there are non-free bits under contribs/, though it's not
  179. ;; clear which ones. libpmi is clearly free (it used to be
  180. ;; under src/api/) and so is pmi2 (lax non-copyleft
  181. ;; license), so remove all of contribs/ except pmi and pmi2.
  182. (substitute* "configure.ac"
  183. (("^[[:space:]]+contribs/(.*)$" all directory)
  184. (if (string-prefix? "pmi" directory)
  185. all
  186. "")))
  187. (rename-file "contribs/pmi" "tmp-pmi")
  188. (rename-file "contribs/pmi2" "tmp-pmi2")
  189. (delete-file-recursively "contribs")
  190. (mkdir "contribs")
  191. (rename-file "tmp-pmi" "contribs/pmi")
  192. (rename-file "tmp-pmi2" "contribs/pmi2")))))
  193. ;; FIXME: More optional inputs could be added,
  194. ;; in particular mysql and gtk+.
  195. (inputs (list freeipmi
  196. `(,hwloc-2 "lib")
  197. json-c
  198. linux-pam
  199. munge
  200. numactl
  201. readline))
  202. (native-inputs
  203. (list autoconf expect perl pkg-config python-wrapper))
  204. (build-system gnu-build-system)
  205. (arguments
  206. (list #:configure-flags
  207. #~(list "--enable-pam" "--sysconfdir=/etc/slurm"
  208. "--disable-static"
  209. (string-append "--with-freeipmi=" #$(this-package-input "freeipmi"))
  210. (string-append "--with-hwloc=" #$(this-package-input "hwloc"))
  211. (string-append "--with-json=" #$(this-package-input "json-c"))
  212. (string-append "--with-munge=" #$(this-package-input "munge"))
  213. ;; 32-bit support is marked as deprecated and needs to be
  214. ;; explicitly enabled.
  215. #$@(if (target-64bit?) '() '("--enable-deprecated")))
  216. #:phases
  217. #~(modify-phases %standard-phases
  218. (add-after 'unpack 'patch-plugin-linker-flags
  219. (lambda _
  220. (substitute* (find-files "src/plugins/" "Makefile.in")
  221. (("_la_LDFLAGS = ")
  222. "_la_LDFLAGS = ../../../api/libslurm.la "))))
  223. (add-after 'patch-plugin-linker-flags 'autoconf
  224. (lambda _ (invoke "autoconf"))) ;configure.ac was patched
  225. (add-after 'install 'install-libpmi
  226. (lambda _
  227. ;; Open MPI expects libpmi to be provided by Slurm so install it.
  228. (invoke "make" "install" "-C" "contribs/pmi")
  229. ;; Others expect pmi2.
  230. (invoke "make" "install" "-C" "contribs/pmi2"))))))
  231. (home-page "https://slurm.schedmd.com/")
  232. (synopsis "Workload manager for cluster computing")
  233. (description
  234. "SLURM is a fault-tolerant and highly scalable cluster management and job
  235. scheduling system for large and small clusters. It allocates access to
  236. resources (computer nodes) to users for some duration of time, provides a
  237. framework for starting, executing, and monitoring work (typically a parallel
  238. job) on a set of allocated nodes, and arbitrates contention for resources
  239. by managing a queue of pending work.")
  240. (license (list license:bsd-2 ; src/common/log.[ch], src/common/uthash
  241. license:expat ; slurm/pmi.h
  242. license:isc ; src/common/strlcpy.c
  243. license:lgpl2.1+ ; hilbert.[ch], src/common/slurm_time.h
  244. license:zlib ; src/common/strnatcmp.c
  245. (license:non-copyleft ;contribs/pmi2, Argonne Natl. Lab.
  246. "https://github.com/SchedMD/slurm/blob/master/contribs/pmi2/COPYRIGHT")
  247. license:gpl2+)))) ; the rest, often with OpenSSL exception
  248. ;; The SLURM client/daemon protocol and file format changes from time to time
  249. ;; in incompatible ways, as noted in
  250. ;; <https://slurm.schedmd.com/troubleshoot.html#network>. Thus, keep older
  251. ;; releases here. See also <https://issues.guix.gnu.org/44387>.
  252. ;; As noted in the link, YY.MM is the release scheme, and the 'maintenance'
  253. ;; digit does not introduce incompatibilities.
  254. (define-public slurm-20.02
  255. (package
  256. (inherit slurm)
  257. (version "20.02.6-1")
  258. (source (origin
  259. (inherit (package-source slurm))
  260. (method url-fetch)
  261. (uri (string-append
  262. "https://download.schedmd.com/slurm/slurm-"
  263. version ".tar.bz2"))
  264. (sha256
  265. (base32
  266. "0qj4blfymrd2ry2qmb58l3jbr4jwygc3adcfw7my27rippcijlyc"))))
  267. (arguments
  268. (substitute-keyword-arguments (package-arguments slurm)
  269. ((#:configure-flags flags ''())
  270. #~(append '("CFLAGS=-O2 -g -fcommon" "LDFLAGS=-fcommon")
  271. #$flags))))))
  272. (define-public slurm-19.05
  273. (package
  274. (inherit slurm-20.02)
  275. (version "19.05.8")
  276. (source (origin
  277. (inherit (package-source slurm))
  278. (method url-fetch)
  279. (uri (string-append
  280. "https://download.schedmd.com/slurm/slurm-"
  281. version ".tar.bz2"))
  282. (sha256
  283. (base32
  284. "10c9j4a9a6d4ibpf75006mn03p8xgpaprc247x2idakysjf2fw43"))))))
  285. ;; Same as Debian 10
  286. (define-public slurm-18.08
  287. (package
  288. (inherit slurm-20.02)
  289. (version "18.08.9")
  290. (source
  291. (origin
  292. (inherit (package-source slurm))
  293. (uri (string-append
  294. "https://download.schedmd.com/slurm/slurm-"
  295. version ".tar.bz2"))
  296. (sha256
  297. (base32
  298. "1bgrpz75m7l4xhirsd0fvnkzlkrl8v2qpmjcz60barc5qm2kn457"))))))
  299. (define-public slurm-drmaa
  300. (package
  301. (name "slurm-drmaa")
  302. (version "1.1.2")
  303. (source (origin
  304. (method url-fetch)
  305. (uri (string-append
  306. "https://github.com/natefoo/slurm-drmaa/releases/download/"
  307. version "/slurm-drmaa-" version ".tar.gz"))
  308. (sha256
  309. (base32
  310. "0dn8ypqxdaq3k4jqwwx7msckxnmr6n2z5j68yffp50yy07ajbzjv"))))
  311. (build-system gnu-build-system)
  312. (arguments `(#:tests? #f)) ; The tests require "bats".
  313. (inputs
  314. (list slurm))
  315. (native-inputs
  316. (list which))
  317. (home-page "https://github.com/natefoo/slurm-drmaa")
  318. (synopsis "Distributed resource management application API for SLURM")
  319. (description
  320. "PSNC DRMAA for Simple Linux Utility for Resource Management (SLURM) is
  321. an implementation of Open Grid Forum DRMAA 1.0 (Distributed Resource
  322. Management Application API) specification for submission and control of jobs
  323. to SLURM. Using DRMAA, grid applications builders, portal developers and ISVs
  324. can use the same high-level API to link their software with different
  325. cluster/resource management systems.")
  326. (license license:gpl3+)))
  327. (define-public python-slurm-magic
  328. (let ((commit "73dd1a2b85799f7dae4b3f1cd9027536eff0c4d7")
  329. (revision "0"))
  330. (package
  331. (name "python-slurm-magic")
  332. (version (git-version "0.0" revision commit))
  333. (home-page "https://github.com/NERSC/slurm-magic")
  334. (source (origin
  335. (method git-fetch)
  336. (uri (git-reference (url home-page)
  337. (commit commit)))
  338. (sha256
  339. (base32
  340. "19pp2vs0wm8mx0arz9n6lw9wgyv70w9wyi4y6b91qc5j3bz5igfs"))
  341. (file-name (git-file-name name version))))
  342. (build-system python-build-system)
  343. (arguments
  344. '(#:phases (modify-phases %standard-phases
  345. (add-before 'build 'set-slurm-path
  346. (lambda* (#:key inputs #:allow-other-keys)
  347. ;; The '_execute' method tries to exec 'salloc'
  348. ;; etc. from $PATH. Record the absolute file name
  349. ;; instead.
  350. (let ((slurm (assoc-ref inputs "slurm")))
  351. (substitute* "slurm_magic.py"
  352. (("name = (.*)$" _ value)
  353. (string-append "name = \""
  354. slurm "/bin/\" + "
  355. value "\n")))
  356. #t))))))
  357. (inputs
  358. (list slurm))
  359. (propagated-inputs
  360. (list python-ipython python-pandas))
  361. (synopsis "Control the SLURM batch scheduler from Jupyter Notebook")
  362. (description
  363. "This package implements Jupyter/IPython
  364. @uref{http://ipython.readthedocs.io/en/stable/interactive/magics.html, magic
  365. commands} for interacting with the SLURM workload manager. SLURM magic simply
  366. wraps command-line executables and the commands themselves should look like
  367. their command-line counterparts. Commands are spawned via @code{subprocess}
  368. and output captured in the notebook. Whatever arguments are accepted by a
  369. SLURM command line executable are also accepted by the corresponding magic
  370. command---e.g., @code{%salloc}, @code{%sbatch}, etc.")
  371. (license license:bsd-3))))
  372. (define-public pthreadpool
  373. ;; This repository has only one tag, 0.1, which is older than what users
  374. ;; such as XNNPACK expect.
  375. (let ((commit "1787867f6183f056420e532eec640cba25efafea")
  376. (version "0.1")
  377. (revision "1"))
  378. (package
  379. (name "pthreadpool")
  380. (version (git-version version revision commit))
  381. (home-page "https://github.com/Maratyszcza/pthreadpool")
  382. (source (origin
  383. (method git-fetch)
  384. (uri (git-reference (url home-page) (commit commit)))
  385. (file-name (git-file-name name version))
  386. (sha256
  387. (base32
  388. "02hdvxfn5krw8zivkgjx3b4rk9p02yr4mpdjlp75lsv6z1xf5yrx"))
  389. (patches (search-patches "pthreadpool-system-libraries.patch"))))
  390. (build-system cmake-build-system)
  391. (arguments '(#:configure-flags '("-DBUILD_SHARED_LIBS=ON")))
  392. (inputs
  393. (list googletest googlebenchmark fxdiv))
  394. (synopsis "Efficient thread pool implementation")
  395. (description
  396. "The pthreadpool library implements an efficient and portable thread
  397. pool, similar to those implemented by OpenMP run-time support libraries for
  398. constructs such as @code{#pragma omp parallel for}, with additional
  399. features.")
  400. (license license:bsd-2))))
  401. (define-public cpuinfo
  402. ;; There's currently no tag on this repo.
  403. (let ((version "0.0")
  404. (revision "1")
  405. (commit "866ae6e5ffe93a1f63be738078da94cf3005cce2"))
  406. (package
  407. (name "cpuinfo")
  408. (version (git-version version revision commit))
  409. (home-page "https://github.com/pytorch/cpuinfo")
  410. (source (origin
  411. (method git-fetch)
  412. (uri (git-reference (url home-page) (commit commit)))
  413. (file-name (git-file-name name version))
  414. (sha256
  415. (base32
  416. "1lmsf4bpkm19a31i40qwcjn46qf7prggziv4pbsi695bkx5as71p"))
  417. (patches (search-patches "cpuinfo-system-libraries.patch"))))
  418. (build-system cmake-build-system)
  419. (arguments '(#:configure-flags '("-DBUILD_SHARED_LIBS=ON")))
  420. (inputs
  421. (list googletest googlebenchmark))
  422. (synopsis "C/C++ library to obtain information about the CPU")
  423. (description
  424. "The cpuinfo library provides a C/C++ and a command-line interface to
  425. obtain information about the CPU being used: supported instruction set,
  426. processor name, cache information, and topology information.")
  427. (license license:bsd-2))))
  428. (define-public psimd
  429. ;; There is currently no tag in this repo.
  430. (let ((commit "072586a71b55b7f8c584153d223e95687148a900")
  431. (version "0.0")
  432. (revision "1"))
  433. (package
  434. (name "psimd")
  435. (version (git-version version revision commit))
  436. (home-page "https://github.com/Maratyszcza/Psimd")
  437. (source (origin
  438. (method git-fetch)
  439. (uri (git-reference (url home-page) (commit commit)))
  440. (file-name (git-file-name name version))
  441. (sha256
  442. (base32
  443. "16mslhvqs0gpqbg7kkq566a8gkn58cgjpqca8ljj9qcv5mk9apwm"))))
  444. (build-system cmake-build-system)
  445. (arguments '(#:tests? #f)) ;there are no tests
  446. (synopsis "Portable 128-bit SIMD intrinsics")
  447. (description
  448. "This header-only C++ library provides a portable interface to
  449. single-instruction multiple-data (SIMD) intrinsics.")
  450. (license license:expat))))