publish.scm 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763
  1. ;;; GNU Guix --- Functional package management for GNU
  2. ;;; Copyright © 2015 David Thompson <davet@gnu.org>
  3. ;;; Copyright © 2020 by Amar M. Singh <nly@disroot.org>
  4. ;;; Copyright © 2016-2022 Ludovic Courtès <ludo@gnu.org>
  5. ;;;
  6. ;;; This file is part of GNU Guix.
  7. ;;;
  8. ;;; GNU Guix is free software; you can redistribute it and/or modify it
  9. ;;; under the terms of the GNU General Public License as published by
  10. ;;; the Free Software Foundation; either version 3 of the License, or (at
  11. ;;; your option) any later version.
  12. ;;;
  13. ;;; GNU Guix is distributed in the hope that it will be useful, but
  14. ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  15. ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16. ;;; GNU General Public License for more details.
  17. ;;;
  18. ;;; You should have received a copy of the GNU General Public License
  19. ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
  20. ;; Avoid interference.
  21. (unsetenv "http_proxy")
  22. (define-module (test-publish)
  23. #:use-module (guix scripts publish)
  24. #:use-module (guix tests)
  25. #:use-module (guix config)
  26. #:use-module (guix utils)
  27. #:use-module (gcrypt hash)
  28. #:use-module (guix store)
  29. #:use-module (guix derivations)
  30. #:use-module (guix gexp)
  31. #:use-module (guix base32)
  32. #:use-module (guix base64)
  33. #:use-module ((guix records) #:select (recutils->alist))
  34. #:use-module ((guix serialization) #:select (restore-file))
  35. #:use-module (gcrypt pk-crypto)
  36. #:use-module ((guix pki) #:select (%public-key-file %private-key-file))
  37. #:use-module (zlib)
  38. #:use-module (lzlib)
  39. #:autoload (zstd) (call-with-zstd-input-port)
  40. #:use-module (web uri)
  41. #:use-module (web client)
  42. #:use-module (web response)
  43. #:use-module (rnrs bytevectors)
  44. #:use-module (ice-9 binary-ports)
  45. #:use-module (srfi srfi-1)
  46. #:use-module (srfi srfi-26)
  47. #:use-module (srfi srfi-64)
  48. #:use-module (ice-9 threads)
  49. #:use-module (ice-9 format)
  50. #:use-module (ice-9 match)
  51. #:use-module (ice-9 rdelim))
  52. (define %store
  53. (open-connection-for-tests))
  54. (define (zstd-supported?)
  55. (resolve-module '(zstd) #t #f #:ensure #f))
  56. (define %reference (add-text-to-store %store "ref" "foo"))
  57. (define %item (add-text-to-store %store "item" "bar" (list %reference)))
  58. (define (http-get-body uri)
  59. (call-with-values (lambda () (http-get uri))
  60. (lambda (response body) body)))
  61. (define (http-get-port uri)
  62. (let ((socket (open-socket-for-uri uri)))
  63. ;; Make sure to use an unbuffered port so that we can then peek at the
  64. ;; underlying file descriptor via 'call-with-gzip-input-port'.
  65. (setvbuf socket 'none)
  66. (call-with-values
  67. (lambda ()
  68. (http-get uri #:port socket #:streaming? #t))
  69. (lambda (response port)
  70. ;; Don't (setvbuf port 'none) because of <http://bugs.gnu.org/19610>
  71. ;; (PORT might be a custom binary input port).
  72. port))))
  73. (define (publish-uri route)
  74. (string-append "http://localhost:6789" route))
  75. (define-syntax-rule (with-separate-output-ports exp ...)
  76. ;; Since ports aren't thread-safe in Guile 2.0, duplicate the output and
  77. ;; error ports to make sure the two threads don't end up stepping on each
  78. ;; other's toes.
  79. (with-output-to-port (duplicate-port (current-output-port) "w")
  80. (lambda ()
  81. (with-error-to-port (duplicate-port (current-error-port) "w")
  82. (lambda ()
  83. exp ...)))))
  84. ;; Run a local publishing server in a separate thread.
  85. (with-separate-output-ports
  86. (call-with-new-thread
  87. (lambda ()
  88. (guix-publish "--port=6789" "-C0")))) ;attempt to avoid port collision
  89. (define (wait-until-ready port)
  90. ;; Wait until the server is accepting connections.
  91. (let ((conn (socket PF_INET SOCK_STREAM 0)))
  92. (let loop ()
  93. (unless (false-if-exception
  94. (connect conn AF_INET (inet-pton AF_INET "127.0.0.1") port))
  95. (loop)))))
  96. (define (wait-for-file file)
  97. ;; Wait until FILE shows up.
  98. (let loop ((i 20))
  99. (cond ((file-exists? file)
  100. #t)
  101. ((zero? i)
  102. (error "file didn't show up" file))
  103. (else
  104. (pk 'wait-for-file file)
  105. (sleep 1)
  106. (loop (- i 1))))))
  107. (define %gzip-magic-bytes
  108. ;; Magic bytes of gzip file.
  109. #vu8(#x1f #x8b))
  110. ;; Wait until the two servers are ready.
  111. (wait-until-ready 6789)
  112. ;; Initialize the public/private key SRFI-39 parameters.
  113. (%public-key (read-file-sexp %public-key-file))
  114. (%private-key (read-file-sexp %private-key-file))
  115. (test-begin "publish")
  116. (test-equal "/nix-cache-info"
  117. (format #f "StoreDir: ~a\nWantMassQuery: 0\nPriority: 100\n"
  118. %store-directory)
  119. (http-get-body (publish-uri "/nix-cache-info")))
  120. (test-equal "/*.narinfo"
  121. (let* ((info (query-path-info %store %item))
  122. (unsigned-info
  123. (format #f
  124. "StorePath: ~a
  125. NarHash: sha256:~a
  126. NarSize: ~d
  127. References: ~a~%"
  128. %item
  129. (bytevector->nix-base32-string
  130. (path-info-hash info))
  131. (path-info-nar-size info)
  132. (basename (first (path-info-references info)))))
  133. (signature (base64-encode
  134. (string->utf8
  135. (canonical-sexp->string
  136. (signed-string unsigned-info))))))
  137. (format #f "~aSignature: 1;~a;~a
  138. URL: nar/~a
  139. Compression: none
  140. FileSize: ~a\n"
  141. unsigned-info (gethostname) signature
  142. (basename %item)
  143. (path-info-nar-size info)))
  144. (utf8->string
  145. (http-get-body
  146. (publish-uri
  147. (string-append "/" (store-path-hash-part %item) ".narinfo")))))
  148. (test-equal "/*.narinfo with properly encoded '+' sign"
  149. ;; See <http://bugs.gnu.org/21888>.
  150. (let* ((item (add-text-to-store %store "fake-gtk+" "Congrats!"))
  151. (info (query-path-info %store item))
  152. (unsigned-info
  153. (format #f
  154. "StorePath: ~a
  155. NarHash: sha256:~a
  156. NarSize: ~d
  157. References: ~%"
  158. item
  159. (bytevector->nix-base32-string
  160. (path-info-hash info))
  161. (path-info-nar-size info)))
  162. (signature (base64-encode
  163. (string->utf8
  164. (canonical-sexp->string
  165. (signed-string unsigned-info))))))
  166. (format #f "~aSignature: 1;~a;~a
  167. URL: nar/~a
  168. Compression: none
  169. FileSize: ~a~%"
  170. unsigned-info (gethostname) signature
  171. (uri-encode (basename item))
  172. (path-info-nar-size info)))
  173. (let ((item (add-text-to-store %store "fake-gtk+" "Congrats!")))
  174. (utf8->string
  175. (http-get-body
  176. (publish-uri
  177. (string-append "/" (store-path-hash-part item) ".narinfo"))))))
  178. (test-equal "/nar/*"
  179. "bar"
  180. (call-with-temporary-output-file
  181. (lambda (temp port)
  182. (let ((nar (utf8->string
  183. (http-get-body
  184. (publish-uri
  185. (string-append "/nar/" (basename %item)))))))
  186. (call-with-input-string nar (cut restore-file <> temp)))
  187. (call-with-input-file temp read-string))))
  188. (test-equal "/nar/gzip/*"
  189. "bar"
  190. (call-with-temporary-output-file
  191. (lambda (temp port)
  192. (let ((nar (http-get-port
  193. (publish-uri
  194. (string-append "/nar/gzip/" (basename %item))))))
  195. (call-with-gzip-input-port nar
  196. (cut restore-file <> temp)))
  197. (call-with-input-file temp read-string))))
  198. (test-equal "/nar/gzip/* is really gzip"
  199. %gzip-magic-bytes
  200. ;; Since 'gzdopen' (aka. 'call-with-gzip-input-port') transparently reads
  201. ;; uncompressed gzip, the test above doesn't check whether it's actually
  202. ;; gzip. This is what this test does. See <https://bugs.gnu.org/30184>.
  203. (let ((nar (http-get-port
  204. (publish-uri
  205. (string-append "/nar/gzip/" (basename %item))))))
  206. (get-bytevector-n nar (bytevector-length %gzip-magic-bytes))))
  207. (test-equal "/nar/lzip/*"
  208. "bar"
  209. (call-with-temporary-output-file
  210. (lambda (temp port)
  211. (let ((nar (http-get-port
  212. (publish-uri
  213. (string-append "/nar/lzip/" (basename %item))))))
  214. (call-with-lzip-input-port nar
  215. (cut restore-file <> temp)))
  216. (call-with-input-file temp read-string))))
  217. (unless (zstd-supported?) (test-skip 1))
  218. (test-equal "/nar/zstd/*"
  219. "bar"
  220. (call-with-temporary-output-file
  221. (lambda (temp port)
  222. (let ((nar (http-get-port
  223. (publish-uri
  224. (string-append "/nar/zstd/" (basename %item))))))
  225. (call-with-zstd-input-port nar
  226. (cut restore-file <> temp)))
  227. (call-with-input-file temp read-string))))
  228. (test-equal "/*.narinfo with compression"
  229. `(("StorePath" . ,%item)
  230. ("URL" . ,(string-append "nar/gzip/" (basename %item)))
  231. ("Compression" . "gzip"))
  232. (let ((thread (with-separate-output-ports
  233. (call-with-new-thread
  234. (lambda ()
  235. (guix-publish "--port=6799" "-C5"))))))
  236. (wait-until-ready 6799)
  237. (let* ((url (string-append "http://localhost:6799/"
  238. (store-path-hash-part %item) ".narinfo"))
  239. (body (http-get-port url)))
  240. (filter (lambda (item)
  241. (match item
  242. (("Compression" . _) #t)
  243. (("StorePath" . _) #t)
  244. (("URL" . _) #t)
  245. (_ #f)))
  246. (recutils->alist body)))))
  247. (test-equal "/*.narinfo with lzip compression"
  248. `(("StorePath" . ,%item)
  249. ("URL" . ,(string-append "nar/lzip/" (basename %item)))
  250. ("Compression" . "lzip"))
  251. (let ((thread (with-separate-output-ports
  252. (call-with-new-thread
  253. (lambda ()
  254. (guix-publish "--port=6790" "-Clzip"))))))
  255. (wait-until-ready 6790)
  256. (let* ((url (string-append "http://localhost:6790/"
  257. (store-path-hash-part %item) ".narinfo"))
  258. (body (http-get-port url)))
  259. (filter (lambda (item)
  260. (match item
  261. (("Compression" . _) #t)
  262. (("StorePath" . _) #t)
  263. (("URL" . _) #t)
  264. (_ #f)))
  265. (recutils->alist body)))))
  266. (test-equal "/*.narinfo for a compressed file"
  267. '("none" "nar") ;compression-less nar
  268. ;; Assume 'guix publish -C' is already running on port 6799.
  269. (let* ((item (add-text-to-store %store "fake.tar.gz"
  270. "This is a fake compressed file."))
  271. (url (string-append "http://localhost:6799/"
  272. (store-path-hash-part item) ".narinfo"))
  273. (body (http-get-port url))
  274. (info (recutils->alist body)))
  275. (list (assoc-ref info "Compression")
  276. (dirname (assoc-ref info "URL")))))
  277. (test-equal "/*.narinfo with lzip + gzip"
  278. `((("StorePath" . ,%item)
  279. ("URL" . ,(string-append "nar/gzip/" (basename %item)))
  280. ("Compression" . "gzip")
  281. ("URL" . ,(string-append "nar/lzip/" (basename %item)))
  282. ("Compression" . "lzip"))
  283. 200
  284. 200)
  285. (call-with-temporary-directory
  286. (lambda (cache)
  287. (let ((thread (with-separate-output-ports
  288. (call-with-new-thread
  289. (lambda ()
  290. (guix-publish "--port=6793" "-Cgzip:2" "-Clzip:2"))))))
  291. (wait-until-ready 6793)
  292. (let* ((base "http://localhost:6793/")
  293. (part (store-path-hash-part %item))
  294. (url (string-append base part ".narinfo"))
  295. (body (http-get-port url)))
  296. (list (filter (match-lambda
  297. (("StorePath" . _) #t)
  298. (("URL" . _) #t)
  299. (("Compression" . _) #t)
  300. (_ #f))
  301. (recutils->alist body))
  302. (response-code
  303. (http-get (string-append base "nar/gzip/"
  304. (basename %item))))
  305. (response-code
  306. (http-get (string-append base "nar/lzip/"
  307. (basename %item))))))))))
  308. (test-equal "custom nar path"
  309. ;; Serve nars at /foo/bar/chbouib instead of /nar.
  310. (list `(("StorePath" . ,%item)
  311. ("URL" . ,(string-append "foo/bar/chbouib/" (basename %item)))
  312. ("Compression" . "none"))
  313. 200
  314. 404)
  315. (let ((thread (with-separate-output-ports
  316. (call-with-new-thread
  317. (lambda ()
  318. (guix-publish "--port=6798" "-C0"
  319. "--nar-path=///foo/bar//chbouib/"))))))
  320. (wait-until-ready 6798)
  321. (let* ((base "http://localhost:6798/")
  322. (part (store-path-hash-part %item))
  323. (url (string-append base part ".narinfo"))
  324. (nar-url (string-append base "foo/bar/chbouib/"
  325. (basename %item)))
  326. (body (http-get-port url)))
  327. (list (filter (lambda (item)
  328. (match item
  329. (("Compression" . _) #t)
  330. (("StorePath" . _) #t)
  331. (("URL" . _) #t)
  332. (_ #f)))
  333. (recutils->alist body))
  334. (response-code (http-get nar-url))
  335. (response-code
  336. (http-get (string-append base "nar/" (basename %item))))))))
  337. (test-equal "/nar/ with properly encoded '+' sign"
  338. "Congrats!"
  339. (let ((item (add-text-to-store %store "fake-gtk+" "Congrats!")))
  340. (call-with-temporary-output-file
  341. (lambda (temp port)
  342. (let ((nar (utf8->string
  343. (http-get-body
  344. (publish-uri
  345. (string-append "/nar/" (uri-encode (basename item))))))))
  346. (call-with-input-string nar (cut restore-file <> temp)))
  347. (call-with-input-file temp read-string)))))
  348. (test-equal "/nar/invalid"
  349. 404
  350. (begin
  351. (call-with-output-file (string-append (%store-prefix) "/invalid")
  352. (lambda (port)
  353. (display "This file is not a valid store item." port)))
  354. (response-code (http-get (publish-uri (string-append "/nar/invalid"))))))
  355. (test-equal "/file/NAME/sha256/HASH"
  356. "Hello, Guix world!"
  357. (let* ((data "Hello, Guix world!")
  358. (hash (call-with-input-string data port-sha256))
  359. (drv (run-with-store %store
  360. (gexp->derivation "the-file.txt"
  361. #~(call-with-output-file #$output
  362. (lambda (port)
  363. (display #$data port)))
  364. #:hash-algo 'sha256
  365. #:hash hash)))
  366. (out (build-derivations %store (list drv))))
  367. (utf8->string
  368. (http-get-body
  369. (publish-uri
  370. (string-append "/file/the-file.txt/sha256/"
  371. (bytevector->nix-base32-string hash)))))))
  372. (test-equal "/file/NAME/sha256/INVALID-NIX-BASE32-STRING"
  373. 404
  374. (let ((uri (publish-uri
  375. "/file/the-file.txt/sha256/not-a-nix-base32-string")))
  376. (response-code (http-get uri))))
  377. (test-equal "/file/NAME/sha256/INVALID-HASH"
  378. 404
  379. (let ((uri (publish-uri
  380. (string-append "/file/the-file.txt/sha256/"
  381. (bytevector->nix-base32-string
  382. (call-with-input-string "" port-sha256))))))
  383. (response-code (http-get uri))))
  384. (test-equal "with cache"
  385. (list #t
  386. `(("StorePath" . ,%item)
  387. ("URL" . ,(string-append "nar/gzip/" (basename %item)))
  388. ("Compression" . "gzip"))
  389. 200 ;nar/gzip/…
  390. #t ;Content-Length
  391. #t ;FileSize
  392. 404) ;nar/…
  393. (call-with-temporary-directory
  394. (lambda (cache)
  395. (let ((thread (with-separate-output-ports
  396. (call-with-new-thread
  397. (lambda ()
  398. (guix-publish "--port=6797" "-C2"
  399. (string-append "--cache=" cache)
  400. "--cache-bypass-threshold=0"))))))
  401. (wait-until-ready 6797)
  402. (let* ((base "http://localhost:6797/")
  403. (part (store-path-hash-part %item))
  404. (url (string-append base part ".narinfo"))
  405. (nar-url (string-append base "nar/gzip/" (basename %item)))
  406. (cached (string-append cache "/gzip/" (basename %item)
  407. ".narinfo"))
  408. (nar (string-append cache "/gzip/"
  409. (basename %item) ".nar"))
  410. (response (http-get url)))
  411. (and (= 404 (response-code response))
  412. ;; We should get an explicitly short TTL for 404 in this case
  413. ;; because it's going to become 200 shortly.
  414. (match (assq-ref (response-headers response) 'cache-control)
  415. ((('max-age . ttl))
  416. (< ttl 3600)))
  417. (wait-for-file cached)
  418. ;; Both the narinfo and nar should be world-readable.
  419. (= #o444 (logand #o444 (stat:perms (lstat cached))))
  420. (= #o444 (logand #o444 (stat:perms (lstat nar))))
  421. (let* ((body (http-get-port url))
  422. (compressed (http-get nar-url))
  423. (uncompressed (http-get (string-append base "nar/"
  424. (basename %item))))
  425. (narinfo (recutils->alist body)))
  426. (list (file-exists? nar)
  427. (filter (lambda (item)
  428. (match item
  429. (("Compression" . _) #t)
  430. (("StorePath" . _) #t)
  431. (("URL" . _) #t)
  432. (_ #f)))
  433. narinfo)
  434. (response-code compressed)
  435. (= (response-content-length compressed)
  436. (stat:size (stat nar)))
  437. (= (string->number
  438. (assoc-ref narinfo "FileSize"))
  439. (stat:size (stat nar)))
  440. (response-code uncompressed)))))))))
  441. (test-equal "with cache, lzip + gzip"
  442. '(200 200 404)
  443. (call-with-temporary-directory
  444. (lambda (cache)
  445. (let ((thread (with-separate-output-ports
  446. (call-with-new-thread
  447. (lambda ()
  448. (guix-publish "--port=6794" "-Cgzip:2" "-Clzip:2"
  449. (string-append "--cache=" cache)
  450. "--cache-bypass-threshold=0"))))))
  451. (wait-until-ready 6794)
  452. (let* ((base "http://localhost:6794/")
  453. (part (store-path-hash-part %item))
  454. (url (string-append base part ".narinfo"))
  455. (nar-url (cute string-append "nar/" <> "/"
  456. (basename %item)))
  457. (cached (cute string-append cache "/" <> "/"
  458. (basename %item) ".narinfo"))
  459. (nar (cute string-append cache "/" <> "/"
  460. (basename %item) ".nar"))
  461. (response (http-get url)))
  462. (wait-for-file (cached "gzip"))
  463. (let* ((body (http-get-port url))
  464. (narinfo (recutils->alist body))
  465. (uncompressed (string-append base "nar/"
  466. (basename %item))))
  467. (and (file-exists? (nar "gzip"))
  468. (file-exists? (nar "lzip"))
  469. (match (pk 'narinfo/gzip+lzip narinfo)
  470. ((("StorePath" . path)
  471. _ ...
  472. ("Signature" . _)
  473. ("URL" . gzip-url)
  474. ("Compression" . "gzip")
  475. ("FileSize" . (= string->number gzip-size))
  476. ("URL" . lzip-url)
  477. ("Compression" . "lzip")
  478. ("FileSize" . (= string->number lzip-size)))
  479. (and (string=? gzip-url (nar-url "gzip"))
  480. (string=? lzip-url (nar-url "lzip"))
  481. (= gzip-size
  482. (stat:size (stat (nar "gzip"))))
  483. (= lzip-size
  484. (stat:size (stat (nar "lzip")))))))
  485. (list (response-code
  486. (http-get (string-append base (nar-url "gzip"))))
  487. (response-code
  488. (http-get (string-append base (nar-url "lzip"))))
  489. (response-code
  490. (http-get uncompressed))))))))))
  491. (let ((item (add-text-to-store %store "fake-compressed-thing.tar.gz"
  492. (random-text))))
  493. (test-equal "with cache, uncompressed"
  494. (list #t
  495. (* 42 3600) ;TTL on narinfo
  496. `(("StorePath" . ,item)
  497. ("URL" . ,(string-append "nar/" (basename item)))
  498. ("Compression" . "none"))
  499. 200 ;nar/…
  500. (* 42 3600) ;TTL on nar/…
  501. (path-info-nar-size
  502. (query-path-info %store item)) ;FileSize
  503. 404) ;nar/gzip/…
  504. (call-with-temporary-directory
  505. (lambda (cache)
  506. (let ((thread (with-separate-output-ports
  507. (call-with-new-thread
  508. (lambda ()
  509. (guix-publish "--port=6796" "-C2" "--ttl=42h"
  510. (string-append "--cache=" cache)
  511. "--cache-bypass-threshold=0"))))))
  512. (wait-until-ready 6796)
  513. (let* ((base "http://localhost:6796/")
  514. (part (store-path-hash-part item))
  515. (url (string-append base part ".narinfo"))
  516. (cached (string-append cache "/none/"
  517. (basename item) ".narinfo"))
  518. (nar (string-append cache "/none/"
  519. (basename item) ".nar"))
  520. (response (http-get url)))
  521. (and (= 404 (response-code response))
  522. (wait-for-file cached)
  523. (let* ((response (http-get url))
  524. (body (http-get-port url))
  525. (compressed (http-get (string-append base "nar/gzip/"
  526. (basename item))))
  527. (uncompressed (http-get (string-append base "nar/"
  528. (basename item))))
  529. (narinfo (recutils->alist body)))
  530. (list (file-exists? nar)
  531. (match (assq-ref (response-headers response)
  532. 'cache-control)
  533. ((('max-age . ttl)) ttl)
  534. (_ #f))
  535. (filter (lambda (item)
  536. (match item
  537. (("Compression" . _) #t)
  538. (("StorePath" . _) #t)
  539. (("URL" . _) #t)
  540. (_ #f)))
  541. narinfo)
  542. (response-code uncompressed)
  543. (match (assq-ref (response-headers uncompressed)
  544. 'cache-control)
  545. ((('max-age . ttl)) ttl)
  546. (_ #f))
  547. (string->number
  548. (assoc-ref narinfo "FileSize"))
  549. (response-code compressed))))))))))
  550. (test-equal "with cache, vanishing item" ;<https://bugs.gnu.org/33897>
  551. 200
  552. (call-with-temporary-directory
  553. (lambda (cache)
  554. (let ((thread (with-separate-output-ports
  555. (call-with-new-thread
  556. (lambda ()
  557. (guix-publish "--port=6795"
  558. (string-append "--cache=" cache)))))))
  559. (wait-until-ready 6795)
  560. ;; Make sure that, even if ITEM disappears, we're still able to fetch
  561. ;; it.
  562. (let* ((base "http://localhost:6795/")
  563. (item (add-text-to-store %store "random" (random-text)))
  564. (part (store-path-hash-part item))
  565. (url (string-append base part ".narinfo"))
  566. (cached (string-append cache "/gzip/"
  567. (basename item)
  568. ".narinfo"))
  569. (response (http-get url)))
  570. (and (= 200 (response-code response)) ;we're below the threshold
  571. (wait-for-file cached)
  572. (begin
  573. (delete-paths %store (list item))
  574. (response-code (pk 'response (http-get url))))))))))
  575. (test-equal "with cache, cache bypass"
  576. 200
  577. (call-with-temporary-directory
  578. (lambda (cache)
  579. (let ((thread (with-separate-output-ports
  580. (call-with-new-thread
  581. (lambda ()
  582. (guix-publish "--port=6788" "-C" "gzip"
  583. (string-append "--cache=" cache)))))))
  584. (wait-until-ready 6788)
  585. (let* ((base "http://localhost:6788/")
  586. (item (add-text-to-store %store "random" (random-text)))
  587. (part (store-path-hash-part item))
  588. (narinfo (string-append base part ".narinfo"))
  589. (nar (string-append base "nar/gzip/" (basename item)))
  590. (cached (string-append cache "/gzip/" (basename item)
  591. ".narinfo")))
  592. ;; We're below the default cache bypass threshold, so NAR and NARINFO
  593. ;; should immediately return 200. The NARINFO request should trigger
  594. ;; caching, and the next request to NAR should return 200 as well.
  595. (and (let ((response (pk 'r1 (http-get nar))))
  596. (and (= 200 (response-code response))
  597. (not (response-content-length response)))) ;not known
  598. (= 200 (response-code (http-get narinfo)))
  599. (begin
  600. (wait-for-file cached)
  601. (let ((response (pk 'r2 (http-get nar))))
  602. (and (> (response-content-length response)
  603. (stat:size (stat item)))
  604. (response-code response))))))))))
  605. (test-equal "with cache, cache bypass, unmapped hash part"
  606. 200
  607. ;; This test reproduces the bug described in <https://bugs.gnu.org/44442>:
  608. ;; the daemon connection would be closed as a side effect of a nar request
  609. ;; for a non-existing file name.
  610. (call-with-temporary-directory
  611. (lambda (cache)
  612. (let ((thread (with-separate-output-ports
  613. (call-with-new-thread
  614. (lambda ()
  615. (guix-publish "--port=6787" "-C" "gzip"
  616. (string-append "--cache=" cache)))))))
  617. (wait-until-ready 6787)
  618. (let* ((base "http://localhost:6787/")
  619. (item (add-text-to-store %store "random" (random-text)))
  620. (part (store-path-hash-part item))
  621. (narinfo (string-append base part ".narinfo"))
  622. (nar (string-append base "nar/gzip/" (basename item)))
  623. (cached (string-append cache "/gzip/" (basename item)
  624. ".narinfo")))
  625. ;; The first response used to be 500 and to terminate the daemon
  626. ;; connection as a side effect.
  627. (and (= (response-code
  628. (http-get (string-append base "nar/gzip/"
  629. (make-string 32 #\e)
  630. "-does-not-exist")))
  631. 404)
  632. (= 200 (response-code (http-get nar)))
  633. (= 200 (response-code (http-get narinfo)))
  634. (begin
  635. (wait-for-file cached)
  636. (response-code (http-get nar)))))))))
  637. (test-equal "/log/NAME"
  638. `(200 #t text/plain (gzip))
  639. (let ((drv (run-with-store %store
  640. (gexp->derivation "with-log"
  641. #~(call-with-output-file #$output
  642. (lambda (port)
  643. (display "Hello, build log!"
  644. (current-error-port))
  645. (display #$(random-text) port)))))))
  646. (build-derivations %store (list drv))
  647. (let* ((response (http-get
  648. (publish-uri (string-append "/log/"
  649. (basename (derivation->output-path drv))))
  650. #:decode-body? #f))
  651. (base (basename (derivation-file-name drv)))
  652. (log (string-append (dirname %state-directory)
  653. "/log/guix/drvs/" (string-take base 2)
  654. "/" (string-drop base 2) ".gz")))
  655. (list (response-code response)
  656. (= (response-content-length response) (stat:size (stat log)))
  657. (first (response-content-type response))
  658. (response-content-encoding response)))))
  659. (test-equal "negative TTL"
  660. `(404 42)
  661. (call-with-temporary-directory
  662. (lambda (cache)
  663. (let ((thread (with-separate-output-ports
  664. (call-with-new-thread
  665. (lambda ()
  666. (guix-publish "--port=6786" "-C0"
  667. "--negative-ttl=42s"))))))
  668. (wait-until-ready 6786)
  669. (let* ((base "http://localhost:6786/")
  670. (url (string-append base (make-string 32 #\z)
  671. ".narinfo"))
  672. (response (http-get url)))
  673. (list (response-code response)
  674. (match (assq-ref (response-headers response) 'cache-control)
  675. ((('max-age . ttl)) ttl)
  676. (_ #f))))))))
  677. (test-equal "no negative TTL"
  678. `(404 #f)
  679. (let* ((uri (publish-uri
  680. (string-append "/" (make-string 32 #\z)
  681. ".narinfo")))
  682. (response (http-get uri)))
  683. (list (response-code response)
  684. (assq-ref (response-headers response) 'cache-control))))
  685. (test-equal "/log/NAME not found"
  686. 404
  687. (let ((uri (publish-uri "/log/does-not-exist")))
  688. (response-code (http-get uri))))
  689. (test-equal "/signing-key.pub"
  690. 200
  691. (response-code (http-get (publish-uri "/signing-key.pub"))))
  692. (test-equal "non-GET query"
  693. '(200 404)
  694. (let ((path (string-append "/" (store-path-hash-part %item)
  695. ".narinfo")))
  696. (map response-code
  697. (list (http-get (publish-uri path))
  698. (http-post (publish-uri path))))))
  699. (test-end "publish")