Mirror of GNU Guix
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

461 lines
18 KiB

  1. ;;; GNU Guix --- Functional package management for GNU
  2. ;;; Copyright © 2012, 2013, 2014, 2015 Ludovic Courtès <ludo@gnu.org>
  3. ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org>
  4. ;;;
  5. ;;; This file is part of GNU Guix.
  6. ;;;
  7. ;;; GNU Guix is free software; you can redistribute it and/or modify it
  8. ;;; under the terms of the GNU General Public License as published by
  9. ;;; the Free Software Foundation; either version 3 of the License, or (at
  10. ;;; your option) any later version.
  11. ;;;
  12. ;;; GNU Guix is distributed in the hope that it will be useful, but
  13. ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14. ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. ;;; GNU General Public License for more details.
  16. ;;;
  17. ;;; You should have received a copy of the GNU General Public License
  18. ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
  19. (define-module (guix build download)
  20. #:use-module (web uri)
  21. #:use-module (web client)
  22. #:use-module (web response)
  23. #:use-module (guix ftp-client)
  24. #:use-module (guix build utils)
  25. #:use-module (rnrs io ports)
  26. #:use-module (srfi srfi-1)
  27. #:use-module (srfi srfi-11)
  28. #:use-module (srfi srfi-19)
  29. #:use-module (srfi srfi-26)
  30. #:use-module (ice-9 match)
  31. #:use-module (ice-9 format)
  32. #:export (open-connection-for-uri
  33. resolve-uri-reference
  34. maybe-expand-mirrors
  35. url-fetch
  36. progress-proc
  37. uri-abbreviation))
  38. ;;; Commentary:
  39. ;;;
  40. ;;; Fetch data such as tarballs over HTTP or FTP (builder-side code).
  41. ;;;
  42. ;;; Code:
  43. (define %http-receive-buffer-size
  44. ;; Size of the HTTP receive buffer.
  45. 65536)
  46. (define (duration->seconds duration)
  47. "Return the number of seconds represented by DURATION, a 'time-duration'
  48. object, as an inexact number."
  49. (+ (time-second duration)
  50. (/ (time-nanosecond duration) 1e9)))
  51. (define (throughput->string throughput)
  52. "Given THROUGHPUT, measured in bytes per second, return a string
  53. representing it in a human-readable way."
  54. (if (> throughput 3e6)
  55. (format #f "~,2f MiB/s" (/ throughput (expt 2. 20)))
  56. (format #f "~,0f KiB/s" (/ throughput 1024.0))))
  57. (define* (progress-proc file size #:optional (log-port (current-output-port)))
  58. "Return a procedure to show the progress of FILE's download, which is
  59. SIZE byte long. The returned procedure is suitable for use as an
  60. argument to `dump-port'. The progress report is written to LOG-PORT."
  61. ;; XXX: Because of <http://bugs.gnu.org/19939> this procedure is often not
  62. ;; called as frequently as we'd like too; this is especially bad with Nginx
  63. ;; on hydra.gnu.org, which returns whole nars as a single chunk.
  64. (let ((start-time #f))
  65. (let-syntax ((with-elapsed-time
  66. (syntax-rules ()
  67. ((_ elapsed body ...)
  68. (let* ((now (current-time time-monotonic))
  69. (elapsed (and start-time
  70. (duration->seconds
  71. (time-difference now
  72. start-time)))))
  73. (unless start-time
  74. (set! start-time now))
  75. body ...)))))
  76. (if (number? size)
  77. (lambda (transferred cont)
  78. (with-elapsed-time elapsed
  79. (let ((% (* 100.0 (/ transferred size)))
  80. (throughput (if elapsed
  81. (/ transferred elapsed)
  82. 0)))
  83. (display #\cr log-port)
  84. (format log-port "~a\t~5,1f% of ~,1f KiB (~a)"
  85. file % (/ size 1024.0)
  86. (throughput->string throughput))
  87. (flush-output-port log-port)
  88. (cont))))
  89. (lambda (transferred cont)
  90. (with-elapsed-time elapsed
  91. (let ((throughput (if elapsed
  92. (/ transferred elapsed)
  93. 0)))
  94. (display #\cr log-port)
  95. (format log-port "~a\t~6,1f KiB transferred (~a)"
  96. file (/ transferred 1024.0)
  97. (throughput->string throughput))
  98. (flush-output-port log-port)
  99. (cont))))))))
  100. (define* (uri-abbreviation uri #:optional (max-length 42))
  101. "If URI's string representation is larger than MAX-LENGTH, return an
  102. abbreviation of URI showing the scheme, host, and basename of the file."
  103. (define uri-as-string
  104. (uri->string uri))
  105. (define (elide-path)
  106. (let ((path (uri-path uri)))
  107. (string-append (symbol->string (uri-scheme uri)) "://"
  108. ;; `file' URIs have no host part.
  109. (or (uri-host uri) "")
  110. (string-append "/.../" (basename path)))))
  111. (if (> (string-length uri-as-string) max-length)
  112. (let ((short (elide-path)))
  113. (if (< (string-length short) (string-length uri-as-string))
  114. short
  115. uri-as-string))
  116. uri-as-string))
  117. (define (ftp-fetch uri file)
  118. "Fetch data from URI and write it to FILE. Return FILE on success."
  119. (let* ((conn (ftp-open (uri-host uri)))
  120. (size (false-if-exception (ftp-size conn (uri-path uri))))
  121. (in (ftp-retr conn (basename (uri-path uri))
  122. (dirname (uri-path uri)))))
  123. (call-with-output-file file
  124. (lambda (out)
  125. (dump-port in out
  126. #:buffer-size %http-receive-buffer-size
  127. #:progress (progress-proc (uri-abbreviation uri) size))))
  128. (ftp-close conn))
  129. (newline)
  130. file)
  131. ;; Autoload GnuTLS so that this module can be used even when GnuTLS is
  132. ;; not available. At compile time, this yields "possibly unbound
  133. ;; variable" warnings, but these are OK: we know that the variables will
  134. ;; be bound if we need them, because (guix download) adds GnuTLS as an
  135. ;; input in that case.
  136. ;; XXX: Use this hack instead of #:autoload to avoid compilation errors.
  137. ;; See <http://bugs.gnu.org/12202>.
  138. (module-autoload! (current-module)
  139. '(gnutls) '(make-session connection-end/client))
  140. (define add-weak-reference
  141. (let ((table (make-weak-key-hash-table)))
  142. (lambda (from to)
  143. "Hold a weak reference from FROM to TO."
  144. (hashq-set! table from to))))
  145. (define (tls-wrap port server)
  146. "Return PORT wrapped in a TLS connection to SERVER. SERVER must be a DNS
  147. host name without trailing dot."
  148. (define (log level str)
  149. (format (current-error-port)
  150. "gnutls: [~a|~a] ~a" (getpid) level str))
  151. (let ((session (make-session connection-end/client)))
  152. ;; Some servers such as 'cloud.github.com' require the client to support
  153. ;; the 'SERVER NAME' extension. However, 'set-session-server-name!' is
  154. ;; not available in older GnuTLS releases. See
  155. ;; <http://bugs.gnu.org/18526> for details.
  156. (if (module-defined? (resolve-interface '(gnutls))
  157. 'set-session-server-name!)
  158. (set-session-server-name! session server-name-type/dns server)
  159. (format (current-error-port)
  160. "warning: TLS 'SERVER NAME' extension not supported~%"))
  161. (set-session-transport-fd! session (fileno port))
  162. (set-session-default-priority! session)
  163. (set-session-credentials! session (make-certificate-credentials))
  164. ;; Uncomment the following lines in case of debugging emergency.
  165. ;;(set-log-level! 10)
  166. ;;(set-log-procedure! log)
  167. (handshake session)
  168. (let ((record (session-record-port session)))
  169. ;; Since we use `fileno' above, the file descriptor behind PORT would be
  170. ;; closed when PORT is GC'd. If we used `port->fdes', it would instead
  171. ;; never be closed. So we use `fileno', but keep a weak reference to
  172. ;; PORT, so the file descriptor gets closed when RECORD is GC'd.
  173. (add-weak-reference record port)
  174. record)))
  175. (define (open-connection-for-uri uri)
  176. "Like 'open-socket-for-uri', but also handle HTTPS connections."
  177. (define https?
  178. (eq? 'https (uri-scheme uri)))
  179. (let-syntax ((with-https-proxy
  180. (syntax-rules ()
  181. ((_ exp)
  182. ;; For HTTPS URIs, honor 'https_proxy', not 'http_proxy'.
  183. ;; FIXME: Proxying is not supported for https.
  184. (let ((thunk (lambda () exp)))
  185. (if (and https?
  186. (module-variable
  187. (resolve-interface '(web client))
  188. 'current-http-proxy))
  189. (parameterize ((current-http-proxy #f))
  190. (when (getenv "https_proxy")
  191. (format (current-error-port)
  192. "warning: 'https_proxy' is ignored~%"))
  193. (thunk))
  194. (thunk)))))))
  195. (with-https-proxy
  196. (let ((s (open-socket-for-uri uri)))
  197. ;; Buffer input and output on this port.
  198. (setvbuf s _IOFBF %http-receive-buffer-size)
  199. (if https?
  200. (tls-wrap s (uri-host uri))
  201. s)))))
  202. ;; XXX: This is an awful hack to make sure the (set-port-encoding! p
  203. ;; "ISO-8859-1") call in `read-response' passes, even during bootstrap
  204. ;; where iconv is not available.
  205. (module-define! (resolve-module '(web response))
  206. 'set-port-encoding!
  207. (lambda (p e) #f))
  208. ;; XXX: Work around <http://bugs.gnu.org/13095>, present in Guile
  209. ;; up to 2.0.7.
  210. (module-define! (resolve-module '(web client))
  211. 'shutdown (const #f))
  212. ;; XXX: Work around <http://bugs.gnu.org/19840>, present in Guile
  213. ;; up to 2.0.11.
  214. (unless (or (> (string->number (major-version)) 2)
  215. (> (string->number (minor-version)) 0)
  216. (> (string->number (micro-version)) 11))
  217. (let ((var (module-variable (resolve-module '(web http))
  218. 'declare-relative-uri-header!)))
  219. ;; If 'declare-relative-uri-header!' doesn't exist, forget it.
  220. (when (and var (variable-bound? var))
  221. (let ((declare-relative-uri-header! (variable-ref var)))
  222. (declare-relative-uri-header! "Location")))))
  223. (define (resolve-uri-reference ref base)
  224. "Resolve the URI reference REF, interpreted relative to the BASE URI, into a
  225. target URI, according to the algorithm specified in RFC 3986 section 5.2.2.
  226. Return the resulting target URI."
  227. (define (merge-paths base-path rel-path)
  228. (let* ((base-components (string-split base-path #\/))
  229. (base-directory-components (match base-components
  230. ((components ... last) components)
  231. (() '())))
  232. (base-directory (string-join base-directory-components "/")))
  233. (string-append base-directory "/" rel-path)))
  234. (define (remove-dot-segments path)
  235. (let loop ((in
  236. ;; Drop leading "." and ".." components from a relative path.
  237. ;; (absolute paths will start with a "" component)
  238. (drop-while (match-lambda
  239. ((or "." "..") #t)
  240. (_ #f))
  241. (string-split path #\/)))
  242. (out '()))
  243. (match in
  244. (("." . rest)
  245. (loop rest out))
  246. ((".." . rest)
  247. (match out
  248. ((or () (""))
  249. (error "remove-dot-segments: too many '..' components" path))
  250. (_
  251. (loop rest (cdr out)))))
  252. ((component . rest)
  253. (loop rest (cons component out)))
  254. (()
  255. (string-join (reverse out) "/")))))
  256. (cond ((or (uri-scheme ref)
  257. (uri-host ref))
  258. (build-uri (or (uri-scheme ref)
  259. (uri-scheme base))
  260. #:userinfo (uri-userinfo ref)
  261. #:host (uri-host ref)
  262. #:port (uri-port ref)
  263. #:path (remove-dot-segments (uri-path ref))
  264. #:query (uri-query ref)
  265. #:fragment (uri-fragment ref)))
  266. ((string-null? (uri-path ref))
  267. (build-uri (uri-scheme base)
  268. #:userinfo (uri-userinfo base)
  269. #:host (uri-host base)
  270. #:port (uri-port base)
  271. #:path (remove-dot-segments (uri-path base))
  272. #:query (or (uri-query ref)
  273. (uri-query base))
  274. #:fragment (uri-fragment ref)))
  275. (else
  276. (build-uri (uri-scheme base)
  277. #:userinfo (uri-userinfo base)
  278. #:host (uri-host base)
  279. #:port (uri-port base)
  280. #:path (remove-dot-segments
  281. (if (string-prefix? "/" (uri-path ref))
  282. (uri-path ref)
  283. (merge-paths (uri-path base)
  284. (uri-path ref))))
  285. #:query (uri-query ref)
  286. #:fragment (uri-fragment ref)))))
  287. (define (http-fetch uri file)
  288. "Fetch data from URI and write it to FILE. Return FILE on success."
  289. (define post-2.0.7?
  290. (or (> (string->number (major-version)) 2)
  291. (> (string->number (minor-version)) 0)
  292. (> (string->number (micro-version)) 7)
  293. (string>? (version) "2.0.7")))
  294. (define headers
  295. '(;; Some web sites, such as http://dist.schmorp.de, would block you if
  296. ;; there's no 'User-Agent' header, presumably on the assumption that
  297. ;; you're a spammer. So work around that.
  298. (User-Agent . "GNU Guile")
  299. ;; Some servers, such as https://alioth.debian.org, return "406 Not
  300. ;; Acceptable" when not explicitly told that everything is accepted.
  301. (Accept . "*/*")))
  302. (let*-values (((connection)
  303. (open-connection-for-uri uri))
  304. ((resp bv-or-port)
  305. ;; XXX: `http-get*' was introduced in 2.0.7, and replaced by
  306. ;; #:streaming? in 2.0.8. We know we're using it within the
  307. ;; chroot, but `guix-download' might be using a different
  308. ;; version. So keep this compatibility hack for now.
  309. (if post-2.0.7?
  310. (http-get uri #:port connection #:decode-body? #f
  311. #:streaming? #t
  312. #:headers headers)
  313. (if (module-defined? (resolve-interface '(web client))
  314. 'http-get*)
  315. (http-get* uri #:port connection #:decode-body? #f
  316. #:headers headers)
  317. (http-get uri #:port connection #:decode-body? #f
  318. #:extra-headers headers))))
  319. ((code)
  320. (response-code resp))
  321. ((size)
  322. (response-content-length resp)))
  323. (case code
  324. ((200) ; OK
  325. (begin
  326. (call-with-output-file file
  327. (lambda (p)
  328. (if (port? bv-or-port)
  329. (begin
  330. (dump-port bv-or-port p
  331. #:buffer-size %http-receive-buffer-size
  332. #:progress (progress-proc (uri-abbreviation uri)
  333. size))
  334. (newline))
  335. (put-bytevector p bv-or-port))))
  336. file))
  337. ((301 ; moved permanently
  338. 302) ; found (redirection)
  339. (let ((uri (resolve-uri-reference (response-location resp) uri)))
  340. (format #t "following redirection to `~a'...~%"
  341. (uri->string uri))
  342. (close connection)
  343. (http-fetch uri file)))
  344. (else
  345. (error "download failed" (uri->string uri)
  346. code (response-reason-phrase resp))))))
  347. (define-syntax-rule (false-if-exception* body ...)
  348. "Like `false-if-exception', but print the exception on the error port."
  349. (catch #t
  350. (lambda ()
  351. body ...)
  352. (lambda (key . args)
  353. #f)
  354. (lambda (key . args)
  355. (print-exception (current-error-port) #f key args))))
  356. (define (uri-vicinity dir file)
  357. "Concatenate DIR, slash, and FILE, keeping only one slash in between.
  358. This is required by some HTTP servers."
  359. (string-append (string-trim-right dir #\/) "/"
  360. (string-trim file #\/)))
  361. (define (maybe-expand-mirrors uri mirrors)
  362. "If URI uses the 'mirror' scheme, expand it according to the MIRRORS alist.
  363. Return a list of URIs."
  364. (case (uri-scheme uri)
  365. ((mirror)
  366. (let ((kind (string->symbol (uri-host uri)))
  367. (path (uri-path uri)))
  368. (match (assoc-ref mirrors kind)
  369. ((mirrors ..1)
  370. (map (compose string->uri (cut uri-vicinity <> path))
  371. mirrors))
  372. (_
  373. (error "unsupported URL mirror kind" kind uri)))))
  374. (else
  375. (list uri))))
  376. (define* (url-fetch url file #:key (mirrors '()))
  377. "Fetch FILE from URL; URL may be either a single string, or a list of
  378. string denoting alternate URLs for FILE. Return #f on failure, and FILE
  379. on success."
  380. (define uri
  381. (append-map (cut maybe-expand-mirrors <> mirrors)
  382. (match url
  383. ((_ ...) (map string->uri url))
  384. (_ (list (string->uri url))))))
  385. (define (fetch uri file)
  386. (format #t "starting download of `~a' from `~a'...~%"
  387. file (uri->string uri))
  388. (case (uri-scheme uri)
  389. ((http https)
  390. (false-if-exception* (http-fetch uri file)))
  391. ((ftp)
  392. (false-if-exception* (ftp-fetch uri file)))
  393. (else
  394. (format #t "skipping URI with unsupported scheme: ~s~%"
  395. uri)
  396. #f)))
  397. ;; Make this unbuffered so 'progress-proc' works as expected. _IOLBF means
  398. ;; '\n', not '\r', so it's not appropriate here.
  399. (setvbuf (current-output-port) _IONBF)
  400. (setvbuf (current-error-port) _IOLBF)
  401. (let try ((uri uri))
  402. (match uri
  403. ((uri tail ...)
  404. (or (fetch uri file)
  405. (try tail)))
  406. (()
  407. (format (current-error-port) "failed to download ~s from ~s~%"
  408. file url)
  409. #f))))
  410. ;;; Local Variables:
  411. ;;; eval: (put 'with-elapsed-time 'scheme-indent-function 1)
  412. ;;; End:
  413. ;;; download.scm ends here