Mirror of GNU Guix
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

481 lines
19 KiB

  1. ;;; GNU Guix --- Functional package management for GNU
  2. ;;; Copyright © 2012, 2013, 2014, 2015 Ludovic Courtès <ludo@gnu.org>
  3. ;;; Copyright © 2015 Mark H Weaver <mhw@netris.org>
  4. ;;;
  5. ;;; This file is part of GNU Guix.
  6. ;;;
  7. ;;; GNU Guix is free software; you can redistribute it and/or modify it
  8. ;;; under the terms of the GNU General Public License as published by
  9. ;;; the Free Software Foundation; either version 3 of the License, or (at
  10. ;;; your option) any later version.
  11. ;;;
  12. ;;; GNU Guix is distributed in the hope that it will be useful, but
  13. ;;; WITHOUT ANY WARRANTY; without even the implied warranty of
  14. ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15. ;;; GNU General Public License for more details.
  16. ;;;
  17. ;;; You should have received a copy of the GNU General Public License
  18. ;;; along with GNU Guix. If not, see <http://www.gnu.org/licenses/>.
  19. (define-module (guix build download)
  20. #:use-module (web uri)
  21. #:use-module ((web client) #:hide (open-socket-for-uri))
  22. #:use-module (web response)
  23. #:use-module (guix ftp-client)
  24. #:use-module (guix build utils)
  25. #:use-module (rnrs io ports)
  26. #:use-module (srfi srfi-1)
  27. #:use-module (srfi srfi-11)
  28. #:use-module (srfi srfi-19)
  29. #:use-module (srfi srfi-26)
  30. #:use-module (ice-9 match)
  31. #:use-module (ice-9 format)
  32. #:export (open-socket-for-uri
  33. open-connection-for-uri
  34. resolve-uri-reference
  35. maybe-expand-mirrors
  36. url-fetch
  37. progress-proc
  38. uri-abbreviation))
  39. ;;; Commentary:
  40. ;;;
  41. ;;; Fetch data such as tarballs over HTTP or FTP (builder-side code).
  42. ;;;
  43. ;;; Code:
  44. (define %http-receive-buffer-size
  45. ;; Size of the HTTP receive buffer.
  46. 65536)
  47. (define (duration->seconds duration)
  48. "Return the number of seconds represented by DURATION, a 'time-duration'
  49. object, as an inexact number."
  50. (+ (time-second duration)
  51. (/ (time-nanosecond duration) 1e9)))
  52. (define (throughput->string throughput)
  53. "Given THROUGHPUT, measured in bytes per second, return a string
  54. representing it in a human-readable way."
  55. (if (> throughput 3e6)
  56. (format #f "~,2f MiB/s" (/ throughput (expt 2. 20)))
  57. (format #f "~,0f KiB/s" (/ throughput 1024.0))))
  58. (define* (progress-proc file size #:optional (log-port (current-output-port)))
  59. "Return a procedure to show the progress of FILE's download, which is
  60. SIZE byte long. The returned procedure is suitable for use as an
  61. argument to `dump-port'. The progress report is written to LOG-PORT."
  62. ;; XXX: Because of <http://bugs.gnu.org/19939> this procedure is often not
  63. ;; called as frequently as we'd like too; this is especially bad with Nginx
  64. ;; on hydra.gnu.org, which returns whole nars as a single chunk.
  65. (let ((start-time #f))
  66. (let-syntax ((with-elapsed-time
  67. (syntax-rules ()
  68. ((_ elapsed body ...)
  69. (let* ((now (current-time time-monotonic))
  70. (elapsed (and start-time
  71. (duration->seconds
  72. (time-difference now
  73. start-time)))))
  74. (unless start-time
  75. (set! start-time now))
  76. body ...)))))
  77. (if (number? size)
  78. (lambda (transferred cont)
  79. (with-elapsed-time elapsed
  80. (let ((% (* 100.0 (/ transferred size)))
  81. (throughput (if elapsed
  82. (/ transferred elapsed)
  83. 0)))
  84. (display #\cr log-port)
  85. (format log-port "~a\t~5,1f% of ~,1f KiB (~a)"
  86. file % (/ size 1024.0)
  87. (throughput->string throughput))
  88. (flush-output-port log-port)
  89. (cont))))
  90. (lambda (transferred cont)
  91. (with-elapsed-time elapsed
  92. (let ((throughput (if elapsed
  93. (/ transferred elapsed)
  94. 0)))
  95. (display #\cr log-port)
  96. (format log-port "~a\t~6,1f KiB transferred (~a)"
  97. file (/ transferred 1024.0)
  98. (throughput->string throughput))
  99. (flush-output-port log-port)
  100. (cont))))))))
  101. (define* (uri-abbreviation uri #:optional (max-length 42))
  102. "If URI's string representation is larger than MAX-LENGTH, return an
  103. abbreviation of URI showing the scheme, host, and basename of the file."
  104. (define uri-as-string
  105. (uri->string uri))
  106. (define (elide-path)
  107. (let ((path (uri-path uri)))
  108. (string-append (symbol->string (uri-scheme uri)) "://"
  109. ;; `file' URIs have no host part.
  110. (or (uri-host uri) "")
  111. (string-append "/.../" (basename path)))))
  112. (if (> (string-length uri-as-string) max-length)
  113. (let ((short (elide-path)))
  114. (if (< (string-length short) (string-length uri-as-string))
  115. short
  116. uri-as-string))
  117. uri-as-string))
  118. (define (ftp-fetch uri file)
  119. "Fetch data from URI and write it to FILE. Return FILE on success."
  120. (let* ((conn (ftp-open (uri-host uri)))
  121. (size (false-if-exception (ftp-size conn (uri-path uri))))
  122. (in (ftp-retr conn (basename (uri-path uri))
  123. (dirname (uri-path uri)))))
  124. (call-with-output-file file
  125. (lambda (out)
  126. (dump-port in out
  127. #:buffer-size %http-receive-buffer-size
  128. #:progress (progress-proc (uri-abbreviation uri) size))))
  129. (ftp-close conn))
  130. (newline)
  131. file)
  132. ;; Autoload GnuTLS so that this module can be used even when GnuTLS is
  133. ;; not available. At compile time, this yields "possibly unbound
  134. ;; variable" warnings, but these are OK: we know that the variables will
  135. ;; be bound if we need them, because (guix download) adds GnuTLS as an
  136. ;; input in that case.
  137. ;; XXX: Use this hack instead of #:autoload to avoid compilation errors.
  138. ;; See <http://bugs.gnu.org/12202>.
  139. (module-autoload! (current-module)
  140. '(gnutls) '(make-session connection-end/client))
  141. (define add-weak-reference
  142. (let ((table (make-weak-key-hash-table)))
  143. (lambda (from to)
  144. "Hold a weak reference from FROM to TO."
  145. (hashq-set! table from to))))
  146. (define (tls-wrap port server)
  147. "Return PORT wrapped in a TLS connection to SERVER. SERVER must be a DNS
  148. host name without trailing dot."
  149. (define (log level str)
  150. (format (current-error-port)
  151. "gnutls: [~a|~a] ~a" (getpid) level str))
  152. (let ((session (make-session connection-end/client)))
  153. ;; Some servers such as 'cloud.github.com' require the client to support
  154. ;; the 'SERVER NAME' extension. However, 'set-session-server-name!' is
  155. ;; not available in older GnuTLS releases. See
  156. ;; <http://bugs.gnu.org/18526> for details.
  157. (if (module-defined? (resolve-interface '(gnutls))
  158. 'set-session-server-name!)
  159. (set-session-server-name! session server-name-type/dns server)
  160. (format (current-error-port)
  161. "warning: TLS 'SERVER NAME' extension not supported~%"))
  162. (set-session-transport-fd! session (fileno port))
  163. (set-session-default-priority! session)
  164. (set-session-credentials! session (make-certificate-credentials))
  165. ;; Uncomment the following lines in case of debugging emergency.
  166. ;;(set-log-level! 10)
  167. ;;(set-log-procedure! log)
  168. (handshake session)
  169. (let ((record (session-record-port session)))
  170. ;; Since we use `fileno' above, the file descriptor behind PORT would be
  171. ;; closed when PORT is GC'd. If we used `port->fdes', it would instead
  172. ;; never be closed. So we use `fileno', but keep a weak reference to
  173. ;; PORT, so the file descriptor gets closed when RECORD is GC'd.
  174. (add-weak-reference record port)
  175. record)))
  176. (define (open-socket-for-uri uri)
  177. "Return an open port for URI. This variant works around
  178. <http://bugs.gnu.org/15368> which affects Guile's 'open-socket-for-uri' up to
  179. 2.0.11 included."
  180. (define rmem-max
  181. ;; The maximum size for a receive buffer on Linux, see socket(7).
  182. "/proc/sys/net/core/rmem_max")
  183. (define buffer-size
  184. (if (file-exists? rmem-max)
  185. (call-with-input-file rmem-max read)
  186. 126976)) ;the default for Linux, per 'rmem_default'
  187. (let ((s ((@ (web client) open-socket-for-uri) uri)))
  188. ;; Work around <http://bugs.gnu.org/15368> by restoring a decent
  189. ;; buffer size.
  190. (setsockopt s SOL_SOCKET SO_RCVBUF buffer-size)
  191. s))
  192. (define (open-connection-for-uri uri)
  193. "Like 'open-socket-for-uri', but also handle HTTPS connections."
  194. (define https?
  195. (eq? 'https (uri-scheme uri)))
  196. (let-syntax ((with-https-proxy
  197. (syntax-rules ()
  198. ((_ exp)
  199. ;; For HTTPS URIs, honor 'https_proxy', not 'http_proxy'.
  200. ;; FIXME: Proxying is not supported for https.
  201. (let ((thunk (lambda () exp)))
  202. (if (and https?
  203. (module-variable
  204. (resolve-interface '(web client))
  205. 'current-http-proxy))
  206. (parameterize ((current-http-proxy #f))
  207. (when (getenv "https_proxy")
  208. (format (current-error-port)
  209. "warning: 'https_proxy' is ignored~%"))
  210. (thunk))
  211. (thunk)))))))
  212. (with-https-proxy
  213. (let ((s (open-socket-for-uri uri)))
  214. ;; Buffer input and output on this port.
  215. (setvbuf s _IOFBF %http-receive-buffer-size)
  216. (if https?
  217. (tls-wrap s (uri-host uri))
  218. s)))))
  219. ;; XXX: This is an awful hack to make sure the (set-port-encoding! p
  220. ;; "ISO-8859-1") call in `read-response' passes, even during bootstrap
  221. ;; where iconv is not available.
  222. (module-define! (resolve-module '(web response))
  223. 'set-port-encoding!
  224. (lambda (p e) #f))
  225. ;; XXX: Work around <http://bugs.gnu.org/13095>, present in Guile
  226. ;; up to 2.0.7.
  227. (module-define! (resolve-module '(web client))
  228. 'shutdown (const #f))
  229. ;; XXX: Work around <http://bugs.gnu.org/19840>, present in Guile
  230. ;; up to 2.0.11.
  231. (unless (or (> (string->number (major-version)) 2)
  232. (> (string->number (minor-version)) 0)
  233. (> (string->number (micro-version)) 11))
  234. (let ((var (module-variable (resolve-module '(web http))
  235. 'declare-relative-uri-header!)))
  236. ;; If 'declare-relative-uri-header!' doesn't exist, forget it.
  237. (when (and var (variable-bound? var))
  238. (let ((declare-relative-uri-header! (variable-ref var)))
  239. (declare-relative-uri-header! "Location")))))
  240. (define (resolve-uri-reference ref base)
  241. "Resolve the URI reference REF, interpreted relative to the BASE URI, into a
  242. target URI, according to the algorithm specified in RFC 3986 section 5.2.2.
  243. Return the resulting target URI."
  244. (define (merge-paths base-path rel-path)
  245. (let* ((base-components (string-split base-path #\/))
  246. (base-directory-components (match base-components
  247. ((components ... last) components)
  248. (() '())))
  249. (base-directory (string-join base-directory-components "/")))
  250. (string-append base-directory "/" rel-path)))
  251. (define (remove-dot-segments path)
  252. (let loop ((in
  253. ;; Drop leading "." and ".." components from a relative path.
  254. ;; (absolute paths will start with a "" component)
  255. (drop-while (match-lambda
  256. ((or "." "..") #t)
  257. (_ #f))
  258. (string-split path #\/)))
  259. (out '()))
  260. (match in
  261. (("." . rest)
  262. (loop rest out))
  263. ((".." . rest)
  264. (match out
  265. ((or () (""))
  266. (error "remove-dot-segments: too many '..' components" path))
  267. (_
  268. (loop rest (cdr out)))))
  269. ((component . rest)
  270. (loop rest (cons component out)))
  271. (()
  272. (string-join (reverse out) "/")))))
  273. (cond ((or (uri-scheme ref)
  274. (uri-host ref))
  275. (build-uri (or (uri-scheme ref)
  276. (uri-scheme base))
  277. #:userinfo (uri-userinfo ref)
  278. #:host (uri-host ref)
  279. #:port (uri-port ref)
  280. #:path (remove-dot-segments (uri-path ref))
  281. #:query (uri-query ref)
  282. #:fragment (uri-fragment ref)))
  283. ((string-null? (uri-path ref))
  284. (build-uri (uri-scheme base)
  285. #:userinfo (uri-userinfo base)
  286. #:host (uri-host base)
  287. #:port (uri-port base)
  288. #:path (remove-dot-segments (uri-path base))
  289. #:query (or (uri-query ref)
  290. (uri-query base))
  291. #:fragment (uri-fragment ref)))
  292. (else
  293. (build-uri (uri-scheme base)
  294. #:userinfo (uri-userinfo base)
  295. #:host (uri-host base)
  296. #:port (uri-port base)
  297. #:path (remove-dot-segments
  298. (if (string-prefix? "/" (uri-path ref))
  299. (uri-path ref)
  300. (merge-paths (uri-path base)
  301. (uri-path ref))))
  302. #:query (uri-query ref)
  303. #:fragment (uri-fragment ref)))))
  304. (define (http-fetch uri file)
  305. "Fetch data from URI and write it to FILE. Return FILE on success."
  306. (define post-2.0.7?
  307. (or (> (string->number (major-version)) 2)
  308. (> (string->number (minor-version)) 0)
  309. (> (string->number (micro-version)) 7)
  310. (string>? (version) "2.0.7")))
  311. (define headers
  312. '(;; Some web sites, such as http://dist.schmorp.de, would block you if
  313. ;; there's no 'User-Agent' header, presumably on the assumption that
  314. ;; you're a spammer. So work around that.
  315. (User-Agent . "GNU Guile")
  316. ;; Some servers, such as https://alioth.debian.org, return "406 Not
  317. ;; Acceptable" when not explicitly told that everything is accepted.
  318. (Accept . "*/*")))
  319. (let*-values (((connection)
  320. (open-connection-for-uri uri))
  321. ((resp bv-or-port)
  322. ;; XXX: `http-get*' was introduced in 2.0.7, and replaced by
  323. ;; #:streaming? in 2.0.8. We know we're using it within the
  324. ;; chroot, but `guix-download' might be using a different
  325. ;; version. So keep this compatibility hack for now.
  326. (if post-2.0.7?
  327. (http-get uri #:port connection #:decode-body? #f
  328. #:streaming? #t
  329. #:headers headers)
  330. (if (module-defined? (resolve-interface '(web client))
  331. 'http-get*)
  332. (http-get* uri #:port connection #:decode-body? #f
  333. #:headers headers)
  334. (http-get uri #:port connection #:decode-body? #f
  335. #:extra-headers headers))))
  336. ((code)
  337. (response-code resp))
  338. ((size)
  339. (response-content-length resp)))
  340. (case code
  341. ((200) ; OK
  342. (begin
  343. (call-with-output-file file
  344. (lambda (p)
  345. (if (port? bv-or-port)
  346. (begin
  347. (dump-port bv-or-port p
  348. #:buffer-size %http-receive-buffer-size
  349. #:progress (progress-proc (uri-abbreviation uri)
  350. size))
  351. (newline))
  352. (put-bytevector p bv-or-port))))
  353. file))
  354. ((301 ; moved permanently
  355. 302) ; found (redirection)
  356. (let ((uri (resolve-uri-reference (response-location resp) uri)))
  357. (format #t "following redirection to `~a'...~%"
  358. (uri->string uri))
  359. (close connection)
  360. (http-fetch uri file)))
  361. (else
  362. (error "download failed" (uri->string uri)
  363. code (response-reason-phrase resp))))))
  364. (define-syntax-rule (false-if-exception* body ...)
  365. "Like `false-if-exception', but print the exception on the error port."
  366. (catch #t
  367. (lambda ()
  368. body ...)
  369. (lambda (key . args)
  370. #f)
  371. (lambda (key . args)
  372. (print-exception (current-error-port) #f key args))))
  373. (define (uri-vicinity dir file)
  374. "Concatenate DIR, slash, and FILE, keeping only one slash in between.
  375. This is required by some HTTP servers."
  376. (string-append (string-trim-right dir #\/) "/"
  377. (string-trim file #\/)))
  378. (define (maybe-expand-mirrors uri mirrors)
  379. "If URI uses the 'mirror' scheme, expand it according to the MIRRORS alist.
  380. Return a list of URIs."
  381. (case (uri-scheme uri)
  382. ((mirror)
  383. (let ((kind (string->symbol (uri-host uri)))
  384. (path (uri-path uri)))
  385. (match (assoc-ref mirrors kind)
  386. ((mirrors ..1)
  387. (map (compose string->uri (cut uri-vicinity <> path))
  388. mirrors))
  389. (_
  390. (error "unsupported URL mirror kind" kind uri)))))
  391. (else
  392. (list uri))))
  393. (define* (url-fetch url file #:key (mirrors '()))
  394. "Fetch FILE from URL; URL may be either a single string, or a list of
  395. string denoting alternate URLs for FILE. Return #f on failure, and FILE
  396. on success."
  397. (define uri
  398. (append-map (cut maybe-expand-mirrors <> mirrors)
  399. (match url
  400. ((_ ...) (map string->uri url))
  401. (_ (list (string->uri url))))))
  402. (define (fetch uri file)
  403. (format #t "starting download of `~a' from `~a'...~%"
  404. file (uri->string uri))
  405. (case (uri-scheme uri)
  406. ((http https)
  407. (false-if-exception* (http-fetch uri file)))
  408. ((ftp)
  409. (false-if-exception* (ftp-fetch uri file)))
  410. (else
  411. (format #t "skipping URI with unsupported scheme: ~s~%"
  412. uri)
  413. #f)))
  414. ;; Make this unbuffered so 'progress-proc' works as expected. _IOLBF means
  415. ;; '\n', not '\r', so it's not appropriate here.
  416. (setvbuf (current-output-port) _IONBF)
  417. (setvbuf (current-error-port) _IOLBF)
  418. (let try ((uri uri))
  419. (match uri
  420. ((uri tail ...)
  421. (or (fetch uri file)
  422. (try tail)))
  423. (()
  424. (format (current-error-port) "failed to download ~s from ~s~%"
  425. file url)
  426. #f))))
  427. ;;; Local Variables:
  428. ;;; eval: (put 'with-elapsed-time 'scheme-indent-function 1)
  429. ;;; End:
  430. ;;; download.scm ends here