123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730 |
- /*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright 2005 Colin Percival
- * All rights reserved
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted providing that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
- * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
- * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
- #include <sys/types.h>
- #include <sys/time.h>
- #include <sys/socket.h>
- #include <ctype.h>
- #include <err.h>
- #include <errno.h>
- #include <fcntl.h>
- #include <limits.h>
- #include <netdb.h>
- #include <stdint.h>
- #include <stdio.h>
- #include <stdlib.h>
- #include <string.h>
- #include <sysexits.h>
- #include <unistd.h>
- static const char * env_HTTP_PROXY;
- static char * env_HTTP_PROXY_AUTH;
- static const char * env_HTTP_USER_AGENT;
- static char * env_HTTP_TIMEOUT;
- static const char * proxyport;
- static char * proxyauth;
- static struct timeval timo = { 15, 0};
- static void
- usage(void)
- {
- fprintf(stderr, "usage: phttpget server [file ...]\n");
- exit(EX_USAGE);
- }
- /*
- * Base64 encode a string; the string returned, if non-NULL, is
- * allocated using malloc() and must be freed by the caller.
- */
- static char *
- b64enc(const char *ptext)
- {
- static const char base64[] =
- "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
- "abcdefghijklmnopqrstuvwxyz"
- "0123456789+/";
- const char *pt;
- char *ctext, *pc;
- size_t ptlen, ctlen;
- uint32_t t;
- unsigned int j;
- /*
- * Encoded length is 4 characters per 3-byte block or partial
- * block of plaintext, plus one byte for the terminating NUL
- */
- ptlen = strlen(ptext);
- if (ptlen > ((SIZE_MAX - 1) / 4) * 3 - 2)
- return NULL; /* Possible integer overflow */
- ctlen = 4 * ((ptlen + 2) / 3) + 1;
- if ((ctext = malloc(ctlen)) == NULL)
- return NULL;
- ctext[ctlen - 1] = 0;
- /*
- * Scan through ptext, reading up to 3 bytes from ptext and
- * writing 4 bytes to ctext, until we run out of input.
- */
- for (pt = ptext, pc = ctext; ptlen; ptlen -= 3, pc += 4) {
- /* Read 3 bytes */
- for (t = j = 0; j < 3; j++) {
- t <<= 8;
- if (j < ptlen)
- t += *pt++;
- }
- /* Write 4 bytes */
- for (j = 0; j < 4; j++) {
- if (j <= ptlen + 1)
- pc[j] = base64[(t >> 18) & 0x3f];
- else
- pc[j] = '=';
- t <<= 6;
- }
- /* If we're done, exit the loop */
- if (ptlen <= 3)
- break;
- }
- return (ctext);
- }
- static void
- readenv(void)
- {
- char *proxy_auth_userpass, *proxy_auth_userpass64, *p;
- char *proxy_auth_user = NULL;
- char *proxy_auth_pass = NULL;
- long http_timeout;
- env_HTTP_PROXY = getenv("HTTP_PROXY");
- if (env_HTTP_PROXY == NULL)
- env_HTTP_PROXY = getenv("http_proxy");
- if (env_HTTP_PROXY != NULL) {
- if (strncmp(env_HTTP_PROXY, "http://", 7) == 0)
- env_HTTP_PROXY += 7;
- p = strchr(env_HTTP_PROXY, '/');
- if (p != NULL)
- *p = 0;
- p = strchr(env_HTTP_PROXY, ':');
- if (p != NULL) {
- *p = 0;
- proxyport = p + 1;
- } else
- proxyport = "3128";
- }
- env_HTTP_PROXY_AUTH = getenv("HTTP_PROXY_AUTH");
- if ((env_HTTP_PROXY != NULL) &&
- (env_HTTP_PROXY_AUTH != NULL) &&
- (strncasecmp(env_HTTP_PROXY_AUTH, "basic:" , 6) == 0)) {
- /* Ignore authentication scheme */
- (void) strsep(&env_HTTP_PROXY_AUTH, ":");
- /* Ignore realm */
- (void) strsep(&env_HTTP_PROXY_AUTH, ":");
- /* Obtain username and password */
- proxy_auth_user = strsep(&env_HTTP_PROXY_AUTH, ":");
- proxy_auth_pass = env_HTTP_PROXY_AUTH;
- }
- if ((proxy_auth_user != NULL) && (proxy_auth_pass != NULL)) {
- asprintf(&proxy_auth_userpass, "%s:%s",
- proxy_auth_user, proxy_auth_pass);
- if (proxy_auth_userpass == NULL)
- err(1, "asprintf");
- proxy_auth_userpass64 = b64enc(proxy_auth_userpass);
- if (proxy_auth_userpass64 == NULL)
- err(1, "malloc");
- asprintf(&proxyauth, "Proxy-Authorization: Basic %s\r\n",
- proxy_auth_userpass64);
- if (proxyauth == NULL)
- err(1, "asprintf");
- free(proxy_auth_userpass);
- free(proxy_auth_userpass64);
- } else
- proxyauth = NULL;
- env_HTTP_USER_AGENT = getenv("HTTP_USER_AGENT");
- if (env_HTTP_USER_AGENT == NULL)
- env_HTTP_USER_AGENT = "phttpget/0.1";
- env_HTTP_TIMEOUT = getenv("HTTP_TIMEOUT");
- if (env_HTTP_TIMEOUT != NULL) {
- http_timeout = strtol(env_HTTP_TIMEOUT, &p, 10);
- if ((*env_HTTP_TIMEOUT == '\0') || (*p != '\0') ||
- (http_timeout < 0))
- warnx("HTTP_TIMEOUT (%s) is not a positive integer",
- env_HTTP_TIMEOUT);
- else
- timo.tv_sec = http_timeout;
- }
- }
- static int
- makerequest(char ** buf, char * path, char * server, int connclose)
- {
- int buflen;
- buflen = asprintf(buf,
- "GET %s%s/%s HTTP/1.1\r\n"
- "Host: %s\r\n"
- "User-Agent: %s\r\n"
- "%s"
- "%s"
- "\r\n",
- env_HTTP_PROXY ? "http://" : "",
- env_HTTP_PROXY ? server : "",
- path, server, env_HTTP_USER_AGENT,
- proxyauth ? proxyauth : "",
- connclose ? "Connection: Close\r\n" : "Connection: Keep-Alive\r\n");
- if (buflen == -1)
- err(1, "asprintf");
- return(buflen);
- }
- static int
- readln(int sd, char * resbuf, int * resbuflen, int * resbufpos)
- {
- ssize_t len;
- while (strnstr(resbuf + *resbufpos, "\r\n",
- *resbuflen - *resbufpos) == NULL) {
- /* Move buffered data to the start of the buffer */
- if (*resbufpos != 0) {
- memmove(resbuf, resbuf + *resbufpos,
- *resbuflen - *resbufpos);
- *resbuflen -= *resbufpos;
- *resbufpos = 0;
- }
- /* If the buffer is full, complain */
- if (*resbuflen == BUFSIZ)
- return -1;
- /* Read more data into the buffer */
- len = recv(sd, resbuf + *resbuflen, BUFSIZ - *resbuflen, 0);
- if ((len == 0) ||
- ((len == -1) && (errno != EINTR)))
- return -1;
- if (len != -1)
- *resbuflen += len;
- }
- return 0;
- }
- static int
- copybytes(int sd, int fd, off_t copylen, char * resbuf, int * resbuflen,
- int * resbufpos)
- {
- ssize_t len;
- while (copylen) {
- /* Write data from resbuf to fd */
- len = *resbuflen - *resbufpos;
- if (copylen < len)
- len = copylen;
- if (len > 0) {
- if (fd != -1)
- len = write(fd, resbuf + *resbufpos, len);
- if (len == -1)
- err(1, "write");
- *resbufpos += len;
- copylen -= len;
- continue;
- }
- /* Read more data into buffer */
- len = recv(sd, resbuf, BUFSIZ, 0);
- if (len == -1) {
- if (errno == EINTR)
- continue;
- return -1;
- } else if (len == 0) {
- return -2;
- } else {
- *resbuflen = len;
- *resbufpos = 0;
- }
- }
- return 0;
- }
- int
- main(int argc, char *argv[])
- {
- struct addrinfo hints; /* Hints to getaddrinfo */
- struct addrinfo *res; /* Pointer to server address being used */
- struct addrinfo *res0; /* Pointer to server addresses */
- char * resbuf = NULL; /* Response buffer */
- int resbufpos = 0; /* Response buffer position */
- int resbuflen = 0; /* Response buffer length */
- char * eolp; /* Pointer to "\r\n" within resbuf */
- char * hln; /* Pointer within header line */
- char * servername; /* Name of server */
- char * fname = NULL; /* Name of downloaded file */
- char * reqbuf = NULL; /* Request buffer */
- int reqbufpos = 0; /* Request buffer position */
- int reqbuflen = 0; /* Request buffer length */
- ssize_t len; /* Length sent or received */
- int nreq = 0; /* Number of next request to send */
- int nres = 0; /* Number of next reply to receive */
- int pipelined = 0; /* != 0 if connection in pipelined mode. */
- int keepalive; /* != 0 if HTTP/1.0 keep-alive rcvd. */
- int sd = -1; /* Socket descriptor */
- int sdflags = 0; /* Flags on the socket sd */
- int fd = -1; /* Descriptor for download target file */
- int error; /* Error code */
- int statuscode; /* HTTP Status code */
- off_t contentlength; /* Value from Content-Length header */
- int chunked; /* != if transfer-encoding is chunked */
- off_t clen; /* Chunk length */
- int firstreq = 0; /* # of first request for this connection */
- int val; /* Value used for setsockopt call */
- /* Check that the arguments are sensible */
- if (argc < 2)
- usage();
- /* Read important environment variables */
- readenv();
- /* Get server name and adjust arg[cv] to point at file names */
- servername = argv[1];
- argv += 2;
- argc -= 2;
- /* Allocate response buffer */
- resbuf = malloc(BUFSIZ);
- if (resbuf == NULL)
- err(1, "malloc");
- /* Look up server */
- memset(&hints, 0, sizeof(hints));
- hints.ai_family = PF_UNSPEC;
- hints.ai_socktype = SOCK_STREAM;
- error = getaddrinfo(env_HTTP_PROXY ? env_HTTP_PROXY : servername,
- env_HTTP_PROXY ? proxyport : "http", &hints, &res0);
- if (error)
- errx(1, "host = %s, port = %s: %s",
- env_HTTP_PROXY ? env_HTTP_PROXY : servername,
- env_HTTP_PROXY ? proxyport : "http",
- gai_strerror(error));
- if (res0 == NULL)
- errx(1, "could not look up %s", servername);
- res = res0;
- /* Do the fetching */
- while (nres < argc) {
- /* Make sure we have a connected socket */
- for (; sd == -1; res = res->ai_next) {
- /* No addresses left to try :-( */
- if (res == NULL)
- errx(1, "Could not connect to %s", servername);
- /* Create a socket... */
- sd = socket(res->ai_family, res->ai_socktype,
- res->ai_protocol);
- if (sd == -1)
- continue;
- /* ... set 15-second timeouts ... */
- setsockopt(sd, SOL_SOCKET, SO_SNDTIMEO,
- (void *)&timo, (socklen_t)sizeof(timo));
- setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO,
- (void *)&timo, (socklen_t)sizeof(timo));
- /* ... disable SIGPIPE generation ... */
- val = 1;
- setsockopt(sd, SOL_SOCKET, SO_NOSIGPIPE,
- (void *)&val, sizeof(int));
- /* ... and connect to the server. */
- if(connect(sd, res->ai_addr, res->ai_addrlen)) {
- close(sd);
- sd = -1;
- continue;
- }
- firstreq = nres;
- }
- /*
- * If in pipelined HTTP mode, put socket into non-blocking
- * mode, since we're probably going to want to try to send
- * several HTTP requests.
- */
- if (pipelined) {
- sdflags = fcntl(sd, F_GETFL);
- if (fcntl(sd, F_SETFL, sdflags | O_NONBLOCK) == -1)
- err(1, "fcntl");
- }
- /* Construct requests and/or send them without blocking */
- while ((nreq < argc) && ((reqbuf == NULL) || pipelined)) {
- /* If not in the middle of a request, make one */
- if (reqbuf == NULL) {
- reqbuflen = makerequest(&reqbuf, argv[nreq],
- servername, (nreq == argc - 1));
- reqbufpos = 0;
- }
- /* If in pipelined mode, try to send the request */
- if (pipelined) {
- while (reqbufpos < reqbuflen) {
- len = send(sd, reqbuf + reqbufpos,
- reqbuflen - reqbufpos, 0);
- if (len == -1)
- break;
- reqbufpos += len;
- }
- if (reqbufpos < reqbuflen) {
- if (errno != EAGAIN)
- goto conndied;
- break;
- } else {
- free(reqbuf);
- reqbuf = NULL;
- nreq++;
- }
- }
- }
- /* Put connection back into blocking mode */
- if (pipelined) {
- if (fcntl(sd, F_SETFL, sdflags) == -1)
- err(1, "fcntl");
- }
- /* Do we need to blocking-send a request? */
- if (nres == nreq) {
- while (reqbufpos < reqbuflen) {
- len = send(sd, reqbuf + reqbufpos,
- reqbuflen - reqbufpos, 0);
- if (len == -1)
- goto conndied;
- reqbufpos += len;
- }
- free(reqbuf);
- reqbuf = NULL;
- nreq++;
- }
- /* Scan through the response processing headers. */
- statuscode = 0;
- contentlength = -1;
- chunked = 0;
- keepalive = 0;
- do {
- /* Get a header line */
- error = readln(sd, resbuf, &resbuflen, &resbufpos);
- if (error)
- goto conndied;
- hln = resbuf + resbufpos;
- eolp = strnstr(hln, "\r\n", resbuflen - resbufpos);
- resbufpos = (eolp - resbuf) + 2;
- *eolp = '\0';
- /* Make sure it doesn't contain a NUL character */
- if (strchr(hln, '\0') != eolp)
- goto conndied;
- if (statuscode == 0) {
- /* The first line MUST be HTTP/1.x xxx ... */
- if ((strncmp(hln, "HTTP/1.", 7) != 0) ||
- ! isdigit(hln[7]))
- goto conndied;
- /*
- * If the minor version number isn't zero,
- * then we can assume that pipelining our
- * requests is OK -- as long as we don't
- * see a "Connection: close" line later
- * and we either have a Content-Length or
- * Transfer-Encoding: chunked header to
- * tell us the length.
- */
- if (hln[7] != '0')
- pipelined = 1;
- /* Skip over the minor version number */
- hln = strchr(hln + 7, ' ');
- if (hln == NULL)
- goto conndied;
- else
- hln++;
- /* Read the status code */
- while (isdigit(*hln)) {
- statuscode = statuscode * 10 +
- *hln - '0';
- hln++;
- }
- if (statuscode < 100 || statuscode > 599)
- goto conndied;
- /* Ignore the rest of the line */
- continue;
- }
- /*
- * Check for "Connection: close" or
- * "Connection: Keep-Alive" header
- */
- if (strncasecmp(hln, "Connection:", 11) == 0) {
- hln += 11;
- if (strcasestr(hln, "close") != NULL)
- pipelined = 0;
- if (strcasestr(hln, "Keep-Alive") != NULL)
- keepalive = 1;
- /* Next header... */
- continue;
- }
- /* Check for "Content-Length:" header */
- if (strncasecmp(hln, "Content-Length:", 15) == 0) {
- hln += 15;
- contentlength = 0;
- /* Find the start of the length */
- while (!isdigit(*hln) && (*hln != '\0'))
- hln++;
- /* Compute the length */
- while (isdigit(*hln)) {
- if (contentlength >= OFF_MAX / 10) {
- /* Nasty people... */
- goto conndied;
- }
- contentlength = contentlength * 10 +
- *hln - '0';
- hln++;
- }
- /* Next header... */
- continue;
- }
- /* Check for "Transfer-Encoding: chunked" header */
- if (strncasecmp(hln, "Transfer-Encoding:", 18) == 0) {
- hln += 18;
- if (strcasestr(hln, "chunked") != NULL)
- chunked = 1;
- /* Next header... */
- continue;
- }
- /* We blithely ignore any other header lines */
- /* No more header lines */
- if (strlen(hln) == 0) {
- /*
- * If the status code was 1xx, then there will
- * be a real header later. Servers may emit
- * 1xx header blocks at will, but since we
- * don't expect one, we should just ignore it.
- */
- if (100 <= statuscode && statuscode <= 199) {
- statuscode = 0;
- continue;
- }
- /* End of header; message body follows */
- break;
- }
- } while (1);
- /* No message body for 204 or 304 */
- if (statuscode == 204 || statuscode == 304) {
- nres++;
- continue;
- }
- /*
- * There should be a message body coming, but we only want
- * to send it to a file if the status code is 200
- */
- if (statuscode == 200) {
- /* Generate a file name for the download */
- fname = strrchr(argv[nres], '/');
- if (fname == NULL)
- fname = argv[nres];
- else
- fname++;
- if (strlen(fname) == 0)
- errx(1, "Cannot obtain file name from %s\n",
- argv[nres]);
- fd = open(fname, O_CREAT | O_TRUNC | O_WRONLY, 0644);
- if (fd == -1)
- errx(1, "open(%s)", fname);
- }
- /* Read the message and send data to fd if appropriate */
- if (chunked) {
- /* Handle a chunked-encoded entity */
- /* Read chunks */
- do {
- error = readln(sd, resbuf, &resbuflen,
- &resbufpos);
- if (error)
- goto conndied;
- hln = resbuf + resbufpos;
- eolp = strstr(hln, "\r\n");
- resbufpos = (eolp - resbuf) + 2;
- clen = 0;
- while (isxdigit(*hln)) {
- if (clen >= OFF_MAX / 16) {
- /* Nasty people... */
- goto conndied;
- }
- if (isdigit(*hln))
- clen = clen * 16 + *hln - '0';
- else
- clen = clen * 16 + 10 +
- tolower(*hln) - 'a';
- hln++;
- }
- error = copybytes(sd, fd, clen, resbuf,
- &resbuflen, &resbufpos);
- if (error) {
- goto conndied;
- }
- } while (clen != 0);
- /* Read trailer and final CRLF */
- do {
- error = readln(sd, resbuf, &resbuflen,
- &resbufpos);
- if (error)
- goto conndied;
- hln = resbuf + resbufpos;
- eolp = strstr(hln, "\r\n");
- resbufpos = (eolp - resbuf) + 2;
- } while (hln != eolp);
- } else if (contentlength != -1) {
- error = copybytes(sd, fd, contentlength, resbuf,
- &resbuflen, &resbufpos);
- if (error)
- goto conndied;
- } else {
- /*
- * Not chunked, and no content length header.
- * Read everything until the server closes the
- * socket.
- */
- error = copybytes(sd, fd, OFF_MAX, resbuf,
- &resbuflen, &resbufpos);
- if (error == -1)
- goto conndied;
- pipelined = 0;
- }
- if (fd != -1) {
- close(fd);
- fd = -1;
- }
- fprintf(stderr, "http://%s/%s: %d ", servername, argv[nres],
- statuscode);
- if (statuscode == 200)
- fprintf(stderr, "OK\n");
- else if (statuscode < 300)
- fprintf(stderr, "Successful (ignored)\n");
- else if (statuscode < 400)
- fprintf(stderr, "Redirection (ignored)\n");
- else
- fprintf(stderr, "Error (ignored)\n");
- /* We've finished this file! */
- nres++;
- /*
- * If necessary, clean up this connection so that we
- * can start a new one.
- */
- if (pipelined == 0 && keepalive == 0)
- goto cleanupconn;
- continue;
- conndied:
- /*
- * Something went wrong -- our connection died, the server
- * sent us garbage, etc. If this happened on the first
- * request we sent over this connection, give up. Otherwise,
- * close this connection, open a new one, and reissue the
- * request.
- */
- if (nres == firstreq)
- errx(1, "Connection failure");
- cleanupconn:
- /*
- * Clean up our connection and keep on going
- */
- shutdown(sd, SHUT_RDWR);
- close(sd);
- sd = -1;
- if (fd != -1) {
- close(fd);
- fd = -1;
- }
- if (reqbuf != NULL) {
- free(reqbuf);
- reqbuf = NULL;
- }
- nreq = nres;
- res = res0;
- pipelined = 0;
- resbufpos = resbuflen = 0;
- continue;
- }
- free(resbuf);
- freeaddrinfo(res0);
- return 0;
- }
|