/*- * Copyright (c) 2000-2004 Dag-Erling Coïdan Smørgrav * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer * in this position and unchanged. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * 3. The name of the author may not be used to endorse or promote products * derived from this software without specific prior written permission. * * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "free2net.h" #include __FBSDID("$FreeBSD: src/lib/libfetch/http.c,v 1.76.2.2 2007/05/29 12:35:26 des Exp $"); /* * The following copyright applies to the base64 code: * *- * Copyright 1997 Massachusetts Institute of Technology * * Permission to use, copy, modify, and distribute this software and * its documentation for any purpose and without fee is hereby * granted, provided that both the above copyright notice and this * permission notice appear in all copies, that both the above * copyright notice and this permission notice appear in all * supporting documentation, and that the name of M.I.T. not be used * in advertising or publicity pertaining to distribution of the * software without specific, written prior permission. M.I.T. makes * no representations about the suitability of this software for any * purpose. It is provided "as is" without express or implied * warranty. * * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''. M.I.T. DISCLAIMS * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE, * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "fetch.h" #include "common.h" #include "httperr.h" #include "free2net.h" /* Maximum number of redirects to follow */ #define MAX_REDIRECT 5 /* Symbolic names for reply codes we care about */ #define HTTP_OK 200 #define HTTP_PARTIAL 206 #define HTTP_MOVED_PERM 301 #define HTTP_MOVED_TEMP 302 #define HTTP_SEE_OTHER 303 #define HTTP_TEMP_REDIRECT 307 #define HTTP_NEED_AUTH 401 #define HTTP_NEED_PROXY_AUTH 407 #define HTTP_BAD_RANGE 416 #define HTTP_PROTOCOL_ERROR 999 #define HTTP_REDIRECT(xyz) ((xyz) == HTTP_MOVED_PERM \ || (xyz) == HTTP_MOVED_TEMP \ || (xyz) == HTTP_TEMP_REDIRECT \ || (xyz) == HTTP_SEE_OTHER) #define HTTP_ERROR(xyz) ((xyz) > 400 && (xyz) < 599) /***************************************************************************** * I/O functions for decoding chunked streams */ struct httpio { conn_t *conn; /* connection */ int chunked; /* chunked mode */ char *buf; /* chunk buffer */ size_t bufsize; /* size of chunk buffer */ ssize_t buflen; /* amount of data currently in buffer */ int bufpos; /* current read offset in buffer */ int eof; /* end-of-file flag */ int error; /* error flag */ size_t chunksize; /* remaining size of current chunk */ #ifndef NDEBUG size_t total; #endif }; /* * Get next chunk header */ static int _http_new_chunk(struct httpio *io) { char *p; if (_fetch_getln(io->conn) == -1) return (-1); if (io->conn->buflen < 2 || !ishexnumber((unsigned)*io->conn->buf)) return (-1); for (p = io->conn->buf; *p && !isspace((unsigned)*p); ++p) { if (*p == ';') break; if (!ishexnumber((unsigned)*p)) return (-1); if (isdigit((unsigned)*p)) { io->chunksize = io->chunksize * 16 + *p - '0'; } else { io->chunksize = io->chunksize * 16 + 10 + tolower((unsigned)*p) - 'a'; } } #ifndef NDEBUG if (fetchDebug) { io->total += io->chunksize; if (io->chunksize == 0) fprintf(stderr, "%s(): end of last chunk\n", __func__); else fprintf(stderr, "%s(): new chunk: %lu (%lu)\n", __func__, (unsigned long)io->chunksize, (unsigned long)io->total); } #endif return (io->chunksize); } /* * Grow the input buffer to at least len bytes */ static inline int _http_growbuf(struct httpio *io, size_t len) { char *tmp; if (io->bufsize >= len) return (0); if ((tmp = realloc(io->buf, len)) == NULL) return (-1); io->buf = tmp; io->bufsize = len; return (0); } /* * Fill the input buffer, do chunk decoding on the fly */ static int _http_fillbuf(struct httpio *io, size_t len) { if (io->error) return (-1); if (io->eof) return (0); if (io->chunked == 0) { if (_http_growbuf(io, len) == -1) return (-1); if ((io->buflen = _fetch_read(io->conn, io->buf, len)) == -1) { io->error = 1; return (-1); } io->bufpos = 0; return (io->buflen); } if (io->chunksize == 0) { switch (_http_new_chunk(io)) { case -1: io->error = 1; return (-1); case 0: io->eof = 1; return (0); } } if (len > io->chunksize) len = io->chunksize; if (_http_growbuf(io, len) == -1) return (-1); if ((io->buflen = _fetch_read(io->conn, io->buf, len)) == -1) { io->error = 1; return (-1); } io->chunksize -= io->buflen; if (io->chunksize == 0) { char endl[2]; if (_fetch_read(io->conn, endl, 2) != 2 || endl[0] != '\r' || endl[1] != '\n') return (-1); } io->bufpos = 0; return (io->buflen); } /* * Read function */ static int _http_readfn(void *v, char *buf, int len) { struct httpio *io = (struct httpio *)v; int l, pos; if (io->error) return (-1); if (io->eof) return (0); for (pos = 0; len > 0; pos += l, len -= l) { /* empty buffer */ if (!io->buf || io->bufpos == io->buflen) if (_http_fillbuf(io, (unsigned) len) < 1) break; l = io->buflen - io->bufpos; if (len < l) l = len; bcopy(io->buf + io->bufpos, buf + pos, (unsigned) l); io->bufpos += l; } if (!pos && io->error) return (-1); return (pos); } /* * Write function */ static int _http_writefn(void *v, const char *buf, int len) { struct httpio *io = (struct httpio *)v; return (_fetch_write(io->conn, buf, (unsigned) len)); } /* * Close function */ static int _http_closefn(void *v) { struct httpio *io = (struct httpio *)v; int r; r = _fetch_close(io->conn); if (io->buf) free(io->buf); free(io); return (r); } /* * Wrap a file descriptor up */ static FILE * _http_funopen(conn_t *conn, int chunked) { struct httpio *io; FILE *f; if ((io = calloc(1, sizeof(*io))) == NULL) { _fetch_syserr(); return (NULL); } io->conn = conn; io->chunked = chunked; f = funopen(io, _http_readfn, _http_writefn, NULL, _http_closefn); if (f == NULL) { _fetch_syserr(); free(io); return (NULL); } return (f); } /***************************************************************************** * Helper functions for talking to the server and parsing its replies */ /* Header types */ typedef enum { hdr_syserror = -2, hdr_error = -1, hdr_end = 0, hdr_unknown = 1, hdr_content_length, hdr_content_range, hdr_last_modified, hdr_location, hdr_transfer_encoding, hdr_www_authenticate } hdr_t; /* Names of interesting headers */ static struct { hdr_t num; const char *name; } hdr_names[] = { { hdr_content_length, "Content-Length" }, { hdr_content_range, "Content-Range" }, { hdr_last_modified, "Last-Modified" }, { hdr_location, "Location" }, { hdr_transfer_encoding, "Transfer-Encoding" }, { hdr_www_authenticate, "WWW-Authenticate" }, { hdr_unknown, NULL }, }; /* * Send a formatted line; optionally echo to terminal */ static int _http_cmd(conn_t *conn, const char *fmt, ...) { va_list ap; size_t len; char *msg; int r; va_start(ap, fmt); len = vasprintf(&msg, fmt, ap); va_end(ap); if (msg == NULL) { errno = ENOMEM; _fetch_syserr(); return (-1); } r = _fetch_putln(conn, msg, len); free(msg); if (r == -1) { _fetch_syserr(); return (-1); } return (0); } /* * Get and parse status line */ static int _http_get_reply(conn_t *conn) { char *p; if (_fetch_getln(conn) == -1) return (-1); /* * A valid status line looks like "HTTP/m.n xyz reason" where m * and n are the major and minor protocol version numbers and xyz * is the reply code. * Unfortunately, there are servers out there (NCSA 1.5.1, to name * just one) that do not send a version number, so we can't rely * on finding one, but if we do, insist on it being 1.0 or 1.1. * We don't care about the reason phrase. */ if (strncmp(conn->buf, "HTTP", 4) != 0) return (HTTP_PROTOCOL_ERROR); p = conn->buf + 4; if (*p == '/') { if (p[1] != '1' || p[2] != '.' || (p[3] != '0' && p[3] != '1')) return (HTTP_PROTOCOL_ERROR); p += 4; } if (*p != ' ' || !isdigit((unsigned)p[1]) || !isdigit((unsigned)p[2]) || !isdigit((unsigned)p[3])) return (HTTP_PROTOCOL_ERROR); conn->err = (p[1] - '0') * 100 + (p[2] - '0') * 10 + (p[3] - '0'); return (conn->err); } /* * Check a header; if the type matches the given string, return a pointer * to the beginning of the value. */ static const char * _http_match(const char *str, const char *hdr) { while (*str && *hdr && tolower((unsigned)*str++) == tolower((unsigned)*hdr++)) /* nothing */; if (*str || *hdr != ':') return (NULL); while (*hdr && isspace((unsigned)*++hdr)) /* nothing */; return (hdr); } /* * Get the next header and return the appropriate symbolic code. */ static hdr_t _http_next_header(conn_t *conn, const char **p) { int i; if (_fetch_getln(conn) == -1) return (hdr_syserror); while (conn->buflen && isspace((unsigned)conn->buf[conn->buflen - 1])) conn->buflen--; conn->buf[conn->buflen] = '\0'; if (conn->buflen == 0) return (hdr_end); /* * We could check for malformed headers but we don't really care. * A valid header starts with a token immediately followed by a * colon; a token is any sequence of non-control, non-whitespace * characters except "()<>@,;:\\\"{}". */ for (i = 0; hdr_names[i].num != hdr_unknown; i++) if ((*p = _http_match(hdr_names[i].name, conn->buf)) != NULL) return (hdr_names[i].num); return (hdr_unknown); } /* * Parse a last-modified header */ static int _http_parse_mtime(const char *p, time_t *mtime) { char locale[64], *r; struct tm tm; strncpy(locale, setlocale(LC_TIME, NULL), sizeof(locale)); setlocale(LC_TIME, "C"); r = strptime(p, "%a, %d %b %Y %H:%M:%S GMT", &tm); /* XXX should add support for date-2 and date-3 */ setlocale(LC_TIME, locale); if (r == NULL) return (-1); DEBUG(fprintf(stderr, "last modified: [%04d-%02d-%02d " "%02d:%02d:%02d]\n", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec)); *mtime = timegm(&tm); return (0); } /* * Parse a content-length header */ static int _http_parse_length(const char *p, off_t *length) { off_t len; for (len = 0; *p && isdigit((unsigned)*p); ++p) len = len * 10 + (*p - '0'); if (*p) return (-1); DEBUG(fprintf(stderr, "content length: [%lld]\n", (long long)len)); *length = len; return (0); } /* * Parse a content-range header */ static int _http_parse_range(const char *p, off_t *offset, off_t *length, off_t *size) { off_t first, last, len; if (strncasecmp(p, "bytes ", 6) != 0) return (-1); p += 6; if (*p == '*') { first = last = -1; ++p; } else { for (first = 0; *p && isdigit((unsigned)*p); ++p) first = first * 10 + *p - '0'; if (*p != '-') return (-1); for (last = 0, ++p; *p && isdigit((unsigned)*p); ++p) last = last * 10 + *p - '0'; } if (first > last || *p != '/') return (-1); for (len = 0, ++p; *p && isdigit((unsigned)*p); ++p) len = len * 10 + *p - '0'; if (*p || len < last - first + 1) return (-1); if (first == -1) { DEBUG(fprintf(stderr, "content range: [*/%lld]\n", (long long)len)); *length = 0; } else { DEBUG(fprintf(stderr, "content range: [%lld-%lld/%lld]\n", (long long)first, (long long)last, (long long)len)); *length = last - first + 1; } *offset = first; *size = len; return (0); } /***************************************************************************** * Helper functions for authorization */ /* * Base64 encoding */ static char * _http_base64(const char *src) { static const char base64[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789+/"; char *str, *dst; size_t l; int r; unsigned t; l = strlen(src); if ((str = malloc(((l + 2) / 3) * 4 + 1)) == NULL) return (NULL); dst = str; r = 0; while (l >= 3) { t = (src[0] << 16) | (src[1] << 8) | src[2]; dst[0] = base64[(t >> 18) & 0x3f]; dst[1] = base64[(t >> 12) & 0x3f]; dst[2] = base64[(t >> 6) & 0x3f]; dst[3] = base64[(t >> 0) & 0x3f]; src += 3; l -= 3; dst += 4; r += 4; } switch (l) { case 2: t = (src[0] << 16) | (src[1] << 8); dst[0] = base64[(t >> 18) & 0x3f]; dst[1] = base64[(t >> 12) & 0x3f]; dst[2] = base64[(t >> 6) & 0x3f]; dst[3] = '='; dst += 4; r += 4; break; case 1: t = src[0] << 16; dst[0] = base64[(t >> 18) & 0x3f]; dst[1] = base64[(t >> 12) & 0x3f]; dst[2] = dst[3] = '='; dst += 4; r += 4; break; case 0: break; } *dst = 0; return (str); } /* * Encode username and password */ static int _http_basic_auth(conn_t *conn, const char *hdr, const char *usr, const char *pwd) { char *upw, *auth; int r; DEBUG(fprintf(stderr, "usr: [%s]\n", usr)); DEBUG(fprintf(stderr, "pwd: [%s]\n", pwd)); if (asprintf(&upw, "%s:%s", usr, pwd) == -1) return (-1); auth = _http_base64(upw); free(upw); if (auth == NULL) return (-1); r = _http_cmd(conn, "%s: Basic %s", hdr, auth); free(auth); return (r); } /* * Send an authorization header */ static int _http_authorize(conn_t *conn, const char *hdr, const char *p) { /* basic authorization */ if (strncasecmp(p, "basic:", 6) == 0) { char *user, *pwd, *str; int r; /* skip realm */ for (p += 6; *p && *p != ':'; ++p) /* nothing */ ; if (!*p || strchr(++p, ':') == NULL) return (-1); if ((str = strdup(p)) == NULL) return (-1); /* XXX */ user = str; pwd = strchr(str, ':'); *pwd++ = '\0'; r = _http_basic_auth(conn, hdr, user, pwd); free(str); return (r); } return (-1); } /***************************************************************************** * Helper functions for connecting to a server or proxy */ /* * Connect to the correct HTTP server or proxy. */ static conn_t * _http_connect(struct url *URL, struct url *purl, const char *flags) { conn_t *conn; int verbose; int af; #ifdef INET6 af = AF_UNSPEC; #else af = AF_INET; #endif verbose = CHECK_FLAG('v'); if (CHECK_FLAG('4')) af = AF_INET; #ifdef INET6 else if (CHECK_FLAG('6')) af = AF_INET6; #endif if (purl && strcasecmp(URL->scheme, SCHEME_HTTPS) != 0) { URL = purl; } else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0) { /* can't talk http to an ftp server */ /* XXX should set an error code */ return (NULL); } if ((conn = _fetch_connect(URL->host, URL->port, af, verbose)) == NULL) /* _fetch_connect() has already set an error code */ return (NULL); if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0 && _fetch_ssl(conn, verbose) == -1) { _fetch_close(conn); /* grrr */ errno = EAUTH; _fetch_syserr(); return (NULL); } #ifdef TCP_NOPUSH { int val; val = 1; setsockopt(conn->sd, IPPROTO_TCP, TCP_NOPUSH, &val, sizeof(val)); } #endif return (conn); } static struct url * _http_get_proxy(const char *flags) { struct url *purl; char *p; if (flags != NULL && strchr(flags, 'd') != NULL) return (NULL); if (((p = getenv("HTTP_PROXY")) || (p = getenv("http_proxy"))) && *p && (purl = fetchParseURL(p))) { if (!*purl->scheme) strcpy(purl->scheme, SCHEME_HTTP); if (!purl->port) purl->port = _fetch_default_proxy_port(purl->scheme); if (strcasecmp(purl->scheme, SCHEME_HTTP) == 0) return (purl); fetchFreeURL(purl); } return (NULL); } static void _http_print_html(FILE *out, FILE *in) { size_t len; char *line, *p, *q; int comment, tag; comment = tag = 0; while ((line = fgetln(in, &len)) != NULL) { while (len && isspace((unsigned)line[len - 1])) --len; for (p = q = line; q < line + len; ++q) { if (comment && *q == '-') { if (q + 2 < line + len && strcmp(q, "-->") == 0) { tag = comment = 0; q += 2; } } else if (tag && !comment && *q == '>') { p = q + 1; tag = 0; } else if (!tag && *q == '<') { if (q > p) fwrite(p, (unsigned)(q - p), 1, out); tag = 1; if (q + 3 < line + len && strcmp(q, "