bda08cd402
Start URL quoting cleanup. All URLs are now quoted correctly on parsing and when appending URLs. URLs without schema and starting with slash are considered to be file:// URLs.
565 lines
12 KiB
C
565 lines
12 KiB
C
/* $NetBSD: fetch.c,v 1.7 2008/04/24 07:55:00 joerg Exp $ */
|
|
/*-
|
|
* Copyright (c) 1998-2004 Dag-Erling Coïdan Smørgrav
|
|
* All rights reserved.
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions
|
|
* are met:
|
|
* 1. Redistributions of source code must retain the above copyright
|
|
* notice, this list of conditions and the following disclaimer
|
|
* in this position and unchanged.
|
|
* 2. Redistributions in binary form must reproduce the above copyright
|
|
* notice, this list of conditions and the following disclaimer in the
|
|
* documentation and/or other materials provided with the distribution.
|
|
* 3. The name of the author may not be used to endorse or promote products
|
|
* derived from this software without specific prior written permission
|
|
*
|
|
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* $FreeBSD: fetch.c,v 1.41 2007/12/19 00:26:36 des Exp $
|
|
*/
|
|
|
|
#include <ctype.h>
|
|
#include <errno.h>
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
|
|
#include "fetch.h"
|
|
#include "common.h"
|
|
|
|
auth_t fetchAuthMethod;
|
|
int fetchLastErrCode;
|
|
char fetchLastErrString[MAXERRSTRING];
|
|
int fetchTimeout;
|
|
int fetchRestartCalls = 1;
|
|
int fetchDebug;
|
|
|
|
|
|
/*** Local data **************************************************************/
|
|
|
|
/*
|
|
* Error messages for parser errors
|
|
*/
|
|
#define URL_MALFORMED 1
|
|
#define URL_BAD_SCHEME 2
|
|
#define URL_BAD_PORT 3
|
|
static struct fetcherr url_errlist[] = {
|
|
{ URL_MALFORMED, FETCH_URL, "Malformed URL" },
|
|
{ URL_BAD_SCHEME, FETCH_URL, "Invalid URL scheme" },
|
|
{ URL_BAD_PORT, FETCH_URL, "Invalid server port" },
|
|
{ -1, FETCH_UNKNOWN, "Unknown parser error" }
|
|
};
|
|
|
|
|
|
/*** Public API **************************************************************/
|
|
|
|
/*
|
|
* Select the appropriate protocol for the URL scheme, and return a
|
|
* read-only stream connected to the document referenced by the URL.
|
|
* Also fill out the struct url_stat.
|
|
*/
|
|
fetchIO *
|
|
fetchXGet(struct url *URL, struct url_stat *us, const char *flags)
|
|
{
|
|
int direct;
|
|
|
|
direct = CHECK_FLAG('d');
|
|
if (us != NULL) {
|
|
us->size = -1;
|
|
us->atime = us->mtime = 0;
|
|
}
|
|
if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
|
|
return (fetchXGetFile(URL, us, flags));
|
|
else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
|
|
return (fetchXGetFTP(URL, us, flags));
|
|
else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
|
|
return (fetchXGetHTTP(URL, us, flags));
|
|
else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
|
|
return (fetchXGetHTTP(URL, us, flags));
|
|
url_seterr(URL_BAD_SCHEME);
|
|
return (NULL);
|
|
}
|
|
|
|
/*
|
|
* Select the appropriate protocol for the URL scheme, and return a
|
|
* read-only stream connected to the document referenced by the URL.
|
|
*/
|
|
fetchIO *
|
|
fetchGet(struct url *URL, const char *flags)
|
|
{
|
|
return (fetchXGet(URL, NULL, flags));
|
|
}
|
|
|
|
/*
|
|
* Select the appropriate protocol for the URL scheme, and return a
|
|
* write-only stream connected to the document referenced by the URL.
|
|
*/
|
|
fetchIO *
|
|
fetchPut(struct url *URL, const char *flags)
|
|
{
|
|
int direct;
|
|
|
|
direct = CHECK_FLAG('d');
|
|
if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
|
|
return (fetchPutFile(URL, flags));
|
|
else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
|
|
return (fetchPutFTP(URL, flags));
|
|
else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
|
|
return (fetchPutHTTP(URL, flags));
|
|
else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
|
|
return (fetchPutHTTP(URL, flags));
|
|
url_seterr(URL_BAD_SCHEME);
|
|
return (NULL);
|
|
}
|
|
|
|
/*
|
|
* Select the appropriate protocol for the URL scheme, and return the
|
|
* size of the document referenced by the URL if it exists.
|
|
*/
|
|
int
|
|
fetchStat(struct url *URL, struct url_stat *us, const char *flags)
|
|
{
|
|
int direct;
|
|
|
|
direct = CHECK_FLAG('d');
|
|
if (us != NULL) {
|
|
us->size = -1;
|
|
us->atime = us->mtime = 0;
|
|
}
|
|
if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
|
|
return (fetchStatFile(URL, us, flags));
|
|
else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
|
|
return (fetchStatFTP(URL, us, flags));
|
|
else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
|
|
return (fetchStatHTTP(URL, us, flags));
|
|
else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
|
|
return (fetchStatHTTP(URL, us, flags));
|
|
url_seterr(URL_BAD_SCHEME);
|
|
return (-1);
|
|
}
|
|
|
|
/*
|
|
* Select the appropriate protocol for the URL scheme, and return a
|
|
* list of files in the directory pointed to by the URL.
|
|
*/
|
|
int
|
|
fetchList(struct url_list *ue, struct url *URL, const char *pattern,
|
|
const char *flags)
|
|
{
|
|
int direct;
|
|
|
|
direct = CHECK_FLAG('d');
|
|
if (strcasecmp(URL->scheme, SCHEME_FILE) == 0)
|
|
return (fetchListFile(ue, URL, pattern, flags));
|
|
else if (strcasecmp(URL->scheme, SCHEME_FTP) == 0)
|
|
return (fetchListFTP(ue, URL, pattern, flags));
|
|
else if (strcasecmp(URL->scheme, SCHEME_HTTP) == 0)
|
|
return (fetchListHTTP(ue, URL, pattern, flags));
|
|
else if (strcasecmp(URL->scheme, SCHEME_HTTPS) == 0)
|
|
return (fetchListHTTP(ue, URL, pattern, flags));
|
|
url_seterr(URL_BAD_SCHEME);
|
|
return -1;
|
|
}
|
|
|
|
/*
|
|
* Attempt to parse the given URL; if successful, call fetchXGet().
|
|
*/
|
|
fetchIO *
|
|
fetchXGetURL(const char *URL, struct url_stat *us, const char *flags)
|
|
{
|
|
struct url *u;
|
|
fetchIO *f;
|
|
|
|
if ((u = fetchParseURL(URL)) == NULL)
|
|
return (NULL);
|
|
|
|
f = fetchXGet(u, us, flags);
|
|
|
|
fetchFreeURL(u);
|
|
return (f);
|
|
}
|
|
|
|
/*
|
|
* Attempt to parse the given URL; if successful, call fetchGet().
|
|
*/
|
|
fetchIO *
|
|
fetchGetURL(const char *URL, const char *flags)
|
|
{
|
|
return (fetchXGetURL(URL, NULL, flags));
|
|
}
|
|
|
|
/*
|
|
* Attempt to parse the given URL; if successful, call fetchPut().
|
|
*/
|
|
fetchIO *
|
|
fetchPutURL(const char *URL, const char *flags)
|
|
{
|
|
struct url *u;
|
|
fetchIO *f;
|
|
|
|
if ((u = fetchParseURL(URL)) == NULL)
|
|
return (NULL);
|
|
|
|
f = fetchPut(u, flags);
|
|
|
|
fetchFreeURL(u);
|
|
return (f);
|
|
}
|
|
|
|
/*
|
|
* Attempt to parse the given URL; if successful, call fetchStat().
|
|
*/
|
|
int
|
|
fetchStatURL(const char *URL, struct url_stat *us, const char *flags)
|
|
{
|
|
struct url *u;
|
|
int s;
|
|
|
|
if ((u = fetchParseURL(URL)) == NULL)
|
|
return (-1);
|
|
|
|
s = fetchStat(u, us, flags);
|
|
|
|
fetchFreeURL(u);
|
|
return (s);
|
|
}
|
|
|
|
/*
|
|
* Attempt to parse the given URL; if successful, call fetchList().
|
|
*/
|
|
int
|
|
fetchListURL(struct url_list *ue, const char *URL, const char *pattern,
|
|
const char *flags)
|
|
{
|
|
struct url *u;
|
|
int rv;
|
|
|
|
if ((u = fetchParseURL(URL)) == NULL)
|
|
return -1;
|
|
|
|
rv = fetchList(ue, u, pattern, flags);
|
|
|
|
fetchFreeURL(u);
|
|
return rv;
|
|
}
|
|
|
|
/*
|
|
* Make a URL
|
|
*/
|
|
struct url *
|
|
fetchMakeURL(const char *scheme, const char *host, int port, const char *doc,
|
|
const char *user, const char *pwd)
|
|
{
|
|
struct url *u;
|
|
|
|
if (!scheme || (!host && !doc)) {
|
|
url_seterr(URL_MALFORMED);
|
|
return (NULL);
|
|
}
|
|
|
|
if (port < 0 || port > 65535) {
|
|
url_seterr(URL_BAD_PORT);
|
|
return (NULL);
|
|
}
|
|
|
|
/* allocate struct url */
|
|
if ((u = calloc(1, sizeof(*u))) == NULL) {
|
|
fetch_syserr();
|
|
return (NULL);
|
|
}
|
|
|
|
if ((u->doc = strdup(doc ? doc : "/")) == NULL) {
|
|
fetch_syserr();
|
|
free(u);
|
|
return (NULL);
|
|
}
|
|
|
|
#define seturl(x) snprintf(u->x, sizeof(u->x), "%s", x)
|
|
seturl(scheme);
|
|
seturl(host);
|
|
seturl(user);
|
|
seturl(pwd);
|
|
#undef seturl
|
|
u->port = port;
|
|
|
|
return (u);
|
|
}
|
|
|
|
int
|
|
fetch_urlpath_safe(char x)
|
|
{
|
|
switch (x) {
|
|
case 'a'...'z':
|
|
case 'A'...'Z':
|
|
case '0'...'9':
|
|
case '$':
|
|
case '-':
|
|
case '_':
|
|
case '.':
|
|
case '+':
|
|
case '!':
|
|
case '*':
|
|
case '\'':
|
|
case '(':
|
|
case ')':
|
|
case ',':
|
|
/* The following are allowed in segment and path components: */
|
|
case '?':
|
|
case ':':
|
|
case '@':
|
|
case '&':
|
|
case '=':
|
|
case '/':
|
|
case ';':
|
|
/* If something is already quoted... */
|
|
case '%':
|
|
return 1;
|
|
default:
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Copy an existing URL.
|
|
*/
|
|
struct url *
|
|
fetchCopyURL(const struct url *src)
|
|
{
|
|
struct url *dst;
|
|
char *doc;
|
|
|
|
/* allocate struct url */
|
|
if ((dst = malloc(sizeof(*dst))) == NULL) {
|
|
fetch_syserr();
|
|
return (NULL);
|
|
}
|
|
if ((doc = strdup(src->doc)) == NULL) {
|
|
fetch_syserr();
|
|
free(dst);
|
|
return (NULL);
|
|
}
|
|
*dst = *src;
|
|
dst->doc = doc;
|
|
|
|
return dst;
|
|
}
|
|
|
|
/*
|
|
* Split an URL into components. URL syntax is:
|
|
* [method:/][/[user[:pwd]@]host[:port]/][document]
|
|
* This almost, but not quite, RFC1738 URL syntax.
|
|
*/
|
|
struct url *
|
|
fetchParseURL(const char *URL)
|
|
{
|
|
const char *p, *q;
|
|
struct url *u;
|
|
size_t i, count;
|
|
|
|
/* allocate struct url */
|
|
if ((u = calloc(1, sizeof(*u))) == NULL) {
|
|
fetch_syserr();
|
|
return (NULL);
|
|
}
|
|
|
|
if (*URL == '/') {
|
|
strcpy(u->scheme, SCHEME_FILE);
|
|
p = URL;
|
|
goto quote_doc;
|
|
}
|
|
if (strncmp(URL, "file:", 5) == 0) {
|
|
strcpy(u->scheme, SCHEME_FILE);
|
|
URL += 5;
|
|
if (URL[0] != '/' || URL[1] != '/' || URL[2] != '/') {
|
|
url_seterr(URL_MALFORMED);
|
|
goto ouch;
|
|
}
|
|
p = URL + 2;
|
|
goto quote_doc;
|
|
}
|
|
if (strncmp(URL, "http:", 5) == 0 ||
|
|
strncmp(URL, "https:", 6) == 0) {
|
|
if (URL[4] == ':') {
|
|
strcpy(u->scheme, SCHEME_HTTP);
|
|
URL += 5;
|
|
} else {
|
|
strcpy(u->scheme, SCHEME_HTTPS);
|
|
URL += 6;
|
|
}
|
|
|
|
if (URL[0] != '/' || URL[1] != '/') {
|
|
url_seterr(URL_MALFORMED);
|
|
goto ouch;
|
|
}
|
|
URL += 2;
|
|
p = URL;
|
|
goto find_hostname;
|
|
}
|
|
if (strncmp(URL, "ftp:", 4) == 0) {
|
|
strcpy(u->scheme, SCHEME_FTP);
|
|
URL += 4;
|
|
if (URL[0] != '/' || URL[1] != '/') {
|
|
url_seterr(URL_MALFORMED);
|
|
goto ouch;
|
|
}
|
|
URL += 2;
|
|
p = URL;
|
|
goto find_user;
|
|
}
|
|
|
|
url_seterr(URL_BAD_SCHEME);
|
|
goto ouch;
|
|
|
|
find_user:
|
|
p = strpbrk(URL, "/@");
|
|
if (p != NULL && *p == '@') {
|
|
/* username */
|
|
for (q = URL, i = 0; (*q != ':') && (*q != '@'); q++) {
|
|
if (i < URL_USERLEN)
|
|
u->user[i++] = *q;
|
|
}
|
|
|
|
/* password */
|
|
if (*q == ':') {
|
|
for (q++, i = 0; (*q != ':') && (*q != '@'); q++)
|
|
if (i < URL_PWDLEN)
|
|
u->pwd[i++] = *q;
|
|
}
|
|
|
|
p++;
|
|
} else {
|
|
p = URL;
|
|
}
|
|
|
|
find_hostname:
|
|
/* hostname */
|
|
#ifdef INET6
|
|
if (*p == '[' && (q = strchr(p + 1, ']')) != NULL &&
|
|
(*++q == '\0' || *q == '/' || *q == ':')) {
|
|
if ((i = q - p - 2) > URL_HOSTLEN)
|
|
i = URL_HOSTLEN;
|
|
strncpy(u->host, ++p, i);
|
|
p = q;
|
|
} else
|
|
#endif
|
|
for (i = 0; *p && (*p != '/') && (*p != ':'); p++)
|
|
if (i < URL_HOSTLEN)
|
|
u->host[i++] = *p;
|
|
|
|
/* port */
|
|
if (*p == ':') {
|
|
for (q = ++p; *q && (*q != '/'); q++)
|
|
if (isdigit((unsigned char)*q))
|
|
u->port = u->port * 10 + (*q - '0');
|
|
else {
|
|
/* invalid port */
|
|
url_seterr(URL_BAD_PORT);
|
|
goto ouch;
|
|
}
|
|
p = q;
|
|
}
|
|
|
|
/* document */
|
|
if (!*p)
|
|
p = "/";
|
|
|
|
quote_doc:
|
|
count = 1;
|
|
for (i = 0; p[i] != '\0'; ++i)
|
|
count += fetch_urlpath_safe(p[i]) ? 1 : 3;
|
|
if ((u->doc = malloc(count)) == NULL) {
|
|
fetch_syserr();
|
|
goto ouch;
|
|
}
|
|
for (i = 0; *p != '\0'; ++p) {
|
|
if (fetch_urlpath_safe(*p))
|
|
u->doc[i++] = *p;
|
|
else {
|
|
u->doc[i++] = '%';
|
|
if ((unsigned char)*p < 160)
|
|
u->doc[i++] = '0' + ((unsigned char)*p) / 16;
|
|
else
|
|
u->doc[i++] = 'a' - 10 + ((unsigned char)*p) / 16;
|
|
if ((unsigned char)*p % 16 < 16)
|
|
u->doc[i++] = '0' + ((unsigned char)*p) % 16;
|
|
else
|
|
u->doc[i++] = 'a' - 10 + ((unsigned char)*p) % 16;
|
|
}
|
|
}
|
|
u->doc[i] = '\0';
|
|
|
|
return (u);
|
|
|
|
ouch:
|
|
free(u);
|
|
return (NULL);
|
|
}
|
|
|
|
/*
|
|
* Free a URL
|
|
*/
|
|
void
|
|
fetchFreeURL(struct url *u)
|
|
{
|
|
free(u->doc);
|
|
free(u);
|
|
}
|
|
|
|
static char
|
|
xdigit2digit(char digit)
|
|
{
|
|
digit = tolower((unsigned char)digit);
|
|
if (digit >= 'a' && digit <= 'f')
|
|
digit = digit - 'a' + 10;
|
|
else
|
|
digit = digit - '0';
|
|
|
|
return digit;
|
|
}
|
|
|
|
/*
|
|
* Extract the file name component of a URL.
|
|
*/
|
|
char *
|
|
fetch_extract_filename(struct url *url)
|
|
{
|
|
char *name, *name_iter;
|
|
const char *last_slash, *iter;
|
|
|
|
last_slash = url->doc;
|
|
if (*last_slash == '\0')
|
|
return strdup("");
|
|
for (iter = last_slash + 1; *iter; ++iter) {
|
|
if (*iter == '#' || *iter == '?')
|
|
break;
|
|
if (*iter == '/')
|
|
last_slash = iter;
|
|
}
|
|
if (last_slash + 1 == iter)
|
|
return strdup("");
|
|
name_iter = name = malloc(iter - last_slash);
|
|
while (++last_slash < iter) {
|
|
if (*last_slash != '%' ||
|
|
!isxdigit((unsigned char)last_slash[1]) ||
|
|
!isxdigit((unsigned char)last_slash[2])) {
|
|
*name_iter++ = *last_slash;
|
|
continue;
|
|
}
|
|
*name_iter++ = xdigit2digit(last_slash[1]) * 16 +
|
|
xdigit2digit(last_slash[2]);
|
|
last_slash += 2;
|
|
}
|
|
*name_iter = '\0';
|
|
return name;
|
|
}
|