Commit 9dc6278c authored by Timo Teräs's avatar Timo Teräs

import libfetch-2.38 from NetBSD

ftp://ftp.fu-berlin.de/unix/NetBSD/packages/current-src/pkgsrc/net/libfetch/files

libfetch comes (at least) in netbsd and freebsd flavors which
differing functionality. Alpine and Arch package netbsd one,
but it's not widely packaged across other distributions.

We need NetBSD version as it does not use funopen(3) which is not
supported in musl, and supports connection pooling.

FreeBSD seems to be the orignal and better maintained version
with support for SSL CAs, client certificate authentication,
proxy authentication, and improved http redirect handling.

So this imports NetBSD version, and future commits will pick up
the needed improvements from FreeBSD tree.

Incidentally, this also fixes #7857 and likes for good.
parent 8f0938a0
# $NetBSD: Makefile,v 1.8 2016/10/27 10:05:38 joerg Exp $
LIB= fetch
SRCS= fetch.c common.c ftp.c http.c file.c
DPSRCS= ftperr.h httperr.h
INCS= fetch.h
MAN= fetch.3
CLEANFILES= ftperr.h httperr.h
MKLINT= no
MKPIC= no
MKPROFILE= no
.include <bsd.own.mk>
CPPFLAGS+= -I.
CPPFLAGS+= -D_LARGEFILE_SOURCE -D_LARGE_FILES -D_FILE_OFFSET_BITS=64
FETCH_WITH_INET6?= no
FETCH_WITH_OPENSSL?= no
.if !empty(FETCH_WITH_INET6:M[yY][eE][sS])
CPPFLAGS+= -DINET6
.endif
.if !empty(FETCH_WITH_OPENSSL:M[yY][eE][sS])
CPPFLAGS+= -DWITH_SSL
LDADD= -lssl -lcrypto
.endif
CPPFLAGS+= -DFTP_COMBINE_CWDS
WARNS?= 4
ftp.o: ftperr.h
http.o: httperr.h
ftperr.h: ${.CURDIR}/ftp.errors ${.CURDIR}/Makefile ${.CURDIR}/errlist.sh
sh ${.CURDIR}/errlist.sh ftp_errlist FTP \
${.CURDIR}/ftp.errors > ${.TARGET}
httperr.h: ${.CURDIR}/http.errors ${.CURDIR}/Makefile ${.CURDIR}/errlist.sh
sh ${.CURDIR}/errlist.sh http_errlist HTTP \
${.CURDIR}/http.errors > ${.TARGET}
.include <bsd.lib.mk>
This diff is collapsed.
/* $NetBSD: common.h,v 1.24 2016/10/20 21:25:57 joerg Exp $ */
/*-
* Copyright (c) 1998-2004 Dag-Erling Codan Smrgrav
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer
* in this position and unchanged.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD: common.h,v 1.30 2007/12/18 11:03:07 des Exp $
*/
#ifndef _COMMON_H_INCLUDED
#define _COMMON_H_INCLUDED
#define FTP_DEFAULT_PORT 21
#define HTTP_DEFAULT_PORT 80
#define FTP_DEFAULT_PROXY_PORT 21
#define HTTP_DEFAULT_PROXY_PORT 3128
#ifdef WITH_SSL
#include <openssl/crypto.h>
#include <openssl/x509.h>
#include <openssl/pem.h>
#include <openssl/ssl.h>
#include <openssl/err.h>
#endif
#if defined(__GNUC__) && __GNUC__ >= 3
#define LIBFETCH_PRINTFLIKE(fmtarg, firstvararg) \
__attribute__((__format__ (__printf__, fmtarg, firstvararg)))
#else
#define LIBFETCH_PRINTFLIKE(fmtarg, firstvararg)
#endif
#if !defined(__sun) && !defined(__hpux) && !defined(__INTERIX) && \
!defined(__digital__) && !defined(__linux) && !defined(__MINT__) && \
!defined(__sgi) && !defined(__minix) && !defined(__CYGWIN__)
#define HAVE_SA_LEN
#endif
/* Connection */
typedef struct fetchconn conn_t;
struct fetchconn {
int sd; /* socket descriptor */
char *buf; /* buffer */
size_t bufsize; /* buffer size */
size_t buflen; /* length of buffer contents */
int buf_events; /* poll flags for the next cycle */
char *next_buf; /* pending buffer, e.g. after getln */
size_t next_len; /* size of pending buffer */
int err; /* last protocol reply code */
#ifdef WITH_SSL
SSL *ssl; /* SSL handle */
SSL_CTX *ssl_ctx; /* SSL context */
X509 *ssl_cert; /* server certificate */
# if OPENSSL_VERSION_NUMBER < 0x00909000L
SSL_METHOD *ssl_meth; /* SSL method */
# else
const SSL_METHOD *ssl_meth; /* SSL method */
# endif
#endif
char *ftp_home;
struct url *cache_url;
int cache_af;
int (*cache_close)(conn_t *);
conn_t *next_cached;
};
/* Structure used for error message lists */
struct fetcherr {
const int num;
const int cat;
const char *string;
};
void fetch_seterr(struct fetcherr *, int);
void fetch_syserr(void);
void fetch_info(const char *, ...) LIBFETCH_PRINTFLIKE(1, 2);
int fetch_default_port(const char *);
int fetch_default_proxy_port(const char *);
int fetch_bind(int, int, const char *);
conn_t *fetch_cache_get(const struct url *, int);
void fetch_cache_put(conn_t *, int (*)(conn_t *));
conn_t *fetch_connect(struct url *, int, int);
conn_t *fetch_reopen(int);
int fetch_ssl(conn_t *, const struct url *, int);
ssize_t fetch_read(conn_t *, char *, size_t);
int fetch_getln(conn_t *);
ssize_t fetch_write(conn_t *, const void *, size_t);
int fetch_close(conn_t *);
int fetch_add_entry(struct url_list *, struct url *, const char *, int);
int fetch_netrc_auth(struct url *url);
int fetch_no_proxy_match(const char *);
int fetch_urlpath_safe(char);
#define ftp_seterr(n) fetch_seterr(ftp_errlist, n)
#define http_seterr(n) fetch_seterr(http_errlist, n)
#define netdb_seterr(n) fetch_seterr(netdb_errlist, n)
#define url_seterr(n) fetch_seterr(url_errlist, n)
fetchIO *fetchIO_unopen(void *, ssize_t (*)(void *, void *, size_t),
ssize_t (*)(void *, const void *, size_t), void (*)(void *));
/*
* I don't really like exporting http_request() and ftp_request(),
* but the HTTP and FTP code occasionally needs to cross-call
* eachother, and this saves me from adding a lot of special-case code
* to handle those cases.
*
* Note that _*_request() free purl, which is way ugly but saves us a
* whole lot of trouble.
*/
fetchIO *http_request(struct url *, const char *,
struct url_stat *, struct url *, const char *);
fetchIO *ftp_request(struct url *, const char *, const char *,
struct url_stat *, struct url *, const char *);
/*
* Check whether a particular flag is set
*/
#define CHECK_FLAG(x) (flags && strchr(flags, (x)))
#endif
#!/bin/sh
# $NetBSD: errlist.sh,v 1.2 2008/10/06 12:58:29 joerg Exp $
printf "static struct fetcherr $1[] = {\n"
while read code type msg; do
[ "${code}" = "#" ] && continue
printf "\t{ ${code}, FETCH_${type}, \"${msg}\" },\n"
done < $3
printf "\t{ -1, FETCH_UNKNOWN, \"Unknown $2 error\" }\n"
printf "};\n"
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
/* $NetBSD: fetch.h,v 1.16 2010/01/22 13:21:09 joerg Exp $ */
/*-
* Copyright (c) 1998-2004 Dag-Erling Codan Smrgrav
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer
* in this position and unchanged.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD: fetch.h,v 1.26 2004/09/21 18:35:20 des Exp $
*/
#ifndef _FETCH_H_INCLUDED
#define _FETCH_H_INCLUDED
#include <sys/types.h>
#include <limits.h>
#include <stdio.h>
#define _LIBFETCH_VER "libfetch/2.0"
#define URL_HOSTLEN 255
#define URL_SCHEMELEN 16
#define URL_USERLEN 256
#define URL_PWDLEN 256
typedef struct fetchIO fetchIO;
struct url {
char scheme[URL_SCHEMELEN + 1];
char user[URL_USERLEN + 1];
char pwd[URL_PWDLEN + 1];
char host[URL_HOSTLEN + 1];
int port;
char *doc;
off_t offset;
size_t length;
time_t last_modified;
};
struct url_stat {
off_t size;
time_t atime;
time_t mtime;
};
struct url_list {
size_t length;
size_t alloc_size;
struct url *urls;
};
/* Recognized schemes */
#define SCHEME_FTP "ftp"
#define SCHEME_HTTP "http"
#define SCHEME_HTTPS "https"
#define SCHEME_FILE "file"
/* Error codes */
#define FETCH_ABORT 1
#define FETCH_AUTH 2
#define FETCH_DOWN 3
#define FETCH_EXISTS 4
#define FETCH_FULL 5
#define FETCH_INFO 6
#define FETCH_MEMORY 7
#define FETCH_MOVED 8
#define FETCH_NETWORK 9
#define FETCH_OK 10
#define FETCH_PROTO 11
#define FETCH_RESOLV 12
#define FETCH_SERVER 13
#define FETCH_TEMP 14
#define FETCH_TIMEOUT 15
#define FETCH_UNAVAIL 16
#define FETCH_UNKNOWN 17
#define FETCH_URL 18
#define FETCH_VERBOSE 19
#define FETCH_UNCHANGED 20
#if defined(__cplusplus)
extern "C" {
#endif
void fetchIO_close(fetchIO *);
ssize_t fetchIO_read(fetchIO *, void *, size_t);
ssize_t fetchIO_write(fetchIO *, const void *, size_t);
/* fetchIO-specific functions */
fetchIO *fetchXGetFile(struct url *, struct url_stat *, const char *);
fetchIO *fetchGetFile(struct url *, const char *);
fetchIO *fetchPutFile(struct url *, const char *);
int fetchStatFile(struct url *, struct url_stat *, const char *);
int fetchListFile(struct url_list *, struct url *, const char *,
const char *);
/* HTTP-specific functions */
fetchIO *fetchXGetHTTP(struct url *, struct url_stat *, const char *);
fetchIO *fetchGetHTTP(struct url *, const char *);
fetchIO *fetchPutHTTP(struct url *, const char *);
int fetchStatHTTP(struct url *, struct url_stat *, const char *);
int fetchListHTTP(struct url_list *, struct url *, const char *,
const char *);
/* FTP-specific functions */
fetchIO *fetchXGetFTP(struct url *, struct url_stat *, const char *);
fetchIO *fetchGetFTP(struct url *, const char *);
fetchIO *fetchPutFTP(struct url *, const char *);
int fetchStatFTP(struct url *, struct url_stat *, const char *);
int fetchListFTP(struct url_list *, struct url *, const char *,
const char *);
/* Generic functions */
fetchIO *fetchXGetURL(const char *, struct url_stat *, const char *);
fetchIO *fetchGetURL(const char *, const char *);
fetchIO *fetchPutURL(const char *, const char *);
int fetchStatURL(const char *, struct url_stat *, const char *);
int fetchListURL(struct url_list *, const char *, const char *,
const char *);
fetchIO *fetchXGet(struct url *, struct url_stat *, const char *);
fetchIO *fetchGet(struct url *, const char *);
fetchIO *fetchPut(struct url *, const char *);
int fetchStat(struct url *, struct url_stat *, const char *);
int fetchList(struct url_list *, struct url *, const char *,
const char *);
/* URL parsing */
struct url *fetchMakeURL(const char *, const char *, int,
const char *, const char *, const char *);
struct url *fetchParseURL(const char *);
struct url *fetchCopyURL(const struct url *);
char *fetchStringifyURL(const struct url *);
void fetchFreeURL(struct url *);
/* URL listening */
void fetchInitURLList(struct url_list *);
int fetchAppendURLList(struct url_list *, const struct url_list *);
void fetchFreeURLList(struct url_list *);
char *fetchUnquotePath(struct url *);
char *fetchUnquoteFilename(struct url *);
/* Connection caching */
void fetchConnectionCacheInit(int, int);
void fetchConnectionCacheClose(void);
/* Authentication */
typedef int (*auth_t)(struct url *);
extern auth_t fetchAuthMethod;
/* Last error code */
extern int fetchLastErrCode;
#define MAXERRSTRING 256
extern char fetchLastErrString[MAXERRSTRING];
/* I/O timeout */
extern int fetchTimeout;
/* Restart interrupted syscalls */
extern volatile int fetchRestartCalls;
/* Extra verbosity */
extern int fetchDebug;
#if defined(__cplusplus)
}
#endif
#endif
/* $NetBSD: file.c,v 1.15 2009/10/15 12:36:57 joerg Exp $ */
/*-
* Copyright (c) 1998-2004 Dag-Erling Codan Smrgrav
* Copyright (c) 2008, 2009 Joerg Sonnenberger <joerg@NetBSD.org>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer
* in this position and unchanged.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. The name of the author may not be used to endorse or promote products
* derived from this software without specific prior written permission
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* $FreeBSD: file.c,v 1.18 2007/12/14 10:26:58 des Exp $
*/
#if HAVE_CONFIG_H
#include "config.h"
#endif
#ifndef NETBSD
#include <nbcompat.h>
#endif
#include <sys/stat.h>
#include <dirent.h>
#include <fcntl.h>
#include <fnmatch.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "fetch.h"
#include "common.h"
static int fetch_stat_file(int, struct url_stat *);
static ssize_t
fetchFile_read(void *cookie, void *buf, size_t len)
{
return read(*(int *)cookie, buf, len);
}
static ssize_t
fetchFile_write(void *cookie, const void *buf, size_t len)
{
return write(*(int *)cookie, buf, len);
}
static void
fetchFile_close(void *cookie)
{
int fd = *(int *)cookie;
free(cookie);
close(fd);
}
fetchIO *
fetchXGetFile(struct url *u, struct url_stat *us, const char *flags)
{
char *path;
fetchIO *f;
struct url_stat local_us;
int if_modified_since, fd, *cookie;
if_modified_since = CHECK_FLAG('i');
if (if_modified_since && us == NULL)
us = &local_us;
if ((path = fetchUnquotePath(u)) == NULL) {
fetch_syserr();
return NULL;
}
fd = open(path, O_RDONLY);
free(path);
if (fd == -1) {
fetch_syserr();
return NULL;
}
if (us && fetch_stat_file(fd, us) == -1) {
close(fd);
fetch_syserr();
return NULL;
}
if (if_modified_since && u->last_modified > 0 &&
u->last_modified >= us->mtime) {
close(fd);
fetchLastErrCode = FETCH_UNCHANGED;
snprintf(fetchLastErrString, MAXERRSTRING, "Unchanged");
return NULL;
}
if (u->offset && lseek(fd, u->offset, SEEK_SET) == -1) {
close(fd);
fetch_syserr();
return NULL;
}
cookie = malloc(sizeof(int));
if (cookie == NULL) {
close(fd);
fetch_syserr();
return NULL;
}
*cookie = fd;
f = fetchIO_unopen(cookie, fetchFile_read, fetchFile_write, fetchFile_close);
if (f == NULL) {
close(fd);
free(cookie);
}
return f;
}
fetchIO *
fetchGetFile(struct url *u, const char *flags)
{
return (fetchXGetFile(u, NULL, flags));
}
fetchIO *
fetchPutFile(struct url *u, const char *flags)
{
char *path;
fetchIO *f;
int fd, *cookie;
if ((path = fetchUnquotePath(u)) == NULL) {
fetch_syserr();
return NULL;
}
if (CHECK_FLAG('a'))
fd = open(path, O_WRONLY | O_APPEND);
else
fd = open(path, O_WRONLY);
free(path);
if (fd == -1) {
fetch_syserr();
return NULL;
}
if (u->offset && lseek(fd, u->offset, SEEK_SET) == -1) {
close(fd);
fetch_syserr();
return NULL;
}
cookie = malloc(sizeof(int));
if (cookie == NULL) {
close(fd);
fetch_syserr();
return NULL;
}
*cookie = fd;
f = fetchIO_unopen(cookie, fetchFile_read, fetchFile_write, fetchFile_close);
if (f == NULL) {
close(fd);
free(cookie);
}
return f;
}
static int
fetch_stat_file(int fd, struct url_stat *us)
{
struct stat sb;
us->size = -1;
us->atime = us->mtime = 0;
if (fstat(fd, &sb) == -1) {
fetch_syserr();
return (-1);
}
us->size = sb.st_size;
us->atime = sb.st_atime;
us->mtime = sb.st_mtime;
return (0);
}
int
fetchStatFile(struct url *u, struct url_stat *us, const char *flags)
{
char *path;
int fd, rv;
if ((path = fetchUnquotePath(u)) == NULL) {
fetch_syserr();
return -1;
}
fd = open(path, O_RDONLY);
free(path);
if (fd == -1) {
fetch_syserr();
return -1;
}
rv = fetch_stat_file(fd, us);
close(fd);
return rv;
}
int
fetchListFile(struct url_list *ue, struct url *u, const char *pattern, const char *flags)
{
char *path;
struct dirent *de;
DIR *dir;
int ret;
if ((path = fetchUnquotePath(u)) == NULL) {
fetch_syserr();
return -1;
}
dir = opendir(path);
free(path);
if (dir == NULL) {
fetch_syserr();
return -1;
}
ret = 0;
while ((de = readdir(dir)) != NULL) {
if (pattern && fnmatch(pattern, de->d_name, 0) != 0)
continue;
ret = fetch_add_entry(ue, u, de->d_name, 0);
if (ret)
break;
}
closedir(dir);
return ret;
}
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment