Gitlab has successfully been upgraded to 14.0.10 🎉. Enjoy!

Commit 32627939 authored by Timo Teräs's avatar Timo Teräs
Browse files

io,url,db: support for if-modified-since

parent 555363f0
...@@ -32,6 +32,12 @@ ...@@ -32,6 +32,12 @@
#define NULL 0L #define NULL 0L
#endif #endif
static inline void *ERR_PTR(long error) { return (void*) error; }
static inline void *ERR_CAST(const void *ptr) { return (void*) ptr; }
static inline int PTR_ERR(const void *ptr) { return (int)(long) ptr; }
static inline int IS_ERR(const void *ptr) { return (unsigned long)ptr >= (unsigned long)-4095; }
static inline int IS_ERR_OR_NULL(const void *ptr) { return IS_ERR(ptr) || !ptr; }
#if defined __GNUC__ && __GNUC__ == 2 && __GNUC_MINOR__ < 96 #if defined __GNUC__ && __GNUC__ == 2 && __GNUC_MINOR__ < 96
#define __builtin_expect(x, expected_value) (x) #define __builtin_expect(x, expected_value) (x)
#endif #endif
......
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
#include <sys/types.h> #include <sys/types.h>
#include <openssl/evp.h> #include <openssl/evp.h>
#include <fcntl.h> #include <fcntl.h>
#include <time.h>
#include "apk_defines.h" #include "apk_defines.h"
#include "apk_blob.h" #include "apk_blob.h"
...@@ -78,7 +79,7 @@ struct apk_ostream *apk_ostream_counter(off_t *); ...@@ -78,7 +79,7 @@ struct apk_ostream *apk_ostream_counter(off_t *);
struct apk_istream *apk_istream_from_fd_pid(int fd, pid_t pid, int (*translate_status)(int)); struct apk_istream *apk_istream_from_fd_pid(int fd, pid_t pid, int (*translate_status)(int));
struct apk_istream *apk_istream_from_file(int atfd, const char *file); struct apk_istream *apk_istream_from_file(int atfd, const char *file);
struct apk_istream *apk_istream_from_file_gz(int atfd, const char *file); struct apk_istream *apk_istream_from_file_gz(int atfd, const char *file);
struct apk_istream *apk_istream_from_fd_url(int atfd, const char *url); struct apk_istream *apk_istream_from_fd_url_if_modified(int atfd, const char *url, time_t since);
struct apk_istream *apk_istream_from_url_gz(const char *url); struct apk_istream *apk_istream_from_url_gz(const char *url);
size_t apk_istream_skip(struct apk_istream *istream, size_t size); size_t apk_istream_skip(struct apk_istream *istream, size_t size);
...@@ -92,13 +93,21 @@ static inline struct apk_istream *apk_istream_from_fd(int fd) ...@@ -92,13 +93,21 @@ static inline struct apk_istream *apk_istream_from_fd(int fd)
} }
static inline struct apk_istream *apk_istream_from_url(const char *url) static inline struct apk_istream *apk_istream_from_url(const char *url)
{ {
return apk_istream_from_fd_url(AT_FDCWD, url); return apk_istream_from_fd_url_if_modified(AT_FDCWD, url, 0);
}
static inline struct apk_istream *apk_istream_from_fd_url(int atfd, const char *url)
{
return apk_istream_from_fd_url_if_modified(atfd, url, 0);
}
static inline struct apk_istream *apk_istream_from_url_if_modified(const char *url, time_t since)
{
return apk_istream_from_fd_url_if_modified(AT_FDCWD, url, since);
} }
struct apk_bstream *apk_bstream_from_istream(struct apk_istream *istream); struct apk_bstream *apk_bstream_from_istream(struct apk_istream *istream);
struct apk_bstream *apk_bstream_from_fd_pid(int fd, pid_t pid, int (*translate_status)(int)); struct apk_bstream *apk_bstream_from_fd_pid(int fd, pid_t pid, int (*translate_status)(int));
struct apk_bstream *apk_bstream_from_file(int atfd, const char *file); struct apk_bstream *apk_bstream_from_file(int atfd, const char *file);
struct apk_bstream *apk_bstream_from_fd_url(int atfd, const char *url); struct apk_bstream *apk_bstream_from_fd_url_if_modified(int atfd, const char *url, time_t since);
struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to, struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const char *to,
apk_progress_cb cb, void *cb_ctx); apk_progress_cb cb, void *cb_ctx);
...@@ -109,7 +118,15 @@ static inline struct apk_bstream *apk_bstream_from_fd(int fd) ...@@ -109,7 +118,15 @@ static inline struct apk_bstream *apk_bstream_from_fd(int fd)
static inline struct apk_bstream *apk_bstream_from_url(const char *url) static inline struct apk_bstream *apk_bstream_from_url(const char *url)
{ {
return apk_bstream_from_fd_url(AT_FDCWD, url); return apk_bstream_from_fd_url_if_modified(AT_FDCWD, url, 0);
}
static inline struct apk_bstream *apk_bstream_from_fd_url(int fd, const char *url)
{
return apk_bstream_from_fd_url_if_modified(fd, url, 0);
}
static inline struct apk_bstream *apk_bstream_from_url_if_modified(const char *url, time_t since)
{
return apk_bstream_from_fd_url_if_modified(AT_FDCWD, url, since);
} }
struct apk_ostream *apk_ostream_to_fd(int fd); struct apk_ostream *apk_ostream_to_fd(int fd);
......
...@@ -605,6 +605,7 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo, ...@@ -605,6 +605,7 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
struct apk_package *pkg, int verify, struct apk_package *pkg, int verify,
apk_progress_cb cb, void *cb_ctx) apk_progress_cb cb, void *cb_ctx)
{ {
struct stat st;
struct apk_istream *is; struct apk_istream *is;
struct apk_bstream *bs; struct apk_bstream *bs;
struct apk_sign_ctx sctx; struct apk_sign_ctx sctx;
...@@ -618,40 +619,42 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo, ...@@ -618,40 +619,42 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
r = apk_pkg_format_cache_pkg(b, pkg); r = apk_pkg_format_cache_pkg(b, pkg);
else else
r = apk_repo_format_cache_index(b, repo); r = apk_repo_format_cache_index(b, repo);
if (r < 0) if (r < 0) return r;
return r;
r = apk_repo_format_real_url(db, repo, pkg, url, sizeof(url)); r = apk_repo_format_real_url(db, repo, pkg, url, sizeof(url));
if (r < 0) if (r < 0) return r;
return r;
apk_message("fetch %s", url); if (fstatat(db->cache_fd, cacheitem, &st, 0) != 0) st.st_mtime = 0;
if (apk_flags & APK_SIMULATE) apk_message("fetch %s", url);
return 0;
if (cb) if (apk_flags & APK_SIMULATE) return 0;
cb(cb_ctx, 0); if (cb) cb(cb_ctx, 0);
if (verify != APK_SIGN_NONE) { if (verify != APK_SIGN_NONE) {
apk_sign_ctx_init(&sctx, APK_SIGN_VERIFY, NULL, db->keys_fd); apk_sign_ctx_init(&sctx, APK_SIGN_VERIFY, NULL, db->keys_fd);
bs = apk_bstream_from_url(url); bs = apk_bstream_from_url_if_modified(url, st.st_mtime);
bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, cb, cb_ctx); bs = apk_bstream_tee(bs, db->cache_fd, tmpcacheitem, cb, cb_ctx);
is = apk_bstream_gunzip_mpart(bs, apk_sign_ctx_mpart_cb, &sctx); is = apk_bstream_gunzip_mpart(bs, apk_sign_ctx_mpart_cb, &sctx);
if (is) r = apk_tar_parse(is, apk_sign_ctx_verify_tar, &sctx, FALSE, &db->id_cache); if (!IS_ERR_OR_NULL(is))
else r = -errno; r = apk_tar_parse(is, apk_sign_ctx_verify_tar, &sctx, FALSE, &db->id_cache);
else
r = PTR_ERR(is) ?: -EIO;
apk_sign_ctx_free(&sctx); apk_sign_ctx_free(&sctx);
} else { } else {
is = apk_istream_from_url(url); is = apk_istream_from_url_if_modified(url, st.st_mtime);
fd = openat(db->cache_fd, tmpcacheitem, O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC, 0644); if (!IS_ERR_OR_NULL(is)) {
fd = openat(db->cache_fd, tmpcacheitem, O_RDWR | O_CREAT | O_TRUNC | O_CLOEXEC, 0644);
if (fd < 0) r = -errno;
} else fd = -1, r = PTR_ERR(is) ?: -EIO;
if (fd >= 0) { if (fd >= 0) {
r = apk_istream_splice(is, fd, APK_SPLICE_ALL, cb, cb_ctx); r = apk_istream_splice(is, fd, APK_SPLICE_ALL, cb, cb_ctx);
close(fd); close(fd);
} else {
r = -errno;
} }
} }
if (is) is->close(is); if (!IS_ERR_OR_NULL(is)) is->close(is);
if (r == -EALREADY) return 0;
if (r < 0) { if (r < 0) {
unlinkat(db->cache_fd, tmpcacheitem, 0); unlinkat(db->cache_fd, tmpcacheitem, 0);
return r; return r;
...@@ -659,7 +662,6 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo, ...@@ -659,7 +662,6 @@ int apk_cache_download(struct apk_database *db, struct apk_repository *repo,
if (renameat(db->cache_fd, tmpcacheitem, db->cache_fd, cacheitem) < 0) if (renameat(db->cache_fd, tmpcacheitem, db->cache_fd, cacheitem) < 0)
return -errno; return -errno;
return 0; return 0;
} }
......
...@@ -149,7 +149,7 @@ struct apk_istream *apk_bstream_gunzip_mpart(struct apk_bstream *bs, ...@@ -149,7 +149,7 @@ struct apk_istream *apk_bstream_gunzip_mpart(struct apk_bstream *bs,
{ {
struct apk_gzip_istream *gis; struct apk_gzip_istream *gis;
if (!bs) return NULL; if (IS_ERR_OR_NULL(bs)) return ERR_CAST(bs);
gis = malloc(sizeof(struct apk_gzip_istream)); gis = malloc(sizeof(struct apk_gzip_istream));
if (!gis) goto err; if (!gis) goto err;
...@@ -233,8 +233,7 @@ struct apk_ostream *apk_ostream_gzip(struct apk_ostream *output) ...@@ -233,8 +233,7 @@ struct apk_ostream *apk_ostream_gzip(struct apk_ostream *output)
{ {
struct apk_gzip_ostream *gos; struct apk_gzip_ostream *gos;
if (output == NULL) if (IS_ERR_OR_NULL(output)) return ERR_CAST(output);
return NULL;
gos = malloc(sizeof(struct apk_gzip_ostream)); gos = malloc(sizeof(struct apk_gzip_ostream));
if (gos == NULL) if (gos == NULL)
......
...@@ -83,8 +83,7 @@ struct apk_istream *apk_istream_from_fd_pid(int fd, pid_t pid, int (*translate_s ...@@ -83,8 +83,7 @@ struct apk_istream *apk_istream_from_fd_pid(int fd, pid_t pid, int (*translate_s
{ {
struct apk_fd_istream *fis; struct apk_fd_istream *fis;
if (fd < 0) if (fd < 0) return NULL;
return NULL;
fis = malloc(sizeof(struct apk_fd_istream)); fis = malloc(sizeof(struct apk_fd_istream));
if (fis == NULL) { if (fis == NULL) {
...@@ -108,8 +107,7 @@ struct apk_istream *apk_istream_from_file(int atfd, const char *file) ...@@ -108,8 +107,7 @@ struct apk_istream *apk_istream_from_file(int atfd, const char *file)
int fd; int fd;
fd = openat(atfd, file, O_RDONLY | O_CLOEXEC); fd = openat(atfd, file, O_RDONLY | O_CLOEXEC);
if (fd < 0) if (fd < 0) return NULL;
return NULL;
return apk_istream_from_fd(fd); return apk_istream_from_fd(fd);
} }
...@@ -272,9 +270,10 @@ struct apk_bstream *apk_bstream_from_istream(struct apk_istream *istream) ...@@ -272,9 +270,10 @@ struct apk_bstream *apk_bstream_from_istream(struct apk_istream *istream)
{ {
struct apk_istream_bstream *isbs; struct apk_istream_bstream *isbs;
if (IS_ERR_OR_NULL(istream)) return ERR_CAST(istream);
isbs = malloc(sizeof(struct apk_istream_bstream)); isbs = malloc(sizeof(struct apk_istream_bstream));
if (isbs == NULL) if (isbs == NULL) return NULL;
return NULL;
isbs->bs = (struct apk_bstream) { isbs->bs = (struct apk_bstream) {
.read = is_bs_read, .read = is_bs_read,
...@@ -331,12 +330,10 @@ static struct apk_bstream *apk_mmap_bstream_from_fd(int fd) ...@@ -331,12 +330,10 @@ static struct apk_bstream *apk_mmap_bstream_from_fd(int fd)
struct stat st; struct stat st;
void *ptr; void *ptr;
if (fstat(fd, &st) < 0) if (fstat(fd, &st) < 0) return NULL;
return NULL;
ptr = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0); ptr = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
if (ptr == MAP_FAILED) if (ptr == MAP_FAILED) return NULL;
return NULL;
mbs = malloc(sizeof(struct apk_mmap_bstream)); mbs = malloc(sizeof(struct apk_mmap_bstream));
if (mbs == NULL) { if (mbs == NULL) {
...@@ -361,8 +358,7 @@ struct apk_bstream *apk_bstream_from_fd_pid(int fd, pid_t pid, int (*translate_s ...@@ -361,8 +358,7 @@ struct apk_bstream *apk_bstream_from_fd_pid(int fd, pid_t pid, int (*translate_s
{ {
struct apk_bstream *bs; struct apk_bstream *bs;
if (fd < 0) if (fd < 0) return NULL;
return NULL;
if (pid == 0) { if (pid == 0) {
bs = apk_mmap_bstream_from_fd(fd); bs = apk_mmap_bstream_from_fd(fd);
...@@ -378,8 +374,7 @@ struct apk_bstream *apk_bstream_from_file(int atfd, const char *file) ...@@ -378,8 +374,7 @@ struct apk_bstream *apk_bstream_from_file(int atfd, const char *file)
int fd; int fd;
fd = openat(atfd, file, O_RDONLY | O_CLOEXEC); fd = openat(atfd, file, O_RDONLY | O_CLOEXEC);
if (fd < 0) if (fd < 0) return NULL;
return NULL;
return apk_bstream_from_fd(fd); return apk_bstream_from_fd(fd);
} }
...@@ -427,7 +422,7 @@ struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const ch ...@@ -427,7 +422,7 @@ struct apk_bstream *apk_bstream_tee(struct apk_bstream *from, int atfd, const ch
struct apk_tee_bstream *tbs; struct apk_tee_bstream *tbs;
int fd; int fd;
if (!from) return NULL; if (IS_ERR_OR_NULL(from)) return ERR_CAST(from);
fd = openat(atfd, to, O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC, fd = openat(atfd, to, O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
...@@ -831,8 +826,8 @@ int apk_move_file(int atfd, const char *from, const char *to) ...@@ -831,8 +826,8 @@ int apk_move_file(int atfd, const char *from, const char *to)
return -errno; return -errno;
is = apk_istream_from_file(atfd, from); is = apk_istream_from_file(atfd, from);
if (is == NULL) if (IS_ERR(is)) return PTR_ERR(is);
return -ENOENT; if (!is) return -ENOENT;
tofd = openat(atfd, to, O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC, tofd = openat(atfd, to, O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC,
S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
......
...@@ -63,34 +63,43 @@ static void fetch_close(void *stream) ...@@ -63,34 +63,43 @@ static void fetch_close(void *stream)
free(fis); free(fis);
} }
static struct apk_istream *apk_istream_fetch(const char *url) static struct apk_istream *apk_istream_fetch(const char *url, time_t since)
{ {
struct apk_fetch_istream *fis; struct apk_fetch_istream *fis;
struct url *u;
fetchIO *io; fetchIO *io;
io = fetchGetURL(url, ""); u = fetchParseURL(url);
if (!io) return NULL; if (!u) return NULL;
u->last_modified = since;
fis = malloc(sizeof(*fis)); io = fetchGet(u, "i");
if (!fis) { fetchFreeURL(u);
fetchIO_close(io); if (!io) {
if (fetchLastErrCode == FETCH_UNCHANGED) return ERR_PTR(-EALREADY);
return NULL; return NULL;
} }
fis = malloc(sizeof(*fis));
if (!fis) goto err;
*fis = (struct apk_fetch_istream) { *fis = (struct apk_fetch_istream) {
.is.read = fetch_read, .is.read = fetch_read,
.is.close = fetch_close, .is.close = fetch_close,
.fetchIO = io, .fetchIO = io,
}; };
fetchFreeURL(u);
return &fis->is; return &fis->is;
err:
if (io) fetchIO_close(io);
return NULL;
} }
struct apk_istream *apk_istream_from_fd_url(int atfd, const char *url) struct apk_istream *apk_istream_from_fd_url_if_modified(int atfd, const char *url, time_t since)
{ {
if (apk_url_local_file(url) != NULL) if (apk_url_local_file(url) != NULL)
return apk_istream_from_file(atfd, apk_url_local_file(url)); return apk_istream_from_file(atfd, apk_url_local_file(url));
return apk_istream_fetch(url); return apk_istream_fetch(url, since);
} }
struct apk_istream *apk_istream_from_url_gz(const char *file) struct apk_istream *apk_istream_from_url_gz(const char *file)
...@@ -98,9 +107,9 @@ struct apk_istream *apk_istream_from_url_gz(const char *file) ...@@ -98,9 +107,9 @@ struct apk_istream *apk_istream_from_url_gz(const char *file)
return apk_bstream_gunzip(apk_bstream_from_url(file)); return apk_bstream_gunzip(apk_bstream_from_url(file));
} }
struct apk_bstream *apk_bstream_from_fd_url(int atfd, const char *url) struct apk_bstream *apk_bstream_from_fd_url_if_modified(int atfd, const char *url, time_t since)
{ {
if (apk_url_local_file(url) != NULL) if (apk_url_local_file(url) != NULL)
return apk_bstream_from_file(atfd, apk_url_local_file(url)); return apk_bstream_from_file(atfd, apk_url_local_file(url));
return apk_bstream_from_istream(apk_istream_fetch(url)); return apk_bstream_from_istream(apk_istream_fetch(url, since));
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment