Commit 6da3e8eb authored by Timo Teräs's avatar Timo Teräs

istream, archive, db: convert db and tar function to use istream

parent 7ca0d146
......@@ -635,14 +635,9 @@ int main(int argc, char **argv)
apk_blob_pull_deps(&b, &db, &db.world);
}
if (test_installed_db != NULL) {
struct apk_bstream *bs = apk_bstream_from_file(AT_FDCWD, test_installed_db);
if (!IS_ERR_OR_NULL(bs)) {
apk_db_index_read(&db, bs, -1);
apk_bstream_close(bs);
}
apk_db_index_read(&db, apk_istream_from_file(AT_FDCWD, test_installed_db), -1);
}
for (i = 0; i < test_repos->num; i++) {
struct apk_bstream *bs;
apk_blob_t spec = APK_BLOB_STR(test_repos->item[i]), name, tag;
int repo_tag = 0, repo = APK_REPOSITORY_FIRST_CONFIGURED + i;
......@@ -659,14 +654,11 @@ int main(int argc, char **argv)
name = spec;
}
bs = apk_bstream_from_file(AT_FDCWD, name.ptr);
if (IS_ERR_OR_NULL(bs)) {
if (apk_db_index_read(&db, apk_istream_from_file(AT_FDCWD, name.ptr), repo) != 0) {
apk_error("Failed to open repository: " BLOB_FMT, BLOB_PRINTF(name));
goto err;
}
apk_db_index_read(&db, bs, repo);
apk_bstream_close(bs);
if (repo != -2) {
if (!(apk_flags & APK_NO_NETWORK))
db.available_repos |= BIT(repo);
......
......@@ -234,7 +234,7 @@ struct apk_package *apk_db_pkg_add(struct apk_database *db, struct apk_package *
struct apk_package *apk_db_get_pkg(struct apk_database *db, struct apk_checksum *csum);
struct apk_package *apk_db_get_file_owner(struct apk_database *db, apk_blob_t filename);
int apk_db_index_read(struct apk_database *db, struct apk_bstream *bs, int repo);
int apk_db_index_read(struct apk_database *db, struct apk_istream *is, int repo);
int apk_db_index_read_file(struct apk_database *db, const char *file, int repo);
int apk_db_index_write(struct apk_database *db, struct apk_ostream *os);
......
......@@ -81,6 +81,9 @@ struct apk_istream *apk_istream_from_fd(int fd);
struct apk_istream *apk_istream_from_fd_url_if_modified(int atfd, const char *url, time_t since);
struct apk_istream *apk_istream_from_url_gz(const char *url);
ssize_t apk_istream_read(struct apk_istream *is, void *ptr, size_t size);
apk_blob_t apk_istream_get(struct apk_istream *is, size_t len);
apk_blob_t apk_istream_get_all(struct apk_istream *is);
apk_blob_t apk_istream_get_delim(struct apk_istream *is, apk_blob_t token);
#define APK_SPLICE_ALL 0xffffffff
ssize_t apk_istream_splice(struct apk_istream *is, int fd, size_t size,
......@@ -107,6 +110,14 @@ static inline void apk_istream_close(struct apk_istream *is)
is->ops->close(is);
}
struct apk_segment_istream {
struct apk_istream is;
struct apk_istream *pis;
size_t bytes_left;
time_t mtime;
};
void apk_istream_segment(struct apk_segment_istream *sis, struct apk_istream *is, size_t len, time_t mtime);
#define APK_BSTREAM_SINGLE_READ 0x0001
#define APK_BSTREAM_EOF 0x0002
......
......@@ -73,55 +73,6 @@ static void put_octal(char *s, size_t l, size_t value)
*(ptr--) = '0';
}
struct apk_tar_entry_istream {
struct apk_istream is;
struct apk_istream *tar_is;
size_t bytes_left;
time_t mtime;
};
static void tar_entry_get_meta(struct apk_istream *is, struct apk_file_meta *meta)
{
struct apk_tar_entry_istream *teis = container_of(is, struct apk_tar_entry_istream, is);
*meta = (struct apk_file_meta) {
.atime = teis->mtime,
.mtime = teis->mtime,
};
}
static ssize_t tar_entry_read(struct apk_istream *is, void *ptr, size_t size)
{
struct apk_tar_entry_istream *teis = container_of(is, struct apk_tar_entry_istream, is);
ssize_t r;
if (size > teis->bytes_left)
size = teis->bytes_left;
if (size == 0)
return 0;
r = apk_istream_read(teis->tar_is, ptr, size);
if (r <= 0) {
/* If inner stream returned zero (end-of-stream), we
* are getting short read, because tar header indicated
* more was to be expected. */
if (r == 0) return -ECONNABORTED;
return r;
}
teis->bytes_left -= r;
return r;
}
static void tar_entry_close(struct apk_istream *is)
{
}
static const struct apk_istream_ops tar_istream_ops = {
.get_meta = tar_entry_get_meta,
.read = tar_entry_read,
.close = tar_entry_close,
};
static int blob_realloc(apk_blob_t *b, size_t newsize)
{
char *tmp;
......@@ -180,12 +131,8 @@ int apk_tar_parse(struct apk_istream *is, apk_archive_entry_parser parser,
void *ctx, struct apk_id_cache *idc)
{
struct apk_file_info entry;
struct apk_tar_entry_istream teis = {
.is.ops = &tar_istream_ops,
.tar_is = is,
};
struct apk_segment_istream segment;
struct tar_header buf;
unsigned long offset = 0;
int end = 0, r;
size_t toskip, paxlen = 0;
apk_blob_t pax = APK_BLOB_NULL, longname = APK_BLOB_NULL;
......@@ -194,7 +141,6 @@ int apk_tar_parse(struct apk_istream *is, apk_archive_entry_parser parser,
memset(&entry, 0, sizeof(entry));
entry.name = buf.name;
while ((r = apk_istream_read(is, &buf, 512)) == 512) {
offset += 512;
if (buf.name[0] == '\0') {
if (end) break;
end++;
......@@ -222,7 +168,6 @@ int apk_tar_parse(struct apk_istream *is, apk_archive_entry_parser parser,
}
buf.mode[0] = 0; /* to nul terminate 100-byte buf.name */
buf.magic[0] = 0; /* to nul terminate 100-byte buf.linkname */
teis.mtime = entry.mtime;
apk_xattr_array_resize(&entry.xattrs, 0);
if (entry.size >= SSIZE_MAX-512) goto err;
......@@ -232,6 +177,7 @@ int apk_tar_parse(struct apk_istream *is, apk_archive_entry_parser parser,
apk_fileinfo_hash_xattr(&entry);
}
toskip = (entry.size + 511) & -512;
switch (buf.typeflag) {
case 'L': /* GNU long name extension */
if ((r = blob_realloc(&longname, entry.size+1)) != 0 ||
......@@ -239,8 +185,7 @@ int apk_tar_parse(struct apk_istream *is, apk_archive_entry_parser parser,
goto err;
entry.name = longname.ptr;
entry.name[entry.size] = 0;
offset += entry.size;
entry.size = 0;
toskip -= entry.size;
break;
case 'K': /* GNU long link target extension - ignored */
break;
......@@ -272,11 +217,10 @@ int apk_tar_parse(struct apk_istream *is, apk_archive_entry_parser parser,
break;
case 'x': /* file specific pax header */
paxlen = entry.size;
entry.size = 0;
if ((r = blob_realloc(&pax, (paxlen + 511) & -512)) != 0 ||
(r = apk_istream_read(is, pax.ptr, paxlen)) != paxlen)
goto err;
offset += paxlen;
toskip -= entry.size;
break;
default:
break;
......@@ -288,26 +232,19 @@ int apk_tar_parse(struct apk_istream *is, apk_archive_entry_parser parser,
goto err;
}
teis.bytes_left = entry.size;
if (entry.mode & S_IFMT) {
r = parser(ctx, &entry, &teis.is);
apk_istream_segment(&segment, is, entry.size, entry.mtime);
r = parser(ctx, &entry, &segment.is);
if (r != 0) goto err;
apk_istream_close(&segment.is);
entry.name = buf.name;
toskip -= entry.size;
paxlen = 0;
}
offset += entry.size - teis.bytes_left;
toskip = teis.bytes_left;
if ((offset + toskip) & 511)
toskip += 512 - ((offset + toskip) & 511);
offset += toskip;
if (toskip != 0) {
if ((r = apk_istream_read(is, NULL, toskip)) != toskip) {
r = -EIO;
goto err;
}
}
if (toskip && (r = apk_istream_read(is, NULL, toskip)) != toskip)
goto err;
}
/* Read remaining end-of-archive records, to ensure we read all of
......
......@@ -174,13 +174,11 @@ int apk_blob_split(apk_blob_t blob, apk_blob_t split, apk_blob_t *l, apk_blob_t
{
char *pos = blob.ptr, *end = blob.ptr + blob.len - split.len + 1;
if (end < pos)
return 0;
if (!pos || end < pos) return 0;
while (1) {
pos = memchr(pos, split.ptr[0], end - pos);
if (pos == NULL)
return 0;
if (!pos) return 0;
if (split.len > 1 && memcmp(pos, split.ptr, split.len) != 0) {
pos++;
......
......@@ -715,27 +715,26 @@ static struct apk_db_dir_instance *find_diri(struct apk_installed_package *ipkg,
return NULL;
}
int apk_db_read_overlay(struct apk_database *db, struct apk_bstream *bs)
int apk_db_read_overlay(struct apk_database *db, struct apk_istream *is)
{
struct apk_db_dir_instance *diri = NULL;
struct hlist_node **diri_node = NULL, **file_diri_node = NULL;
struct apk_package *pkg;
struct apk_installed_package *ipkg;
apk_blob_t token = APK_BLOB_STR("\n"), line, bdir, bfile;
int r = -1;
if (IS_ERR_OR_NULL(bs)) return -1;
if (IS_ERR_OR_NULL(is)) return -1;
pkg = apk_pkg_new();
if (pkg == NULL)
return -1;
if (pkg == NULL) goto err;
ipkg = apk_pkg_install(db, pkg);
if (ipkg == NULL)
return -1;
if (ipkg == NULL) goto err;
diri_node = hlist_tail_ptr(&ipkg->owned_dirs);
while (!APK_BLOB_IS_NULL(line = apk_bstream_read(bs, token))) {
while (!APK_BLOB_IS_NULL(line = apk_istream_get_delim(is, token))) {
if (!apk_blob_rsplit(line, '/', &bdir, &bfile))
break;
......@@ -752,11 +751,13 @@ int apk_db_read_overlay(struct apk_database *db, struct apk_bstream *bs)
(void) apk_db_file_get(db, diri, bfile, &file_diri_node);
}
}
return 0;
r = 0;
err:
apk_istream_close(is);
return r;
}
int apk_db_index_read(struct apk_database *db, struct apk_bstream *bs, int repo)
int apk_db_index_read(struct apk_database *db, struct apk_istream *is, int repo)
{
struct apk_package *pkg = NULL;
struct apk_installed_package *ipkg = NULL;
......@@ -772,7 +773,9 @@ int apk_db_index_read(struct apk_database *db, struct apk_bstream *bs, int repo)
gid_t gid;
int field, r, lineno = 0;
while (!APK_BLOB_IS_NULL(l = apk_bstream_read(bs, token))) {
if (IS_ERR_OR_NULL(is)) return PTR_ERR(is);
while (!APK_BLOB_IS_NULL(l = apk_istream_get_delim(is, token))) {
lineno++;
if (l.len < 2 || l.ptr[1] != ':') {
......@@ -893,13 +896,16 @@ int apk_db_index_read(struct apk_database *db, struct apk_bstream *bs, int repo)
}
if (APK_BLOB_IS_NULL(l)) goto bad_entry;
}
apk_istream_close(is);
return 0;
old_apk_tools:
/* Installed db should not have unsupported fields */
apk_error("This apk-tools is too old to handle installed packages");
return -1;
goto err;
bad_entry:
apk_error("FDB format error (line %d, entry '%c')", lineno, field);
err:
apk_istream_close(is);
return -1;
}
......@@ -1114,14 +1120,16 @@ static void apk_db_triggers_write(struct apk_database *db, struct apk_ostream *o
}
}
static void apk_db_triggers_read(struct apk_database *db, struct apk_bstream *bs)
static void apk_db_triggers_read(struct apk_database *db, struct apk_istream *is)
{
struct apk_checksum csum;
struct apk_package *pkg;
struct apk_installed_package *ipkg;
apk_blob_t l;
while (!APK_BLOB_IS_NULL(l = apk_bstream_read(bs, APK_BLOB_STR("\n")))) {
if (IS_ERR_OR_NULL(is)) return;
while (!APK_BLOB_IS_NULL(l = apk_istream_get_delim(is, APK_BLOB_STR("\n")))) {
apk_blob_pull_csum(&l, &csum);
apk_blob_pull_char(&l, ' ');
......@@ -1136,12 +1144,12 @@ static void apk_db_triggers_read(struct apk_database *db, struct apk_bstream *bs
list_add_tail(&ipkg->trigger_pkgs_list,
&db->installed.triggers);
}
apk_istream_close(is);
}
static int apk_db_read_state(struct apk_database *db, int flags)
{
struct apk_istream *is;
struct apk_bstream *bs;
apk_blob_t blob, world;
int r;
......@@ -1153,26 +1161,16 @@ static int apk_db_read_state(struct apk_database *db, int flags)
*/
if (!(flags & APK_OPENF_NO_WORLD)) {
blob = world = apk_blob_from_file(db->root_fd, apk_world_file);
if (APK_BLOB_IS_NULL(blob))
return -ENOENT;
if (APK_BLOB_IS_NULL(blob)) return -ENOENT;
blob = apk_blob_trim(blob);
apk_blob_pull_deps(&blob, db, &db->world);
free(world.ptr);
}
if (!(flags & APK_OPENF_NO_INSTALLED)) {
bs = apk_bstream_from_file(db->root_fd, apk_installed_file);
if (!IS_ERR_OR_NULL(bs)) {
r = apk_db_index_read(db, bs, -1);
apk_bstream_close(bs);
if (r != 0) return -1;
}
bs = apk_bstream_from_file(db->root_fd, apk_triggers_file);
if (!IS_ERR_OR_NULL(bs)) {
apk_db_triggers_read(db, bs);
apk_bstream_close(bs);
}
r = apk_db_index_read(db, apk_istream_from_file(db->root_fd, apk_installed_file), -1);
if (r != 0) return -1;
apk_db_triggers_read(db, apk_istream_from_file(db->root_fd, apk_triggers_file));
}
if (!(flags & APK_OPENF_NO_SCRIPTS)) {
......@@ -1514,7 +1512,6 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
{
const char *msg = NULL;
struct apk_repository_list *repo = NULL;
struct apk_bstream *bs;
struct statfs stfs;
apk_blob_t blob;
int r, fd, write_arch = FALSE;
......@@ -1662,8 +1659,7 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
if (apk_flags & APK_OVERLAY_FROM_STDIN) {
apk_flags &= ~APK_OVERLAY_FROM_STDIN;
apk_db_read_overlay(db, apk_bstream_from_istream(
apk_istream_from_fd(STDIN_FILENO)));
apk_db_read_overlay(db, apk_istream_from_fd(STDIN_FILENO));
}
r = apk_db_read_state(db, dbopts->open_flags);
......@@ -1682,11 +1678,7 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
if (!(dbopts->open_flags & APK_OPENF_NO_INSTALLED_REPO)) {
if (apk_db_cache_active(db)) {
bs = apk_bstream_from_file(db->cache_fd, "installed");
if (!IS_ERR_OR_NULL(bs)) {
apk_db_index_read(db, bs, -2);
apk_bstream_close(bs);
}
apk_db_index_read(db, apk_istream_from_file(db->cache_fd, "installed"), -2);
}
}
......@@ -2165,7 +2157,6 @@ static int load_apkindex(void *sctx, const struct apk_file_info *fi,
struct apk_istream *is)
{
struct apkindex_ctx *ctx = (struct apkindex_ctx *) sctx;
struct apk_bstream *bs;
struct apk_repository *repo;
int r;
......@@ -2179,11 +2170,7 @@ static int load_apkindex(void *sctx, const struct apk_file_info *fi,
repo->description = apk_blob_from_istream(is, fi->size);
} else if (strcmp(fi->name, "APKINDEX") == 0) {
ctx->found = 1;
bs = apk_bstream_from_istream(is);
if (!IS_ERR_OR_NULL(bs)) {
apk_db_index_read(ctx->db, bs, ctx->repo);
apk_bstream_close(bs);
}
apk_db_index_read(ctx->db, is, ctx->repo);
}
return 0;
......@@ -2212,11 +2199,7 @@ static int load_index(struct apk_database *db, struct apk_bstream *bs,
if (r >= 0 && ctx.found == 0)
r = -ENOMSG;
} else {
bs = apk_bstream_from_istream(apk_bstream_gunzip(bs));
if (!IS_ERR_OR_NULL(bs)) {
apk_db_index_read(db, bs, repo);
apk_bstream_close(bs);
}
apk_db_index_read(db, apk_bstream_gunzip(bs), repo);
}
return r;
}
......@@ -2440,9 +2423,9 @@ static int apk_db_install_archive_entry(void *_ctx,
if (ae->name[0] == '.') {
/* APK 2.0 format */
if (strcmp(ae->name, ".PKGINFO") == 0) {
apk_blob_t blob = apk_blob_from_istream(is, ae->size);
apk_blob_for_each_segment(blob, "\n", read_info_line, ctx);
free(blob.ptr);
apk_blob_t l, token = APK_BLOB_STR("\n");
while (!APK_BLOB_IS_NULL(l = apk_istream_get_delim(is, token)))
read_info_line(ctx, l);
return 0;
}
type = apk_script_type(&ae->name[1]);
......
......@@ -35,7 +35,7 @@
#define HAVE_FGETGRENT_R
#endif
size_t apk_io_bufsize = 2*1024;
size_t apk_io_bufsize = 8*1024;
static void apk_file_meta_from_fd(int fd, struct apk_file_meta *meta)
{
......@@ -96,6 +96,157 @@ ssize_t apk_istream_read(struct apk_istream *is, void *ptr, size_t size)
return size - left;
}
static int __apk_istream_fill(struct apk_istream *is)
{
ssize_t sz;
if (is->err) return is->err;
if (is->ptr != is->buf) {
sz = is->end - is->ptr;
memmove(is->buf, is->ptr, sz);
is->ptr = is->buf;
is->end = is->buf + sz;
}
sz = is->ops->read(is, is->end, is->buf + is->buf_size - is->end);
if (sz <= 0) {
is->err = sz ?: 1;
return is->err;
}
is->end += sz;
return 0;
}
apk_blob_t apk_istream_get(struct apk_istream *is, size_t len)
{
apk_blob_t ret = APK_BLOB_NULL;
do {
if (is->end - is->ptr >= len) {
ret = APK_BLOB_PTR_LEN((char*)is->ptr, len);
break;
}
if (is->err>0 || is->end-is->ptr == is->buf_size) {
ret = APK_BLOB_PTR_LEN((char*)is->ptr, is->end - is->ptr);
break;
}
} while (!__apk_istream_fill(is));
if (!APK_BLOB_IS_NULL(ret)) {
is->ptr = (uint8_t*)ret.ptr + ret.len;
return ret;
}
return (struct apk_blob) { .len = is->err < 0 ? is->err : 0 };
}
apk_blob_t apk_istream_get_all(struct apk_istream *is)
{
if (is->ptr == is->end)
__apk_istream_fill(is);
if (is->ptr != is->end) {
apk_blob_t ret = APK_BLOB_PTR_LEN((char*)is->ptr, is->end - is->ptr);
is->ptr = is->end = 0;
return ret;
}
return (struct apk_blob) { .len = is->err < 0 ? is->err : 0 };
}
apk_blob_t apk_istream_get_delim(struct apk_istream *is, apk_blob_t token)
{
apk_blob_t ret = APK_BLOB_NULL, left = APK_BLOB_NULL;
do {
if (apk_blob_split(APK_BLOB_PTR_LEN((char*)is->ptr, is->end - is->ptr), token, &ret, &left))
break;
if (is->end - is->ptr == is->buf_size) {
is->err = -ENOBUFS;
break;
}
} while (!__apk_istream_fill(is));
/* Last segment before end-of-file. Return also zero length non-null
* blob if eof comes immediately after the delimiter. */
if (is->ptr && is->err > 0)
ret = APK_BLOB_PTR_LEN((char*)is->ptr, is->end - is->ptr);
if (!APK_BLOB_IS_NULL(ret)) {
is->ptr = (uint8_t*)left.ptr;
is->end = (uint8_t*)left.ptr + left.len;
return ret;
}
return (struct apk_blob) { .len = is->err < 0 ? is->err : 0 };
}
static void segment_get_meta(struct apk_istream *is, struct apk_file_meta *meta)
{
struct apk_segment_istream *sis = container_of(is, struct apk_segment_istream, is);
*meta = (struct apk_file_meta) {
.atime = sis->mtime,
.mtime = sis->mtime,
};
}
static ssize_t segment_read(struct apk_istream *is, void *ptr, size_t size)
{
struct apk_segment_istream *sis = container_of(is, struct apk_segment_istream, is);
ssize_t r;
if (size > sis->bytes_left) size = sis->bytes_left;
if (size == 0) return 0;
r = sis->pis->ops->read(sis->pis, ptr, size);
if (r <= 0) {
/* If inner stream returned zero (end-of-stream), we
* are getting short read, because tar header indicated
* more was to be expected. */
if (r == 0) r = -ECONNABORTED;
} else {
sis->bytes_left -= r;
}
return r;
}
static void segment_close(struct apk_istream *is)
{
struct apk_segment_istream *sis = container_of(is, struct apk_segment_istream, is);
if (sis->bytes_left) {
apk_istream_read(sis->pis, NULL, sis->bytes_left);
sis->bytes_left = 0;
}
}
static const struct apk_istream_ops segment_istream_ops = {
.get_meta = segment_get_meta,
.read = segment_read,
.close = segment_close,
};
void apk_istream_segment(struct apk_segment_istream *sis, struct apk_istream *is, size_t len, time_t mtime)
{
*sis = (struct apk_segment_istream) {
.is.ops = &segment_istream_ops,
.is.buf = is->buf,
.is.buf_size = is->buf_size,
.is.ptr = is->ptr,
.is.end = is->end,
.pis = is,
.bytes_left = len,
.mtime = mtime,
};
if (sis->is.end - sis->is.ptr > len) {
sis->is.end = sis->is.ptr + len;
is->ptr += len;
} else {
is->ptr = is->end = 0;
}
sis->bytes_left -= sis->is.end - sis->is.ptr;
}
struct apk_fd_istream {
struct apk_istream is;
int fd;
......@@ -630,7 +781,6 @@ int apk_fileinfo_get(int atfd, const char *filename, unsigned int flags,
struct apk_file_info *fi)
{
struct stat64 st;
struct apk_bstream *bs;
unsigned int checksum = flags & 0xff;
unsigned int xattr_checksum = (flags >> 8) & 0xff;