bstream: make tokenizable and load index using bstream

some fixes on index reading code too.

src/apk_blob.h

@@ -22,7 +22,7 @@ struct apk_blob {
 typedef struct apk_blob apk_blob_t;
 typedef int (*apk_blob_cb)(void *ctx, apk_blob_t blob);
 
-#define APK_BLOB_IS_NULL(blob)		(blob.ptr == NULL)
+#define APK_BLOB_IS_NULL(blob)		((blob).ptr == NULL)
 
 #define APK_BLOB_NULL			((apk_blob_t){0, NULL})
 #define APK_BLOB_BUF(buf)		((apk_blob_t){sizeof(buf), (char *)(buf)})
@@ -39,11 +39,12 @@ static inline apk_blob_t APK_BLOB_STR(const char *str)
 char *apk_blob_cstr(apk_blob_t str);
 int apk_blob_spn(apk_blob_t blob, const char *accept, apk_blob_t *l, apk_blob_t *r);
 int apk_blob_cspn(apk_blob_t blob, const char *reject, apk_blob_t *l, apk_blob_t *r);
-int apk_blob_splitstr(apk_blob_t blob, const char *split, apk_blob_t *l, apk_blob_t *r);
+int apk_blob_split(apk_blob_t blob, apk_blob_t split, apk_blob_t *l, apk_blob_t *r);
 int apk_blob_rsplit(apk_blob_t blob, char split, apk_blob_t *l, apk_blob_t *r);
-unsigned apk_blob_uint(apk_blob_t blob, int base);
 unsigned long apk_blob_hash(apk_blob_t str);
 int apk_blob_compare(apk_blob_t a, apk_blob_t b);
+unsigned int apk_blob_parse_uint(apk_blob_t *b, int radix);
+int apk_blob_parse_char(apk_blob_t *b);
 
 int apk_blob_for_each_segment(apk_blob_t blob, const char *split,
 			      apk_blob_cb cb, void *ctx);

src/apk_database.h

@@ -135,7 +135,7 @@ struct apk_package *apk_db_pkg_add(struct apk_database *db, struct apk_package *
 struct apk_package *apk_db_get_pkg(struct apk_database *db, csum_t sum);
 struct apk_package *apk_db_get_file_owner(struct apk_database *db, apk_blob_t filename);
 
-int apk_db_index_read(struct apk_database *db, struct apk_istream *is, int repo);
+int apk_db_index_read(struct apk_database *db, struct apk_bstream *bs, int repo);
 int apk_db_index_write(struct apk_database *db, struct apk_ostream *os);
 
 int apk_db_add_repository(apk_database_t db, apk_blob_t repository);

src/apk_io.h

@@ -35,7 +35,7 @@ struct apk_istream {
 };
 
 struct apk_bstream {
-	size_t (*read)(void *stream, void **ptr);
+	apk_blob_t (*read)(void *stream, apk_blob_t token);
 	void (*close)(void *stream, size_t *size);
 };
 

src/archive.c

@@ -44,7 +44,12 @@ struct tar_header {
 	char padding[12];   /* 500-512 */
 };
 
-#define GET_OCTAL(s) apk_blob_uint(APK_BLOB_PTR_LEN(s, sizeof(s)), 8)
+#define GET_OCTAL(s) get_octal(s, sizeof(s))
+static int get_octal(char *s, size_t l)
+{
+	apk_blob_t b = APK_BLOB_PTR_LEN(s, l);
+	return apk_blob_parse_uint(&b, 8);
+}
 
 struct apk_tar_entry_istream {
 	struct apk_istream is;

src/blob.c

@@ -75,26 +75,25 @@ int apk_blob_rsplit(apk_blob_t blob, char split, apk_blob_t *l, apk_blob_t *r)
 	return 1;
 }
 
-int apk_blob_splitstr(apk_blob_t blob, const char *split, apk_blob_t *l, apk_blob_t *r)
+int apk_blob_split(apk_blob_t blob, apk_blob_t split, apk_blob_t *l, apk_blob_t *r)
 {
-	int splitlen = strlen(split);
-	char *pos = blob.ptr, *end = blob.ptr + blob.len - splitlen + 1;
+	char *pos = blob.ptr, *end = blob.ptr + blob.len - split.len + 1;
 
 	if (end < pos)
 		return 0;
 
 	while (1) {
-		pos = memchr(pos, split[0], end - pos);
+		pos = memchr(pos, split.ptr[0], end - pos);
 		if (pos == NULL)
 			return 0;
 
-		if (memcmp(pos, split, splitlen) != 0) {
+		if (split.len > 1 && memcmp(pos, split.ptr, split.len) != 0) {
 			pos++;
 			continue;
 		}
 
 		*l = APK_BLOB_PTR_PTR(blob.ptr, pos-1);
-		*r = APK_BLOB_PTR_PTR(pos+splitlen, blob.ptr+blob.len-1);
+		*r = APK_BLOB_PTR_PTR(pos+split.len, blob.ptr+blob.len-1);
 		return 1;
 	}
 }
@@ -122,11 +121,11 @@ int apk_blob_compare(apk_blob_t a, apk_blob_t b)
 int apk_blob_for_each_segment(apk_blob_t blob, const char *split,
 			      int (*cb)(void *ctx, apk_blob_t blob), void *ctx)
 {
-	apk_blob_t l, r;
+	apk_blob_t l, r, s = APK_BLOB_STR(split);
 	int rc;
 
 	r = blob;
-	while (apk_blob_splitstr(r, split, &l, &r)) {
+	while (apk_blob_split(r, s, &l, &r)) {
 		rc = cb(ctx, l);
 		if (rc != 0)
 			return rc;
@@ -136,7 +135,7 @@ int apk_blob_for_each_segment(apk_blob_t blob, const char *split,
 	return 0;
 }
 
-static int dx(int c)
+static inline int dx(int c)
 {
 	if (c >= '0' && c <= '9')
 		return c - '0';
@@ -147,24 +146,39 @@ static int dx(int c)
 	return -1;
 }
 
-unsigned apk_blob_uint(apk_blob_t blob, int base)
+unsigned int apk_blob_parse_uint(apk_blob_t *blob, int base)
 {
-	unsigned val;
-	int i, ch;
+	unsigned int val;
+	int ch;
 
 	val = 0;
-	for (i = 0; i < blob.len; i++) {
-		if (blob.ptr[i] == 0)
-			break;
-		ch = dx(blob.ptr[i]);
+	while (blob->len && blob->ptr[0] != 0) {
+		ch = dx(blob->ptr[0]);
 		if (ch < 0 || ch >= base)
-			return 0;
+			break;
 		val *= base;
 		val += ch;
+
+		blob->ptr++;
+		blob->len--;
 	}
+
 	return val;
 }
 
+int apk_blob_parse_char(apk_blob_t *blob)
+{
+	int r;
+
+	if (blob->len == 0 || blob->ptr == NULL)
+		return -1;
+	r = blob->ptr[0];
+	blob->ptr++;
+	blob->len--;
+
+	return r;
+}
+
 int apk_hexdump_parse(apk_blob_t to, apk_blob_t from)
 {
 	int i;

src/database.c

@@ -205,7 +205,8 @@ static struct apk_db_dir *apk_db_dir_get(struct apk_database *db,
 		return apk_db_dir_ref(dir);
 
 	db->installed.stats.dirs++;
-	dir = calloc(1, sizeof(*dir) + name.len + 1);
+	dir = malloc(sizeof(*dir) + name.len + 1);
+	memset(dir, 0, sizeof(*dir));
 	dir->refs = 1;
 	memcpy(dir->dirname, name.ptr, name.len);
 	dir->dirname[name.len] = 0;
@@ -311,7 +312,8 @@ static struct apk_db_file *apk_db_file_get(struct apk_database *db,
 	if (file != NULL)
 		return file;
 
-	file = calloc(1, sizeof(*file) + name.len + 1);
+	file = malloc(sizeof(*file) + name.len + 1);
+	memset(file, 0, sizeof(*file));
 	memcpy(file->filename, name.ptr, name.len);
 	file->filename[name.len] = 0;
 
@@ -376,116 +378,107 @@ struct apk_package *apk_db_pkg_add(struct apk_database *db, struct apk_package *
 	return idb;
 }
 
-int apk_db_index_read(struct apk_database *db, struct apk_istream *is, int repo)
+int apk_db_index_read(struct apk_database *db, struct apk_bstream *bs, int repo)
 {
 	struct apk_package *pkg = NULL;
 	struct apk_db_dir_instance *diri = NULL;
 	struct apk_db_file *file = NULL;
 	struct hlist_node **diri_node = NULL;
 	struct hlist_node **file_diri_node = NULL;
+	apk_blob_t token = APK_BLOB_STR("\n"), l;
+	int field;
 
-	char buf[1024];
-	apk_blob_t l, r;
-	int n, field;
-
-	r = APK_BLOB_PTR_LEN(buf, 0);
-	while (1) {
-		n = is->read(is, &r.ptr[r.len], sizeof(buf) - r.len);
-		if (n <= 0)
-			break;
-		r.len += n;
-
-		while (apk_blob_splitstr(r, "\n", &l, &r)) {
-			if (l.len < 2 || l.ptr[1] != ':') {
-				if (pkg == NULL)
-					continue;
+	while (!APK_BLOB_IS_NULL(l = bs->read(bs, token))) {
+		if (l.len < 2 || l.ptr[1] != ':') {
+			if (pkg == NULL)
+				continue;
 
-				if (repo != -1)
-					pkg->repos |= BIT(repo);
-				else
-					apk_pkg_set_state(db, pkg, APK_PKG_INSTALLED);
+			if (repo != -1)
+				pkg->repos |= BIT(repo);
+			else
+				apk_pkg_set_state(db, pkg, APK_PKG_INSTALLED);
 
-				if (apk_db_pkg_add(db, pkg) != pkg && repo == -1) {
-					apk_error("Installed database load failed");
-					return -1;
-				}
-				pkg = NULL;
-				continue;
+			if (apk_db_pkg_add(db, pkg) != pkg && repo == -1) {
+				apk_error("Installed database load failed");
+				return -1;
 			}
+			pkg = NULL;
+			continue;
+		}
 
-			/* Get field */
-			field = l.ptr[0];
-			l.ptr += 2;
-			l.len -= 2;
-
-			/* If no package, create new */
-			if (pkg == NULL) {
-				pkg = apk_pkg_new();
-				diri = NULL;
-				diri_node = hlist_tail_ptr(&pkg->owned_dirs);
-				file_diri_node = NULL;
-			}
+		/* Get field */
+		field = l.ptr[0];
+		l.ptr += 2;
+		l.len -= 2;
+
+		/* If no package, create new */
+		if (pkg == NULL) {
+			pkg = apk_pkg_new();
+			diri = NULL;
+			diri_node = hlist_tail_ptr(&pkg->owned_dirs);
+			file_diri_node = NULL;
+		}
 
-			/* Standard index line? */
-			if (apk_pkg_add_info(db, pkg, field, l) == 0)
-				continue;
+		/* Standard index line? */
+		if (apk_pkg_add_info(db, pkg, field, l) == 0)
+			continue;
 
-			if (repo != -1) {
-				apk_error("Invalid index entry '%c'", field);
+		if (repo != -1) {
+			apk_error("Invalid index entry '%c'", field);
+			return -1;
+		}
+
+		/* Check FDB special entries */
+		switch (field) {
+		case 'F':
+			if (pkg->name == NULL) {
+				apk_error("FDB directory entry before package entry");
 				return -1;
 			}
-
-			/* Check FDB special entries */
-			switch (field) {
-			case 'F':
-				if (pkg->name == NULL) {
-					apk_error("FDB directory entry before package entry");
-					return -1;
-				}
-				diri = apk_db_diri_new(db, pkg, l, &diri_node);
-				file_diri_node = &diri->owned_files.first;
-				break;
-			case 'M':
-				if (diri == NULL) {
-					apk_error("FDB directory metadata entry before directory entry");
-					return -1;
-				}
-				/* FIXME: sscanf may touch unallocated area */
-				if (sscanf(l.ptr, "%d:%d:%o",
-					   &diri->uid, &diri->gid, &diri->mode) != 3) {
-					apk_error("FDB bad directory mode entry");
-					return -1;
-				}
-				break;
-			case 'R':
-				if (diri == NULL) {
-					apk_error("FDB file entry before directory entry");
-					return -1;
-				}
-				file = apk_db_file_get(db, diri, l,
-						       &file_diri_node);
-				break;
-			case 'Z':
-				if (file == NULL) {
-					apk_error("FDB checksum entry before file entry");
-					return -1;
-				}
-				if (apk_hexdump_parse(APK_BLOB_BUF(file->csum), l)) {
-					apk_error("Not a valid checksum");
-					return -1;
-				}
-				break;
-			default:
-				apk_error("FDB entry '%c' unsupported", n);
+			diri = apk_db_diri_new(db, pkg, l, &diri_node);
+			file_diri_node = &diri->owned_files.first;
+			break;
+		case 'M':
+			if (diri == NULL) {
+				apk_error("FDB directory metadata entry before directory entry");
+				return -1;
+			}
+			diri->uid = apk_blob_parse_uint(&l, 10);
+			if (apk_blob_parse_char(&l) != ':')
+				goto bad_mode;
+			diri->gid = apk_blob_parse_uint(&l, 10);
+			if (apk_blob_parse_char(&l) != ':')
+				goto bad_mode;
+			diri->mode = apk_blob_parse_uint(&l, 8);
+			break;
+		case 'R':
+			if (diri == NULL) {
+				apk_error("FDB file entry before directory entry");
 				return -1;
 			}
+			file = apk_db_file_get(db, diri, l,
+					       &file_diri_node);
+			break;
+		case 'Z':
+			if (file == NULL) {
+				apk_error("FDB checksum entry before file entry");
+				return -1;
+			}
+			if (apk_hexdump_parse(APK_BLOB_BUF(file->csum), l)) {
+				apk_error("Not a valid checksum");
+				return -1;
+			}
+			break;
+		default:
+			apk_error("FDB entry '%c' unsupported", field);
+			return -1;
 		}
-
-		memcpy(&buf[0], r.ptr, r.len);
-		r = APK_BLOB_PTR_LEN(buf, r.len);
 	}
 
 	return 0;
+bad_mode:
+	apk_error("FDB bad directory mode entry");
+	return -1;
 }
 
 static int apk_db_write_fdb(struct apk_database *db, struct apk_ostream *os)
@@ -581,6 +574,7 @@ static int apk_db_scriptdb_read(struct apk_database *db, struct apk_istream *is)
 static int apk_db_read_state(struct apk_database *db, int flags)
 {
 	struct apk_istream *is;
+	struct apk_bstream *bs;
 	apk_blob_t blob;
 	int i;
 
@@ -606,10 +600,10 @@ static int apk_db_read_state(struct apk_database *db, int flags)
 	}
 
 	if (!(flags & APK_OPENF_NO_INSTALLED)) {
-		is = apk_istream_from_file("var/lib/apk/installed");
-		if (is != NULL) {
-			apk_db_index_read(db, is, -1);
-			is->close(is);
+		bs = apk_bstream_from_file("var/lib/apk/installed");
+		if (bs != NULL) {
+			apk_db_index_read(db, bs, -1);
+			bs->close(bs, NULL);
 		}
 	}
 
@@ -1013,7 +1007,6 @@ int apk_repository_update(struct apk_database *db, struct apk_repository *repo)
 int apk_db_add_repository(apk_database_t _db, apk_blob_t repository)
 {
 	struct apk_database *db = _db.db;
-	struct apk_istream *is = NULL;
 	struct apk_bstream *bs = NULL;
 	struct apk_repository *repo;
 	int r, n = 1;
@@ -1050,13 +1043,13 @@ int apk_db_add_repository(apk_database_t _db, apk_blob_t repository)
 	} else {
 		bs = apk_repository_file_open(repo, apk_index_gz);
 	}
-	is = apk_bstream_gunzip(bs, 1);
-	if (is == NULL) {
+	bs = apk_bstream_from_istream(apk_bstream_gunzip(bs, TRUE));
+	if (bs == NULL) {
 		apk_warning("Failed to open index for %s", repo->url);
 		return -1;
 	}
-	apk_db_index_read(db, is, r);
-	is->close(is);
+	apk_db_index_read(db, bs, r);
+	bs->close(bs, NULL);
 
 	return 0;
 }
@@ -1276,6 +1269,7 @@ static int apk_db_gzip_part(void *pctx, EVP_MD_CTX *mdctx, int part)
 {
 	struct install_ctx *ctx = (struct install_ctx *) pctx;
 
+	fprintf(stderr, "part %d\n", part);
 	switch (part) {
 	case APK_MPART_BEGIN:
 		EVP_DigestInit_ex(mdctx, EVP_md5(), NULL);

src/gunzip.c

@@ -49,12 +49,16 @@ static size_t gz_read(void *stream, void *ptr, size_t size)
 
 	while (gis->zs.avail_out != 0 && gis->z_err == Z_OK) {
 		if (gis->zs.avail_in == 0) {
+			apk_blob_t blob;
+
 			if (gis->cb != NULL && gis->mdblock != NULL) {
 				/* Digest the inflated bytes */
 				EVP_DigestUpdate(&gis->mdctx, gis->mdblock,
 						 (void *)gis->zs.next_in - gis->mdblock);
 			}
-			gis->zs.avail_in = gis->bs->read(gis->bs, &gis->mdblock);
+			blob = gis->bs->read(gis->bs, APK_BLOB_NULL);
+			gis->mdblock = blob.ptr;
+			gis->zs.avail_in = blob.len;
 			gis->zs.next_in = (void *) gis->mdblock;
 			if (gis->zs.avail_in < 0) {
 				gis->z_err = Z_DATA_ERROR;

src/index.c

@@ -41,15 +41,15 @@ static int index_parse(void *ctx, int optch, int optindex, const char *optarg)
 
 static int index_read_file(struct apk_database *db, struct index_ctx *ictx)
 {
-	struct apk_istream *is;
+	struct apk_bstream *bs;
 	int r;
 	if (ictx->index_file == NULL)
 		return 0;
-	is = apk_bstream_gunzip(apk_bstream_from_url(ictx->index_file), 1);
-	if (is == NULL)
+	bs = apk_bstream_from_istream(apk_bstream_gunzip(apk_bstream_from_url(ictx->index_file), 1));
+	if (bs == NULL)
 		return -1;
-	r = apk_db_index_read(db, is, -1);
-	is->close(is);
+	r = apk_db_index_read(db, bs, -1);
+	bs->close(bs, NULL);
 	return r;
 }
 

src/io.c

@@ -150,24 +150,60 @@ err:
 struct apk_istream_bstream {
 	struct apk_bstream bs;
 	struct apk_istream *is;
-	unsigned char buffer[8*1024];
+	apk_blob_t left;
+	char buffer[8*1024];
 	size_t size;
 };
 
-static size_t is_bs_read(void *stream, void **ptr)
+static apk_blob_t is_bs_read(void *stream, apk_blob_t token)
 {
 	struct apk_istream_bstream *isbs =
 		container_of(stream, struct apk_istream_bstream, bs);
-	size_t size;
+	ssize_t size;
+	apk_blob_t ret;
+
+	/* If we have cached stuff, first check if it full fills the request */
+	if (isbs->left.len != 0) {
+		if (!APK_BLOB_IS_NULL(token)) {
+			/* If we have tokenized thingy left, return it */
+			if (apk_blob_split(isbs->left, token, &ret, &isbs->left))
+				goto ret;
+		} else
+			goto ret_all;
+	}
 
-	size = isbs->is->read(isbs->is, isbs->buffer, sizeof(isbs->buffer));
-	if (size <= 0)
-		return size;
+	/* If we've exchausted earlier, it's end of stream */
+	if (APK_BLOB_IS_NULL(isbs->left))
+		return APK_BLOB_NULL;
 
-	isbs->size += size;
+	/* We need more data */
+	if (isbs->left.len != 0)
+		memcpy(isbs->buffer, isbs->left.ptr, isbs->left.len);
+	isbs->left.ptr = isbs->buffer;
+	size = isbs->is->read(isbs->is, isbs->buffer + isbs->left.len,
+			      sizeof(isbs->buffer) - isbs->left.len);
+	if (size > 0) {
+		isbs->size += size;
+		isbs->left.len += size;
+	} else if (size == 0) {
+		if (isbs->left.len == 0)
+			isbs->left = APK_BLOB_NULL;
+		goto ret_all;
+	}
 
-	*ptr = isbs->buffer;
-	return size;
+	if (!APK_BLOB_IS_NULL(token)) {
+		/* If we have tokenized thingy left, return it */
+		if (apk_blob_split(isbs->left, token, &ret, &isbs->left))
+			goto ret;
+		/* No token found; just return the full buffer */
+	}
+
+ret_all:
+	/* Return all that is in cache */
+	ret = isbs->left;
+	isbs->left.len = 0;
+ret:
+	return ret;
 }
 
 static void is_bs_close(void *stream, size_t *size)
@@ -195,6 +231,8 @@ struct apk_bstream *apk_bstream_from_istream(struct apk_istream *istream)
 		.close = is_bs_close,
 	};
 	isbs->is = istream;
+	isbs->left = APK_BLOB_PTR_LEN(isbs->buffer, 0),
+	isbs->size = 0;
 
 	return &isbs->bs;
 }
@@ -204,23 +242,24 @@ struct apk_mmap_bstream {
 	int fd;
 	size_t size;
 	unsigned char *ptr;
-	size_t pos;
+	apk_blob_t left;
 };
 
-static size_t mmap_read(void *stream, void **ptr)
+static apk_blob_t mmap_read(void *stream, apk_blob_t token)
 {
 	struct apk_mmap_bstream *mbs =
 		container_of(stream, struct apk_mmap_bstream, bs);
-	size_t size;
+	apk_blob_t ret;
 
-	size = mbs->size - mbs->pos;
-	if (size > 1024*1024)
-		size = 1024*1024;
+	if (!APK_BLOB_IS_NULL(token) && !APK_BLOB_IS_NULL(mbs->left)) {
+		if (apk_blob_split(mbs->left, token, &ret, &mbs->left))
+			return ret;
+	}
 
-	*ptr = (void *) &mbs->ptr[mbs->pos];
-	mbs->pos += size;
+	ret = mbs->left;
+	mbs->left = APK_BLOB_NULL;
 
-	return size;
+	return ret;
 }
 
 static void mmap_close(void *stream, size_t *size)
@@ -245,7 +284,7 @@ static struct apk_bstream *apk_mmap_bstream_from_fd(int fd)
 	if (fstat(fd, &st) < 0)
 		return NULL;
 
-	ptr = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+	ptr = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
 	if (ptr == MAP_FAILED)
 		return NULL;
 
@@ -262,7 +301,7 @@ static struct apk_bstream *apk_mmap_bstream_from_fd(int fd)
 	mbs->fd = fd;
 	mbs->size = st.st_size;
 	mbs->ptr = ptr;
-	mbs->pos = 0;
+	mbs->left = APK_BLOB_PTR_LEN(ptr, mbs->size);
 
 	return &mbs->bs;
 }
@@ -301,17 +340,17 @@ struct apk_tee_bstream {
 	size_t size;
 };
 
-static size_t tee_read(void *stream, void **ptr)
+static apk_blob_t tee_read(void *stream, apk_blob_t token)
 {
 	struct apk_tee_bstream *tbs =
 		container_of(stream, struct apk_tee_bstream, bs);
-	ssize_t size;
+	apk_blob_t blob;
 
-	size = tbs->inner_bs->read(tbs->inner_bs