db: unify handling of special packages

make cache a special kind of repository, and automatically cache
special packages (virtual packages, or ones installed from command
line). add test cases for handling virtual packages. fixes #1617.

src/add.c

@@ -93,7 +93,6 @@ static int add_main(void *ctx, struct apk_database *db, int argc, char **argv)
 		virtpkg->version = apk_blob_atomize(APK_BLOB_STR("0"));
 		virtpkg->description = strdup("virtual meta package");
 		virtpkg->arch = apk_blob_atomize(APK_BLOB_STR("noarch"));
-		virtpkg->repos |= db->repo_tags[virtdep.repository_tag].allowed_repos;
 		virtpkg = apk_db_pkg_add(db, virtpkg);
 	}
 

src/apk.c

@@ -476,7 +476,7 @@ int main(int argc, char **argv)
 	for (i = 0; i < test_repos->num; i++) {
 		struct apk_bstream *bs;
 		apk_blob_t spec = APK_BLOB_STR(test_repos->item[i]), name, tag;
-		int repo_tag = 0;
+		int repo_tag = 0, repo = APK_REPOSITORY_FIRST_CONFIGURED + i;
 
 		if (apk_blob_split(spec, APK_BLOB_STR(":"), &tag, &name)) {
 			repo_tag = apk_db_get_tag_id(&db, tag);
@@ -486,9 +486,11 @@ int main(int argc, char **argv)
 
 		bs = apk_bstream_from_file(AT_FDCWD, name.ptr);
 		if (bs != NULL) {
-			apk_db_index_read(&db, bs, i);
-			db.repo_tags[repo_tag].allowed_repos |= BIT(i);
+			apk_db_index_read(&db, bs, repo);
 			bs->close(bs, NULL);
+			if (!(apk_flags & APK_NO_NETWORK))
+				db.available_repos |= BIT(repo);
+			db.repo_tags[repo_tag].allowed_repos |= BIT(repo);
 		}
 	}
 #endif

src/apk_database.h

@@ -101,7 +101,6 @@ struct apk_name {
 struct apk_repository {
 	char *url;
 	struct apk_checksum csum;
-
 	apk_blob_t description;
 };
 
@@ -120,6 +119,9 @@ struct apk_db_options {
 	struct list_head repository_list;
 };
 
+#define APK_REPOSITORY_CACHED		0
+#define APK_REPOSITORY_FIRST_CONFIGURED	1
+
 #define APK_DEFAULT_REPOSITORY_TAG	0
 #define APK_DEFAULT_PINNING_MASK	BIT(APK_DEFAULT_REPOSITORY_TAG)
 
@@ -135,7 +137,7 @@ struct apk_database {
 	const char *cache_dir;
 	char *cache_remount_dir;
 	apk_blob_t *arch;
-	unsigned int local_repos, bad_repos;
+	unsigned int local_repos, available_repos;
 	int performing_self_update : 1;
 	int permanent : 1;
 	int compat_newfeatures : 1;
@@ -217,8 +219,6 @@ int apk_db_index_write(struct apk_database *db, struct apk_ostream *os);
 int apk_db_add_repository(apk_database_t db, apk_blob_t repository);
 struct apk_repository *apk_db_select_repo(struct apk_database *db,
 					  struct apk_package *pkg);
-int apk_repository_update(struct apk_database *db, struct apk_repository *repo);
-int apk_repo_is_remote(struct apk_repository *repo);
 int apk_repo_format_filename(char *buf, size_t len,
 			     const char *repourl, apk_blob_t *arch,
 			     const char *pkgfile);

src/apk_defines.h

@@ -80,7 +80,7 @@ extern char **apk_argv;
 #define APK_DEFAULT_ARCH        "noarch"
 #endif
 
-#define APK_MAX_REPOS		31	/* see struct apk_package */
+#define APK_MAX_REPOS		32	/* see struct apk_package */
 #define APK_MAX_TAGS		16	/* see solver; unsigned short */
 #define APK_CACHE_CSUM_BYTES	4
 

src/apk_package.h

@@ -99,7 +99,6 @@ struct apk_package {
 	size_t installed_size, size;
 	time_t build_time;
 	unsigned int topology_hard;
-	unsigned in_cache : 1;
 	unsigned repos : APK_MAX_REPOS;
 	struct apk_checksum csum;
 };

src/cache.c

@@ -43,11 +43,11 @@ static int cache_download(struct apk_database *db)
 		change = &changeset.changes->item[i];
 		pkg = change->newpkg;
 
-		if (pkg->in_cache)
+		if (pkg->repos & db->local_repos)
 			continue;
 
 		repo = apk_db_select_repo(db, pkg);
-		if (repo == NULL || !apk_repo_is_remote(repo))
+		if (repo == NULL)
 			continue;
 
 		apk_pkg_format_cache(pkg, APK_BLOB_BUF(cacheitem));

src/database.c

@@ -531,6 +531,11 @@ struct apk_package *apk_db_pkg_add(struct apk_database *db, struct apk_package *
 	if (pkg->license == NULL)
 		pkg->license = apk_blob_atomize(APK_BLOB_NULL);
 
+	/* Set as "cached" if installing from specified file, and
+	 * for virtual packages */
+	if (pkg->filename != NULL || pkg->installed_size == 0)
+		pkg->repos |= BIT(APK_REPOSITORY_CACHED);
+
 	idb = apk_hash_get(&db->available.packages, APK_BLOB_CSUM(pkg->csum));
 	if (idb == NULL) {
 		idb = pkg;
@@ -1124,7 +1129,7 @@ static int apk_db_index_write_nr_cache(struct apk_database *db)
 	ctx.os = os;
 	list_for_each_entry(ipkg, &db->installed.packages, installed_pkgs_list) {
 		struct apk_package *pkg = ipkg->pkg;
-		if (pkg->repos != 0 || !pkg->in_cache)
+		if (pkg->repos != BIT(APK_REPOSITORY_CACHED))
 			continue;
 		r = write_index_entry(pkg, &ctx);
 		if (r != 0)
@@ -1326,7 +1331,7 @@ static void mark_in_cache(struct apk_database *db, int dirfd, const char *name,
 	if (pkg == NULL)
 		return;
 
-	pkg->in_cache = 1;
+	pkg->repos |= BIT(APK_REPOSITORY_CACHED);
 }
 
 static int add_repos_from_file(void *ctx, int dirfd, const char *file)
@@ -1350,6 +1355,27 @@ static int add_repos_from_file(void *ctx, int dirfd, const char *file)
 	return 0;
 }
 
+static void apk_db_setup_repositories(struct apk_database *db)
+{
+	int repo_tag;
+
+	db->repos[APK_REPOSITORY_CACHED] = (struct apk_repository) {
+		.url = "cache",
+		.csum.data = {
+			0xb0,0x35,0x92,0x80,0x6e,0xfa,0xbf,0xee,0xb7,0x09,
+			0xf5,0xa7,0x0a,0x7c,0x17,0x26,0x69,0xb0,0x05,0x38 },
+		.csum.type = APK_CHECKSUM_SHA1,
+	};
+
+	db->num_repos = APK_REPOSITORY_FIRST_CONFIGURED;
+	db->local_repos |= BIT(APK_REPOSITORY_CACHED);
+	db->available_repos |= BIT(APK_REPOSITORY_CACHED);
+
+	/* Get first repository tag (the NULL tag) */
+	repo_tag = apk_db_get_tag_id(db, APK_BLOB_NULL);
+	db->repo_tags[repo_tag].allowed_repos |= BIT(APK_REPOSITORY_CACHED);
+}
+
 int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
 {
 	const char *msg = NULL;
@@ -1381,8 +1407,7 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
 	apk_protected_path_array_init(&db->protected_paths);
 	db->permanent = 1;
 
-	/* Get first repository tag (the NULL tag) */
-	apk_db_get_tag_id(db, APK_BLOB_NULL);
+	apk_db_setup_repositories(db);
 
 	db->root = strdup(dbopts->root ?: "/");
 	db->root_fd = openat(AT_FDCWD, db->root, O_RDONLY | O_CLOEXEC);
@@ -1652,7 +1677,7 @@ void apk_db_close(struct apk_database *db)
 		}
 	}
 
-	for (i = 0; i < db->num_repos; i++) {
+	for (i = APK_REPOSITORY_FIRST_CONFIGURED; i < db->num_repos; i++) {
 		free(db->repos[i].url);
 		free(db->repos[i].description.ptr);
 	}
@@ -1879,11 +1904,6 @@ int apk_repo_format_filename(char *buf, size_t len,
 	return n;
 }
 
-int apk_repo_is_remote(struct apk_repository *repo)
-{
-	return repo->csum.type != APK_CHECKSUM_NONE;
-}
-
 static struct apk_bstream *apk_repo_file_open(struct apk_repository *repo,
 					      apk_blob_t *arch,
 					      const char *file,
@@ -1893,9 +1913,6 @@ static struct apk_bstream *apk_repo_file_open(struct apk_repository *repo,
 
 	apk_repo_format_filename(buf, buflen, url, arch, file);
 
-	if ((apk_flags & APK_NO_NETWORK) && apk_repo_is_remote(repo))
-		return NULL;
-
 	/* We should not get called for non-repository files */
 	if (strcmp(repo->url, "cache") == 0)
 		return NULL;
@@ -1906,49 +1923,30 @@ static struct apk_bstream *apk_repo_file_open(struct apk_repository *repo,
 struct apk_repository *apk_db_select_repo(struct apk_database *db,
 					  struct apk_package *pkg)
 {
-	static struct apk_repository cache_repo = {
-		.url = "cache",
-		.csum.data = {
-			0xb0,0x35,0x92,0x80,0x6e,0xfa,0xbf,0xee,0xb7,0x09,
-			0xf5,0xa7,0x0a,0x7c,0x17,0x26,0x69,0xb0,0x05,0x38 },
-		.csum.type = APK_CHECKSUM_SHA1,
-	};
-	unsigned int repos = pkg->repos & ~(db->bad_repos);
+	unsigned int repos;
 	int i;
 
-	/* Uninstalled and unavailable? */
-	if (repos == 0 && pkg->ipkg == NULL)
+	/* Select repositories to use */
+	repos = pkg->repos & db->available_repos;
+	if (repos == 0)
 		return NULL;
 
-	if ((repos & db->local_repos) == 0) {
-		/* Network repository (or installed and unavailable) */
-		if (apk_flags & APK_NO_NETWORK) {
-			if (pkg->in_cache)
-				return &cache_repo;
-			return NULL;
-		}
-	} else {
-		/* Local repositories; don't use network */
+	if (repos & db->local_repos)
 		repos &= db->local_repos;
-	}
 
 	/* Pick first repository providing this package */
-	for (i = 0; i < APK_MAX_REPOS; i++) {
+	for (i = APK_REPOSITORY_FIRST_CONFIGURED; i < APK_MAX_REPOS; i++) {
 		if (repos & BIT(i))
 			return &db->repos[i];
 	}
-
-	return NULL;
+	return &db->repos[APK_REPOSITORY_CACHED];
 }
 
-int apk_repository_update(struct apk_database *db, struct apk_repository *repo)
+static int apk_repository_update(struct apk_database *db, struct apk_repository *repo)
 {
 	char cacheitem[PATH_MAX];
 	int r;
 
-	if (!apk_repo_is_remote(repo))
-		return 0;
-
 	apk_cache_format_index(APK_BLOB_BUF(cacheitem), repo);
 	r = apk_cache_download(db, repo->url, db->arch, apkindex_tar_gz, cacheitem,
 			       (apk_flags & APK_ALLOW_UNTRUSTED) ?
@@ -2055,7 +2053,7 @@ int apk_db_add_repository(apk_database_t _db, apk_blob_t _repository)
 		repo = &db->repos[repo_num];
 		if (strcmp(url, repo->url) == 0) {
 			db->repo_tags[tag_id].allowed_repos |=
-				BIT(repo_num) & ~db->bad_repos;
+				BIT(repo_num) & db->available_repos;
 			free(url);
 			return 0;
 		}
@@ -2071,29 +2069,32 @@ int apk_db_add_repository(apk_database_t _db, apk_blob_t _repository)
 		.url = url,
 	};
 
-	if (apk_url_local_file(repo->url) == NULL) {
-		apk_blob_checksum(brepo, apk_checksum_default(), &repo->csum);
-
-		if (apk_flags & APK_UPDATE_CACHE)
-			apk_repository_update(db, repo);
+	apk_blob_checksum(brepo, apk_checksum_default(), &repo->csum);
 
+	if (apk_url_local_file(repo->url) == NULL) {
+		if (!(apk_flags & APK_NO_NETWORK)) {
+			db->available_repos |= BIT(repo_num);
+			if (apk_flags & APK_UPDATE_CACHE)
+				apk_repository_update(db, repo);
+		}
 		apk_cache_format_index(APK_BLOB_BUF(buf), repo);
 		bs = apk_bstream_from_file(db->cache_fd, buf);
 	} else {
 		db->local_repos |= BIT(repo_num);
+		db->available_repos |= BIT(repo_num);
 		bs = apk_repo_file_open(repo, db->arch, apkindex_tar_gz, buf, sizeof(buf));
 	}
-	if (bs != NULL) {
+	if (bs != NULL)
 		r = load_index(db, bs, targz, repo_num);
-	} else
+	else
 		r = -ENOENT;
 
 	if (r != 0) {
 		apk_warning("Ignoring %s: %s", buf, apk_error_str(r));
-		db->bad_repos |= BIT(repo_num);
+		db->available_repos &= ~BIT(repo_num);
 		r = 0;
 	} else {
-		db->repo_tags[tag_id].allowed_repos |= BIT(repo_num);
+		db->repo_tags[tag_id].allowed_repos |= BIT(repo_num) & db->available_repos;
 	}
 
 	return 0;
@@ -2517,16 +2518,14 @@ static int apk_db_unpack_pkg(struct apk_database *db,
 			return -1;
 		}
 
-		if (apk_db_cache_active(db) && apk_repo_is_remote(repo)) {
+		if (strcmp(repo->url, "cache") == 0) {
 			apk_pkg_format_cache(pkg, APK_BLOB_BUF(file));
 			bs = apk_bstream_from_file(db->cache_fd, file);
-		}
-
-		if (bs == NULL) {
+		} else {
 			apk_pkg_format_plain(pkg, APK_BLOB_BUF(item));
 			bs = apk_repo_file_open(repo, pkg->arch,
 						item, file, sizeof(file));
-			if (apk_repo_is_remote(repo))
+			if (!(pkg->repos & db->local_repos))
 				need_copy = TRUE;
 		}
 	} else {
@@ -2570,7 +2569,7 @@ static int apk_db_unpack_pkg(struct apk_database *db,
 	if (need_copy) {
 		if (r == 0) {
 			renameat(db->cachetmp_fd, item, db->cache_fd, item);
-			pkg->in_cache = 1;
+			pkg->repos |= BIT(APK_REPOSITORY_CACHED);
 		} else {
 			unlinkat(db->cachetmp_fd, item, 0);
 		}

src/solver.c

@@ -198,14 +198,7 @@ static inline struct apk_package *decision_to_pkg(struct apk_decision *d)
 
 static inline int pkg_available(struct apk_database *db, struct apk_package *pkg)
 {
-	/* virtual packages - only deps used; no real .apk */
-	if (pkg->installed_size == 0)
-		return TRUE;
-	/* obviously present */
-	if (pkg->in_cache || pkg->filename != NULL || (pkg->repos & db->local_repos))
-		return TRUE;
-	/* can download */
-	if ((pkg->repos & ~db->bad_repos) && !(apk_flags & APK_NO_NETWORK))
+	if (pkg->repos & db->available_repos)
 		return TRUE;
 	return FALSE;
 }
@@ -686,6 +679,7 @@ static inline void assign_name(
 	    (p.version != &apk_null_blob || name->ss.chosen.version != &apk_null_blob)) {
 		/* Assigning locked name with version is a problem;
 		 * generally package providing same name twice */
+		dbg_printf("%s: re-provided by %s\n", name->name, p.pkg->name->name);
 		name->ss.locked++;
 		ss->impossible_state = 1;
 		return;

test/basic8.test

+@ARGS
+--no-network
+add -t .virtual
+@EXPECT
+(1/1) Installing .virtual (0)
+OK: 0 MiB in 0 packages

test/basic9.test

+@ARGS
+--no-network
+--test-instdb basic.installed
+--test-world a
+add -t .virtual a
+@EXPECT
+(1/1) Installing .virtual (0)
+OK: 0 MiB in 2 packages