Commit aec93ee7 authored by Timo Teräs's avatar Timo Teräs

db: unify handling of special packages

make cache a special kind of repository, and automatically cache
special packages (virtual packages, or ones installed from command
line). add test cases for handling virtual packages. fixes #1617.
parent 035da870
...@@ -93,7 +93,6 @@ static int add_main(void *ctx, struct apk_database *db, int argc, char **argv) ...@@ -93,7 +93,6 @@ static int add_main(void *ctx, struct apk_database *db, int argc, char **argv)
virtpkg->version = apk_blob_atomize(APK_BLOB_STR("0")); virtpkg->version = apk_blob_atomize(APK_BLOB_STR("0"));
virtpkg->description = strdup("virtual meta package"); virtpkg->description = strdup("virtual meta package");
virtpkg->arch = apk_blob_atomize(APK_BLOB_STR("noarch")); virtpkg->arch = apk_blob_atomize(APK_BLOB_STR("noarch"));
virtpkg->repos |= db->repo_tags[virtdep.repository_tag].allowed_repos;
virtpkg = apk_db_pkg_add(db, virtpkg); virtpkg = apk_db_pkg_add(db, virtpkg);
} }
......
...@@ -476,7 +476,7 @@ int main(int argc, char **argv) ...@@ -476,7 +476,7 @@ int main(int argc, char **argv)
for (i = 0; i < test_repos->num; i++) { for (i = 0; i < test_repos->num; i++) {
struct apk_bstream *bs; struct apk_bstream *bs;
apk_blob_t spec = APK_BLOB_STR(test_repos->item[i]), name, tag; apk_blob_t spec = APK_BLOB_STR(test_repos->item[i]), name, tag;
int repo_tag = 0; int repo_tag = 0, repo = APK_REPOSITORY_FIRST_CONFIGURED + i;
if (apk_blob_split(spec, APK_BLOB_STR(":"), &tag, &name)) { if (apk_blob_split(spec, APK_BLOB_STR(":"), &tag, &name)) {
repo_tag = apk_db_get_tag_id(&db, tag); repo_tag = apk_db_get_tag_id(&db, tag);
...@@ -486,9 +486,11 @@ int main(int argc, char **argv) ...@@ -486,9 +486,11 @@ int main(int argc, char **argv)
bs = apk_bstream_from_file(AT_FDCWD, name.ptr); bs = apk_bstream_from_file(AT_FDCWD, name.ptr);
if (bs != NULL) { if (bs != NULL) {
apk_db_index_read(&db, bs, i); apk_db_index_read(&db, bs, repo);
db.repo_tags[repo_tag].allowed_repos |= BIT(i);
bs->close(bs, NULL); bs->close(bs, NULL);
if (!(apk_flags & APK_NO_NETWORK))
db.available_repos |= BIT(repo);
db.repo_tags[repo_tag].allowed_repos |= BIT(repo);
} }
} }
#endif #endif
......
...@@ -101,7 +101,6 @@ struct apk_name { ...@@ -101,7 +101,6 @@ struct apk_name {
struct apk_repository { struct apk_repository {
char *url; char *url;
struct apk_checksum csum; struct apk_checksum csum;
apk_blob_t description; apk_blob_t description;
}; };
...@@ -120,6 +119,9 @@ struct apk_db_options { ...@@ -120,6 +119,9 @@ struct apk_db_options {
struct list_head repository_list; struct list_head repository_list;
}; };
#define APK_REPOSITORY_CACHED 0
#define APK_REPOSITORY_FIRST_CONFIGURED 1
#define APK_DEFAULT_REPOSITORY_TAG 0 #define APK_DEFAULT_REPOSITORY_TAG 0
#define APK_DEFAULT_PINNING_MASK BIT(APK_DEFAULT_REPOSITORY_TAG) #define APK_DEFAULT_PINNING_MASK BIT(APK_DEFAULT_REPOSITORY_TAG)
...@@ -135,7 +137,7 @@ struct apk_database { ...@@ -135,7 +137,7 @@ struct apk_database {
const char *cache_dir; const char *cache_dir;
char *cache_remount_dir; char *cache_remount_dir;
apk_blob_t *arch; apk_blob_t *arch;
unsigned int local_repos, bad_repos; unsigned int local_repos, available_repos;
int performing_self_update : 1; int performing_self_update : 1;
int permanent : 1; int permanent : 1;
int compat_newfeatures : 1; int compat_newfeatures : 1;
...@@ -217,8 +219,6 @@ int apk_db_index_write(struct apk_database *db, struct apk_ostream *os); ...@@ -217,8 +219,6 @@ int apk_db_index_write(struct apk_database *db, struct apk_ostream *os);
int apk_db_add_repository(apk_database_t db, apk_blob_t repository); int apk_db_add_repository(apk_database_t db, apk_blob_t repository);
struct apk_repository *apk_db_select_repo(struct apk_database *db, struct apk_repository *apk_db_select_repo(struct apk_database *db,
struct apk_package *pkg); struct apk_package *pkg);
int apk_repository_update(struct apk_database *db, struct apk_repository *repo);
int apk_repo_is_remote(struct apk_repository *repo);
int apk_repo_format_filename(char *buf, size_t len, int apk_repo_format_filename(char *buf, size_t len,
const char *repourl, apk_blob_t *arch, const char *repourl, apk_blob_t *arch,
const char *pkgfile); const char *pkgfile);
......
...@@ -80,7 +80,7 @@ extern char **apk_argv; ...@@ -80,7 +80,7 @@ extern char **apk_argv;
#define APK_DEFAULT_ARCH "noarch" #define APK_DEFAULT_ARCH "noarch"
#endif #endif
#define APK_MAX_REPOS 31 /* see struct apk_package */ #define APK_MAX_REPOS 32 /* see struct apk_package */
#define APK_MAX_TAGS 16 /* see solver; unsigned short */ #define APK_MAX_TAGS 16 /* see solver; unsigned short */
#define APK_CACHE_CSUM_BYTES 4 #define APK_CACHE_CSUM_BYTES 4
......
...@@ -99,7 +99,6 @@ struct apk_package { ...@@ -99,7 +99,6 @@ struct apk_package {
size_t installed_size, size; size_t installed_size, size;
time_t build_time; time_t build_time;
unsigned int topology_hard; unsigned int topology_hard;
unsigned in_cache : 1;
unsigned repos : APK_MAX_REPOS; unsigned repos : APK_MAX_REPOS;
struct apk_checksum csum; struct apk_checksum csum;
}; };
......
...@@ -43,11 +43,11 @@ static int cache_download(struct apk_database *db) ...@@ -43,11 +43,11 @@ static int cache_download(struct apk_database *db)
change = &changeset.changes->item[i]; change = &changeset.changes->item[i];
pkg = change->newpkg; pkg = change->newpkg;
if (pkg->in_cache) if (pkg->repos & db->local_repos)
continue; continue;
repo = apk_db_select_repo(db, pkg); repo = apk_db_select_repo(db, pkg);
if (repo == NULL || !apk_repo_is_remote(repo)) if (repo == NULL)
continue; continue;
apk_pkg_format_cache(pkg, APK_BLOB_BUF(cacheitem)); apk_pkg_format_cache(pkg, APK_BLOB_BUF(cacheitem));
......
...@@ -531,6 +531,11 @@ struct apk_package *apk_db_pkg_add(struct apk_database *db, struct apk_package * ...@@ -531,6 +531,11 @@ struct apk_package *apk_db_pkg_add(struct apk_database *db, struct apk_package *
if (pkg->license == NULL) if (pkg->license == NULL)
pkg->license = apk_blob_atomize(APK_BLOB_NULL); pkg->license = apk_blob_atomize(APK_BLOB_NULL);
/* Set as "cached" if installing from specified file, and
* for virtual packages */
if (pkg->filename != NULL || pkg->installed_size == 0)
pkg->repos |= BIT(APK_REPOSITORY_CACHED);
idb = apk_hash_get(&db->available.packages, APK_BLOB_CSUM(pkg->csum)); idb = apk_hash_get(&db->available.packages, APK_BLOB_CSUM(pkg->csum));
if (idb == NULL) { if (idb == NULL) {
idb = pkg; idb = pkg;
...@@ -1124,7 +1129,7 @@ static int apk_db_index_write_nr_cache(struct apk_database *db) ...@@ -1124,7 +1129,7 @@ static int apk_db_index_write_nr_cache(struct apk_database *db)
ctx.os = os; ctx.os = os;
list_for_each_entry(ipkg, &db->installed.packages, installed_pkgs_list) { list_for_each_entry(ipkg, &db->installed.packages, installed_pkgs_list) {
struct apk_package *pkg = ipkg->pkg; struct apk_package *pkg = ipkg->pkg;
if (pkg->repos != 0 || !pkg->in_cache) if (pkg->repos != BIT(APK_REPOSITORY_CACHED))
continue; continue;
r = write_index_entry(pkg, &ctx); r = write_index_entry(pkg, &ctx);
if (r != 0) if (r != 0)
...@@ -1326,7 +1331,7 @@ static void mark_in_cache(struct apk_database *db, int dirfd, const char *name, ...@@ -1326,7 +1331,7 @@ static void mark_in_cache(struct apk_database *db, int dirfd, const char *name,
if (pkg == NULL) if (pkg == NULL)
return; return;
pkg->in_cache = 1; pkg->repos |= BIT(APK_REPOSITORY_CACHED);
} }
static int add_repos_from_file(void *ctx, int dirfd, const char *file) static int add_repos_from_file(void *ctx, int dirfd, const char *file)
...@@ -1350,6 +1355,27 @@ static int add_repos_from_file(void *ctx, int dirfd, const char *file) ...@@ -1350,6 +1355,27 @@ static int add_repos_from_file(void *ctx, int dirfd, const char *file)
return 0; return 0;
} }
static void apk_db_setup_repositories(struct apk_database *db)
{
int repo_tag;
db->repos[APK_REPOSITORY_CACHED] = (struct apk_repository) {
.url = "cache",
.csum.data = {
0xb0,0x35,0x92,0x80,0x6e,0xfa,0xbf,0xee,0xb7,0x09,
0xf5,0xa7,0x0a,0x7c,0x17,0x26,0x69,0xb0,0x05,0x38 },
.csum.type = APK_CHECKSUM_SHA1,
};
db->num_repos = APK_REPOSITORY_FIRST_CONFIGURED;
db->local_repos |= BIT(APK_REPOSITORY_CACHED);
db->available_repos |= BIT(APK_REPOSITORY_CACHED);
/* Get first repository tag (the NULL tag) */
repo_tag = apk_db_get_tag_id(db, APK_BLOB_NULL);
db->repo_tags[repo_tag].allowed_repos |= BIT(APK_REPOSITORY_CACHED);
}
int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts) int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
{ {
const char *msg = NULL; const char *msg = NULL;
...@@ -1381,8 +1407,7 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts) ...@@ -1381,8 +1407,7 @@ int apk_db_open(struct apk_database *db, struct apk_db_options *dbopts)
apk_protected_path_array_init(&db->protected_paths); apk_protected_path_array_init(&db->protected_paths);
db->permanent = 1; db->permanent = 1;
/* Get first repository tag (the NULL tag) */ apk_db_setup_repositories(db);
apk_db_get_tag_id(db, APK_BLOB_NULL);
db->root = strdup(dbopts->root ?: "/"); db->root = strdup(dbopts->root ?: "/");
db->root_fd = openat(AT_FDCWD, db->root, O_RDONLY | O_CLOEXEC); db->root_fd = openat(AT_FDCWD, db->root, O_RDONLY | O_CLOEXEC);
...@@ -1652,7 +1677,7 @@ void apk_db_close(struct apk_database *db) ...@@ -1652,7 +1677,7 @@ void apk_db_close(struct apk_database *db)
} }
} }
for (i = 0; i < db->num_repos; i++) { for (i = APK_REPOSITORY_FIRST_CONFIGURED; i < db->num_repos; i++) {
free(db->repos[i].url); free(db->repos[i].url);
free(db->repos[i].description.ptr); free(db->repos[i].description.ptr);
} }
...@@ -1879,11 +1904,6 @@ int apk_repo_format_filename(char *buf, size_t len, ...@@ -1879,11 +1904,6 @@ int apk_repo_format_filename(char *buf, size_t len,
return n; return n;
} }
int apk_repo_is_remote(struct apk_repository *repo)
{
return repo->csum.type != APK_CHECKSUM_NONE;
}
static struct apk_bstream *apk_repo_file_open(struct apk_repository *repo, static struct apk_bstream *apk_repo_file_open(struct apk_repository *repo,
apk_blob_t *arch, apk_blob_t *arch,
const char *file, const char *file,
...@@ -1893,9 +1913,6 @@ static struct apk_bstream *apk_repo_file_open(struct apk_repository *repo, ...@@ -1893,9 +1913,6 @@ static struct apk_bstream *apk_repo_file_open(struct apk_repository *repo,
apk_repo_format_filename(buf, buflen, url, arch, file); apk_repo_format_filename(buf, buflen, url, arch, file);
if ((apk_flags & APK_NO_NETWORK) && apk_repo_is_remote(repo))
return NULL;
/* We should not get called for non-repository files */ /* We should not get called for non-repository files */
if (strcmp(repo->url, "cache") == 0) if (strcmp(repo->url, "cache") == 0)
return NULL; return NULL;
...@@ -1906,49 +1923,30 @@ static struct apk_bstream *apk_repo_file_open(struct apk_repository *repo, ...@@ -1906,49 +1923,30 @@ static struct apk_bstream *apk_repo_file_open(struct apk_repository *repo,
struct apk_repository *apk_db_select_repo(struct apk_database *db, struct apk_repository *apk_db_select_repo(struct apk_database *db,
struct apk_package *pkg) struct apk_package *pkg)
{ {
static struct apk_repository cache_repo = { unsigned int repos;
.url = "cache",
.csum.data = {
0xb0,0x35,0x92,0x80,0x6e,0xfa,0xbf,0xee,0xb7,0x09,
0xf5,0xa7,0x0a,0x7c,0x17,0x26,0x69,0xb0,0x05,0x38 },
.csum.type = APK_CHECKSUM_SHA1,
};
unsigned int repos = pkg->repos & ~(db->bad_repos);
int i; int i;
/* Uninstalled and unavailable? */ /* Select repositories to use */
if (repos == 0 && pkg->ipkg == NULL) repos = pkg->repos & db->available_repos;
if (repos == 0)
return NULL; return NULL;
if ((repos & db->local_repos) == 0) { if (repos & db->local_repos)
/* Network repository (or installed and unavailable) */
if (apk_flags & APK_NO_NETWORK) {
if (pkg->in_cache)
return &cache_repo;
return NULL;
}
} else {
/* Local repositories; don't use network */
repos &= db->local_repos; repos &= db->local_repos;
}
/* Pick first repository providing this package */ /* Pick first repository providing this package */
for (i = 0; i < APK_MAX_REPOS; i++) { for (i = APK_REPOSITORY_FIRST_CONFIGURED; i < APK_MAX_REPOS; i++) {
if (repos & BIT(i)) if (repos & BIT(i))
return &db->repos[i]; return &db->repos[i];
} }
return &db->repos[APK_REPOSITORY_CACHED];
return NULL;
} }
int apk_repository_update(struct apk_database *db, struct apk_repository *repo) static int apk_repository_update(struct apk_database *db, struct apk_repository *repo)
{ {
char cacheitem[PATH_MAX]; char cacheitem[PATH_MAX];
int r; int r;
if (!apk_repo_is_remote(repo))
return 0;
apk_cache_format_index(APK_BLOB_BUF(cacheitem), repo); apk_cache_format_index(APK_BLOB_BUF(cacheitem), repo);
r = apk_cache_download(db, repo->url, db->arch, apkindex_tar_gz, cacheitem, r = apk_cache_download(db, repo->url, db->arch, apkindex_tar_gz, cacheitem,
(apk_flags & APK_ALLOW_UNTRUSTED) ? (apk_flags & APK_ALLOW_UNTRUSTED) ?
...@@ -2055,7 +2053,7 @@ int apk_db_add_repository(apk_database_t _db, apk_blob_t _repository) ...@@ -2055,7 +2053,7 @@ int apk_db_add_repository(apk_database_t _db, apk_blob_t _repository)
repo = &db->repos[repo_num]; repo = &db->repos[repo_num];
if (strcmp(url, repo->url) == 0) { if (strcmp(url, repo->url) == 0) {
db->repo_tags[tag_id].allowed_repos |= db->repo_tags[tag_id].allowed_repos |=
BIT(repo_num) & ~db->bad_repos; BIT(repo_num) & db->available_repos;
free(url); free(url);
return 0; return 0;
} }
...@@ -2071,29 +2069,32 @@ int apk_db_add_repository(apk_database_t _db, apk_blob_t _repository) ...@@ -2071,29 +2069,32 @@ int apk_db_add_repository(apk_database_t _db, apk_blob_t _repository)
.url = url, .url = url,
}; };
if (apk_url_local_file(repo->url) == NULL) { apk_blob_checksum(brepo, apk_checksum_default(), &repo->csum);
apk_blob_checksum(brepo, apk_checksum_default(), &repo->csum);
if (apk_flags & APK_UPDATE_CACHE)
apk_repository_update(db, repo);
if (apk_url_local_file(repo->url) == NULL) {
if (!(apk_flags & APK_NO_NETWORK)) {
db->available_repos |= BIT(repo_num);
if (apk_flags & APK_UPDATE_CACHE)
apk_repository_update(db, repo);
}
apk_cache_format_index(APK_BLOB_BUF(buf), repo); apk_cache_format_index(APK_BLOB_BUF(buf), repo);
bs = apk_bstream_from_file(db->cache_fd, buf); bs = apk_bstream_from_file(db->cache_fd, buf);
} else { } else {
db->local_repos |= BIT(repo_num); db->local_repos |= BIT(repo_num);
db->available_repos |= BIT(repo_num);
bs = apk_repo_file_open(repo, db->arch, apkindex_tar_gz, buf, sizeof(buf)); bs = apk_repo_file_open(repo, db->arch, apkindex_tar_gz, buf, sizeof(buf));
} }
if (bs != NULL) { if (bs != NULL)
r = load_index(db, bs, targz, repo_num); r = load_index(db, bs, targz, repo_num);
} else else
r = -ENOENT; r = -ENOENT;
if (r != 0) { if (r != 0) {
apk_warning("Ignoring %s: %s", buf, apk_error_str(r)); apk_warning("Ignoring %s: %s", buf, apk_error_str(r));
db->bad_repos |= BIT(repo_num); db->available_repos &= ~BIT(repo_num);
r = 0; r = 0;
} else { } else {
db->repo_tags[tag_id].allowed_repos |= BIT(repo_num); db->repo_tags[tag_id].allowed_repos |= BIT(repo_num) & db->available_repos;
} }
return 0; return 0;
...@@ -2517,16 +2518,14 @@ static int apk_db_unpack_pkg(struct apk_database *db, ...@@ -2517,16 +2518,14 @@ static int apk_db_unpack_pkg(struct apk_database *db,
return -1; return -1;
} }
if (apk_db_cache_active(db) && apk_repo_is_remote(repo)) { if (strcmp(repo->url, "cache") == 0) {
apk_pkg_format_cache(pkg, APK_BLOB_BUF(file)); apk_pkg_format_cache(pkg, APK_BLOB_BUF(file));
bs = apk_bstream_from_file(db->cache_fd, file); bs = apk_bstream_from_file(db->cache_fd, file);
} } else {
if (bs == NULL) {
apk_pkg_format_plain(pkg, APK_BLOB_BUF(item)); apk_pkg_format_plain(pkg, APK_BLOB_BUF(item));
bs = apk_repo_file_open(repo, pkg->arch, bs = apk_repo_file_open(repo, pkg->arch,
item, file, sizeof(file)); item, file, sizeof(file));
if (apk_repo_is_remote(repo)) if (!(pkg->repos & db->local_repos))
need_copy = TRUE; need_copy = TRUE;
} }
} else { } else {
...@@ -2570,7 +2569,7 @@ static int apk_db_unpack_pkg(struct apk_database *db, ...@@ -2570,7 +2569,7 @@ static int apk_db_unpack_pkg(struct apk_database *db,
if (need_copy) { if (need_copy) {
if (r == 0) { if (r == 0) {
renameat(db->cachetmp_fd, item, db->cache_fd, item); renameat(db->cachetmp_fd, item, db->cache_fd, item);
pkg->in_cache = 1; pkg->repos |= BIT(APK_REPOSITORY_CACHED);
} else { } else {
unlinkat(db->cachetmp_fd, item, 0); unlinkat(db->cachetmp_fd, item, 0);
} }
......
...@@ -198,14 +198,7 @@ static inline struct apk_package *decision_to_pkg(struct apk_decision *d) ...@@ -198,14 +198,7 @@ static inline struct apk_package *decision_to_pkg(struct apk_decision *d)
static inline int pkg_available(struct apk_database *db, struct apk_package *pkg) static inline int pkg_available(struct apk_database *db, struct apk_package *pkg)
{ {
/* virtual packages - only deps used; no real .apk */ if (pkg->repos & db->available_repos)
if (pkg->installed_size == 0)
return TRUE;
/* obviously present */
if (pkg->in_cache || pkg->filename != NULL || (pkg->repos & db->local_repos))
return TRUE;
/* can download */
if ((pkg->repos & ~db->bad_repos) && !(apk_flags & APK_NO_NETWORK))
return TRUE; return TRUE;
return FALSE; return FALSE;
} }
...@@ -686,6 +679,7 @@ static inline void assign_name( ...@@ -686,6 +679,7 @@ static inline void assign_name(
(p.version != &apk_null_blob || name->ss.chosen.version != &apk_null_blob)) { (p.version != &apk_null_blob || name->ss.chosen.version != &apk_null_blob)) {
/* Assigning locked name with version is a problem; /* Assigning locked name with version is a problem;
* generally package providing same name twice */ * generally package providing same name twice */
dbg_printf("%s: re-provided by %s\n", name->name, p.pkg->name->name);
name->ss.locked++; name->ss.locked++;
ss->impossible_state = 1; ss->impossible_state = 1;
return; return;
......
@ARGS
--no-network
add -t .virtual
@EXPECT
(1/1) Installing .virtual (0)
OK: 0 MiB in 0 packages
@ARGS
--no-network
--test-instdb basic.installed
--test-world a
add -t .virtual a
@EXPECT
(1/1) Installing .virtual (0)
OK: 0 MiB in 2 packages
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment