solver: new package selection logic (which is not yet used)

 * basic code for a backtracking, forward checking dependency satisfier
 * works better when there are tricky dependencies to solve
   (when can't just upgrade everything to most preferred versions)
 * the new code always evaluates all of 'world' constraints
   (old code just does incremental updates based on heuristics)
 * is probably somewhat slower than old code (probably unnoticeable
   difference in most cases)
 * makes easier to write support for provides and repository pinning
 * test applet and a bunch of test cases added which uses the new code
 * from the old feature set install_if is not yet implemented

.gitignore

-apk
-apk.static
+src/apk
+src/apk_test
+src/apk.static
 *.o
 *.d
 *.cmd

Makefile

@@ -36,5 +36,9 @@ install:
 	$(INSTALLDIR) $(DESTDIR)$(DOCDIR)
 	$(INSTALL) README $(DESTDIR)$(DOCDIR)
 
+test: FORCE
+	$(Q)$(MAKE) TEST=y
+	$(Q)$(MAKE) -C test
+
 static:
 	$(Q)$(MAKE) STATIC=y

src/Makefile

@@ -23,7 +23,13 @@ apk-objs		:= apk.o add.o del.o fix.o update.o info.o \
 			   audit.o verify.o dot.o
 
 libapk.so-objs		:= common.o state.o database.o package.o archive.o \
-			   version.o io.o url.o gunzip.o blob.o hash.o print.o
+			   version.o io.o url.o gunzip.o blob.o hash.o print.o \
+			   solver.o topology.o
+
+ifeq ($(TEST),y)
+progs-y			+= apk_test
+apk_test-objs		:= apk.o test.o $(libapk.so-objs)
+endif
 
 ifeq ($(SHARED_LIBAPK),)
 apk-objs		+= $(libapk.so-objs)
@@ -47,15 +53,16 @@ progs-$(STATIC)		+= apk.static
 apk.static-objs		:= $(filter-out apk.o,$(apk-objs)) apk-static.o
 LDFLAGS_apk.static	:= -static
 LDFLAGS_apk		+= -nopie -L$(obj)
+LDFLAGS_apk_test	+= -nopie -L$(obj)
 
 CFLAGS_ALL		+= $(shell pkg-config --cflags $(PKGDEPS))
 LIBS			:= -Wl,--as-needed \
 				$(shell pkg-config --libs $(PKGDEPS)) \
 			   -Wl,--no-as-needed
 
-$(obj)/apk:	$(LIBAPK-y)
+$(obj)/apk: $(LIBAPK-y)
 
-$(obj)/apk.so:	$(obj)/libapk.so
+$(obj)/apk.so: $(obj)/libapk.so
 
 install: $(obj)/apk $(LIBAPK-y) $(LUA_LIB-y)
 	$(INSTALLDIR) $(DESTDIR)$(SBINDIR)

src/apk_package.h

@@ -80,6 +80,7 @@ struct apk_installed_package {
 
 struct apk_package {
 	apk_hash_node hash_node;
+	unsigned int topology_sort;
 	struct apk_name *name;
 	struct apk_installed_package *ipkg;
 	apk_blob_t *version, *arch, *license;

src/apk_solver.h

+/* apk_solver.h - Alpine Package Keeper (APK)
+ *
+ * Copyright (C) 2005-2008 Natanael Copa <n@tanael.org>
+ * Copyright (C) 2008 Timo Teräs <timo.teras@iki.fi>
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation. See http://www.gnu.org/ for details.
+ */
+
+#ifndef APK_SOLVER_H
+#define APK_SOLVER_H
+
+struct apk_change {
+	struct apk_package *oldpkg;
+	struct apk_package *newpkg;
+};
+APK_ARRAY(apk_change_array, struct apk_change);
+
+struct apk_changeset {
+	struct apk_change_array *changes;
+};
+
+void apk_solver_sort(struct apk_database *db);
+int apk_solver_solve(struct apk_database *db, struct apk_dependency_array *world,
+		 struct apk_package_array **solution);
+int apk_solver_generate_changeset(struct apk_database *db,
+				  struct apk_package_array *solution,
+				  struct apk_changeset *changeset);
+
+#endif
+

src/test.c

+/* test.c - Alpine Package Keeper (APK)
+ *
+ * Copyright (C) 2011 Timo Teräs <timo.teras@iki.fi>
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation. See http://www.gnu.org/ for details.
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include "apk_applet.h"
+#include "apk_database.h"
+#include "apk_solver.h"
+#include "apk_io.h"
+#include "apk_print.h"
+
+struct test_ctx {
+	const char *installed_db;
+	struct apk_string_array *repos;
+};
+
+static int test_parse(void *pctx, struct apk_db_options *dbopts,
+		      int optch, int optindex, const char *optarg)
+{
+	struct test_ctx *ctx = (struct test_ctx *) pctx;
+
+	switch (optch) {
+	case 0x10000:
+		ctx->installed_db = optarg;
+		break;
+	case 0x10001:
+		if (ctx->repos == NULL)
+			apk_string_array_init(&ctx->repos);
+		*apk_string_array_add(&ctx->repos) = (char*) optarg;
+		break;
+	case 'u':
+		apk_flags |= APK_UPGRADE;
+		break;
+	case 'a':
+		apk_flags |= APK_PREFER_AVAILABLE;
+		break;
+	default:
+		return -1;
+	}
+	return 0;
+}
+
+static inline void print_change(struct apk_package *oldpkg,
+				struct apk_package *newpkg)
+{
+	const char *msg = NULL;
+	struct apk_name *name;
+	int r;
+
+	if (oldpkg != NULL)
+		name = oldpkg->name;
+	else
+		name = newpkg->name;
+
+	if (oldpkg == NULL) {
+		apk_message("Installing %s (" BLOB_FMT ")",
+			    name->name,
+			    BLOB_PRINTF(*newpkg->version));
+	} else if (newpkg == NULL) {
+		apk_message("Purging %s (" BLOB_FMT ")",
+			    name->name,
+			    BLOB_PRINTF(*oldpkg->version));
+	} else {
+		r = apk_pkg_version_compare(newpkg, oldpkg);
+		switch (r) {
+		case APK_VERSION_LESS:
+			msg = "Downgrading";
+			break;
+		case APK_VERSION_EQUAL:
+			if (newpkg == oldpkg)
+				msg = "Re-installing";
+			else
+				msg = "Replacing";
+			break;
+		case APK_VERSION_GREATER:
+			msg = "Upgrading";
+			break;
+		}
+		apk_message("%s %s (" BLOB_FMT " -> " BLOB_FMT ")",
+			    msg, name->name,
+			    BLOB_PRINTF(*oldpkg->version),
+			    BLOB_PRINTF(*newpkg->version));
+	}
+}
+
+static int test_main(void *pctx, struct apk_database *db, int argc, char **argv)
+{
+	struct test_ctx *ctx = (struct test_ctx *) pctx;
+	struct apk_bstream *bs;
+	struct apk_package_array *solution = NULL;
+	struct apk_changeset changeset;
+	int i;
+
+	if (argc != 1)
+		return -EINVAL;
+
+	/* load installed db */
+	if (ctx->installed_db != NULL) {
+		bs = apk_bstream_from_file(AT_FDCWD, ctx->installed_db);
+		apk_db_index_read(db, bs, -1);
+		bs->close(bs, NULL);
+	}
+
+	/* load additional indexes */
+	if (ctx->repos) {
+		for (i = 0; i < ctx->repos->num; i++) {
+			bs = apk_bstream_from_file(AT_FDCWD, ctx->repos->item[i]);
+			apk_db_index_read(db, bs, i);
+			bs->close(bs, NULL);
+		}
+	}
+
+	/* construct new world */
+	apk_deps_parse(db, &db->world, APK_BLOB_STR(argv[0]));
+
+	/* run solver */
+	apk_solver_sort(db);
+	if (apk_solver_solve(db, db->world, &solution) != 0)
+		return 1;
+
+	memset(&changeset, 0, sizeof(changeset));
+	if (apk_solver_generate_changeset(db, solution, &changeset) == 0) {
+		/* dump changeset */
+		for (i = 0; i < changeset.changes->num; i++) {
+			struct apk_change *c = &changeset.changes->item[i];
+			print_change(c->oldpkg, c->newpkg);
+		}
+	}
+
+	return 0;
+}
+
+static struct apk_option test_options[] = {
+	{ 0x10000,	"installed",	"Installed database",
+			required_argument, "DB" },
+	{ 0x10001,	"raw-repository", "Add unsigned test repository index",
+			required_argument, "INDEX" },
+	{ 'u',		"upgrade",	"Prefer to upgrade package" },
+	{ 'a',		"available",
+	  "Re-install or downgrade if currently installed package is not "
+	  "currently available from any repository" },
+};
+
+static struct apk_applet test_applet = {
+	.name = "test",
+	.help = "Test dependency graph solver (uses simple repository and installed db)",
+	.arguments = "'WORLD'",
+	.open_flags = APK_OPENF_READ | APK_OPENF_NO_STATE | APK_OPENF_NO_REPOS,
+	.context_size = sizeof(struct test_ctx),
+	.num_options = ARRAY_SIZE(test_options),
+	.options = test_options,
+	.parse = test_parse,
+	.main = test_main,
+};
+
+APK_DEFINE_APPLET(test_applet);

src/topology.c

+/* topology.c - Alpine Package Keeper (APK)
+ * Topological sorting of database packages
+ *
+ * Copyright (C) 2011 Timo Teräs <timo.teras@iki.fi>
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation. See http://www.gnu.org/ for details.
+ */
+
+#include "apk_database.h"
+
+static int sort_pkg(apk_hash_item item, void *ctx)
+{
+	struct apk_package *pkg = (struct apk_package *) item;
+	unsigned int *sort_order = (unsigned int *) ctx;
+	int i, j;
+
+	/* Avoid recursion to same package */
+	if (pkg->topology_sort != 0) {
+		/* if pkg->topology_sort==-1 we have encountered a
+		 * dependency loop. Just silently ignore it and pick a
+		 * random topology sorting. */
+		return 0;
+	}
+	pkg->topology_sort = -1;
+
+	/* Sort all dependants before self */
+	for (i = 0; i < pkg->depends->num; i++) {
+		struct apk_dependency *dep = &pkg->depends->item[i];
+		struct apk_name *name0 = dep->name;
+
+		/* FIXME: sort names in order of ascending preference */
+		for (j = 0; j < name0->pkgs->num; j++) {
+			struct apk_package *pkg0 = name0->pkgs->item[j];
+			if (!apk_dep_is_satisfied(dep, pkg0))
+				continue;
+			sort_pkg(pkg0, ctx);
+		}
+	}
+
+	/* FIXME: install_if, provides, etc.*/
+
+	/* Finally assign a topology sorting order */
+	pkg->topology_sort = ++(*sort_order);
+
+	return 0;
+}
+
+void apk_solver_sort(struct apk_database *db)
+{
+	unsigned int sort_order = 0;
+
+	apk_hash_foreach(&db->available.packages, sort_pkg, &sort_order);
+}

test/Makefile

@@ -13,7 +13,7 @@ SYSREPO ?= http://alpinelinux.org/cgi-bin/dl.cgi/edge/main
 LD_LIBRARY_PATH = ../src
 export LD_LIBRARY_PATH SYSREPO
 
-all:	tests
+all: tests
 
 $(repos):
 	@echo "Building $@"
@@ -50,11 +50,17 @@ repos.stamp: $(repos)
 		$(SUDO) rm -rf $(testroot); \
 		touch $@
 
-tests: repos.stamp
-	@echo "== Testing `$(APK) --version` =="
+root-tests: repos.stamp
+	@echo "== Testing `$(APK) --version` (tests that require root permission) =="
 	@for i in test*.sh; do \
 		rm -f $${i%.sh}.ok ;\
 		$(SUDO) $(MAKE) --no-print-directory $${i%.sh}.ok SYSREPO=$(SYSREPO); \
 	done
 
-.PHONY:	$(repos)
+tests:
+	@echo "== Testing `$(APK) --version` =="
+	@for i in $(sort $(filter-out test%.sh,$(wildcard *.sh))); do \
+		./$$i || exit 1 ; \
+	done
+
+.PHONY:	$(repos) tests

test/basic.installed

+C:Q1EyN5AdpAOBJWKMR89pp/C66o+OE=
+P:a
+V:1
+S:1
+I:1
+D:b
+
+C:Q1C4uoV7SdMdDhYg4OCVmI71D8HIA=
+P:b
+V:1
+S:1
+I:1
+

test/basic.repo

+C:Q1EyN5AdpAOBJWKMR89pp/C66o+OE=
+P:a
+V:1
+S:1
+I:1
+D:b
+
+C:Q1eVpkasfqZAukAXFYbgwt4xAMZWU=
+P:a
+V:2
+S:1
+I:1
+D:b
+
+C:Q1C4uoV7SdMdDhYg4OCVmI71D8HIA=
+P:b
+V:1
+S:1
+I:1
+
+C:Q1hdUpqRv5mYgJEqW52UmVsvmyysE=
+P:b
+V:2
+S:1
+I:1
+

test/basic1.expect

+Installing a (2)
+Installing b (2)

test/basic1.test

+--raw-repository basic.repo a

test/basic2.test

+--raw-repository basic.repo --installed basic.installed a

test/basic3.expect

+Upgrading a (1 -> 2)
+Upgrading b (1 -> 2)

test/basic3.test

+--raw-repository basic.repo --installed basic.installed -u a

test/basic4.expect

+Purging a (1)

test/basic4.test

+--raw-repository basic.repo --installed basic.installed b

test/complicated1.expect

+Installing a (2)
+Installing b (1)
+Installing c (1)
+Installing d (1.5)

test/complicated1.installed

+C:Q16m4HrGizBiH4lG6Mxd5EL239L2U=
+P:d
+V:1.0
+S:1
+I:1
+

test/complicated1.repo

+C:Q1EyN5AdpAOBJWKMR89pp/C66o+OE=
+P:a
+V:2
+S:1
+I:1
+D:b c
+
+C:Q1eVpkasfqZAukAXFYbgwt4xAMZWU=
+P:a
+V:3
+S:1
+I:1
+D:b c d>1.5
+
+C:Q1C4uoV7SdMdDhYg4OCVmI71D8HIA=
+P:b
+V:1
+S:1
+I:1
+D:c d<2.0
+
+C:Q1hdUpqRv5mYgJEqW52UmVsvmyysE=
+P:c
+V:1
+S:1
+I:1
+D:d>1.0
+
+C:Q16m4HrGizBiH4lG6Mxd5EL239L2U=
+P:d
+V:1.0
+S:1
+I:1
+
+C:Q1/hQ3eH2AguTwJVGOz+keypXhXKY=
+P:d
+V:1.5
+S:1
+I:1
+
+C:Q19uA/Cwc6UfrQs95TWVDETyAeEYM=
+P:d
+V:2.0
+S:1
+I:1
+

test/complicated1.test

+--raw-repository complicated1.repo a

test/complicated2.expect

+Installing b (1)
+Installing c (1)
+Installing d (1.5)

test/complicated2.test

+--raw-repository complicated1.repo b

test/complicated3.expect

+Installing c (1)
+Installing d (2.0)

test/complicated3.test

+--raw-repository complicated1.repo c

test/complicated4.expect

+Installing a (2)
+Installing b (1)
+Installing c (1)
+Upgrading d (1.0 -> 1.5)

test/complicated4.test

+--raw-repository complicated1.repo --installed complicated1.installed a

test/solver.sh

+#!/bin/sh
+
+APK_TEST=../src/apk_test
+
+fail=0
+for test in *.test; do
+	bn=$(basename $test .test)
+	$APK_TEST $(cat $test) &> $bn.got
+	if ! cmp $bn.expect $bn.got 2> /dev/null; then
+		fail=$((fail+1))
+		echo "FAIL: $test"
+		diff -ru $bn.expect $bn.got
+	else
+		echo "OK: $test"
+	fi
+done
+
+if [ "$fail" != "0" ]; then
+	echo "FAIL: $fail failed test cases"
+fi
+
+exit $fail
+

test/version.sh

 #!/bin/sh
 
 fail=0
-while read a result b rest ; do
+cat version.data | while read a result b rest ; do
 	output="$(../src/apk version -t "$a" "$b")"
 	if [ "$output" != "$result" ] ; then
 		echo "$a $result $b, but got $output"
@@ -9,5 +9,9 @@ while read a result b rest ; do
 	fi
 done
 
+if [ "$fail" == "0" ]; then
+	echo "OK: version checking works"
+fi
+
 exit $fail