db: cache index files, 'update' applet

Cache non-local index files always locally.
Introduce 'update' applet to force refresh of cached index files.
Fixes #19.

src/Makefile

@@ -2,8 +2,8 @@ progs-y			+= apk
 apk-objs		:= state.o database.o package.o archive.o \
 			   version.o io.o url.o gunzip.o blob.o \
 			   hash.o md5.o apk.o \
-			   add.o del.o ver.o index.o info.o search.o \
-			   fetch.o audit.o
+			   add.o del.o update.o info.o search.o \
+			   ver.o index.o fetch.o audit.o
 CFLAGS_apk.o		:= -DAPK_VERSION=\"$(FULL_VERSION)\"
 
 progs-$(STATIC)		+= apk.static

src/apk_database.h

@@ -67,6 +67,7 @@ struct apk_name {
 
 struct apk_repository {
 	char *url;
+	char *cache;
 };
 
 struct apk_database {
@@ -124,6 +125,8 @@ struct apk_package *apk_db_get_file_owner(struct apk_database *db, apk_blob_t fi
 int apk_db_index_write(struct apk_database *db, struct apk_ostream *os);
 
 int apk_db_add_repository(apk_database_t db, apk_blob_t repository);
+int apk_repository_update(struct apk_database *db, struct apk_repository *repo);
+
 int apk_db_install_pkg(struct apk_database *db,
 		       struct apk_package *oldpkg,
 		       struct apk_package *newpkg,

src/apk_io.h

@@ -69,5 +69,7 @@ apk_blob_t apk_blob_from_istream(struct apk_istream *istream, size_t size);
 apk_blob_t apk_blob_from_file(const char *file);
 
 int apk_file_get_info(const char *filename, struct apk_file_info *fi);
+int apk_url_download(const char *url, const char *file);
+const char *apk_url_local_file(const char *url);
 
 #endif

src/database.c

@@ -803,8 +803,11 @@ void apk_db_close(struct apk_database *db)
 		}
 	}
 
-	for (i = 0; i < db->num_repos; i++)
+	for (i = 0; i < db->num_repos; i++) {
 		free(db->repos[i].url);
+		if (db->repos[i].cache != NULL)
+			free(db->repos[i].cache);
+	}
 	if (db->protected_paths) {
 		for (i = 0; i < db->protected_paths->num; i++)
 			free(db->protected_paths->item[i]);
@@ -896,13 +899,77 @@ int apk_db_index_write(struct apk_database *db, struct apk_ostream *os)
 	return ctx.count;
 }
 
-int apk_db_add_repository(apk_database_t _db, apk_blob_t repository)
+static struct apk_bstream *apk_db_cache_open(struct apk_database *db,
+					     const char *file)
 {
-	struct apk_database *db = _db.db;
-	struct apk_istream *is;
 	char tmp[256];
+
+	if (db->root == NULL)
+		return NULL;
+
+	snprintf(tmp, sizeof(tmp), "%svar/lib/apk/%s", db->root, file);
+	return apk_bstream_from_file(tmp);
+}
+
+static int apk_db_cache_has(struct apk_database *db, const char *file)
+{
+	char tmp[256];
+	struct apk_file_info fi;
+	int r;
+
+	if (db->root == NULL)
+		return FALSE;
+
+	snprintf(tmp, sizeof(tmp), "%svar/lib/apk/%s", db->root, file);
+	r = apk_file_get_info(tmp, &fi);
+	if (r < 0)
+		return FALSE;
+
+	return TRUE;
+}
+
+static struct apk_bstream *apk_repository_file_open(struct apk_repository *repo,
+						    const char *file)
+{
+	char tmp[256];
+
+	snprintf(tmp, sizeof(tmp), "%s/%s", repo->url, file);
+
+	return apk_bstream_from_url(tmp);
+}
+
+int apk_repository_update(struct apk_database *db, struct apk_repository *repo)
+{
+	char tmp[256], tmp2[256];
 	int r;
 
+	if (repo->cache == NULL)
+		return 0;
+
+	apk_message("fetch index %s", repo->url);
+
+	snprintf(tmp, sizeof(tmp), "%s/APK_INDEX.gz", repo->url);
+	snprintf(tmp2, sizeof(tmp2), "%svar/lib/apk/%s.new",
+		 db->root, repo->cache);
+	r = apk_url_download(tmp, tmp2);
+	if (r < 0)
+		return r;
+
+	snprintf(tmp2, sizeof(tmp2), "%svar/lib/apk/%s",
+		 db->root, repo->cache);
+	if (rename(tmp2, tmp) < 0)
+		return -errno;
+
+	return 0;
+}
+
+int apk_db_add_repository(apk_database_t _db, apk_blob_t repository)
+{
+	struct apk_database *db = _db.db;
+	struct apk_istream *is = NULL;
+	char buf[2*sizeof(csum_t)+32], *name;
+	int r, n;
+
 	if (repository.ptr == NULL || *repository.ptr == '\0' 
 			|| *repository.ptr == '#')
 		return 0;
@@ -911,14 +978,37 @@ int apk_db_add_repository(apk_database_t _db, apk_blob_t repository)
 		return -1;
 
 	r = db->num_repos++;
-	db->repos[r] = (struct apk_repository){
-		.url = apk_blob_cstr(repository)
+	db->repos[r] = (struct apk_repository) {
+		.url = apk_blob_cstr(repository),
+		.cache = NULL,
 	};
 
-	snprintf(tmp, sizeof(tmp), "%s/APK_INDEX.gz", db->repos[r].url);
-	is = apk_istream_from_url_gz(tmp);
+	if (apk_url_local_file(db->repos[r].url) == NULL) {
+		csum_ctx_t csum;
+		csum_t res;
+
+		csum_init(&csum);
+		csum_process(&csum, repository.ptr, repository.len);
+		csum_finish(&csum, res);
+
+		n = apk_hexdump_format(sizeof(buf), buf, APK_BLOB_BUF(res)) - 1;
+		snprintf(&buf[n], sizeof(buf) - n, ".index.gz");
+
+		db->repos[r].cache = strdup(buf);
+
+		if (!apk_db_cache_has(db, db->repos[r].cache)) {
+			n = apk_repository_update(db, &db->repos[r]);
+			if (n < 0)
+				return n;
+		}
+		name = db->repos[r].cache;
+		is = apk_bstream_gunzip(apk_db_cache_open(db, db->repos[r].cache), 1);
+	} else {
+		name = "APK_INDEX.gz";
+		is = apk_bstream_gunzip(apk_repository_file_open(&db->repos[r], name), 1);
+	}
 	if (is == NULL) {
-		apk_error("Failed to open index file %s", tmp);
+		apk_error("Failed to open index file %s", name);
 		return -1;
 	}
 	apk_db_index_read(db, is, r);

src/update.c

+/* update.c - Alpine Package Keeper (APK)
+ *
+ * Copyright (C) 2005-2008 Natanael Copa <n@tanael.org>
+ * Copyright (C) 2008 Timo Teräs <timo.teras@iki.fi>
+ * All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it 
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation. See http://www.gnu.org/ for details.
+ */
+
+#include <stdio.h>
+#include "apk_defines.h"
+#include "apk_applet.h"
+#include "apk_database.h"
+#include "apk_version.h"
+
+static int update_main(void *ctx, int argc, char **argv)
+{
+	struct apk_database db;
+	int i;
+
+	if (apk_db_open(&db, apk_root, APK_OPENF_READ) < 0)
+		return -1;
+
+	for (i = 0; i < db.num_repos; i++)
+		apk_repository_update(&db, &db.repos[i]);
+
+	apk_db_close(&db);
+
+	return 0;
+}
+
+static struct apk_applet apk_update = {
+	.name = "update",
+	.usage = "",
+	.main = update_main,
+};
+
+APK_DEFINE_APPLET(apk_update);
+

src/url.c

@@ -13,10 +13,11 @@
 #include <fcntl.h>
 #include <stdlib.h>
 #include <unistd.h>
+#include <sys/wait.h>
 
 #include "apk_io.h"
 
-static const char *url_is_file(const char *url)
+const char *apk_url_local_file(const char *url)
 {
 	if (strncmp(url, "file:", 5) == 0)
 		return &url[5];
@@ -59,8 +60,8 @@ static int fork_wget(const char *url)
 
 struct apk_istream *apk_istream_from_url(const char *url)
 {
-	if (url_is_file(url) != NULL)
-		return apk_istream_from_file(url_is_file(url));
+	if (apk_url_local_file(url) != NULL)
+		return apk_istream_from_file(apk_url_local_file(url));
 
 	return apk_istream_from_fd(fork_wget(url));
 }
@@ -72,9 +73,34 @@ struct apk_istream *apk_istream_from_url_gz(const char *file)
 
 struct apk_bstream *apk_bstream_from_url(const char *url)
 {
-	if (url_is_file(url))
+	if (apk_url_local_file(url))
 		return apk_bstream_from_file(url);
 
 	return apk_bstream_from_fd(fork_wget(url));
 }
 
+int apk_url_download(const char *url, const char *file)
+{
+	pid_t pid;
+	int status;
+
+	pid = fork();
+	if (pid == -1)
+		return -1;
+
+	if (pid == 0) {
+		setsid();
+		dup2(open("/dev/null", O_RDONLY), STDIN_FILENO);
+		execlp("wget", "wget", "-q", "-O", file, url, NULL);
+		exit(0);
+	}
+
+	waitpid(pid, &status, 0);
+	if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
+		unlink(file);
+		return -1;
+	}
+
+	return 0;
+}
+