Commit c979575f authored by Jakub Jirutka's avatar Jakub Jirutka

Add integration with Anitya - automatically flag outdated packages

parent 239a396b
......@@ -6,7 +6,7 @@ This application makes use of the [Turbo](http://turbolua.org) (Lua) framework.
On Alpine Linux it should be enough to install turbo and deps by:
apk add luajit lua5.1 lua-turbo lua-sqlite lua-lustache lua-socket
apk add luajit lua5.1 lua-turbo lua-sqlite lua-lustache lua-socket lua-lzmq lua-gversion
Copy config.sample.lua to config.lua and edit it.
You can start the application by starting ./aports.lua or on Alpine Linux with turbo's init.d (see conf.d/turbo for settings)
......
---------
-- Common module for anitya-check-all and anitya-watch with integration
-- to aports.
local gversion = require 'gversion'
local log = require 'turbo.log'
local conf = require 'config'
local db = require 'db'
local cntrl = require 'controller'
local format = string.format
local normalize_version = gversion.normalize
local parse_version = gversion.parse
local pre_suffixes = {'alpha', 'beta', 'pre', 'rc'}
gversion.set_suffixes(pre_suffixes, {'cvs', 'svn', 'git', 'hg', 'p'})
--- Flags `pkg` as outdated and send email to its maintainer.
--
-- @tparam table pkg The package table as from `db:getPackage`.
-- @tparam string new_version
local function flag_package(pkg, new_version)
local flag_fields = {
from = conf.mail.from,
new_version = new_version,
message = conf.anitya.flag_message
}
assert(db:flagOrigin(flag_fields, pkg),
'Failed to flag package: '..pkg.origin)
assert(cntrl:flagMail(flag_fields, pkg))
end
--- Compares the current version with new version and returns true if the
-- current version is outdated.
--
-- This function has special handling of pre-release versions. If `new_ver` is
-- higher than `curr_ver`, but contains pre-release suffix lower than the
-- `curr_ver`, then it returns false. The point is to ignore e.g. beta releases
-- unless the current version is also beta or lower (alpha).
--
-- @tparam gversion.Version curr_ver The current version.
-- @tparam gversion.Version new_ver The new version.
-- @treturn boolean
local function is_outdated(curr_ver, new_ver)
if new_ver <= curr_ver then
return false
end
local allow_pre = false
for _, suffix in pairs(pre_suffixes) do
if curr_ver[suffix] then
allow_pre = true
end
if new_ver[suffix] then
return allow_pre
end
end
return true
end
local M = {}
--- Yields names of aports (i.e. origin packages) in the specified branch.
-- If the database is not opened, then it opens it and close after finish.
function M.each_aport_name(branch)
local close_db = db:open()
local stmt = db:raw_db():prepare [[
SELECT DISTINCT origin
FROM packages
WHERE branch = ?
]]
stmt:bind_values(branch)
return coroutine.wrap(function()
for name in stmt:urows() do
coroutine.yield(name)
end
stmt:finalize()
if close_db then db:close() end
end)
end
--- Flags packages with origin `origin_name` in the edge branch that have older
-- version than `new_ver`, are not flagged yet or the flag specifies an older
-- version than `new_ver`.
--
-- If the database is not opened, then it opens it and close after finish.
--
-- @tparam string origin_name
-- @tparam string new_ver
-- @raise Error if `new_ver` or current version cannot be parsed, if fail to
-- flag a package or send email to maintainer.
function M.flag_outdated_pkgs(origin_name, new_ver)
new_ver = assert(parse_version(normalize_version(new_ver)),
'Malformed new version: '..new_ver)
local close_db = db:open()
local stmt = db:raw_db():prepare [[
SELECT DISTINCT p.version, f.new_version
FROM packages p
LEFT JOIN flagged f ON f.fid = p.fid
WHERE p.origin = ? AND p.branch = ?
]]
stmt:bind_values(origin_name, 'edge')
local finalize = function()
stmt:finalize()
if close_db then db:close() end
end
for curr_ver_s, flag_ver_s in stmt:urows() do
local curr_ver = parse_version(curr_ver_s)
if not curr_ver then
finalize()
error('Malformed current version: '..curr_ver_s)
end
local flag_ver = parse_version(normalize_version(flag_ver_s or ''))
if (not flag_ver or new_ver > flag_ver) and is_outdated(curr_ver, new_ver) then
local pkg = db:getPackage { origin = origin_name, version = curr_ver_s }
log.notice(format('Flagging package %s-%s, new version is %s',
pkg.origin, pkg.version, new_ver))
local ok, err = pcall(flag_package, pkg, tostring(new_ver))
if not ok then
finalize()
error(err, 2)
end
end
end
finalize()
end
return M
local ioloop = require 'turbo.ioloop'
local M = {}
--- Runs function for each value concurrently with optional limit
-- of concurrent tasks.
--
-- @tparam function iterator The iterator that yields values to be passed
-- into `func`.
-- @tparam function func The function to run.
-- @tparam int limit Maximum number of tasks to run concurrently.
function M.foreach(iterator, func, limit)
local io = ioloop.instance()
local scheduled = 1 -- number of scheduled tasks (callbacks)
io:add_callback(function()
for value in iterator do
-- Do not schedule more callbacks than the limit.
while limit ~= nil and scheduled > limit do
coroutine.yield()
end
io:add_callback(function()
local ok, res = pcall(func, value)
scheduled = scheduled - 1
if not ok then error(res, 2) end
end)
scheduled = scheduled + 1
end
scheduled = scheduled - 1
end)
-- Close IO loop when all work is done.
io:set_interval(250, function()
if scheduled == 0 then
io:close()
end
end)
io:start()
end
return M
......@@ -84,5 +84,25 @@ config.cache.clear = true
config.cache.path = "/var/lib/nginx/cache"
-- the max depth to traverse (can be set by nginx cache settings)
config.cache.depth = 3
----
---- settings for anitya (https://release-monitoring.org)
----
config.anitya = {}
-- name of the distribution on anitya
config.anitya.distro = "Alpine"
-- base uri of the anitya restful api
config.anitya.api_uri = "https://release-monitoring.org/api"
-- number of http requests to send concurrently
config.anitya.api_concurrency = 20
-- uri of the anitya fedmsg/zeromq interface
config.anitya.fedmsg_uri = "tcp://release-monitoring.org:9940"
-- text of the message to be sent to maintainer of an outdated package
config.anitya.flag_message = [[
This package has been flagged automatically on the basis of notification from
Anitya <https://release-monitoring.org/>.
Please note that integration with Anitya is in experimental phase. If the
provided information is incorrect, please let us know on IRC or write directly
to jakub@jirutka.cz. Thanks!]]
return config
......@@ -117,7 +117,7 @@ function cntrl:flagMail(args, pkg)
mail:set_subject(subject)
local body = lustache:render(self:tpl("mail_body.tpl"), m)
mail:set_body(body)
mail:send()
return mail:send()
end
function cntrl:flagged(args)
......
......@@ -7,13 +7,22 @@ local model = require('model')
local db = class('db')
function db:open()
self.db = sqlite3.open(conf.db.path)
if self.db and self.db:isopen() then
return false
else
self.db = assert(sqlite3.open(conf.db.path))
return true
end
end
function db:close()
self.db:close()
end
function db:raw_db()
return self.db
end
function db:getDistinct(tbl,col)
local r = {}
local sql = string.format("SELECT DISTINCT %s from %s", col, tbl)
......
#!/usr/bin/env luajit
---------
-- This script checks versions of all packages in the edge branch on Anitya
-- <https://release-monitoring.org/> and flags outdated packages. It should
-- be run for the first check or after outages. For periodic checks use
-- anitya-watch.
--
-- ## How does it work
--
-- It iterates over abuild names (i.e. origin packages) in the edge branch and
-- requests Anitya's resource `GET /project/{distro}/{pkgname}` for each
-- pkgname and the configured distro. If a project with the distro's mapping
-- is found, then it takes version of the latest release from the response.
-- Then looks into the aports database; if there's an older version that is not
-- flagged yet, or the flag contains older new version, then it flags the
-- package and sends email to its maintainer.
--
TURBO_SSL = true
local _ = require 'turbo'
local async = require 'turbo.async'
local escape = require 'turbo.escape'
local log = require 'turbo.log'
local aports = require 'anitya_aports'
local cc = require 'concurrent'
local conf = require 'config'
local HTTPClient = async.HTTPClient
local json_decode = escape.json_decode
local yield = coroutine.yield
local anitya_distro_pkg_uri = ("%s/project/%s/%s"):format(
conf.anitya.api_uri, conf.anitya.distro, '%s')
--- Gets project from Anitya by the given distro's package name and returns
-- response as a table.
local function fetch_distro_pkg(pkgname)
local url = anitya_distro_pkg_uri:format(pkgname)
local res = yield(HTTPClient():fetch(url))
if res.error then
error(res.error)
elseif res.code == 200 then
return json_decode(res.body)
end
end
-------- M a i n --------
log.notice 'Checking outdated packages using Anitya...'
cc.foreach(aports.each_aport_name('edge'), function(pkgname)
local proj = fetch_distro_pkg(pkgname)
if proj and proj.version then
log.debug(("Found %s %s"):format(pkgname, proj.version))
aports.flag_outdated_pkgs(pkgname, proj.version)
else
log.debug('Did not find '..pkgname)
end
end, conf.anitya.api_concurrency)
log.success 'Completed'
#!/usr/bin/env luajit
---------
-- This script connects to fedmsg/zeromq interface of Anitya
-- <https://release-monitoring.org> and flags packages in the edge branch
-- for which a new version is released. It should be run as a daemon. Before
-- first run or after an outage run anitya-check-all to check all packages.
--
-- ## How does it work
--
-- Anitya watches registered projects and when a new release is found, then it
-- sends a message via fedmsg. This scripts consumes these messages. If the
-- updated project contains mapping for the configured distro, then it takes
-- distro's package name and new version from the message. Then looks into the
-- aports database; if there's an older version that is not flagged yet, or the
-- flag contains older new version, then it flags the package and sends email
-- to its maintainer.
--
local log = require 'turbo.log'
local escape = require 'turbo.escape'
local zmq = require 'lzmq'
local zpoller = require 'lzmq.poller'
local aports = require 'anitya_aports'
local conf = require 'config'
local utils = require 'utils'
local get = utils.get
local distro = conf.anitya.distro
local json_decode = escape.json_decode
--- Receives multipart message and returns decoded JSON payload.
local function receive_json_msg(sock)
local resp, err = sock:recv_multipart()
if err then
return nil, err
end
log.debug('Received message from topic '..resp[1])
local ok, res = pcall(json_decode, resp[2])
if not ok then
return nil, 'Failed to parse message as JSON: '..res
end
return res
end
--- Handles message from topic `anitya.project.map.new`; if it's mapping for
-- our distro, then flags the package if it's outdated.
local function handle_map_new(msg)
if get(msg, 'distro.name') ~= distro then
return nil
end
local pkgname = get(msg, 'message.new')
local version = get(msg, 'project.version')
if pkgname and version then
log.info(("Received version update: %s %s"):format(pkgname, version))
aports.flag_outdated_pkgs(pkgname, version)
end
end
--- Handles message from topic `anitya.project.version.update`; if the project
-- contains mapping for our distro, then flags the package if it's outdated.
local function handle_version_update(msg)
local pkgname = nil
for _, pkg in ipairs(get(msg, 'message.packages') or {}) do
if pkg.distro == distro then
pkgname = pkg.package_name
break
end
end
local version = get(msg, 'message.upstream_version')
if pkgname and version then
log.info(("Received version update: %s %s"):format(pkgname, version))
aports.flag_outdated_pkgs(pkgname, version)
end
end
-------- M a i n --------
local sock, err = zmq.context():socket(zmq.SUB, {
connect = conf.anitya.fedmsg_uri
})
zmq.assert(sock, err)
local handlers = {
['org.release-monitoring.prod.anitya.project.version.update'] = handle_version_update,
['org.release-monitoring.prod.anitya.project.map.new'] = handle_map_new,
}
for topic, _ in pairs(handlers) do
sock:subscribe(topic)
end
local poller = zpoller.new(1)
poller:add(sock, zmq.POLLIN, function()
local payload, err = receive_json_msg(sock)
if err then
log.error('Failed to receive message from fedmsg: '..err)
end
local ok, err = pcall(handlers[payload.topic], payload.msg)
if not ok then
log.error(err)
end
end)
log.notice('Connecting to '..conf.anitya.fedmsg_uri)
poller:start()
......@@ -24,6 +24,21 @@ function M.escape_uri(str)
:gsub(' ', '+')
end
--- Returns a value from (nested) table at the specified path.
--
-- @tparam table tab The table to operate on.
-- @tparam string path A dot separated sequence of fields.
-- @treturn A value of the last field specified in `path`, or `nil` if some
-- field doesn't exist in `tab`.
function M.get(tab, path)
local res = tab
for field in string.gmatch(path, '[^%.]+') do
if res == nil then return nil end
res = res[field]
end
return res
end
--- Formats email address as "Display Name <address@domain.tld>" or
-- "address@domain.tld" if `display_name` is empty.
function M.format_email_addr(display_name, email)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment