Commit 4640b887 authored by Ted Trask's avatar Ted Trask

No need to have four duplicate importlogentry functions

parent 5f39a2dc
......@@ -31,8 +31,8 @@ local database_creation_script = {
"CREATE TABLE pubweblog(sourcename character varying(40), clientip inet NOT NULL, clientuserid character varying(64) NOT NULL, logdatetime timestamp(3) without time zone NOT NULL, uri text NOT NULL, bytes bigint NOT NULL, reason text, score integer, shortreason text, badyesno int, deniedyesno int, bypassyesno int, wordloc text, goodwordloc text, selected boolean, id serial)",
"CREATE TABLE pubweblog_history(sourcename character varying(40), clientip inet NOT NULL, clientuserid character varying(64) NOT NULL, logdatetime timestamp(3) without time zone NOT NULL, uri text NOT NULL, bytes bigint NOT NULL, reason text, score integer, shortreason text, badyesno int, deniedyesno int, bypassyesno int, wordloc text, goodwordloc text, selected boolean, id int)",
"CREATE TABLE source (sourcename character varying(40) NOT NULL, method character varying(100) NOT NULL, userid character varying(32), passwd character varying(255), source character varying(255) NOT NULL, tzislocal boolean, enabled boolean)",
"CREATE TABLE usagestat(sourcename character varying(40) NOT NULL, date timestamp(0) without time zone NOT NULL, numrequest integer, numblock integer)",
"CREATE TABLE watchlist(clientuserid character varying(64) NOT NULL, expiredatetime timestamp(0) without time zone NOT NULL)",
"CREATE TABLE usagestat (sourcename character varying(40) NOT NULL, date timestamp(0) without time zone NOT NULL, numrequest integer, numblock integer)",
"CREATE TABLE watchlist (clientuserid character varying(64) NOT NULL, expiredatetime timestamp(0) without time zone NOT NULL)",
"ALTER TABLE ONLY source ADD CONSTRAINT source_pkey PRIMARY KEY (sourcename)",
"CREATE INDEX dbhistlogdatetimeidx ON dbhistlog USING btree (logdatetime)",
"CREATE INDEX pubweblogclientdateidx ON pubweblog USING btree (logdatetime, clientuserid)",
......@@ -179,42 +179,16 @@ local listhistorylogentries = function()
return entries
end
local importsquidlog = function(entry, sourcename)
local importlogentry = function(entry, sourcename)
if entry then
local sql = string.format("INSERT INTO pubweblog VALUES ('%s', '%s', '%s', '%s', '%s', '%s','%s','%s','%s','%s','%s','%s','%s')",
escape(sourcename), escape(entry.clientip), escape(entry.clientuserid, 64):lower(),
escape(entry.logdatetime), escape(entry.URL), escape(entry.bytes), escape(entry.reason), escape(entry.score), escape(entry.shortreason), escape(entry.badyesno), escape(entry.deniedyesno), escape(entry.bypassyesno), escape(entry.wordloc), escape(entry.goodwordloc))
local res = assert (con:execute(sql))
end
end
local importsquarklog = function(entry, sourcename)
if entry then
local sql = string.format("INSERT INTO pubweblog VALUES ('%s', '%s', '%s', '%s', '%s', '%s','%s','%s','%s','%s','%s','%s','%s')",
escape(sourcename), escape(entry.clientip), escape(entry.clientuserid, 64):lower(),
escape(entry.logdatetime), escape(entry.URL), escape(entry.bytes), escape(entry.reason), escape(entry.score), escape(entry.shortreason), escape(entry.badyesno), escape(entry.deniedyesno), escape(entry.bypassyesno), escape(entry.wordloc), escape(entry.gwordloc))
local res = assert (con:execute(sql))
end
end
local importdglog = function(entry, sourcename)
if entry then
local sql = string.format("INSERT INTO pubweblog VALUES ('%s', '%s', '%s', '%s', '%s', '%s','%s','%s','%s','%s','%s','%s','%s')",
escape(sourcename), escape(entry.clientip), escape(entry.clientuserid, 64):lower(),
escape(entry.logdatetime), escape(entry.URL), escape(entry.bytes), escape(entry.reason), escape(entry.score), escape(entry.shortreason), escape(entry.badyesno), escape(entry.deniedyesno), escape(entry.bypassyesno), escape(entry.wordloc), escape(entry.gwordloc))
local res = assert (con:execute(sql))
end
end
local importdumplog = function(entry, sourcename)
if entry then
local sql = string.format("INSERT INTO pubweblog VALUES ('%s', '%s', '%s', '%s', '%s', '%s','%s','%s','%s','%s','%s','%s','%s')",
escape(sourcename), escape(entry.clientip), escape(entry.clientuserid, 64):lower(),
escape(entry.logdatetime), escape(entry.URL), escape(entry.bytes), escape(entry.reason), escape(entry.score), escape(entry.shortreason), escape(entry.badyesno), escape(entry.deniedyesno), escape(entry.bypassyesno), escape(entry.wordloc), escape(entry.gwordloc))
local res = assert (con:execute(sql))
end
end
local listsourceentries = function(sourcename)
local sources = {}
-- retrieve a cursor
......@@ -1194,9 +1168,8 @@ function testsource(sourcename)
return result
end
-- import either squid or dg log file.
-- delete logfile after
local function importlogfile(source, cookiesfile, file, parselog_func, importlog_func)
-- import a logfile and delete logfile after
local function importlogfile(source, cookiesfile, file, parselog_func)
logme("Getting " .. file )
local loghandle = openlogfile(source, cookiesfile, file)
logme("Processing " .. file )
......@@ -1206,7 +1179,7 @@ local function importlogfile(source, cookiesfile, file, parselog_func, importlog
assert(con:execute("SAVEPOINT before_line"))
local res2, err2 = pcall(function()
local logentry = parselog_func(line)
importlog_func(logentry, source.sourcename)
importlogentry(logentry, source.sourcename)
end)
if not res2 then
if (config.stoponerror == "true") then
......@@ -1275,19 +1248,19 @@ function importlogs()
for j,file in ipairs(files) do
if string.match(file, "dansguardian/access%.log[%.%-]") then
count = count + 1
success = importlogfile(source, cookiesfile, file, parsedglog, importdglog) and success
success = importlogfile(source, cookiesfile, file, parsedglog) and success
end
if string.match(file, "squark/access%.log[%.%-]") then
count = count + 1
success = importlogfile(source, cookiesfile, file, parsesquarklog, importsquarklog) and success
success = importlogfile(source, cookiesfile, file, parsesquarklog) and success
end
if string.match(file, "squid/access%.log[%.%-]") then
count = count + 1
success = importlogfile(source, cookiesfile, file, parsesquidlog, importsquidlog) and success
success = importlogfile(source, cookiesfile, file, parsesquidlog) and success
end
if string.match(file, "dump/access%.log[%.%-]") then
count = count + 1
success = importlogfile(source, cookiesfile, file, parsedumplog, importdumplog) and success
success = importlogfile(source, cookiesfile, file, parsedumplog) and success
end
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment