Commit 9df4e8c1 authored by Ted Trask's avatar Ted Trask

Fixed http/https download typo, simplified importing code.

Changed parse and import functions to work on one line at a time, removed iterator functions.
parent 33459f27
......@@ -178,26 +178,22 @@ local listhistorylogentries = function()
return entries
end
local importsquidlog = function(logentries, sourcename)
con:execute("START TRANSACTION")
for entry in logentries do
local importsquidlog = function(entry, sourcename)
if entry then
local sql = string.format("INSERT INTO weblog VALUES ('%s', '%s', '%s', '%s', '%s', '%s')",
escape(sourcename), escape(entry.clientip), escape(entry.clientuserid:lower()),
escape(entry.logdatetime), escape(entry.URL), escape(entry.bytes))
local res = assert (con:execute(sql))
end
con:execute("COMMIT")
end
local importdglog = function(logentries, sourcename)
con:execute("START TRANSACTION")
for entry in logentries do
local importdglog = function(entry, sourcename)
if entry then
local sql = string.format("INSERT INTO blocklog VALUES ('%s', '0.0.0.0', '%s', '%s', '%s', '%s', '%s', '%s', '%s')",
escape(sourcename), escape(entry.clientuserid:lower()), escape(entry.logdatetime), escape(entry.URL),
escape(entry.bytes), escape(entry.reason), escape(entry.score or "0"), escape(entry.shortreason))
local res = assert (con:execute(sql))
end
con:execute("COMMIT")
end
local listsourceentries = function(sourcename)
......@@ -552,7 +548,7 @@ end
-- ################################################################################
-- LOG FILE FUNCTIONS
local function parsesquidlog_line(line)
local function parsesquidlog(line)
-- Format of squid log (space separated):
-- time elapsed remotehost code/status bytes method URL rfc931 peerstatus/peerhost
local words = {}
......@@ -572,51 +568,27 @@ local function parsesquidlog_line(line)
peerhost=string.match(words[9], "[^/]*$")}
logentry.logdatetime = os.date("%Y-%m-%d %H:%M:%S", logentry.logdatetime)..string.match(logentry.logdatetime, "%..*")
return logentry
end
local function parsesquidlog_iter(f)
return function()
while true do
line = f:read("*line")
if line == nil then
return nil
end
local logentry = parsesquidlog_line(line)
-- Don't care about local requests (from DG)
if logentry.clientip ~= "127.0.0.1" then
return logentry
end
end
-- Don't care about local requests (from DG)
if logentry.clientip ~= "127.0.0.1" then
return logentry
end
return nil
end
local function parsedglog_line(line)
local function parsedglog(line)
local words = format.string_to_table(line, "\t")
return { logdatetime=words[1], clientuserid=words[2], clientip=words[3],
local logentry = { logdatetime=words[1], clientuserid=words[2], clientip=words[3],
URL=words[4], reason=words[5], method=words[6], bytes=words[7],
shortreason=words[9]}
end
local function parsedglog_iter(f)
return function()
while true do
line = f:read("*line")
if line == nil then
return nil
end
local logentry = parsedglog_line(line)
if logentry.reason ~= "" then
if logentry.shortreason == "" then
logentry.shortreason = logentry.reason
end
logentry.score = string.match(logentry.reason, "^.*: ([0-9]+) ")
logentry.logdatetime = string.gsub(logentry.logdatetime, "%.", "-")
return logentry
end
if logentry.reason ~= "" then
if logentry.shortreason == "" then
logentry.shortreason = logentry.reason
end
logentry.score = string.match(logentry.reason, "^.*: ([0-9]+) ")
logentry.logdatetime = string.gsub(logentry.logdatetime, "%.", "-")
return logentry
end
return nil
end
-- ################################################################################
......@@ -878,13 +850,16 @@ end
-- import either squid or dg log file.
-- delete logfile after
function importlogfile(source, cookiesfile, file, parselog_iter, importlog_func)
local logentries
local function importlogfile(source, cookiesfile, file, parselog_func, importlog_func)
logme("Processing " .. file )
logme("Getting " .. file )
loghandle = openlogfile(source, cookiesfile, file)
logentries = parselog_iter(loghandle)
importlog_func(logentries, source.sourcename)
local loghandle = openlogfile(source, cookiesfile, file)
con:execute("START TRANSACTION")
for line in loghandle:lines() do
local logentry = parselog_func(line)
importlog_func(logentry, source.sourcename)
end
con:execute("COMMIT")
loghandle:close()
logme("Deleting " .. file )
deletelogfile(source, cookiesfile, file)
......@@ -909,10 +884,10 @@ function importlogs()
for j,file in ipairs(files) do
if string.match(file, "dansguardian/access%.log[%.%-]") then
count = count + 1
importlogfile(source, cookeisfile, file, parsedglog_iter, importdglog)
importlogfile(source, cookiesfile, file, parsedglog, importdglog)
elseif string.match(file, "squid/access%.log[%.%-]") then
count = count + 1
importlogfile(source, cookeisfile, file, parsesquidlog_iter, importsquidlog)
importlogfile(source, cookiesfile, file, parsesquidlog, importsquidlog)
end
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment