Commit 12abd7f8 authored by Natanael Copa's avatar Natanael Copa

save ram by parse logfiles line by line

rather than read everything into a big buffer, we open and parse the log
using a file handle. This should reduce the memory usage dramatically.
parent 01fa13ec
......@@ -552,9 +552,9 @@ end
-- ################################################################################
-- LOG FILE FUNCTIONS
local parsesquidlog = function(logdata)
local parsesquidlog = function(f)
local logentries = {}
for line in string.gmatch(logdata, "[^\n]+") do
for line in f:lines() do
-- Format of squid log (space separated):
-- time elapsed remotehost code/status bytes method URL rfc931 peerstatus/peerhost
local words = {}
......@@ -571,9 +571,9 @@ local parsesquidlog = function(logdata)
return logentries
end
local parsedglog = function(logdata)
local parsedglog = function(f)
local logentries = {}
for line in string.gmatch(logdata, "[^\n]+") do
for line in f:lines() do
local words = format.string_to_table(line, "\t")
local logentry = {logdatetime=words[1], clientuserid=words[2], clientip=words[3], URL=words[4], reason=words[5], method=words[6], bytes=words[7], shortreason=words[9]}
if logentry.reason ~= "" then
......@@ -630,27 +630,23 @@ local getlogcandidates = function(source, cookiesfile)
return candidates
end
local getlogfile = function(source, cookiesfile, logfile)
local filecontent
local openlogfile = function(source, cookiesfile, logfile)
local handle
if source.method == "http" or source.method == "https" then
local cmd = "wget -O - --no-check-certificate --load-cookies "..cookiesfile.." --post-data 'name="..logfile.."' '"..source.method.."://"..source.source.."/cgi-bin/acf/alpine-baselayout/logfiles/download' 2>/dev/null"
if string.find(logfile, "%.gz$") then
cmd = cmd.." | gunzip -c"
end
local f = io.popen(cmd)
filecontent = f:read("*a")
f:close()
handle = io.popen(cmd)
elseif source.method == "local" then
if string.find(logfile, "%.gz$") then
local cmd = "gunzip -c "..logfile
local f = io.popen(cmd)
filecontent = f:read("*a")
f:close()
handle = io.popen(cmd)
else
filecontent = fs.read_file(logfile)
handle = io.open(logfile)
end
end
return filecontent
return handle
end
local deletelogfile = function(source, cookiesfile, logfile)
......@@ -853,9 +849,10 @@ end
function importlogfile(source, cookiesfile, file, parselog_func, importlog_func)
logme("Processing " .. file )
logme("Getting " .. file )
logcontent = getlogfile(source, cookiesfile, file)
logentries = parselog_func(logcontent)
importlog(logentries, source.sourcename)
loghandle = openlogfile(source, cookiesfile, file)
logentries = parselog_func(loghandle)
importlog_func(logentries, source.sourcename)
loghandle:close()
logme("Deleting " .. file )
deletelogfile(source, cookiesfile, file)
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment