Commit fe120099 authored by Ted Trask's avatar Ted Trask

Modified import to better handle errors.

Better error reporting for the line that caused the error.
Continue to do the purging even if import fails.
parent 3bcebade
......@@ -855,23 +855,40 @@ end
-- import either squid or dg log file.
-- delete logfile after
local function importlogfile(source, cookiesfile, file, parselog_func, importlog_func)
logme("Processing " .. file )
local tmp
logme("Getting " .. file )
local loghandle = openlogfile(source, cookiesfile, file)
con:execute("START TRANSACTION")
for line in loghandle:lines() do
local logentry = parselog_func(line)
importlog_func(logentry, source.sourcename)
logme("Processing " .. file )
local res, err = pcall(function()
con:execute("START TRANSACTION")
for line in loghandle:lines() do
tmp = line
local logentry = parselog_func(line)
importlog_func(logentry, source.sourcename)
end
con:execute("COMMIT")
end)
if not res then
pcall(function() con:execute("ROLLBACK") end)
if tmp then
pcall(function() logme("Exception on line:"..tmp) end)
end
if err then
pcall(function() logme(err) end)
end
end
con:execute("COMMIT")
loghandle:close()
logme("Deleting " .. file )
deletelogfile(source, cookiesfile, file)
if res then
logme("Deleting " .. file )
deletelogfile(source, cookiesfile, file)
end
return res
end
function importlogs()
local result = cfe({ label="Weblog Import Logs Result" })
local count = 0
local success = true
local res, err = pcall(function()
databaseconnect(DatabaseOwner, config.password)
......@@ -888,10 +905,10 @@ function importlogs()
for j,file in ipairs(files) do
if string.match(file, "dansguardian/access%.log[%.%-]") then
count = count + 1
importlogfile(source, cookiesfile, file, parsedglog, importdglog)
success = importlogfile(source, cookiesfile, file, parsedglog, importdglog) and success
elseif string.match(file, "squid/access%.log[%.%-]") then
count = count + 1
importlogfile(source, cookiesfile, file, parsesquidlog, importsquidlog)
success = importlogfile(source, cookiesfile, file, parsesquidlog, importsquidlog) and success
end
end
end
......@@ -900,23 +917,26 @@ function importlogs()
end
-- Process the logs
addtowatchlist()
updateusagestats()
importpubweblog()
if success then
addtowatchlist()
updateusagestats()
importpubweblog()
groomwatchlist()
end
-- Purge old database entries
groomwatchlist()
groomusagestat()
groomdbhistlog()
groompublogs()
databasedisconnect()
end)
if not res then
if not res or not success then
result.errtxt = "Import Logs Failure"
if err then
pcall(function() logme(err) end)
result.errtxt = result.errtxt .. "\n" .. err
end
pcall(function() databasedisconnect() end)
end
result.value = "Imported "..tostring(count).." logs"
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment