Commit a1225776 authored by Ted Trask's avatar Ted Trask

Started cleanup, not complete

parent 28804449
......@@ -96,7 +96,7 @@ function downloadselected(self)
end
function checkselected(self)
self.conf.viewtype = "silent"
-- TT self.conf.viewtype = "silent"
return self.model.editselected(self.clientdata.chkdata)
--return file
end
......@@ -109,6 +109,18 @@ function viewselected(self)
return self.model.getweblog(self.clientdata.activelog, self.clientdata.clientuserid, self.clientdata.starttime, self.clientdata.endtime, self.clientdata.clientip, self.clientdata.badyesno, self.clientdata.deniedyesno, self.clientdata.bypassyesno, self.clientdata.score, self.clientdata.urisearch, self.clientdata.sortby, self.clientdata.getselected, clientdata.focus)
end
function viewusagestats(self)
return self.model.getusagestats()
end
function viewauditstats(self)
return self.model.getauditstats()
end
function completeaudit(self)
return self:redirect_to_referrer(self.model.completeaudit(self.clientdata.auditend))
end
function adhocquery(self)
return controllerfunctions.handle_form(self, self.model.getnewadhocquery, self.model.adhocquery, self.clientdata, "Submit", "Submit ad-hoc query")
end
......
weblog-editsource-html.lsp
\ No newline at end of file
<% local form, viewlibrary, page_info = ...
require("viewfunctions")
%>
<H1><%= html.html_escape(form.label) %></H1>
<%
form.action = page_info.script .. page_info.prefix .. page_info.controller .. "/" .. page_info.action
form.value.passwd.type = "password"
if page_info.action == "editsource" then
form.value.sourcename.readonly = true
end
local order = {"sourcename", "enabled", "source", "method", "userid", "passwd"}
displayform(form, order)
%>
../filedetails-html.lsp
\ No newline at end of file
<% local form, viewlibrary, page_info = ... %>
<% require("viewfunctions") %>
<% if form.type == "form" then %>
<H1>Configuration</H1>
<H2>Expert Configuration</H2>
<% else %>
<H1>View File</H1>
<% end %>
<H3>File Details</H3>
<DL>
<%
displayitem(form.value.filename)
displayitem(form.value.filesize)
displayitem(form.value.mtime)
%>
</DL>
<H3>File Content</H3>
<% if form.type == "form" then %>
<% form.action = page_info.script .. page_info.prefix .. page_info.controller .. "/" .. page_info.action %>
<% displayformstart(form) %>
<input type="hidden" name="filename" value="<%= html.html_escape(form.value.filename.value) %>">
<% else %>
<DL>
<% end %>
<textarea name="filecontent">
<%= html.html_escape(form.value.filecontent.value) %>
</textarea>
<% if form.value.filecontent.errtxt then %><P CLASS='error'><%= string.gsub(html.html_escape(form.value.filecontent.errtxt), "\n", "<BR>") %></P><% end %>
<% if form.value.filecontent.descr then %><P CLASS='descr'><%= string.gsub(html.html_escape(form.value.filecontent.descr), "\n", "<BR>") %></P><% end %>
<% if form.type == "form" then %>
<% displayformend(form) %>
<% else %>
</DL>
<% end %>
......@@ -8,7 +8,7 @@ require("validator")
require("luasql.postgres")
require("date")
local DatabaseName = "webproxylog"
local DatabaseName = "weblog"
local DatabaseOwner = "weblogowner"
local DatabaseUser = "webloguser"
......@@ -133,7 +133,7 @@ local runSQLscript = function(filename)
end
-- Create the database and tables
-- pg_dump -U postgres -c webproxylog > makeweblog.postgres
-- pg_dump -U postgres -c weblog > makeweblog.postgres
--runSQLscript("/root/work/weblog/makeweblog.postgres")
local databaseconnect = function(username, password)
......@@ -180,10 +180,20 @@ local importsquidlog = function(entry, sourcename)
local sql = string.format("INSERT INTO pubweblog VALUES ('%s', '%s', '%s', '%s', '%s', '%s','%s','%s','%s','%s','%s','%s','%s')",
escape(sourcename), escape(entry.clientip), escape(entry.clientuserid, 64):lower(),
escape(entry.logdatetime), escape(entry.URL), escape(entry.bytes), escape(entry.reason), escape(entry.score), escape(entry.shortreason), escape(entry.badyesno), escape(entry.deniedyesno), escape(entry.bypassyesno), escape(entry.wordloc), escape(entry.goodwordloc))
local res = assert (con:execute(sql))
end
end
local importsquarklog = function(entry, sourcename)
if entry then
local sql = string.format("INSERT INTO pubweblog VALUES ('%s', '%s', '%s', '%s', '%s', '%s','%s','%s','%s','%s','%s','%s','%s')",
escape(sourcename), escape(entry.clientip), escape(entry.clientuserid, 64):lower(),
escape(entry.logdatetime), escape(entry.URL), escape(entry.bytes), escape(entry.reason), escape(entry.score), escape(entry.shortreason), escape(entry.badyesno), escape(entry.deniedyesno), escape(entry.bypassyesno), escape(entry.wordloc), escape(entry.gwordloc))
local res = assert (con:execute(sql))
end
end
local importdglog = function(entry, sourcename)
if entry then
local sql = string.format("INSERT INTO pubweblog VALUES ('%s', '%s', '%s', '%s', '%s', '%s','%s','%s','%s','%s','%s','%s','%s')",
......@@ -192,6 +202,14 @@ local importdglog = function(entry, sourcename)
local res = assert (con:execute(sql))
end
end
local importdumplog = function(entry, sourcename)
if entry then
local sql = string.format("INSERT INTO pubweblog VALUES ('%s', '%s', '%s', '%s', '%s', '%s','%s','%s','%s','%s','%s','%s','%s')",
escape(sourcename), escape(entry.clientip), escape(entry.clientuserid, 64):lower(),
escape(entry.logdatetime), escape(entry.URL), escape(entry.bytes), escape(entry.reason), escape(entry.score), escape(entry.shortreason), escape(entry.badyesno), escape(entry.deniedyesno), escape(entry.bypassyesno), escape(entry.wordloc), escape(entry.gwordloc))
local res = assert (con:execute(sql))
end
end
local listsourceentries = function(sourcename)
local sources = {}
......@@ -301,6 +319,21 @@ local groompublogs = function()
res = assert (con:execute(sql))
logme("Deleted " .. res .. " old records to from pubweblog")
-- purge anything older than startddate+historydays+watchdays
--local temp = config.auditstart
--if not temp or temp == "" then temp = os.date("%Y-%m-%d %H:%M:%S") end
--logme("Purge date since last audit is " .. tostring(watchdays+historydays) .. " days before " .. temp .. ".")
--sql = "delete from pubweblog where logdatetime < (timestamp '"..temp.."' - INTERVAL '"..tostring(watchdays+historydays).." days')"
--res = assert (con:execute(sql))
--logme("removed " .. res .. " old pubweblog records that are older than history+watchdays")
-- purge good people after historydays
--logme("The delete date for non-watchlist users is " .. tostring(historydays) .. " days before " .. temp .. ".")
--sql = "delete from pubweblog where logdatetime < (timestamp '".. temp.."' - INTERVAL '"..tostring(historydays).." days') and clientuserid NOT IN (select clientuserid from watchlist)"
--res = assert (con:execute(sql))
--logme("removed " .. res .. " records for users not on the watchlist.")
end
local listwatchlistentries = function()
......@@ -475,7 +508,6 @@ local printtableentries = function(tablename)
end
-- ################################################################################
-- LOG FILE FUNCTIONS
local function parsesquidlog(line)
......@@ -501,8 +533,7 @@ local function parsesquidlog(line)
if not thisline then
break
end
_,instcnt = string.lower(words[7]):gsub(format.escapespecialcharacters(thisline), " ")
--if string.find(words[7],thisline) ~= nil then
_,instcnt = string.lower(words[7]):gsub(thisline, " ")
if instcnt ~= 0 then
ignoreme = true
break
......@@ -510,14 +541,16 @@ local function parsesquidlog(line)
end
if ignoreme ~= true then
--proceed with record analysis, badwords first
--proceed with record analysis
for thisline in io.lines("/etc/weblog/badwords") do
if not thisline then
break
end
_,instcnt = string.lower(words[7]):gsub(format.escapespecialcharacters(thisline), " ")
_,instcnt = string.lower(words[7]):gsub(thisline, " ")
--if string.find(words[7],thisline) ~= nil then
if instcnt ~= 0 then
-- logme("instcnt = "..instcnt)
isbad=1
wrdcnt= wrdcnt + instcnt
if badwordloc ~= "" then
......@@ -528,25 +561,22 @@ local function parsesquidlog(line)
end
if (words[11] and words[11]~= nil and string.find(words[11],"," )) then
--logme("squid says "..words[11])
prxarray = split(words[11],",")
for r,s in pairs(prxarray) do
if string.find(s,"blocked") then
isdenied=1
elseif string.find(s,"overridden") then
isbypass=1
end
end
end
if string.find(words[7],"*DENIED*") then
logme("*Denied*")
isdenied=1
elseif string.find(words[7],"GBYPASS") then
logme("GBYPASS")
isbypass=1
elseif string.find(words[7],"*OVERRIDE*") then
logme("*OVERRIDE*")
isbypass=1
end
end
--and now a good words search in mitigation of severity
for goodline in io.lines("/etc/weblog/goodwords") do
if not goodline then
break
end
_,instcnt = string.lower(words[7]):gsub(format.escapespecialcharacters(goodline), " ")
_,instcnt = string.lower(words[7]):gsub(goodline, " ")
--if string.find(words[7],goodline) then
if instcnt ~= 0 then
if wrdcnt ~= 0 then
......@@ -560,19 +590,18 @@ local function parsesquidlog(line)
end
end
end
if (r and r~=nil) then
reason=r
else
reason=words[6]
end
-- Reset bad to reduce number of bad hits if score is zero
-- if wrdcnt == 0 then
-- isbad=0
-- end
local logentry = {logdatetime=words[1],
elapsed=words[2],
clientip=words[3],
code=string.match(words[4] or "", "^[^/]*"),
status=string.match(words[4] or "", "[^/]*$"),
bytes=words[5],
method=reason,
method=words[6],
URL=words[7],
clientuserid=words[8],
peerstatus=string.match(words[9] or "", "^[^/]*"),
......@@ -594,6 +623,109 @@ local function parsesquidlog(line)
return nil
end
local function parsesquarklog(line)
-- Format of squid log (space separated):
-- time elapsed remotehost code/status bytes method URL rfc931 peerstatus/peerhost
local words = {}
for word in string.gmatch(line, "%S+") do
words[#words+1] = word
end
local goodwordloc=""
local badwordloc=""
local wrdcnt=0
local isbad=0
local isdenied=0
local isbypass=0
local ignoreme=false
--check for ignored records first
for thisline in io.lines("/etc/weblog/ignorewords") do
if not thisline then
break
end
_,instcnt = string.lower(words[5]):gsub(thisline, " ")
if instcnt ~= 0 then
ignoreme = true
break
end
end
if ignoreme ~= true then
for thisline in io.lines(baseurl.."badwords") do
if not thisline then
-- logme("This line is apparently empty...")
break
end
_,instcnt = string.lower(words[5]):gsub(thisline, " ")
-- if string.find(words[5],thisline) ~= nil then
--logme("checking "..thisline.." against "..words[5])
if instcnt ~= 0 then
isbad=1
wrdcnt = wrdcnt + instcnt
if badwordloc ~= "" then
badwordloc = badwordloc.."|"..thisline
else
badwordloc=thisline
end
-- logme("bad "..badwordloc)
end
if string.find(words[6],"*DENIED*") then
isdenied=1
end
if string.find(words[6],"*OVERRIDE*") then
isbypass=1
end
end
for goodline in io.lines(baseurl.."goodwords") do
if not goodline then
-- logme("This line is apparently empty...")
break
end
_,instcnt = string.lower(words[5]):gsub(goodline, " ")
--if string.find(words[4],goodline) then
if instcnt ~= 0 then
if wrdcnt ~= 0 then
wrdcnt = wrdcnt - instcnt
if goodwordloc ~= "" then
goodwordloc = goodwordloc.."|"..goodline
else
goodwordloc = goodline
end
end
end
end
end
local words = format.string_to_table(line, "\t")
local logentry = {logdatetime=words[1],
clientuserid=words[2],
clientip=words[3],
URL=words[4],
reason=words[5],
method=words[6],
bytes=words[7],
shortreason=words[9],
score=wrdcnt,
badyesno=isbad,
deniedyesno=isdenied,
bypassyesno=isbypass,
wordloc=badwordloc,
gwordloc=goodwordloc}
if logentry.reason and logentry.reason ~= "" then
if logentry.shortreason == "" then
logentry.shortreason = logentry.reason
end
return logentry
end
return nil
end
local function parsedglog(line)
-- Format of squid log (space separated):
-- time elapsed remotehost code/status bytes method URL rfc931 peerstatus/peerhost
......@@ -617,7 +749,7 @@ local function parsedglog(line)
if not thisline then
break
end
_,instcnt = string.lower(words[4]):gsub(format.escapespecialcharacters(thisline), " ")
_,instcnt = string.lower(words[4]):gsub(thisline, " ")
if instcnt ~= 0 then
ignoreme = true
break
......@@ -700,6 +832,107 @@ local function parsedglog(line)
return nil
end
local function parsedumplog(line)
-- Format of squid log (space separated):
-- time elapsed remotehost code/status bytes method URL rfc931 peerstatus/peerhost
local words = {}
for word in string.gmatch(line, "%S+") do
words[#words+1] = word
end
goodwordloc=""
badwordloc=""
wrdcnt=0
isbad=0
isdenied=0
isbypass=0
for thisline in io.lines("/etc/weblog/badwords") do
if not thisline then
logme("This line is apparently empty...")
break
end
_,instcnt = string.lower(words[5]):gsub(thisline, " ")
if instcnt ~= 0 then
isbad=1
wrdcnt = wrdcnt + instcnt
if badwordloc ~= "" then
badwordloc = badwordloc.."|"..thisline
else
badwordloc=thisline
end
-- logme("bad "..badwordloc)
end
if string.find(words[6],"*DENIED*") then
isdenied=1
end
if string.find(words[5],"GBYPASS") then
isbypass=1
elseif string.find(words[6],"*OVERRIDE*") then
isbypass=1
end
end
for goodline in io.lines("/etc/weblog/goodwords") do
if not goodline then
-- logme("This line is apparently empty...")
break
end
_,instcnt = string.lower(words[4]):gsub(goodline, " ")
--if string.find(words[4],goodline) then
if instcnt ~= 0 then
if wrdcnt ~= 0 then
wrdcnt = wrdcnt - instcnt
if goodwordloc ~= "" then
goodwordloc = goodwordloc.."|"..goodline
else
goodwordloc = goodline
end
end
end
end
local words = format.string_to_table(line, "\t")
local logentry = {logdatetime=words[1],
clientuserid=words[2],
clientip=words[3],
URL=words[4],
reason=words[6],
method=words[5],
bytes=words[7],
shortreason=words[9],
score=wrdcnt,
badyesno=isbad,
deniedyesno=isdenied,
bypassyesno=isbypass,
wordloc=badwordloc,
gwordloc=goodwordloc}
if logentry.reason and logentry.reason ~= "" then
if logentry.shortreason == "" then
logentry.shortreason = logentry.reason
end
return logentry
end
return nil
end
--local function parsedglog(line)
-- local words = format.string_to_table(line, "\t")
-- local logentry = { logdatetime=words[1], clientuserid=words[2], clientip=words[3],
-- URL=words[4], reason=words[5], method=words[6], bytes=words[7],
-- shortreason=words[9]}
-- if logentry.reason and logentry.reason ~= "" then
-- if logentry.shortreason == "" then
-- logentry.shortreason = logentry.reason
-- end
-- logentry.score = string.match(logentry.reason, "^.*: ([0-9]+) ")
-- logentry.logdatetime = string.gsub(logentry.logdatetime, "%.", "-")
-- return logentry
-- end
-- return nil
--end
-- ################################################################################
-- DOWNLOAD FILE FUNCTIONS
......@@ -960,9 +1193,9 @@ end
-- import either squid or dg log file.
-- delete logfile after
local function importlogfile(source, cookiesfile, file, parselog_func, importlog_func)
--logme("Getting " .. file )
logme("Getting " .. file )
local loghandle = openlogfile(source, cookiesfile, file)
--logme("Processing " .. file )
logme("Processing " .. file )
local res, err = pcall(function()
con:execute("START TRANSACTION")
for line in loghandle:lines() do
......@@ -1035,10 +1268,18 @@ function importlogs()
count = count + 1
success = importlogfile(source, cookiesfile, file, parsedglog, importdglog) and success
end
if string.match(file, "squark/access%.log[%.%-]") then
count = count + 1
success = importlogfile(source, cookiesfile, file, parsesquarklog, importsquarklog) and success
end
if string.match(file, "squid/access%.log[%.%-]") then
count = count + 1
success = importlogfile(source, cookiesfile, file, parsesquidlog, importsquidlog) and success
end
if string.match(file, "dump/access%.log[%.%-]") then
count = count + 1
success = importlogfile(source, cookiesfile, file, parsedumplog, importdumplog) and success
end
end
end
os.remove(cookiesfile)
......@@ -1267,15 +1508,15 @@ function editselected(chkdata)
keycnt = 0
sql = "UPDATE pubweblog SET selected = "
databaseconnect(DatabaseOwner)
idarray = split(chkdata,"|")
for key,x in pairs(idarray) do
keycnt = keycnt + 1
if keycnt == 1 then
sql = sql..x.." WHERE id = "
else
sql = sql..x
end
end
idarray = format.string_to_table(chkdata, "|")
for key,x in pairs(idarray) do
keycnt = keycnt + 1
if keycnt == 1 then
sql = sql..x.." WHERE id = "
else
sql = sql..x
end
end
assert (con:execute(sql))
databasedisconnect()
end
......@@ -1557,7 +1798,6 @@ function create_database(database)
return database
end
function listfiles()
local retval = cfe({ type="list", value={}, label="Weblog Files" })
if not fs.is_dir(baseurl) then fs.create_directory(baseurl) end
......@@ -1570,12 +1810,10 @@ function listfiles()
table.sort(retval.value)
return retval
end
function getnewfile()
local filename = cfe({ label="File Name", descr="Must be in "..baseurl })
return cfe({ type="group", value={filename=filename}, label="Weblog File" })
end
function readfile(filename)
return modelfunctions.getfiledetails(filename, listfiles().value)
end
......@@ -1597,28 +1835,3 @@ function deletefile(filename)
return retval
end
-- Split a string to an array by delimiter or pattern
function split(str, pat)
if string.find(str, pat) == nil then
return str
end
local t = {}
local fpat = "(.-)" .. pat
local last_end = 1
local s, e, cap = str:find(fpat, 1)
while s do
if s ~= 1 or cap ~= "" then
table.insert(t,cap)
end
last_end = e+1
s, e, cap = str:find(fpat, last_end)
end
if last_end <= #str then
cap = str:sub(last_end)
table.insert(t, cap)
end
return t
end
<% local data, viewlibrary, page_info, session = ... %>
<% require("viewfunctions")%>
<% require("weblogviewfunctions")%>
<%
-- Insert a string into another string
function string_insert(value, insert, place)
if place == nil then
place = string.len(value)+1
end
return string.sub(value, 1,place-1) .. tostring(insert) .. string.sub(value, place, string.len(value))
end
--Highlight occurences of a word in a string
function string_highlight(txtvalue, searchval, fcolour, bcolour)
if txtvalue ~=nil and searchval ~= nil then
sStart = string.find(string.lower(txtvalue),string.lower(searchval))
if sStart ~= nil then
sEnd = sStart + string.len(searchval)
txtvalue = string.insert(txtvalue,"</font>", sEnd)
txtvalue = string.insert(txtvalue,"<font style='color:"..fcolour.."; background-color:"..bcolour..";'>", sStart)
end
end
return txtvalue
end
%>
<% if data.value.focus.value ~= "" then %>
<script type="text/javascript" src="<%= html.html_escape(page_info.wwwprefix) %>/js/jquery-latest.js"></script>
<script type="text/javascript">
......@@ -33,6 +58,7 @@
#content table tr.mark { background: #E9E9E9; }
#content table tr.selected { background: #FC9A01; }
</style>
<H1>Search Parameters</H1>
<DL>
<% if data.errtxt then %><p class="error"><%= html.html_escape(data.errtxt) %></p><% end %>
......@@ -170,23 +196,23 @@ end
<TD <% if data.value.clientip.value == watch.clientip then %> style="font-weight:bold;" <% end %> ><%= html.html_escape(watch.clientip) %></TD>
<TD <% if data.value.clientuserid.value == watch.clientuserid then %> style="font-weight:bold;" <% end %> ><%= html.html_escape(watch.clientuserid) %></TD>
<TD><%= html.html_escape(watch.bytes) %></TD>
<TD WIDTH="2%"><% if watch.badyesno == "1" then %><IMG SRC='<%= html.html_escape(page_info.wwwprefix..page_info.staticdir) %>/dodgy.gif' width='13' height='13'><% end %></TD>
<TD WIDTH="2%"><% if watch.deniedyesno ~= "0" then %> <IMG SRC='<%= html.html_escape(page_info.wwwprefix..page_info.staticdir) %>/denied.gif' width='13' height='13'><% end %></TD>
<TD WIDTH="2%"><% if watch.bypassyesno ~= "0" then %> <IMG SRC='<%= html.html_escape(page_info.wwwprefix..page_info.staticdir) %>/bypass.gif' width='13' height='13'><% end %></TD>
<TD WIDTH="2%"><% if watch.badyesno == "1" then %><IMG SRC='<%= html.html_escape(page_info.wwwprefix..page_info.staticdir) %>/dodgy.png' width='13' height='13'><% end %></TD>
<TD WIDTH="2%"><% if watch.deniedyesno ~= "0" then %> <IMG SRC='<%= html.html_escape(page_info.wwwprefix..page_info.staticdir) %>/denied.png' width='13' height='13'><% end %></TD>
<TD WIDTH="2%"><% if watch.bypassyesno ~= "0" then %> <IMG SRC='<%= html.html_escape(page_info.wwwprefix..page_info.staticdir) %>/bypass.png' width='13' height='13'><% end %></TD>
<TD><%= html.html_escape(watch.score) %></TD>
<TD><%= html.html_escape(watch.reason) %></TD>
<TD style="word-wrap: break-word" width="500"><% highlight_uri=html.html_escape(watch.uri)
if watch.wordloc ~= nil then
if string.find(watch.wordloc,"|") then
badwords = split(watch.wordloc,"|")
for key,wrd in pairs(badwords) do
highlight_uri = string.highlight(highlight_uri, wrd, "yellow","red")
end
else
highlight_uri = string.highlight(highlight_uri, watch.wordloc, "yellow","red")
end
end %>
<%= highlight_uri %></TD>
<TD style="word-wrap: break-word" width="500"><% highlight_uri=html.html_escape(watch.uri)