Commit 28804449 authored by Luke Stuart's avatar Luke Stuart Committed by Ted Trask

Weblog Updates including Squark support and revamped analysis.

parent 9377ebd8
APP_NAME=weblog
PACKAGE=acf-$(APP_NAME)
VERSION=0.5.9
VERSION=0.6.0
CRON_FILE=weblogimport
......
#!/bin/bash
mkdir /etc/weblog
touch /etc/weblog/goodwords
touch /etc/weblog/badwords
touch /etc/weblog/ignorewords
This diff is collapsed.
......@@ -4,7 +4,7 @@ require("viewfunctions")
<style type="text/css">
#content table { border-collapse: collapse; width: 100%; }
#content table td { white-space: nowrap; padding-right:20px; border-bottom:1px solid #999; }
#content table td { white-space: normal; padding-right:20px; border-bottom:1px solid #999; }
#content table tr.mark { background: #E9E9E9; }
</style>
......
<% local form, viewlibrary, page_info = ...
require("viewfunctions")
%>
<H1><%= html.html_escape(form.label) %></H1>
<%
local order = {"auditstart", "auditend", "historydays", "watchdays", "purgedays", "window", "shorturi", "shortreason", "groupby"}
displayform(form, order, nil, page_info)
local order = {"auditstart", "auditend", "badyesno", "sortby", "minimumscore", "window", "shorturi", "shortreason", "historydays", "watchdays", "purgedays" }
displayform(form, order, nil, page_info)
%>
......@@ -50,37 +50,63 @@ function deletewatchlistentry(self)
end
function viewweblog(self)
return self.model.getweblog(self.clientdata.clientuserid, self.clientdata.starttime, self.clientdata.endtime, self.clientdata.clientip, clientdata.focus)
return self.model.getweblog(self.clientdata.activelog, self.clientdata.clientuserid, self.clientdata.starttime, self.clientdata.endtime, self.clientdata.clientip, self.clientdata.badyesno, self.clientdata.deniedyesno, self.clientdata.bypassyesno, self.clientdata.score, self.clientdata.urisearch, self.clientdata.sortby, self.clientdata.selected, clientdata.focus)
end
function downloadweblog(self)
self.conf.viewtype = "stream"
local retval = viewweblog(self)
local file = cfe({ type="longtext", value="", label=retval.value.clientuserid.value .. ".log" })
local content = {"sourcename\tclientip\tclientuserid\tlogdatetime\turi\tbytes\treason\tscore"}
local file = cfe({ type="longtext", value="", label=retval.value.clientuserid.value .. ".csv" })
local content = {"clientuserid,clientip,logdatetime,uri,bytes,reason,score,reason,badyesno,deniedyesno,bypassyesno"}
for i,log in ipairs(retval.value.log.value) do
content[#content+1] = string.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
log.sourcename, log.clientip, log.clientuserid, log.logdatetime,
log.uri, log.bytes, log.reason or "", log.score or "0")
content[#content+1] = string.format("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,",
log.clientuserid, log.clientip, log.logdatetime, log.uri, log.bytes, log.reason, log.score or "0", log.reason, log.badyesno, log.deniedyesno, log.bypassyesno )
end
file.value = table.concat(content, "\n")
return file
end
function viewblocklog(self)
return self.model.getblocklog(self.clientdata.clientuserid, self.clientdata.starttime, self.clientdata.endtime, self.clientdata.clientip, clientdata.focus)
function downloadlogview(self)
thisdate=os.date()
self.conf.viewtype = "stream"
local retval = viewweblog(self)
local file = cfe({ type="longtext", value="", label="Weblog-resnet-"..thisdate..".tab" })
local content = {"clientuserid\tclientip\tlogdatetime\turi\tbytes\treason\tscore\treason\tbadyesno\tdeniedyesno\tbypassyesno"}
for i,log in ipairs(retval.value.log.value) do
content[#content+1] = string.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
log.clientuserid, log.clientip, log.logdatetime, log.uri, log.bytes, log.reason, log.score or "0", log.reason, log.badyesno, log.deniedyesno, log.bypassyesno )
end
file.value = table.concat(content, "\n")
return file
end
function viewusagestats(self)
return self.model.getusagestats()
function downloadselected(self)
thisdate=os.date()
self.conf.viewtype = "stream"
local retval = viewselected(self)
local file = cfe({ type="longtext", value="", label="Weblog-resnet-selected-"..thisdate..".tab" })
local content = {"clientuserid\tclientip\tlogdatetime\turi\tbytes\treason\tscore\treason\tbadyesno\tdeniedyesno\tbypassyesno"}
for i,log in ipairs(retval.value.log.value) do
content[#content+1] = string.format("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t",
log.clientuserid, log.clientip, log.logdatetime, log.uri, log.bytes, log.reason, log.score or "0", log.reason, log.badyesno, log.deniedyesno, log.bypassyesno )
end
file.value = table.concat(content, "\n")
return file
end
function viewauditstats(self)
return self.model.getauditstats()
function checkselected(self)
self.conf.viewtype = "silent"
return self.model.editselected(self.clientdata.chkdata)
--return file
end
function clearselected(self)
return self:redirect_to_referrer(self.model.clearselected())
end
function completeaudit(self)
return self:redirect_to_referrer(self.model.completeaudit(self.clientdata.auditend))
function viewselected(self)
--return self.model.getselected(self.clientdata.csvdata, self.clientdata.starttime, self.clientdata.endtime)
return self.model.getweblog(self.clientdata.activelog, self.clientdata.clientuserid, self.clientdata.starttime, self.clientdata.endtime, self.clientdata.clientip, self.clientdata.badyesno, self.clientdata.deniedyesno, self.clientdata.bypassyesno, self.clientdata.score, self.clientdata.urisearch, self.clientdata.sortby, self.clientdata.getselected, clientdata.focus)
end
function adhocquery(self)
......@@ -120,3 +146,18 @@ end
function createdatabase(self)
return controllerfunctions.handle_form(self, self.model.getnewdatabase, self.model.create_database, self.clientdata, "Create", "Create New Database", "Database Created")
end
function listfiles(self)
return self.model.listfiles(self)
end
function createfile(self)
return controllerfunctions.handle_form(self, self.model.getnewfile, self.model.createfile, self.clientdata, "Create", "Create New Weblog File", "Weblog File Created")
end
function editfile(self)
return controllerfunctions.handle_form(self, function() return self.model.readfile(self.clientdata.filename) end, self.model.updatefile, self.clientdata, "Save", "Edit Weblog File", "Weblog File Saved" )
end
function deletefile(self)
return self:redirect_to_referrer(self.model.deletefile(self.clientdata.filename))
end
../filedetails-html.lsp
\ No newline at end of file
<% local view, viewlibrary, page_info, session = ...
require("viewfunctions")
%>
<% displaycommandresults({"editfile", "deletefile", "startstop"}, session) %>
<% displaycommandresults({"createfile"}, session, true) %>
<H1>File List</H1>
<DL>
<TABLE>
<TR style="background:#eee;font-weight:bold;">
<TD style="padding-right:20px;white-space:nowrap;text-align:left;" class="header">Action</TD>
<TD style="white-space:nowrap;text-align:left;" class="header">File</TD>
</TR>
<% for i,file in ipairs(view.value) do %>
<TR>
<TD style="padding-right:20px;white-space:nowrap;">
<% if viewlibrary.check_permission("editfile") then %>
<%= html.link{value=page_info.script..page_info.prefix..page_info.controller.."/editfile?filename="..file.."&redir="..page_info.orig_action, label="Edit "} %>
<% end %>
<% if viewlibrary.check_permission("deletefile") then %>
<%= html.link{value=page_info.script..page_info.prefix..page_info.controller.."/deletefile?filename="..file, label="Delete "} %>
<% end %>
</TD>
<TD style="white-space:nowrap;"><%= html.html_escape(file) %></TD>
</TR>
<% end %>
</TABLE>
</DL>
<% if viewlibrary and viewlibrary.dispatch_component and viewlibrary.check_permission("createfile") then
local createform = viewlibrary.dispatch_component("createfile", nil, true) %>
<H2><%= html.html_escape(createform.label) %></H2>
<%
createform.action = page_info.script .. page_info.prefix .. page_info.controller .. "/createfile"
displayform(createform)
end %>
<% if viewlibrary and viewlibrary.dispatch_component and viewlibrary.check_permission("startstop") then
viewlibrary.dispatch_component("startstop")
end %>
<% local form, viewlibrary, page_info = ...
require("viewfunctions")
%>
<style type="text/css">
#content table { border-collapse: collapse; width: 100%; }
#content table td { white-space: normal; padding-right:20px; border-bottom:1px solid #999; }
#content table tr.mark { background: #E9E9E9; }
</style>
<H1>Maintenance</H1>
<% displayformstart(form, page_info) %>
<% displayformitem(form.value.query, "query") %>
<% displayformend(form) %>
<H2>Purge Database Operations</H2>
<H3>Analysis Operations</H3>
<H4>Actions</H4>
This diff is collapsed.
<% local form, viewlibrary, page_info = ...
require("viewfunctions")
%>
<style type="text/css">
#content table { border-collapse: collapse; width: 100%; }
#content table td { white-space: normal; padding-right:20px; border-bottom:1px solid #999; }
#content table tr.mark { background: #E9E9E9; }
</style>
<H1>Summary</H1>
<H2>Last 24 Hours</H2>
<H2>Last 7 Days</H2>
<H2>Last 30 Days</
<% displayformstart(form, page_info) %>
<% displayformitem(form.value.query, "query") %>
<% displayformend(form) %>
<% local data, viewlibrary, page_info, session = ... %>
<% require("viewfunctions") %>
<script type="text/javascript" src="<%= html.html_escape(page_info.wwwprefix) %>/js/jquery-latest.js"></script>
<script type="text/javascript" src="<%= html.html_escape(page_info.wwwprefix) %>/js/jquery.tablesorter.js"></script>
<script type="text/javascript">
$(document).ready(function() {
$("#audit").tablesorter({headers: {1:{sorter:'digit'}, 2:{sorter:'digit'}}});
});
</script>
<% displaycommandresults({"completeaudit"}, session) %>
<H1>Audit Parameters</H1>
<DL>
<% displayitem(data.value.auditstart) %>
<% displayitem(data.value.auditend) %>
</DL>
<H1><%= html.html_escape(data.label) %></H1>
<DL>
<TABLE id="audit" class="tablesorter"><THEAD>
<TR style="font-weight:bold;">
<TH><% if data.value.groupby.value == "clientip" then %>Client IP<% else %>User ID<% end %></TH>
<TH>Blocks</TH>
<TH>Maximum Score</TH>
</TR>
</THEAD><TBODY>
<% for i,stat in ipairs(data.value.stats.value) do %>
<TR><TD><%= html.link{value = "viewblocklog?"..data.value.groupby.value.."="..stat[data.value.groupby.value], label=stat[data.value.groupby.value]} %></TD>
<TD><%= html.html_escape(stat.numblock) %></TD>
<TD><%= html.html_escape(stat.maxscore) %></TD></TR>
<% end %>
</TBODY></TABLE>
<% if data.errtxt then %>
<p class='error'><%= html.html_escape(data.errtxt) %></p>
<% end %>
<% if #data.value.stats.value == 0 then %>
<p>No blocks, try adjusting the audit dates</p>
<% end %>
<form action="<%= html.html_escape(page_info.script .. page_info.prefix .. page_info.controller .. "/completeaudit") %>">
<DT>Complete Audit</DT>
<DD><input class="submit" type="submit" value="Complete"></DD>
</form>
</DL>
<% local data, viewlibrary, page_info, session = ... %>
<% if data.value.focus.value ~= "" then %>
<script type="text/javascript" src="<%= html.html_escape(page_info.wwwprefix) %>/js/jquery-latest.js"></script>
<script type="text/javascript">
$(function(){
if ($("#focus").length) {
var top = $("#focus").offset().top;
$("html,body").scrollTop(top);
}
});
</script>
<% end %>
<style type="text/css">
#content table { border-collapse: collapse; width: 100%; }
#content table td { border-bottom: none; white-space: nowrap; padding-right:20px; }
#content table tr.mark { background: #E9E9E9; }
#content table tr.markfocus { background: #E9E966; }
#content table tr.focus { background: #E9E900; }
#content table tr.bypass { background: #eaa; }
#content table td.reason { color: #000; padding-left: 20px; border-bottom:1px solid #999; }
</style>
<H1>Search Parameters</H1>
<DL>
<% if data.errtxt then %><p class="error"><%= html.html_escape(data.errtxt) %></p><% end %>
<form action="<%= html.html_escape(page_info.script .. page_info.prefix .. page_info.controller .. "/" .. page_info.action) %>" method="POST">
<DT>Start Time</DT>
<DD><input class="text" type="text" name="starttime" value="<%= html.html_escape(data.value.starttime.value) %>" >
<p><%= html.html_escape(data.value.starttime.value) %></p>
<% if data.value.starttime.errtxt then %><p class="error"><%= html.html_escape(data.value.starttime.errtxt) %></p><% end %>
</DD>
<DT>User ID</DT>
<DD><input class="text" type="text" name="clientuserid" value="<%= html.html_escape(data.value.clientuserid.value) %>" >
<p><%= html.html_escape(data.value.clientuserid.value) %></p>
<% if data.value.clientuserid.errtxt then %><p class="error"><%= html.html_escape(data.value.clientuserid.errtxt) %></p><% end %>
</DD>
<DT>Client IP</DT>
<DD><input class="text" type="text" name="clientip" value="<%= html.html_escape(data.value.clientip.value) %>" >
<p><%= html.html_escape(data.value.clientip.value) %></p>
<% if data.value.clientip.errtxt then %><p class="error"><%= html.html_escape(data.value.clientip.errtxt) %></p><% end %>
</DD>
<DT>End Time</DT>
<DD><input class="text" type="text" name="endtime" value="<%= html.html_escape(data.value.endtime.value) %>" >
<p><%= html.html_escape(data.value.endtime.value) %></p>
<% if data.value.endtime.errtxt then %><p class="error"><%= html.html_escape(data.value.endtime.errtxt) %></p><% end %>
</DD>
<DT></DT><DD><input class="submit" type="submit" name="Update" value="Update"></DD>
</FORM>
</DL>
<%
local clientinfo = ""
if data.value.clientuserid.value ~= "" then
clientinfo = clientinfo .. "clientuserid="..data.value.clientuserid.value.."&"
end
if data.value.clientip.value ~= "" then
clientinfo = clientinfo .. "clientip="..data.value.clientip.value.."&"
end
%>
<H1><%= html.html_escape(data.label) %></H1>
<DL><TABLE>
<TR class="header">
<TD class="header">Timestamp</TD>
<TD class="header">Client IP</TD>
<TD class="header">User ID</TD>
<TD class="header">Size</TD>
<TD WIDTH="90%" class="header">URL</TD>
</TR>
<% for i,watch in ipairs(data.value.log.value) do %>
<% local a,b = math.modf((i/2))
local mark = ''
if (b == 0) then mark=mark..'class="mark"' end
if string.find(watch.uri, "GBYPASS=") then mark='class="bypass"' end
%>
<% local time = {}
time.year, time.month, time.day, time.hour, time.min, time.sec =
string.match(watch.logdatetime, "(%d+)%-(%d+)-(%d+)%s+(%d+):(%d+):(%d+)")
time = os.time(time) %>
<TR <%= mark %>>
<TD <% if data.value.focus.value == watch.logdatetime then %> style="font-weight:bold;" id="focus" <% end %> ><%= html.link{value = "viewweblog?"..clientinfo..
"starttime="..os.date("%Y-%m-%d %H:%M:%S", time - 60*(tonumber(data.value.window.value)))..
"&endtime="..os.date("%Y-%m-%d %H:%M:%S", time + 60*(tonumber(data.value.window.value)))..
"&focus="..watch.logdatetime,
label=watch.logdatetime} %></TD>
<TD <% if data.value.clientip.value == watch.clientip then %> style="font-weight:bold;" <% end %> ><%= html.html_escape(watch.clientip) %></TD>
<TD <% if data.value.clientuserid.value == watch.clientuserid then %> style="font-weight:bold;" <% end %> ><%= html.html_escape(watch.clientuserid) %></TD>
<TD><%= html.html_escape(watch.bytes) %></TD>
<TD title="<%= html.html_escape(watch.uri) %>" >
<%= html.link{value = watch.uri, label=watch.shorturi or watch.uri} %></TD>
</TR>
<TR <%= mark %>>
<TD class='reason' <% if (watch.shortreason and watch.shortreason ~= "") or (watch.reason and watch.reason ~= "") then %>
title="<%= html.html_escape(watch.reason) %>"<% end %> colspan=5>
<% if watch.score and watch.score ~= "0" then %><%= html.html_escape(watch.score) %> - <% end %>
<% if (watch.shortreason and watch.shortreason ~= "") then %>
<%= html.html_escape(watch.shortreason) %>
<% elseif (watch.reason and watch.reason ~= "") then %>
<%= html.html_escape(watch.reason) %>
<% end %>
</TD></TR>
<% end %>
</TABLE>
<% if data.errtxt then %>
<p class="error"><%= html.html_escape(data.errtxt) %></p>
<% end %>
<% if #data.value.log.value == 0 then %>
<p>No results, try adjusting search parameters</p>
<% end %>
<% if page_info.action == "viewweblog" then %>
<form action="<%= html.html_escape(page_info.script .. page_info.prefix .. page_info.controller .. "/downloadweblog") %>" method="POST">
<input type="hidden" name="starttime" value="<%= html.html_escape(data.value.starttime.value) %>" >
<input type="hidden" name="clientuserid" value="<%= html.html_escape(data.value.clientuserid.value) %>" >
<input type="hidden" name="clientip" value="<%= html.html_escape(data.value.clientip.value) %>" >
<input type="hidden" name="endtime" value="<%= html.html_escape(data.value.endtime.value) %>" >
<DT>Download log</DT><DD><input class="submit" type="submit" name="Download" value="Download"></DD>
</FORM>
<% end %>
</DL>
<% local data, viewlibrary, page_info, session = ... %>
<% require("viewfunctions") %>
<% local subdata, pagedata = paginate(data.value, page_info.clientdata, 100) %>
<H1><%= html.html_escape(data.label) %></H1>
<DL>
<% displaypagination(pagedata, page_info) %>
<TABLE>
<TR style="background:#eee;font-weight:bold;">
<TD style="padding-right:20px;white-space:nowrap;" class="header">Date</TD>
<TD style="padding-right:20px;white-space:nowrap;" class="header">Source</TD>
<TD style="padding-right:20px;white-space:nowrap;" class="header">Requests</TD>
<TD style="white-space:nowrap;" WIDTH="90%" class="header">Blocks</TD>
</TR>
<% for i,stat in ipairs(subdata) do %>
<TR>
<TD><%= html.html_escape(stat.date) %></TD>
<TD><%= html.html_escape(stat.sourcename) %></TD>
<TD><%= html.html_escape(stat.numrequest) %></TD>
<TD><%= html.html_escape(stat.numblock) %></TD>
</TR>
<% end %>
</TABLE>
<% if data.errtxt then %>
<p class='error'><%= html.html_escape(data.errtxt) %></p>
<% end %>
<% if #data.value == 0 then %>
<p>No usage stats found</p>
<% end %>
</DL>
......@@ -24,7 +24,7 @@
<% for i,watch in ipairs(data.value) do %>
<TR>
<TD><%= html.link{value = "deletewatchlistentry?clientuserid="..watch.clientuserid, label="Delete "} %></TD>
<TD><%= html.link{value = "viewblocklog?clientuserid="..watch.clientuserid, label=watch.clientuserid} %></TD>
<TD><%= html.link{value = "viewweblog?clientuserid="..watch.clientuserid, label=watch.clientuserid} %></TD>
<TD><%= html.html_escape(watch.expiredatetime) %></TD>
</TR>
<% end %>
......
weblog-viewblocklog-html.lsp
\ No newline at end of file
This diff is collapsed.
#CAT GROUP/DESC TAB ACTION
Applications 41Weblog Status status
Applications 41Weblog Audit viewauditstats
#Applications 41Weblog Audit viewauditstats
Applications 41Weblog View_Log viewweblog
Applications 41Weblog Config config
Applications 41Weblog File_List listfiles
Applications 41Weblog Watch_List viewwatchlist
Applications 41Weblog Sources listsources
Applications 41Weblog Usage viewusagestats
Applications 41Weblog History viewactivitylog
Applications 41Weblog Ad-Hoc_Query adhocquery
USER=weblog:viewauditstats,weblog:completeaudit,weblog:viewactivitylog,weblog:viewwatchlist,weblog:viewweblog,weblog:downloadweblog,weblog:viewblocklog,weblog:viewusagestats,weblog:deletewatchlistentry,weblog:createwatchlistentry,weblog:adhocquery,weblog:downloadadhocquery
EXPERT=weblog:config,weblog:listsources,weblog:createsource,weblog:deletesource,weblog:editsource,weblog:testsource,weblog:importlogs,weblog:status,weblog:createdatabase
ADMIN=weblog:config,weblog:viewauditstats,weblog:completeaudit,weblog:viewactivitylog,weblog:viewwatchlist,weblog:viewweblog,weblog:downloadweblog,weblog:viewblocklog,weblog:viewusagestats,weblog:listsources,weblog:createsource,weblog:deletesource,weblog:editsource,weblog:testsource,weblog:importlogs,weblog:deletewatchlistentry,weblog:createwatchlistentry,weblog:adhocquery,weblog:downloadadhocquery,weblog:status,weblog:createdatabase
EXPERT=weblog:config,weblog:viewweblog,weblog:listsources,weblog:createsource,weblog:deletesource,weblog:editsource,weblog:testsource,weblog:importlogs,weblog:status,weblog:createdatabase,weblog:listfile,weblog:editfile
ADMIN=weblog:config,weblog:viewauditstats,weblog:completeaudit,weblog:viewactivitylog,weblog:viewwatchlist,weblog:viewweblog,weblog:viewselected,weblog:downloadweblog,weblog:downloadlogview,weblog:downloadselected,weblog:checkselected,weblog:editselected,weblog:clearselected,weblog:viewblocklog,weblog:viewusagestats,weblog:listsources,weblog:createsource,weblog:deletesource,weblog:editsource,weblog:testsource,weblog:importlogs,weblog:deletewatchlistentry,weblog:createwatchlistentry,weblog:adhocquery,weblog:downloadadhocquery,weblog:status,weblog:createdatabase,weblog:listfiles,weblog:editfile
require("html")
require("session")
-- Split a string to an array by delimiter or pattern
function split(str, pat)
if string.find(str, pat) == nil then
return str
end
local t = {}
local fpat = "(.-)" .. pat
local last_end = 1
local s, e, cap = str:find(fpat, 1)
while s do
if s ~= 1 or cap ~= "" then
table.insert(t,cap)
end
last_end = e+1
s, e, cap = str:find(fpat, last_end)
end
if last_end <= #str then
cap = str:sub(last_end)
table.insert(t, cap)
end
return t
end
-- Insert a string into another string
function string.insert(value, insert, place)
if place == nil then
place = string.len(value)+1
end
return string.sub(value, 1,place-1) .. tostring(insert) .. string.sub(value, place, string.len(value))
end
--Highlight occurences of a word in a string
function string.highlight(txtvalue, searchval, fcolour, bcolour)
if txtvalue ~=nil and searchval ~= nil then
sStart = string.find(string.lower(txtvalue),string.lower(searchval))
if sStart ~= nil then
sEnd = sStart + string.len(searchval)
txtvalue = string.insert(txtvalue,"</font>", sEnd)
txtvalue = string.insert(txtvalue,"<font style='color:"..fcolour.."; background-color:"..bcolour..";'>", sStart)
end
end
return txtvalue
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment