\documentclass[11pt]{beamer}

\usetheme{Rochester}                    % No navigation - simple theme

\usepackage{times}                      
\usefonttheme{structurebold}            % make titles bolder

\usepackage[english]{babel}             % not necessary?
\usepackage{pgf}                        % graphics
\usepackage{amsmath,amssymb}
\usepackage[latin1]{inputenc}

\setbeamercovered{dynamic}              % no idea

% -------- ADDITIONAL PACKAGES -------------
\usepackage{forestcolor}
\usepackage{mdwlist}
\usepackage{fancyvrb}
\usepackage[T1]{fontenc}
\usepackage[scaled]{beramono}

% -------- LOCAL COMMANDS ------------

% HACK HACK HACK -- instead of using "forestbrown" I should be 
% referencing "palettes" from the styles. 
\newcommand{\code}[1]{{\color{forestbrown}\texttt{#1}}}
\newcommand{\highlight}[1]{{\color{forestbrown}{#1}}}
\newcommand{\ed}[1]{\alert{(#1 -P)}}


\DefineVerbatimEnvironment%
  {codeblock}{Verbatim}
  {formatcom=\color{forestbrown},frame=single,framerule=1pt,%
  rulecolor=\color{forestgreen}}

% -------- PAGE LENGTHS -------------------

\addtolength{\parskip}{1.5ex}

% ------- PRESENTATION, FOO --------------

\title{Remote Logging with Rsyslog}
\subtitle{Or, How I Learned to Start Worrying and Love the Panopticon}
\author{Paul Nijjar}
\institute[KWLUG]{Kitchener-Waterloo Linux User Group}
\date{August 10, 2009}

\begin{document}

% ----------------------------
\frame{\maketitle}

\section{Introduction}


% ----------------------------
\begin{frame}[plain]

\begin{center}
\includegraphics[scale=0.5]{png/network-01-okay}
\end{center}

\end{frame}


% ----------------------------
\begin{frame}[plain]

\begin{center}
\includegraphics[scale=0.5]{png/network-02-threats}
\end{center}

\end{frame}

% ----------------------------
\begin{frame}[plain]

\begin{center}
\includegraphics[scale=0.5]{png/network-03-nointernet}
\end{center}

\end{frame}

% ----------------------------
\begin{frame}[plain]

\begin{center}
\includegraphics[scale=0.5]{png/network-04-devicealerts}
\end{center}

\end{frame}

% ----------------------------
\begin{frame}[plain]

\begin{center}
\includegraphics[scale=0.5]{png/network-05-logserver}
\end{center}

\end{frame}

% ----------------------------
\begin{frame}[plain]

\begin{center}
\includegraphics[scale=0.5]{png/network-06-logcollect}
\end{center}

\end{frame}

% ----------------------------
\begin{frame}[plain]

\begin{center}
\includegraphics[scale=0.5]{png/network-07-logsend}
\end{center}

\end{frame}




% ----------------------------
\begin{frame}

\frametitle{Goals}

\begin{description}

\item [Centralize Logging:] Look in one place, using one set of tools.

\item [Archive Logs:] Keep logs around for at least a year.

\item [Generate Alerts:] Tell me when something goes wrong.

\item [Identify Trends:] Tell me what ``business as usual'' looks
like.

\end{description}

The last two of these goals are still works in progress. 

Another goal: do this on the cheap, preferably with FLOSS.

\end{frame} 


% ----------------------------
\section{Rsyslog}
\begin{frame}

\frametitle{Rsyslog}


\begin{center}
\includegraphics[scale=0.5]{png/rsyslog-splash}
\end{center}

\end{frame}


% ----------------------------
\subsection{About Rsyslog}
\begin{frame}

\frametitle{About Syslog}

Syslogd is a logging interface used by many Linux programs to write
log files. It is responsible for:

\begin{itemize*}

\item Many of the files in \code{/var/log}: \code{messages},
\code{debug}, \code{syslog}, etc.

\item Messages sent to the system console. 

\item Messages forwarded to other systems.

\item Emergency log messages printed on everybody's screens 

\end{itemize*}

\end{frame}


% ----------------------------
\begin{frame}

\frametitle{About Rsyslog}

\highlight{Rsyslog} is a drop-in replacement for regular syslog. It adds
a bunch of features: 

\begin{itemize*}

\item Better security controls

\item More filtering options/syntax

\item More reliable transport mechanisms 

\item Writing to databases

\end{itemize*}

Rsyslog is now the default syslogging daemon for Fedora and Debian. 

\end{frame}


% ----------------------------
\subsection{Configuring Rsyslog Overview}
\begin{frame}

\frametitle{Configuring Rsyslog}

\begin{enumerate}

\item Enable remote logging

\item Write templates for filenames and log formats

\item Filter messages from different hosts to different files

\item Rotate and archive files using \code{logrotate}

\item Debug the collection process

\end{enumerate}

\end{frame}


% ----------------------------
\begin{frame}[fragile]

\frametitle{Config Files}

In Debian, configuration is done in \code{/etc/rsyslog.conf} and
\code{/etc/rsyslog.d/*.conf} 

Order matters, so I prepend configuration snippets with numbers:

\begin{itemize}

\item \code{/etc/rsyslog.d/00-AllowedHosts.conf}

\item \code{/etc/rsyslog.d/40-Windows-Servers.conf}

\item \code{/etc/rsyslog.d/99-EverythingElse.conf}

\end{itemize}

In general rules need to begin in the first column (no spaces) and
they should be on one line. 

\end{frame}  


% ----------------------------
\subsection{Enabling Remote Logging}
\begin{frame}[fragile]

\frametitle{Enabling Remote Logging}

In \code{/etc/rsyslog.conf}, uncomment the following lines:

\begin{codeblock}
$ModLoad imudp
$UDPServerRun 514
\end{codeblock}

UDP on port 514 is the standard syslog port.

You may need to open this
port on your firewalls if you are logging from remote subnets/devices.


\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Allowing Remote Hosts to Syslog}

In \code{/etc/rsyslog.d/00-AllowedHosts.conf}, allow some hosts.
You can specify IP addresses, subnets, or hostnames:

\begin{codeblock}
# One server or router
$AllowedSender UDP, 192.168.1.4

# Everything in a subnet
$AllowedSender UDP, 192.168.2.0/24

# Everything (claiming to be) from Microsoft
$AllowedSender UDP, *.microsoft.com
\end{codeblock}

\end{frame}

% ----------------------------
\subsection{Text Log Goals}
\begin{frame}

\frametitle{Text Log Goals}

My goal: put one or two logfiles per host in 
\code{/var/log/remote-logs/} 

I don't want to touch the local logging (e.g.
\code{/var/log/messages}) at all. 

I want to keep the logs for at least a year, and archive them 
in \code{/var/log/remote-logs/oldlogs/}

\end{frame}

% ----------------------------
\subsection{Properties}
\begin{frame}

\frametitle{Log Message Properties}

Every log message comes with some attributes called
\highlight{properties}. Here are a few useful ones: 

\begin{description}

\item[msg] Message body.

\item[rawmsg] The message text as sent over the wire.

\item[HOSTNAME] The host that generated the message.  

\item[FROMHOST] The host that last relayed this message.

\item[syslogtag] The service that reported the message. e.g.
\code{kernel:}, \code{security[success]}

\item[PRI] The facility.priority of the message. e.g. \code{mail.debug}

\end{description}

There are others that can be useful for auditing, such as
\code{timereported}, \code{syslogfacility-text}...

\end{frame}

% ----------------------------
\begin{frame}

\frametitle{HOSTNAME va FROMHOST}

\begin{center}
\includegraphics[scale=0.33]{png/hostname-vs-fromhost}
\end{center}

\end{frame}

% ----------------------------
\subsection{Templates}
\begin{frame}[fragile]

\frametitle{Templates}

\highlight{Templates} are formatted strings. They 
can be used to name destination files and
rewrite the format of messages that go to the syslog server. 

\begin{codeblock}
$template BoringServerLog, 
  "/var/log/remote-logs/%HOSTNAME%-boring.log"
\end{codeblock}

e.g. \code{Aug  7 04:29:49 localhost su[2569]: pam\_authenticate: Authentication failure}

\begin{codeblock}
$template TraditionalFormat,
  "%timegenerated% %HOSTNAME% %syslogtag%%msg%\n"
\end{codeblock}

\end{frame}

% ----------------------------
\subsection{Destination Files}
\begin{frame}[fragile]

\frametitle{Message Destinations}

You can send messages to files (with an optional format):
\begin{codeblock}
*.*             /var/log/everything.log
*.debug         ?DebugLog
mail.*          ?MailLog;MailFormat
\end{codeblock}

To stop processing messages send them to the \code{\~} destination:
\begin{codeblock}
*.*             ~
\end{codeblock}

\end{frame}
% ----------------------------
\subsection{Selection Methods}
\begin{frame}

\frametitle{Filtering Messages With Selection Rules}

Rsyslog provides four mechanisms for filtering messages into files: 

\begin{description}

\item[BSD blocks] Filter messages by hostname or program name

\item[Traditional] Filter by severity and facility

\item[Property based] Look at the message properties

\item[Expression based] If-then statements

\end{description}

I could get the first three of these to work. 

Note that you cannot mix these methods on one line (but you can put
other rules inside a BSD block)

\end{frame}

% ----------------------------
\subsection{BSD Blocks}
\begin{frame}[fragile]

\frametitle{BSD blocks}

Specify a hostname for which all following rules will apply:
\begin{codeblock}
+mailserver 

*.*     /var/log/remote-logs/mailserver.log
*.*     ~
\end{codeblock}

You can make rules for all but a certain host
\begin{codeblock}
-mailserver 

*.*     /var/log/remote-logs/allbutmail.log
\end{codeblock}

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{BSD Blocks}

You can unset the code block afterwards to allow all hosts.
\begin{codeblock}
+*

*.*     /var/log/everybody.log
\end{codeblock}

There is also syntax that allows you to make blocks based on program
name: 

\begin{codeblock}
!sudo

*.*     /var/log/sudostuff.log
\end{codeblock}

\end{frame}

% ----------------------------
\subsection{Traditional Selectors}
\begin{frame}

\frametitle{Traditional Selectors}

This is the standard \code{facility.priority} filtering from regular
syslog. 

Some facilities: \code{auth, authpriv, cron, daemon, local0, local1,
local7, user}

Some priorities: \code{debug, info, notice, warning, err, crit, alert,
emerg} . 

By default specifying a priority includes messages from higher
priorities to the same file. 

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Selector Examples}

\begin{codeblock}
*.*             /var/log/everything.log

# daemon messages of priority err to emerg
daemon.err      /var/log/daemon-warning.log

# Only messages of priority crit
*.=crit         /var/log/critical.log

# Emergencies get printed on everybody's screen
*.emerg         * 
\end{codeblock}

\end{frame}

% ----------------------------
\subsection{Property-based filters}
\begin{frame}[fragile]

\frametitle{Property Based Filters}

These allow you to filter based on message properties. They begin 
with a colon. 

\begin{codeblock}
:msg, contains, "RGFW-OUT: ACCEPT (ICMP type 8" 
  /var/log/remote-logs/pubrouter-stupid.log
\end{codeblock}


Property-based filters are slower than traditional ones, but I used
them a lot. 

\end{frame}

% ----------------------------
\begin{frame}

\frametitle{Property Operators}

The following operators are defined: 

\begin{description}

\item[isequal] Does the property match exactly?

\item[contains] Does the property contain a string?

\item[startswith] Does the property start with a certain string?

\item[regex] Does the property match a given regular expression?

\end{description}


\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Property Filter Examples}

If \code{HOSTNAME} is not defined I often filtered like this:
\begin{codeblock}
:FROMHOST, isequal, "192.168.1.20"  
   /var/log/remote-logs/pubrouter.log
\end{codeblock}

Some messages in my router were of the form \code{192.168.1.42:28268 ->
192.161.1.3:443} for DNS lookups. 
\begin{codeblock}
:msg, regex, ".*:443$" ?BoringDNSLog
\end{codeblock}

\end{frame}

% ----------------------------
\subsection{If-Then Expressions}
\begin{frame}[fragile]

\frametitle{If-Then Expression Filters}

You are supposed to be able to use expressions filters like this:

\begin{codeblock}
if $FROMHOST isequal '192.168.1.20' 
  and $msg contains 'RGFW-OUT' 
  /var/log/router-out.log 
\end{codeblock}

I could never get these to work, but maybe I am just dumb. As Rsyslog
matures this is supposed to get more powerful. 

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Putting It Together}

Some of \code{/etc/rsyslog.d/40-winservers.conf}

\begin{codeblock}
+dc1

:syslogtag, startswith, "DHCP"  
  /var/log/remote-logs/dc1-dhcp.log
:syslogtag, startswith, "DHCP"  ~

*.*     /var/log/remote-logs/dc1.log
*.*     ~
\end{codeblock}

\end{frame}



% ----------------------------
\subsection{Rsyslog and MySQL}
\begin{frame}[fragile]

\frametitle{Another Approach: MySQL}

If you install the \code{rsyslog-mysql} package, you can write logs to
a MySQL database. 

\alert{Caution:} On Debian, this package creates an \code{rsyslog}
database user that is more powerful than it needs to be.

The package puts a file called \code{mysql.conf} in
\code{/etc/rsyslog.d/}, which I copied to a file called 
\code{07-mysql.conf} . 


\end{frame}



% ----------------------------
\begin{frame}[fragile]

\frametitle{07-mysql.conf}

\begin{codeblock}
$ModLoad ommysql

# <dbserver>,<dbname>,<dbuser>,<dbpass>;<template>
*.* :ommysql:localhost,Syslog,rsyslog,dbpassword
\end{codeblock}

The template is optional -- there is a default schema and template
used. 

\end{frame}

% ----------------------------
\subsection{phpLogCon}
\begin{frame}

\frametitle{phpLogCon}

You can download a PHP frontend to the Rsyslog MySQL database
called from \url{http://www.phplogcon.org} 

Installation is manual but pretty easy: untar scripts into
\code{/var/www} and run a configuration script.

Dependencies: \code{rsyslog-mysql}, \code{php5-mysql}, \code{php5-gd}, 
\code{libapache2-mod-php5}

\end{frame}


% ----------------------------
\begin{frame}

\frametitle{phpLogCon interface}


\begin{center}
\includegraphics[scale=0.3]{png/phplogcon-top}
\end{center}

\end{frame}


% ----------------------------
\begin{frame}

\frametitle{phpLogCon Messages}


\begin{center}
\includegraphics[scale=0.3]{png/phplogcon-detail}
\end{center}

\end{frame}

% ----------------------------
\begin{frame}

\frametitle{phpLogCon Host Graph}


\begin{center}
\includegraphics[scale=0.5]{png/toptenhost}
\end{center}

\end{frame}


% ----------------------------
\begin{frame}

\frametitle{phpLogCon Severity Graph}


\begin{center}
\includegraphics[scale=0.5]{png/toptenseverity}
\end{center}

\end{frame}

% ----------------------------
\begin{frame}

\frametitle{phpLogCon SyslogTag Graph}


\begin{center}
\includegraphics[scale=0.5]{png/toptensyslogtag}
\end{center}

\end{frame}

% ----------------------------
\subsection{Archiving Logs}
\begin{frame}[fragile]

\frametitle{Archiving Logfiles}

To archive logfiles I had to manually edit \code{/etc/logrotate.conf}.
Most of it is pretty standard.

\begin{codeblock}
/var/log/remote-logs/*.log
{
  rotate 60
  weekly
  missingok
  notifempty
  compress
  delaycompress
\end{codeblock}

This says: keep 60 weeks of logs. Compress old files, but wait a 
week before doing so. Don't archive empty files and don't complain
about them.

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{}

\begin{codeblock}
  sharedscripts
  postrotate
    invoke-rc.d rsyslog reload > /dev/null
  endscript
  olddir /var/log/remote-logs/oldlogs
}
\end{codeblock}

This says: restart \code{rsyslog} once after moving all files. Put the
files in the \code{oldlogs} directory. 

\end{frame}
% ----------------------------
\subsection{Debugging Rsyslog}
\begin{frame}

\frametitle{Debugging}

Debugging can be hideous. Here are some tools to make it easier. 

\begin{itemize}

\item Listing logs by update time

\item DEBUG template

\item Using logger to send messages locally

\item Rsyslog in verbose mode

\item Wireshark/TCPDump

\end{itemize}

\pgfputat{\pgfxy(8,2)}{\pgfimage[height=2cm]{png/debug}}

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Listing logs by update time}

This is suprisingly handy to see if a particular host has been 
writing files recently. It sorts files by modification time. 

\begin{codeblock}
ls -ltc 
\end{codeblock}

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{DEBUG template}

In \code{rsyslog.d/05-DebugTemplate.conf} add the following template 
(given in the documentation): 

\begin{codeblock}
$template DEBUG,"Debug line with all properties:
\nFROMHOST: '%FROMHOST%', HOSTNAME: '%HOSTNAME%', 
PRI: %PRI%,\nsyslogtag '%syslogtag%', 
programname: '%programname%', 
APP-NAME: '%APP-NAME%', PROCID: '%PROCID%', 
MSGID: '%MSGID%',\nTIMESTAMP: '%TIMESTAMP%', 
STRUCTURED-DATA: '%STRUCTURED-DATA%',
\nmsg: '%msg%'
\nescaped msg: '%msg:::drop-cc%'
\nrawmsg: '%rawmsg%'\n\n"
\end{codeblock}


\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Use DEBUG template}

Now in \code{/rsyslog.d/70-EverythingElse.conf} log every remote 
message that has not been logged already: 

\begin{codeblock}
-logserver

$template RemoteHostLog, 
  "/var/log/remote-logs/uncaught.log"

*.*     ?RemoteHostLog;DEBUG
*.*     ~
\end{codeblock}

You can also activate this for particular hosts, or for hosts 
that do not have a \code{HOSTNAME} defined. 

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Using logger to send messages locally}

You can use the \code{logger} command to write syslog messages
manually: 

\begin{codeblock}
# Send with priority user.info
logger 'I hate test messages!'

logger -p kern.emerg 'Everything is broken!'
\end{codeblock}

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Rsyslog in debug mode}

This will produce a HUGE amount of information. It can be useful in
checking whether your messages are getting to the daemon.

\begin{codeblock}
/etc/init.d/rsyslog stop 
script /tmp/output.txt

rsyslogd -c3 -d 
<ctrl>+C

exit
/etc/init.d/rsyslog start 
\end{codeblock}

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Wireshark and tcpdump}

This is useful to see whether messages are getting to the syslog
server. Use the following filter to see what is coming in on UDP port
514: 

\begin{codeblock}
udp.port == 514
\end{codeblock}

The equivalent filter for \code{tcpdump} is:

\begin{codeblock}
tcpdump udp port 514
\end{codeblock}

\end{frame}




% ----------------------------
\section{Sending Logs}
\begin{frame}

\frametitle{Sending Logs from Computers and Devices}

\begin{center}
\includegraphics[scale=0.5]{png/collectlogs-splash}
\end{center}

\end{frame}

% ----------------------------
\subsection{Sending logs from UNIX}
\begin{frame}[fragile]

\frametitle{Sending logs from UNIX/Linux}

In the \code{syslogd.conf} of the client, add the following line
before any log messages are thrown away: 

\begin{codeblock}
*.*     @192.168.1.40
\end{codeblock}

This forwards messages using UDP over the default port. Many
sysloggers support TCP as well (with \code{@@}). 

Your client does not need to run rsyslog for this to work. Most
sysloggers will work.  

\end{frame}

% ----------------------------
\subsection{Sending logs from devices}
\begin{frame}

\frametitle{Sending logs from devices}

\begin{center}
\includegraphics[scale=0.3]{png/dlink-circled}
\end{center}


\end{frame}

% ----------------------------
\begin{frame}

\frametitle{Enabling Devices}

\begin{center}
\includegraphics[scale=0.3]{png/dlink-syslog}
\end{center}


\end{frame}
% ----------------------------
\begin{frame}[plain]

\begin{center}
{\color{red}\huge{WARNING}}

The following slides contain depictions of proprietary software use
and may not be suitable for all viewers. Viewer discretion is advised.

\end{center}



\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Windows AUGH}

Naturally, Windows does not speak syslog format natively. However,
there are tools to convert Windows event logs to syslog format. 

Windows Vista/2008 introduced an XML format \code{.evtx} which I don't
care about (yet).

\end{frame}

% ----------------------------
\subsection{SysLogAgent}
\begin{frame}

\frametitle{SyslogAgent}

This is commercial software released under the GPL. Get it from
\url{http://syslogserver.com/syslogagent.html}

This runs as a system service. 

There are a few other syslog agents available. (The Rsyslog guy makes
a proprietary one.) 

I found SysLogAgent 
lightweight, easy to install, and good enough for my purposes. 

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{SysLogAgent main screen}

\begin{center}
\includegraphics[scale=0.5]{png/syslogagent-front}
\end{center}

\end{frame}


% ----------------------------
\begin{frame}[fragile]

\frametitle{SysLogAgent: Specifying Messages to Send}

\begin{center}
\includegraphics[scale=0.5]{png/syslogagent-auditfailure}
\end{center}

\end{frame}



% ----------------------------
\subsection{Generating Windows Events}
\begin{frame}[fragile]

\frametitle{Generating Windows Events}

There is a commandline interface to generate Windows System Log
events called \code{eventcreate.exe}

\begin{codeblock}
eventcreate /t ERROR /id 666 
  /d "Our stock price is falling!" 
\end{codeblock}


\end{frame}


% ----------------------------
\subsection{Text Logs and MS Log Parser}
\begin{frame}[fragile]

\frametitle{A Story: MS DHCP Logs}


\begin{center}
\includegraphics[scale=0.4]{png/dhcp-splash}
\end{center}

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{A Sad Story}

Microsoft's DHCP server can write out pretty good logs. 

Naturally, they don't show up as events in Event Viewer. 

Instead, they are textfiles in 
\code{c:$\backslash$windows$\backslash$system32$\backslash$dhcp$\backslash$} 

How can we get them into the syslog server?


\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{More Sadness}

The first 30 lines in every logfile are purely informational: 

\begin{Verbatim}[fontsize=\small]
                     Microsoft DHCP Service Activity Log


Event ID  Meaning
00        The log was started.
01        The log was stopped.
02        The log was temporarily paused due to low disk space.
...
32        DNS update successful
50+       Codes above 50 are used for Rogue Server Detection information.

ID,Date,Time,Description,IP Address,Host Name,MAC Address
24,08/06/09,00:00:57,Database Cleanup Begin,,,,
30,08/06/09,00:00:57,DNS Update Request,85.1.168.192,EM14temp.,,
25,08/06/09,00:00:57,0 leases expired and 0 leases deleted,,,,
\end{Verbatim}

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Even Sadder}


\begin{center}
\includegraphics[scale=0.5]{png/dhcp-filelist}
\end{center}

\end{frame}


% ----------------------------
\begin{frame}[fragile]

\frametitle{Logs everywhere!}


\begin{center}
\includegraphics[scale=0.5]{png/dhcp-filelist-lognames}
\end{center}

\end{frame}



% ----------------------------
\begin{frame}[fragile]

\frametitle{SysLogAgent!}

\begin{center}
\includegraphics[scale=0.5]{png/syslogagent-dhcplog}
\end{center}


\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Foiled!}

\begin{center}
\includegraphics[scale=0.5]{png/dhcp-filelist-timestamps}
\end{center}

\end{frame}



% ----------------------------
\begin{frame}

\frametitle{Log Parser 2.2}

Microsoft has a freeware utility called \highlight{Log Parser} which
can help. (Microsoft employees get frustrated by Windows too.) 

It is a commandline ``any-to-any'' log converter with SQLesque syntax. 

You can run it every minute with Task Scheduler

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Log Parser Magic Syntax}

\begin{codeblock}
LogParser.exe -i:TEXTLINE -iCheckPoint:check.lpc 
  -o:SYSLOG -hostname:dc1 -processName:DHCP[info] 
  "SELECT * INTO @192.168.1.40 
  FROM DhcpSrvLog-*.log WHERE Index > 30"
\end{codeblock}

\begin{description}

\item[-i:TEXTLINE] Text file input

\item[-iCheckPoint] Remember the last location

\item[-o:SYSLOG] Syslog format output

\item[INTO @192.168.1.40] Send to logserver

\item[WHERE Index > 30] Skip first 30 lines

\end{description}


\end{frame}


% ----------------------------
\section{Tenshi}
\begin{frame}[fragile]

\frametitle{Tenshi}

\begin{center}
\includegraphics[scale=0.4]{png/tenshi-splash}
\end{center}

\end{frame}

% ----------------------------
\begin{frame}

\frametitle{Who Watches the Logs?}

My goals: be lazy but informed

\begin{itemize}

\item Get alerted when important things happen 

\item Get summaries of interesting log events

\item Format the stuff so I will actually read it without feeling 
swamped.

\end{itemize}

\end{frame}

% ----------------------------
\begin{frame}

\frametitle{Log Watchers}

There are lots of them: \code{swatch}, \code{logwatch}, 
\code{sec}, \code{log2mail}, \code{logsentry}...

My arbitrary choice: \code{tenshi}

\end{frame}

% ----------------------------
\begin{frame}

\frametitle{Tenshi concepts}

Tenshi collects log messages into \highlight{queues} . 

Identical messages are \highlight{tallied} in reports. 

You can use \highlight{masks} to filter irrelevant information and
make different messages appear identical to Tenshi.

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Basic configuration}

In \code{/etc/tenshi/tenshi.conf}

Specify logfiles to watch:
\begin{codeblock}
set logfile /var/log/auth.log
set logfile /var/log/remote-logs/dc1-dhcp.log
\end{codeblock}

Limit report size from host (default is 800)
\begin{codeblock}
set limit 80 
\end{codeblock}
This says that a host may produce 80 lines of information per report.

\end{frame}


% ----------------------------
\subsection{Tenshi queues}
\begin{frame}[fragile]

\frametitle{Queues}

Queues are used to sort messages and send them at different
frequencies and in different ways. Syntax:

\begin{codeblock}
set queue <queue_name> <mail_from> <mail_to> 
  <interval>
\end{codeblock}

This queue will be flushed at most every two minutes. If there are no
alerts it will do nothing. The subject will be ``Log Alert!''
\begin{codeblock}
set queue important 
  tenshi@localhost alerts@contoso.com  
  [*/2 * * * *] Log alert!
\end{codeblock}

\end{frame}


% ----------------------------
\begin{frame}[fragile]

\frametitle{More Queues}

This queue goes out every Wednesday at 4:20pm with the default subject
(which can be set in \code{tenshi.conf})
\begin{codeblock}
set queue report tenshi@localhost 
  list@contoso.com  [20 16 * * Wed]
\end{codeblock}

This queue goes out immediately and is sent to a pager and a mailing
list with the subject ``Emergency!''
\begin{codeblock}
set queue emergency tenshi@localhost
  pager:37337@pager.com,alert@contoso.com
  [now] Emergency!
\end{codeblock}

A builtin queue called \code{trash} is used to ignore messages
entirely.

\end{frame}

% ----------------------------
\subsection{Tenshi rules}
\begin{frame}[fragile]

\frametitle{Tenshi rules}

\vspace*{2cm}

Rules for filtering messages go in \code{/etc/tenshi/includes-active/}

They are specified using regular expressions. 

Like \code{rsyslog}, order matters. 

Unlike \code{rsyslog} the first rule that applies ``eats'' the
message. 

\pgfputat{\pgfxy(7,5.7)}
{\pgfimage[height=3cm]{png/regular-expressions-clipped}}


\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Sample Rules}

Context: Firewall messages look like this: 

\begin{codeblock}
pf: 138214 rule 60/0(match): block in on 
em0: (tos 0x0, ttl 118, id 49601, offset 0, 
flags [DF], proto TCP (6), length 58) 
209.73.191.147.1935 > 174.113.185.28.59609: 
P, cksum 0x8198 (correct), 0:18(18) ack 
1 win 65535
\end{codeblock}

They all begin with \code{pf:}

I am largely interested in the IP addresses and ports: 
\begin{codeblock}
209.73.191.147.1935 > 174.113.185.28.59609: 
\end{codeblock}

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Code blocks}

Apply the following rules onto to messages beginning with \code{pf:}
\begin{codeblock}
group ^pf:
\end{codeblock}

which you end with 
\begin{codeblock}
group_end
\end{codeblock}

Report any message that comes from an address and port 6669
\begin{codeblock}
important \d+\.\d+\.\d+\.\d+\.6669[ :]
\end{codeblock}

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{}

Mask out almost everything about firewall traffic that passes using 
the parentheses. 


\begin{codeblock}
report (\d+) .+? pass in on .+?: (.+)
\end{codeblock}

Sample output:
\begin{codeblock}
___ rule 54/0(match): pass in on xl0: ___ 
\end{codeblock}

\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Down the slippery slope}

Actual tenshi rules. They match things like: 

\begin{codeblock}
DHCP[info]c:\WINDOWS\system32
\dhcp\DhcpSrvLog-Tue.log 
78 8 10,08/04/09,09:29:04,Assign,
192.168.1.66,58tf-loftX.,00065BCAF8BD,
\end{codeblock}

\begin{codeblock}
report ^DHCP\[.+?](.+?) 
  \d\d,(\d\d\/\d\d\/\d\d,\d\d:\d\d:\d\d,)
  Renew

dhcp ^DHCP\[.+?](c:.+?\.log) .*?,Assign

important ^DHCP\[.+?](c:.+?\.log) .*?,Conflict
\end{codeblock}

\end{frame}

% ----------------------------
\begin{frame}

\frametitle{``I know regular expressions!''}

\begin{center}
Some people, when confronted with a problem, think ``I know,
I'll use regular expressions.''  Now they have two problems.
\end{center}
--Jamie Zawinski, August 1997

If you disagree, you might check out the \code{logwatch-database}
package. 

\end{frame}

% ----------------------------
\section{Lessons and Conclusions}
\begin{frame}[fragile]

\frametitle{Lessons Learned}

\begin{itemize}

\item Collecting and archiving logs can be worthwhile. 

\item \code{rsyslog} offers lots of new features and flexibility over
standard syslog. 

\item You can get syslog files from a lot of places (but the
formatting is often wretched).

\item Alerts for expected events work okay.

\item I'm still unhappy with reporting. Regular expressions are not
the right tool.

\end{itemize}


\end{frame}

% ----------------------------
\begin{frame}[fragile]

\frametitle{Thoughts and Future Work}

\highlight{Thought:} Log reporting is like spam filtering. 

\highlight{Idea:} Use database reporting to flag messages that I want
reported always, and to report any brand new messages I have never
seen. 

\end{frame}

% ----------------------------
\subsection{Thank You}
\begin{frame}[fragile]

\frametitle{Thank You!}

\begin{itemize}

\item \highlight{OpenClipArt} and its many contributors 
for releasing beautiful images I can use for free

\item \highlight{NetDirect} for the projector

\item \highlight{The Working Centre} for not firing me 
even though I embezzled
hardware and company time for this presentation  

\item \highlight{Randall Munroe}  at \url{xkcd.com} and \highlight{Jamie Zawinski}
for quotations

\item The authors of \code{rsyslog}, \code{SysLogAgent}, \code{tenshi}
and many other tools for giving me software to present about

\item The \highlight{\LaTeX}, \code{latex-beamer}, \highlight{GIMP}, 
and \highlight{Inkscape} people for giving
me tools to make this presentation. 

\end{itemize}


\end{frame}



% ----------------------------
\begin{frame}

\frametitle{OpenClipArt Credits}

In no particular order. I used real names when I could conveniently
find them, handles on \url{openclipart.org} or the filenames otherwise. 

\begin{itemize}


\item  Nicubunu (smileys)

\item Gerald G (log/campfire)

\item nicolas (Wireless box)

\item Luiz Araujo (alert)

\item Linda Kim (crossbones)

\item Francesco Rollandin (dinosaurs)

\item Andrew Fitzsimon (laptop, printer)

\item denco (firewall)

\item Chris Goerner (fly)


\end{itemize}

\end{frame}

% ----------------------------
\begin{frame}

\frametitle{More OpenClipArt Credits}

Still in no particular order: 

\begin{itemize}


\item glenn rolla (workstation) 

\item Nicolas cl (Internet cloud)

\item Nicu Buculei  (no symbol)

\item teudimundo (server)

\item Jarna Vasmaa (paper)

\item mimooh (server)

\item Feth Arezki (xbill) 


\end{itemize}


\end{frame}


% ----------------------------
\begin{frame}

\frametitle{The End}


\begin{center}
\includegraphics[scale=0.3]{png/theend-splash}
\end{center}

\end{frame}



\end{document}
