#!/usr/bin/perl ##### # PIX log summarizer # # version 0.2, 2004-04-02, changes for PIX blade, Matthew.Dalton@rochester.edu # version 0.1, 2002-06-06, the original, Liudvikas Bukys ##### $help_text = "PIX log summarizer options: -s summary_file To provide list of options for summarizing down events related to certain hosts, ports, foreign host/local host combinations, etc. -S Don't summarize events (use if you only want per-conduit statistics) -c conduit_file To provide list of conduits in place -C conduit_threhold Set threshold where conduit statistics print. Default: 0 (print stats for all conduits) Useful: 1 (print only conduits that have some events associated with them) "; ##### # It reads the most popular PIX syslog entries, and summarizes all the events found there. # It also predicts what conduits are used to decide access for each event. # (It can be used for after-the-fact analysis, or before-the fact examination of what will result # from new conduits.) # It summarizare both outbound and inbound events, but it only understands inbound conduits # (not outbound access lists). # Log summarization rules can be used to collapse numerous events into much smaller bins. # Note: it is limited by the way that the PIX logs information: # - Certain conduits dynamically open temporary access to TCP ports (e.g. FTP data connections) # Statically looking at the conduits suggests that "this shouldn't be permitted", you have to # read between the lines # Due to a fortuitous bug, dynamic FTP connections get looged with TCP port 0, which makes # them easier to spot and understant. # Unfortunately, the same is not true of UDP DNS traffic, which looks all over the map. # - ICMP traffic not logged. ##### # sample log entries: # Oct 28 13:45:57 router Oct 28 1999 13:47:30: %PIX-6-302001: Built inbound TCP connection 512495 for faddr 128.151.81.205/1057 gaddr 128.151.215.5/23 laddr 128.151.215.5/23 # Oct 28 13:46:04 router Oct 28 1999 13:47:38: %PIX-6-302001: Built outbound TCP connection 512508 for faddr 128.151.225.95/113 gaddr 128.151.223.3/45966 laddr 128.151.223.3/45966 # Oct 28 13:46:04 router Oct 28 1999 13:47:38: %PIX-6-302005: Built UDP connection for faddr 128.151.7.6/1 gaddr 128.151.223.3/34847 laddr 128.151.223.3/34847 # Feb 28 12:11:59 router Feb 28 2000 12:15:26: %PIX-2-106001: Inbound TCP connection denied from 209.138.199.165/2048 to 128.151.215.5/23 flags SYN # Mar 2 13:08:20 router Mar 02 2000 13:11:51: %PIX-2-106012: Deny IP from 63.84.162.97 to 128.151.223.3, IP options 0x803c11c4 ############################################################################### sub aton { my $str = shift(@_); my @oct = split(/\./, $str); return ($oct[0]<<24)|($oct[1]<<16)|($oct[2]<<8)|($oct[3]); } sub matchip { my $host = shift(@_); my $addr = shift(@_); my $mask = shift(@_); #$hoststr = ($host>>24&255).'.'.($host>>16&255).'.'.($host>>8&255).'.'.($host&255); #$addrstr = ($addr>>24&255).'.'.($addr>>16&255).'.'.($addr>>8&255).'.'.($addr&255); #$maskstr = ($mask>>24&255).'.'.($mask>>16&255).'.'.($mask>>8&255).'.'.($mask&255); #print "match $hoststr $addrstr $maskstr ", ($host&$mask) == ($addr&$mask), "\n"; return ($host&$mask) == ($addr&$mask); } sub matchport { my $port = shift(@_); my $not = shift(@_); my $min = shift(@_); my $max = shift(@_); my $result = ($min <= $port) && ($port <= $max); if ($not) { #print "matchport $port ($not $min $max) = not $result\n"; return !$result; } else { #print "matchport $port ($not $min $max) = $result\n"; return $result; } } sub matchproto { my $packetproto = shift(@_); my $conduitproto = shift(@_); #print "matchproto $packetproto $conduitproto ", ($conduitproto eq 'IP') || ($conduitproto eq $packetproto), "\n"; return ($conduitproto eq 'IP') || ($conduitproto eq $packetproto); } sub conduits_by_lhostaddr { my $lhostaddr = shift(@_); my $listref; my @list; my $c; if ($listref = $conduit_lhostaddr_cache{$lhostaddr}) { @list = @{$listref}; #print "OLD $lhostaddr $#list:"; foreach $c (@list) { print " $c"; } print "\n"; } else { @list = (); for ($c = 1; $c <= $conduit_count; $c++) { if (matchip($lhostaddr, $c_inside[$c], $c_inmask[$c])) { push @list, $c; } } #print "NEW $lhostaddr $#list:"; foreach $c (@list) { print " $c"; } print "\n"; $conduit_lhostaddr_cache{$lhostaddr} = [ @list ]; } return @list; } ############################################################################### #select(STDOUT); $| = 1; # make unbuffered $start_time = time(); ($start_cpu, $start_sys, $child_cpu, $child_sys) = times(); ############################################################################### while ($option = shift @ARGV) { if ($option eq "-s") { $summary_file = shift @ARGV; } elsif ($option eq "-c") { $conduit_file = shift @ARGV; } elsif ($option eq "-C") { $conduit_threshold = shift @ARGV; } elsif ($option eq "-S") { $dont_summarize = 1; } elsif ($option eq "-h") { print $help_text; exit 0; } else { print "ERROR: unrecognized command line option '$option'\n"; print $help_text; exit 1; } } ############################################################################### # read in summary exceptions from pix-summary.txt $summary_file eq "" || open(SUMMARY, $summary_file) || die("ERROR: can't read summary file $summary_file\n"); $summary_line = 0; while () { $summary_line++; s/[\r\n]+$//; s/#.*//; @args = split(/[ \t]+/); next if $#args < 0; $mask = 0; while ($name = shift @args) { $arg = shift @args; if ($arg eq "") { print "ERROR: missing argument after '$name' in summary file line $summary_line '$_'\n"; next; } if ($name eq "options") { $mask |= 1; $options = $arg; } elsif ($name eq "fhost") { $mask |= 2; $fhost = $arg; } elsif ($name eq "lhost") { $mask |= 4; $lhost = $arg; } elsif ($name eq "fport") { $mask |= 8; $fport = $arg; } elsif ($name eq "lport") { $mask |= 16; $lport = $arg; } else { print "ERROR: bad keyword '$name' in summary file line $summary_line '$_'\n"; next; } } if (($mask&1) == 0) { print "ERROR: missing options in summary file line $summary_line '$_'\n"; next; } if ($mask == 3) { if ($old = $match_fhost{"$fhost"}) {; print "WARNING: new options overriding '$old' in summary file line $summary_line '$_'\n"; } $match_fhost{"$fhost"} = $options; } elsif ($mask == 7) { if ($old = $match_fhost_lhost{"$fhost $lhost"}) {; print "WARNING: new options overriding '$old' in summary file line $summary_line '$_'\n"; } $match_fhost_lhost{"$fhost $lhost"} = $options; } elsif ($mask == 9) { if ($old = $match_fport{"$fport"}) {; print "WARNING: new options overriding '$old' in summary file line $summary_line '$_'\n"; } $match_fport{"$fport"} = $options; } elsif ($mask == 17) { if ($old = $match_lport{"$lport"}) {; print "WARNING: new options overriding '$old' in summary file line $summary_line '$_'\n"; } $match_lport{"$lport"} = $options; } elsif ($mask == 21) { if ($old = $match_lhost_lport{"$lhost $lport"}) {; print "WARNING: new options overriding '$old' in summary file line $summary_line '$_'\n"; } $match_lhost_lport{"$lhost $lport"} = $options; } else { print "ERROR: bad combination of keywords in summary file line $summary_line '$_'\n"; } } ############################################################################### $conduit_file eq "" || open(CONDUITS, $conduit_file) || die("ERROR: can't read $conduit_file\n"); # read in conduit lists from pix-conduits.txt while () { s/[\r\n]+$//; s/\(.*\)//g; s/#.*//; s/\s+/ /g; s/^\s//; last if (/conduit end/); next if (/^$/); if (!/^conduit/) { print "ERROR: not a conduit '$_'\n"; next; } $orig = $_; s/any/0.0.0.0 0.0.0.0/g; s/host (\d*\.\d*\.\d*\.\d*)/$1 255.255.255.255/g; s/bgp/179/; s/biff/512/; s/bootpc/68/; s/bootps/67/; s/chargen/19/; s/cmd/514/; s/daytime/13/; s/discard/9/; s/dnsix/195/; s/domain/53/; s/echo/7/; s/exec/512/; s/finger/79/; s/ftp/21/; s/ftp-data/20/; s/gopher/70/; s/hostname/101/; s/ident/113/; s/irc/194/; s/isakmp/500/; s/klogin/543/; s/kshell/544/; s/lpd/515/; s/login/513/; s/mobile-ip/434/; s/nameserver/42/; s/netbios-ns/137/; s/netbios-dgm/138/; s/nntp/119/; s/ntp/123/; s/pim-auto-rp/496/; s/pop2/109/; s/pop3/110/; s/rip/520/; s/smtp/25/; s/snmp/161/; s/snmptrap/162/; s/sqlnet/1521/; s/sunrpc/111/; s/syslog/514/; s/tacacs/49/; s/talk/517/; s/telnet/23/; s/tftp/69/; s/time/37/; s/uucp/540/; s/who/513/; s/whois/43/; s/www/80/; s/xdmcp/177/; $not = 0; if (/eq [^\d]/) { print "WARNING: NEED NUMERIC PORT NUMBER for '$_'\n" } s/eq (\d*)/range $1 $1/; if (/lt (\d*)/) { s/lt (\d*)/range $1 65536/; $not = 1; } if (/gt (\d*)/) { s/gt (\d*)/range 0 $1/; $not = 1; } if (/neq (\d*)/) { s/eq (\d*)/range $1 $1/; $not = 1; } if (/range/) { ($conduit, $permitq, $proto, $inside, $inmask, $range, $min, $max, $outside, $outmask) = split; } else { ($conduit, $permitq, $proto, $inside, $inmask, $outside, $outmask) = split; $min = 0; $max = 65536; } $proto =~ tr/a-z/A-Z/; ++$c; $c_orig[$c] = $orig; $c_permitq[$c] = "$permitq"; $c_proto[$c] = $proto; $c_inside[$c] = aton($inside); $c_inmask[$c] = aton($inmask); $c_not[$c] = $not; $c_min[$c] = $min; $c_max[$c] = $max; $c_outside[$c] = aton($outside); $c_outmask[$c] = aton($outmask); $c_count[$c] = 0; $c_first[$c] = $c_last[$c] = ' ' x 20; # print "\n$orig\n"; # print "$c $conduit $permitq $proto $inside $inmask ($not $min $max) $outside $outmask\n"; } # default deny $conduit_count = ++$c; $c_orig[$c] = "CONDUIT DEFAULT DENY"; $c_permitq[$c] = "deny"; $c_proto[$c] = "IP"; $c_inside[$c] = aton('0.0.0.0'); $c_inmask[$c] = aton('0.0.0.0'); $c_not[$c] = 0; $c_min[$c] = 0; $c_max[$c] = 65536; $c_outside[$c] = aton('0.0.0.0'); $c_outmask[$c] = aton('0.0.0.0'); $c_count[$c] = 0; $c_first[$c] = $c_last[$c] = ' ' x 20; ############################################################################### # accumulate packet counts and unique proto/src/dst/dstport events $event_count = 0; $ignored_count = 0; $eventmax = 0; while (<>) { s/[\r\n]+$//; ($mon, $day, $time, $host, $fwmon, $fwday, $fwyear, $fwtime, $fwcode, $fwtext) = split(/[ \t,]+/,$_,10); $fwtime =~ s/:$//; if ( $fwcode eq "%PIX-6-302001:" || $fwcode eq "%FWSM-6-302001:" || $fwcode eq "%PIX-6-302005:" || $fwcode eq "%FWSM-6-302005:" || $fwcode eq "%PIX-2-106001:" || $fwcode eq "%PIX-2-106006:" || $fwcode eq "%PIX-2-106007:" || $fwcode eq "%FWSM-4-106010:" || $fwcode eq "%FWSM-4-106023:" || $fwcode eq "%FWSM-6-106015:" || $fwcode eq "%PIX-2-106012:") { $last = "$fwmon $fwday $fwyear $fwtime"; $first = $last if (!$first); $event_count++; if ($fwcode eq "%PIX-6-302001:" || $fwcode eq "%FWSM-6-302001:") { ($built, $dir, $proto, $connection, $connection_num, $for, $faddr, $fhost, $fport, $gaddr, $ghost, $gport, $laddr, $lhost, $lport) = split(/[ ,\/]+/,$fwtext); $other = ""; } elsif ($fwcode eq "%PIX-6-302005:" || $fwcode eq "%FWSM-6-302005:") { ($built, $proto, $connection, $for, $faddr, $fhost, $fport, $gaddr, $ghost, $gport, $laddr, $lhost, $lport) = split(/[ ,\/]+/,$fwtext); $dir = "packets"; $other = ""; } elsif ($fwcode eq "%PIX-2-106001:") { ($dir, $proto, $connection, $denied, $from, $fhost, $fport, $to, $lhost, $lport) = split(/[ ,\/]+/,$fwtext); $dir = "DENIED-$dir"; $other = ""; } elsif ($fwcode eq "%FWSM-4-106023:") { @line = split(/[ ,\/:]+/,$fwtext); $proto = $line[1]; $dir = $line[$#line]; $dir =~ s/"//g; $fhost = $line[4]; $fport = $line[5]; $lhost = $line[8]; $lport = $line[9]; $dir = "DENIED-$dir"; $other = ""; } elsif ($fwcode eq "%PIX-2-106006:" || $fwcode eq "%PIX-2-106007:") { ($deny, $dir, $proto, $from, $fhost, $fport, $to, $lhost, $lport, $other) = split(/[ ,\/]+/,$fwtext,10); $dir = "DENIED-$dir"; } elsif ($fwcode eq "%FWSM-4-106010:") { ($deny, $dir, $proto, $from, $finterface, $fhost, $fport, $to, $linterface, $lhost, $lport, $other) = split(/[ ,\/\:]+/,$fwtext,10); $dir = "DENIED-$dir"; } elsif ($fwcode eq "%PIX-2-106012:") { ($deny, $proto, $from, $fhost, $to, $lhost, $other) = split(/[ ,\/]+/,$fwtext,7); $dir = "DENIED-IP"; $fport = $lport = "-"; $other =~ s/options .*/options .../; } if ($dir eq "outbound") { $predict = "outbound"; $conduit = ""; } else { $conduit_shortkey = "$lhost $fhost $proto"; if (($c = $conduit_cache{$conduit_shortkey})==0) { $conduit_cache_misses++; $lhostaddr = aton($lhost); $fhostaddr = aton($fhost); foreach $x (conduits_by_lhostaddr($lhostaddr)) { $c = $x; if (matchip($fhostaddr, $c_outside[$c], $c_outmask[$c]) && matchproto($proto, $c_proto[$c])) { if (!defined($conduit_cache{$conduit_shortkey})) { if ($c_min[$c] == 0 && $c_max[$c] == 65536 && $c_not[$c]==0) { $conduit_cache{$conduit_shortkey} = $c; #print "conduit#$c cache '$conduit_shortkey'\n"; } else { $conduit_cache{$conduit_shortkey} = 0; #print "conduit#$c no cache '$conduit_shortkey' $c_min[$c] $c_max[$c]\n"; } } #else { print "conduit#$c repeat '$conduit_shortkey' $c_min[$c] $c_max[$c]\n"; } if (matchport($lport, $c_not[$c], $c_min[$c], $c_max[$c])) { last; } } } } $predict = $c_permitq[$c]; $conduit = $c; if ($c_count[$c]++ == 0) { $c_first[$c] = "$fwmon $fwday $fwyear $fwtime"; } $c_last[$c] = $last; } if (!$dont_summarize) { if ($dir eq "inbound" || $dir eq "DENIED-Inbound" || $dir eq "DENIED-inbound") { $arrow = "=>"; $fport = "*"; } elsif ($dir eq "outbound") { $arrow = "<="; $lport = "*"; } else { $arrow = "<=>"; } if ($action = ($match_fhost{$fhost} . $match_fport{$fport} . $match_lport{$lport} . $match_fhost_lhost{"$fhost $lhost"} . $match_lhost_lport{"$lhost $lport"}) ) { if ($action =~ /F/g) { $fhost =~ s/(128\.151)\.[0-9*]+\.[0-9*]+/$1.*.*(UR)/; $fhost =~ s/(207\.127\.120)\.[0-9*]+/$1.*(EDC)/; $fhost =~ s/(207\.127)\.12[1-7]\.[0-9*]+/$1.12[1-7].*(HIGHLAND)/; $fhost =~ s/(207\.127)\.22[4-6]\.[0-9*]+/$1.22[4-6].*(HIGHLAND)/; $fhost =~ s/(207\.127\.227)\.[0-9*]+/$1.*(COLGATE)/; $fhost =~ s/(172\.1[6-9])\.[0-9*]+\.[0-9*]+/$1.*.*(PRIVATE)/; $fhost =~ s/(172\.2[0-9])\.[0-9*]+\.[0-9*]+/$1.*.*(PRIVATE)/; $fhost =~ s/(172\.3[0-1])\.[0-9*]+\.[0-9*]+/$1.*.*(PRIVATE)/; $fhost =~ s/[0-9*]+\.[0-9*]+\.[0-9*]+\.[0-9*]+$/*.*.*.*(INTERNET)/; } $fport = "***" if ($action =~ /f/g); $lhost =~ s/[0-9]*$/***/ if ($action =~ /L/g); if ($action =~ /M/g) { $lhost =~ s/(128\.151)\.[0-9*]+\.[0-9*]+/$1.*.*(UR)/; $lhost =~ s/(207\.127\.120)\.[0-9*]+/$1.*(EDC)/; $lhost =~ s/(207\.127)\.12[1-7]\.[0-9*]+/$1.12[1-7].*(HIGHLAND)/; $lhost =~ s/(207\.127)\.22[4-6]\.[0-9*]+/$1.22[4-6].*(HIGHLAND)/; $lhost =~ s/(207\.127\.227)\.[0-9*]+/$1.*(COLGATE)/; $lhost =~ s/(172\.1[6-9])\.[0-9*]+\.[0-9*]+/$1.*.*(PRIVATE)/; $lhost =~ s/(172\.2[0-9])\.[0-9*]+\.[0-9*]+/$1.*.*(PRIVATE)/; $lhost =~ s/(172\.3[0-1])\.[0-9*]+\.[0-9*]+/$1.*.*(PRIVATE)/; $lhost =~ s/[0-9*]+\.[0-9*]+\.[0-9*]+\.[0-9*]+$/*.*.*.*(INTERNET)/; } $lport = "***" if ($action =~ /l/g); } $event = "$predict#$conduit $dir $proto $fhost port $fport $arrow $lhost port $lport $other"; if (++$event_counts{$event} == 1) { $key = "$fhost $lport $predict $lhost $dir $fport $proto $conduit"; $key =~ s/\d+/0000$&/g; $key =~ s/\d+(\d{5})/$1/g; $event_keys{$event} = $key; $event_break{$event} = "$fhost $lport"; $event_first{$event} = "$fwmon $fwday $fwyear $fwtime"; $first = $event_first{$event} if (!$first); } $event_last{$event} = $last; if ($fhost =~ /128\.151\.[0-9*]+\.[0-9*]+/) { $site = "UR "; } elsif ($fhost =~ /207\.127\.120\.[0-9*]+/) { $site = "EDC "; } elsif ($fhost =~ /207\.127\.12[1-7]\.[0-9*]+/) { $site = "HIGHLAND"; } elsif ($fhost =~ /207\.127\.22[4-6]\.[0-9*]+/) { $site = "HIGHLAND"; } elsif ($fhost =~ /207\.127\.227\.[0-9*]+/) { $site = "COLGATE "; } elsif ($fhost =~ /172\.1[6-9]\.[0-9*]+\.[0-9*]+/) { $site = "PRIVATE "; } elsif ($fhost =~ /172\.2[0-9]\.[0-9*]+\.[0-9*]+/) { $site = "PRIVATE "; } elsif ($fhost =~ /172\.3[0-1]\.[0-9*]+\.[0-9*]+/) { $site = "PRIVATE "; } else { $site = "* "; } $dstport = sprintf("%5s", $lport); $class = "port $dstport from $site"; $event = "$class $fhost"; $summary_totals{$class} += 1; if (++$summary_events{$event} == 1) { $summary_distinct{$class}++; } } } else { if ($fwcode !~ /%PIX.*/ && $fwcode !~ /%FWSM.*/) { ($fwcode,$junk) = split(/\b/,$fwmon); } $ignored_example{$fwcode} = "$fwmon $fwday $fwyear $fwtime $fwcode $fwtext"; $ignored_counts{$fwcode}++; $ignored_count++; } } # output the results print "DATES $first - $last\tEVENTS SUMMARIZED $event_count IGNORED $ignored_count\n"; foreach $class (sort keys %summary_distinct) { print "$class\tdistinct source hosts\t$summary_distinct{$class}\ttotal events\t$summary_totals{$class}\n"; } sub byladdr { $event_keys{$a} cmp $event_keys{$b}; } $prev = ""; foreach $event (sort byladdr keys %event_keys) { if (($cur = $event_break{$event}) ne $prev) { print "\n"; $prev = $cur; } print "$event_first{$event} - $event_last{$event}\t$event_counts{$event}\t$event\n"; } print "\n \n"; for ($c = 1; $c <= $conduit_count; $c++) { print "$c_first[$c] - $c_last[$c]\t$c_count[$c]\tconduit#$c $c_orig[$c]\n" if $c_count[$c] >= $conduit_threshold; } print "\n \n"; foreach $fwcode (sort keys %ignored_counts) { print "ignored $fwcode count $ignored_counts{$fwcode} example '$ignored_example{$fwcode}'\n"; } ############################################################################### $hits = $event_count-$conduit_cache_misses; $hitrate = sprintf("%.1f", (100.0*$hits/$event_count)) if $event_count>0; $stop_time = time(); ($stop_cpu, $stop_sys, $child_cpu, $child_sys) = times(); $secs = $stop_time - $start_time; $cpus = $stop_cpu - $start_cpu; if ($secs > 0 && $cpus > 0) { $rate = int($event_count/$secs); $rate_cpu = int($event_count/$cpus); print "\n$event_count events / $secs seconds = $rate events/sec, $event_count events / $cpus cpu-seconds = $rate_cpu events/cpu-sec, $hitrate% cache hit rate\n"; } else { print "\n$event_count events / $secs seconds, $event_count events / $cpus seconds cpu, $hitrate% cache hit rate\n"; }