#!/usr/bin/env php
<?php

/*------------------------------------------------------------------\
| LORG v0.41 | Sat Jun 15 20:20:22 CEST 2013                        |
| Logfile Outlier Recognition and Gathering                         |
|                                                                   |
| Author: Jens Mueller {jens.a.mueller@rub.de}, Ruhr-Uni Bochum     |
| License: GPL v2 (http://www.gnu.org/licenses/gpl-2.0.html)        |
|                                                                   |
| **************************** INSTALL **************************** |
| STEP 1: (optional) get PHPIDS from http://phpids.org, gunzip      |
|         and untar, then move IDS/ into the following directory    |
|         */ static $phpids_path = './phpids/'; /*                  |
| STEP 2: (optional) for geotargeting, download php-x.xx.tar.gz     |
|         and GeoLiteCity.dat.gz from http://geolite.maxmind.com/   |
|         and gunzip/untar all files into the following directory   |
|         */ static $geoip_path = './geoip/'; /*                    |
| STEP 3: (optional) for some nice graphics in HTML reports, get    |
|         the php5-gd library and pChart from http://pchart.net/    |
|         and gunzip/untar all files into the following directory   |
|         */ static $pchart_path = './pchart/'; /*                  |
| STEP 4: run ./lorg.php access.log                                 |
|                                                                   |
| *************************** CONFIGURE *************************** |
| you can define your own Apache-style logline formats, e.g.        |
| 'custom' => '%h %l %u %t \"%r\" %>s %b %{X-Forwarded-For}'        |
| (see http://httpd.apache.org/docs/mod/mod_log_config.html)        |
\------------------------------------------------------------------*/

# define allowed input formats (apache style, feel free to complement)
static $allowed_input_types = array(
  'common'     => '%h %l %u %t \"%r\" %>s %b',
  'combined'   => '%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"',
  'vhost'      => '%v %h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"',
  'logio'      => '%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\ %I %O"',
  'cookie'     => '%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" \"%{Cookie}i\"'
);

# define output types, client identifiers and attack vectors (don't change!)
static $allowed_output_types = array('html', 'json', 'xml', 'csv');
static $allowed_client_ident = array('host', 'session', 'user', 'logname', 'all');
static $additional_attack_vectors = array('path', 'argnames', 'cookie', 'agent', 'all');

# define dnsbl types and corresponding servers (feel free to modify)
static $allowed_dnsbl_types = array(
  'tor'    => array('tor.dnsbl.sectoor.de'),
  'proxy'  => array('dnsbl.proxybl.org', 'http.dnsbl.sorbs.net', 'socks.dnsbl.sorbs.net'),
  'zombie' => array('xbl.spamhaus.org', 'zombie.dnsbl.sorbs.net'),
  'spam'   => array('b.barracudacentral.org', 'spam.dnsbl.sorbs.net', 'sbl.spamhaus.org'), # 'sbl.nszones.com': slow!
  'dialup' => array('dyn.nszones.com'), # slow!
  'all'    => ''
);

# define implemented modes of anomaly detection (don't change!)
static $allowed_detect_modes = array('chars', 'phpids', 'mcshmm', 'dnsbl', 'geoip', 'all'); # 'length'

# define implemented modes of attack quantification (don't change!)
static $allowed_quantify_types = array('status', 'bytes', 'replay', 'all');

# define known HTTP/1.1 methods (RFC 2616)
static $allowed_http_methods = array('HEAD', 'GET', 'POST', 'PUT', 'TRACE', 'OPTIONS', 'CONNECT');

# define session id strings to look for (case-insensitive)
# note: you can also use a regex like: '(\w)*SESS(ION)?ID(\w)*'
static $session_identifiers = array('SID', 'SESSID', 'PHPSESSID', 'JSESSIONID', 'ASP.NET_SessionId');

# define client inactivity time span until new session starts
static $max_session_duration = 3600;

# define maximum number of attack replays for a single client
static $max_replay_per_client = 1000;

# define maximum number of harmless requests to show between attacks
static $max_harmless_summarize = 100;

# define file extensions considered as web applications
static $web_app_extensions = array('cgi', 'php[3-5]?', 'phtml', 'pl', 'jsp', 'aspx?', 'cfm', 'exe');

# only detect attacks made to web applications (matched by file extension)
static $only_check_webapps = false;

# always normalize requests with PHPIDS (even for other detect modes)
static $use_phpids_converter = false;

# create pie chart of most noisy clients, if pChart and GD libraries installed
static $use_pchart_library = true;

// ------------------------- CHARS settings -------------------------//

# define minimum number of required observations
static $var_min_learn = 10;

# define maximum variance (above, algorithm will perform badly)
static $var_citical_val = 10;

// ------------------------- MCSHMM settings -------------------------//

# define minimum number of required observations
# (if not hit, we will not make a detection statement)
static $hmm_min_learn = 50;

# define maximum number of observations to learn
# (if hit, aggregation stops since we have enough training data)
static $hmm_max_learn = 150;

# define maximal number of iterations for training
# (if hit, training stops even HMM is not yet precise)
static $hmm_max_iter = 100;

# define tolerance for testing convergence function
static $hmm_tolerance = 1.0E-5;

# define probability of validity for unknown symbols
static $hmm_decrease = 1.0E-10;

# define the number of models (HMMs) per ensemble
static $hmm_num_models = 5;

// ------------------------- GEOIP settings ------------------------- //

# define minimum number of items to learn for LOF algorihm
static $lof_min_learn = 40;

# define maximum number of items to learn for LOF algorihm
static $lof_max_learn = 150;

# define lower and upper bounds of k-nearest neighbors
static $lof_minpts_lb = 10;
static $lof_minpts_ub = 20;

// ------------------------- LORG switches -------------------------- //

# by default, use a moderate detection threshold (see -t)
# note: this will also be the result for DNSBL detect mode
$default_threshold = 10;

# by default, don't be to chatty (see '-v')
$default_verbosity = 1;

# by default, don't do attack quantification (see '-q')
$quantify_type = array();

# by default, summarize detection results (see '-n')
$do_summarize = true;

# by default, don't try to determine hostnames (see '-h')
$dns_lookup = false;

# by default, don't do dnsbl lookups (see '-b')
$dnsbl_lookup = false;

# by default, don't do geoip lookups (see '-g')
$geoip_lookup = false;

# by default, don't try to urldecode requests (see '-u')
$url_decode = false;

# by default, don't use additional attack vectors (see '-a')
$add_vector = false;

# by default, don't do naive logfile tamper detection (see '-p')
$tamper_test = false;

// ------------------------------------------------------------------ //

# check if running from the command line
if (!defined("STDIN"))
  die("[!] Please run this programm from the CLI\n\n");

# set threshold to default
$GLOBALS['verbosity'] = $default_verbosity;

// ------------------------------------------------------------------ //

# remove first element first of argv (scriptname)
array_shift($argv);

# parse command line options and flags
foreach(getopt("i:o:d:a:c:b:q:t:v:nuhgp") as $opt => $value)
{
  ### echo "\n======================= <DEBUG: \$argv $opt -> $value> =======================\n";
  ### print_r($argv);
  ### echo "======================= </DEBUG: \$argv ======================\n";

  switch ($opt)
  {
    case 'i':
      $input_type = $value;
      // check for correct input file type
      if (array_key_exists($input_type, $allowed_input_types) == false)
      {
        print_message(0, "[!] Input format '$input_type' not allowed\n");
        usage_die();
      }
      // remove switch and option from argument vector
      if (strlen($argv[0]) <= 2) array_shift($argv); array_shift($argv);
      break;

    case 'o':
      $output_type = $value;
      // check for correct output file type
      if (in_array($output_type, $allowed_output_types) == false)
      {
        print_message(0, "[!] Output format '$output_type' not allowed\n");
        usage_die();
      }
      // remove switch and option from argument vector
      if (strlen($argv[0]) <= 2) array_shift($argv); array_shift($argv);
      break;

    case 'd':
      $detect_mode = (is_array($value) ? $value : array($value));
      foreach ($detect_mode as $mode)
      {
        // check for correct detection mode
        if (in_array($mode, $allowed_detect_modes) == false)
        {
          print_message(0, "[!] Detect mode '$mode' not allowed\n");
          usage_die();
        }
        // remove switch and option from argument vector
        if (strlen($argv[0]) <= 2) array_shift($argv); array_shift($argv);
      }
      // if 'all' selected, create array of detection modes
      if (in_array('all', $detect_mode))
      {
        $detect_mode = $allowed_detect_modes;
        array_pop($detect_mode); // remove element 'all'
      }
      break;

    case 'a':
      $add_vector = array($value);
      // check for correct attack vector
      if (in_array($add_vector[0], $additional_attack_vectors) == false)
      {
        print_message(0, "[!] Additional attack vector '$add_vector[0]' not allowed\n");
        usage_die();
      }
      // if all selected, create array of additional attack vectors
      if ($add_vector[0] == 'all')
      {
        $add_vector = $additional_attack_vectors;
        array_pop($add_vector); // remove element 'all'
		  }
      // remove switch and option from argument vector
      if (strlen($argv[0]) <= 2) array_shift($argv); array_shift($argv);
      break;

    case 'c':
      $client_ident = $value;
      // check for correct client identifier type
      if (in_array($client_ident, $allowed_client_ident) == false)
      {
        print_message(0, "[!] Remote ID '$client_ident' not allowed\n");
        usage_die();
      }
      // remove switch and option from argument vector
      if (strlen($argv[0]) <= 2) array_shift($argv); array_shift($argv);
      break;

    case 'b':
      $dnsbl_type = array($value);
      // check for correct dnsbl type
      if (array_key_exists($dnsbl_type[0], $allowed_dnsbl_types) == false)
      {
        print_message(0, "[!] DNSBL type '$dnsbl_type[0]' not allowed\n");
        usage_die();
      }
      // if all selected, create array of dnsbl types
      if ($dnsbl_type[0] == 'all')
      {
        $dnsbl_type = array_keys($allowed_dnsbl_types);
        array_pop($dnsbl_type); // remove element 'all'
		  }
      $dnsbl_lookup = true;
      // remove switch and option from argument vector
      if (strlen($argv[0]) <= 2) array_shift($argv); array_shift($argv);
      break;

    case 'q':
      $quantify_type = array($value);
      // check for correct attack vector
      if (in_array($quantify_type[0], $allowed_quantify_types) == false)
      {
        print_message(0, "[!] Quantification type '$quantify_type[0]' not allowed\n");
        usage_die();
      }
      // if all selected, create array of additional attack vectors
      if ($quantify_type[0] == 'all')
      {
        $quantify_type = $allowed_quantify_types;
        array_pop($quantify_type); // remove element 'all'
		  }
      // remove switch and option from argument vector
      if (strlen($argv[0]) <= 2) array_shift($argv); array_shift($argv);
      break;

    case 't':
      $threshold = $value;
      // check for correct $threshold (= minimum impact) type and value
      if (filter_var($threshold, FILTER_VALIDATE_INT, array('options' => array('min_range' => 0))) === false)
      {
        print_message(0, "[!] Threshold must be a positive integer or zero\n");
        usage_die();
      }
      // remove switch and option from argument vector
      if (strlen($argv[0]) <= 2) array_shift($argv); array_shift($argv);
      break;

    case 'v':
      $GLOBALS['verbosity'] = $value;
      // check for correct verbosity level type and value
      if (filter_var($value, FILTER_VALIDATE_INT, array('options' => array('min_range' => 0, 'max_range' => 3))) === false)
      {
        print_message(0, "[!] Verbosity must be an integer value from 0 to 3\n");
        usage_die();
      }
      // remove switch and option from argument vector
      if (strlen($argv[0]) <= 2) array_shift($argv); array_shift($argv);
      break;

    case 'n':
      $do_summarize = false;
      // we cannot handle two flags at a time
      if (strlen($argv[0]) > 2)
        usage_die();
      // remove flag from argument vector
      array_shift($argv);
      break;

    case 'u':
      $url_decode = true;
      // we cannot handle two flags at a time
      if (strlen($argv[0]) > 2)
        usage_die();
      // remove flag from argument vector
      array_shift($argv);
      break;

    case 'h':
      $dns_lookup = true;
      // we cannot handle two flags at a time
      if (strlen($argv[0]) > 2)
        usage_die();
      // remove flag from argument vector
      array_shift($argv);
      break;

    case 'g':
      $geoip_lookup = true;
      // we cannot handle two flags at a time
      if (strlen($argv[0]) > 2)
        usage_die();
      // remove flag from argument vector
      array_shift($argv);
      break;

    case 'p':
      $tamper_test = true;
      // we cannot handle two flags at a time
      if (strlen($argv[0]) > 2)
        usage_die();
      // remove flag from argument vector
      array_shift($argv);
      break;
  }
}

# parse command line arguments
if (isset($argv[0]))
  $input_file = $argv[0];
if (isset($argv[1]))
  $output_file = $argv[1];

// ------------------------------------------------------------------ //

# print newline to stdout
print_message(1, "\n");

# turn off PHP warnings in non-verbose mode
if ($GLOBALS['verbosity'] <= 1)
  error_reporting(E_ERROR | E_PARSE);

// ------------------------------------------------------------------ //

# exit, if no input file given
if (!isset($input_file))
{
  print_message(0, "[!] Specify at least an input logfile\n");
  usage_die();
}

# set input file basename
$input_file_basename = basename($input_file);

// ------------------------------------------------------------------ //

# try to open input file for reading
if (($input_stream = fopen($input_file, 'r')) == false)
{
  print_message(0, "[!] Cannot read from input file '$input_file'\n");
  usage_die();
}

// ------------------------------------------------------------------ //

# try auto-detection of input format, if none given
if (!isset($input_type))
{
  if ($input_type = detect_logformat($input_stream, $allowed_input_types))
    print_message(1, "[#] No input file format given - guessing '$input_type'\n");
  else
  {
    print_message(0, "[!] Cannot auto-detect input format of '$input_file_basename'\n");
    usage_die();
  }  
}

// ------------------------------------------------------------------ //

# set regex for given input format
format_to_regex($allowed_input_types[$input_type], $regex_fields, $regex_string, $num_fields);

# set geoip objects to zero-value
$geoip_stream = null; $geoip_data = null;

// ------------------------------------------------------------------ //

# set default output format, if none given
if (!isset($output_type))
{
  $output_type = $allowed_output_types[0];
  print_message(1, "[#] No output file format given - using '$output_type'\n");
}

// ------------------------------------------------------------------ //

# set default output filename, if none given
if (!isset($output_file))
{
  $output_file = 'report_' . date("d-M-Y-His") . '.' . $output_type;
  print_message(1, "[#] No output file given - using '$output_file'\n");
}

// ------------------------------------------------------------------ //

# check if pChart-installation is present, if needed
if (($output_type == 'html') and $do_summarize and $use_pchart_library)
{
  // try to include pChart framework
  $pchart_requires = array('pData.class.php', 'pDraw.class.php', 'pPie.class.php', 'pImage.class.php');
  $pchart_included = (extension_loaded('gd') and function_exists('gd_info'));
  $pchart_included = $pchart_included ? include_multiple("$pchart_path/class/", $pchart_requires) : false;

  // print errors in case pChart-installation not found
  if ($pchart_included)
  {
    print_message(2, "[*] pChart/php5-gd library found in '$pchart_path'\n");
  }
  else
  {
    print_message(1, "[#] Cannot find pChart/GD library - charting disabled\n");
    $use_pchart_library = false;
  }
}

// ------------------------------------------------------------------ //

# set default detection mode, if none given
if (!isset($detect_mode))
{
  $detect_mode = array($allowed_detect_modes[0]);
  print_message(1, "[#] No detect mode given - using '$detect_mode[0]'\n");
}

// ------------------------------------------------------------------ //

# check if PHPIDS-installation is present, if needed
if (in_array('phpids', $detect_mode) or $use_phpids_converter)
{
  # try to include PHPIDS framework
  $phpids_requires = array('Init.php', 'Event.php', 'Filter.php', 'Report.php', 'Converter.php');
  $phpids_included = include_multiple("$phpids_path/", $phpids_requires);

  # print errors in case PHPIDS-installation not found
  if ($phpids_included)
  {
    print_message(2, "[*] PHPIDS installation found in '$phpids_path'\n");
    // overwrite some configs so we can always find the PHPIDS directory
    $phpids_init = IDS_Init::init($phpids_path . '/Config/Config.ini.php');
    $phpids_init->config['General']['base_path'] = $phpids_path . '/';
    $phpids_init->config['General']['use_base_path'] = true;
  }
  else
  {
    // in case we're using PHPIDS detection
    if (in_array('phpids', $detect_mode))
    {
      unset($detect_mode[array_search('phpids', $detect_mode)]);
      // quit, if no other dectection modes are enabled
      if (count($detect_mode) == 0)
        die("[!] No PHPIDS installation found - exiting!\n\n");
      else
        print_message(0, "[#] No PHPIDS installation found - detection disabled\n");
    }

    // else, just disable PHPIDS request conversion
    if ($use_phpids_converter)
    {
      print_message(1, "[#] No PHPIDS installation found - request conversion disabled\n");
      $use_phpids_converter = false;
    }
  }
}
else // do not leave variable unset
  $phpids_init = null;

// ------------------------------------------------------------------ //

# ask for target host, if active replay enabled
if (in_array('replay', $quantify_type))
{
  print_message(0, "[#] Active replay enabled - this will re-probe all attacks!\n");
  // eh, screw good practics (...and use GOTO)
  REPLAY: $target = readline("    Please enter target host: ");
  // HINT: You need to configure PHP --with-readline
  // TODO: maybe migrate to stream_get_line()

  // add http:// substring to target URI, if not given
  if (!preg_match("/^https?:\/\//", $target))
    $target = 'http://' . $target;
  // try to connect to HTTP(S) server
  $body = @file_get_contents( $target . '/', false);
  if ($body === FALSE)
  {
    print_message(0, "[!] Cannot connect to '$target'\n");
    goto REPLAY; // a little GOTO here and there
  }
}
else
  $target = null;

// ------------------------------------------------------------------ //

# set default threshold, if none given
if (!isset($threshold))
{
  // process all requests with threshold greater ten
  $threshold = $default_threshold;
  print_message(1, "[#] No threshold given - using default value '$default_threshold'\n");
}
else
{
  if ($threshold > 0)
    print_message(1, "[#] Ignoring requests with a threshold less than '$threshold'\n");
  else
    print_message(1, "[#] Threshold set to zero - Logging everything!\n");
}

// ------------------------------------------------------------------ //

# set default client identifier, if none given
if (!isset($client_ident))
{
  $client_ident = $allowed_client_ident[0];
  print_message(1, "[#] No client identifier given - using '$client_ident'\n");
}

// ------------------------------------------------------------------ //

# check if geoip-information is present, if used
if ($geoip_lookup)
{
  // workaround: program crashes if php5-geoip is installed!
  if (function_exists('geoip_record_by_name'))
  {
    print_message(1, "[#] Incompatible php5-geoip installation found - geotargeting disabled\n");
    $geoip_lookup = false;
  }
  else
  {
    /*------------------------------------------------------------------------\
    | files needed for geotargeting:                                          |
    | *********************************************************************** |
    | - http://geolite.maxmind.com/download/geoip/api/php/php-x.xx.tar.gz     |
    | - http://geolite.maxmind.com/download/geoip/database/GeoLiteCity.dat.gz |
    |   (don't forget to gunzip/untar all files into $geoip_path folder)      |
    \------------------------------------------------------------------------*/

    # try to include 'maxmind geoip city' framework
    $geoip_requires = array('geoipregionvars.php', 'geoip.inc', 'geoipcity.inc');
    $geoip_included = include_multiple("$geoip_path/", $geoip_requires);
    $geoip_database = $geoip_path.'/GeoLiteCity.dat';

    # print errors in case geoip-installation not found
    if ($geoip_included and file_exists($geoip_database))
    {
      print_message(2, "[*] GeoIP information found in '$geoip_path'\n");
      # for speeding this up a bit, you might want to try GEOIP_SHARED_MEMORY or GEOIP_MEMORY_CACHE
      $geoip_stream = geoip_open($geoip_database, GEOIP_STANDARD);
    }
    else
    {
      print_message(1, "[#] No GeoIP information available - geotargeting disabled\n");
      $geoip_lookup = false;
    }
  }
}

// ------------------------------------------------------------------ //

# remove geoip from detection modes, if geoip lookup disabled
if (in_array('geoip', $detect_mode))
{
  if ($geoip_lookup == false)  
  {
    unset($detect_mode[array_search('geoip', $detect_mode)]);
    print_message(0, "[!] To use GeoIP detection, you have to enable geotargeting (-g)\n");
    // quit, if no other dectection modes are enabled
    if (count($detect_mode) == 0)
      die("\n");
  }
}

// ------------------------------------------------------------------ //

# remove dnsbl from detection modes, if dnsbl lookup disabled
if (in_array('dnsbl', $detect_mode))
  if ($dnsbl_lookup == false)  
  {
    unset($detect_mode[array_search('dnsbl', $detect_mode)]);
    print_message(0, "[!] To use DNSBL detection, you have to choose a DNSBL type (-b)\n");
    // quit, if no other dectection modes are enabled
    if (count($detect_mode) == 0)
      die("\n");
  }

// ------------------------------------------------------------------ //

# show speed warning, if dnsbl lookup enabled
if ($dnsbl_lookup)
  print_message(1, "[#] DNSBL lookup enabled - this might be a significant slowdown\n");

// ------------------------------------------------------------------ //

# show speed warning, if hostname lookup enabled
if ($dns_lookup)
  print_message(1, "[#] Hostname lookup enabled - this might be a significant slowdown\n");

// ------------------------------------------------------------------ //

# show speed warning, if additional attack vectors used
if ($add_vector)
  print_message(1, "[#] Scanning additional attack vectors - this might be a significant slowdown\n");

// ------------------------------------------------------------------ //
  
# show information, if url-decoding enabled
if ($url_decode)
  print_message(1, "[#] Non-binary urlencoded requests will be decoded\n");

// ------------------------------------------------------------------ //

# show message in case summarization is disabled
if ($do_summarize == false)
  print_message(1, "[#] Summarization and robot detection disabled\n");

// ------------------------------------------------------------------ //

# set counters for statistics
$line_index = 0; $progress = -1; $request_count = 0; $attack_count = 0;
$tags_count = 0; $tag_stats = $replay_count = array(); $success = null;
$pathes = array(); $dates = array();

# set if tags are to shown in output
$add_tags = in_array('phpids', $detect_mode) ? true : false;

# poor man's dns/dnsbl/geoip/lof cache
$dns_cache = $dnsbl_cache = $geoip_cache = $lof_cache = null;

# main dataset and client collection
$dataset = $clients = array();

// --------------------------------------------------------------

# breakpoint: read logfile, count it's lines and aggregate dataset if needed
function pre_processing() {}

// decide if we have to create a dataset
$do_aggregate = (in_array('chars', $detect_mode)
                 or in_array('length', $detect_mode)
                 or in_array('mcshmm', $detect_mode)
                 or in_array('geoip', $detect_mode)
                 or in_array('bytes', $quantify_type));

// decide if we have to read all lines of logfile
$do_readlines = ($do_aggregate or $tamper_test);

// save logfile modification time
$last_modified = filemtime($input_file);

// number of lines
$line_count = 0;

// number of vectors
$vector_count = 0;

// needed for variance calculation
$avg_delay = $avg_request = $avg_subst = $var_delay = $var_request = $var_subst = 0.0; $index_delay = $index_request = $index_subst = 0; 

// print feedback (important, when dealing with huge files)
print_message(1, '[>] ' . ($do_aggregate ? 'Creating dataset' : ($do_readlines ?
                 'Gathering statistics' : 'Counting number of lines')) . " of '$input_file_basename'" . ($do_readlines ? '' : "\n"));

// reset position indicator to zero
fseek($input_stream, 0);

// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

if ($do_readlines)
{
  // loop over input file
  while ($line = fgets($input_stream))
  {
    // increment number of lines
    $line_count++;

    // remove junk like MS-wordwraps
    $line = trim($line);

    // ignore empty lines
    if (empty($line))
      continue;

    // convert logline to HTTP object
    $data = logline_to_httpdata($line, $regex_string, $regex_fields, $num_fields);

    // create dataset
    if (isset($data))
    {
      // extract request (+additional attack vectors) and path from HTTP-data
      $vector = httpdata_to_vector($data, $add_vector, $use_phpids_converter, $phpids_path, $phpids_init,
                                   $detect_mode, $only_check_webapps, $web_app_extensions, $input_file, null);

      // get request used for detection
      $request = $vector[0];

      // get path of the web application
      $path = $vector[1];

      // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

      # calculate variance of inter request time delay, based on Welford1962 online algorithm
      if ($tamper_test and (isset($data['Date'])))
      {
        // convert date to unix timestamp format
        $current_date = strtotime(date("r", apachedate_to_timestamp($data['Date'])));

        if (isset($last_date))
        {
          // get inter-request time delay
          $value = $current_date - $last_date;

          // set maximum delay, if new max
          $max_delay = max($value, (isset($max_delay) ? $max_delay : 0));

          // online variance and mean calculation of delay
          online_variance($avg_delay, $var_delay, $index_delay, $value);
        }
        // set last date to current date
        $last_date = $current_date;
      }

      // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

      # aggregate dataset needed for mcshmm/geoip detection and attack quantification
      if ($do_aggregate)
      {
        // if request contains a remote-host entry
        if (array_key_exists('Remote-Host', $data))
        {
          // this might be an ip address _or_ a hostname
          $remote_host = $data['Remote-Host'];
          // convert hostnames to ip addresses (needed for geoip lookups)
          if (in_array('geoip', $detect_mode))
            $ipaddr = hostname_to_ipaddr($data['Remote-Host'], $dns_cache);
        }
        else
          // set remote-host to current line if client identified by ip address, else unknown
          $remote_host = ($client_ident == 'host') ? "client_$line_count" : 'unknown_host';

        // try to retrieve client's identity
        $client = client_identification($data, $client_ident, $remote_host, $session_identifiers);

        // if status code is valid (if present)
        if (isset($data['Final-Status']) ? preg_match("/^(2|3)[0-9]+$/", $data['Final-Status']) : true)
        {
          if (in_array('chars', $detect_mode)) # add information needed for chars detection
            aggregate_chars($dataset, $data, $path, $vector, $client, $request, $avg_subst, $var_subst, $index_subst);

          if (in_array('length', $detect_mode)) # add information needed for length detection
            aggregate_length($dataset, $data, $path, $vector, $client, $request, $avg_request, $var_request, $index_request);

          if (in_array('mcshmm', $detect_mode)) # add information needed for mcshmm detection
            aggregate_mcshmm($dataset, $path, $request, $vector_count, $add_vector, $client, $hmm_max_learn);

          if (in_array('geoip', $detect_mode)) # add information needed for geoip detection
            aggregate_geoip($dataset, $geoip_cache, $geoip_stream, $ipaddr, $lof_max_learn);

          if (in_array('bytes', $quantify_type)) # add information needed for attack quantification;
            aggregate_bytes($dataset, $data, $path, $vector, $client, $lof_max_learn);
        }
      }
    }
  }

  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

  # chose random samples for geoip detection
  if (in_array('geoip', $detect_mode))
  {
    // calculate number of geoip samples to choose
    $sample_count = isset($dataset['geolocation']) ? count($dataset['geolocation']) : 0;
    $sample_count = ($sample_count >= $lof_max_learn) ? $lof_max_learn : $sample_count;

    // in case we have collected enough samples for geoip detection
    if ($sample_count >= $lof_min_learn)
    {
      // randomize geoip samples
      $dataset['geolocation'] = array_rand_multi($dataset['geolocation'], $sample_count);
      carriage_return(1, "[#] [GeoIP detect] Choosing $sample_count random geolocations\n", $line_count, $input_file_basename);
    }
    else // remove geoip from detection modes, 
    {
      unset($detect_mode[array_search('geoip', $detect_mode)]); $dataset['geolocation'] = null;
      carriage_return(0, "[!] Not enough geolocations available - GeoIP detection disabled", $line_count, $input_file_basename);
      // die, if geoip is only used detection mode
      if (count($detect_mode) == 0)
        die("\n");
    }
  }

  # chose random samples for bytes quantification
  if (in_array('bytes', $quantify_type) and isset($dataset['query']))
  {
    // for all webapps do
    foreach ($dataset['query'] as $key => &$path)
    {
      // calculate number of bytes samples to choose
      $sample_count = isset($path['bytes']) ? count($path['bytes']) : 0;
      $sample_count = ($sample_count >= $lof_max_learn) ? $lof_max_learn : $sample_count;

      // in case we have collected enough samples for bytes quantification
      if ($sample_count >= $lof_min_learn)
      {
        // randomize bytes samples
        $path['bytes'] = array_rand_multi($path['bytes'], $sample_count);
      }
      else // unset bytes-quantification of webapp
      {
        carriage_return(2, "[*] [Quantification] Not enough samples available to do attack quantification for '$key'", $line_count, $input_file_basename);
        // skip attack quantification if 'bytes-sent' dataset ist not large enough
        $path['bytes'] = null;
      }
    }
  }

  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

  # final calculation for variance/mean of delay
  if ($tamper_test)
  {
    $avg_delay = round($avg_delay, 2);
    $var_delay = round(($index_delay > 1) ? $var_delay / ($index_delay-1) : $var_delay, 2); // avoid division by zero if logfile contains a single line
    carriage_return(2, "[*] [Tamper test] Mean of inter-request time delay is $avg_delay\n"
                     . "[*] [Tamper test] Variance of inter-request time delay is $var_delay\n", $line_count, $input_file_basename);
    // tmp variable needs to be non-set later
    unset($last_date);
  }

  # final calculation for variance/mean of request
  if (in_array('chars', $detect_mode))
  {
    $avg_subst = round($avg_subst, 2);
    $var_subst = round(($index_subst > 1) ? $var_subst / ($index_subst-1) : $var_subst, 2); // avoid division by zero if logfile contains a single line
    carriage_return(2, "[*] [Chars detect] Mean of requests is $avg_subst\n"
                     . "[*] [Chars detect] Variance of requests is $var_subst\n", $line_count, $input_file_basename);
    if ($var_subst > $var_citical_val)
      carriage_return(1, "[#] High variance - You should use to another detect mode than 'chars'!\n", $line_count, $input_file_basename);
  }
  # final calculation for variance/mean of request
  if (in_array('length', $detect_mode))
  {
    $avg_request = round($avg_request, 2);
    $var_request = round(($index_request > 1) ? $var_request / ($index_request-1) : $var_request, 2); // avoid division by zero if logfile contains a single line
    carriage_return(2, "[*] [Length detect] Mean of requests is $avg_request\n"
                     . "[*] [Length detect] Variance of requests is $var_request\n", $line_count, $input_file_basename);
  }
}
else
  # loop fast over input file and count lines
  while (fgets($input_stream))
   $line_count++;

### echo "\n======================= <DEBUG: \$dataset> =======================\n";
### print_r($dataset);
### echo "======================= </DEBUG: \$dataset> ======================\n";

// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

// reset position indicator to zero
fseek($input_stream, 0);

// emtpy file status cache
clearstatcache();
// check if logfile has been modified
if (filemtime($input_file) != $last_modified)
  carriage_return(1, "[#] Logfile '$input_file_basename' changed while reading\n", $line_count, $input_file_basename);

// ------------------------------------------------------------------ //

# train mcshmm data for anomaly detection with hidden markov models
if (in_array('mcshmm', $detect_mode))
  $list_of_ensembles = training_mcshmm($dataset, $add_vector, $hmm_min_learn, $hmm_max_iter, $hmm_tolerance, $vector_count, $detect_mode, $hmm_num_models);

// ------------------------------------------------------------------ //

# try to open output file for writing
if (($output_stream = fopen($output_file, 'w')) == false)
{
  print_message(0, "[!] Cannot write to output file '$output_file'\n");
  usage_die();
}

// -------------------- MAIN PROGRAM STARTS HERE -------------------- //

# insert header data into output file
if ($do_summarize == false)
{
  log_header($output_type, $output_stream, $regex_fields, $client_ident, $dns_lookup,
             $geoip_lookup, $dnsbl_lookup, $quantify_type, $add_tags, $do_summarize, $input_file);
  log_sub_header($output_type, $output_stream, $regex_fields, null, $client_ident, $dns_lookup, $geoip_lookup,
                 $dnsbl_lookup, $detect_mode, null, null, $quantify_type, $add_tags, $do_summarize, '0');
}

# catch SIGINT (CTRL+C) to do a clean exit (e.g. log footer)
declare(ticks = 1); pcntl_signal(SIGINT, "clean_exit");

# in non-summarization mode, results are directly written to disk
if ($do_summarize == false)
  carriage_return(1, "[#] Press Ctrl-C to view partial results\n", $line_count, $input_file_basename);

// ------------------------------------------------------------------ //

# breakpoint: main programm starts here
function main_processing() {}

# main loop over input logfile
while ($line = fgets($input_stream))
{
  // increment line index
  $line_index++;

  // remove junk like MS-wordwraps
  $line = trim($line);

  // ignore empty lines
  if (empty($line)) continue;

  // convert logline to HTTP object
  $data = logline_to_httpdata($line, $regex_string, $regex_fields, $num_fields);

  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

  # check if line is not crippled, else print error message
  if ($data == null)
    print_badline($line, $line_index, $line_count, $input_type, $input_file_basename);
  else
  {
    # extract request (+additional attack vectors) and path from HTTP-data
    $vector = httpdata_to_vector($data, $add_vector, $use_phpids_converter, $phpids_path, $phpids_init,
                                 $detect_mode, $only_check_webapps, $web_app_extensions, $input_file, $line_index);

    // get request used for detection
    $request = $vector[0];

    // get path of the web application
    $path = $vector[1];

    # convert date to unix timestamp format
    if (array_key_exists('Date', $data))
      $date = $data['Date'] = date("r", apachedate_to_timestamp($data['Date']));

    // aggregate dates for 'traffic over time' statistics
    # if ($do_summarize)
    # {
    #   $day = date("Y/m/d", strtotime($date));
    #   $dates['traffic'][$day] ? $dates['traffic'][$day]++ : $dates['traffic'][$day] = 1;
    # }

    # do a naive tamper test based on time delays
    if ($tamper_test)
    {
      if (isset($last_date))
        tampter_test_grubbs($date, $last_date, $avg_delay, $var_delay, $index_delay, $max_delay, $line_count, $input_file, $input_file_basename);
      // set last date to current date
      $last_date = $date;
    }

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    # check if request contains non-null attack vectors
    if (array_filter($request))
    {
      // increment overall number of requests by one
      $request_count++;

      // reset results and tags
      $result = array(); $tags = null;

      # detect attacks for detect modes based on request inspection:
      # a) CHARS detect mode
      if (in_array('chars', $detect_mode))
        $result['Chars'] = detection_chars($threshold, $dataset, $path, $request, $avg_subst, $var_subst, $var_min_learn);
      # b) LENTH detect mode
      if (in_array('length', $detect_mode))
        $result['Length'] = detection_length($threshold, $dataset, $path, $request, $avg_request, $var_request, $var_min_learn);
      # c) PHPIDS detect mode
      if (in_array('phpids', $detect_mode))
        $result['PHPIDS'] = detection_phpids($threshold, $request, $phpids_path, $phpids_init, $tag_stats, $tags, $tags_count);
      # d) MCSHMM detect mode
      if (in_array('mcshmm', $detect_mode))
        $result['MCSHMM'] = detection_mcshmm($threshold, $dataset, $path, $request, $list_of_ensembles, $hmm_min_learn, $hmm_max_iter, $hmm_decrease);

      # calculate overall result by adding up results of different detect modes
      $result_sum = array_sum($result);

      // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

      # do only in case of an attack or if GeoIP/DNSBL detect modes enabled
      if (($result_sum >= $threshold) or in_array('geoip', $detect_mode) or in_array('dnsbl', $detect_mode))
      {
        // reset geoip and dnsbl data
        $geoip_data = null; $dnsbl_data = null;

        # receive remote host and do dns/dnsbl/geoip lookups
        if (array_key_exists('Remote-Host', $data))
        {
          // this might be an ip address _or_ a hostname
          $remote_host = $data['Remote-Host'];

          # convert hostnames to ip addresses (needed for geoip/dnsbl)
          if ($geoip_lookup or $dnsbl_lookup)
          {
            $ipaddr = hostname_to_ipaddr($remote_host, $dns_cache);
            // show speed warning once, if logfile contains hostname
            if (!isset($dns_warning_once) and ($ipaddr != $remote_host))
            {
              carriage_return(1, "[#] Logfile contains hostnames - this might be a significant slowdown\n", $line_count, $input_file_basename);
              $dns_warning_once = true;
            }
          }

          # retrieve geoip-information
          if ($geoip_lookup)
            $geoip_data = geo_targeting($ipaddr, $geoip_stream, $geoip_cache);

          # lookup ip address in dns blacklist
          if ($dnsbl_lookup)
            $dnsbl_data = ipaddr_to_dnsbl($ipaddr, $dnsbl_type, $allowed_dnsbl_types, $dnsbl_cache);

          # convert ip address to hostname right now, if summarization disabled
          if ($dns_lookup and ($do_summarize == false))
            $remote_host = ipaddr_to_hostname($remote_host, $dns_cache);

          // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

          # detect attacks for detect modes based on geolocation/ip address inspection:
          # e) GeoIP detect mode
          if (in_array('geoip', $detect_mode))
            $result['GEOIP'] = detection_geoip($threshold, $dataset, $geoip_data, $ipaddr, $lof_minpts_lb, $lof_minpts_ub, $lof_cache);
          # f) DNSBL detect mode
          if (in_array('dnsbl', $detect_mode))
            $result['DNSBL'] = detection_dnsbl($threshold, $default_threshold, $ipaddr, $dnsbl_cache);
        }
        else
        {
          // show speed warning once, if logfile does not contain remote-hosts
          if (!isset($remote_host))
            carriage_return(1, "[#] Logfile does not contain 'Remote-Host' field - client identification hindered\n", $line_count, $input_file_basename);

          # set remote host to unknown, if not in logfile
          $remote_host = 'unknown_host';
    	  }
      }

      // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

      # calculate overall result by adding up result of different detect modes
      $result_sum = array_sum($result);

      # in case the overall result is above threshold level
      if ($result_sum >= $threshold)
      {
        // increment attack index
        $attack_count++;

        # if logline entry contains zero-information, mark aus useless
        $regex_fields = mark_regex_fields_useful($regex_fields, $data);

        # try to retrieve client's identity
        $client = client_identification($data, $client_ident, $remote_host, $session_identifiers);

        # try to evaluate the success of attacks
        if (!empty($quantify_type))
          $success = attack_quantification($dataset, $request, $quantify_type, $data, $path, $client, $lof_minpts_lb, $lof_minpts_ub,
                                           $lof_cache, $lof_min_learn, $max_replay_per_client, $replay_count, $target, $web_app_extensions);

        // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

        # summarization is enabled (default)
        if ($do_summarize)
        {
          # create new client object, if not yet exists
          if (!isset($clients[$client]))
          {
            $clients[$client] = new Client($client);
            print_message(2, "[*] Added client to list of troublemakers: $client\n");
          }

          # create action, containing data + result/tags
          $actions[$client][md5(serialize($data))] = new Action($date, $data, $path, $result_sum, $tags, $success, $remote_host, $geoip_data, $dnsbl_data);

          # add current result sum to overall results
          $clients[$client]->result += $result_sum;

          # add current success to overall quantification
          if (isset($success) and ($success != '-'))
          {
            $clients[$client]->severity++;
            if (!in_array($success, $clients[$client]->quantification))
              $clients[$client]->quantification[] = $success;
          }
        }
        # summarization is disabled
        else
        {
          // increment client index or first-time add client
          // (note: $client is now an array, not an object)
          $clients[$client] = isset($clients[$client]) ? $clients[$client]+1 : 1;

          # add current incident to output file
          log_incident($output_type, $output_stream, $client, $client_ident, $dns_lookup, $geoip_data, $dnsbl_lookup, $dnsbl_data,
                       $data, $url_decode, $attack_count, $allowed_http_methods, false, $result_sum, $threshold, $tags, $add_tags, $success, $quantify_type, 0);
        }
      }
    }
  }

  // show progress bar, while waiting :)
  $progress = progress_bar($line_index, $line_count, $progress, $input_file_basename);
}

### echo "\n======================= <DEBUG: \$clients> =======================\n";
### print_r($clients);
### echo "======================= </DEBUG: \$clients> ======================\n";

### echo "\n======================= <DEBUG: dns/dnsbl/geoip cache> =======================\n";
### print_r($dns_cache);
### print_r($dnsbl_cache);
### print_r($geoip_cache);
### echo "======================= </DEBUG: dns/dnsbl/geoip cache> ======================\n";

// - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

# breakpoint: read logfile, collect all requests of a client and reconstruct sessions
function post_processing() {}

# summarization is enabled (default)
if ($do_summarize)
{
  // number of lines
  $line_count = 0;

  // print feedback (important, when dealing with huge files)
  print_message(1, "[>] Creating summary for '$input_file_basename'\n");

  // reset position indicator to zero
  fseek($input_stream, 0);

  // loop over input file
  while ($line = fgets($input_stream))
  {
    // increment number of lines
    $line_count++;

    // remove junk like MS-wordwraps
    $line = trim($line);

    // ignore empty lines
    if (empty($line))
      continue;

    // convert logline to HTTP object
    $data = logline_to_httpdata($line, $regex_string, $regex_fields, $num_fields);

    // create dataset
    if (isset($data))
    {
      // extract request (+additional attack vectors) and path from HTTP-data
      $vector = httpdata_to_vector($data, $add_vector, $use_phpids_converter, $phpids_path, $phpids_init,
                                   $detect_mode, $only_check_webapps, $web_app_extensions, $input_file, null);

      // get request used for detection
      $request = $vector[0];

      // get path of the web application
      $path = $vector[1];

      // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

      # check if request exists
      if (isset($request))
      {
        // reset geoip and dnsbl data
        $geoip_data = null; $dnsbl_data = null;

        # receive remote host and do dns/dnsbl/geoip lookups
        if (array_key_exists('Remote-Host', $data))
        {
          // this might be an ip address _or_ a hostname
          $remote_host = $data['Remote-Host'];

          // try to retrieve client's identity
          $client = client_identification($data, $client_ident, $remote_host, $session_identifiers);

          // if client is in the list of troublemakers
          if (isset($clients[$client]))
          {

            # convert hostnames to ip addresses (needed for geoip/dnsbl)
            if ($geoip_lookup or $dnsbl_lookup)
              $ipaddr = hostname_to_ipaddr($remote_host, $dns_cache);

            # retrieve geoip-information
            if ($geoip_lookup)
              $geoip_data = geo_targeting($ipaddr, $geoip_stream, $geoip_cache);

            # lookup ip address in dns blacklist
            if ($dnsbl_lookup)
              $dnsbl_data = ipaddr_to_dnsbl($ipaddr, $dnsbl_type, $allowed_dnsbl_types, $dnsbl_cache);

            # convert ip address to hostname
            if ($dns_lookup)
            {
              $remote_host = ipaddr_to_hostname($remote_host, $dns_cache);
              // don't forget to reset client's name
              $clients[$client]->name = client_identification($data, $client_ident, $remote_host, $session_identifiers);
            }

            // - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
 
            # convert date to unix timestamp format
            if (array_key_exists('Date', $data))
              $date = $data['Date'] = date("r", apachedate_to_timestamp($data['Date']));

            # create md5sum of current data
            $md5_key = md5(serialize($data));

            # collect all actions (even non-suspicious) of client
            if (array_key_exists($md5_key, $actions[$client]))
            {
              # get supsicous action from cache, containing data + result/tags
              $action = $actions[$client][$md5_key];
              // increment counter for most-attacked web applications
              isset($pathes[$path]) ? $pathes[$path]++ : $pathes[$path] = 1;
              // aggregate dates for 'attacks over time' statistics
              # $day = date("Y/m/d", strtotime($date));
              # $dates['attacks'][$day] ? $dates['attacks'][$day]++ : $dates['attacks'][$day] = 1;
            }       
            else
            {
              # create harmless action, needed for statistics (troublemaking clients only)
              $action = new Action($date, $data, $path, 0, array('none'), '-', $remote_host, $geoip_data, $dnsbl_data);
              // increment counter of harmless requests
              $clients[$client]->harmless_requests++;
            }

            # add current action, to the client's actions
            $clients[$client]->add_action($action);
          }
        }
      }
    }
  }
  // free some cache we don't need anymore
  unset($actions);
}

// ------------------------------------------------------------------ //

if ($do_summarize)
{
  session_classification($clients, $request_count, $allowed_http_methods, $web_app_extensions, $max_session_duration);

  # insert summarized data into output file
  log_summarized($output_type, $output_stream, $regex_fields, $clients, $client_ident, $dns_lookup, $geoip_lookup,
                 $dnsbl_lookup, $detect_mode, $quantify_type, $url_decode, $add_tags, $allowed_http_methods, $attack_count,
                 $threshold, $web_app_extensions, $max_session_duration, $pathes, $dates, $max_harmless_summarize, $pchart_path, $use_pchart_library, $input_file);
}

// ------------------------------------------------------------------ //

# insert footer data into output file
if ($do_summarize == false)
{
  log_sub_footer($output_type, $output_stream, $regex_fields, $client_ident, $dns_lookup,
                 $geoip_lookup, $dnsbl_lookup, $quantify_type, $add_tags, false, false, '0');
  log_footer($output_type, $output_stream);
}

# print some statistics
print_statistics($attack_count, $tags_count, count($clients), $tag_stats, $output_file, $do_summarize, 'complete');

// ------------------------------------------------------------------ //

# close i/o streams
fclose($input_stream);
fclose($output_stream);
if (isset($geoip_stream))
  geoip_close($geoip_stream);

// ------------------------ END MAIN PROGRAM ------------------------ //

# function: print usage and exit
function usage_die()
{
  // define those variables global, we have to deal with
  global $allowed_input_types, $allowed_output_types, $allowed_detect_modes, $allowed_quantify_types, $allowed_client_ident, $additional_attack_vectors, $allowed_dnsbl_types, $default_threshold, $default_verbosity;

  echo("\nUsage: lorg [-i input_type] [-o output_type] [-d detect_mode]");
  echo("\n            [-a add_vector] [-c client_ident] [-b dnsbl_type]");
  echo("\n            [-q quantification] [-t threshold] [-v verbosity]");
  echo("\n            [-n] [-u] [-h] [-g] [-p] input_file [output_file]");
  echo("\n\n -i allowed input formats:");
  foreach($allowed_input_types as $key => $val)
    echo " $key";
  echo("\n -o allowed output formats:");
  foreach($allowed_output_types as $val)
    echo " $val";
  echo("\n -d allowed detect modes:");
  foreach($allowed_detect_modes as $val)
    echo " $val";
  echo("\n -a additional attack vectors:");
  foreach($additional_attack_vectors as $val)
    echo " $val";
  echo("\n -c allowed client identfiers:");
  foreach($allowed_client_ident as $val)
    echo " $val";
  echo("\n -b allowed dnsbl types:");
  foreach($allowed_dnsbl_types as $key => $val)
    echo " $key";
  echo("\n -q allowed quantification types:");
  foreach($allowed_quantify_types as $val)
    echo " $val";
  echo("\n -t threshold level as value from 0 to n (default: $default_threshold)");
  echo("\n -v verbosity level as value from 0 to 3 (default: $default_verbosity)");
  echo("\n -n do not summarize results, output single incidents");
  echo("\n -u urldecode encoded requests (affects reports only)");
  echo("\n -h try to convert numerical addresses into hostnames");
  echo("\n -g enable geotargeting (separate files are needed!)");
  echo("\n -p perform a naive tamper detection test on logfile");
  die("\n\n");
}

// ------------------------------------------------------------------ //

# function: auto-detect logfile format
function detect_logformat($stream, $allowed_input_types)
{
  // try detection for the first couple of lines
  for ($line_index = 0; $line_index < 10; $line_index++)
  {
    // get next line of logfile and remove junk
    $line = fgets($stream);
    $line = trim($line);

    // try to auto-detect format
    foreach($allowed_input_types as $key => $val)
    {
      $regex_fields = null;
      format_to_regex($val, $regex_fields, $regex_string, $num_fields);
      $data = logline_to_httpdata($line, $regex_string, $regex_fields, $num_fields);
      if (isset($data))
        return $key;
    }
  }
  // in case auto-detection failed
  return null;
}

// ------------------------------------------------------------------ //

# function: parse logline into HTTP-data array
function logline_to_httpdata($line, $regex_string, $regex_fields, $num_fields)
{
  // line cannot be parsed for some reason
  if (preg_match($regex_string, $line, $matches) !== 1)
    return null;

  // create HTTP-data array
  reset($regex_fields);
  for ($n = 0; $n < $num_fields; ++$n)
  {
    $field = each($regex_fields);
    $data[$field['key']] = $matches[$n + 1];
  }

  ### echo "\n======================= <DEBUG: \$data> =======================\n";
  ### print_r($data);
  ### echo "======================= </DEBUG: \$data> ======================\n";

  return $data;
}

// ------------------------------------------------------------------ //

# function: extract request (+additional attack vectors) and path from HTTP-data
function httpdata_to_vector($data, $add_vector, $use_phpids_converter, $phpids_path, $phpids_init, $detect_mode, $only_check_webapps, $web_app_extensions, $input_file, $line_index)
{
  if (array_key_exists('Request', $data))
  {
    if (preg_match("/^(\S+) (.*?) HTTP\/[0-9]\.[0-9]\z/", $data['Request'], $match))
    {
      // parse query part of given url
      $url_query = parse_url($match[2], PHP_URL_QUERY);

      // use whole url, if query is crippled and non-harmless
      if ((!$url_query) and (preg_match('/[^\w!\/~#+-.]/', $match[2])))
        $url_query = $match[2];

      // a request to inspect might consists of
      //
      // [0] url query (e.g. foo=bar) as array
      //     (note: parse_str also urldecodes)
      parse_str($url_query, $parameters);
      // [1] url path (e.g. /index.php) as string
      $path = parse_url($match[2], PHP_URL_PATH);
      // [2] argument names (e.g. foo) as array
      $argnames = array_keys($parameters);
    }
    else
    {
      // inspect the whole request, if malformed / non-rfc2616
      parse_str($data['Request'], $parameters);
      // in this case, we have no path/argnames
      $path = null; $argnames = null;
      // be chatty, when running inside main loop in verbose mode
      if (isset($line_index))
        print_message(2, "[*] Line '$line_index' contains malformed, rfc2616-incompatible request:\n    '" . $data['Request'] . "'\n");
    }

    # workaround for a bug in PHP's urldecode() which threads url query values as arrays if they contain '[...]'
    foreach ($parameters as $key => &$val)
      if (is_array($val))
        $val = implode_recursive('', $val);

    // a request to inspect might consists of
    //
    // [3] cookie(s) (e.g. PHPSESSID) as string
    $cookie = (array_key_exists('Cookie', $data) and ($data['Cookie'] != '-')) ? $data['Cookie'] : '';
    // [4] user agent (e.g. Mozilla/5.0) as string
    $agent = (array_key_exists('User-Agent', $data) and ($data['User-Agent'] != '-')) ? $data['User-Agent'] : '';

    // if nonexistent, don't even pass request to detect modes (speed boost!)
    $request = null;

    // by default, only inspect url query (+ do optional web app check)
    if (!$only_check_webapps or (preg_match("/.*(" . implode('|', $web_app_extensions) . ")$/", $path)))
      $request['query'] = !empty($parameters) ? $parameters : null;

    // additionaly, inspect url path, argument names, cookie(s), user agent
    if ($add_vector)
      foreach ($add_vector as $vector)
        $request[$vector] = (!empty($$vector)) ? $$vector : null;

    // normalize requests, using the PHPIDS converter
    if ($use_phpids_converter and isset($request)) // for PHPIDS detect mode, conversion is done automatically
      if (in_array('chars', $detect_mode) or in_array('mcshmm', $detect_mode))
        array_walk_recursive($request, 'convert_using_phpids', array($phpids_path, $phpids_init));

    ### echo "\n======================= <DEBUG: \$request> =======================\n";
    ### print_r($request);
    ### echo "======================= </DEBUG: \$request> ======================\n";

    // return array to inspect
    return array($request, $path);
  }
  else
    // if request nonexitent, we've nothing to inspect
    return null;
}

// ------------------------------------------------------------------ //

# function: do a naive tamper test based on inter-request time delays
function tampter_test_grubbs($date, $last_date, $mean, $variance, $index_delay, $max_delay, $line_count, $input_file, $input_file_basename)
{
  // calc inter-request time delay
  $delay = strtotime($date) - strtotime($last_date);

  // if we've gotten the maximum delay
  if (($delay == $max_delay) and ($index_delay > 2) and ($variance != 0))
  {
    // do Grubbs' test for maximum value outliers
    $grubbs = ($delay - $mean) / sqrt($variance);

    // G distribution of critical values for alpha = 0.05
    $g_dist = array(3 =>   1.1531, 4 =>   1.4625, 5 =>   1.6714, 6 =>   1.8221, 7 =>   1.9381,
                    8 =>   2.0317, 9 =>   2.1096, 10 =>  2.1761, 11 =>  2.2339, 12 =>  2.2850,
                    13 =>  2.3305, 14 =>  2.3717, 15 =>  2.4090, 16 =>  2.4433, 17 =>  2.4748,
                    18 =>  2.5040, 19 =>  2.5312, 20 =>  2.5566, 25 =>  2.6629, 30 =>  2.7451,
                    40 =>  2.8675, 50 =>  2.9570, 60 =>  3.0269, 70 =>  3.0839, 80 =>  3.1319,
                    90 =>  3.1733, 100 => 3.2095, 120 => 3.2706, 140 => 3.3208, 160 => 3.3633,
                    180 => 3.4001, 200 => 3.4324, 300 => 3.5525, 400 => 3.6339, 500 => 3.6952);

    // match population of delays to critical value in G distribution
    foreach ($g_dist as $key => $val)
    {
      // set critical value to current g-value
      $critical_value = $val;
      // break, if we've reached a population key that matches
      if ($key >= $index_delay)
        break;
    }

    // format gmdate string, if delay has to be measured in min, hours, days
    $gmdate_string = 's \s\e\c'; // default: only seconds
    $gmdate_string = ((($delay / 60) >= 1) ? 'i \m\i\\n, ' : '') . $gmdate_string;
    $gmdate_string = ((($delay / 3600) >= 1) ? 'H \h\o\u\\r\s, ' : '') . $gmdate_string;
    $gmdate_string = ((($delay / 86400) >= 1) ? 'd \d\a\y\s, ' : '') . $gmdate_string;

    if ($grubbs > $critical_value) // print message, if we have an 'outlier' to the max
      carriage_return(1, "[!] Possible logfile tamper at $last_date:\n    Input file contains no activity for " . gmdate($gmdate_string, $delay) . "\n", $line_count, $input_file_basename);
    else
      carriage_return(1, "[#] Could not find any evidence for logfile tampering\n", $line_count, $input_file_basename);
  }
}

// ------------------------------------------------------------------ //

# function: add information needed for geoip detection to dataset
function aggregate_geoip(&$dataset, &$geoip_cache, $geoip_stream, $ipaddr, $lof_max_learn)
{
  // if no geoip-data retreived yet or number of samples below maximum times ten
  if (!isset($dataset['geolocation']) or (count($dataset['geolocation']) < $lof_max_learn * 10))
  {
    // retrieve geoip-information
    $geoip_data = geo_targeting($ipaddr, $geoip_stream, $geoip_cache);
    // extract client's geolocation
    $geolocation = explode(',', $geoip_data[1]);
    // if we've got a valid geolocation
    if ($geolocation[0] != '-')
      // if client does not yet exist in dataset
      if (!isset($dataset['geolocation'][$ipaddr]))
        // add it as client => latitude/longitude array to dataset
        $dataset['geolocation'][$ipaddr] = array($geolocation[0] + 0.01 * rand(0, 99), $geolocation[1] + 0.01 * rand(0, 99));
  }
}

// ------------------------------------------------------------------ //

function aggregate_bytes(&$dataset, $data, $path, $vector, $client, $lof_max_learn)
{
  // break, in case we are dealing static content (= no url query parameters)
  if (!isset($vector[0]['query']))
    return null;

  // if request contains a path and a bytes-sent entry
  if (isset($path) and isset($data['Bytes-Sent']))
  {
    // create reference to the bytes-sent list for the current path
    $bytes = &$dataset['query'][$path]['bytes'];

    // every client is only allowed to contribute once, to avoid test data poisioning
    if (isset($bytes[$client]))
      return null;

    // if no samples retreived yet or number of samples below maximum times ten
    if (!isset($bytes) or (count($bytes) < $lof_max_learn * 10))
    {
      // add value for current bytes-sent to dataset
      $bytes[$client] = ($data['Bytes-Sent'] != '-') ? $data['Bytes-Sent'] + 0.01 * rand(0,99) : 0;
    }
  }
}

// ------------------------------------------------------------------ //

# function: add information needed for chars detection to dataset
function aggregate_chars(&$dataset, $data, $path, $vector, $client, $request, &$avg, &$var, &$idx)
{
  // break, in case we are dealing static content (= no url query parameters)
  if (!isset($vector[0]['query']))
    return null;

  // if request contains a valid path
  if (isset($path))
  {
    // create reference to the chars detection values for the current path
    $chars = &$dataset['query'][$path]['chars'];

    // every client is only allowed to contribute once, to avoid test data poisioning
    if (isset($chars['clients'][$client]))
      return null;

    // get length of concatenated vectors of request (without alphanumeric)
    $value = strlen(remove_alphanumeric(implode_recursive('', $request)));

    // global online variance and mean calculation of substituted request's length
    online_variance($avg, $var, $idx, $value);

    // local online variance and mean calculation of substituted request's length
    online_variance($chars['avg'], $chars['var'], $chars['idx'], $value);

    // mark current client as 'has contributed'
    $chars['clients'][$client] = true;
  }
}

// ------------------------------------------------------------------ //

# function: add information needed for length detection to dataset
function aggregate_length(&$dataset, $data, $path, $vector, $client, $request, &$avg, &$var, &$idx)
{
  // break, in case we are dealing static content (= no url query parameters)
  if (!isset($vector[0]['query']))
    return null;

  // if request contains a valid path
  if (isset($path))
  {
    // create reference to the length detection values for the current path
    $length = &$dataset['query'][$path]['length'];

    // every client is only allowed to contribute once, to avoid test data poisioning
    if (isset($length['clients'][$client]))
      return null;

    // get length of concatenated vectors of request
    $value = strlen(implode_recursive('', $request));

    // global online variance and mean calculation of request's length
    online_variance($avg, $var, $idx, $value);

    // local online variance and mean calculation of request's length
    online_variance($length['avg'], $length['var'], $length['idx'], $value);

    // mark current client as 'has contributed'
    $length['clients'][$client] = true;
  }
}

// ------------------------------------------------------------------ //

# function: add information needed for mcshmm detection to dataset
function aggregate_mcshmm(&$dataset, $path, $request, &$vector_count, $add_vector, $client, $hmm_max_learn)
{
  // add query parameters to attack vectors
  $add_vector[] = 'query';

  // aggregate training data for query parameters, argument names, url path, cookie(s), user agent
  foreach ($add_vector as $vector)
  {
    // create reference to the value list for the current vector
    switch ($vector)
    {
      // reference to parameters of query
      case 'query':
        $vectors = &$dataset['query'][$path]['parameters'];
        break;
      // reference to argnames of query
      case 'argnames':
        $vectors = &$dataset['query'][$path]['argnames'];
        break;
      // reference to path, cookie or agent
      default:
        $vectors = &$dataset[$vector];
    }

    // some vectors (e.g. query parameters) cannot be found in all requests
    if (!isset($request[$vector]))
      continue;

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    // if request contains a query, aggregate it's parameters' values
    if (($vector == 'query') and is_array($request['query']))
    {
      foreach($request['query'] as $parameter => $value)
      {
        // create reference to the value list for the current parameter
        $parameters = &$vectors[$parameter];

        // every client is only allowed to contribute once, to avoid test data poisioning
        if (isset($parameters[$client]))
          continue;

        // convert and add values, if not enough samples retreived yet
        if (!isset($parameters) or (count($parameters) < $hmm_max_learn))
        {
          // substitute alphanumeric elements of value
          $value_subst = convert_alphanumeric($value);
          // add value for current parameter to dataset
          $parameters[$client] = str_split($value_subst);
          // increase number of parameters/vector samples
          $vector_count++;
        }
      }
    }

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    // every client is only allowed to contribute once, to avoid test data poisioning
    if (isset($vectors[$client]))
      continue;

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    // if request contains argnames, aggregate it's values
    if ($vector == 'argnames' and is_array($request['argnames']))
    {
      foreach ($request[$vector] as $argname)
      {
        // if no samples retreived yet or number of samples below maximum
        if (!isset($vectors) or (count($vectors) < $hmm_max_learn))
        {
          $vectors[$client] = $argname;
          // increase number of parameters/vector samples
          $vector_count++;
        }
      }
    }

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    if (($vector == 'cookie') or ($vector == 'agent') or ($vector == 'path'))
    {
      // if no samples retreived yet or number of samples below maximum
      if (!isset($vectors) or (count($vectors) < $hmm_max_learn))
      {
        // substitute alphanumeric elements of path, cookie or agent's values
        $vectors[$client] = str_split(convert_alphanumeric($request[$vector]));
        // increase number of parameters/vector samples
        $vector_count++;
      }
    }
  }
}

// ------------------------------------------------------------------ //

# function: train dataset for anomaly detection with hidden markov models
function training_mcshmm(&$dataset, $add_vector, $hmm_min_learn, $hmm_max_iter, $hmm_tolerance, $vector_count, $detect_mode, $hmm_num_models)
{
  # set counter for process bar
  $vector_index = 0; $progress = -1; $observations = 0;

  // insert newline
  print_message(1, "\n");

  // we need at least a set of parameters
  if (!isset($dataset['query']))
  {
    unset($detect_mode[array_search('mcshmm', $detect_mode)]);
    print_message(0, "[!] Training dataset empty - MCSHMM detection disabled\n");
    // quit, if no other dectection modes are enabled
    if (count($detect_mode) == 0)
      die("\n");
    return null;
  }
  // add query parameters to attack vectors
  $add_vector[] = 'query';

  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

  # training phase: make each HMM ensemble learn query parameters, argument names, url path, cookie(s), user agent
  foreach ($add_vector as $vector)
  {
    // show progress bar, while waiting :)  
    progress_bar($vector_index, $vector_count, $progress, '');

    if ($vector == 'query')
    {
      foreach($dataset['query'] as $path => $query)
      {
        if (!isset($query['parameters']))
          continue;

        foreach($query['parameters'] as $parameter => $training_set)
        {
          // increment parameter index
          $vector_index++;

          $training_set_count = count($training_set);

          // train HMM ensemble if training set is large enough
          if ($training_set_count >= $hmm_min_learn)
          {
            // merge and unique value char arrays
            $universe = array_unique(call_User_Func_Array('array_merge', $training_set));

            print_message(2, "[*] [MCSHMM detect] Training parameter '$parameter' of '$path' with $training_set_count observations\n");
            print_message(3, "[-] [MCSHMM detect] Universe: '" . implode($universe) . "'\n");
            print_message(3, "[-] [MCSHMM detect] Training set:"); foreach($training_set as $value) print_message(3, " '" . implode($value) . "'"); print_message(2, "\n");

            // create and train HMM ensemble
            $hmm_ensemble = new Ensemble($universe, $training_set, $path, $parameter, $hmm_num_models);
            $hmm_ensemble->train($training_set, $hmm_max_iter, $universe, $hmm_tolerance);
            $list_of_ensembles['query'][$path][$parameter] = $hmm_ensemble;

            ### echo "\n======================= <DEBUG: hmm_ensemble for parameter '$parameter'> =======================\n";
            ### print_r($hmm_ensemble);
            ### echo "======================= </DEBUG: hmm_ensemble for parameter '$parameter'> ======================\n";
          }
          else
          {
            $observations = ($training_set_count > $observations) ? $training_set_count : $observations;
            print_message(2, "[*] [MCSHMM detect] Not enough ($training_set_count) observations available to train parameter '$parameter' of '$path'\n");
          }
        }
      }
      if ($observations < $hmm_min_learn)
      {
        carriage_return(1, "[#] Logfile does not contain enough training data for 'mcshmm' mode!\n", 40, '');
        carriage_return(1, "[#] You should consider using another detection mode (e.g. phpids)!\n", 40, '');
        carriage_return(1, "[#] To use 'mcshmm' anyway, set \$hmm_min_learn to less than " . ($observations+1) . "\n", 40, '');
      }
    }
  }

  // if we have at least on trained HMM
  if (isset($list_of_ensembles))
    return $list_of_ensembles;
}

// ------------------------------------------------------------------ //

# function: try to retrieve client's identity
function client_identification($data, $client_ident, $remote_host, $session_identifiers)
{
  // set defaults
  $session = ''; $user = ''; $logname = '';

  # try to retrieve session id from cookie
  if (array_key_exists('Cookie', $data))
  {
    $entries = preg_split("/;(\ )*/", $data['Cookie']);
    foreach($entries as $entry)
    {
      $cookie = explode('=', $entry);
      if (isset($cookie[1]) and (preg_match("/^" . implode('|', $session_identifiers) . "$/i", $cookie[0])))
        $session = "'" . $cookie[1] . "'";
    }
  }

  # try to retrieve session id from url query
  if (empty($session))
  {
    $query = explode(" ", parse_url($data['Request'], PHP_URL_QUERY));
    parse_str($query[0], $query_parsed);
    foreach ($query_parsed as $parameter => $value)
      if (preg_match("/^" . implode('|', $session_identifiers) . "$/i", $parameter))
        $session = $value;
  }

  # try to retrieve username (%u taken from auth)
  if (array_key_exists('Remote-User', $data) and ($data['Remote-User'] != '-'))
    $user = $data['Remote-User'];

  # try to retrieve logname (%l taken from identd)
  if (array_key_exists('Remote-Logname', $data) and ($data['Remote-Logname'] != '-'))
    $logname = $data['Remote-Logname'];

  # set ident to address, session, user, logname or all
  switch ($client_ident)
  {
    case 'host':
      $ident = $remote_host;
      break;
    case 'session':
      $ident = empty($session) ? $remote_host : $session;
      break;
    case 'user':
      $ident = empty($user) ? $remote_host : $user;
      break;
    case 'logname':
      $ident = empty($logname) ? $remote_host : $logname;
      break;
    case 'all':
      $ident = $remote_host;
      $ident .= !empty($session) ? " {" . $session . "}" : '';
      $ident .= !empty($user) ? " (" . $user . ")" : '';
      $ident .= !empty($logname) ? " [" . $logname . "]" : '';
      break;
  }

  ### echo "\n======================= <DEBUG: \$ident> =======================\n";
  ### echo "$ident\n";
  ### echo "======================= </DEBUG: \$ident> ======================\n";

  return $ident;
}

// ------------------------------------------------------------------ //

# function: try to reconstruct, if new session was spawned by client
function session_identification($client, $action, $last_action, $max_session_duration)
{
  // set default
  $new_session = false;

  if (isset($last_action))
  {
    // calculate client's current inter-request time delay
    $delay = strtotime($action->date) - strtotime($last_action->date);

    // new session, if inter-request time delay exceeds max session duration
    if ($delay > $max_session_duration)
      $new_session = 'max_session_duration_exceeded';

    // extract user agent
    $agent = isset($action->data['User-Agent']) ? $action->data['User-Agent'] : '';
    $last_agent = isset($last_action->data['User-Agent']) ? $last_action->data['User-Agent'] : '';

    // new session, change of user agent and unusual delay or request change
    if ($agent != $last_agent)
    {
      if (($delay > (60 + $client->avg_time_delay + 3 * $client->std_time_delay))
      and ($action->path != $last_action->path))
        $new_session = 'user_agent_change';
    }
  }
  else // new session, if it's the first time, we see this client
    $new_session = 'client_first_seen';

  if ($new_session)
    print_message(2, "[*] [Classification] Spawning new session (reason: $new_session)\n");

  return $new_session;
}

// ------------------------------------------------------------------ //

# function: simple, fast (and inaccurate) statistical anomaly detection
function detection_chars($threshold, &$dataset, $path, $request, $mean, $variance, $var_min_learn)
{
  // set default;
  $result_chars = 0;

  // create reference to the chars detection values for the current path
  if (isset($dataset['query'][$path]['chars']))
    $chars = $dataset['query'][$path]['chars'];

  // get length of concatenated vectors of request (without alphanumeric)
  $value = strlen(remove_alphanumeric(implode_recursive('', $request)));

  if (isset($chars['clients']) and ($chars['clients'] >= $var_min_learn))
  {
    if ($value > $chars['avg'] and ($value > 2))
      $result_chars = round(log($value, 1.25) + 1);
  }
  else
  {
    if ($value > $mean and ($mean > 0) and ($value > 2))
      $result_chars = round(log($value/$mean));
  }

  if ($result_chars >= $threshold)
    print_message(3, "[*] [Chars detect] Result +$result_chars (found request with $value suspicious chars)\n");

  ### echo "\n======================= <DEBUG: \$result_chars> =======================\n";
  ### echo "$result_chars\n";
  ### echo "======================= </DEBUG: \$result_chars> ======================\n";

  return($result_chars);
}

// ------------------------------------------------------------------ //

# function: simple, fast (and inaccurate) statistical anomaly detection
function detection_length($threshold, &$dataset, $path, $request, $mean, $variance, $var_min_learn)
{
  // set default;
  $result_length = 0;

  // create reference to the length detection values for the current path
  // create reference to the chars detection values for the current path
  if (isset($dataset['query'][$path]['length']))
    $length = $dataset['query'][$path]['length'];

  // get length of concatenated vectors of request
  $value = strlen(implode_recursive('', $request));

  if (isset($length['clients']) and ($length['clients'] >= $var_min_learn))
  {
    # use Chebyshev inequality to calculate probability of request of certain length
    if (($value > $length['avg']) and ($length['var'] != 0))
      $result_length = round(log(1 / ($length['var'] / pow($value - $length['avg'], 2)), 1.15) + 1);
  }
  else
  {
    if (($value > $mean)  and ($variance != 0))
        $result_length = round(log(1 / ($variance / pow($value - $mean, 2)), 1.15) + 1);
  }

  if ($result_length >= $threshold)
    print_message(3, "[*] [Length detect] Result +$result_length (found request with $value suspicious chars)\n");

  ### echo "\n======================= <DEBUG: \$result_length> =======================\n";
  ### echo "$result_length\n";
  ### echo "======================= </DEBUG: \$result_length> ======================\n";

  return($result_length);
}

// ------------------------------------------------------------------ //

# function: pipe request through PHPIDS-filter
function detection_phpids($threshold, $request, $phpids_path, $phpids_init, &$tag_stats, &$tags, &$tags_count)
{
  try // pipe request through PHPIDS-filter
  {
    // initiate the PHPIDS and fetch/return the results
    $ids = new IDS_Monitor($request, $phpids_init);
    $result_phpids = $ids->run();
  }
  catch (Exception $e)
  {
    // sth went terribly wrong
    print_message(0, "[!] PHPIDS error occured: " . $e->getMessage() . "\n");
    return null;
  }

  // if non-suspicious, create harmless result
  if ($result_phpids->isEmpty())
    $result_phpids = fake_phpids_result();
  // if suspicious, result is impact plus tags
  elseif ($result_phpids->getImpact() >= $threshold)
  {
    // count tags for statistics
    foreach($result_phpids->getTags() as $tag)
    {
      // first-time add tag
      if (!(isset($tag_stats[$tag])))
        $tag_stats[$tag] = 1;
      // increment tag index
      else
        $tag_stats[$tag]++;

      /*--------------------------------------\
      | possible tags are (PHPIDS 0.7):       |
      | ************************************* |
      | - 'xss' (cross-site scripting)        |
      | - 'sqli' (sql injection)              |
      | - 'csrf' (cross-site request forgery) |
      | - 'dos' (denial of service)           |
      | - 'dt' (directory traversal)          |
      | - 'spam' (mail header injections)     |
      | - 'id' (information disclosure)       |
      | - 'rfe' (remote file execution)       |
      | - 'lfi' (local file inclusion)        |
      | - 'command execution'                 |
      | - 'format string'                     |
      \--------------------------------------*/

      // increment attack index
      $tags_count++;
    }
  }
  ### echo "\n======================= <DEBUG: \$result_phpids> =======================\n";
  ### print_r ($result_phpids);
  ### echo "======================= </DEBUG: \$result_phpids> ======================\n";

  if ($result_phpids->getImpact() >= $threshold)
    print_message(3, "[*] [PHPIDS detect] Result +" . $result_phpids->getImpact() . " (tags: " . implode(', ', $result_phpids->getTags()) . ")\n");

  $tags = $result_phpids->getTags();
  return $result_phpids->getImpact();
}

// ------------------------------------------------------------------ //

# function: anomaly detection using hidden markov models (testing phase)
function detection_mcshmm($threshold, &$dataset, $path, $request, &$list_of_ensembles, $hmm_min_learn, $hmm_max_iter, $hmm_decrease)
{
  // check if request contains a valid query
  if (isset($request['query']) and is_array($request['query']))
  {
    // determine observed values probabilities of a web application
    foreach($request['query'] as $parameter => $value)
    {
      // substitute alphanumeric elements of value
      $value_subst = convert_alphanumeric($value);
      // convert to array
      $test_set = str_split($value_subst);

      if (isset($list_of_ensembles['query'][$path][$parameter]))
      {
        print_message(3, "[-] [MCSHMM detect] Testing observation '$value_subst' for parameter '$parameter'\n");
        // merge and unique value char arrays
        $universe = array_unique(call_User_Func_Array('array_merge', $dataset['query'][$path]['parameters'][$parameter]));
        // sum up probabilies (Corona takes the maximum value instead)
        $result_mcshmm[] = $list_of_ensembles['query'][$path][$parameter]->test($test_set, $universe, $hmm_decrease);
      }
    }

    // set result to the value of the url querythe that is most suspicious (= least likely)
    if (isset($result_mcshmm))
    {
      if (min($result_mcshmm) == 0)
        $result_mcshmm = 100;
      else
      $result_mcshmm = round(log(1 / min($result_mcshmm), 40));

      if ($result_mcshmm < 0)
        $result_mcshmm = 0;

      if ($result_mcshmm >= $threshold) 
        print_message(3, "[*] [MCSHMM detect] Result +$result_mcshmm (found suspicious request)\n");

      return($result_mcshmm);
    }
  }
}

// ------------------------------------------------------------------ //

# function: anomaly detection based on GeoIP information
function detection_geoip($threshold, &$dataset, $geoip_data, $ipaddr, $lof_minpts_lb, $lof_minpts_ub, &$lof_cache)
{
  // set default
  $result_geoip = 0;

  // extract client's geolocation
  $geolocation = explode(',', $geoip_data[1]);

  // try to get lof info from cache
  if (isset($lof_cache['lof'][$geoip_data[1]]))
    $result_geoip = $lof_cache['lof'][$geoip_data[1]];
  else
  {
    // if we've got a valid geolocation
    if ($geolocation[0] != '-')
    {
      // set reference to dataset of geolocations
      $lof_data = &$dataset['geolocation'];

      // for lower to upper bound of k-nearest neighbors do
      for ($lof_neighbors=$lof_minpts_lb; $lof_neighbors <= $lof_minpts_ub; $lof_neighbors++)
      {
        // create and initialize object of class LOF
        $lof = new LOF($lof_data, $lof_neighbors, $lof_cache);

        // get local outlier factor of geolocation
        $lof_values[$lof_neighbors] = $lof->run($ipaddr, $geolocation);
      }
      // get maximum of collected LOFs
      $result_geoip = round(max($lof_values));

      // add result to lof cache
      $lof_cache['lof'][$geoip_data[1]] = $result_geoip;
    }
  }

  if ($result_geoip >= $threshold)
    print_message(3, "[*] [GeoIP detect] Result +$result_geoip (found '$ipaddr' from $geoip_data[0])\n");

  ### echo "\n======================= <DEBUG: \$result_geoip> =======================\n";
  ### echo "$result_geoip\n";
  ### echo "======================= </DEBUG: \$result_geoip> ======================\n";
      
  return($result_geoip);
}

// ------------------------------------------------------------------ //

# function: anomaly detection based on dnsbl information
function detection_dnsbl($threshold, $default_threshold, $ipaddr, &$dnsbl_cache)
{
  // ip address is invalid (e.g. hostname that could not be resolved)
  if (!filter_var($ipaddr, FILTER_VALIDATE_IP))
    return null;

  // ip address found within dnsbl cache and contains a blacklist type
  if (isset($dnsbl_cache[$ipaddr]))
  {
    print_message(3, "[*] [DNSBL detect] Result +$default_threshold ('$ipaddr' listed as '" . $dnsbl_cache[$ipaddr] . "')\n");
    return $default_threshold;
  }
}

// ------------------------------------------------------------------ //

# function: try to evaluate the success of attacks
function attack_quantification(&$dataset, $request, $quantify_type, $data, $path, $client, $lof_minpts_lb, $lof_minpts_ub, &$lof_cache, $lof_min_learn, $max_replay_per_client, $replay_count, $target, $web_app_extensions)
{
  // extract request's status-code and bytes-sent
  $status = isset($data['Final-Status']) ? $data['Final-Status'] : null;
  $bytes = isset($data['Bytes-Sent']) ? (($data['Bytes-Sent'] != '-') ? $data['Bytes-Sent'] : 0) : null;

  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

  // success evaluation based on status-codes
  if (in_array('status', $quantify_type))
  {
    if (preg_match("/^(404)+$/", $status))
      $success[] = 'unsuccessful scan?';
    if (preg_match("/^(401|403)+$/", $status))
      $success[] = 'unsuccessful http-auth';
    if (preg_match("/^(400|408|503)$/", $status))
      $success[] = 'denial of service?';
    if (preg_match("/^(500)$/", $status))
      $success[] = 'buffer overflow?';
    if (preg_match("/^(414)$/", $status))
      $success[] = 'unsuccessful buffer overflow?';
    // highlight requests to yet unknown webapp (CHARS detect mode only)
    if (preg_match("/^(200)+$/", $status)
      and isset($dataset['query'][$path]['chars']['clients'])
      and (count($dataset['query'][$path]['chars']['clients']) == 1)
      and (preg_match("/.*(" . implode('|', $web_app_extensions) . ")$/", $path)))
        $success[] = 'potential webshell?';
  }

  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

  // success evaluation based on bytes-sent outliers
  if (in_array('bytes', $quantify_type))
  {
    // set defaults
    $lof_bytes = 0;
    $lof_data = null;
    // try to get lof info from cache
    if (isset($lof_cache['lof'][$path][$bytes]))
      $lof_bytes = $lof_cache['lof'][$path][$bytes];
    else
    {
      // set reference to dataset of bytes-sent for current path
      if (isset($dataset['query'][$path]['bytes']))
        $lof_data = &$dataset['query'][$path]['bytes'];

      // if we've got a valid bytes-sent value and corresponding dataset
      if (isset($bytes) and !is_null($lof_data))
      {
        // for lower to upper bound of k-nearest neighbors do
        for ($lof_neighbors=$lof_minpts_lb; $lof_neighbors <= $lof_minpts_ub; $lof_neighbors++)
        {
          // create and initialize object of class LOF
          $lof = new LOF($lof_data, $lof_neighbors, $lof_cache);

          // get local outlier factor of bytes-sent
          $lof_values[$lof_neighbors] = $lof->run($client, $bytes);
        }
        // get maximum of collected LOFs
        $lof_bytes = round(max($lof_values));

        // add result to lof cache
        $lof_cache['lof'][$path][$bytes] = $lof_bytes;
      }
    }
    // success evaluation formula
    if ($lof_bytes > 1)
    {
      $success[] = 'Bytes-sent outlier by factor ' . round($lof_bytes);
      print_message(2, "[*] [Quantification] Bytes-sent outlier by factor " . round($lof_bytes) . " from $client on $path, status $status \n");
    }
  }
  
  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

  // check if maximum number of attack replays of current client is already reached
  $max_replay_not_reached = (isset($replay_count[$client]) ? ($replay_count[$client] < $max_replay_per_client) : true);

  // success evaluation based on active replay
  if (in_array('replay', $quantify_type) and $max_replay_not_reached)
  {
    // increase counter for attack replays of current client
    $replay_count[$client] = isset($replay_count[$client]) ? $replay_count[$client]++ : 0;

    if (preg_match("/^(\S+) (.*?) HTTP\/[0-9]\.[0-9]\z/", $data['Request'], $match))
      // extract url path + url query as payload
      $payload = $match[2];
    else
      // this might falsify the result on crippled requests, but we cannot yet handle them with file_get_contents()
      $payload = $data['Request'];

    // set user-agent, timeout, error handling, ...
    $opts = array('http' => array('user_agent' => 'LORG active-replay quantification', 'timeout' => 3, 'ignore_errors' => true));
    // connect to HTTP(S) server and replay attack
    print_message(2, "[*] [Quantification] Fetching contents of http://" . $target . $payload. "\n");
    // retreive content to be inspected from server
    $body = @file_get_contents($target . $path, false, stream_context_create($opts));

    // regular expressions indicating a successful attack
    $signatures = array( # credits for many signatures go to w3af, skipfish, zaproxy and nikto
      // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      'File disclosure: UNIX /etc/passwd' => 'root\:x\:0\:0\:.+\:[0-9a-zA-Z\/]+',
      'File disclosure: Windows boot.ini' => '\[boot loader\](.*)\[operating systems\]',
      // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      'File disclosure: Apache access log' => '0\] "GET \/',
      'File disclosure: Apache error log' => '\[error\] \[client ',
      'File disclosure: IIS access log' => '0, GET, \/',
      // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      'File disclosure: Java source' => 'import java\.',
      'File disclosure: C/C++ source' => '#include',
      'File disclosure: Shell script' => '#\!\/(.*)bin\/',
      'File disclosure: PHP source' => '\<\? ?php(.*)\?\>', # better mime:application/x-httpd-php-source
      'File disclosure: JSP source' => '\<%@(.*)%\>',
      'File disclosure: ASP source' => '\<%(.*)%\>',
      // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      'File disclosure: DSA/RSA private key' => '\-\-\-\-\-BEGIN (D|R)SA PRIVATE KEY\-\-\-\-\-',
      'File disclosure: SQL configuration/logs' => 'ADDRESS\=\(PROTOCOL\=',
      'File disclosure: web.xml config file' => '\<web\-app',
      'File disclosure: SVN RCS data' => 'svn\:special svn',
      'File disclosure: MySQL dump' => '\-\- MySQL dump',
      'File disclosure: phpMyAdmin dump' => ' phpMyAdmin (My)?SQL(\-| )Dump',
      // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      'Info disclosure: Environment variables' => 'REQUEST_URI\=' . preg_quote($path, '/'),
      'Info disclosure: directory listing' => '[iI]ndex [oO]f(.*)\/"\>Parent Directory\<\/a\>',
      'Info disclosure: phpinfo() page' => '\<title\>phpinfo\(\)\<\/title\>\<meta name\=',
      'Info disclosure: Apache mod_status' => '(\<title\>(Apache Status|Server Information)\<\/title\>)(.*)Server Version\:',
      'Info disclosure: ODBC password' => '\(Data Source\=\|Driver\=\|Provider\=\)(.*)(;Password\=|;Pwd\=)',
      // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      'Info disclosure: PHP exception' => 'PHP (Notice|Warning|Error)',
      'Info disclosure: Java IO exception' => 'java.io.FileNotFoundException: ',
      'Info disclosure: Python IO exception' => 'Traceback (most recent call last):',
      // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      'Info disclosure: file system path' => 'Call to undefined function.*\(\) in \/',
      'Info disclosure: web root path' => '\: failed to open stream\: ', # file inclusion attempt?
      'Info disclosure: file inclusion error: ' => 'Warning(?:\<\/b\>)?\:\s+(?:include|require)(?:_once)?\(', # file inclusion attempt?
      'Info disclosure: DB connection error' => '(mysql|pgp|sqlite|mssql)_p?(connect|open|query)\(',
      // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      'Info disclosure: Access error (1)' => 'Sybase message\:',
      'Info disclosure: Access error (2)' => 'Syntax error in query expression',
      'Info disclosure: Access error (3)' => 'Data type mismatch in criteria expression\.',
      'Info disclosure: Access error (4)' => 'Microsoft JET Database Engine',
      'Info disclosure: Access error (5)' => '\[Microsoft\]\[ODBC Microsoft Access Driver\]',
      'Info disclosure: ASP / MSSQL error (1)' => 'System\.Data\.OleDb\.OleDbException',
      'Info disclosure: ASP / MSSQL error (2)' => '\[SQL Server\]',
      'Info disclosure: ASP / MSSQL error (3)' => '\[Microsoft\]\[ODBC SQL Server Driver\]',
      'Info disclosure: ASP / MSSQL error (4)' => '\[SQLServer JDBC Driver\]',
      'Info disclosure: ASP / MSSQL error (5)' => '\[SqlException',
      'Info disclosure: ASP / MSSQL error (6)' => 'System\.Data\.SqlClient\.SqlException',
      'Info disclosure: ASP / MSSQL error (7)' => 'Unclosed quotation mark (before|after) the character string',
      'Info disclosure: ASP / MSSQL error (8)' => '\'80040e14\'',
      'Info disclosure: ASP / MSSQL error (9)' => 'mssql_query\(\)',
      'Info disclosure: ASP / MSSQL error (10)' => 'odbc_exec\(\)',
      'Info disclosure: ASP / MSSQL error (11)' => 'Microsoft OLE DB Provider for',
      'Info disclosure: ASP / MSSQL error (12)' => 'Incorrect syntax near',
      'Info disclosure: ASP / MSSQL error (13)' => 'Syntax error in string in query expression',
      'Info disclosure: ASP / MSSQL error (14)' => 'Procedure \'\[^\'\]+\' requires parameter \'\[^\'\]+\'',
      'Info disclosure: ASP / MSSQL error (15)' => 'ADODB\.(Field \(0x800A0BCD\)\<br\>|Recordset\')',
      'Info disclosure: ASP / MSSQL error (16)' => 'Unclosed quotation mark before the character string',
      'Info disclosure: Coldfusion SQL error' => '\[Macromedia\]\[SQLServer JDBC Driver\]',
      'Info disclosure: DB2 error' => '(SQLCODE|DB2 SQL error\:|SQLSTATE|\[CLI Driver\]\[DB2\/6000\])',
      'Info disclosure: DML error (1)' => '\[DM_QUERY_E_SYNTAX\]',
      'Info disclosure: DML error (2)' => 'has occurred in the vicinity of\:',
      'Info disclosure: DML error (3)' => 'A Parser Error \(syntax error\)',
      'Info disclosure: Generic SQL error' => '(INSERT INTO|SELECT|UPDATE) \.\*?( (FROM|SET) \.\*?)?',
      'Info disclosure: Informix error (1)' => 'com\.informix\.jdbc',
      'Info disclosure: Informix error (2)' => 'Dynamic Page Generation Error\:',
      'Info disclosure: Informix error (3)' => 'An illegal character has been found in the statement',
      'Info disclosure: Informix error (4)' => '\<b\>Warning\<\/b\>\:  ibase_',
      'Info disclosure: Informix error (5)' => 'Dynamic SQL Error',
      'Info disclosure: Java SQL error (1)' => 'java\.sql\.SQLException',
      'Info disclosure: Java SQL error (2)' => 'Unexpected end of command in statement',
      'Info disclosure: MySQL error (1)' => 'supplied argument is not a valid MySQL',
      'Info disclosure: MySQL error (2)' => 'Column count doesn\'t match value count at row',
      'Info disclosure: MySQL error (3)' => 'mysql_fetch_array\(\)',
      'Info disclosure: MySQL error (4)' => 'on MySQL result index',
      'Info disclosure: MySQL error (5)' => 'You have an error in your SQL syntax(;| near)',
      'Info disclosure: MySQL error (5)' => 'MySQL server version for the right syntax to use',
      'Info disclosure: MySQL error (7)' => '\[MySQL\]\[ODBC',
      'Info disclosure: MySQL error (8)' => 'the used select statements have different number of columns',
      'Info disclosure: MySQL error (9)' => 'Table \'[^\']+\' doesn\'t exist',
      'Info disclosure: MySQL error (10)' => 'DBD\:\:mysql\:\:(db|st)(.*)failed',
      'Info disclosure: ORACLE error (6)' => '(PLS|ORA)\-[0-9][0-9][0-9][0-9]',
      'Info disclosure: PostgreSQL error (1)' => 'PostgreSQL query failed\:',
      'Info disclosure: PostgreSQL error (2)' => 'supplied argument is not a valid PostgreSQL result',
      'Info disclosure: PostgreSQL error (3)' => 'pg_(exec|query)\(\) \[\:');

      /* TODO: CHANGE THIS TO FILE MAGIC
       * 
       * >> finfo_buffer($finfo = finfo_open(FILEINFO_MIME_TYPE);
       *
       * MAGICFILE: use /usr/share/misc/magic
       * 
       * FILETYPES: GNOME keyring, OpenSSH RSA1 private key, PEM RSA private key, PEM DSA private key, Microsoft Access Database,
       *            SQLite 2.x database, SQLite 3.x database, PostgreSQL custom database dump, dBaseIV with SQL Table,
       *            MySQL table definition file, MySQL MISAM index file, MySQL MISAM compressed data file,
       *            MySQL ISAM index file, MySQL ISAM compressed data file, MySQL replication log
       *
       * MIMETYPES: text/PGP, application/pgp, application/pgp-encrypted, application/pgp-keys, application/pgp-signature
       *            application/x-executable, application/x-gnucash, application/x-pgp-keyring, application/x-ruby
       *            text/x-lisp, text/x-lua, text/x-msdos-batch, text/x-perl, text/x-php, text/x-shellscript
       */

    // add cross-site-scripting to quantification strings
    if (is_array($request['query']))
    {
      foreach ($request['query'] as $parameter => $value)
      {
        // if we have a XSS attack, match for it's payload in the server's response
        if (preg_match('/\<script/', $value))
          $signatures['XSS might have been sucessful'] = preg_quote($value, '/');
      }
    }

    // match server's response with quantification strings
    foreach ($signatures as $key => $val)
    {
      if (preg_match('/' . $val . '/', $body))
      {
        print_message(1, "[#] [Quantification] Successful attack of type '$key', matching $val\n");
        $success[] = $key;
        // break loop, if successful attack detected
        break;
      }
    }
  }

  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

  ### echo "\n======================= <DEBUG: bytes, status, is_attack> =======================\n";
  ### echo "bytes: $bytes\t | lof: " . abs(round($lof_bytes)) . "\t | status: $status\t | success: $success\t | client: $client   \t | path: $path\n";
  ### echo "======================= </DEBUG:: bytes, status, is_attack> =======================\n";

  return(isset($success) ? implode(' | ', $success) : '-');
}

// ------------------------------------------------------------------ //

# function: classify clients and their sessions, optionally create a nice pchart
function session_classification(&$clients, $request_count, $allowed_methods, $web_app_extensions, $max_session_duration)
{
  // for all clients do
  foreach ($clients as $name => &$client)
  {
    # 1st loop over client's actions
    foreach ($client->actions as &$action)
    {
      # aggregate data used to classify client as night-/working-time visitors
      $client->reset_properties($action, $web_app_extensions, $max_session_duration);
    }

    # classify client as creature of the night or working-time visitor
    $client->classify();

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    // reset last action
    $last_action = null;

    # 2nd loop over client's actions
    foreach ($client->actions as &$action)
    {
      # try to reconstruct, if new session was spawned by client
      if (session_identification($client, $action, $last_action, $max_session_duration))
        $session = $action->new_session = new Session();

      # aggregate data used to classify current session as spawned by human or machine
      $session->reset_properties($action, $allowed_methods, $web_app_extensions, $max_session_duration);

      // set last action to current action
      $last_action = $action;
    }

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    # 3rd loop over client's actions
    foreach ($client->actions as &$action)
    {
      if (isset($action->new_session))
      {
        # Workarround!!! there's a serious bug here, that has to be tracked later...
        if (is_object($action->new_session))
          {
          # set session classification as spawned by human or machine
          $action->new_session = $action->new_session->classify($request_count);
 
          # add session classification to client, if not yet existent
          if (!in_array($action->new_session, $client->classification))
            $client->classification[] = $action->new_session;
        }
      }
    }
  }
}

// ------------------------------------------------------------------ //

# function: insert summarized data into output file
function log_summarized($output_type, $output_stream, $regex_fields, &$clients, $client_ident, $dns_lookup, $geoip_lookup, $dnsbl_lookup, $detect_mode, $quantify_type, $url_decode, $add_tags, $allowed_methods, $attack_count, $threshold, $web_app_extensions, $max_session_duration, $pathes, $dates, $max_harmless_summarize, $pchart_path, $use_pchart_library, $input_file)
{
  // some nice colors for session classificaion
  $session_colors      = array('human attacker' => 'ffeeff', 'random scan' => 'ddddff', 'targeted scan' => 'ccf0ff'); # 'static scan' => 'ccffcc'
  $session_colors_dark = array('human attacker' => 'ffd9ff', 'random scan' => 'ddccff', 'targeted scan' => 'cce0ff'); # 'static scan' => 'bbf0bb'

  // break, if no attacks found
  if ($attack_count == 0)
    return null;

  // reset counter for table names
  $client_index = 0;

  // count number of troublemaking clients
  $client_count = count($clients);

  # insert header data into output file
  log_header($output_type, $output_stream, $regex_fields, $client_ident, $dns_lookup,
             $geoip_lookup, $dnsbl_lookup, $quantify_type, $add_tags, true, $input_file);

  # create a summary, if html report requested
  if ($output_type == 'html')
  {
    // create pie chart of most noisy clients (PNG, base64-encoded)
    $image_clients_base64 = $use_pchart_library ? create_clients_pchart($clients, $pchart_path) : '';

    // create pie chart of most noisy clients (PNG, base64-encoded)
    $image_pathes_base64 = $use_pchart_library ? create_pathes_pchart($pathes, $pchart_path) : '';

    // create bar chart of of traffic/attacks (PNG, base64-encoded)
    # $image_dates_base64 = $use_pchart_library ? create_dates_pchart($dates, $pchart_path) : '';

    // create a summary of client's behaviour
    $logstr = '<h4>Summary: ' . ($attack_count > 1 ? $attack_count . ' incidents' : 'one incident')
            . ' discovered from ' . ($client_count > 1 ? $client_count . ' clients' : 'one client')
            . '</h4><p></p><p>'
            . (empty($image_clients_base64) ? '' : '<img src="data:image/png;base64,' . $image_clients_base64 . '"> ')
            . (empty($image_pathes_base64) ? '' : '<img src="data:image/png;base64,' . $image_pathes_base64 . '"> ')
            # . (empty($image_dates_base64) ? '' : '<img src="data:image/png;base64,' . $image_dates_base64 . '">')
            . "</p>\n";
    fputs($output_stream, $logstr);
  }

  # sort clients by severity of their attacks
  uasort($clients, 'sort_by_severity'); 

  # for all clients do
  foreach ($clients as $name => $client)
  {
    // count client's actions/incidents
    $attack_count = count($client->actions) - $client->harmless_requests;

    // print some message informing about attacks
    print_message(2, "[*] Found " . $attack_count . " attacks from $name, first seen at " . $client->first_seen . "\n");

    # insert html table header into output file
    log_sub_header($output_type, $output_stream, $regex_fields, $client, $client_ident, $dns_lookup, $geoip_lookup,
                   $dnsbl_lookup, $detect_mode, $attack_count, $session_colors, $quantify_type, $add_tags, true, $client_index);

    // reset harmless action counter and session color
    $harmless_count = 0;
    $session_color = null;

    foreach ($client->actions as $action)
    {
      $date = $action->date;
      $data = $action->data;
      $result = $action->result;
      $tags = $action->tags;
      $success = $action->quantification;
      $geoip_data = $action->geoip_data; 
      $dnsbl_data = $action->dnsbl_data;

      if ($action->new_session)
      {
        // if old session color (= session classification) is new one
        // in case we have to sessions of the same type
        if (isset($session_color) and ($session_color == $session_colors[$action->new_session]))
          $session_color = $session_colors_dark[$action->new_session];
        // ...
        else
          $session_color = $session_colors[$action->new_session];

      }
      # add current incident to output file
      if (($action->result > 0) or (($output_type == 'html') and ($harmless_count <= $max_harmless_summarize)))
        log_incident($output_type, $output_stream, $client->name, $client_ident, $dns_lookup, $geoip_data, $dnsbl_lookup, $dnsbl_data,
                     $data, $url_decode, $client_index, $allowed_methods, $session_color, $result, $threshold, $tags, $add_tags, $success, $quantify_type, $client_index);

      // increase counter for harmless action
      if ($action->result >= $threshold)
        $harmless_count = 0;
      else
        $harmless_count++;

      # we have enough data for for exhibit visualiziation
      if ($output_type == 'json')
        break 1;
    }

    # insert html table footer into output file
    log_sub_footer($output_type, $output_stream, $regex_fields, $client_ident, $dns_lookup, $geoip_lookup,
                   $dnsbl_lookup, $quantify_type, $add_tags, true, $client->multiple_remote_hosts, $client_index);

    // increase counter for table names
    $client_index++;
  }

  # insert footer data into output file
  log_footer($output_type, $output_stream);
}

// ------------------------------------------------------------------ //

# function: insert header data into output file
function log_header($type, $stream, $fields, $client_ident, $dns_lookup, $geoip_lookup, $dnsbl_lookup, $quantify_type, $add_tags, $do_summarize, $file)
{
  switch ($type)
  {
    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -//
    case 'csv':
      $logstr = 'Impact;';
      // if PHPIDS is used, add field for tags
      if ($add_tags)
        $logstr .= 'Tags;';
      // add field for attack quantification
      if (!empty($quantify_type))
        $logstr .= 'Quantification';
      // add client-field if not same as remote-host
      if ($client_ident != 'host' or $dns_lookup)
        $logstr .= 'Client;';
      if ($geoip_lookup)
        $logstr .= 'Remote-City;Remote-Location;';
      if ($dnsbl_lookup)
        $logstr .= 'DNSBL;';
      foreach($fields as $key => $val)
        $logstr .= $key . ';';
      break;
    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -//
    case 'html':
      // this is modern times, ppl want stylesheets ;)
      $style = new Style();
      $logstr = '<html><head>
                 <meta content="text/html;charset=utf-8" http-equiv="Content-Type">
                 <meta content="utf-8" http-equiv="encoding">
                 <title>LORG Detection Results [' . basename($file) . ']</title>
                 <style type="text/css">' . $style->get_stylesheet() . '</style>
                 <script>' . $style->get_filterable() . '</script>
                 <script>' . $style->get_sortable() . '</script>';
      $logstr .= $do_summarize ? '
                 <script>' . $style->get_coltable() . '</script>' : '';
      $logstr .= '</head>
                 <body>';
      break;
    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -//
    case 'xml':
      $logstr = '<?xml version="1.0" encoding="ISO-8859-1"?>';
      $logstr .= '<data title="' . basename($file) . '">';
      break;
    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -//
    case 'json':
      $logstr = '{ items: [';
      break;
  }
  fputs($stream, $logstr . "\n");
}

// ------------------------------------------------------------------ //

# function: insert html table header into output file
function log_sub_header($type, $stream, $fields, $client, $client_ident, $dns_lookup, $geoip_lookup, $dnsbl_lookup, $detect_mode, $attack_count, $session_colors, $quantify_type, $add_tags, $do_summarize, $table)
{
  // this is only relevant for html output
  if ($type != 'html')
    return null;

  # define base64-encoded icons
  $icons_base64 = array(
    'dialup'  => '[encoded data, 5888 characters]',
    'missing' => '[encoded data, 5608 characters]',
    'mixed'   => '[encoded data, 5660 characters]',
    'night'   => '[encoded data, 5440 characters]',
    'other'   => '[encoded data, 5400 characters]',
    'proxy'   => '[encoded data, 4864 characters]',
    'spam'    => '[encoded data, 5500 characters]',
    'tor'     => '[encoded data, 5000 characters]',
    'regular' => '[encoded data, 5360 characters]',
    'zombie'  => '[encoded data, 5308 characters]'
  );

  # write summary, in case summarization enabled
  if ($do_summarize)
  {
    // set geolocation, if not in dsnbl and we have valid, single geoip data
    $geolocation = (!$client->first_dnsbl_data and !$client->multiple_geolocations and $geoip_lookup and ($client->first_geoip_data[0] != '-')) ? $client->first_geoip_data[0] : false;
    $countryname = explode(',', $geolocation); $countryname = $countryname[0];

    // set dnsbl type, if we have a valid dns blacklist entry
    $dnsbl_type = (isset($client->first_dnsbl_data) ? $client->first_dnsbl_data : false);

    // set nighttime, if not in dsnbl entry and attack carried out during the night
    $nighttime = (!isset($client->first_dnsbl_data) and $client->night_time_visitor);

    // convert timestamp into human readable, relative date
    $relative_date = relative_date($client->first_seen);

    // merge results of attack quantification
    $quantification = str_replace('?', '', implode(', ', $client->quantification));

    // set classification of client's sessions
    foreach ($client->classification as $val)
      $classification[] = '<font style="background-color:#' . $session_colors[$val] . '">' . $val . '</font>';
    $classification = $classification ? implode(' and ', $classification) : $classification;

    $icon_type = ($dnsbl_type ? $dnsbl_type : ($nighttime ? 'night' : 'regular'));
    $logstr = '<table border="0" align="center" cellpadding="4" cellspacing="0">
    <tr>
      <td bgcolor="#999999" colspan="3"></td>
    </tr>
    <tr bgcolor="#eeeeee">
      <td rowspan="2"><img title="' . $icon_type . ' visitor" src="data:image/png;base64,' . $icons_base64[$icon_type] . '"></td>
      <td><strong>' . $client->name . ' | ' . $attack_count . ' incident' . ($attack_count > 1 ? 's' : '')
      . (!empty($quantify_type) ? ' | <font color="' . ($quantification ? 'red' : 'green') . '">severity: ' . ($quantification ? 'high' : 'low') . '</font>' : '')
      . ' | impact &sum;: ' . $client->result . '</strong></td>
      <td bgcolor="#eeeeee" align="right"><input id="chk_' . $table . '" type="checkbox" name="box_attacks" value="true" checked="checked" onclick="toggle_visibility_benign(\'' . $table . '\');"> attacks only</td>
    </tr>
    <tr bgcolor="#eeeeee">
      <td style="white-space:normal" width="80%">Description: A '
      . $classification . ($geolocation ? ' from ' . $geolocation : '') . ' ' . $relative_date . '.'
      . ' A total of ' . ($attack_count > 1 ? $attack_count . ' incidents where' : 'one incident was') . ' discovered by ' . implode(' and ', $detect_mode) . ' detection modules.'
      . ($dnsbl_type ? ' The attacker covered his identity using a ' . $dnsbl_type . ' node, so tracking might be difficult.' : '')
      . ($quantification ? ' Attack quantification has found this: ' . $quantification . '. You might want to manually check it.' : '')
      . ($nighttime ? ' The incident happened during unusual working hours in ' . $countryname . ', so it might be carried out by a hobbyist on caffeine.' : '')
      . '<td bgcolor="#eeeeee" align="right"><input id="lnk_' . $table . '" type="button" value="[+] show more" onclick="toggle_visibility_details(\'' . $table . '\',\'lnk_' . $table . '\');"></td>
    </tr>
    <tr>
      <td colspan="3">';
    fputs($stream, $logstr);
  }

  # write html table header to output file
  $logstr = '<table id="' . $table . '" ' . ($do_summarize ? 'style="display:none" ' : '') . 'class="sortable">
            <thead><tr><th>Impact</th>';
  // if PHPIDS is used, add field for tags
  if ($add_tags)
    $logstr .= '<th>Tags</th>';
  // add field for attack quantification
  if (!empty($quantify_type))
    $logstr .= '<th>Quantification</th>';
  // add client-field if not same as remote-host
  if ($client_ident != 'host' or $dns_lookup)
    $logstr .= '<th>Client</th>';
  if ($geoip_lookup)
    $logstr .= '<th>Remote-City</th>';
  if ($dnsbl_lookup)
    $logstr .= '<th>DNSBL</th>';
  foreach($fields as $key => $val)
    $logstr .= '<th>' . $key . '</th>';
  $logstr .= '</tr></thead><tbody>';
  fputs($stream, $logstr . "\n");
}

// ------------------------------------------------------------------ //

# function: insert current incident to output file
function log_incident($type, $stream, $client, $client_ident, $dns_lookup, $geoip_data, $dnsbl_lookup, $dnsbl_data, $data, $url_decode, $attack_count, $allowed_methods, $session_color, $result, $threshold, $tags, $add_tags, $quantification, $quantify_type, $client_index)
{
  // decode URL-encoded requests (for visualization only!)
  if ($url_decode)
    $data['Request'] = urldecode($data['Request']);

  // don't fuck up the reports with 'special characters' (binary)
  // (might especially happen when -u flag is used for urlencode)
  foreach($data as $key => $val)
  {
    $data[$key] = filter_var($val, FILTER_SANITIZE_STRING, FILTER_FLAG_STRIP_LOW);
    $data[$key] = empty($data[$key]) ? '-' : $data[$key];
  }

  switch ($type)
  {
    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -//
    case 'csv':
      $logstr = '"' . $result . '";';
      // if PHPIDS is used, add it's tags
      if ($add_tags)
      {
        $logstr .= '"';
        foreach($tags as $tag)
          $logstr .= $tag . ' ';
        $logstr .= '";';
      }
      // add attack quantification
      if (!empty($quantify_type))
        $logstr .= '"' . $quantification . '";';
      // add client-entry if not same as remote-host
      if ($client_ident != 'host' or $dns_lookup)
        $logstr .= '"' . $client . '";';
      if (isset($geoip_data))
        $logstr .= '"' . $geoip_data[0] . '";' . '"' . $geoip_data[1]  . '";';
      if ($dnsbl_lookup)
        $logstr .= '"' . $dnsbl_data . '";';
      // escape csv-reserved characters
      foreach($data as $key => $val)
        $logstr .= '"' . str_replace('\"', '""', $val) . '";';
      break;
    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -//

    case 'html':
      $logstr = '<tr name="' . (($result >= $threshold) ? 'malicious_' : 'benign_') . $client_index . '" style="background-color:#' . $session_color
                . '"><td' . (($result >= $threshold) ? ' bgcolor="#ffffcc"' : '') . '>' . $result . '</td>';
      // if PHPIDS is used, add it's tags
      if ($add_tags)
      {
        $logstr .= '<td>';
        foreach($tags as $tag)
          $logstr .= $tag . ' ';
        $logstr .= '</td>';
      }
      // add attack quantification
      if (!empty($quantify_type))
        $logstr .= '<td' . (($quantification != '-') ? ' bgcolor="#ffbbbb"' : '') . '>' . $quantification . '</td>';
      // add client-entry if not same as remote-host
      if ($client_ident != 'host' or $dns_lookup)
        $logstr .= '<td>' . $client . '</td>';
      if (isset($geoip_data))
        $logstr .= '<td>' . $geoip_data[0] . '</td>';
      if ($dnsbl_lookup)
        $logstr .= '<td>' . $dnsbl_data . '</td>';

      foreach($data as $key => $val)
      {
        // set max size for <td>...</td> content
        $td_limit_break = 4096;
        $td_limit_tooltip = 52;
        // escape html-reserved characters (else we might re-open the payload!)
        $val_filtered = filter_var($val, FILTER_SANITIZE_SPECIAL_CHARS);
        // trim really long lines
        if (strlen($val_filtered) > $td_limit_break)
          $val_filtered = substr($val_filtered, 0 , $td_limit_break);
        // show tooltip for long lines
        if (strlen($val_filtered) > $td_limit_tooltip)
          $logstr .= '<td title="' . $val_filtered . '">' . $val_filtered . '</td>';
        else
          $logstr .= '<td>' . $val_filtered . '</td>';
      }
      $logstr .= '</tr>';
      break;
    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -//
    case 'xml':
      $logstr = '<event start="' . $data['Date'] . '" title="' . $client . ' (' . $result . ')">';
      // this makes perfect sense, dont't touch :)
      $logstr .= 'Impact: ' . $result . '&lt;br&gt;';

      // if PHPIDS is used, add it's tags
      if ($add_tags)
      {
        $logstr .= 'Tags: ';
        foreach($tags as $tag)
          $logstr .= $tag . ' ';
        $logstr .= '&lt;br&gt;';
      }
      // add attack quantification
      if (!empty($quantify_type))
        $logstr .= 'Quantification: ' . $quantification . '&lt;br&gt;';
      // add client-entry if not same as remote-host
      if ($client_ident != 'host' or $dns_lookup)
        $logstr .= 'Remote-City: ' . $client . '&lt;br&gt;';
      if (isset($geoip_data))
        $logstr .= 'Remote-City: ' . $geoip_data[0] . '&lt;br&gt;Remote-Location: '  . $geoip_data[1] . '&lt;br&gt;';
      if ($dnsbl_lookup)
        $logstr .= 'DNSBL: ' . $dnsbl_data . '&lt;br&gt;';
      foreach($data as $key => $val)
        // escape html-reserved characters (else we might re-open the payload!)
        $logstr .= $key . ': ' . htmlentities(filter_var($val, FILTER_SANITIZE_SPECIAL_CHARS), ENT_QUOTES) . '&lt;br&gt;';
      $logstr .= '</event>';
      break;
    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -//
    case 'json':
      // add basic data and categorize result in intervals of ten
      $logstr = '      {
        label: "' . $client . ' [Incident #' . $attack_count . ']' . '",
        type: "Incident",
        index: ' . $attack_count . ',
        client: "' . $client . '",
        impact: [ ' . (($result == 0) ? '-1' : $result) . ' ],
        severity: "' . (($result == 0) ? '0' : '< ' . 10 * ceil(($result +1)/10)) . '",';
      // if PHPIDS is used, add it's tags
      if ($add_tags)
      {
        $logstr .= '
        tags: [ ';
        foreach($tags as $tag)
          $logstr .= '"' . $tag . '",';
        $logstr .= ' ],';
      }
      else
      $logstr .= '
        tags: [ "unknown" ],';
      // add attack quantification
      if (!empty($quantify_type))
      $logstr .= '
        quantification: "' . $quantification . '",';
      // add DNSBL entry
      $logstr .= '
        dnsbl: "' . $dnsbl_data . '",';
      if ($dnsbl_lookup)
      $logstr .= '
        usebl: "true",';
      // add HTTP status-code
      if (isset($data['Final-Status']))
        $logstr .= '
        status: "' . $data['Final-Status'] . '",';
      // add HTTP request-method
      $method = explode(' ', $data['Request']);
      if (in_array($method[0], $allowed_methods) == false)
        $method[0] = 'non-rfc2616';
      $logstr .= '
        method: "' . $method[0] . '",';
      // set max size for <td>...</td> content
      $td_limit_break = 1024;
      // escape html-reserved characters (we're re-opening the payload!)
      $request_filtered = filter_var($data['Request'], FILTER_SANITIZE_SPECIAL_CHARS);
        // trim really long lines
      if (strlen($request_filtered) > $td_limit_break)
        $request_filtered = substr($request_filtered, 0 , $td_limit_break);
      $logstr .= '
        request: "' . $request_filtered . '",
        data: "';
      foreach($data as $key => $val)
        if ($key != 'Request' and $key != 'Date' and $val != '-')
        {
          // escape html-reserved characters (we're re-opening data defined by client)
          $val_filtered = filter_var($val, FILTER_SANITIZE_SPECIAL_CHARS);
          // trim really long lines
          if (strlen($val_filtered) > $td_limit_break)
            $val_filtered = substr($val_filtered, 0 , $td_limit_break);
          $logstr .= $key . ': ' . $val_filtered . '\<br\>';
        }
      $logstr .= '",';
      if (isset($geoip_data))
      {
        // escape html-reserved characters
        $logstr .= '
        remoteCity: "' . htmlentities($geoip_data[0]) . '",
        location: "' . $geoip_data[1] . '",';
      }
      $logstr .= '
      },
      {
        label: "' . $attack_count . '",
        index: ' . $attack_count . ',
        date: "' . date("Y-m-d H:i:s O", strtotime($data['Date'])) . '",
      },';
      break;
  }
  fputs($stream, $logstr . "\n");
}

// ------------------------------------------------------------------ //

# function: insert html table footer into output file
function log_sub_footer($type, $stream, $fields, $client_ident, $dns_lookup, $geoip_lookup, $dnsbl_lookup, $quantify_type, $add_tags, $do_summarize, $multiple_hosts, $table)
{
  // this is only relevant for html output
  if ($type != 'html')
    return null;

  # write html table footer to output file
  $logstr = '</tbody></table><script>';
  // leave space for col 'Impact'
  $num_cols = 1;
  // leave space for PHPIDS 'Tags'
  if ($add_tags)
    $num_cols++;
  // leave space for extra-col 'Quantification'
  if (!empty($quantify_type))
    $num_cols++;
  // leave space for extra-col 'Client'
  if ($client_ident != 'host' or $dns_lookup)
    $num_cols++;
  // leave space for extra-col 'Remote-City'
  if ($geoip_lookup)
    $num_cols++;
  // leave space for extra-col 'DNSBL'
  if ($dnsbl_lookup)
    $num_cols++;
  // hide zero-information columns
  $logstr .= 'var useless_cols = new Array(); ';
  // no need for remote-host field if single host and summarization enabled
  if (!$multiple_hosts and $do_summarize and ($client_ident == 'host'))
    $fields['Remote-Host'] = false;
  foreach($fields as $key => $val)
  {
    if ($val != true)
      $logstr .= 'useless_cols.push("' . $num_cols . '"); ';
    $num_cols++;
  }
  $logstr .= 'setFilterGrid("' . $table . '", {rows_counter: ' . ($do_summarize ? 'false' : 'true') . '});</script>';
  
  # close sub-table, in case summarization enabled
  if ($do_summarize)
    $logstr .= '</td></tr></table>';

  fputs($stream, $logstr);
}

// ------------------------------------------------------------------ //

# function: insert footer data into output file
function log_footer($type, $stream)
{
  switch ($type)
  {
    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -//
    case 'csv':
      $logstr = '';
      break;
    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -//
    case 'html':
      $logstr = '</body></html>';
      break;
    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -//
    case 'xml':
      $logstr = '</data>' . "\n";
      break;
    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -//
    case 'json':
      $logstr = '] }';
      break;
  }
  fputs($stream, $logstr);
}

// ------------------------------------------------------------------ //

# function: print info dependent on current verbosity level
function print_message($level, $msg)
{
  if (($GLOBALS['verbosity']) >= $level)
    echo($msg);
}

// ------------------------------------------------------------------ //

# function: clear last message, then print new message
function carriage_return($level, $msg, $line_count, $input_file_basename)
{
  if ($GLOBALS['verbosity'] == 1)
  {
    print_message($level, "\r");
    // fill up current line with blanks
    for ($char = 0; $char <= 45 + strlen($input_file_basename) + strlen($line_count); $char++)
      print_message($level, " ");
  // print carriage return
  print_message($level, "\r");
  }
  else // print newline
    print_message($level, "\n");

  // print message
  print_message($level, $msg);
}

// ------------------------------------------------------------------ //

# function: print error on crippled lines
function print_badline($line_binary, $line_index, $line_count, $input_type, $input_file_basename)
{
  // don't fuck up the terminal with 'special characters' (binary)
  $line_ascii = preg_replace('/[^(\x20-\x7E)]*/', '', $line_binary);
  carriage_return(1, "[#] Line '$line_index' crippled or not of type '$input_type':\n    '$line_ascii'\n", $line_count, $input_file_basename);
}

// ------------------------------------------------------------------ //

# function: print some statistics
function print_statistics($attack_count, $tags_count, $client_count, $tag_stats, $output_file, $do_summarize, $completeness)
{
  $str_tags = (!empty($tag_stats)) ? " ($tags_count tags)" : '';
  print_message(1, "\n    Found $attack_count incidents$str_tags from $client_count clients");
  # tag statistics
  $tag_index = 0;
  foreach($tag_stats as $tag => $val)
  {
    // insert newline
    if ($tag_index % 3 == 0)
      print_message(1, "\n    | ");
    // print tag and value
    print_message(1, sprintf("%-9s%8d | ", "$tag:", "$val"));
    // increment tag index
    $tag_index++;
  }
  if ($do_summarize and ($completeness == 'partial'))
    print_message(1, "\n\n[>] No partial report written because summarization is enabled\n\n");
  else
    print_message(1, "\n\n[>] Check out '$output_file' for a $completeness report\n\n");
}

// ------------------------------------------------------------------ //

# function: show console progress bar
function progress_bar($index, $count, $progress, $input_file_basename)
{
  // show process bar, only when in verbosity level set to one
  if ($GLOBALS['verbosity'] < 2)
  {
    $progress_new = ceil($index * 100 / $count);
    if ($progress_new != $progress)
    {
      if (empty($input_file_basename)) // show progress bar dataset training
        print_message(1, "\r[>] Training dataset with MCSHMM [$progress_new%]");
      else // show progress bar for main logfile processing loop
        print_message(1, "\r[>] Processing $count lines of input file '$input_file_basename' [$progress_new%]");
      // print newline if we've reached 100%
      if ($progress_new == 100)
        print_message(1, "\n");
    }
    return $progress_new;
  }
}

// ------------------------------------------------------------------ //

# function: parse apache custom log format into regex
#           credits go to Hamish Morgan's apachelogregex
function format_to_regex($format, &$regex_fields, &$regex_string, &$num_fields)
{
  $format = preg_replace(array('/[ \t]+/', '/^ /', '/ $/'), array(' ', '', ''), $format);
  $regex_elements = array();

  foreach(explode(' ', $format) as $element)
  {
    $quotes = preg_match('/^\\\"/', $element);
    if($quotes)
      $element = preg_replace(array('/^\\\"/', '/\\\"$/'), '', $element );

    $regex_fields[formatstr_to_desc($element)] = null;

    if($quotes)
    {
      if ($element == '%r'
      or (preg_match('/{(.*)}/', $element)))
        $x = '\"([^\"\\\\]*(?:\\\\.[^\"\\\\]*)*)\"';
      else
        $x = '\"([^\"]*)\"';
    }
    elseif ( preg_match('/^%.*t$/', $element) )
      $x = '(\[[^\]]+\])';
    else
      $x = '(\S*)';

    $regex_elements[] = $x;
  }

  $regex_string = '/^' . implode(' ', $regex_elements ) . '$/';
  $regex = array($regex_fields, $regex_string);

  ### echo "\n======================= <DEBUG: \$regex> =======================\n";
  ### print_r($regex);
  ### echo "======================= </DEBUG: \$regex> ======================\n";

  // return regex data by reference
  $regex_fields = $regex[0];
  $regex_string = $regex[1];
  $num_fields = count($regex_fields);
}

// ------------------------------------------------------------------ //

# function: convert apache format strings to description
#           credits go to Hamish Morgan's apachelogregex
function formatstr_to_desc($field)
{
  static $orig_val_default = array('s', 'U', 'T', 'D', 'r');
  static $trans_names = array(
    '%' => '',
    'a' => 'Remote-IP',
    'A' => 'Local-IP',
    'B' => 'Bytes-Sent-X',
    'b' => 'Bytes-Sent',
    'c' => 'Connection-Status', // <= 1.3
    'C' => 'Cookie', // >= 2.0
    'D' => 'Time-Taken-MS',
    'e' => 'Env-Var',
    'f' => 'Filename',
    'h' => 'Remote-Host',
    'H' => 'Request-Protocol',
    'i' => 'Request-Header',
    'I' => 'Bytes-Received', // requires mod_logio
    'l' => 'Remote-Logname',
    'm' => 'Request-Method',
    'n' => 'Note',
    'o' => 'Reply-Header',
    'O' => 'Bytes-Sent', // requires mod_logio
    'p' => 'Port',
    'P' => 'Process-Id', // {format} >= 2.0
    'q' => 'Query-String',
    'r' => 'Request',
    's' => 'Status',
    't' => 'Date',
    'T' => 'Time-Taken-S',
    'u' => 'Remote-User',
    'U' => 'Request-Path',
    'v' => 'Server-Name',
    'V' => 'Server-Name-X',
    'X' => 'Connection-Status', // >= 2.0
    );

    foreach($trans_names as $find => $name)
    {
      if(preg_match("/^%([!\d,]+)*([<>])?(?:\\{([^\\}]*)\\})?$find$/", $field, $matches))
      {
        if (!empty($matches[2]) and $matches[2] === '<' and !in_array($find, $orig_val_default, true))
          $chooser = "Original-";
        elseif (!empty($matches[2]) and $matches[2] === '>' and in_array($find, $orig_val_default, true))
          $chooser = "Final-";
        else
          $chooser = '';
        $name = "{$chooser}" . (!empty($matches[3]) ? "$matches[3]" : $name) . (!empty($matches[1]) ? "($matches[1])" : '');
        break;
      }
    }
    if(empty($name))
      return $field;

    // returns original name if there is a problem
    return $name;
}

// ------------------------------------------------------------------ //

# function: convert 'standard english format' to unix timestamp
function apachedate_to_timestamp($time)
{
  list($d, $M, $y, $h, $m, $s, $z) = sscanf($time, "[%2d/%3s/%4d:%2d:%2d:%2d %5s]");
  return strtotime("$d $M $y $h:$m:$s $z");
}

// ------------------------------------------------------------------ //

# function: convert timestamp into human readable, relative date
function relative_date($timestamp)
{
  if(!ctype_digit($timestamp))
    $timestamp = strtotime($timestamp);

  $diff = time() - $timestamp;

  if($diff > 0)
  {
    $day_diff = floor($diff / 86400);
    if($day_diff == 0)
    {
      if($diff < 60) return 'just now';
      if($diff < 120) return 'a minute ago';
      if($diff < 3600) return floor($diff / 60) . ' minutes ago';
      if($diff < 7200) return 'an hour ago';
      if($diff < 86400) return floor($diff / 3600) . ' hours ago';
    }
    if($day_diff == 1) return 'yesterday';
    if($day_diff < 7) return $day_diff . ' days ago';
    if($day_diff < 31) return ceil($day_diff / 7) . ' weeks ago';
    if($day_diff < 60) return 'last month';
    if($day_diff < 365) return ceil($day_diff / 31) . ' month ago';
    if($day_diff < 739) return 'last year';
    return ceil($day_diff / 365) . ' years ago';
    }
    else
      return 'in the future (wrong clock settings?)';
}


// ------------------------------------------------------------------ //

# function: create pie chart of most noisy clients (PNG, base64-encoded)
function create_clients_pchart($clients, $pchart_path)
{
  // extract client's results
  foreach ($clients as $client)
    $datapoints[$client->name] = $client->result;

  // sort clients by result
  asort($datapoints);

  // get the top 10 noisy clients
  $datapoints_top = array_slice($datapoints, -10, 10);

  // create and populate the pData object
  $MyData = new pData();
  $MyData->addPoints($datapoints_top, "ScoreA");
  $MyData->setSerieDescription("ScoreA", "Application A");

  // define the absissa serie
  $MyData->addPoints(array_keys($datapoints_top), "Labels");
  $MyData->setAbscissa("Labels");

  // create the pChart object
  $myPicture = new pImage(490, 260, $MyData);

  // draw a solid background
  $Settings = array("R" => 48, "G" => 48, "B" => 48, "Dash" => 1, "DashR" => 68, "DashG" => 68, "DashB" => 68);
  $myPicture->drawFilledRectangle(0, 0, 490, 300, $Settings);

  // overlay with a gradient
  $Settings = array("StartR" => 138, "StartG" => 138, "StartB" => 138, "EndR" => 217, "EndG" => 217, "EndB" => 217, "Alpha" => 50);
  $myPicture->drawGradientArea(0, 0, 490, 260, DIRECTION_VERTICAL, $Settings);
  $myPicture->drawGradientArea(0, 0, 490, 20, DIRECTION_VERTICAL, array("StartR" => 0, "StartG" => 0, "StartB" => 0, "EndR" => 50, "EndG" => 50, "EndB" => 50, "Alpha" => 100));

  // add a border to the picture
  $myPicture->drawRectangle(0, 0, 489, 259, array("R" => 0, "G" => 0, "B" => 0));

  // write the picture title
  $myPicture->setFontProperties(array("FontName" => $pchart_path."/fonts/Silkscreen.ttf", "FontSize" => 6));
  $myPicture->drawText(10, 13, "Noise per client", array("R" => 255, "G" => 255, "B" => 255));

  // set the default font properties
  $myPicture->setFontProperties(array("FontName" => $pchart_path."/fonts/Forgotte.ttf", "FontSize" => 10, "R" => 255, "G" => 255, "B" => 255));

  // enable shadow computing
  $myPicture->setShadow(TRUE, array("X" => 2, "Y" => 2, "R" => 0, "G" => 0, "B" => 0 , "Alpha" => 50));

  // create the pPie object
  $PieChart = new pPie($myPicture, $MyData);

  // draw an AA pie chart
  $PieChart->draw2DPie(300, 140, array("DrawLabels" => TRUE, "LabelStacked" => TRUE, "Border" => TRUE));

  // write the legend box
  $myPicture->setShadow(FALSE);
  $PieChart->drawPieLegend(15, 40, array("Alpha" => 20));

  // turn on output buffering, render image as PNG and base64-encode
  ob_start();
  imagepng($myPicture->Picture);
  $image_base64 = base64_encode(ob_get_contents());
  ob_end_clean();

  // return base64-encoded image
  return $image_base64;
}

// ------------------------------------------------------------------ //

# function: create pie chart of most attacked web apps (PNG, base64-encoded)
function create_pathes_pchart($pathes, $pchart_path)
{
  $datapoints = &$pathes;

  // sort clients by result
  asort($datapoints);

  // get the top 10 noisy clients
  $datapoints_top = array_slice($datapoints, -8, 8);

  // create and populate the pData object
  $MyData = new pData();
  $MyData->addPoints($datapoints_top, "ScoreA");
  $MyData->setSerieDescription("ScoreA", "Application A");

  // define the absissa serie
  $MyData->addPoints(array_keys($datapoints_top), "Labels");
  $MyData->setAbscissa("Labels");

  // create the pChart object
  $myPicture = new pImage(490, 260, $MyData);

  // draw a solid background
  $Settings = array("R" => 48, "G" => 48, "B" => 48, "Dash" => 1, "DashR" => 68, "DashG" => 68, "DashB" => 68);
  $myPicture->drawFilledRectangle(0, 0, 490, 300, $Settings);

  // overlay with a gradient
  $Settings = array("StartR" => 138, "StartG" => 138, "StartB" => 138, "EndR" => 217, "EndG" => 217, "EndB" => 217, "Alpha" => 50);
  $myPicture->drawGradientArea(0, 0, 490, 260, DIRECTION_VERTICAL, $Settings);
  $myPicture->drawGradientArea(0, 0, 490, 20, DIRECTION_VERTICAL, array("StartR" => 0, "StartG" => 0, "StartB" => 0, "EndR" => 50, "EndG" => 50, "EndB" => 50, "Alpha" => 100));

  // add a border to the picture
  $myPicture->drawRectangle(0, 0, 489, 259, array("R" => 0, "G" => 0, "B" => 0));

  // write the picture title
  $myPicture->setFontProperties(array("FontName" => $pchart_path."/fonts/Silkscreen.ttf", "FontSize" => 6));
  $myPicture->drawText(10, 13, "Noise per webapp", array("R" => 255, "G" => 255, "B" => 255));

  // set the default font properties
  $myPicture->setFontProperties(array("FontName" => $pchart_path."/fonts/Forgotte.ttf", "FontSize" => 10, "R" => 255, "G" => 255, "B" => 255));

  // enable shadow computing
  $myPicture->setShadow(TRUE, array("X" => 2, "Y" => 2, "R" => 0, "G" => 0, "B" => 0 , "Alpha" => 50));

  // create the pPie object
  $PieChart = new pPie($myPicture, $MyData);

  // draw an AA pie chart
  $PieChart->draw2DPie(300, 140, array("DrawLabels" => TRUE, "LabelStacked" => TRUE, "Border" => TRUE));

  // write the legend box
  $myPicture->setShadow(FALSE);
  $PieChart->drawPieLegend(15, 40, array("Alpha" => 20));

  // turn on output buffering, render image as PNG and base64-encode
  ob_start();
  imagepng($myPicture->Picture);
  $image_base64 = base64_encode(ob_get_contents());
  ob_end_clean();

  // return base64-encoded image
  return $image_base64;
}


// ------------------------------------------------------------------ //

# function: create bar chart of of traffic/attacks (PNG, base64-encoded)
function create_dates_pchart($dates, $pchart_path)
{
  $datapoints = &$dates;

  // create and populate the pData object
  $MyData = new pData();
  $MyData->addPoints($datapoints['traffic'], "ScoreA");
  $MyData->addPoints($datapoints['attacks'], "ScoreB");
  $MyData->addPoints(array_keys($datapoints['traffic']), "Labels");
  $MyData->setSerieDescription("ScoreA", "Application A");
  $MyData->setSerieDescription("ScoreB", "Application B");

  // define the absissa serie
  $MyData->setAbscissa("Labels");

  // create the pChart object
  $myPicture = new pImage(490, 260, $MyData);

  // draw a solid background
  $Settings = array("R" => 48, "G" => 48, "B" => 48, "Dash" => 1, "DashR" => 68, "DashG" => 68, "DashB" => 68);
  $myPicture->drawFilledRectangle(0, 0, 490, 300, $Settings);

  // overlay with a gradient
  $Settings = array("StartR" => 138, "StartG" => 138, "StartB" => 138, "EndR" => 217, "EndG" => 217, "EndB" => 217, "Alpha" => 50);
  $myPicture->drawGradientArea(0, 0, 490, 260, DIRECTION_VERTICAL, $Settings);
  $myPicture->drawGradientArea(0, 0, 490, 20, DIRECTION_VERTICAL, array("StartR" => 0, "StartG" => 0, "StartB" => 0, "EndR" => 50, "EndG" => 50, "EndB" => 50, "Alpha" => 100));

  // add a border to the picture
  $myPicture->drawRectangle(0, 0, 489, 259, array("R" => 0, "G" => 0, "B" => 0));

  // write the picture title
  $myPicture->setFontProperties(array("FontName" => $pchart_path."/fonts/Silkscreen.ttf", "FontSize" => 6));
  $myPicture->drawText(10, 13, "Traffic over time", array("R" => 255, "G" => 255, "B" => 255));

  // set the default font properties
  $myPicture->setFontProperties(array("FontName" => $pchart_path."/fonts/Forgotte.ttf", "FontSize" => 10, "R" => 255, "G" => 255, "B" => 255));

  // enable shadow computing
  $myPicture->setShadow(TRUE, array("X" => 2, "Y" => 2, "R" => 0, "G" => 0, "B" => 0 , "Alpha" => 50));

  // create the pPie object
  $PieChart = new pPie($myPicture, $MyData);

  // draw an AA pie chart
  $PieChart->draw2DPie(300, 140, array("DrawLabels" => TRUE, "LabelStacked" => TRUE, "Border" => TRUE));

  // write the legend box
  $myPicture->setShadow(FALSE);
  $PieChart->drawPieLegend(15, 40, array("Alpha" => 20));

  // turn on output buffering, render image as PNG and base64-encode
  ob_start();
  imagepng($myPicture->Picture);
  $image_base64 = base64_encode(ob_get_contents());
  ob_end_clean();

  // return base64-encoded image
  return $image_base64;
}

// ------------------------------------------------------------------ //

# function: if logline entry contains zero-information, mark aus useless
#           (according to output type, useless entries will not be shown)
  function mark_regex_fields_useful($regex_fields, $data)
{
  foreach ($data as $key => $val)
    if (!(($val == '-')))
      $regex_fields[$key] = true;
  return $regex_fields;
}

// ------------------------------------------------------------------ //

# function: normalize parts of a request, using the PHPIDS converter
function convert_using_phpids(&$str, $key, $phpids_data)
{
  $phpids_path = &$phpids_data[0];
  $phpids_init = &$phpids_data[1];

  try // pipe request through PHPIDS-converter
  {
    // normalize string using the PHPIDS-converter
    $ids = new IDS_Monitor(array(), $phpids_init);
    $converter = new IDS_Converter();
    $str = $converter->runAll($str, $ids);
  }
  catch (Exception $e)
  {
    // sth went terribly wrong
    print_message(0, "[!] PHPIDS error occured: " . $e->getMessage() . "\n");
    return null;
  }
}

// ------------------------------------------------------------------ //

# function: substitute alphanumeric elements of a string
function convert_alphanumeric($str)
{
  $str_subst = preg_replace('/[\p{L}äöüÄÖÜß]/', 'A', $str); // replace letters a-Z with A
  $str_subst = preg_replace('/\d/', 'N', $str_subst); // replace digits 0..9 with N
  return $str_subst;
}

// ------------------------------------------------------------------ //

# function: remove alphanumeric and other less suspicious chars from string
function remove_alphanumeric($str)
{
  $str_subst = preg_replace('#\\.\\.#','**', $str); // replace '..' with double-supspicious char '$$'
  $str_subst = preg_replace('/[\p{L}äöüÄÖÜß\d]/', '', $str_subst); // remove 'less suspicious' chars

  # RFC2396 reserved = alphanum | "-" | "_" | "." | "!" | "~" | "*" | "'" | "(" | ")"
  # RFC2396 unreserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | "$" | ","

  return $str_subst;
}

// ------------------------------------------------------------------ //

# function: try to retrieve client's geo-infos
function geo_targeting($ipaddr, $geoip_stream, &$geoip_cache)
{
  // try to get geoip entry from cache
  if (isset($geoip_cache[$ipaddr]))
    return $geoip_cache[$ipaddr];
  else
  // return sth, if geoip is enabled
  if (isset($geoip_stream))
  {
    // set defaults
    $remote_city = '-'; $remote_location = '-'; $country_code = null;

    // do the geoiplookup
    $geoip_record = geoip_record_by_addr($geoip_stream, $ipaddr);

    // build the geoip-info
    if (isset($geoip_record->country_name))
      $remote_city = $geoip_record->country_name;
    if (isset($geoip_record->city))
      $remote_city = $remote_city . ', ' . $geoip_record->city;
    if (isset($geoip_record->latitude) and isset($geoip_record->longitude))
      $remote_location = number_format($geoip_record->latitude, 4) . ',' . number_format($geoip_record->longitude, 4);
    if (isset($geoip_record->country_code))
      $country_code = $geoip_record->country_code;

    // geoip-info contains remote city/county and geocordinates
    $geoip_data = array($remote_city, $remote_location, $country_code);

    // add entry to geoip cache
    $geoip_cache[$ipaddr] = $geoip_data;

    ### echo "\n======================= <DEBUG: geo_targeting()> =======================\n";
    ### echo "ipaddr: $ipaddr | remote_city: $remote_city\n";
    ### echo "======================= </DEBUG: geo_targeting()> ======================\n";

    return $geoip_data;
  }
  else return null;
}

// ------------------------------------------------------------------ //

# function: lookup ip address in dns blacklist(s)
function ipaddr_to_dnsbl($ipaddr, $dnsbl_type, $allowed_dnsbl_types, &$dnsbl_cache)
{
  // try to get dnsbl entry from cache
  if (isset($dnsbl_cache[$ipaddr]))
    return $dnsbl_cache[$ipaddr];
  else
  {
    // set defaults
    $dnsbl_data = null;

	  // convert ip address to reverse IN-ADDR entry
    $reverse_ipaddr = implode('.', array_reverse(explode('.', $ipaddr)));
    // for all blacklist types/catgories
    foreach ($dnsbl_type as $type)
      // for all blacklist servers of a type
      foreach($allowed_dnsbl_types[$type] as $listname)
        // lookup dnsbl entry in current blacklist
        if (checkdnsrr($reverse_ipaddr . '.' . $listname . '.', 'A'))
        {
          $dnsbl_data = $type;
          break 2;
        }

    // add entry to dnsbl cache
    $dnsbl_cache[$ipaddr] = $dnsbl_data;

    ### echo "\n======================= <DEBUG: \$dnsbl> =======================\n";
    ### echo "[$type] $ipaddr is listed in dnsbl $dnsbl\n";
    ### echo "dnsbl-cache: "; print_r($dnsbl_cache);
    ### echo "======================= </DEBUG: \$dnsbl> ======================\n";

    return $dnsbl_data;
  }
}

// ------------------------------------------------------------------ //

# function: convert ip address to hostname, if possible
function ipaddr_to_hostname($ipaddr, &$dns_cache)
{
  // return argument, if it already contains a hostname
  if (preg_match("/^.*[a-zA-Z]$/", $ipaddr))
    return $ipaddr;

  // try to get dns info from cache
  if (isset($dns_cache[$ipaddr]))
    $hostname = $dns_cache[$ipaddr];
  else
  {
    // convert ip address to reverse IN-ADDR entry
    $reverse_ipaddr = implode('.', array_reverse(explode('.', $ipaddr))) . '.in-addr.arpa';
    // lookup PTR-record, but faster than gethostbyaddr()
    $record = dns_get_record($reverse_ipaddr, DNS_PTR);
    // check if ip address resolved to hostname
    if (isset($record[0]['target']))
      $hostname = $record[0]['target'];
    // if not, return ip address
    else
      $hostname = $ipaddr;
    // add entry to dns cache
    $dns_cache[$ipaddr] = $hostname;
  }
  ### echo "\n======================= <DEBUG: \$ipaddr> =======================\n";
  ### echo "ipaddr: $ipaddr | hostname: $hostname\n";
  ### echo "dns_cache: "; print_r($dns_cache);
  ### echo "======================= </DEBUG: \$ipaddr> ======================\n";

  return $hostname;
}

// ------------------------------------------------------------------ //

# function: convert hostname to ip address, if possible
function hostname_to_ipaddr($hostname, &$dns_cache)
{
  // return argument, if it already contains an ip address
  if (filter_var($hostname, FILTER_VALIDATE_IP))
    return $hostname;

  // try to get dns info from cache
  if (isset($dns_cache[$hostname]))
    $ipaddr = $dns_cache[$hostname];
  else
  {
    // lookup A-record, but faster than gethostbyname()
    $record = dns_get_record($hostname, DNS_A);
    // check if hostname resolved to ip address
    if (isset($record[0]['ip']))
      $ipaddr = $record[0]['ip'];
    // if not, return hostname
    else
      $ipaddr = $hostname;
    // add entry to dns cache
    $dns_cache[$hostname] = $ipaddr;
  }
  ### echo "\n======================= <DEBUG: \$hostname> =======================\n";
  ### echo "hostname: $hostname | ipaddr: $ipaddr\n";
  ### echo "dns_cache: "; print_r($dns_cache);
  ### echo "======================= </DEBUG: \$hostname> ======================\n";

  return $ipaddr;
}

// ------------------------------------------------------------------ //

# function: create a fake PHPIDS result with zero impact
function fake_phpids_result()
{
  $filter = new IDS_Filter('-1', '(.*)', 'dummy', array(0 => 'none'), 0);
  $event = new IDS_Event('dummy', 'none', array($filter));
  $result = new IDS_Report(array($event));
  return $result;
}

// ------------------------------------------------------------------ //

# calculate variance and mean, based on Welford1962 online algorithm
function online_variance(&$avg, &$var, &$index, $value)
{
  $index++;
  $delta = $value - $avg;
  $avg += $delta/$index;
  $var += $delta * ($value - $avg);
}

// ------------------------------------------------------------------ //

# function: sort clients by severity of their attacks
function sort_by_severity($client1, $client2)
{
  // 1st criterion: quantification sum
  $severity = $client2->severity - $client1->severity;
  // 2nd criterion: impact sum
  $impact = $client2->result - $client1->result;

  return ($severity != 0) ? $severity : $impact;
}

// ------------------------------------------------------------------ //

# function: include one or more files within given path
function include_multiple($path, array $files)
{
  // reset include path
  set_include_path(get_include_path().PATH_SEPARATOR.$path);

  // try to include files
  foreach($files as $file)
  {
    if (!(file_exists("$path/$file")) or !((include_once("$path/$file"))))
	  return false;
  }

  // all files included successfully
  return true;
}

// ------------------------------------------------------------------ //

# function: join (multi-dimensional) array elements with a string
function implode_recursive($glue, array $multi_array)
{
  $str = '';
  foreach ($multi_array as $key => $val)
    $str .= is_array($val) ? implode_recursive($glue, $val) : $glue . $val;
  return $str;
}

// ------------------------------------------------------------------ //

# function: pick one or more random entries out of an multi-dimensional array
function array_rand_multi($array, $limit = 2)
{
  uksort($array, 'callback_rand');
  return array_slice($array, 0, $limit, true); 
}

// ------------------------------------------------------------------ //

# function: callback for function array_rand_multi()
function callback_rand()
{ 
  return rand() > rand();
}

// ------------------------------------------------------------------ //

# function: do a clean exit when reveiving SIGINT
function clean_exit()
{
  // define those variables global, we have to deal with
  global $output_file, $output_stream, $output_type, $regex_fields, $client_ident, $dns_lookup, $geoip_lookup, $dnsbl_lookup,
         $quantify_type, $add_tags, $attack_index, $attack_count, $clients, $tags_count, $tag_stats, $do_summarize;

  print_message(0, "\n[!] SIGINT received - " . ($do_summarize ? 'exiting' : 'writing report and exiting'));

  if ($do_summarize)
  { // delete (empty) report file
    unlink($output_file);
  }
  else
  { // insert footer data into output file
    log_sub_footer($output_type, $output_stream, $regex_fields, $client_ident, $dns_lookup,
                   $geoip_lookup, $dnsbl_lookup, $quantify_type, $add_tags, false, false, '0');
    log_footer($output_type, $output_stream);
  }

  // print some statistics
  print_statistics($attack_count, $tags_count, count($clients), $tag_stats, $output_file, $do_summarize, 'partial');

  // time to say goodbye
  exit();
}

// ============================= CLIENT ============================= //

class Client
{
  public $name;
  public $result = 0;
  public $severity = 0;
  public $actions = array();
  public $quantification = array();
  public $classification = array();
  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  public $first_seen = null;
  public $first_geoip_data = null;
  public $first_dnsbl_data = null;
  public $first_remote_host = null;
  public $multiple_remote_hosts = false;
  public $multiple_geolocations = false;
  public $last_non_autoload_visit = null;
  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  public $harmless_requests = 0;
  public $avg_time_delay = 0.0;
  public $std_time_delay = 0.0;
  public $index_time_delay = 0;
  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  public $night_time_visitor = false;

// ------------------------------------------------------------------ //

  # function: construct client and set name
  public function __construct($name)
  {
    $this->name = $name;
  }

// ------------------------------------------------------------------ //

  # function: add action to client's actions
  public function add_action(Action $action)
  {
    array_push($this->actions, $action);
  }

// ------------------------------------------------------------------ //

  # function: aggregate data used to classify client as night-/working-time visitors
  function reset_properties($action, $web_app_extensions, $max_session_duration)
  {
    // set date of first rendezvous
    if (!isset($this->first_seen))
      $this->first_seen = $action->date;

    // set first seen remote host
    if (!isset($this->first_remote_host))
      $this->first_remote_host = $action->remote_host;

    // set first geoip-data (might change if client ident not address)
    if (!isset($this->first_geoip_data))
      $this->first_geoip_data = $action->geoip_data;

    // set first dnsbl-data (might change if client ident not address)
    if (!isset($this->first_dnsbl_entry))
      $this->first_dnsbl_data = $action->dnsbl_data;

    // set if client origins from multiple hosts
    if ($action->remote_host != $this->first_remote_host)
      $this->multiple_remote_hosts = true;

    // set if client origins from multiple geolocations
    if ($action->geoip_data[1] != $this->first_geoip_data[1])
      $this->multiple_geolocations = true;

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    // if we can act on the assumtion of a non-autoload request
    if ((is_string($action->path)) and (preg_match("/.*(html?|\/|" . implode('|', $web_app_extensions) . ")$/", $action->path)))
    {
      // get inter-request time delay
      $delay = strtotime($action->date) - strtotime($this->last_non_autoload_visit);

      // if we're within a session's timespan and don't have a future timestamp in the future
      if (($delay <= $max_session_duration) and ($delay >= 0))
      {
        // online variance calculation of inter-request time delay
        online_variance($this->avg_time_delay, $this->std_time_delay, $this->index_time_delay, $delay);

        // set time of last non-autoload request
        $this->last_non_autoload_visit = $action->date; 
      }
      // first-time set last non-autoload request
      if (!isset($this->last_non_autoload_visit))
        $this->last_non_autoload_visit = $action->date;
    }
  }

// ------------------------------------------------------------------ //

  # function: classify client as creature of the night or working-time visitor
  function classify()
  {
    # final calculation for standard deviation of client's inter-request time delay
    $this->std_time_delay = sqrt(($this->index_time_delay > 1) ? $this->std_time_delay / ($this->index_time_delay - 1) : $this->std_time_delay);

    // check if geoip lookup enabled
    if (isset($this->first_geoip_data))
    {
      // extract client's geolocation
      $geolocation = explode(',', $this->first_geoip_data[1]);

      // if we've got a valid geolocation
      if ($geolocation[0] != '-')
      {
        // get date of first rendez-vous with client
        $date = strtotime($this->first_seen);
        // determine sun info in client's geolocation at that date
        $sun_info = date_sun_info($date, $geolocation[0], $geolocation[1]);
        // determine if attack happend during 'unusual' working-time, which we define as 6 hours before
        // and 10 hours after transit; note: one could also define this as the time between sunrise and
        // sunset, but it won't work out e.g. for all-sunshine hackers from the polar circle at midummer
        if (($date  < ($sun_info['transit']) - 6*3600) or ($date  > ($sun_info['transit'] + 10*3600)))
          $this->night_time_visitor = true;
        print_message(2, "[*] [Classification] Classifying client $this->name as " . ($this->night_time_visitor ? "creature of the night" : "working-time visitor" ) . "\n");
      }
    }
  }
}

// ============================= SESSION ============================= //

class Session
{
  public $classification = array();
  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  #!# public $requests_made = array();
  #!# public $dirnames_visited = array();
  public $last_non_autoload_visit = null;
  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  public $number_of_requests = 0;
  #!# public $rel_number_of_requests = 0.0;
  ### public $avg_requests_per_path = 0.0;
  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  public $number_of_webapp_requests = 0;
  #!# public $ratio_of_webapp_requests = 0.0;
  #!# public $ratio_of_image_requests = 0.0;
  #!# public $ratio_of_repeated_requests = 0.0;
  public $requests_robots_file = false;
  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  public $avg_time_delay = null;
  public $std_time_delay = null;
  public $index_time_delay = 0;
  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  public $ratio_of_GET_requests = null;
  public $ratio_of_HEAD_requests = null;
  public $ratio_of_POST_requests = null;
  public $ratio_of_other_requests = null;
  public $ratio_of_non_rfc2616_requests = null;
  // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  public $ratio_of_status_code_4xx = 0.0;

  // ---------------------------------------------------------------- //

  # function: aggregate data used to classify session as spawned by human or machine
  function reset_properties($action, $allowed_methods, $web_app_extensions, $max_session_duration)
  {
    // determine method
    $method = explode(' ', $action->data['Request']);
    $method = in_array($method[0], $allowed_methods) ? $method[0] : 'non_rfc2616';

    // determine status
    $status = isset($action->data['Final-Status']) ? $action->data['Final-Status'] : null;

    // determine referer
    $referer = isset($action->data['Referer']) ? $action->data['Referer'] : null;

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    // increment number of requests in this session
    $this->number_of_requests++;

    // check if robots.txt file requested
    if ($action->path == '/robots.txt')
      $this->requests_robots_file = true;

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    // if we can act on the assumtion of a non-autoload request
    if ((is_string($action->path)) and (preg_match("/.*(html?|\/|" . implode('|', $web_app_extensions) . ")$/", $action->path)))
    {
      // increase number of webapps requested
      $this->number_of_webapp_requests++;

      if (isset($this->last_non_autoload_visit) and isset($action->date))
      {
        // get inter-request time delay
        $delay = strtotime($action->date) - strtotime($this->last_non_autoload_visit);

        // if we're within a valid session's timespan
        if ($delay <= $max_session_duration)
        {
          // online variance calculation of inter-request time delay
          online_variance($this->avg_time_delay, $this->std_time_delay, $this->index_time_delay, $delay);;
        }
      }
      $this->last_non_autoload_visit = $action->date;
    }

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    // increment counter for used HTTP methods
    if (in_array($method, array('GET', 'POST', 'HEAD', 'non_rfc2616')))
    {
      $var = 'ratio_of_'.$method.'_requests';
      $this->$var++;
    }
    else
      $this->ratio_of_other_requests++;

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    // increase number of 4xx response codes
    if (preg_match("/^(4)[0-9]+$/", $status))
      $this->ratio_of_status_code_4xx++;
  }

  // ---------------------------------------------------------------- //

  # function: classify session as spawned by human or machine (robot, scanner)
  function classify($request_count)
  {
    $this->ratio_of_status_code_4xx /= $this->number_of_requests;

    // set client's ratio of GET/HEAD/POST/non-rfc2616/other requests
    $this->ratio_of_GET_requests /= $this->number_of_requests;
    $this->ratio_of_HEAD_requests /= $this->number_of_requests;
    $this->ratio_of_POST_requests /= $this->number_of_requests;
    $this->ratio_of_non_rfc2616_requests /= $this->number_of_requests;
    $this->ratio_of_other_requests /= $this->number_of_requests;

    # final calculation for standard deviation of a session's inter-request time delay
    $this->std_time_delay = sqrt(($this->index_time_delay > 1) ? $this->std_time_delay / ($this->index_time_delay - 1) : $this->std_time_delay);

    // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

    // human attacker by default
    $classification = 'human attacker'; # targeted probing

    // static or fuzzy scan
    if ($this->requests_robots_file
    or ($this->avg_time_delay < 1 and !is_null($this->avg_time_delay))
    or ($this->number_of_webapp_requests > 1000)
    or ($this->ratio_of_status_code_4xx > sqrt($this->number_of_requests) / $this->number_of_requests)
    or ($this->ratio_of_HEAD_requests + $this->ratio_of_non_rfc2616_requests + $this->ratio_of_other_requests > 0))
      $classification = 'targeted scan';

    // random scan for specific vulnerability, if client has made only one request
    if (($this->number_of_requests == 1))
      $classification = 'random scan';

    print_message(2, "[*] [Classification] Classifying session as $classification\n");

    ### echo "\n======================= <DEBUG: session properties> =======================\n";
    ### print_r($this);
    ### echo "======================= </DEBUG: session properties> ======================\n";

    return $classification;
  }
}

// ============================= ACTION ============================= //

class Action
{
  public $date;
  public $data;
  public $path;
  public $result;
  public $tags;
  public $quantification;
  public $remote_host;
  public $geoip_data;
  public $dnsbl_data;
  public $new_session;

  # function: construct action
  public function __construct($date, $data, $path, $result, $tags, $quantification, $remote_host, $geoip_data, $dnsbl_data)
  {
    $this->date = $date;
    $this->data = $data;
    $this->path = $path;
    $this->result = $result;
    $this->tags = $tags;
    $this->quantification = $quantification;
    $this->remote_host = $remote_host;
    $this->geoip_data = $geoip_data;
    $this->dnsbl_data = $dnsbl_data;
  }
}

// ============================ ENSEMBLE ============================ //

# Algorithms ported to PHP, based on "HMM-Web: a framework for the #
# detection of attacks against Web applications" from [Corona2009] #

class Ensemble
{
  // models within this ensemble
  protected $models = array();

  # function: construct ensemble of HMMs from a set of labels
  public function __construct(array $labels, $all_values, $path, $parameter, $hmm_num_models)
  {
    $sum_len = 0;
    foreach($all_values as $value)
    {
      $char_lst = array();
      foreach ($value as $char)
      {
        if (!in_array ($char, $char_lst))
        {
          $char_lst[] = $char;
        }
      }
      $sum_len += count($char_lst);
    }
    $states = round($sum_len / count($all_values));

    if ($states < 1)
    {
      print_message(1, "[#] Number of states is zero, setting to one\n");
      $states = 1;
    }

    # create HMMs of ensemble
    for ($i=0; $i<$hmm_num_models; $i++)
    {
      // create an HMM with n states
      $hmm = new HMM($states, $labels);
      $hmm->randomize();
      $this->models[] = $hmm;

      ### echo "\n======================= <DEBUG: HMM ensemble =======================\n";
      ### print_r($this);
      ### echo "======================= </DEBUG: HMM ensemble> ======================\n";
    }
  }

// ------------------------------------------------------------------ //

  # function: train ensemble of HMMs
  public function train($observations, $max_iter, $universe, $tolerance)
  {
    // for all HMMs in ensemble do
    foreach($this->models as $hmm)
      // make HMM learn using the Baum-Welch algorithm
			$hmm->train_baum_welch($observations, $universe, $max_iter, $tolerance);
  }

// ------------------------------------------------------------------ //

  # function: test ensemble of HMMs
  public function test($observation, $universe, $hmm_decrease)
  {
    // for all HMMs in ensemble do
    foreach($this->models as $hmm)
      // get probability of validity of observation
      $probabilities[] = $hmm->get_probability($observation, $universe, $hmm_decrease);

    // use maximum rule for MCS classification
    return max($probabilities);
  }    
}

// ============================== HMM =============================== //

# Algorithms ported to PHP, based on "HMM-Web: a framework for the #
# detection of attacks against Web applications" from [Corona2009] #

class HMM
{
  // number of states of the HMM
  protected $states;
  // node labels in the HMM
  protected $labels = array();
  // start probabilities as an adjazence matrix
  protected $start = array();
  // transition probabilities as an adjazence matrix
  protected $transition = array();
  // emission probabilities as an adjazence matrix
  protected $emission = array();

// ------------------------------------------------------------------ //

  # function: construct hidden markov model from a set of labels
  public function __construct($states, array $labels)
  {
    // initialize properties of HMM
    $this->states     = $states;
    $this->labels     = array_values($labels);
    $this->start      = array_fill(0, $states, 1 / $states);
    $this->transition = array_fill(0, $states, array_fill(0, $states, 1 / $states));
    $this->emission   = array_fill(0, $states, array_fill(0, count($labels), 1 / count($labels)));

    print_message(2, "[*] [MCSHMM detect] HMM created with $states states\n");
  }

// ------------------------------------------------------------------ //

  # function: create random transition, emission and start probabilities
  public function randomize()
  {
    for ($i = 0; $i < $this->states; ++$i)
    {
      $this->transition[$i] = $this->get_random_array($this->states);
      $this->emission[$i]   = $this->get_random_array(count($this->labels));
    }
    $this->start = $this->get_random_array($this->states);
  }

// ------------------------------------------------------------------ //

  # function: get array of random values which sum up to 1
  protected function get_random_array($size)
  {
    $return = array();
    $left   = 1;
    for ($i = 0; $i < $size; ++$i)
    {
      if ($i === ($size - 1))
      {
        $return[] = $left;
        break;
      }
      $return[] = $v = mt_rand(0, $left * 10000) / 10000;
      $left    -= $v;
    }
    return $return;
  }

// ------------------------------------------------------------------ //

  # function: map labels to their indexes
  public function map_labels(array $labels)
  {
    $indexes = array();
    foreach ($labels as $label)
    {
      if (($key = array_search($label, $this->labels)) === false)
        throw new OutOfBoundsException();
      $indexes[] = $key;
    }
    return $indexes;
  }

// ------------------------------------------------------------------ //

  # function: get probability of validity of observation
  public function get_probability(array $observation, array $universe, $hmm_decrease)
  {
    // define observation as valid
    $probability = 1;

    // define known and unknown symbols
    $known_symbols = $unknown_symbols = array();

    // for each symbol in the observation
    foreach ($observation as $symbol)
    {
      // check if symbol is within universe
      if (in_array($symbol, $universe))
        // add to known symbols
        $known_symbols[] = $symbol;
      else
        // add to unknown symbols
        $unknown_symbols[] = $symbol;
    }

    // if observation contains known symbols
    if ($known_symbols)
      // apply logarithmic version of viterbi algorithm
      $probability = $this->test_viterbi($known_symbols);

    // if observation contains symbols never seen before
    if ($unknown_symbols)
    {
      print_message(2, "[*] [MCSHMM detect] Unknown symbols in observation: '" . implode($observation) . "'\n");
      $probability = ($probability + 5) * $hmm_decrease; // decrease probability of validity
    }

    // return double value between 1.0 (valid) and 0.0 (suspicious)
    return $probability;
  }

// ------------------------------------------------------------------ //

  # function: logarithmic version of viterbi algorithm
  public function test_viterbi(array $observation)
  {
    $len_observation = count($observation);
    $sequence = $this->map_labels($observation);
    $path = array();
    $ksi = array(array());

    # initialize base cases (t == 0)
    for ($y = 0; $y < $this->states; $y++)
    {
      $ksi[0][$y] = $this->start[$y] * $this->emission[$y][$sequence[0]];
      $path[$y] = array($y);
    }
 
    # run Viterbi for t > 0
    for ($t = 1; $t < $len_observation; $t++)
    {
      $obs_t = $sequence[$t];
      $newpath = array();
      # for all states do
      for ($y = 0; $y < $this->states; $y++)
      {
        # calculate state/probability tuple, where probablility is max
        $probability = 0.0;
        $state = 0;
        for ($y0 = 0; $y0 < $this->states; $y0++)
        {
          $new_probability = $ksi[$t - 1][$y0] * $this->transition[$y0][$y] * $this->emission[$y][$obs_t];
          ### echo "\nstate: $y0 | new_probability: $new_probability";
          if ($new_probability > $probability)
          {
            ### echo " <<< SET!";
            $probability = $new_probability;
            $state = $y0;
          }
        }
        ### echo "\n------------------------------------------------------";
        $ksi[$t][$y] = $probability;
        $newpath[$y] = $path[$state] + array($y);
      }
      # don't need to remember the old paths
      $path = $newpath;
    }

    # calculate state/probability tuple, where probablility is max
    $probability = 0.0;
    $state = 0;
    for ($y = 0; $y < $this->states; $y++)
    {
      $new_probability = ($ksi[$len_observation-1][$y]);
      ### echo "\nstate: $y0 | new_probability: $new_probability";
      if ($new_probability > $probability)
      {
        ### echo " <<< SET!";
        $probability = $new_probability;
        $state = $y;
      }
    }
    print_message(3, "[-] [MCSHMM detect] Probability of value " . implode($observation) . ": $probability\n");
    return $probability;
  }

// ------------------------------------------------------------------ //

  # function: calculate (scaled) forward variables for a given sequence
  public function forward_scaled(array $sequence, $len_sequence, &$scale)
  {
    // 1st: initialization
    for ($i = 0; $i < $this->states; ++$i)
      $scale[0] += $fwd[0][$i] = $this->start[$i] * $this->emission[$i][$sequence[0]];

    // scaling
    if ($scale[0] != 0)
      for ($i = 0; $i < $this->states; $i++)
        $fwd[0][$i] = $fwd[0][$i] / $scale[0];
    else
     // avoid divistion by zero
     $scale[0] = 1;

    // 2nd: induction
    for ($t = 1; $t < $len_sequence; $t++)
    {
      for ($i = 0; $i < $this->states; $i++)
      {
        $sum = 0.0;
        for ($j = 0; $j < $this->states; $j++)
          $sum += $fwd[$t-1][$j] * $this->transition[$j][$i];
        $fwd[$t][$i] = $sum * $this->emission[$i][$sequence[$t]];
        $scale[$t] += $fwd[$t][$i]; // scaling coefficient
      }
      // scaling
      if ($scale[$t] != 0)
        for ($i = 0; $i < $this->states; $i++)
          $fwd[$t][$i] = $fwd[$t][$i] / $scale[$t];
      else
      {
        $scale[$t] = 1;
        print_message(3, "[-] [MCSHMM detect] [convergence algorithm] Found zero scale - set to one.\n");
      }
    }
    return $fwd;
  }
  
// ------------------------------------------------------------------ //

  # function: calculate (scaled) backward variables for a given sequence
  public function backward_scaled(array $sequence, $len_sequence, $scale)
  {
    // 1st: initialization
    for ($i = 0; $i < $this->states; $i++)
      $bwd[$len_sequence-1][$i] = 1.0 / $scale[$len_sequence-1];

    // 2nd: induction
    for ($t = $len_sequence - 2; $t >= 0; $t--)
      for ($i = 0; $i < $this->states; $i++)
      {
        $sum = 0.0;
        for ($j = 0; $j < $this->states; $j++)
          $sum += $this->transition[$i][$j] * $this->emission[$j][$sequence[$t+1]] * $bwd[$t+1][$j];
        // no more undefined offsets
        if (!isset($bwd[$t][$i]))
          $bwd[$t][$i] = 0.0;
        $bwd[$t][$i] += $sum / $scale[$t];
      }
    return $bwd;
  }

// ------------------------------------------------------------------ //

  # function: modified version of Baum-Welch algorithm to learn probabilities
  # on multiple observations sequences as described by Rabiner and Juang ('89)
  public function train_baum_welch(array $observations, array $universe, $max_iter, $tolerance)
  {
    // print some information
    print_message(2, "[*] [MCSHMM detect] Starting Baum-Welch training with " . count($observations) . " observations\n");

    // set number of observations and symbols in universe
    $num_observations = count($observations);
    $num_universe = count($universe);

    // randomize observations (and remove their key)
    shuffle($observations);

    // calculate initial model log-likelihood
    $likelihood = 0; $old_likelihood = -1;

    // first, map labels and count sequences for all observations
    for ($i = 0; $i < $num_observations; $i++)
    {
      $sequences[$i] = $this->map_labels($observations[$i]);
      $len_sequences[$i] = count($sequences[$i]);
    }

    // loop until convergence or max iterations is reached
    for ($iter=0; $iter < $max_iter; $iter++)
    {
      // for each sequence in the observations input
      for ($cur_observation = 0; $cur_observation < $num_observations; $cur_observation++)
      {
        // current sequence
        $sequence = $sequences[$cur_observation];
        $len_sequence = $len_sequences[$cur_observation];

        // scale factors
        $scale = array_fill(0, $len_sequence, 0.0);

        // calculate forward and backward variables
        $fwd = $this->forward_scaled($sequence, $len_sequence, $scale);
        $bwd = $this->backward_scaled($sequence, $len_sequence, $scale);

        ### echo "-------------------------------------------------------------------------------------------------------------------------\n";
        // calculate gamma values for next computations
        for ($t = 0; $t < $len_sequence; $t++)
        {
          $s = 0;
          for ($cur_state = 0; $cur_state < $this->states; $cur_state++)
            $s += $gamma[$cur_observation][$t][$cur_state] = $fwd[$t][$cur_state] * $bwd[$t][$cur_state];
          if ($s != 0) // scaling
            for ($cur_state = 0; $cur_state < $this->states; $cur_state++)
            {
              ### $old = $gamma[$cur_observation][$t][$cur_state];
              $gamma[$cur_observation][$t][$cur_state] /= $s;
              ### echo "[calculation gamma]\tcur_observation = $cur_observation\tt = $t\t$cur_state = $cur_state\t\t| value = " . $gamma[$cur_observation][$t][$cur_state] . "\t| old = $old\n";
            }
        }

        ### echo "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n";
        // calculate ksi values for next computations
        for ($t = 0; $t < $len_sequence - 1; $t++)
        {
          $s = 0;
          for ($cur_state = 0; $cur_state < $this->states; $cur_state++)
            for ($l = 0; $l < $this->states; $l++)
              $s += $ksi[$cur_observation][$t][$cur_state][$l] = $fwd[$t][$cur_state] * $this->transition[$cur_state][$l] * $bwd[$t + 1][$l] * $this->emission[$l][$sequence[$t + 1]];
          if ($s != 0) // scaling
          {
            for ($cur_state = 0; $cur_state < $this->states; $cur_state++)
              for ($l = 0; $l < $this->states; $l++)
              {
                ### $old = $ksi[$cur_observation][$t][$cur_state][$l];
                $ksi[$cur_observation][$t][$cur_state][$l] /= $s;
                ### echo "[calculation ksi]\tcur_observation = $cur_observation\tt = $t\tcur_state = $cur_state\tl = $l\t| value = " . $ksi[$cur_observation][$t][$cur_state][$l] . "\t| old = $old\n";
              }
          }
        }

        // compute log-likelihood for the given sequence
        for ($t = 0; $t < $len_sequence; $t++)
          $likelihood += log($scale[$t]);
      }

      // average likelihood for all sequences
      $likelihood /= $num_observations;

      // check if the model has converged or we should stop
      if ($this->check_convergence($old_likelihood, $likelihood, $iter, $max_iter, $tolerance))
      {
        print_message(2, "[*] [MCSHMM detect] Overall likelihood: $likelihood\n");
        return $likelihood;
      }

      // continue with parameter re-estimation
      $old_likelihood = $likelihood; $likelihood = 0.0;

      for ($cur_state = 0; $cur_state < $this->states; $cur_state++)
      {
        ### echo "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n";

        // re-estimation of initial state probabilities
        $sum_start = 0.0;
        for ($cur_observation = 0; $cur_observation < $num_observations; $cur_observation++)
          $sum_start += $gamma[$cur_observation][0][$cur_state];
        ### $old = $this->start[$cur_state];
        $this->start[$cur_state] = $sum_start / $num_observations;
        ### echo "[re-estimation pi]\tcur_state = $cur_state\t\t\t\t| value = $sum_start \t| old = $old\n";
        ### echo "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n";

        // re-estimation of transition probabilities 
        for ($j = 0; $j < $this->states; $j++)
        {
          $den_transition = 0.0;
          $num_transition = 0.0;

          for ($cur_observation = 0; $cur_observation < $num_observations; $cur_observation++)
          {
            $len_sequence = $len_sequences[$cur_observation];
            for ($l = 0; $l < $len_sequence - 1; $l++)
            {
              $num_transition += $ksi[$cur_observation][$l][$cur_state][$j];
              $den_transition += $gamma[$cur_observation][$l][$cur_state];
            }
          }

          ### $old = $this->transition[$cur_state][$j];
          $this->transition[$cur_state][$j] = ($den_transition != 0) ? $num_transition / $den_transition : 0.0;
          ### echo "[re-estimation A]\tcur_state = $cur_state\tj = $j\t\t\t| value = " . $num_transition / $den_transition . " \t| old = $old\n";
        }
        ### echo "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -\n";

        // re-estimation of emission probabilities
        for ($j = 0; $j < $num_universe; $j++)
        {
          $den_emission = 0.0;
          $num_emission = 0.0;

          for ($cur_observation = 0; $cur_observation < $num_observations; $cur_observation++)
          {
            $len_sequence = $len_sequences[$cur_observation];
            $sequence = $sequences[$cur_observation];
            for ($l = 0; $l < $len_sequence; $l++)
            {
              if ($sequence[$l] == $j)
                $num_emission += $gamma[$cur_observation][$l][$cur_state];
              $den_emission += $gamma[$cur_observation][$l][$cur_state];
            }
          }

          ### $old = $this->emission[$cur_state][$j];
          // avoid locking a parameter in zero
          $this->emission[$cur_state][$j] = ($num_emission == 0) ? 1e-10 : $num_emission / $den_emission;
          ### echo "[re-estimation B]\tcur_state = $cur_state\tj = $j\t\t\t| value = " . ($num_emission / $den_emission) . " \t| old = $old\n";
        }
      }
    }
  }

// ------------------------------------------------------------------ //

  # function: check if a model has converged given the likelihoods between
  # two iterations of the Baum-Welch algorithm and criteria for convergence
  function check_convergence($old_likelihood, $likelihood, $iter, $max_iter, $tolerance)
  {
    ### print_message(3, "-------------------------------------------------------------------------------------------------------------------------\n");
    print_message(3, "[-] [MCSHMM detect] [convergence check]\titeration = $iter\ttolerance = $tolerance\t| likelihood = $likelihood\t| old = $old_likelihood\n");
    // Update and verify stop criteria
    if ($tolerance > 0)
    {
      // Stopping criteria is likelihood convergence
      if (abs($old_likelihood - $likelihood) <= $tolerance)
      {
        print_message(2, "[*] [MCSHMM detect] HMM converged after $iter iterations\n");
        return true;
      }
      if ($max_iter > 0)
        // Maximum iterations should also be respected
        if ($iter >= $max_iter)
        {
          print_message(2, "[*] [MCSHMM detect] HMM did not converge, but maximum iterations reached\n");
          return true;
        }
    }
    else
      // stopping criteria is number of iterations
      if ($iter == $max_iter)
      {
        print_message(2, "[*] [MCSHMM detect] HMM did not converge, but maximum iterations reached\n");
        return true;
      }
    // check if we have reached an invalid state1
    if (is_nan($likelihood) || is_infinite($likelihood))
    {
      print_message(2, "[*] [MCSHMM detect] HMM is lost in an invalid state\n");
      return true;
    }
    return false;
  }
}

// ============================== LOF =============================== //

class LOF
{
  // number of elements
  protected $N;
  // number of nearest neighbors
  protected $k;
  // data to be inspected
  protected $data = array();
  // cache for log, lrd and distances
  protected $cache = array();
  // database of nearest neighbors
  protected $nearest_neighbor = array();
  // defines if we deal with values or arrays
  protected $is_values = true;

// ------------------------------------------------------------------ //

  # function: construct instance of local outlier factor from data
  public function __construct(array $lof_data, $lof_neighbors, &$lof_cache)
  {
    // initialize properties of LOF
    $this->data  = $lof_data;
    $this->cache = $lof_cache;
    $this->k     = $lof_neighbors;
    $this->N     = count($lof_data);

    // switch between value and array distance calculaten
    if (is_array(end($this->data)))
      $this->is_values = false;

    print_message(3, "[-] [LOF] LOF created with $this->k neighbors ($this->N items)\n");
  }

// ------------------------------------------------------------------ //

  # function: return local outlier factor for item
  public function run($item, $value)
  {
    // add current item to dataset
    $this->data[$item] = $value;

    // find k nearest neighbors of item
    $this->set_nearest_neighbors($item);

    // calculate local outlier factor (lof)
    $lof = $this->local_outlier_factor($item);

    return $lof;
  }

// ------------------------------------------------------------------ //

  # function: find k nearest neighbors of A
  protected function set_nearest_neighbors($A)
  {
    foreach ($this->data as $A => $valA)
    {
      // fill up the item's nearest neighbor database with infinity
      $this->nearest_neighbor[$A] = array_fill(0, $this->k, INF);

      // find k nearest neighbors (if 'tie', add one more neighbor)
      foreach ($this->data as $B => $valB)
      {
        // calculate distance between A and B
        $distance = $this->get_distance($valA, $valB);

        // if distance lower than highest seen distance
        if ($distance < end($this->nearest_neighbor[$A]))
        {
          // remove last distance element of array, if no 'tie'
          if (!in_array($distance, $this->nearest_neighbor[$A]))
            array_pop($this->nearest_neighbor[$A]);

          // push new A-B distance onto the end of array
          $this->nearest_neighbor[$A][$B] = $distance;

          // re-sort array by distances
          asort($this->nearest_neighbor[$A]);
        }
      }
      // if infinite values are left at the end, remove them
      while (is_infinite(end($this->nearest_neighbor[$A])))
        array_pop($this->nearest_neighbor[$A]);
    }
  }

// ------------------------------------------------------------------ //

  # function: calculate reachability distance (rd)
  protected function reachability_distance($A, $B)
  {
    // calculate Euclidean distance between A and B
    $distance = $this->get_distance($this->data[$A], $this->data[$B]);
    // get distance of B to it's k nearest neighbor
    $k_distance = end($this->nearest_neighbor[$B]);
    // calculate reachability distance between A and B
    $rd = max($distance, $k_distance);

    return $rd;
  }

// ------------------------------------------------------------------ //

  # function: calculate local reachability density (lrd)
  protected function local_reachability_distance($A)
  {
    $sum = 1; // avoid division by zero if zero-distance
    // calculate sum of reachability distances for A-B
    foreach ($this->nearest_neighbor[$A] as $B => $distance)
      $sum += $this->reachability_distance($A, $B);

    // calculate local reachability density
    $lrd = 1 / ($sum / count(array_unique($this->nearest_neighbor[$A])));
    print_message(3, "[-] [LOF] Local reachability distance of '$A': $lrd\n");

    return $lrd;
  }

// ------------------------------------------------------------------ //

  # function: calculate local outlier factor (lof)
  protected function local_outlier_factor($A)
  {
    $sum = 0;
    // calculate sum of local reachability densities for A
    foreach ($this->nearest_neighbor[$A] as $B => $distance)
      $sum += $this->local_reachability_distance($B);

    // calculate local outlier factor
    $lof = ($sum / count(array_unique($this->nearest_neighbor[$A]))) / $this->local_reachability_distance($A);
    print_message(3, "[-] [LOF] Local outlier factor of '$A': $lof\n");

    return $lof;
  }

// ------------------------------------------------------------------ //

  # function: get distance of values or geolocations
  protected function get_distance(&$valA, &$valB)
  {
    if ($this->is_values) // assume values (e.g. bytes-sent)
      return $this->get_distance_val($valA, $valB);
    else // assume arrays (in our case: geo-coordinates)
      return $this->get_distance_geo($valA, $valB);
  }

// ------------------------------------------------------------------ //

  # function: calculate distance of two values (e.g. bytes-sent)
  protected function get_distance_val($valA, $valB)
  {
    // try to get distance from cache
    if (isset($this->cache['distance'][$valA][$valB]))
      return $this->cache['distance'][$valA][$valB];

    // calculate absolute value of difference
    $distance = abs($valA - $valB);

    // add distance to cache
    $this->cache['distance'][$valA][$valB] = $distance;

    return $distance;
  }

// ------------------------------------------------------------------ //

  # function: calculate distance of two geolocations
  protected function get_distance_geo($valA, $valB)
  {
    // try to get distance from cache
    $a = implode(',', $valA); $b = implode(',', $valB);
    if (isset($this->cache['distance'][$a][$b]))
      return $this->cache['distance'][$a][$b];

    // get longitude/latitude for both items
    $lat1 = $valA[0]; $lon1 = $valA[1];
    $lat2 = $valB[0]; $lon2 = $valB[1];

    $earth_radius = 16371; // mean radius of the earth in km
    $deg_per_rad = 57.29578; // number of degrees/radian
    $distance = $earth_radius * pi() * sqrt(($lat1 - $lat2)
                                          * ($lat1 - $lat2)
                                          + cos($lat1 / $deg_per_rad)
                                          * cos($lat2 / $deg_per_rad)
                                          * ($lon1 - $lon2)
                                          * ($lon1 - $lon2)) / 180;
    // add distance to cache
    $this->cache['distance'][$a][$b] = $distance;

    return $distance;
  }
}

// ============================= Style ============================== //

class Style
{
  # compressed CSS syle for HTML table and document
  protected $stylesheet = 'body{font-family:verdana,helvetica,arial,sans-serif;color:#444}input{border:0;background:0;font-size:120%;width:100px}table{font-family:verdana,helvetica,arial,sans-serif;font-size:9px;background-color:#fefefe;border:1px solid #d4d4d4;color:#555;width:100%;max-width:100%}td{font-size:9px;padding-left:2px;padding-right:2px;width:1%;max-width:230px;white-space:nowrap;word-wrap:break-word;overflow:hidden;text-overflow:ellipsis}table thead{background-color:#eee;font-weight:bold;cursor:default}.fltrow{height:20px;background-color:#f4f4f4}.btnflt{font-size:10px;margin:0 2px 0 2px;padding:0 1px 0 1px;width:35%;text-decoration:none;color:#fff;background-color:#666}.flt{background-color:#f4f4f4;border:1px inset #ccc;margin:0;width:100%}.flt_s{background-color:#f4f4f4;border:1px inset #ccc;margin:0;width:60%}.inf{clear:both;width:auto;height:20px;background:#f4f4f4;font-size:11px;margin:0;padding:1px 3px 1px 3px;border:1px solid #ccc}.ldiv{float:left;width:30%;position:inherit}.mdiv{float:left;width:30%;position:inherit;text-align:center}.rdiv{float:right;width:30%;position:inherit;text-align:right}.loader{position:absolute;padding:15px 0 15px 0;margin-top:7%;width:200px;left:40%;z-index:1000;font-size:14px;font-weight:bold;border:1px solid #666;background:#f4f4f4;text-align:center;vertical-align:middle}div.mdiv select{height:20px}div.inf a{color:#c00}div.inf a:hover{text-decoration:none}.tot{font-weight:bold}.even{background-color:#fff}.odd{background-color:#f4f4f4}';

  # closure compiled code to toggle visibility of a table's cols
  protected $coltable = 'function toggle_visibility_details(a){document.all?document.getElementById(a).style.display="block"==document.getElementById(a).style.display?"none":"block":document.getElementById(a).style.display="table"==document.getElementById(a).style.display?"none":"table";toggle_visibility_benign(a);document.getElementById("lnk_"+a).value="[-] show less"==document.getElementById("lnk_"+a).value?"[+] show more":"[-] show less"} function toggle_visibility_benign(a){if("none"!=document.getElementById(a).style.display){var b=document.getElementById("chk_"+a).checked;a=document.getElementsByName("benign_"+a);if(b)for(b=0;b<a.length;b++)a[b].style.display="none";else for(b=0;b<a.length;b++)a[b].style.display=""}};';

  # closure compiled version of J. Struebig's sort_table.js
  protected $sortable = 'SortTable.ok=!0;document.getElementsByTagName||(SortTable.ok=!1);SortTable.up=" "+String.fromCharCode(9660);SortTable.alt_up=" sort upwards";SortTable.down=" "+String.fromCharCode(9650);SortTable.alt_down=" sort downwards";SortTable.pointer_color="#222";SortTable.className="sortable"; SortTable.init=function(){for(var d=document.getElementsByTagName("table"),g=[],k=RegExp("\\\\b"+SortTable.className+"\\\\b","i"),f=0;f<d.length;f++)SortTable.ok&&(d[f].className&&k.test(d[f].className))&&g.push(new SortTable(d[f]));return g}; function SortTable(d){function g(a,d,c){if(a.getAttribute("my_key"))a=a.getAttribute("my_key");else if(0<a.childNodes.length){var b=a.getElementsByTagName("input")[0];a=b&&"text"==b.type?b.value:a.getElementsByTagName("select")[0]?a.getElementsByTagName("select")[0].value:a.innerHTML.stripTags()}else a=a.firstChild?a.firstChild.data:"";a=a.trim();if(c)return d?a.toLowerCase():a;c=a.match(f);return a==parseFloat(a)?parseFloat(a):c?(new Date(c[3]+"/"+c[2]+"/"+c[1])).getTime():!isNaN(Date.parse(a))? Date.parse(a):d?a.toLowerCase():a}var k=this,f=/(\\d{1,2})\\.(\\d{1,2})\\.(\\d{2,4})/,q=/\\bzebra\\b/i.test(d.className),h=d.tBodies[0],j=d.tHead,l=1;k.onstart=k.onsort=function(){};this.length=function(){return h.rows.length};this.sort=function(a,c){l=c?1:-1;j.rows[0].cells[a].onclick()};j||(j=d.createTHead(),j.appendChild(h.rows[0]),h=d.tBodies[1]||d.tBodies[0]);var b=j.rows[0].cells;j.rows[1]&&(b=j.rows[1].cells);for(var p,m=0,c=0;c<b.length;c++)if(!b[c].className||!/\\bno_sort\\b/i.test(b[c].className))b[c].onclick= function(){var a=document.createElement("span");a.style.fontFamily="Arial";a.style.fontSize="80%";a.style.visibility="hidden";a.innerHTML=SortTable.down;b[c].appendChild(a);var d=c+m,f="ignore_case"==(b[c].getAttribute("ignore_case")||b[c].title),j=!(!b[c].className||!/\\bsort_string\\b/i.test(b[c].className));return function(){k.onstart(new Date,this);for(var c=l,b=[],n=h.rows,m=h.rows.length,e=0;e<m;e++)b.push({elem:n[e],value:g(n[e].cells[d],f,j)});b.sort(function(a,b){return a.value.localeCompare? c*a.value.localeCompare(b.value):a.value==b.value?0:a.value>b.value?c:-c});n=h.cloneNode(!1);for(e=0;e<m;e++)q&&(b[e].elem.className=b[e].elem.className.replace(/( ?odd)/,""),e%2&&(b[e].elem.className+=" odd")),n.appendChild(b[e].elem);h.parentNode.replaceChild(n,h);h=n;l=-l;p!=a&&(p&&(p.style.visibility="hidden"),a.style.visibility="",p=a);a.style.color=SortTable.pointer_color;a.innerHTML=0>l?SortTable.down:SortTable.up;this.title=0>l?SortTable.alt_down:SortTable.alt_up;k.onsort(new Date);return!1}}(), b[c].style.cursor="pointer",b[c].getAttribute("colspan")&&(m+=b[c].getAttribute("colspan")-1)}String.prototype.stripTags=function(){return this.replace(/(<.*[\'"])([^\'"]*)([\'"]>)/g,function(d,g,k,f){return g+f}).replace(/<\\/?[^>]+>/gi,"")};String.prototype.trim=function(d){if(!this.length)return"";var g=this.strip_nl().ltrim().rtrim();return d?g.replace(/ +/g," "):g};String.prototype.rtrim=function(){return!this.length?"":this.replace(/\\s+$/g,"")}; String.prototype.ltrim=function(){return!this.length?"":this.replace(/^\\s+/g,"")};String.prototype.strip_nl=function(){return!this.length?"":this.replace(/[\\n\\r]/g,"")};window.onload=function(){SortTable.init()};function filterTable(d,a){dehighlight(a);for(var b=d.value.toLowerCase().split(" "),c=1;c<a.rows.length;c++)for(var f="",e=0;e<b.length;e++)0>a.rows[c].innerHTML.replace(/<[^>]+>/g,"").toLowerCase().indexOf(b[e])?f="none":b[e].length&&highlight(b[e],a.rows[c]),a.rows[c].style.display=f} function dehighlight(d){for(var a=0;a<d.childNodes.length;a++){var b=d.childNodes[a];if(b.attributes&&b.attributes["class"]&&"highlighted"==b.attributes["class"].value){b.parentNode.parentNode.replaceChild(document.createTextNode(b.parentNode.innerHTML.replace(/<[^>]+>/g,"")),b.parentNode);break}else 3!=b.nodeType&&dehighlight(b)}} function highlight(d,a){for(var b=0;b<a.childNodes.length;b++){var c=a.childNodes[b];if(3==c.nodeType){var f=c.data,e=f.toLowerCase();if(0<=e.indexOf(d)){var g=document.createElement("span");for(c.parentNode.replaceChild(g,c);-1!=(c=e.indexOf(d));)g.appendChild(document.createTextNode(f.substr(0,c))),g.appendChild(create_node(document.createTextNode(f.substr(c,d.length)))),f=f.substr(c+d.length),e=e.substr(c+d.length);g.appendChild(document.createTextNode(f))}}else highlight(d,c)}} function create_node(d){var a=document.createElement("span");a.setAttribute("class","highlighted");a.attributes["class"].value="highlighted";a.appendChild(d);return a}tables=document.getElementsByTagName("table"); for(var t=0;t<tables.length;t++)if(element=tables[t],element.attributes["class"]&&"sortable"==element.attributes["class"].value){var form=document.createElement("form");form.setAttribute("class","filter");form.attributes["class"].value="filter";var input=document.createElement("input");input.onkeyup=function(){filterTable(input,element)};form.appendChild(input);element.parentNode.insertBefore(form,element)};';

  # closure compiled version of Max Guglielmi's tablefilter.js
  protected $filterable = 'var TblId,SearchFlt,SlcArgs;TblId=[];SlcArgs=[];function setFilterGrid(b){var d=grabEBI(b),c,f;if(null!=d&&"table"==d.nodeName.toLowerCase()){if(1<arguments.length)for(var g=0;g<arguments.length;g++)switch((typeof arguments[g]).toLowerCase()){case "number":c=arguments[g];break;case "object":f=arguments[g]}void 0==c?c=2:c+=2;g=getCellsNb(b,c);d.tf_ncells=g;void 0==d.tf_ref_row&&(d.tf_ref_row=c);d.tf_Obj=f;hasGrid(b)||AddGrid(b)}} function AddGrid(b){TblId.push(b);var d=grabEBI(b),c=d.tf_Obj,f=d.tf_ncells,g,k,l,j,q,r,v,D,p,s,m,E,x,z,A,F,B,n,H,I,J,K,L;void 0!=c&&!1==c.grid?k=!1:k=!0;void 0!=c&&!0==c.btn?l=!0:l=!1;void 0!=c&&void 0!=c.btn_text?j=c.btn_text:j="find";void 0!=c&&!1==c.enter_key?q=!1:q=!0;void 0!=c&&c.mod_filter_fn?r=!0:r=!1;void 0!=c&&void 0!=c.display_all_text?v=c.display_all_text:v="";void 0!=c&&!1==c.on_change?D=!1:D=!0;void 0!=c&&!0==c.rows_counter?p=!0:p=!1;void 0!=c&&void 0!=c.rows_counter_text?s=c.rows_counter_text: s="Incidents found: ";void 0!=c&&!0==c.btn_reset?m=!0:m=!1;void 0!=c&&void 0!=c.btn_reset_text?E=c.btn_reset_text:E="Reset";void 0!=c&&!0==c.sort_select?x=!0:x=!1;void 0!=c&&!0==c.paging?z=!0:z=!1;void 0!=c&&void 0!=c.paging_length?A=c.paging_length:A=10;void 0!=c&&!0==c.loader?F=!0:F=!1;void 0!=c&&void 0!=c.loader_text?B=c.loader_text:B="Loading...";void 0!=c&&!0==c.exact_match?n=!0:n=!1;void 0!=c&&!0==c.alternate_rows?H=!0:H=!1;void 0!=c&&c.col_operation?I=!0:I=!1;void 0!=c&&c.rows_always_visible? J=!0:J=!1;void 0!=c&&c.col_width?K=!0:K=!1;void 0!=c&&c.bind_script?L=!0:L=!1;d.tf_fltGrid=k;d.tf_displayBtn=l;d.tf_btnText=j;d.tf_enterKey=q;d.tf_isModfilter_fn=r;d.tf_display_allText=v;d.tf_on_slcChange=D;d.tf_rowsCounter=p;d.tf_rowsCounter_text=s;d.tf_btnReset=m;d.tf_btnReset_text=E;d.tf_sortSlc=x;d.tf_displayPaging=z;d.tf_pagingLength=A;d.tf_displayLoader=F;d.tf_loadText=B;d.tf_exactMatch=n;d.tf_alternateBgs=H;d.tf_startPagingRow=0;r&&(d.tf_modfilter_fn=c.mod_filter_fn);if(k){var O=d.insertRow(0); O.className="fltrow";for(var u=0;u<f;u++){var M=O.insertCell(u);u==f-1&&!0==l?g="flt_s":g="flt";for(var C=0;C<useless_cols.length;C++)if(useless_cols[C]==u)for(n=0;n<d.rows.length;n++)d.rows[n].cells[u].style.display="none";if(void 0==c||void 0==c["col_"+u]||"none"==c["col_"+u]){var N;void 0==c||void 0==c["col_"+u]?N="text":N="hidden";n=createElm("input",["id","flt"+u+"_"+b],["type",N],["class",g]);n.className=g;M.appendChild(n);q&&(n.onkeypress=DetectKey)}else"select"==c["col_"+u]&&(n=createElm("select", ["id","flt"+u+"_"+b],["class",g]),n.className=g,M.appendChild(n),PopulateOptions(b,u),z&&(C=[],C.push(b),C.push(u),C.push(f),C.push(v),C.push(x),C.push(z),SlcArgs.push(C)),q&&(n.onkeypress=DetectKey),D&&(!r?n.onchange=function(){Filter(b)}:n.onchange=c.mod_filter_fn));u==f-1&&!0==l&&(n=createElm("input",["id","btn"+u+"_"+b],["type","button"],["value",j],["class","btnflt"]),n.className="btnflt",M.appendChild(n),!r?n.onclick=function(){Filter(b)}:n.onclick=c.mod_filter_fn)}}if(p||m||z||F){f=createElm("div", ["id","inf_"+b],["class","inf"]);f.className="inf";d.parentNode.insertBefore(f,d);if(p){var G;n=createElm("div",["id","ldiv_"+b]);p?n.className="ldiv":n.style.display="none";z?G=A:G=getRowsNb(b);p=createElm("span",["id","totrows_span_"+b],["class","tot"]);p.className="tot";p.appendChild(createText(G));s=createText(s);n.appendChild(s);n.appendChild(p);f.appendChild(n)}F&&(s=createElm("div",["id","load_"+b],["class","loader"]),s.className="loader",s.style.display="none",s.appendChild(createText(B)), f.appendChild(s));if(z){B=createElm("div",["id","mdiv_"+b]);z?B.className="mdiv":B.style.display="none";f.appendChild(B);s=d.tf_ref_row;n=grabTag(d,"tr");B=n.length;A=Math.ceil((B-s)/A);G=createElm("select",["id","slcPages_"+b]);G.onchange=function(){F&&showLoader(b,"");d.tf_startPagingRow=this.value;GroupByPage(b);F&&showLoader(b,"none")};p=createElm("span",["id","pgspan_"+b]);grabEBI("mdiv_"+b).appendChild(createText(" Page "));grabEBI("mdiv_"+b).appendChild(G);grabEBI("mdiv_"+b).appendChild(createText(" of ")); p.appendChild(createText(A+" "));grabEBI("mdiv_"+b).appendChild(p);for(A=s;A<B;A++)n[A].setAttribute("validRow","true");setPagingInfo(b);F&&showLoader(b,"none")}m&&k&&(k=createElm("div",["id","reset_"+b]),m?k.className="rdiv":k.style.display="none",m=createElm("a",["href","javascript:clearFilters(\'"+b+"\');Filter(\'"+b+"\');"]),m.appendChild(createText(E)),k.appendChild(m),f.appendChild(k))}K&&(d.tf_colWidth=c.col_width,setColWidths(b));H&&!z&&setAlternateRows(b);I&&(d.tf_colOperation=c.col_operation, setColOperation(b));J&&(d.tf_rowVisibility=c.rows_always_visible,z&&setVisibleRows(b));L&&(d.tf_bindScript=c.bind_script,void 0!=d.tf_bindScript&&void 0!=d.tf_bindScript.target_fn&&d.tf_bindScript.target_fn.call(null,b))} function PopulateOptions(b,d){var c=grabEBI(b),f=c.tf_ncells,g=c.tf_display_allText,k=c.tf_sortSlc,l=c.tf_ref_row,c=grabTag(c,"tr"),j=[],q=0,g=new Option(g,"",!1,!1);for(grabEBI("flt"+d+"_"+b).options[q]=g;l<c.length;l++){var g=getChildElms(c[l]).childNodes,r=g.length;c[l].getAttribute("paging");if(r==f)for(var v=0;v<r;v++)if(d==v){var D=getCellText(g[v]),p=!1;for(w in j)D==j[w]&&(p=!0);p||j.push(D)}}k&&j.sort();for(y in j)q++,g=new Option(j[y],j[y],!1,!1),grabEBI("flt"+d+"_"+b).options[q]=g} function Filter(b){showLoader(b,"");SearchFlt=getFilters(b);var d=grabEBI(b);void 0!=d.tf_Obj?fprops=d.tf_Obj:fprops=[];var c=[],f=getCellsNb(b);getRowsNb(b);for(var g=0,k=d.tf_exactMatch,l=d.tf_displayPaging,j=0;j<SearchFlt.length;j++)c.push(grabEBI(SearchFlt[j]).value.toLowerCase());for(var q=d.tf_ref_row,j=grabTag(d,"tr");q<j.length;q++){"none"==j[q].style.display&&(j[q].style.display="");var r=getChildElms(j[q]).childNodes,v=r.length;if(v==f){for(var D=[],p=[],s=!0,m=0;m<v;m++){var E=getCellText(r[m]).toLowerCase(); D.push(E);if(""!=c[m]){var x=parseFloat(E);/<=/.test(c[m])&&!isNaN(x)?x<=parseFloat(c[m].replace(/<=/,""))?p[m]=!0:p[m]=!1:/>=/.test(c[m])&&!isNaN(x)?x>=parseFloat(c[m].replace(/>=/,""))?p[m]=!0:p[m]=!1:/</.test(c[m])&&!isNaN(x)?x<parseFloat(c[m].replace(/</,""))?p[m]=!0:p[m]=!1:/>/.test(c[m])&&!isNaN(x)?x>parseFloat(c[m].replace(/>/,""))?p[m]=!0:p[m]=!1:(x=k||"select"==fprops["col_"+m]?RegExp("(^)"+regexpEscape(c[m])+"($)","gi"):RegExp(regexpEscape(c[m]),"gi"),p[m]=x.test(E))}}for(r=0;r<f;r++)""!= c[r]&&!p[r]&&(s=!1)}s?(j[q].style.display="",l&&j[q].setAttribute("validRow","true")):(j[q].style.display="none",g++,l&&j[q].setAttribute("validRow","false"))}d.tf_nRows=parseInt(getRowsNb(b))-g;l||applyFilterProps(b);l&&(d.tf_startPagingRow=0,setPagingInfo(b))} function setPagingInfo(b){for(var d=grabEBI(b),c=parseInt(d.tf_ref_row),f=d.tf_pagingLength,g=grabTag(d,"tr"),k=grabEBI("mdiv_"+b),d=grabEBI("slcPages_"+b),l=grabEBI("pgspan_"+b),j=0;c<g.length;c++)"true"==g[c].getAttribute("validRow")&&j++;c=Math.ceil(j/f);l.innerHTML=c;d.innerHTML="";if(0<c){k.style.visibility="visible";for(k=0;k<c;k++)l=new Option(k+1,k*f,!1,!1),d.options[k]=l}else k.style.visibility="hidden";GroupByPage(b)} function GroupByPage(b){showLoader(b,"");for(var d=grabEBI(b),c=parseInt(d.tf_ref_row),f=parseInt(d.tf_pagingLength),g=parseInt(d.tf_startPagingRow),f=g+f,k=grabTag(d,"tr"),l=0,j=[];c<k.length;c++)"true"==k[c].getAttribute("validRow")&&j.push(c);for(h=0;h<j.length;h++)h>=g&&h<f?(l++,k[j[h]].style.display=""):k[j[h]].style.display="none";d.tf_nRows=parseInt(l);applyFilterProps(b)} function applyFilterProps(b){t=grabEBI(b);var d=t.tf_nRows,c=t.tf_rowVisibility,f=t.tf_alternateBgs,g=t.tf_colOperation;t.tf_rowsCounter&&showRowsCounter(b,parseInt(d));c&&setVisibleRows(b);f&&setAlternateRows(b);g&&setColOperation(b);showLoader(b,"none")}function hasGrid(b){var d=!1,c=grabEBI(b);if(null!=c&&"table"==c.nodeName.toLowerCase())for(i in TblId)b==TblId[i]&&(d=!0);return d} function getCellsNb(b,d){var c=grabEBI(b),c=void 0==d?grabTag(c,"tr")[0]:grabTag(c,"tr")[d];return getChildElms(c).childNodes.length}function getRowsNb(b){var d=grabEBI(b);b=d.tf_ref_row;d=grabTag(d,"tr").length;return parseInt(d-b)}function getFilters(b){var d=[],c=grabEBI(b);b=grabTag(c,"tr")[0].childNodes;if(c.tf_fltGrid)for(c=0;c<b.length;c++)d.push(b[c].firstChild.getAttribute("id"));return d}function clearFilters(b){SearchFlt=getFilters(b);for(i in SearchFlt)grabEBI(SearchFlt[i]).value=""} function showLoader(b,d){var c=grabEBI("load_"+b);null!=c&&"none"==d?setTimeout("grabEBI(\'load_"+b+"\').style.display = \'"+d+"\'",150):null!=c&&"none"!=d&&(c.style.display=d)}function showRowsCounter(b,d){var c=grabEBI("totrows_span_"+b);null!=c&&"span"==c.nodeName.toLowerCase()&&(c.innerHTML=d)}function getChildElms(b){if(1==b.nodeType){for(var d=b.childNodes,c=0;c<d.length;c++){var f=d[c];3==f.nodeType&&b.removeChild(f)}return b}} function getCellText(b){var d="";b=b.childNodes;for(var c=0;c<b.length;c++)var f=b[c],d=3==f.nodeType?d+f.data:d+getCellText(f);return d}function getColValues(b,d,c){var f=grabEBI(b),g=grabTag(f,"tr"),k=g.length,l=parseInt(f.tf_ref_row);b=getCellsNb(b,l);for(f=[];l<k;l++){var j=getChildElms(g[l]).childNodes,q=j.length;if(q==b)for(var r=0;r<q;r++)if(r==d&&""==g[l].style.display){var v=getCellText(j[r]).toLowerCase();c?f.push(parseFloat(v)):f.push(v)}}return f} function setColWidths(b){if(hasGrid(b)){var d=grabEBI(b);d.style.tableLayout="fixed";var c=d.tf_colWidth,f=parseInt(d.tf_ref_row),d=grabTag(d,"tr")[0];b=getCellsNb(b,f);for(f=0;f<c.length;f++)for(var g=0;g<b;g++)cell=d.childNodes[g],g==f&&(cell.style.width=c[f])}} function setVisibleRows(b){if(hasGrid(b)){var d=grabEBI(b);b=grabTag(d,"tr");for(var c=b.length,f=d.tf_displayPaging,d=d.tf_rowVisibility,g=0;g<d.length;g++)d[g]<=c&&(f&&b[d[g]].setAttribute("validRow","true"),b[d[g]].style.display="")}}function setAlternateRows(b){if(hasGrid(b)){var d=grabEBI(b);b=grabTag(d,"tr");for(var c=b.length,f=[],d=parseInt(d.tf_ref_row);d<c;d++)""==b[d].style.display&&f.push(d);for(c=0;c<f.length;c++)0==c%2?b[f[c]].className="even":b[f[c]].className="odd"}} function setColOperation(b){if(hasGrid(b)){var d=grabEBI(b),c=d.tf_colOperation.id,f=d.tf_colOperation.col,g=d.tf_colOperation.operation,k=d.tf_colOperation.write_method;if("object"==(typeof c).toLowerCase()&&"object"==(typeof f).toLowerCase()&&"object"==(typeof g).toLowerCase()){grabTag(d,"tr");d=parseInt(d.tf_ref_row);getCellsNb(b,d);for(var d=[],l=0;l<f.length;l++)d.push(getColValues(b,f[l],!0));for(b=0;b<d.length;b++){for(var j=l=f=0;j<d[b].length;j++){var q=d[b][j];if(!isNaN(q))switch(g[b].toLowerCase()){case "sum":f+= parseFloat(q);break;case "mean":l++,f+=parseFloat(q)}}switch(g[b].toLowerCase()){case "mean":f/=l}if(void 0!=k&&"object"==(typeof k).toLowerCase()){if(f=f.toFixed(2),void 0!=grabEBI(c[b]))switch(k[b].toLowerCase()){case "innerhtml":grabEBI(c[b]).innerHTML=f;break;case "setvalue":grabEBI(c[b]).value=f;break;case "createtextnode":l=grabEBI(c[b]).firstChild,f=createText(f),grabEBI(c[b]).replaceChild(f,l)}}else try{grabEBI(c[b]).innerHTML=f.toFixed(2)}catch(r){}}}}} function grabEBI(b){return document.getElementById(b)}function grabTag(b,d){return b.getElementsByTagName(d)}function regexpEscape(b){chars="\\\\[^$.|?*+()".split("");for(e in chars){var d=chars[e];a=RegExp("\\\\"+d,"g");b=b.replace(a,"\\\\"+d)}return b}function createElm(b){var d=document.createElement(b);if(1<arguments.length)for(var c=0;c<arguments.length;c++)switch((typeof arguments[c]).toLowerCase()){case "object":2==arguments[c].length&&d.setAttribute(arguments[c][0],arguments[c][1])}return d} function createText(b){return document.createTextNode(b)}function DetectKey(b){if((b=b?b:window.event?window.event:null)&&"13"==(b.charCode?b.charCode:b.keyCode?b.keyCode:b.which?b.which:0)){var d;b=this.getAttribute("id");d=this.getAttribute("id").split("_")[0];b=b.substring(d.length+1,b.length);t=grabEBI(b);t.tf_isModfilter_fn?t.tf_modfilter_fn.call():Filter(b)}} function importScript(b,d){for(var c=!1,f=grabTag(document,"script"),g=0;g<f.length;g++)if(f[g].src.match(d)){c=!0;break}c||(c=grabTag(document,"head")[0],f=createElm("script",["id",b],["type","text/javascript"],["src",d]),c.appendChild(f))}function TF_GetFilterIds(){return TblId}function TF_HasGrid(b){return hasGrid(b)}function TF_GetFilters(b){try{return getFilters(b)}catch(d){alert("TF_GetFilters() fn: table id not found")}} function TF_GetStartRow(b){try{return grabEBI(b).tf_ref_row}catch(d){alert("TF_GetStartRow() fn: table id not found")}}function TF_GetColValues(b,d,c){if(hasGrid(b))return getColValues(b,d,c);alert("TF_GetColValues() fn: table id not found")}function TF_Filter(b){grabEBI(b);TF_HasGrid(b)?Filter(b):alert("TF_Filter() fn: table id not found")} function TF_RemoveFilterGrid(b){if(TF_HasGrid(b)){var d=grabEBI(b);clearFilters(b);null!=grabEBI("inf_"+b)&&d.parentNode.removeChild(d.previousSibling);for(var c=grabTag(d,"tr"),f=0;f<c.length;f++){c[f].style.display="";try{c[f].hasAttribute("validRow")&&c[f].removeAttribute("validRow")}catch(g){for(var k=0;k<c[f].attributes.length;k++)"validrow"==c[f].attributes[k].nodeName.toLowerCase()&&c[f].removeAttribute("validRow")}}if(d.tf_alternateBgs)for(f=0;f<c.length;f++)c[f].className="";d.tf_fltGrid&& d.deleteRow(0);for(i in TblId)b==TblId[i]&&TblId.splice(i,1)}else alert("TF_RemoveFilterGrid() fn: table id not found")}function TF_ClearFilters(b){TF_HasGrid(b)?clearFilters(b):alert("TF_ClearFilters() fn: table id not found")}function TF_SetFilterValue(b,d,c){if(TF_HasGrid(b))for(i in b=getFilters(b),b)i==d&&(grabEBI(b[i]).value=c);else alert("TF_SetFilterValue() fn: table id not found")};';

  public function get_stylesheet()
  {
    return $this->stylesheet;
  }

  public function get_coltable()
  {
    return $this->coltable;
  }

  public function get_sortable()
  {
    return $this->sortable;
  }

  public function get_filterable()
  {
    return $this->filterable;
  }
}
// ------------------------------ EOF ------------------------------- //

?>
