elB4RTO/LogDoctor
elB4RTO
/
LogDoctor
2
2
Fork 0
Code Issues 6 Pull Requests Projects 1 Releases 8 Wiki Activity

Improvements and updates

This commit is contained in:
Valentino Orlandi 2024-02-08 22:25:41 +01:00
parent 53fe197279
commit 236b7eeb33
Signed by: elB4RTO
GPG Key ID: 1719E976DB2D4E71
10 changed files with 620 additions and 649 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
logdoctor/modules/craplog/craplog.cpp
View File

@ -17,6 +17,7 @@
#include "modules/blacklists/blacklists.h" #include "modules/blacklists/blacklists.h"
#include "modules/craplog/modules/lib.h"
#include "modules/craplog/modules/donuts.h" #include "modules/craplog/modules/donuts.h"
#include "modules/craplog/modules/logs.h" #include "modules/craplog/modules/logs.h"
#include "modules/craplog/modules/workers/lister.h" #include "modules/craplog/modules/workers/lister.h"

2
logdoctor/modules/craplog/craplog.h
View File

@ -4,13 +4,13 @@
#include <QtCharts> #include <QtCharts>
#include "modules/craplog/modules/lib.h"
#include "modules/craplog/modules/hash.h" #include "modules/craplog/modules/hash.h"
#include "modules/craplog/modules/formats.h" #include "modules/craplog/modules/formats.h"
#include "modules/craplog/modules/workers/lib.h" #include "modules/craplog/modules/workers/lib.h"
struct LogFile;
struct Blacklists; struct Blacklists;
class QWaitCondition; class QWaitCondition;

155
logdoctor/modules/craplog/modules/datetime.cpp
View File

@ -1,6 +1,8 @@
#include "datetime.h" #include "datetime.h"
#include "lib.h"
#include "modules/exceptions.h" #include "modules/exceptions.h"
#include "utilities/strings.h" #include "utilities/strings.h"
@ -57,12 +59,12 @@ const std::string convertMonth( std::string_view month )
} // namespace (private) } // namespace (private)
std::vector<std::string> processDateTime( std::string_view datetime_, std::string_view format ) std::vector<std::string> processDateTime( std::string_view datetime_, const LogsFormatField format )
{ {
std::string aux, datetime{datetime_}; std::string datetime{datetime_};
std::string year, month, day, hour, minute, second; std::string year, month, day, hour, minute, second;
if ( format == "ncsa" ) { if ( format == date_time_ncsa ) {
datetime = StringOps::strip( datetime, "[ ]" ); datetime = StringOps::strip( datetime, "[ ]" );
day = datetime.substr( 0ul, 2ul ); day = datetime.substr( 0ul, 2ul );
month = convertMonth( datetime.substr( 3ul, 3ul ) ); month = convertMonth( datetime.substr( 3ul, 3ul ) );
@ -71,7 +73,7 @@ std::vector<std::string> processDateTime( std::string_view datetime_, std::strin
minute = datetime.substr( 15ul, 2ul ); minute = datetime.substr( 15ul, 2ul );
second = datetime.substr( 18ul, 2ul ); second = datetime.substr( 18ul, 2ul );
} else if ( format == "mcs" ) { } else if ( format == date_time_mcs ) {
month = convertMonth( datetime.substr( 4ul, 3ul ) ); month = convertMonth( datetime.substr( 4ul, 3ul ) );
day = datetime.substr( 8ul, 2ul ); day = datetime.substr( 8ul, 2ul );
hour = datetime.substr( 11ul, 2ul ); hour = datetime.substr( 11ul, 2ul );
@ -79,7 +81,7 @@ std::vector<std::string> processDateTime( std::string_view datetime_, std::strin
second = datetime.substr( 17ul, 2ul ); second = datetime.substr( 17ul, 2ul );
year = datetime.substr( datetime.size()-4 ); year = datetime.substr( datetime.size()-4 );
} else if ( format == "gmt" ) { } else if ( format == date_time_gmt ) {
size_t start{ datetime.find( ", " ) + 2ul }; size_t start{ datetime.find( ", " ) + 2ul };
day = datetime.substr( start, 2ul ); day = datetime.substr( start, 2ul );
start += 3ul; start += 3ul;
@ -93,7 +95,7 @@ std::vector<std::string> processDateTime( std::string_view datetime_, std::strin
start += 3ul; start += 3ul;
second = datetime.substr( start, 2ul ); second = datetime.substr( start, 2ul );
} else if ( StringOps::startsWith( format, "iso" ) ) { } else if ( format == date_time_iso ) {
year = datetime.substr( 0ul, 4ul ); year = datetime.substr( 0ul, 4ul );
month = datetime.substr( 5ul, 2ul ); month = datetime.substr( 5ul, 2ul );
day = datetime.substr( 8ul, 2ul ); day = datetime.substr( 8ul, 2ul );
@ -101,31 +103,36 @@ std::vector<std::string> processDateTime( std::string_view datetime_, std::strin
minute = datetime.substr( 14ul, 2ul ); minute = datetime.substr( 14ul, 2ul );
second = datetime.substr( 17ul, 2ul ); second = datetime.substr( 17ul, 2ul );
} else if ( StringOps::startsWith( format, "utc" ) ) { } else if ( _DATE_TIME_UTC & format ) {
if ( format == "utc_d" ) { if ( format == date_time_utc_d ) {
// date // date
year = datetime.substr( 0ul, 4ul ); year = datetime.substr( 0ul, 4ul );
month = datetime.substr( 5ul, 2ul ); month = datetime.substr( 5ul, 2ul );
day = datetime.substr( 8ul, 2ul ); day = datetime.substr( 8ul, 2ul );
} else { } else if ( format == date_time_utc_t ) {
// time // time
hour = datetime.substr( 0ul, 2ul ); hour = datetime.substr( 0ul, 2ul );
minute = datetime.substr( 3ul, 2ul ); minute = datetime.substr( 3ul, 2ul );
second = datetime.substr( 6ul, 2ul ); second = datetime.substr( 6ul, 2ul );
} else [[unlikely]] {
// wronthing went some ...
throw DateTimeException("Unexpected DateTime UTC: "+std::string{datetime_}+" - format: "+std::to_string(format));
} }
} else if ( StringOps::startsWith( format, "epoch_" ) ) { } else if ( _DATE_TIME_EPOCH & format ) {
aux = format.substr( 6ul );
// convert to seconds // convert to seconds
if ( aux == "us" ) { if ( format == date_time_epoch_us ) {
// from microseconds // from microseconds
datetime.resize( datetime.size()-6ul ); datetime.resize( datetime.size()-6ul );
} else if ( aux == "ms" ) { } else if ( format == date_time_epoch_ms ) {
// from milliseconds // from milliseconds
datetime.resize( datetime.size()-3ul ); datetime.resize( datetime.size()-3ul );
} else if ( aux == "s.ms" ) { } else if ( format == date_time_epoch_s_ms ) {
// from seconds.milliseconds // from seconds.milliseconds
datetime = std::to_string( std::stoi( datetime ) ); datetime = std::to_string( std::stoi( datetime ) );
} else if ( format != date_time_epoch_s ) [[unlikely]] {
// wronthing went some ...
throw DateTimeException("Unexpected DateTime EPOCH: "+std::string{datetime_}+" - format: "+std::to_string(format));
} }
// convert to iso date format // convert to iso date format
const QDateTime e{ QDateTime::fromSecsSinceEpoch( std::stoi( datetime ) ) }; const QDateTime e{ QDateTime::fromSecsSinceEpoch( std::stoi( datetime ) ) };
@ -139,20 +146,20 @@ std::vector<std::string> processDateTime( std::string_view datetime_, std::strin
minute = datetime.substr( 14ul, 2ul ); minute = datetime.substr( 14ul, 2ul );
second = datetime.substr( 17ul, 2ul ); second = datetime.substr( 17ul, 2ul );
} else { } else if ( _DATE_TIME_DATE & format ) {
if ( format == "YYYYMMDD" ) { if ( format == date_time_yyyymmdd ) {
year = datetime.substr( 0ul, 4ul ); year = datetime.substr( 0ul, 4ul );
month = datetime.substr( 5ul, 2ul ); month = datetime.substr( 5ul, 2ul );
day = datetime.substr( 8ul, 2ul ); day = datetime.substr( 8ul, 2ul );
} else if ( format == "MMDDYY" ) { } else if ( format == date_time_mmddyy ) {
const int y{ std::stoi( datetime.substr( 6ul, 2ul ) ) }; const int y{ std::stoi( datetime.substr( 6ul, 2ul ) ) };
month = datetime.substr( 0ul, 2ul ); month = datetime.substr( 0ul, 2ul );
day = datetime.substr( 3ul, 2ul ); day = datetime.substr( 3ul, 2ul );
year = (y<70) ? "20" : "19"; year = (y<70) ? "20" : "19";
year += (y<10) ? "0"+std::to_string( y ) : std::to_string( y ); year += (y<10) ? "0"+std::to_string( y ) : std::to_string( y );
} else if ( format == "MDYYYY" ) { } else if ( format == date_time_mdyyyy ) {
size_t aux_; size_t aux_;
if ( datetime.at(2) == '/' ) { if ( datetime.at(2) == '/' ) {
month = datetime.substr( 0ul, 2ul ); month = datetime.substr( 0ul, 2ul );
@ -170,63 +177,65 @@ std::vector<std::string> processDateTime( std::string_view datetime_, std::strin
} }
year = datetime.substr( aux_ ); year = datetime.substr( aux_ );
} else if ( StringOps::startsWith( format, "year" ) ) { } else [[unlikely]] {
year = datetime;
if ( format == "year_short" ) {
const int y{ std::stoi( year ) };
year = (y<70) ? "20" : "19";
year += year;
}
} else if ( StringOps::startsWith( format, "month" ) ) {
if ( format.size() <= 5ul ) {
month = datetime;
} else {
datetime.resize( 3ul ); // may be the full name
month = convertMonth( datetime );
}
} else if ( format == "day" ) {
day = datetime;
} else if ( StringOps::startsWith( format, "clock_" ) ) {
aux = format.substr( 6ul );
if ( aux == "24" ) {
hour = datetime.substr( 0ul, 2ul );
minute = datetime.substr( 3ul, 2ul );
second = datetime.substr( 6ul, 2ul );
} else if ( aux == "12" ) {
hour = datetime.substr( 0ul, 2ul );
minute = datetime.substr( 3ul, 2ul );
second = datetime.substr( 6ul, 2ul );
if ( datetime.substr( 9ul, 2ul ) == "pm" ) {
hour = std::to_string( 12 + std::stoi(hour) );
}
} else if ( aux == "short" ) {
hour = datetime.substr( 0ul, 2ul );
minute = datetime.substr( 3ul, 2ul );
} else if ( aux == "meridian" ) {
if ( datetime == "pm" ) {
hour = "PM"; // to mark for final update
}
}
} else if ( format == "hour" ) {
hour = datetime;
} else if ( format == "minute" ) {
minute = datetime;
} else if ( format == "second" ) {
second = datetime;
} else {
// wronthing went some ... // wronthing went some ...
throw DateTimeException("Unexpected DateTime format: "+std::string{datetime_}); throw DateTimeException("Unexpected DateTime DATE: "+std::string{datetime_}+" - format: "+std::to_string(format));
} }
} else if ( _DATE_TIME_CLOCK & format ) {
if ( format == date_time_clock_24 ) {
hour = datetime.substr( 0ul, 2ul );
minute = datetime.substr( 3ul, 2ul );
second = datetime.substr( 6ul, 2ul );
} else if ( format == date_time_clock_12 ) {
hour = datetime.substr( 0ul, 2ul );
minute = datetime.substr( 3ul, 2ul );
second = datetime.substr( 6ul, 2ul );
if ( datetime.substr( 9ul, 2ul ) == "pm" ) {
hour = std::to_string( 12 + std::stoi(hour) );
}
} else if ( format == date_time_clock_short ) {
hour = datetime.substr( 0ul, 2ul );
minute = datetime.substr( 3ul, 2ul );
} else [[unlikely]] {
// wronthing went some ...
throw DateTimeException("Unexpected DateTime CLOCK: "+std::string{datetime_}+" - format: "+std::to_string(format));
}
} else if ( _DATE_TIME_YEAR & format ) {
year = datetime;
if ( format == date_time_year_short ) {
const int y{ std::stoi( year ) };
year = (y<70) ? "20" : "19";
year += year;
}
} else if ( _DATE_TIME_MONTH & format ) {
if ( format == date_time_month ) {
month = datetime;
} else {
datetime.resize( 3ul ); // may be the full name
month = convertMonth( datetime );
}
} else if ( format == date_time_day ) {
day = datetime;
} else if ( format == date_time_hour ) {
hour = datetime;
} else if ( format == date_time_minute ) {
minute = datetime;
} else if ( format == date_time_second ) {
second = datetime;
} else [[unlikely]] {
// wronthing went some ...
throw DateTimeException("Unexpected DateTime: "+std::string{datetime_}+" - format: "+std::to_string(format));
} }
return { year, month, day, hour, minute, second }; return { year, month, day, hour, minute, second };

6
logdoctor/modules/craplog/modules/datetime.h
View File

@ -2,10 +2,14 @@
#define LOGDOCTOR__CRAPLOG__MODULES__DATETIME_H #define LOGDOCTOR__CRAPLOG__MODULES__DATETIME_H
#include <stdint.h>
#include <string> #include <string>
#include <vector> #include <vector>
enum LogsFormatField : uint32_t;
//! DateTimeOps //! DateTimeOps
/*! /*!
Operations for the dates Operations for the dates
@ -20,7 +24,7 @@ namespace DateTimeOps
\throw DateTimeException \throw DateTimeException
\return The list of items \return The list of items
*/ */
std::vector<std::string> processDateTime( std::string_view datetime, std::string_view format ); std::vector<std::string> processDateTime( std::string_view datetime, const LogsFormatField format );
} // namespace DateTimeOps } // namespace DateTimeOps

50
logdoctor/modules/craplog/modules/formats.cpp
View File

@ -273,18 +273,20 @@ LogsFormat FormatOps::processApacheFormatString( const std::string& f_str ) cons
const auto& f_map_v { this->APACHE_ALF_V }; const auto& f_map_v { this->APACHE_ALF_V };
std::string initial, final; std::string initial, final;
std::vector<std::string> separators, fields; std::vector<std::string> separators;
std::vector<LogsFormatField> fields;
// parse the string to convert keyargs in craplog's fields format // parse the string to convert keyargs in craplog's fields format
bool is_strftime_sep; bool is_strftime_sep{ false };
int n_fld{ 0 }; int n_fld{ 0 };
size_t start, stop{0ul}, aux, aux_start, aux_stop; size_t start, stop{0ul}, aux, aux_start, aux_stop;
const size_t max{ f_str.size()-1ul }; const size_t max{ f_str.size()-1ul };
std::string aux_fld, aux_fld_v, cur_fld, cur_sep; std::string aux_fld, aux_fld_v, cur_sep;
LogsFormatField cur_fld{ _INVALID };
// find and convert any field // find and convert any field
while (true) { while (true) {
// start after the last found field // start after the last found field
start = stop; start = stop;
if ( cur_fld == "date_time_ncsa" ) { if ( cur_fld == date_time_ncsa ) {
// NCAS time format is always enclosed inside brackets // NCAS time format is always enclosed inside brackets
cur_sep += "]"; cur_sep += "]";
} }
@ -381,7 +383,7 @@ LogsFormat FormatOps::processApacheFormatString( const std::string& f_str ) cons
const auto& aux_map{ f_map_v.at( aux_fld_v ) }; const auto& aux_map{ f_map_v.at( aux_fld_v ) };
if ( aux_map.empty() ) { if ( aux_map.empty() ) {
// module not considered and always giving out something, even if invalid varname is passed // module not considered and always giving out something, even if invalid varname is passed
cur_fld = "NONE"; cur_fld = _DISCARDED;
} else if ( aux_fld.empty() ) { } else if ( aux_fld.empty() ) {
// no need to check further, the dafault is used in this case // no need to check further, the dafault is used in this case
@ -393,11 +395,12 @@ LogsFormat FormatOps::processApacheFormatString( const std::string& f_str ) cons
} else if ( aux_fld_v == "p" || aux_fld_v == "P" || aux_fld_v == "T" ) { } else if ( aux_fld_v == "p" || aux_fld_v == "P" || aux_fld_v == "T" ) {
// still not considered (except 'T'), but invalid fields get used as text // still not considered (except 'T'), but invalid fields get used as text
// field concatenation not allowed, whole content used as varname // field concatenation not allowed, whole content used as varname
if ( aux_map.find( aux_fld ) != aux_map.end() ) { if ( aux_map.contains( aux_fld ) ) {
// valid varname // valid varname
cur_fld = aux_map.at( aux_fld ); cur_fld = aux_map.at( aux_fld );
} else { } else {
// invalid varname, use as text // invalid varname, use as text
cur_fld = _INVALID;
cur_sep += aux_fld; cur_sep += aux_fld;
start = stop = aux_stop; start = stop = aux_stop;
continue; continue;
@ -406,15 +409,15 @@ LogsFormat FormatOps::processApacheFormatString( const std::string& f_str ) cons
} else if ( aux_fld_v == "a" || aux_fld_v == "h" ) { } else if ( aux_fld_v == "a" || aux_fld_v == "h" ) {
// whatever the varname is (valid, invalid, empty), always returns the client // whatever the varname is (valid, invalid, empty), always returns the client
// field concatenation not allowed, the entire content is used as varname // field concatenation not allowed, the entire content is used as varname
cur_fld = "client" ; cur_fld = client;
} else if ( aux_fld_v == "i" ) { } else if ( aux_fld_v == "i" ) {
// always giving a result, may the varname be valid or not ('-' if invalid/empty) // always giving a result, may the varname be valid or not ('-' if invalid/empty)
// field concatenation not allowed, the entire content is used as varname // field concatenation not allowed, the entire content is used as varname
if ( aux_map.find( aux_fld ) != aux_map.end() ) { if ( aux_map.contains( aux_fld ) ) {
cur_fld = aux_map.at( aux_fld ); cur_fld = aux_map.at( aux_fld );
} else { } else {
cur_fld = "NONE"; cur_fld = _DISCARDED;
} }
} else /*if ( aux_fld_v == "t" )*/ { } else /*if ( aux_fld_v == "t" )*/ {
@ -423,12 +426,13 @@ LogsFormat FormatOps::processApacheFormatString( const std::string& f_str ) cons
if ( aux_aux == std::string::npos ) { if ( aux_aux == std::string::npos ) {
// no concatenation, only valid fields used, anything else used as text // no concatenation, only valid fields used, anything else used as text
// whole content used as varname // whole content used as varname
if ( aux_map.find( aux_fld ) != aux_map.end() ) { if ( aux_map.contains( aux_fld ) ) {
// valid // valid
cur_fld = aux_map.at( aux_fld ); cur_fld = aux_map.at( aux_fld );
is_strftime_sep = true; is_strftime_sep = true;
} else { } else {
// invalid, append to current separator // invalid, append to current separator
cur_fld = _INVALID;
cur_sep += aux_fld; cur_sep += aux_fld;
start = stop = aux_stop; start = stop = aux_stop;
continue; continue;
@ -475,7 +479,7 @@ LogsFormat FormatOps::processApacheFormatString( const std::string& f_str ) cons
aux_aux_fld = aux_fld.substr( aux_aux, 2ul ); aux_aux_fld = aux_fld.substr( aux_aux, 2ul );
aux_aux_stop = aux_aux+2ul; aux_aux_stop = aux_aux+2ul;
// check if the field is valid // check if the field is valid
if ( aux_map.find( aux_aux_fld ) != aux_map.end() ) { if ( aux_map.contains( aux_aux_fld ) ) {
// valid, append // valid, append
cur_fld = aux_map.at( aux_aux_fld ); cur_fld = aux_map.at( aux_aux_fld );
// append the separator // append the separator
@ -517,10 +521,10 @@ LogsFormat FormatOps::processApacheFormatString( const std::string& f_str ) cons
++ aux_stop; ++ aux_stop;
} }
// check if the module is valid // check if the module is valid
if ( f_map.find( aux_fld ) != f_map.end() ) { if ( f_map.contains( aux_fld ) ) {
// valid // valid
cur_fld = f_map.at( aux_fld ); cur_fld = f_map.at( aux_fld );
if ( cur_fld == "date_time_ncsa" ) { if ( cur_fld == date_time_ncsa ) {
// apache's NCSA time format is always enclosed inside brackets // apache's NCSA time format is always enclosed inside brackets
cur_sep += "["; cur_sep += "[";
} }
@ -544,7 +548,7 @@ LogsFormat FormatOps::processApacheFormatString( const std::string& f_str ) cons
break; break;
} }
if ( cur_fld.empty() ) { if ( cur_fld == _INVALID ) {
// invalid field, used as text (namely, added to current separator) // invalid field, used as text (namely, added to current separator)
continue; continue;
} }
@ -608,7 +612,8 @@ LogsFormat FormatOps::processNginxFormatString( const std::string& f_str ) const
const auto& f_map{ this->NGINX_ALF }; const auto& f_map{ this->NGINX_ALF };
std::string initial, final; std::string initial, final;
std::vector<std::string> separators, fields; std::vector<std::string> separators;
std::vector<LogsFormatField> fields;
// parse the string to convert keyargs in craplog's fields format // parse the string to convert keyargs in craplog's fields format
bool finished{ false }; bool finished{ false };
size_t start, aux, stop{0ul}; size_t start, aux, stop{0ul};
@ -654,14 +659,14 @@ LogsFormat FormatOps::processNginxFormatString( const std::string& f_str ) const
} }
// check if the field is valid // check if the field is valid
if ( f_map.find( cur_fld ) != f_map.end() ) { if ( const auto it{ f_map.find( cur_fld ) }; it != f_map.end() ) {
// valid, append // valid, append
if ( start == 0ul ) { if ( start == 0ul ) {
initial = parseNginxEscapes( cur_sep ); initial = parseNginxEscapes( cur_sep );
} else { } else {
separators.push_back( parseNginxEscapes( cur_sep ) ); separators.push_back( parseNginxEscapes( cur_sep ) );
} }
fields.push_back( f_map.at( cur_fld ) ); fields.push_back( it->second );
if ( finished ) { if ( finished ) {
// this was the last field // this was the last field
break; break;
@ -705,18 +710,19 @@ LogsFormat FormatOps::processIisFormatString( const std::string& f_str, const in
{ {
checkIisString( f_str ); checkIisString( f_str );
std::string initial, final; std::string initial, final;
std::vector<std::string> separators, fields; std::vector<std::string> separators;
std::vector<LogsFormatField> fields;
switch ( l_mod ) { switch ( l_mod ) {
case 2: case 2:
// IIS logging module // IIS logging module
final = ","; final = ",";
separators = {", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", "}; separators = {", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", "};
fields = {"client","NONE","date_time_MDYYYY","date_time_utc_t","NONE","NONE","NONE","time_taken_ms","bytes_received","bytes_sent","response_code","NONE","request_method","request_uri","request_query"}; fields = {client,_DISCARDED,date_time_mdyyyy,date_time_utc_t,_DISCARDED,_DISCARDED,_DISCARDED,time_taken_ms,bytes_received,bytes_sent,response_code,_DISCARDED,request_method,request_uri,request_query};
break; break;
case 1: case 1:
// NCSA logging module // NCSA logging module
separators = {" "," "," [","] \"","\" "," "}; separators = {" "," "," [","] \"","\" "," "};
fields = {"client","NONE","NONE","date_time_ncsa","request_full","response_code","bytes_sent"}; fields = {client,_DISCARDED,_DISCARDED,date_time_ncsa,request_full,response_code,bytes_sent};
break; break;
case 0: case 0:
// W3C logging module // W3C logging module
@ -745,9 +751,9 @@ LogsFormat FormatOps::processIisFormatString( const std::string& f_str, const in
++ stop; ++ stop;
// check if the module is valid // check if the module is valid
if ( f_map.find( cur_fld ) != f_map.end() ) { if ( const auto it{ f_map.find( cur_fld ) }; it != f_map.end() ) {
// valid, append // valid, append
fields.push_back( f_map.at( cur_fld ) ); fields.push_back( it->second );
if ( ! finished ) { if ( ! finished ) {
separators.push_back( cur_sep ); separators.push_back( cur_sep );
} else { } else {

662
logdoctor/modules/craplog/modules/formats.h
View File

@ -2,6 +2,8 @@
#define LOGDOCTOR__CRAPLOG__MODULES__FORMATS_H #define LOGDOCTOR__CRAPLOG__MODULES__FORMATS_H
#include "lib.h"
#include <QString> #include <QString>
#include <unordered_map> #include <unordered_map>
@ -15,6 +17,10 @@ struct LogsFormat;
*/ */
class FormatOps final class FormatOps final
{ {
using simple_fields_umap_t = std::unordered_map<std::string, LogsFormatField>;
using composed_fields_umap_t = std::unordered_map<std::string, std::unordered_map<std::string, LogsFormatField>>;
using samples_umap_t = std::unordered_map<LogsFormatField, QString>;
public: public:
//! Processes the given string to extrapolate the format for Apache2 //! Processes the given string to extrapolate the format for Apache2
@ -79,362 +85,362 @@ private:
///////////////// /////////////////
//// APACHE2 //// //// APACHE2 ////
//!< Access logs fields formats //! Access logs fields formats
const std::unordered_map<std::string, std::string> APACHE_ALF{ const simple_fields_umap_t APACHE_ALF{
{"a", "NONE"}, {"a", _DISCARDED},
{"A", "NONE"}, {"A", _DISCARDED},
{"b", "NONE"}, {"b", _DISCARDED},
{"B", "NONE"}, {"B", _DISCARDED},
{"C", "NONE"}, {"C", _DISCARDED},
{"D", "time_taken_ms"}, {"D", time_taken_ms},
{"e", "NONE"}, {"e", _DISCARDED},
{"f", "NONE"}, {"f", _DISCARDED},
{"h", "client"}, {"h", client},
{"H", "request_protocol"}, {"H", request_protocol},
{"i", "NONE"}, {"i", _DISCARDED},
{"I", "bytes_received"}, {"I", bytes_received},
{"k", "NONE"}, {"k", _DISCARDED},
{"l", "NONE"}, {"l", _DISCARDED},
{"L", "NONE"}, {"L", _DISCARDED},
{"m", "request_method"}, {"m", request_method},
{"n", "NONE"}, {"n", _DISCARDED},
{"o", "NONE"}, {"o", _DISCARDED},
{"O", "bytes_sent"}, {"O", bytes_sent},
{"p", "NONE"}, {"p", _DISCARDED},
{"P", "NONE"}, {"P", _DISCARDED},
{"q", "request_query"}, {"q", request_query},
{"r", "request_full"}, {"r", request_full},
{"R", "NONE"}, {"R", _DISCARDED},
{"s", "response_code"}, {"s", response_code},
{"<s", "response_code"}, {"<s", response_code},
{">s", "response_code"}, {">s", response_code},
{"S", "NONE"}, {"S", _DISCARDED},
{"t", "date_time_ncsa"}, {"t", date_time_ncsa},
{"T", "time_taken_s"}, {"T", time_taken_s},
{"u", "NONE"}, {"u", _DISCARDED},
{"U", "request_uri"}, {"U", request_uri},
{"v", "NONE"}, {"v", _DISCARDED},
{"V", "NONE"}, {"V", _DISCARDED},
{"X", "NONE"} }; {"X", _DISCARDED} };
//!< Composed access logs fields formats //! Composed access logs fields formats
const std::unordered_map<std::string, std::unordered_map<std::string, std::string>> APACHE_ALF_V{ const composed_fields_umap_t APACHE_ALF_V{
{"a", { {"", "client"}, // as %h (from single fields) {"a", { {"", client}, // as %h (from single fields)
{"c", "client"}}}, {"c", client}}},
{"h", { {"", "client"}, // as %h (from single fields) {"h", { {"", client}, // as %h (from single fields)
{"c", "client"}}}, {"c", client}}},
{"i", { {"", "NONE"}, {"i", { {"", _DISCARDED},
{"Cookie", "cookie"}, {"Cookie", cookie},
{"Referer", "referer"}, {"Referer", referer},
{"User-agent", "user_agent"}}}, {"User-agent", user_agent}}},
{"t", { {"", "date_time_ncsa"}, // as %t (from single fields) {"t", { {"", date_time_ncsa}, // as %t (from single fields)
//{"%%", "NONE"}, // the percent sign //{"%%", _DISCARDED}, // the percent sign
//{"%n", "NONE"}, // line feed //{"%n", _DISCARDED}, // line feed
//{"%t", "NONE"}, // horizontal tab //{"%t", _DISCARDED}, // horizontal tab
{"sec", "date_time_epoch_s"}, {"sec", date_time_epoch_s},
{"msec", "date_time_epoch_ms"}, {"msec", date_time_epoch_ms},
{"usec", "date_time_epoch_us"}, {"usec", date_time_epoch_us},
{"msec_frac", "NONE"}, // milliseconds fraction {"msec_frac", _DISCARDED}, // milliseconds fraction
{"usec_frac", "NONE"}, // microseconds fraction {"usec_frac", _DISCARDED}, // microseconds fraction
{"%a", "NONE"}, // abbreviated weekday name {"%a", _DISCARDED}, // abbreviated weekday name
{"%A", "NONE"}, // weekday name {"%A", _DISCARDED}, // weekday name
{"%b", "date_time_month_str"}, {"%b", date_time_month_str},
{"%B", "date_time_month_str"}, {"%B", date_time_month_str},
{"%c", "date_time_mcs"}, {"%c", date_time_mcs},
{"%C", "NONE"}, // year (first 2 digits, aka centuries) {"%C", _DISCARDED}, // year (first 2 digits, aka centuries)
{"%d", "date_time_day"}, {"%d", date_time_day},
{"%D", "date_time_MMDDYY"}, {"%D", date_time_mmddyy},
{"%e", "date_time_day"}, {"%e", date_time_day},
{"%F", "date_time_YYYYMMDD"}, {"%F", date_time_yyyymmdd},
{"%g", "NONE"}, // weel-based year (last 2 digits) {"%g", _DISCARDED}, // weel-based year (last 2 digits)
{"%G", "NONE"}, // week-based year, namely the year which contains the current week {"%G", _DISCARDED}, // week-based year, namely the year which contains the current week
{"%h", "date_time_month_str"}, {"%h", date_time_month_str},
{"%H", "date_time_hour"}, {"%H", date_time_hour},
{"%I", "NONE"}, // hour (12h format) {"%I", _DISCARDED}, // hour (12h format)
{"%j", "NONE"}, // day of the year number {"%j", _DISCARDED}, // day of the year number
{"%k", "date_time_hour"}, // ?!? hour (24h format) !?! no documentation ?!? {"%k", date_time_hour}, // ?!? hour (24h format) !?! no documentation ?!?
{"%l", "NONE"}, // hour (12h format) {"%l", _DISCARDED}, // hour (12h format)
{"%m", "date_time_month"}, {"%m", date_time_month},
{"%M", "date_time_minute"}, {"%M", date_time_minute},
{"%p", "NONE"}, // AM or PM {"%p", _DISCARDED}, // AM or PM
{"%r", "date_time_clock_12"}, {"%r", date_time_clock_12},
{"%R", "date_time_clock_short"}, {"%R", date_time_clock_short},
{"%s", "date_time_epoch_s"}, {"%s", date_time_epoch_s},
{"%S", "date_time_second"}, {"%S", date_time_second},
{"%T", "date_time_clock_24"}, {"%T", date_time_clock_24},
{"%u", "NONE"}, // weekday number (1-7, Monday is 1) {"%u", _DISCARDED}, // weekday number (1-7, Monday is 1)
{"%U", "NONE"}, // week of the year number, with the first Sunday as the first day of week one {"%U", _DISCARDED}, // week of the year number, with the first Sunday as the first day of week one
{"%V", "NONE"}, // week of the year number {"%V", _DISCARDED}, // week of the year number
{"%w", "NONE"}, // weekday number (0-6, Sunday is 0) {"%w", _DISCARDED}, // weekday number (0-6, Sunday is 0)
{"%W", "NONE"}, // week of the year number, with the first Monday as the first day of week one {"%W", _DISCARDED}, // week of the year number, with the first Monday as the first day of week one
{"%x", "date_time_MMDDYY"}, {"%x", date_time_mmddyy},
{"%X", "date_time_clock_24"}, {"%X", date_time_clock_24},
{"%y", "date_time_year_short"}, {"%y", date_time_year_short},
{"%Y", "date_time_year"}, {"%Y", date_time_year},
{"%z", "NONE"}, // timezone offset from UTC (1 minute=1, 1 hour=100) {"%z", _DISCARDED}, // timezone offset from UTC (1 minute=1, 1 hour=100)
{"%Z", "NONE"}}}, // timezone name abbreviation {"%Z", _DISCARDED}}}, // timezone name abbreviation
{"T", { {"", "time_taken_s"}, // as s {"T", { {"", time_taken_s}, // as s
{"s", "time_taken_s"}, {"s", time_taken_s},
{"ms", "time_taken_ms"}, {"ms", time_taken_ms},
{"us", "time_taken_us"}}}, {"us", time_taken_us}}},
// composed not in use // composed not in use
{"C", {}}, {"C", {}},
{"e", {}}, {"e", {}},
{"L", {}}, {"L", {}},
{"n", {}}, {"n", {}},
{"o", {}}, {"o", {}},
{"p", {{"", "NONE"}, // as canonical {"p", {{"", _DISCARDED}, // as canonical
{"canonical", "NONE"}, {"canonical", _DISCARDED},
{"local", "NONE"}, {"local", _DISCARDED},
{"remote", "NONE"}}}, {"remote", _DISCARDED}}},
{"P", {{"", "NONE"}, // as pid {"P", {{"", _DISCARDED}, // as pid
{"pid", "NONE"}, {"pid", _DISCARDED},
{"tid", "NONE"}, {"tid", _DISCARDED},
{"hextid", "NONE"}}}, {"hextid", _DISCARDED}}},
{"^ti", {}}, {"^ti", {}},
{"^to", {}} }; {"^to", {}} };
// Access logs fields formats samples // Access logs fields formats samples
const std::unordered_map<std::string, QString> APACHE_ALF_SAMPLES{ const samples_umap_t APACHE_ALF_SAMPLES{
{"NONE", "<span style=\"color:#7f7f7f\">DISCARDED</span>"}, {_DISCARDED, R"(<span style="color:#7f7f7f">DISCARDED</span>)"},
{"date_time_epoch_s", "<b><span style=\"color:#00cc6f\">946771199</span></b>"}, {date_time_epoch_s, R"(<b><span style="color:#00cc6f">946771199</span></b>)"},
{"date_time_epoch_ms", "<b><span style=\"color:#00cc6f\">946771199000</span></b>"}, {date_time_epoch_ms, R"(<b><span style="color:#00cc6f">946771199000</span></b>)"},
{"date_time_epoch_us", "<b><span style=\"color:#00cc6f\">946771199000000</span></b>"}, {date_time_epoch_us, R"(<b><span style="color:#00cc6f">946771199000000</span></b>)"},
{"date_time_ncsa", "<b><span style=\"color:#00cc6f\">01/Jan/2000:23:59:59 +0000</span></b>"}, {date_time_ncsa, R"(<b><span style="color:#00cc6f">01/Jan/2000:23:59:59 +0000</span></b>)"},
{"date_time_mcs", "<b><span style=\"color:#00cc6f\">Sat Jan 01 23:59:59 2000</span></b>"}, {date_time_mcs, R"(<b><span style="color:#00cc6f">Sat Jan 01 23:59:59 2000</span></b>)"},
{"date_time_YYYYMMDD", "<b><span style=\"color:#00cc6f\">2000-01-01</span></b>"}, {date_time_yyyymmdd, R"(<b><span style="color:#00cc6f">2000-01-01</span></b>)"},
{"date_time_MMDDYY", "<b><span style=\"color:#00cc6f\">01/01/00</span></b>"}, {date_time_mmddyy, R"(<b><span style="color:#00cc6f">01/01/00</span></b>)"},
{"date_time_year", "<b><span style=\"color:#00cc6f\">2000</span></b>"}, {date_time_year, R"(<b><span style="color:#00cc6f">2000</span></b>)"},
{"date_time_year_short", "<b><span style=\"color:#00cc6f\">00</span></b>"}, {date_time_year_short, R"(<b><span style="color:#00cc6f">00</span></b>)"},
{"date_time_month_str", "<b><span style=\"color:#00cc6f\">January</span></b>"}, {date_time_month_str, R"(<b><span style="color:#00cc6f">January</span></b>)"},
{"date_time_month", "<b><span style=\"color:#00cc6f\">01</span></b>"}, {date_time_month, R"(<b><span style="color:#00cc6f">01</span></b>)"},
{"date_time_day", "<b><span style=\"color:#00cc6f\">01</span></b>"}, {date_time_day, R"(<b><span style="color:#00cc6f">01</span></b>)"},
{"date_time_clock_12", "<b><span style=\"color:#00cc6f\">11:59:59 pm</span></b>"}, {date_time_clock_12, R"(<b><span style="color:#00cc6f">11:59:59 pm</span></b>)"},
{"date_time_clock_24", "<b><span style=\"color:#00cc6f\">23:59:59</span></b>"}, {date_time_clock_24, R"(<b><span style="color:#00cc6f">23:59:59</span></b>)"},
{"date_time_clock_short", "<b><span style=\"color:#00cc6f\">23:59</span></b>"}, {date_time_clock_short, R"(<b><span style="color:#00cc6f">23:59</span></b>)"},
{"date_time_hour", "<b><span style=\"color:#00cc6f\">23</span></b>"}, {date_time_hour, R"(<b><span style="color:#00cc6f">23</span></b>)"},
{"date_time_minute", "<b><span style=\"color:#00cc6f\">59</span></b>"}, {date_time_minute, R"(<b><span style="color:#00cc6f">59</span></b>)"},
{"date_time_second", "<b><span style=\"color:#00cc6f\">59</span></b>"}, {date_time_second, R"(<b><span style="color:#00cc6f">59</span></b>)"},
{"request_full", "<b><span style=\"color:#00cc6f\">GET /index.php?query=x HTTP/1.1</span></b>"}, {request_full, R"(<b><span style="color:#00cc6f">GET /index.php?query=x HTTP/1.1</span></b>)"},
{"request_protocol", "<b><span style=\"color:#00cc6f\">HTTP/1.1</span></b>"}, {request_protocol, R"(<b><span style="color:#00cc6f">HTTP/1.1</span></b>)"},
{"request_method", "<b><span style=\"color:#00cc6f\">GET</span></b>"}, {request_method, R"(<b><span style="color:#00cc6f">GET</span></b>)"},
{"request_uri", "<b><span style=\"color:#00cc6f\">/index.php</span></b>"}, {request_uri, R"(<b><span style="color:#00cc6f">/index.php</span></b>)"},
{"request_query", "<b><span style=\"color:#00cc6f\">query=x</span></b>"}, {request_query, R"(<b><span style="color:#00cc6f">query=x</span></b>)"},
{"response_code", "<b><span style=\"color:#00cc6f\">404</span></b>"}, {response_code, R"(<b><span style="color:#00cc6f">404</span></b>)"},
{"bytes_sent", "<b><span style=\"color:#00cc6f\">1234</span></b>"}, {bytes_sent, R"(<b><span style="color:#00cc6f">1234</span></b>)"},
{"bytes_received", "<b><span style=\"color:#00cc6f\">123</span></b>"}, {bytes_received, R"(<b><span style="color:#00cc6f">123</span></b>)"},
{"time_taken_s", "<b><span style=\"color:#00cc6f\">1</span></b>"}, {time_taken_s, R"(<b><span style="color:#00cc6f">1</span></b>)"},
{"time_taken_ms", "<b><span style=\"color:#00cc6f\">1000</span></b>"}, {time_taken_ms, R"(<b><span style="color:#00cc6f">1000</span></b>)"},
{"time_taken_us", "<b><span style=\"color:#00cc6f\">1000000</span></b>"}, {time_taken_us, R"(<b><span style="color:#00cc6f">1000000</span></b>)"},
{"referer", "<b><span style=\"color:#00cc6f\">http://www.referrer.site</span></b>"}, {referer, R"(<b><span style="color:#00cc6f">http://www.referrer.site</span></b>)"},
{"cookie", "<b><span style=\"color:#00cc6f\">aCookie=abc123</span></b>"}, {cookie, R"(<b><span style="color:#00cc6f">aCookie=abc123</span></b>)"},
{"user_agent", "<b><span style=\"color:#00cc6f\">UserAgent/3.0 (Details stuff) Info/123</span></b>"}, {user_agent, R"(<b><span style="color:#00cc6f">UserAgent/3.0 (Details stuff) Info/123</span></b>)"},
{"client", "<b><span style=\"color:#00cc6f\">192.168.1.123</span></b>"} }; {client, R"(<b><span style="color:#00cc6f">192.168.1.123</span></b>)"} };
/////////////// ///////////////
//// NGINX //// //// NGINX ////
//!< Access logs fields formats //! Access logs fields formats
const std::unordered_map<std::string, std::string> NGINX_ALF{ const simple_fields_umap_t NGINX_ALF{
{"remote_addr", "client"}, {"remote_addr", client},
{"realip_remote_addr", "client"}, {"realip_remote_addr", client},
{"time_local", "date_time_ncsa"}, {"time_local", date_time_ncsa},
{"time_iso8601", "date_time_iso"}, {"time_iso8601", date_time_iso},
{"date_gmt", "date_time_gmt"}, {"date_gmt", date_time_gmt},
{"msec", "date_time_epoch_s.ms"}, {"msec", date_time_epoch_s_ms},
{"request", "request_full"}, {"request", request_full},
{"server_protocol", "request_protocol"}, {"server_protocol", request_protocol},
{"request_method", "request_method"}, {"request_method", request_method},
{"request_uri", "request_uri_query"}, {"request_uri", request_uri_query},
{"uri", "request_uri"}, {"uri", request_uri},
{"query_string", "request_query"}, {"query_string", request_query},
{"status", "response_code"}, {"status", response_code},
{"bytes_sent", "bytes_sent"}, {"bytes_sent", bytes_sent},
{"request_length", "bytes_received"}, {"request_length", bytes_received},
{"request_time", "time_taken_s.ms"}, {"request_time", time_taken_s_ms},
{"http_referer", "referer"}, {"http_referer", referer},
{"cookie_", "cookie"}, {"cookie_", cookie},
{"http_user_agent", "user_agent"}, {"http_user_agent", user_agent},
// not in use, will be discarded // not in use, will be discarded
{"ancient_browser", "NONE"}, {"ancient_browser", _DISCARDED},
{"arg_", "NONE"}, {"arg_", _DISCARDED},
{"args", "NONE"}, {"args", _DISCARDED},
{"binary_remote_addr", "NONE"}, {"binary_remote_addr", _DISCARDED},
{"body_bytes_sent", "NONE"}, {"body_bytes_sent", _DISCARDED},
{"connection", "NONE"}, {"connection", _DISCARDED},
{"connection_requests", "NONE"}, {"connection_requests", _DISCARDED},
{"connections_active", "NONE"}, {"connections_active", _DISCARDED},
{"connections_reading", "NONE"}, {"connections_reading", _DISCARDED},
{"connections_waiting", "NONE"}, {"connections_waiting", _DISCARDED},
{"connections_writing", "NONE"}, {"connections_writing", _DISCARDED},
{"content_length", "NONE"}, {"content_length", _DISCARDED},
{"content_type", "NONE"}, {"content_type", _DISCARDED},
{"date_local", "NONE"}, {"date_local", _DISCARDED},
{"document_root", "NONE"}, {"document_root", _DISCARDED},
{"document_uri", "NONE"}, {"document_uri", _DISCARDED},
{"fastcgi_path_info", "NONE"}, {"fastcgi_path_info", _DISCARDED},
{"fastcgi_script_name", "NONE"}, {"fastcgi_script_name", _DISCARDED},
{"geoip_area_code", "NONE"}, {"geoip_area_code", _DISCARDED},
{"geoip_city", "NONE"}, {"geoip_city", _DISCARDED},
{"geoip_city_continent_code", "NONE"}, {"geoip_city_continent_code", _DISCARDED},
{"geoip_city_country_code", "NONE"}, {"geoip_city_country_code", _DISCARDED},
{"geoip_city_country_code3", "NONE"}, {"geoip_city_country_code3", _DISCARDED},
{"geoip_city_country_name", "NONE"}, {"geoip_city_country_name", _DISCARDED},
{"geoip_country_code", "NONE"}, {"geoip_country_code", _DISCARDED},
{"geoip_country_code3", "NONE"}, {"geoip_country_code3", _DISCARDED},
{"geoip_country_name", "NONE"}, {"geoip_country_name", _DISCARDED},
{"geoip_dma_code", "NONE"}, {"geoip_dma_code", _DISCARDED},
{"geoip_latitude", "NONE"}, {"geoip_latitude", _DISCARDED},
{"geoip_longitude", "NONE"}, {"geoip_longitude", _DISCARDED},
{"geoip_org", "NONE"}, {"geoip_org", _DISCARDED},
{"geoip_postal_code", "NONE"}, {"geoip_postal_code", _DISCARDED},
{"geoip_region", "NONE"}, {"geoip_region", _DISCARDED},
{"geoip_region_name", "NONE"}, {"geoip_region_name", _DISCARDED},
{"gzip_ratio", "NONE"}, {"gzip_ratio", _DISCARDED},
{"host", "NONE"}, {"host", _DISCARDED},
{"hostname", "NONE"}, {"hostname", _DISCARDED},
{"http2", "NONE"}, {"http2", _DISCARDED},
{"http_", "NONE"}, {"http_", _DISCARDED},
{"https", "NONE"}, {"https", _DISCARDED},
{"invalid_referer", "NONE"}, {"invalid_referer", _DISCARDED},
{"is_args", "NONE"}, {"is_args", _DISCARDED},
{"limit_rate", "NONE"}, {"limit_rate", _DISCARDED},
{"memcached_key", "NONE"}, {"memcached_key", _DISCARDED},
{"modern_browser", "NONE"}, {"modern_browser", _DISCARDED},
{"msie", "NONE"}, {"msie", _DISCARDED},
{"nginx_version", "NONE"}, {"nginx_version", _DISCARDED},
{"pid", "NONE"}, {"pid", _DISCARDED},
{"pipe", "NONE"}, {"pipe", _DISCARDED},
{"proxy_add_x_forwarded_for", "NONE"}, {"proxy_add_x_forwarded_for", _DISCARDED},
{"proxy_host", "NONE"}, {"proxy_host", _DISCARDED},
{"proxy_port", "NONE"}, {"proxy_port", _DISCARDED},
{"proxy_protocol_addr", "NONE"}, {"proxy_protocol_addr", _DISCARDED},
{"proxy_protocol_port", "NONE"}, {"proxy_protocol_port", _DISCARDED},
{"realip_remote_port", "NONE"}, {"realip_remote_port", _DISCARDED},
{"realpath_root", "NONE"}, {"realpath_root", _DISCARDED},
{"remote_port", "NONE"}, {"remote_port", _DISCARDED},
{"remote_user", "NONE"}, {"remote_user", _DISCARDED},
{"request_body", "NONE"}, {"request_body", _DISCARDED},
{"request_body_file", "NONE"}, {"request_body_file", _DISCARDED},
{"request_completion", "NONE"}, {"request_completion", _DISCARDED},
{"request_filename", "NONE"}, {"request_filename", _DISCARDED},
{"request_id", "NONE"}, {"request_id", _DISCARDED},
{"scheme", "NONE"}, {"scheme", _DISCARDED},
{"secure_link", "NONE"}, {"secure_link", _DISCARDED},
{"secure_link_expires", "NONE"}, {"secure_link_expires", _DISCARDED},
{"sent_http_", "NONE"}, {"sent_http_", _DISCARDED},
{"server_addr", "NONE"}, {"server_addr", _DISCARDED},
{"server_name", "NONE"}, {"server_name", _DISCARDED},
{"server_port", "NONE"}, {"server_port", _DISCARDED},
{"session_log_binary_id", "NONE"}, {"session_log_binary_id", _DISCARDED},
{"session_log_id", "NONE"}, {"session_log_id", _DISCARDED},
{"slice_range", "NONE"}, {"slice_range", _DISCARDED},
{"spdy", "NONE"}, {"spdy", _DISCARDED},
{"spdy_request_priority", "NONE"}, {"spdy_request_priority", _DISCARDED},
{"ssl_cipher", "NONE"}, {"ssl_cipher", _DISCARDED},
{"ssl_client_cert", "NONE"}, {"ssl_client_cert", _DISCARDED},
{"ssl_client_fingerprint", "NONE"}, {"ssl_client_fingerprint", _DISCARDED},
{"ssl_client_i_dn", "NONE"}, {"ssl_client_i_dn", _DISCARDED},
{"ssl_client_raw_cert", "NONE"}, {"ssl_client_raw_cert", _DISCARDED},
{"ssl_client_s_dn", "NONE"}, {"ssl_client_s_dn", _DISCARDED},
{"ssl_client_serial", "NONE"}, {"ssl_client_serial", _DISCARDED},
{"ssl_client_verify", "NONE"}, {"ssl_client_verify", _DISCARDED},
{"ssl_protocol", "NONE"}, {"ssl_protocol", _DISCARDED},
{"ssl_server_name", "NONE"}, {"ssl_server_name", _DISCARDED},
{"ssl_session_id", "NONE"}, {"ssl_session_id", _DISCARDED},
{"ssl_session_reused", "NONE"}, {"ssl_session_reused", _DISCARDED},
{"tcpinfo_rtt", "NONE"}, {"tcpinfo_rtt", _DISCARDED},
{"tcpinfo_rttvar", "NONE"}, {"tcpinfo_rttvar", _DISCARDED},
{"tcpinfo_snd_cwnd", "NONE"}, {"tcpinfo_snd_cwnd", _DISCARDED},
{"tcpinfo_rcv_space", "NONE"}, {"tcpinfo_rcv_space", _DISCARDED},
{"uid_got", "NONE"}, {"uid_got", _DISCARDED},
{"uid_reset", "NONE"}, {"uid_reset", _DISCARDED},
{"uid_set", "NONE"}, {"uid_set", _DISCARDED},
{"upstream_addr", "NONE"}, {"upstream_addr", _DISCARDED},
{"upstream_cache_status", "NONE"}, {"upstream_cache_status", _DISCARDED},
{"upstream_connect_time", "NONE"}, {"upstream_connect_time", _DISCARDED},
{"upstream_cookie_", "NONE"}, {"upstream_cookie_", _DISCARDED},
{"upstream_header_time", "NONE"}, {"upstream_header_time", _DISCARDED},
{"upstream_http_", "NONE"}, {"upstream_http_", _DISCARDED},
{"upstream_response_length", "NONE"}, {"upstream_response_length", _DISCARDED},
{"upstream_response_time", "NONE"}, {"upstream_response_time", _DISCARDED},
{"upstream_status", "NONE"} }; {"upstream_status", _DISCARDED} };
// Access logs fields formats samples // Access logs fields formats samples
const std::unordered_map<std::string, QString> NGINX_ALF_SAMPLES{ const samples_umap_t NGINX_ALF_SAMPLES{
{"NONE", "<span style=\"color:#7f7f7f\">DISCARDED</span>"}, {_DISCARDED, R"(<span style="color:#7f7f7f">DISCARDED</span>)"},
{"date_time_epoch_s.ms", "<b><span style=\"color:#00cc6f\">946771199.000</span></b>"}, {date_time_epoch_s_ms, R"(<b><span style="color:#00cc6f">946771199.000</span></b>)"},
{"date_time_ncsa", "<b><span style=\"color:#00cc6f\">01/Jan/2000:23:59:59 +0000</span></b>"}, {date_time_ncsa, R"(<b><span style="color:#00cc6f">01/Jan/2000:23:59:59 +0000</span></b>)"},
{"date_time_iso", "<b><span style=\"color:#00cc6f\">2000-01-01T23:59:59+00:00</span></b>"}, {date_time_iso, R"(<b><span style="color:#00cc6f">2000-01-01T23:59:59+00:00</span></b>)"},
{"date_time_gmt", "<b><span style=\"color:#00cc6f\">Saturday, 01-Jan-2000 23:59:59 UTC</span></b>"}, {date_time_gmt, R"(<b><span style="color:#00cc6f">Saturday, 01-Jan-2000 23:59:59 UTC</span></b>)"},
{"request_full", "<b><span style=\"color:#00cc6f\">GET /index.php?query=x HTTP/1.1</span></b>"}, {request_full, R"(<b><span style="color:#00cc6f">GET /index.php?query=x HTTP/1.1</span></b>)"},
{"request_protocol", "<b><span style=\"color:#00cc6f\">HTTP/1.1</span></b>"}, {request_protocol, R"(<b><span style="color:#00cc6f">HTTP/1.1</span></b>)"},
{"request_method", "<b><span style=\"color:#00cc6f\">GET</span></b>"}, {request_method, R"(<b><span style="color:#00cc6f">GET</span></b>)"},
{"request_uri_query", "<b><span style=\"color:#00cc6f\">/index.php?query=x</span></b>"}, {request_uri_query, R"(<b><span style="color:#00cc6f">/index.php?query=x</span></b>)"},
{"request_uri", "<b><span style=\"color:#00cc6f\">/index.php</span></b>"}, {request_uri, R"(<b><span style="color:#00cc6f">/index.php</span></b>)"},
{"request_query", "<b><span style=\"color:#00cc6f\">query=x</span></b>"}, {request_query, R"(<b><span style="color:#00cc6f">query=x</span></b>)"},
{"response_code", "<b><span style=\"color:#00cc6f\">404</span></b>"}, {response_code, R"(<b><span style="color:#00cc6f">404</span></b>)"},
{"bytes_sent", "<b><span style=\"color:#00cc6f\">1234</span></b>"}, {bytes_sent, R"(<b><span style="color:#00cc6f">1234</span></b>)"},
{"bytes_received", "<b><span style=\"color:#00cc6f\">123</span></b>"}, {bytes_received, R"(<b><span style="color:#00cc6f">123</span></b>)"},
{"time_taken_s.ms", "<b><span style=\"color:#00cc6f\">1.000</span></b>"}, {time_taken_s_ms, R"(<b><span style="color:#00cc6f">1.000</span></b>)"},
{"referer", "<b><span style=\"color:#00cc6f\">http://www.referrer.site</span></b>"}, {referer, R"(<b><span style="color:#00cc6f">http://www.referrer.site</span></b>)"},
{"cookie", "<b><span style=\"color:#00cc6f\">aCookie=abc123</span></b>"}, {cookie, R"(<b><span style="color:#00cc6f">aCookie=abc123</span></b>)"},
{"user_agent", "<b><span style=\"color:#00cc6f\">UserAgent/3.0 (Details stuff) Info/123</span></b>"}, {user_agent, R"(<b><span style="color:#00cc6f">UserAgent/3.0 (Details stuff) Info/123</span></b>)"},
{"client", "<b><span style=\"color:#00cc6f\">192.168.1.123</span></b>"} }; {client, R"(<b><span style="color:#00cc6f">192.168.1.123</span></b>)"} };
///////////// /////////////
//// IIS //// //// IIS ////
//!< Access logs fields formats (W3C) //! Access logs fields formats (W3C)
const std::unordered_map<std::string, std::string> IIS_ALF{ const simple_fields_umap_t IIS_ALF{
{"date", "date_time_utc_d"}, {"date", date_time_utc_d},
{"time", "date_time_utc_t"}, {"time", date_time_utc_t},
{"cs-version", "request_protocol"}, {"cs-version", request_protocol},
{"cs-method", "request_method"}, {"cs-method", request_method},
{"cs-uri-stem", "request_uri"}, {"cs-uri-stem", request_uri},
{"cs-uri-query", "request_query"}, {"cs-uri-query", request_query},
{"sc-status", "response_code"}, {"sc-status", response_code},
{"sc-bytes", "bytes_sent"}, {"sc-bytes", bytes_sent},
{"cs-bytes", "bytes_received"}, {"cs-bytes", bytes_received},
{"time-taken", "time_taken_ms"}, {"time-taken", time_taken_ms},
{"cs(Referer)", "referer"}, {"cs(Referer)", referer},
{"cs(Cookie)", "cookie"}, {"cs(Cookie)", cookie},
{"cs(User-Agent)", "user_agent"}, {"cs(User-Agent)", user_agent},
{"c-ip", "client"}, {"c-ip", client},
// not in use, will be discarded // not in use, will be discarded
{"s-sitename", "NONE"}, {"s-sitename", _DISCARDED},
{"s-computername", "NONE"}, {"s-computername", _DISCARDED},
{"s-ip", "NONE"}, {"s-ip", _DISCARDED},
{"s-port", "NONE"}, {"s-port", _DISCARDED},
{"cs-username", "NONE"}, {"cs-username", _DISCARDED},
{"cs-host", "NONE"}, {"cs-host", _DISCARDED},
{"sc-substatus", "NONE"}, {"sc-substatus", _DISCARDED},
{"sc-win32-status", "NONE"}, {"sc-win32-status", _DISCARDED},
{"streamid", "NONE"} }; {"streamid", _DISCARDED} };
// Access logs fields formats samples // Access logs fields formats samples
const std::unordered_map<std::string, QString> IIS_ALF_SAMPLES{ const samples_umap_t IIS_ALF_SAMPLES{
{"NONE", "<span style=\"color:#7f7f7f\">DISCARDED</span>"}, {_DISCARDED, R"(<span style="color:#7f7f7f">DISCARDED</span>)"},
{"date_time_ncsa", "<b><span style=\"color:#00cc6f\">01/Jan/2000:23:59:59 +0000</span></b>"}, {date_time_ncsa, R"(<b><span style="color:#00cc6f">01/Jan/2000:23:59:59 +0000</span></b>)"},
{"date_time_MDYYYY", "<b><span style=\"color:#00cc6f\">1/1/2000</span></b>"}, {date_time_mdyyyy, R"(<b><span style="color:#00cc6f">1/1/2000</span></b>)"},
{"date_time_utc_d", "<b><span style=\"color:#00cc6f\">2000-01-01</span></b>"}, {date_time_utc_d, R"(<b><span style="color:#00cc6f">2000-01-01</span></b>)"},
{"date_time_utc_t", "<b><span style=\"color:#00cc6f\">23:59:59</span></b>"}, {date_time_utc_t, R"(<b><span style="color:#00cc6f">23:59:59</span></b>)"},
{"request_full", "<b><span style=\"color:#00cc6f\">GET /index.php?query=x HTTP/1.1</span></b>"}, {request_full, R"(<b><span style="color:#00cc6f">GET /index.php?query=x HTTP/1.1</span></b>)"},
{"request_protocol", "<b><span style=\"color:#00cc6f\">HTTP/1.1</span></b>"}, {request_protocol, R"(<b><span style="color:#00cc6f">HTTP/1.1</span></b>)"},
{"request_method", "<b><span style=\"color:#00cc6f\">GET</span></b>"}, {request_method, R"(<b><span style="color:#00cc6f">GET</span></b>)"},
{"request_uri", "<b><span style=\"color:#00cc6f\">/index.php</span></b>"}, {request_uri, R"(<b><span style="color:#00cc6f">/index.php</span></b>)"},
{"request_query", "<b><span style=\"color:#00cc6f\">query=x</span></b>"}, {request_query, R"(<b><span style="color:#00cc6f">query=x</span></b>)"},
{"response_code", "<b><span style=\"color:#00cc6f\">404</span></b>"}, {response_code, R"(<b><span style="color:#00cc6f">404</span></b>)"},
{"bytes_sent", "<b><span style=\"color:#00cc6f\">1234</span></b>"}, {bytes_sent, R"(<b><span style="color:#00cc6f">1234</span></b>)"},
{"bytes_received", "<b><span style=\"color:#00cc6f\">123</span></b>"}, {bytes_received, R"(<b><span style="color:#00cc6f">123</span></b>)"},
{"time_taken_ms", "<b><span style=\"color:#00cc6f\">1000</span></b>"}, {time_taken_ms, R"(<b><span style="color:#00cc6f">1000</span></b>)"},
{"referer", "<b><span style=\"color:#00cc6f\">http://www.referrer.site</span></b>"}, {referer, R"(<b><span style="color:#00cc6f">http://www.referrer.site</span></b>)"},
{"cookie", "<b><span style=\"color:#00cc6f\">aCookie=abc123</span></b>"}, {cookie, R"(<b><span style="color:#00cc6f">aCookie=abc123</span></b>)"},
{"user_agent", "<b><span style=\"color:#00cc6f\">UserAgent/3.0+(Details+stuff)+Info/123</span></b>"}, {user_agent, R"(<b><span style="color:#00cc6f">UserAgent/3.0+(Details+stuff)+Info/123</span></b>)"},
{"client", "<b><span style=\"color:#00cc6f\">192.168.1.123</span></b>"} }; {client, R"(<b><span style="color:#00cc6f">192.168.1.123</span></b>)"} };
}; };

41
logdoctor/modules/craplog/modules/lib.h
View File

@ -36,30 +36,31 @@ struct LogFile final
LogFile& operator=(LogFile&& other) noexcept = default; LogFile& operator=(LogFile&& other) noexcept = default;
LogFile(const LogFile& other) noexcept = default; LogFile(const LogFile& other) noexcept = default;
LogFile& operator=(const LogFile& other) noexcept = default; LogFile& operator=(const LogFile& other) noexcept = default;
//! Wheter the file has been selected to be used or not
//! Returns whether the file has been selected to be used or not
inline bool isSelected() const noexcept inline bool isSelected() const noexcept
{ return this->selected; } { return this->selected; }
//! Sets the file as selected //! Sets the file as selected
inline void setSelected() noexcept inline void setSelected() noexcept
{ this->selected |= true; } { this->selected |= true; }
//! Sets the file as unselected //! Sets the file as unselected
inline void setUnselected() noexcept inline void setUnselected() noexcept
{ this->selected &= false; } { this->selected &= false; }
//! Wheter the file has been used already or not //! Returns whether the file has been used already or not
inline bool hasBeenUsed() const noexcept inline bool hasBeenUsed() const noexcept
{ return this->used_already; } { return this->used_already; }
//! The size of the file //! Returns the size of the file
inline size_t size() const noexcept inline size_t size() const noexcept
{ return this->size_; } { return this->size_; }
//! The name of the file, to be displayed in the list //! Returns the name of the file, to be displayed in the list
inline const QString& name() const noexcept inline const QString& name() const noexcept
{ return this->name_; } { return this->name_; }
//! The sha256 hash of the content //! Returns the sha256 hash of the content
inline const std::string& hash() const noexcept inline const std::string& hash() const noexcept
{ return this->hash_; } { return this->hash_; }
//! The path of the file, including the file name //! Returns the path of the file, including the file name
inline const std::string& path() const noexcept inline const std::string& path() const noexcept
{ return this->path_; } { return this->path_; }
private: private:
bool selected; bool selected;
bool used_already; bool used_already;
@ -149,16 +150,16 @@ enum LogsFormatField : uint32_t {
//! Holds informations about a log format //! Holds informations about a log format
struct LogsFormat final struct LogsFormat final
{ {
explicit LogsFormat() noexcept = default;
explicit LogsFormat
(const std::string& str,std::string&& itl,std::string&& fnl,std::vector<std::string>&& seps,std::vector<LogsFormatField>&& flds,const unsigned nl) noexcept
:string{str},initial{std::move(itl)},final{std::move(fnl)},separators{std::move(seps)},fields{std::move(flds)},new_lines{nl}{}
std::string string; //!< The logs format string std::string string; //!< The logs format string
std::string initial; //!< The initial separator std::string initial; //!< The initial separator
std::string final; //!< The final separator std::string final; //!< The final separator
std::vector<std::string> separators; //!< The separators in the middle std::vector<std::string> separators; //!< The separators in the middle
std::vector<LogsFormatField> fields; //!< The fields std::vector<LogsFormatField> fields; //!< The logged fields
size_t new_lines; //!< The number of new lines size_t new_lines; //!< The number of new lines within the string
explicit LogsFormat() noexcept = default;
explicit LogsFormat(const std::string& str,std::string&& itl,std::string&& fnl,std::vector<std::string>&& seps,std::vector<LogsFormatField>&& flds,const size_t nl) noexcept
:string{str},initial{std::move(itl)},final{std::move(fnl)},separators{std::move(seps)},fields{std::move(flds)},new_lines{nl}{}
}; };

152
logdoctor/modules/craplog/modules/workers/impl/loglinedata.cpp
View File

@ -13,7 +13,8 @@
LogLineData::LogLineData(const std::string& line, const LogsFormat& logs_format) LogLineData::LogLineData(const std::string& line, const LogsFormat& logs_format)
{ {
bool add_pm{false}; using F = LogsFormatField;
size_t start, stop{logs_format.initial.size()}, size_t start, stop{logs_format.initial.size()},
sep_i{0}; sep_i{0};
const size_t line_size{ line.size()-1ul }, const size_t line_size{ line.size()-1ul },
@ -46,11 +47,11 @@ LogLineData::LogLineData(const std::string& line, const LogsFormat& logs_format)
// should be unreachable // should be unreachable
throw ("Unexpected section reached"); throw ("Unexpected section reached");
} }
const size_t sep_size = sep.size(); const size_t sep_size{ sep.size() };
// get the field // get the field
const std::string& fld = logs_format.fields.at( sep_i ); const F fld{ logs_format.fields.at( sep_i ) };
if ( fld != "NONE" ) { if ( _DISCARDED | fld ) {
// only parse the considered fields // only parse the considered fields
std::string fld_str{ line.substr(start, stop-start) }; std::string fld_str{ line.substr(start, stop-start) };
@ -59,34 +60,30 @@ LogLineData::LogLineData(const std::string& line, const LogsFormat& logs_format)
// not the last separator, check for mistakes // not the last separator, check for mistakes
size_t aux_stop = stop; size_t aux_stop = stop;
if ( sep == " " ) { if ( (_MAY_HAVE_SPACES & fld) && sep == " " ) {
// check the fields with whitespace-separated values // check the fields with whitespace-separated values
const size_t n{ fld == "request_full" ? 2ul const size_t n{ fld == request_full ? 2ul
: fld == "date_time_ncsa" ? 1ul : fld & _COUNT_SPACES };
: fld == "date_time_mcs" ? 4ul
: fld == "date_time_gmt" ? 3ul size_t c{ StringOps::count( fld_str, ' ' ) };
: 0ul }; if ( c < n ) {
if ( n > 0ul ) { // loop until the correct number of whitespaces is reached
size_t c{ StringOps::count( fld_str, ' ' ) }; size_t aux_start = line[stop+1ul] == ' ' ? stop : stop+1ul;
if ( c < n ) { while ( c < n ) {
// loop until the correct number of whitespaces is reached aux_stop = line.find( sep, aux_start );
size_t aux_start = line[stop+1ul] == ' ' ? stop : stop+1ul; if ( aux_stop == std::string::npos ) {
while ( c < n ) { // not found
aux_stop = line.find( sep, aux_start ); throw LogParserException( "Separator not found", std::string{sep} );
if ( aux_stop == std::string::npos ) {
// not found
throw LogParserException( "Separator not found", std::string{sep} );
}
aux_start = aux_stop+1ul;
++c;
} }
} else if ( c > n ) [[unlikely]] { aux_start = aux_stop+1ul;
// should be unreachable ++c;
throw LogParserException( "Unexpected count for separator", std::string{sep} );
} }
} else if ( c > n ) [[unlikely]] {
// should be unreachable
throw LogParserException( "Unexpected count for separator", std::string{sep} );
} }
} else if ( sep.front() == '"' && fld == "user_agent" ) { } else if ( sep.front() == '"' && fld == F::user_agent ) {
// atm the only support is for escaped quotes // atm the only support is for escaped quotes
if ( fld_str.back() == '\\' ) { if ( fld_str.back() == '\\' ) {
// the found separator is not actually the separator but is part of the user-agent string // the found separator is not actually the separator but is part of the user-agent string
@ -115,20 +112,19 @@ LogLineData::LogLineData(const std::string& line, const LogsFormat& logs_format)
if ( ! fld_str.empty() ) { if ( ! fld_str.empty() ) {
// process the field // process the field
const int& fld_id{ this->field2id.at(fld) }; if ( _NO_PARSE_NEEDED & fld ) {
if ( fld_id > 0 ) {
// no need to process, append directly if non-empty // no need to process, append directly if non-empty
if ( fld_id == 13 && fld_str == "-" ) { if ( fld == request_query && fld_str == "-" ) {
continue; continue;
} }
this->data( fld_id ) = FieldData( std::move(fld_str) ); this->data( fld ) = FieldData( std::move(fld_str) );
} else { } else {
// process the field // process the field
// process the date to get year, month, day, hour and minute // process the date to get year, month, day, hour and minute
if ( StringOps::startsWith( fld, "date_time" ) ) { if ( _DATE_TIME & fld ) {
auto dt = DateTimeOps::processDateTime( fld_str, fld.substr( 10 ) ); // cut away the "date_time_" part auto dt = DateTimeOps::processDateTime( fld_str, fld ); // cut away the "date_time_" part
if ( auto& year{ dt.at(0) }; !year.empty() ) { if ( auto& year{ dt.at(0) }; !year.empty() ) {
// year // year
this->year = FieldData( std::move(year) ); this->year = FieldData( std::move(year) );
@ -143,11 +139,7 @@ LogLineData::LogLineData(const std::string& line, const LogsFormat& logs_format)
} }
if ( auto& hour{ dt.at(3) }; !hour.empty() ) { if ( auto& hour{ dt.at(3) }; !hour.empty() ) {
// hour // hour
if ( hour == "PM" ) { this->hour = FieldData( std::move(hour) );
add_pm |= true;
} else {
this->hour = FieldData( std::move(hour) );
}
} }
if ( auto& minute{ dt.at(4) }; !minute.empty() ) { if ( auto& minute{ dt.at(4) }; !minute.empty() ) {
// minute // minute
@ -159,8 +151,21 @@ LogLineData::LogLineData(const std::string& line, const LogsFormat& logs_format)
} }
// process the time taken to convert to milliseconds
} else if ( _TIME_TAKEN & fld ) {
float t{ std::stof( fld_str ) };
if ( fld == time_taken_us ) {
// from microseconds
t /= 1000.0f;
} else if ( fld & time_taken_s ) {
// from seconds or seconds.milliseconds
t *= 1000.0f;
}
this->time_taken = FieldData( std::to_string( static_cast<int>( t ) ) );
// process the request to get the protocol, method, resource and query // process the request to get the protocol, method, resource and query
} else if ( fld == "request_full" ) { } else if ( fld == request_full ) {
// check whether the request string has the proper number of spaces // check whether the request string has the proper number of spaces
const size_t n_spaces{ StringOps::count( fld_str, ' ' ) }; const size_t n_spaces{ StringOps::count( fld_str, ' ' ) };
@ -212,7 +217,7 @@ LogLineData::LogLineData(const std::string& line, const LogsFormat& logs_format)
// process the request to get uri and query // process the request to get uri and query
} else if ( fld == "request_uri_query" ) { } else if ( fld == request_uri_query ) {
// search for the query // search for the query
std::string uri, query; std::string uri, query;
const size_t aux_{ fld_str.find( '?' ) }; const size_t aux_{ fld_str.find( '?' ) };
@ -231,24 +236,10 @@ LogLineData::LogLineData(const std::string& line, const LogsFormat& logs_format)
} }
// process the time taken to convert to milliseconds
} else if ( fld.rfind("time_taken_",0ul) == 0ul ) {
float t{ std::stof( fld_str ) };
const std::string u{ fld.substr( 11ul ) };
if ( u == "us" ) {
// from microseconds
t /= 1000.0f;
} else if ( u == "s" || u == "s.ms" ) {
// from seconds
t *= 1000.0f;
}
this->time_taken = FieldData( std::to_string( static_cast<int>( t ) ) );
// something went wrong // something went wrong
} else { } else {
// hmmm.. no... // hmmm.. no...
throw LogParserException( "Unexpected LogField", fld ); throw LogParserException( "Unexpected LogFormatField", std::to_string(fld) );
} }
} }
} }
@ -263,15 +254,6 @@ LogLineData::LogLineData(const std::string& line, const LogsFormat& logs_format)
} }
} }
if ( add_pm ) {
try {
// add +12 hours for PM
this->hour = FieldData( std::to_string( 12 + std::stoi(*this->hour) ) );
} catch (...) {
// no hour data
}
}
} }
void LogLineData::storeUriQuery(std::string&& str) noexcept void LogLineData::storeUriQuery(std::string&& str) noexcept
@ -532,46 +514,48 @@ size_t LogLineData::size() const noexcept
+ this->cookie; + this->cookie;
} }
FieldData& LogLineData::data(const int& id) FieldData& LogLineData::data(const LogsFormatField id)
{ {
using F = LogsFormatField;
switch (id) { switch (id) {
case 1: case F::date_time_year:
return this->year; return this->year;
case 2: case F::date_time_month:
return this->month; return this->month;
case 3: case F::date_time_day:
return this->day; return this->day;
case 4: case F::date_time_hour:
return this->hour; return this->hour;
case 5: case F::date_time_minute:
return this->minute; return this->minute;
case 6: case F::date_time_second:
return this->second; return this->second;
case 10: case F::request_protocol:
return this->protocol; return this->protocol;
case 11: case F::request_method:
return this->method; return this->method;
case 12: case F::request_uri:
return this->uri; return this->uri;
case 13: case F::request_query:
return this->query; return this->query;
case 14: case F::response_code:
return this->response_code; return this->response_code;
case 15: case F::time_taken_s:
return this->time_taken; return this->time_taken;
case 16: case F::bytes_sent:
return this->bytes_sent; return this->bytes_sent;
case 17: case F::bytes_received:
return this->bytes_received; return this->bytes_received;
case 18: case F::referer:
return this->referrer; return this->referrer;
case 20: case F::client:
return this->client; return this->client;
case 21: case F::user_agent:
return this->user_agent; return this->user_agent;
case 22: case F::cookie:
return this->cookie; return this->cookie;
default: default:
throw LogParserException( "Unexpected LogField ID", std::to_string(id) ); throw LogParserException( "Unexpected LogFormatField", std::to_string(id) );
} }
} }

104
logdoctor/modules/craplog/modules/workers/lib.h
View File

@ -5,6 +5,8 @@
#include <QMetaType> #include <QMetaType>
enum LogsFormatField : uint32_t;
struct LogsFormat; struct LogsFormat;
@ -44,13 +46,17 @@ struct FieldData final
FieldData& operator=(FieldData&& rhs) noexcept = default; FieldData& operator=(FieldData&& rhs) noexcept = default;
Q_DISABLE_COPY(FieldData) Q_DISABLE_COPY(FieldData)
//! Returns whether the field contains data
inline operator bool() const noexcept inline operator bool() const noexcept
{ return this->is_set; } { return this->is_set; }
//! Returns a reference to the field's data
inline const std::string& operator *() const noexcept inline const std::string& operator *() const noexcept
{ return this->data; } { return this->data; }
//! Returns the sum of the field's data size with another field's data size
inline size_t operator +(const FieldData& rhs) const noexcept inline size_t operator +(const FieldData& rhs) const noexcept
{ return this->data.size() + rhs.data.size(); } { return this->data.size() + rhs.data.size(); }
//! Returns the sum of the field's data size with the given size
friend inline size_t operator +(const size_t lhs, const FieldData& rhs) noexcept;
private: private:
bool is_set; bool is_set;
@ -60,7 +66,7 @@ private:
inline size_t operator +(const size_t lhs, const FieldData& rhs) noexcept inline size_t operator +(const size_t lhs, const FieldData& rhs) noexcept
{ {
return lhs + (*rhs).size(); return lhs + rhs.data.size();
} }
@ -77,87 +83,41 @@ struct LogLineData final
LogLineData& operator=(LogLineData&& rhs) noexcept = delete; LogLineData& operator=(LogLineData&& rhs) noexcept = delete;
Q_DISABLE_COPY(LogLineData) Q_DISABLE_COPY(LogLineData)
//! Returns the total size of all the fields' data
size_t size() const noexcept; size_t size() const noexcept;
// date and time // date and time
FieldData year; // 1 FieldData year;
FieldData month; // 2 FieldData month;
FieldData day; // 3 FieldData day;
FieldData hour; // 4 FieldData hour;
FieldData minute; // 5 FieldData minute;
FieldData second; // 6 FieldData second;
// request // request
FieldData protocol; // 10 FieldData protocol;
FieldData method; // 11 FieldData method;
FieldData uri; // 12 FieldData uri;
FieldData query; // 13 FieldData query;
// server // server
FieldData response_code; // 14 FieldData response_code;
FieldData time_taken; // 15 FieldData time_taken;
FieldData bytes_sent; // 16 FieldData bytes_sent;
FieldData bytes_received; // 17 FieldData bytes_received;
// client // client
FieldData client; // 20 FieldData client;
FieldData cookie; // 22 FieldData cookie;
FieldData user_agent; // 21 FieldData user_agent;
FieldData referrer; // 18 FieldData referrer;
private: private:
FieldData& data(const int& id); //! Returns a reference to the field data corresponding to the given field identifier
FieldData& data(const LogsFormatField id);
void storeUriQuery(std::string&& str) noexcept; void storeUriQuery(std::string&& str) noexcept;
void storeMalformedRequestOneSpace(std::string&& str) noexcept; void storeMalformedRequestOneSpace(std::string&& str) noexcept;
void storeMalformedRequestMultiSpace(std::string&& str) noexcept; void storeMalformedRequestMultiSpace(std::string&& str) noexcept;
inline static const std::unordered_map<std::string, int> field2id{
// date-time
{"date_time_year", 1},
{"date_time_month", 2},
{"date_time_day", 3},
{"date_time_hour", 4},
{"date_time_minute", 5},
{"date_time_second", 6},
{"date_time_ncsa", 0},
{"date_time_iso", 0},
{"date_time_mcs", 0},
{"date_time_gmt", 0},
{"date_time_utc_d", 0},
{"date_time_utc_t", 0},
{"date_time_epoch_s", 0},
{"date_time_epoch_s.ms", 0},
{"date_time_epoch_ms", 0},
{"date_time_epoch_us", 0},
{"date_time_YYYYMMDD", 0},
{"date_time_MMDDYY", 0},
{"date_time_MDYYYY", 0},
{"date_time_year_short", 0},
{"date_time_month_str", 0},
{"date_time_clock_12", 0},
{"date_time_clock_24", 0},
{"date_time_clock_short", 0},
// request
{"request_protocol", 10},
{"request_method", 11},
{"request_uri", 12},
{"request_query", 13},
{"response_code", 14},
{"request_full", 0},
// performance
{"time_taken_ms", 15},
{"time_taken_us", 0},
{"time_taken_s.ms", 0},
{"time_taken_s", 0},
{"bytes_sent", 16},
{"bytes_received", 17},
// referer
{"referer", 18},
// client data
{"client", 20},
{"user_agent", 21},
{"cookie", 22}
};
inline static const std::vector<std::string> valid_methods{ inline static const std::vector<std::string> valid_methods{
"GET", "GET",
"POST", "POST",

96
logdoctor/tests/white_box.cpp
View File

@ -392,39 +392,39 @@ void testCraplogModules()
const std::string e_str{ std::to_string( e ) }; const std::string e_str{ std::to_string( e ) };
const std::string epochs[4]{ e_str, e_str+".000", e_str+"000", e_str+"000000" }; const std::string epochs[4]{ e_str, e_str+".000", e_str+"000", e_str+"000000" };
std::vector<std::string> target{"2000","01","01","23","59","59"}; std::vector<std::string> target{"2000","01","01","23","59","59"};
assert( DateTimeOps::processDateTime("[01/Jan/2000:23:59:59 +0000]", "ncsa") == target ); assert( DateTimeOps::processDateTime("[01/Jan/2000:23:59:59 +0000]", date_time_ncsa) == target );
assert( DateTimeOps::processDateTime("Sat Jan 01 23:59:59 2000", "mcs") == target ); assert( DateTimeOps::processDateTime("Sat Jan 01 23:59:59 2000", date_time_mcs) == target );
assert( DateTimeOps::processDateTime("Saturday, 01-Jan-2000 23:59:59 UTC", "gmt") == target ); assert( DateTimeOps::processDateTime("Saturday, 01-Jan-2000 23:59:59 UTC", date_time_gmt) == target );
assert( DateTimeOps::processDateTime("2000-01-01T23:59:59+00:00", "iso") == target ); assert( DateTimeOps::processDateTime("2000-01-01T23:59:59+00:00", date_time_iso) == target );
assert( DateTimeOps::processDateTime(epochs[0], "epoch_s") == target ); assert( DateTimeOps::processDateTime(epochs[0], date_time_epoch_s) == target );
assert( DateTimeOps::processDateTime(epochs[1], "epoch_s.ms") == target ); assert( DateTimeOps::processDateTime(epochs[1], date_time_epoch_s_ms) == target );
assert( DateTimeOps::processDateTime(epochs[2], "epoch_ms") == target ); assert( DateTimeOps::processDateTime(epochs[2], date_time_epoch_ms) == target );
assert( DateTimeOps::processDateTime(epochs[3], "epoch_us") == target ); assert( DateTimeOps::processDateTime(epochs[3], date_time_epoch_us) == target );
target = {"2000","01","01","","",""}; target = {"2000","01","01","","",""};
assert( DateTimeOps::processDateTime("2000-01-01", "utc_d") == target ); assert( DateTimeOps::processDateTime("2000-01-01", date_time_utc_d) == target );
assert( DateTimeOps::processDateTime("2000-01-01", "YYYYMMDD") == target ); assert( DateTimeOps::processDateTime("2000-01-01", date_time_yyyymmdd) == target );
assert( DateTimeOps::processDateTime("01/01/00", "MMDDYY") == target ); assert( DateTimeOps::processDateTime("01/01/00", date_time_mmddyy) == target );
assert( DateTimeOps::processDateTime("1/1/2000", "MDYYYY") == target ); assert( DateTimeOps::processDateTime("1/1/2000", date_time_mdyyyy) == target );
target = {"","","","23","59","59"}; target = {"","","","23","59","59"};
assert( DateTimeOps::processDateTime("23:59:59", "utc_t") == target ); assert( DateTimeOps::processDateTime("23:59:59", date_time_utc_t) == target );
assert( DateTimeOps::processDateTime("11:59:59 pm", "clock_12") == target ); assert( DateTimeOps::processDateTime("11:59:59 pm", date_time_clock_12) == target );
assert( DateTimeOps::processDateTime("23:59:59", "clock_24") == target ); assert( DateTimeOps::processDateTime("23:59:59", date_time_clock_24) == target );
target = {"","","","23","59",""}; target = {"","","","23","59",""};
assert( DateTimeOps::processDateTime("23:59", "clock_short") == target ); assert( DateTimeOps::processDateTime("23:59", date_time_clock_short) == target );
target = {"","","","PM","",""}; /*target = {"","","","PM","",""};
assert( DateTimeOps::processDateTime("pm", "clock_meridian") == target ); assert( DateTimeOps::processDateTime("pm", date_time_clock_meridian) == target );*/
target = {"2000","","","","",""}; target = {"2000","","","","",""};
assert( DateTimeOps::processDateTime("2000", "year") == target ); assert( DateTimeOps::processDateTime("2000", date_time_year) == target );
target = {"","01","","","",""}; target = {"","01","","","",""};
assert( DateTimeOps::processDateTime("01", "month") == target ); assert( DateTimeOps::processDateTime("01", date_time_month) == target );
target = {"","","01","","",""}; target = {"","","01","","",""};
assert( DateTimeOps::processDateTime("01", "day") == target ); assert( DateTimeOps::processDateTime("01", date_time_day) == target );
target = {"","","","23","",""}; target = {"","","","23","",""};
assert( DateTimeOps::processDateTime("23", "hour") == target ); assert( DateTimeOps::processDateTime("23", date_time_hour) == target );
target = {"","","","","59",""}; target = {"","","","","59",""};
assert( DateTimeOps::processDateTime("59", "minute") == target ); assert( DateTimeOps::processDateTime("59", date_time_minute) == target );
target = {"","","","","","59"}; target = {"","","","","","59"};
assert( DateTimeOps::processDateTime("59", "second") == target ); assert( DateTimeOps::processDateTime("59", date_time_second) == target );
} }
T_PRINT("DateTimeOps::processDateTime"); T_PRINT("DateTimeOps::processDateTime");
@ -435,11 +435,11 @@ void testCraplogModules()
FormatOps fo; FormatOps fo;
LogsFormat lf; LogsFormat lf;
std::string format_string; std::string format_string;
std::vector<std::string> fields; std::vector<LogsFormatField> fields;
std::vector<std::string> separators; std::vector<std::string> separators;
// test the default string // test the default string
format_string = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\""; format_string = "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"";
fields = {"client","NONE","NONE","date_time_ncsa","request_full","response_code","NONE","referer","user_agent"}; fields = {client,_DISCARDED,_DISCARDED,date_time_ncsa,request_full,response_code,_DISCARDED,referer,user_agent};
separators = {" "," "," [","] \"","\" "," "," \"","\" \""}; separators = {" "," "," [","] \"","\" "," "," \"","\" \""};
lf = fo.processApacheFormatString(format_string); lf = fo.processApacheFormatString(format_string);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -448,7 +448,7 @@ void testCraplogModules()
assert( lf.final == "\"" ); assert( lf.final == "\"" );
// test all simple fields // test all simple fields
format_string = "%%%h %% %t\t%r\n%H %m [%U%%%q} <%s> %<s %>s %O %I %T %D %a %A %b %B %e %f %k %l %L %p %P %R %S %u %v %V %% %X%%"; format_string = "%%%h %% %t\t%r\n%H %m [%U%%%q} <%s> %<s %>s %O %I %T %D %a %A %b %B %e %f %k %l %L %p %P %R %S %u %v %V %% %X%%";
fields = {"client","date_time_ncsa","request_full","request_protocol","request_method","request_uri","request_query","response_code","response_code","response_code","bytes_sent","bytes_received","time_taken_s","time_taken_ms","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE"}; fields = {client,date_time_ncsa,request_full,request_protocol,request_method,request_uri,request_query,response_code,response_code,response_code,bytes_sent,bytes_received,time_taken_s,time_taken_ms,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED};
separators = {" % [","]\t","\n"," "," [","%","} <","> "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," % "}; separators = {" % [","]\t","\n"," "," [","%","} <","> "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," % "};
lf = fo.processApacheFormatString(format_string); lf = fo.processApacheFormatString(format_string);
assert( lf.initial == "%" ); assert( lf.initial == "%" );
@ -470,7 +470,7 @@ void testCraplogModules()
} }
// test client related composed fields // test client related composed fields
format_string = "%{}a %{c}a %{}h %{c}h %{Cookie}i %200{Cookie}i %{User-agent}i %302,400{User-agent}i %!200{Referer}i %,200{Referer}i %{Referer}i"; format_string = "%{}a %{c}a %{}h %{c}h %{Cookie}i %200{Cookie}i %{User-agent}i %302,400{User-agent}i %!200{Referer}i %,200{Referer}i %{Referer}i";
fields = {"client","client","client","client","cookie","cookie","user_agent","user_agent","referer","referer","referer"}; fields = {client,client,client,client,cookie,cookie,user_agent,user_agent,referer,referer,referer};
separators = {" "," "," "," "," "," "," "," "," "," "}; separators = {" "," "," "," "," "," "," "," "," "," "};
lf = fo.processApacheFormatString(format_string); lf = fo.processApacheFormatString(format_string);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -479,7 +479,7 @@ void testCraplogModules()
assert( lf.final.empty() ); assert( lf.final.empty() );
// test unexisting/unsupported client related composed fields // test unexisting/unsupported client related composed fields
format_string = "%{ }a %{x}a %{NOPE}a %{ }h %{y}h %{NOPE}h %{}i %{ }i %{Random}i %{Cookies}i"; format_string = "%{ }a %{x}a %{NOPE}a %{ }h %{y}h %{NOPE}h %{}i %{ }i %{Random}i %{Cookies}i";
fields = {"client","client","client","client","client","client","NONE","NONE","NONE","NONE"}; fields = {client,client,client,client,client,client,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED};
separators = {" "," "," "," "," "," "," "," "," "}; separators = {" "," "," "," "," "," "," "," "," "};
lf = fo.processApacheFormatString(format_string); lf = fo.processApacheFormatString(format_string);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -488,7 +488,7 @@ void testCraplogModules()
assert( lf.final.empty() ); assert( lf.final.empty() );
// test date-time composed fields // test date-time composed fields
format_string = "%{%%}t %{%n}t %{%t}t %{}t %{sec}t %{msec}t %{usec}t %{msec_frac}t %{usec_frac}t %{%a}t %{%A}t %{%b}t %{%B}t %{%c}t %{%C}t %{%d}t %{%D}t %{%e}t %{%F}t %{%g}t %{%G}t %{%h}t %{%H}t %{%I}t %{%j}t %{%k}t %{%m}t %{%M}t %{%p}t %{%r}t %{%R}t %{%S}t %{%T}t %{%u}t %{%U}t %{%V}t %{%w}t %{%W}t %{%x}t %{%X}t %{%y}t %{%Y}t %{%z}t %{%Z}t"; format_string = "%{%%}t %{%n}t %{%t}t %{}t %{sec}t %{msec}t %{usec}t %{msec_frac}t %{usec_frac}t %{%a}t %{%A}t %{%b}t %{%B}t %{%c}t %{%C}t %{%d}t %{%D}t %{%e}t %{%F}t %{%g}t %{%G}t %{%h}t %{%H}t %{%I}t %{%j}t %{%k}t %{%m}t %{%M}t %{%p}t %{%r}t %{%R}t %{%S}t %{%T}t %{%u}t %{%U}t %{%V}t %{%w}t %{%W}t %{%x}t %{%X}t %{%y}t %{%Y}t %{%z}t %{%Z}t";
fields = {"date_time_ncsa","date_time_epoch_s","date_time_epoch_ms","date_time_epoch_us","NONE","NONE","NONE","NONE","date_time_month_str","date_time_month_str","date_time_mcs","NONE","date_time_day","date_time_MMDDYY","date_time_day","date_time_YYYYMMDD","NONE","NONE","date_time_month_str","date_time_hour","NONE","NONE","date_time_hour","date_time_month","date_time_minute","NONE","date_time_clock_12","date_time_clock_short","date_time_second","date_time_clock_24","NONE","NONE","NONE","NONE","NONE","date_time_MMDDYY","date_time_clock_24","date_time_year_short","date_time_year","NONE","NONE"}; fields = {date_time_ncsa,date_time_epoch_s,date_time_epoch_ms,date_time_epoch_us,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,date_time_month_str,date_time_month_str,date_time_mcs,_DISCARDED,date_time_day,date_time_mmddyy,date_time_day,date_time_yyyymmdd,_DISCARDED,_DISCARDED,date_time_month_str,date_time_hour,_DISCARDED,_DISCARDED,date_time_hour,date_time_month,date_time_minute,_DISCARDED,date_time_clock_12,date_time_clock_short,date_time_second,date_time_clock_24,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,date_time_mmddyy,date_time_clock_24,date_time_year_short,date_time_year,_DISCARDED,_DISCARDED};
separators = {"] "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "}; separators = {"] "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "};
lf = fo.processApacheFormatString(format_string); lf = fo.processApacheFormatString(format_string);
assert( lf.initial == "% \n \t [" ); assert( lf.initial == "% \n \t [" );
@ -497,7 +497,7 @@ void testCraplogModules()
assert( lf.final.empty() ); assert( lf.final.empty() );
// test date-time composed fields, with one field only // test date-time composed fields, with one field only
format_string = "%{}t"; format_string = "%{}t";
fields = {"date_time_ncsa"}; fields = {date_time_ncsa};
lf = fo.processApacheFormatString(format_string); lf = fo.processApacheFormatString(format_string);
assert( lf.initial == "[" ); assert( lf.initial == "[" );
assert( lf.fields == fields ); assert( lf.fields == fields );
@ -505,7 +505,7 @@ void testCraplogModules()
assert( lf.final == "]" ); assert( lf.final == "]" );
// test date-time composed fields, with many aggreagated fields // test date-time composed fields, with many aggreagated fields
format_string = "%{%%%Y_%m_%e%t%H@%M@%S%%}t"; format_string = "%{%%%Y_%m_%e%t%H@%M@%S%%}t";
fields = {"date_time_year","date_time_month","date_time_day","date_time_hour","date_time_minute","date_time_second"}; fields = {date_time_year,date_time_month,date_time_day,date_time_hour,date_time_minute,date_time_second};
separators = {"_","_","\t","@","@"}; separators = {"_","_","\t","@","@"};
lf = fo.processApacheFormatString(format_string); lf = fo.processApacheFormatString(format_string);
assert( lf.initial == "%" ); assert( lf.initial == "%" );
@ -514,7 +514,7 @@ void testCraplogModules()
assert( lf.final == "%" ); assert( lf.final == "%" );
// test date-time composed fields, with all fields aggeregated in one // test date-time composed fields, with all fields aggeregated in one
format_string = "%{%% %n %t %a %A %b %B %c %C %d %D %e %F %g %G %h %H %I %j %k %m %M %p %r %R %S %T %u %U %V %w %W %x %X %y %Y %z %Z}t"; format_string = "%{%% %n %t %a %A %b %B %c %C %d %D %e %F %g %G %h %H %I %j %k %m %M %p %r %R %S %T %u %U %V %w %W %x %X %y %Y %z %Z}t";
fields = {"NONE","NONE","date_time_month_str","date_time_month_str","date_time_mcs","NONE","date_time_day","date_time_MMDDYY","date_time_day","date_time_YYYYMMDD","NONE","NONE","date_time_month_str","date_time_hour","NONE","NONE","date_time_hour","date_time_month","date_time_minute","NONE","date_time_clock_12","date_time_clock_short","date_time_second","date_time_clock_24","NONE","NONE","NONE","NONE","NONE","date_time_MMDDYY","date_time_clock_24","date_time_year_short","date_time_year","NONE","NONE"}; fields = {_DISCARDED,_DISCARDED,date_time_month_str,date_time_month_str,date_time_mcs,_DISCARDED,date_time_day,date_time_mmddyy,date_time_day,date_time_yyyymmdd,_DISCARDED,_DISCARDED,date_time_month_str,date_time_hour,_DISCARDED,_DISCARDED,date_time_hour,date_time_month,date_time_minute,_DISCARDED,date_time_clock_12,date_time_clock_short,date_time_second,date_time_clock_24,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,date_time_mmddyy,date_time_clock_24,date_time_year_short,date_time_year,_DISCARDED,_DISCARDED};
separators = {" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "}; separators = {" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "};
lf = fo.processApacheFormatString(format_string); lf = fo.processApacheFormatString(format_string);
assert( lf.initial == "% \n \t " ); assert( lf.initial == "% \n \t " );
@ -543,7 +543,7 @@ void testCraplogModules()
} }
// test time taken related composed fields // test time taken related composed fields
format_string = "%{}T %{s}T %{ms}T %{us}T"; format_string = "%{}T %{s}T %{ms}T %{us}T";
fields = {"time_taken_s","time_taken_s","time_taken_ms","time_taken_us"}; fields = {time_taken_s,time_taken_s,time_taken_ms,time_taken_us};
separators = {" "," "," "}; separators = {" "," "," "};
lf = fo.processApacheFormatString(format_string); lf = fo.processApacheFormatString(format_string);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -560,7 +560,7 @@ void testCraplogModules()
assert( lf.final.empty() ); assert( lf.final.empty() );
// test unused composed fields // test unused composed fields
format_string = "%{}C %{}e %{}L %{}n %{}o %{}p %{canonical}p %{local}p %{remote}p %{}P %{pid}P %{tid}P %{hextid}P %{}^ti %{}^to"; format_string = "%{}C %{}e %{}L %{}n %{}o %{}p %{canonical}p %{local}p %{remote}p %{}P %{pid}P %{tid}P %{hextid}P %{}^ti %{}^to";
fields = {"NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE"}; fields = {_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED};
separators = {" "," "," "," "," "," "," "," "," "," "," "," "," "," "}; separators = {" "," "," "," "," "," "," "," "," "," "," "," "," "," "};
lf = fo.processApacheFormatString(format_string); lf = fo.processApacheFormatString(format_string);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -569,7 +569,7 @@ void testCraplogModules()
assert( lf.final.empty() ); assert( lf.final.empty() );
// test unused composed fields, with random content // test unused composed fields, with random content
format_string = "%{TEST}C %{TEST}e %{TEST}L %{TEST}n %{TEST}o %{TEST}p %{TEST}P %{TEST}^ti %{TEST}^to"; format_string = "%{TEST}C %{TEST}e %{TEST}L %{TEST}n %{TEST}o %{TEST}p %{TEST}P %{TEST}^ti %{TEST}^to";
fields = {"NONE","NONE","NONE","NONE","NONE","NONE","NONE"}; fields = {_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED};
separators = {" "," "," "," "," TEST TEST "," "}; separators = {" "," "," "," "," TEST TEST "," "};
lf = fo.processApacheFormatString(format_string); lf = fo.processApacheFormatString(format_string);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -578,7 +578,7 @@ void testCraplogModules()
assert( lf.final.empty() ); assert( lf.final.empty() );
// test the default string with dumb logging: no characters to enclose the full request // test the default string with dumb logging: no characters to enclose the full request
format_string = "%h %l %u %t %r %>s %b \"%{Referer}i\" \"%{User-agent}i\""; format_string = "%h %l %u %t %r %>s %b \"%{Referer}i\" \"%{User-agent}i\"";
fields = {"client","NONE","NONE","date_time_ncsa","request_full","response_code","NONE","referer","user_agent"}; fields = {client,_DISCARDED,_DISCARDED,date_time_ncsa,request_full,response_code,_DISCARDED,referer,user_agent};
separators = {" "," "," [","] "," "," "," \"","\" \""}; separators = {" "," "," [","] "," "," "," \"","\" \""};
lf = fo.processApacheFormatString(format_string); lf = fo.processApacheFormatString(format_string);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -599,11 +599,11 @@ void testCraplogModules()
FormatOps fo; FormatOps fo;
LogsFormat lf; LogsFormat lf;
std::string format_string; std::string format_string;
std::vector<std::string> fields; std::vector<LogsFormatField> fields;
std::vector<std::string> separators; std::vector<std::string> separators;
// test the default string // test the default string
format_string = "$remote_addr - $remote_user [$time_local] \"$request\" $status $bytes_sent \"$http_referer\" \"$http_user_agent\""; format_string = "$remote_addr - $remote_user [$time_local] \"$request\" $status $bytes_sent \"$http_referer\" \"$http_user_agent\"";
fields = {"client","NONE","date_time_ncsa","request_full","response_code","bytes_sent","referer","user_agent"}; fields = {client,_DISCARDED,date_time_ncsa,request_full,response_code,bytes_sent,referer,user_agent};
separators = {" - "," [","] \"","\" "," "," \"","\" \""}; separators = {" - "," [","] \"","\" "," "," \"","\" \""};
lf = fo.processNginxFormatString(format_string); lf = fo.processNginxFormatString(format_string);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -612,7 +612,7 @@ void testCraplogModules()
assert( lf.final == "\"" ); assert( lf.final == "\"" );
// test all the considered fields // test all the considered fields
format_string = "$remote_addr $realip_remote_addr $time_local $time_iso8601 $date_gmt $msec $request $server_protocol $request_method $request_uri $uri $query_string $status $bytes_sent $request_length $request_time $http_referer $cookie_ $http_user_agent"; format_string = "$remote_addr $realip_remote_addr $time_local $time_iso8601 $date_gmt $msec $request $server_protocol $request_method $request_uri $uri $query_string $status $bytes_sent $request_length $request_time $http_referer $cookie_ $http_user_agent";
fields = {"client","client","date_time_ncsa","date_time_iso","date_time_gmt","date_time_epoch_s.ms","request_full","request_protocol","request_method","request_uri_query","request_uri","request_query","response_code","bytes_sent","bytes_received","time_taken_s.ms","referer","cookie","user_agent"}; fields = {client,client,date_time_ncsa,date_time_iso,date_time_gmt,date_time_epoch_s_ms,request_full,request_protocol,request_method,request_uri_query,request_uri,request_query,response_code,bytes_sent,bytes_received,time_taken_s_ms,referer,cookie,user_agent};
separators = {" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "}; separators = {" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "};
lf = fo.processNginxFormatString(format_string); lf = fo.processNginxFormatString(format_string);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -621,7 +621,7 @@ void testCraplogModules()
assert( lf.final.empty() ); assert( lf.final.empty() );
// test all the non-considered fields // test all the non-considered fields
format_string = "$ancient_browser $arg_ $args $binary_remote_addr $body_bytes_sent $connection $connection_requests $connections_active $connections_reading $connections_waiting $connections_writing $content_length $content_type $date_local $document_root $document_uri $fastcgi_path_info $fastcgi_script_name $geoip_area_code $geoip_city $geoip_city_continent_code $geoip_city_country_code $geoip_city_country_code3 $geoip_city_country_name $geoip_country_code $geoip_country_code3 $geoip_country_name $geoip_dma_code $geoip_latitude $geoip_longitude $geoip_org $geoip_postal_code $geoip_region $geoip_region_name $gzip_ratio $host $hostname $http2 $http_ $https $invalid_referer $is_args $limit_rate $memcached_key $modern_browser $msie $nginx_version $pid $pipe $proxy_add_x_forwarded_for $proxy_host $proxy_port $proxy_protocol_addr $proxy_protocol_port $realip_remote_port $realpath_root $remote_port $remote_user $request_body $request_body_file $request_completion $request_filename $request_id $scheme $secure_link $secure_link_expires $sent_http_ $server_addr $server_name $server_port $session_log_binary_id $session_log_id $slice_range $spdy $spdy_request_priority $ssl_cipher $ssl_client_cert $ssl_client_fingerprint $ssl_client_i_dn $ssl_client_raw_cert $ssl_client_s_dn $ssl_client_serial $ssl_client_verify $ssl_protocol $ssl_server_name $ssl_session_id $ssl_session_reused $tcpinfo_rtt $tcpinfo_rttvar $tcpinfo_snd_cwnd $tcpinfo_rcv_space $uid_got $uid_reset $uid_set $upstream_addr $upstream_cache_status $upstream_connect_time $upstream_cookie_ $upstream_header_time $upstream_http_ $upstream_response_length $upstream_response_time $upstream_status"; format_string = "$ancient_browser $arg_ $args $binary_remote_addr $body_bytes_sent $connection $connection_requests $connections_active $connections_reading $connections_waiting $connections_writing $content_length $content_type $date_local $document_root $document_uri $fastcgi_path_info $fastcgi_script_name $geoip_area_code $geoip_city $geoip_city_continent_code $geoip_city_country_code $geoip_city_country_code3 $geoip_city_country_name $geoip_country_code $geoip_country_code3 $geoip_country_name $geoip_dma_code $geoip_latitude $geoip_longitude $geoip_org $geoip_postal_code $geoip_region $geoip_region_name $gzip_ratio $host $hostname $http2 $http_ $https $invalid_referer $is_args $limit_rate $memcached_key $modern_browser $msie $nginx_version $pid $pipe $proxy_add_x_forwarded_for $proxy_host $proxy_port $proxy_protocol_addr $proxy_protocol_port $realip_remote_port $realpath_root $remote_port $remote_user $request_body $request_body_file $request_completion $request_filename $request_id $scheme $secure_link $secure_link_expires $sent_http_ $server_addr $server_name $server_port $session_log_binary_id $session_log_id $slice_range $spdy $spdy_request_priority $ssl_cipher $ssl_client_cert $ssl_client_fingerprint $ssl_client_i_dn $ssl_client_raw_cert $ssl_client_s_dn $ssl_client_serial $ssl_client_verify $ssl_protocol $ssl_server_name $ssl_session_id $ssl_session_reused $tcpinfo_rtt $tcpinfo_rttvar $tcpinfo_snd_cwnd $tcpinfo_rcv_space $uid_got $uid_reset $uid_set $upstream_addr $upstream_cache_status $upstream_connect_time $upstream_cookie_ $upstream_header_time $upstream_http_ $upstream_response_length $upstream_response_time $upstream_status";
fields = {"NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE"}; fields = {_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED};
separators = {" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "}; separators = {" "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "," "};
lf = fo.processNginxFormatString(format_string); lf = fo.processNginxFormatString(format_string);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -642,11 +642,11 @@ void testCraplogModules()
FormatOps fo; FormatOps fo;
LogsFormat lf; LogsFormat lf;
std::string format_string; std::string format_string;
std::vector<std::string> fields; std::vector<LogsFormatField> fields;
std::vector<std::string> separators; std::vector<std::string> separators;
// test the default string for the W3C module // test the default string for the W3C module
format_string = "date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) cs(Referer) sc-status sc-substatus sc-win32-status time-taken"; format_string = "date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs(User-Agent) cs(Referer) sc-status sc-substatus sc-win32-status time-taken";
fields = {"date_time_utc_d","date_time_utc_t","NONE","request_method","request_uri","request_query","NONE","NONE","client","user_agent","referer","response_code","NONE","NONE","time_taken_ms"}; fields = {date_time_utc_d,date_time_utc_t,_DISCARDED,request_method,request_uri,request_query,_DISCARDED,_DISCARDED,client,user_agent,referer,response_code,_DISCARDED,_DISCARDED,time_taken_ms};
separators = {" "," "," "," "," "," "," "," "," "," "," "," "," "," "}; separators = {" "," "," "," "," "," "," "," "," "," "," "," "," "," "};
lf = fo.processIisFormatString(format_string, 0); lf = fo.processIisFormatString(format_string, 0);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -655,7 +655,7 @@ void testCraplogModules()
assert( lf.final.empty() ); assert( lf.final.empty() );
// test all the considered fields for the W3C module // test all the considered fields for the W3C module
format_string = "date time cs-version cs-method cs-uri-stem cs-uri-query sc-status sc-bytes cs-bytes time-taken cs(Referer) cs(Cookie) cs(User-Agent) c-ip"; format_string = "date time cs-version cs-method cs-uri-stem cs-uri-query sc-status sc-bytes cs-bytes time-taken cs(Referer) cs(Cookie) cs(User-Agent) c-ip";
fields = {"date_time_utc_d","date_time_utc_t","request_protocol","request_method","request_uri","request_query","response_code","bytes_sent","bytes_received","time_taken_ms","referer","cookie","user_agent","client"}; fields = {date_time_utc_d,date_time_utc_t,request_protocol,request_method,request_uri,request_query,response_code,bytes_sent,bytes_received,time_taken_ms,referer,cookie,user_agent,client};
separators = {" "," "," "," "," "," "," "," "," "," "," "," "," "}; separators = {" "," "," "," "," "," "," "," "," "," "," "," "," "};
lf = fo.processIisFormatString(format_string, 0); lf = fo.processIisFormatString(format_string, 0);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -664,7 +664,7 @@ void testCraplogModules()
assert( lf.final.empty() ); assert( lf.final.empty() );
// test all the non-considered fields for the W3C module // test all the non-considered fields for the W3C module
format_string = "s-sitename s-computername s-ip s-port cs-username cs-host sc-substatus sc-win32-status streamid"; format_string = "s-sitename s-computername s-ip s-port cs-username cs-host sc-substatus sc-win32-status streamid";
fields = {"NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE","NONE"}; fields = {_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED,_DISCARDED};
separators = {" "," "," "," "," "," "," "," "}; separators = {" "," "," "," "," "," "," "," "};
lf = fo.processIisFormatString(format_string, 0); lf = fo.processIisFormatString(format_string, 0);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -680,7 +680,7 @@ void testCraplogModules()
assert( lf.final.empty() ); assert( lf.final.empty() );
// test the the NCSA module // test the the NCSA module
format_string = "some random useless text"; format_string = "some random useless text";
fields = {"client","NONE","NONE","date_time_ncsa","request_full","response_code","bytes_sent"}; fields = {client,_DISCARDED,_DISCARDED,date_time_ncsa,request_full,response_code,bytes_sent};
separators = {" "," "," [","] \"","\" "," "}; separators = {" "," "," [","] \"","\" "," "};
lf = fo.processIisFormatString(format_string, 1); lf = fo.processIisFormatString(format_string, 1);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -696,7 +696,7 @@ void testCraplogModules()
assert( lf.final.empty() ); assert( lf.final.empty() );
// test the the IIS module // test the the IIS module
format_string = "some random useless text"; format_string = "some random useless text";
fields = {"client","NONE","date_time_MDYYYY","date_time_utc_t","NONE","NONE","NONE","time_taken_ms","bytes_received","bytes_sent","response_code","NONE","request_method","request_uri","request_query"}; fields = {client,_DISCARDED,date_time_mdyyyy,date_time_utc_t,_DISCARDED,_DISCARDED,_DISCARDED,time_taken_ms,bytes_received,bytes_sent,response_code,_DISCARDED,request_method,request_uri,request_query};
separators = {", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", "}; separators = {", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", ",", "};
lf = fo.processIisFormatString(format_string, 2); lf = fo.processIisFormatString(format_string, 2);
assert( lf.initial.empty() ); assert( lf.initial.empty() );
@ -717,7 +717,7 @@ void testCraplogModules()
//// LOGS TYPE //// //// LOGS TYPE ////
{ {
LogsFormat lf{ "","","]",{" ","_"},{"","",""},0 }; LogsFormat lf{ "","","]",{" ","_"},{_DISCARDED,_DISCARDED,_DISCARDED},0 };
assert( LogOps::defineFileType({"ok ok_ok]","a a_a]","TEST TEST_TEST]"}, lf) == LogType::Access ); assert( LogOps::defineFileType({"ok ok_ok]","a a_a]","TEST TEST_TEST]"}, lf) == LogType::Access );
assert( LogOps::defineFileType({"no no no!","some thing wrong","with this file!"}, lf) == LogType::Discarded ); assert( LogOps::defineFileType({"no no no!","some thing wrong","with this file!"}, lf) == LogType::Discarded );
assert( LogOps::defineFileType({}, lf) == LogType::Failed ); assert( LogOps::defineFileType({}, lf) == LogType::Failed );