elB4RTO/LogDoctor
elB4RTO
/
LogDoctor
2
2
Fork 0
Code Issues 6 Pull Requests Projects 1 Releases 8 Wiki Activity

Improvements 

Added new Warnlists/Blacklists related methods.
Added new LogsPaths related methods.
Added new LogsFormats related methods.
This commit is contained in:
Valentino Orlandi 2022-08-03 21:33:43 +02:00
parent ed93c0509c
commit 93dbec95d6
Signed by: elB4RTO
GPG Key ID: 1719E976DB2D4E71
2 changed files with 158 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
logdoctor/tools/craplog/craplog.cpp
View File

@ -38,25 +38,25 @@ Craplog::Craplog()
}
// default format strings
this->logs_format_stings.emplace( this->APACHE_ID, std::unordered_map<int, std::string>() );
this->logs_format_stings.at( this->APACHE_ID ).emplace( this->ACCESS_LOGS, "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" );
this->logs_format_stings.at( this->APACHE_ID ).emplace( this->ERROR_LOGS, "[%t] [%l] [pid %P] %F: %E: [client %a] %M" );
this->logs_format_stings.emplace( this->NGINX_ID, std::unordered_map<int, std::string>() );
this->logs_format_stings.at( this->NGINX_ID ).emplace( this->ACCESS_LOGS, "$remote_addr - $remote_user [$time_local] \"$request\" $status $bytes_sent \"$http_referer\" \"$http_user_agent\"" );
this->logs_format_stings.at( this->NGINX_ID ).emplace( this->ERROR_LOGS, "$time_iso8601 [$error_level] $pid: *$cid $error_message" );
this->logs_format_stings.emplace( this->IIS_ID, std::unordered_map<int, std::string>() );
this->logs_format_stings.at( this->IIS_ID ).emplace( this->ACCESS_LOGS, "date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs-version cs(User-Agent) cs(Cookie) cs(Referer) cs-host sc-status sc-substatus sc-win32-status sc-bytes cs-bytes time-taken" );
this->logs_format_stings.at( this->IIS_ID ).emplace( this->ERROR_LOGS, "" );
this->logs_format_strings.emplace( this->APACHE_ID, std::unordered_map<int, std::string>() );
this->logs_format_strings.at( this->APACHE_ID ).emplace( this->ACCESS_LOGS, "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" );
this->logs_format_strings.at( this->APACHE_ID ).emplace( this->ERROR_LOGS, "[%t] [%l] [pid %P] %F: %E: [client %a] %M" );
this->logs_format_strings.emplace( this->NGINX_ID, std::unordered_map<int, std::string>() );
this->logs_format_strings.at( this->NGINX_ID ).emplace( this->ACCESS_LOGS, "$remote_addr - $remote_user [$time_local] \"$request\" $status $bytes_sent \"$http_referer\" \"$http_user_agent\"" );
this->logs_format_strings.at( this->NGINX_ID ).emplace( this->ERROR_LOGS, "$time_iso8601 [$error_level] $pid: *$cid $error_message" );
this->logs_format_strings.emplace( this->IIS_ID, std::unordered_map<int, std::string>() );
this->logs_format_strings.at( this->IIS_ID ).emplace( this->ACCESS_LOGS, "date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs-version cs(User-Agent) cs(Cookie) cs(Referer) cs-host sc-status sc-substatus sc-win32-status sc-bytes cs-bytes time-taken" );
this->logs_format_strings.at( this->IIS_ID ).emplace( this->ERROR_LOGS, "" );
// initialize format strings
this->logs_formats.emplace( this->APACHE_ID, std::unordered_map<int, FormatOps::LogsFormat>() );
this->logs_formats.at( this->APACHE_ID ).emplace( this->ACCESS_LOGS, this->formatOps.processApacheFormatString( this->logs_format_stings.at(this->APACHE_ID).at(this->ACCESS_LOGS), this->ACCESS_LOGS ) );
this->logs_formats.at( this->APACHE_ID ).emplace( this->ERROR_LOGS, this->formatOps.processApacheFormatString( this->logs_format_stings.at(this->APACHE_ID).at(this->ERROR_LOGS), this->ERROR_LOGS ) );
this->logs_formats.at( this->APACHE_ID ).emplace( this->ACCESS_LOGS, this->formatOps.processApacheFormatString( this->logs_format_strings.at(this->APACHE_ID).at(this->ACCESS_LOGS), this->ACCESS_LOGS ) );
this->logs_formats.at( this->APACHE_ID ).emplace( this->ERROR_LOGS, this->formatOps.processApacheFormatString( this->logs_format_strings.at(this->APACHE_ID).at(this->ERROR_LOGS), this->ERROR_LOGS ) );
this->logs_formats.emplace( this->NGINX_ID, std::unordered_map<int, FormatOps::LogsFormat>() );
this->logs_formats.at( this->NGINX_ID ).emplace( this->ACCESS_LOGS, this->formatOps.processNginxFormatString( this->logs_format_stings.at(this->NGINX_ID).at(this->ACCESS_LOGS), this->ACCESS_LOGS ) );
this->logs_formats.at( this->NGINX_ID ).emplace( this->ERROR_LOGS, this->formatOps.processNginxFormatString( this->logs_format_stings.at(this->NGINX_ID).at(this->ERROR_LOGS), this->ERROR_LOGS ) );
this->logs_formats.at( this->NGINX_ID ).emplace( this->ACCESS_LOGS, this->formatOps.processNginxFormatString( this->logs_format_strings.at(this->NGINX_ID).at(this->ACCESS_LOGS), this->ACCESS_LOGS ) );
this->logs_formats.at( this->NGINX_ID ).emplace( this->ERROR_LOGS, this->formatOps.processNginxFormatString( this->logs_format_strings.at(this->NGINX_ID).at(this->ERROR_LOGS), this->ERROR_LOGS ) );
this->logs_formats.emplace( this->IIS_ID, std::unordered_map<int, FormatOps::LogsFormat>() );
this->logs_formats.at( this->IIS_ID ).emplace( this->ACCESS_LOGS, this->formatOps.processIisFormatString( this->logs_format_stings.at(this->IIS_ID).at(this->ACCESS_LOGS), 3 ) );
this->logs_formats.at( this->IIS_ID ).emplace( this->ACCESS_LOGS, this->formatOps.processIisFormatString( this->logs_format_strings.at(this->IIS_ID).at(this->ACCESS_LOGS), 3 ) );
this->logs_formats.at( this->IIS_ID ).emplace( this->ERROR_LOGS, FormatOps::LogsFormat {} );
this->current_ALF = this->logs_formats.at( this->APACHE_ID ).at( this->ACCESS_LOGS );
@ -128,7 +128,7 @@ Craplog::Craplog()
//////////////////
//// SETTINGS ////
const int Craplog::getDialogLevel()
const int Craplog::getDialogsLevel()
{
return this->dialog_level;
}
@ -167,48 +167,150 @@ void Craplog::setWarningSize( const int new_size )
}
////////////////////
//// WARN/BLACK ////
const bool Craplog::isBlacklistUsed( const int web_server_id, const int log_type, const int log_field_id )
{
return this->blacklists.at( this->current_WS ).at( log_type ).at( log_field_id ).used;
}
const bool Craplog::isWarnlistUsed( const int web_server_id, const int log_type, const int log_field_id )
{
return this->warnlists.at( this->current_WS ).at( log_type ).at( log_field_id ).used;
}
void Craplog::setBlacklistUsed( const int web_server_id, const int log_type, const int log_field_id, const bool used )
{
}
void Craplog::setWarnlistUsed( const int web_server_id, const int log_type, const int log_field_id, const bool used )
{
}
const std::vector<std::string>& Craplog::getBlacklist( const int web_server_id, const int log_type, const int log_field_id )
{
return this->blacklists.at( this->current_WS ).at( log_type ).at( log_field_id ).list;
}
const std::vector<std::string>& Craplog::getWarnlist( const int web_server_id, const int log_type, const int log_field_id )
{
return this->warnlists.at( this->current_WS ).at( log_type ).at( log_field_id ).list;
}
void Craplog::blacklistAdd( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item )
{
}
void Craplog::warnlistAdd( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item )
{
}
void Craplog::blacklistRemove( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item )
{
}
void Craplog::warnlistRemove( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item )
{
}
void Craplog::blacklistMoveUp( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item )
{
}
void Craplog::warnlistMoveUp( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item )
{
}
void Craplog::blacklistMoveDown( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item )
{
}
void Craplog::warnlistMoveDown( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item )
{
}
/////////////////
//// FORMATS ////
// get the logs format string
const std::string& Craplog::getAccessLogsFormatString( const int web_server_id )
{
return this->logs_format_strings.at( web_server_id ).at( this->ACCESS_LOGS );
}
const std::string& Craplog::getErrorLogsFormatString( const int web_server_id )
{
return this->logs_format_strings.at( web_server_id ).at( this->ERROR_LOGS );
}
// get the logs format
const FormatOps::LogsFormat& Craplog::getAccessLogsFormat( const int web_server_id )
{
return this->logs_formats.at( web_server_id ).at( 1 );
return this->logs_formats.at( web_server_id ).at( this->ACCESS_LOGS );
}
const FormatOps::LogsFormat& Craplog::getErrorLogsFormat( const int web_server_id )
{
return this->logs_formats.at( web_server_id ).at( 2 );
return this->logs_formats.at( web_server_id ).at( this->ERROR_LOGS );
}
// set the logs format
void Craplog::setApacheALF( const std::string& format_string )
{
// apache
this->logs_format_strings.at( this->APACHE_ID ).at( this->ACCESS_LOGS ) = format_string;
this->logs_formats.at( this->APACHE_ID ).at( this->ACCESS_LOGS ) =
this->formatOps.processApacheFormatString( format_string, this->ACCESS_LOGS );
}
void Craplog::setNginxALF( const std::string& format_string )
{
// nginx
this->logs_format_strings.at( this->NGINX_ID ).at( this->ACCESS_LOGS ) = format_string;
this->logs_formats.at( this->NGINX_ID ).at( this->ACCESS_LOGS ) =
this->formatOps.processApacheFormatString( format_string, this->ACCESS_LOGS );
}
void Craplog::setIisALF( const std::string& format_string, const int log_module )
{
// iis
this->logs_format_strings.at( this->IIS_ID ).at( this->ACCESS_LOGS ) = format_string;
this->logs_formats.at( this->IIS_ID ).at( this->ACCESS_LOGS ) =
this->formatOps.processApacheFormatString( format_string, log_module );
}
void Craplog::setApacheELF( const std::string& format_string )
{
// apache
this->logs_format_strings.at( this->APACHE_ID ).at( this->ERROR_LOGS ) = format_string;
this->logs_formats.at( this->APACHE_ID ).at( this->ERROR_LOGS ) =
this->formatOps.processApacheFormatString( format_string, this->ERROR_LOGS );
}
void Craplog::setNginxELF( const std::string& format_string )
{
// nginx
this->logs_format_strings.at( this->NGINX_ID ).at( this->ERROR_LOGS ) = format_string;
this->logs_formats.at( this->NGINX_ID ).at( this->ERROR_LOGS ) =
this->formatOps.processApacheFormatString( format_string, this->ERROR_LOGS );
}
const QString Craplog::getLogsFormatSample( const int web_server_id, const int log_type )
{
QString sample;
if ( web_server_id == this->APACHE_ID ) {
sample = this->formatOps.getApacheLogSample( this->logs_formats.at( web_server_id ).at( this->APACHE_ID ), log_type );
} else if ( web_server_id == this->NGINX_ID ) {
sample = this->formatOps.getNginxLogSample( this->logs_formats.at( web_server_id ).at( this->NGINX_ID ), log_type );
} else if ( web_server_id == this->IIS_ID ) {
if ( log_type == this->ACCESS_LOGS ) {
sample = this->formatOps.getIisLogSample( this->logs_formats.at( web_server_id ).at( this->IIS_ID )/*, log_type*/ );
}
} else {
// unexpected WebServer
throw (&"Unexpected WebServer: "[web_server_id]);
}
return sample;
}
// set the current Web Server
void Craplog::setCurrentWSID( const int web_server_id )
@ -246,6 +348,17 @@ const FormatOps::LogsFormat& Craplog::getCurrentELF()
}
///////////////////
//// LOGS PATH ////
const std::string& Craplog::getLogsPath( const int web_server, const int log_type )
{
return this->logs_paths.at( web_server ).at( log_type );
}
void Craplog::setLogsPath( const int web_server, const int log_type, const std::string& new_path )
{
this->logs_paths.at( web_server ).at( log_type ) = new_path;
}
///////////////////
//// LOGS LIST ////
@ -755,23 +868,6 @@ void Craplog::parseLogLines()
}
const bool Craplog::isBlacklistUsed( const int web_server_id, const int log_type, const int log_field_id )
{
return this->blacklists.at( this->current_WS ).at( log_type ).at( log_field_id ).used;
}
const bool Craplog::isWarnlistUsed( const int web_server_id, const int log_type, const int log_field_id )
{
return this->warnlists.at( this->current_WS ).at( log_type ).at( log_field_id ).used;
}
const std::vector<std::string>& Craplog::getBlacklist( const int web_server_id, const int log_type, const int log_field_id )
{
return this->blacklists.at( this->current_WS ).at( log_type ).at( log_field_id ).list;
}
const std::vector<std::string>& Craplog::getWarnlist( const int web_server_id, const int log_type, const int log_field_id )
{
return this->warnlists.at( this->current_WS ).at( log_type ).at( log_field_id ).list;
}
void Craplog::storeLogLines()
{

39
logdoctor/tools/craplog/craplog.h
View File

@ -21,18 +21,17 @@ public:
void run();
const int getDialogLevel();
const int getDialogsLevel();
void setDialogLevel( const int new_level );
void setChartsTheme( const int new_theme_id );
const std::string
& getStatsDatabasePath(),
& getHashesDatabasePath();
void setStatsDatabasePath( const std::string& path ),
setHashesDatabasePath( const std::string& path );
// web servers ID constants
const unsigned int APACHE_ID=11, NGINX_ID=12, IIS_ID=13;
// logs formats web server specific settings
void setApacheALF( const std::string& format_string );
void setNginxALF( const std::string& format_string );
@ -40,16 +39,21 @@ public:
void setApacheELF( const std::string& format_string );
void setNginxELF( const std::string& format_string );
//void setIisELF( const std::string& format_string );
const std::string& getAccessLogsFormatString( const int web_server_id );
const std::string& getErrorLogsFormatString( const int web_server_id );
const FormatOps::LogsFormat& getAccessLogsFormat( const int web_server_id );
const FormatOps::LogsFormat& getErrorLogsFormat( const int web_server_id );
const QString getLogsFormatSample( const int web_server_id, const int log_type );
// currently used WS and LF
void setCurrentWSID( const int web_server_id );
const int getCurrentWSID();
const FormatOps::LogsFormat& getCurrentALF();
const FormatOps::LogsFormat& getCurrentELF();
// log type constants
const unsigned int FAILED=0, ACCESS_LOGS=1, ERROR_LOGS=2;
// logs paths
const std::string& getLogsPath( const int web_server, const int log_type );
void setLogsPath( const int web_server, const int log_type, const std::string& new_path );
// log file item's infoes
struct LogFile {
bool selected;
@ -87,9 +91,19 @@ public:
};
const bool isBlacklistUsed( const int web_server_id, const int log_type, const int log_field_id ),
isWarnlistUsed( const int web_server_id, const int log_type, const int log_field_id );
void setBlacklistUsed( const int web_server_id, const int log_type, const int log_field_id, const bool used ),
setWarnlistUsed( const int web_server_id, const int log_type, const int log_field_id, const bool used );
const std::vector<std::string>
& getBlacklist( const int web_server_id, const int log_type, const int log_field_id ),
& getWarnlist( const int web_server_id, const int log_type, const int log_field_id );
&getBlacklist( const int web_server_id, const int log_type, const int log_field_id ),
&getWarnlist( const int web_server_id, const int log_type, const int log_field_id );
void blacklistAdd( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ),
warnlistAdd( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ),
blacklistRemove( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ),
warnlistRemove( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ),
blacklistMoveUp( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ),
warnlistMoveUp( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ),
blacklistMoveDown( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ),
warnlistMoveDown( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item );
// job related
const bool checkStuff();
@ -115,6 +129,11 @@ private:
// quantity of informational dialogs to display
int dialog_level = 2; // 0: essential, 1: usefull, 2: explanatory
// web servers ID constants
const unsigned int APACHE_ID=11, NGINX_ID=12, IIS_ID=13;
// log type constants
const unsigned int FAILED=0, ACCESS_LOGS=1, ERROR_LOGS=2;
// databases paths
std::string db_stats_path,
db_hashes_path;
@ -166,8 +185,6 @@ private:
std::string configs_path;
// control related
bool delete_old_hashes = false;
int old_hashes_months = 12;
int warning_size = 1'048'576 +1; //104'857'600; // in Bytes ( => 100 MiB ) // !!! RESTORE !!!
// black/warn-list
// { web_server_id : { log_type : { log_field_id : BWlist } } }
@ -193,7 +210,7 @@ private:
// logs format related
FormatOps formatOps;
std::unordered_map<int, std::unordered_map<int, std::string>> logs_format_stings;
std::unordered_map<int, std::unordered_map<int, std::string>> logs_format_strings;
std::unordered_map<int, std::unordered_map<int, FormatOps::LogsFormat>> logs_formats;
void setCurrentALF();
void setCurrentELF();