From 93dbec95d659a3a74918510229f8e5d83d14c0b2 Mon Sep 17 00:00:00 2001 From: Valentino Orlandi Date: Wed, 3 Aug 2022 21:33:43 +0200 Subject: [PATCH] Improvements Added new Warnlists/Blacklists related methods. Added new LogsPaths related methods. Added new LogsFormats related methods. --- logdoctor/tools/craplog/craplog.cpp | 164 ++++++++++++++++++++++------ logdoctor/tools/craplog/craplog.h | 39 +++++-- 2 files changed, 158 insertions(+), 45 deletions(-) diff --git a/logdoctor/tools/craplog/craplog.cpp b/logdoctor/tools/craplog/craplog.cpp index 966b6356..2a9767f1 100644 --- a/logdoctor/tools/craplog/craplog.cpp +++ b/logdoctor/tools/craplog/craplog.cpp @@ -38,25 +38,25 @@ Craplog::Craplog() } // default format strings - this->logs_format_stings.emplace( this->APACHE_ID, std::unordered_map() ); - this->logs_format_stings.at( this->APACHE_ID ).emplace( this->ACCESS_LOGS, "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" ); - this->logs_format_stings.at( this->APACHE_ID ).emplace( this->ERROR_LOGS, "[%t] [%l] [pid %P] %F: %E: [client %a] %M" ); - this->logs_format_stings.emplace( this->NGINX_ID, std::unordered_map() ); - this->logs_format_stings.at( this->NGINX_ID ).emplace( this->ACCESS_LOGS, "$remote_addr - $remote_user [$time_local] \"$request\" $status $bytes_sent \"$http_referer\" \"$http_user_agent\"" ); - this->logs_format_stings.at( this->NGINX_ID ).emplace( this->ERROR_LOGS, "$time_iso8601 [$error_level] $pid: *$cid $error_message" ); - this->logs_format_stings.emplace( this->IIS_ID, std::unordered_map() ); - this->logs_format_stings.at( this->IIS_ID ).emplace( this->ACCESS_LOGS, "date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs-version cs(User-Agent) cs(Cookie) cs(Referer) cs-host sc-status sc-substatus sc-win32-status sc-bytes cs-bytes time-taken" ); - this->logs_format_stings.at( this->IIS_ID ).emplace( this->ERROR_LOGS, "" ); + this->logs_format_strings.emplace( this->APACHE_ID, std::unordered_map() ); + this->logs_format_strings.at( this->APACHE_ID ).emplace( this->ACCESS_LOGS, "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"" ); + this->logs_format_strings.at( this->APACHE_ID ).emplace( this->ERROR_LOGS, "[%t] [%l] [pid %P] %F: %E: [client %a] %M" ); + this->logs_format_strings.emplace( this->NGINX_ID, std::unordered_map() ); + this->logs_format_strings.at( this->NGINX_ID ).emplace( this->ACCESS_LOGS, "$remote_addr - $remote_user [$time_local] \"$request\" $status $bytes_sent \"$http_referer\" \"$http_user_agent\"" ); + this->logs_format_strings.at( this->NGINX_ID ).emplace( this->ERROR_LOGS, "$time_iso8601 [$error_level] $pid: *$cid $error_message" ); + this->logs_format_strings.emplace( this->IIS_ID, std::unordered_map() ); + this->logs_format_strings.at( this->IIS_ID ).emplace( this->ACCESS_LOGS, "date time s-ip cs-method cs-uri-stem cs-uri-query s-port cs-username c-ip cs-version cs(User-Agent) cs(Cookie) cs(Referer) cs-host sc-status sc-substatus sc-win32-status sc-bytes cs-bytes time-taken" ); + this->logs_format_strings.at( this->IIS_ID ).emplace( this->ERROR_LOGS, "" ); // initialize format strings this->logs_formats.emplace( this->APACHE_ID, std::unordered_map() ); - this->logs_formats.at( this->APACHE_ID ).emplace( this->ACCESS_LOGS, this->formatOps.processApacheFormatString( this->logs_format_stings.at(this->APACHE_ID).at(this->ACCESS_LOGS), this->ACCESS_LOGS ) ); - this->logs_formats.at( this->APACHE_ID ).emplace( this->ERROR_LOGS, this->formatOps.processApacheFormatString( this->logs_format_stings.at(this->APACHE_ID).at(this->ERROR_LOGS), this->ERROR_LOGS ) ); + this->logs_formats.at( this->APACHE_ID ).emplace( this->ACCESS_LOGS, this->formatOps.processApacheFormatString( this->logs_format_strings.at(this->APACHE_ID).at(this->ACCESS_LOGS), this->ACCESS_LOGS ) ); + this->logs_formats.at( this->APACHE_ID ).emplace( this->ERROR_LOGS, this->formatOps.processApacheFormatString( this->logs_format_strings.at(this->APACHE_ID).at(this->ERROR_LOGS), this->ERROR_LOGS ) ); this->logs_formats.emplace( this->NGINX_ID, std::unordered_map() ); - this->logs_formats.at( this->NGINX_ID ).emplace( this->ACCESS_LOGS, this->formatOps.processNginxFormatString( this->logs_format_stings.at(this->NGINX_ID).at(this->ACCESS_LOGS), this->ACCESS_LOGS ) ); - this->logs_formats.at( this->NGINX_ID ).emplace( this->ERROR_LOGS, this->formatOps.processNginxFormatString( this->logs_format_stings.at(this->NGINX_ID).at(this->ERROR_LOGS), this->ERROR_LOGS ) ); + this->logs_formats.at( this->NGINX_ID ).emplace( this->ACCESS_LOGS, this->formatOps.processNginxFormatString( this->logs_format_strings.at(this->NGINX_ID).at(this->ACCESS_LOGS), this->ACCESS_LOGS ) ); + this->logs_formats.at( this->NGINX_ID ).emplace( this->ERROR_LOGS, this->formatOps.processNginxFormatString( this->logs_format_strings.at(this->NGINX_ID).at(this->ERROR_LOGS), this->ERROR_LOGS ) ); this->logs_formats.emplace( this->IIS_ID, std::unordered_map() ); - this->logs_formats.at( this->IIS_ID ).emplace( this->ACCESS_LOGS, this->formatOps.processIisFormatString( this->logs_format_stings.at(this->IIS_ID).at(this->ACCESS_LOGS), 3 ) ); + this->logs_formats.at( this->IIS_ID ).emplace( this->ACCESS_LOGS, this->formatOps.processIisFormatString( this->logs_format_strings.at(this->IIS_ID).at(this->ACCESS_LOGS), 3 ) ); this->logs_formats.at( this->IIS_ID ).emplace( this->ERROR_LOGS, FormatOps::LogsFormat {} ); this->current_ALF = this->logs_formats.at( this->APACHE_ID ).at( this->ACCESS_LOGS ); @@ -128,7 +128,7 @@ Craplog::Craplog() ////////////////// //// SETTINGS //// -const int Craplog::getDialogLevel() +const int Craplog::getDialogsLevel() { return this->dialog_level; } @@ -167,48 +167,150 @@ void Craplog::setWarningSize( const int new_size ) } +//////////////////// +//// WARN/BLACK //// + +const bool Craplog::isBlacklistUsed( const int web_server_id, const int log_type, const int log_field_id ) +{ + return this->blacklists.at( this->current_WS ).at( log_type ).at( log_field_id ).used; +} +const bool Craplog::isWarnlistUsed( const int web_server_id, const int log_type, const int log_field_id ) +{ + return this->warnlists.at( this->current_WS ).at( log_type ).at( log_field_id ).used; +} + +void Craplog::setBlacklistUsed( const int web_server_id, const int log_type, const int log_field_id, const bool used ) +{ + +} +void Craplog::setWarnlistUsed( const int web_server_id, const int log_type, const int log_field_id, const bool used ) +{ + +} + +const std::vector& Craplog::getBlacklist( const int web_server_id, const int log_type, const int log_field_id ) +{ + return this->blacklists.at( this->current_WS ).at( log_type ).at( log_field_id ).list; +} +const std::vector& Craplog::getWarnlist( const int web_server_id, const int log_type, const int log_field_id ) +{ + return this->warnlists.at( this->current_WS ).at( log_type ).at( log_field_id ).list; +} + +void Craplog::blacklistAdd( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ) +{ + +} +void Craplog::warnlistAdd( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ) +{ + +} + +void Craplog::blacklistRemove( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ) +{ + +} +void Craplog::warnlistRemove( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ) +{ + +} + +void Craplog::blacklistMoveUp( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ) +{ + +} +void Craplog::warnlistMoveUp( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ) +{ + +} + +void Craplog::blacklistMoveDown( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ) +{ + +} +void Craplog::warnlistMoveDown( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ) +{ + +} + + +///////////////// +//// FORMATS //// +// get the logs format string +const std::string& Craplog::getAccessLogsFormatString( const int web_server_id ) +{ + return this->logs_format_strings.at( web_server_id ).at( this->ACCESS_LOGS ); +} +const std::string& Craplog::getErrorLogsFormatString( const int web_server_id ) +{ + return this->logs_format_strings.at( web_server_id ).at( this->ERROR_LOGS ); +} + // get the logs format const FormatOps::LogsFormat& Craplog::getAccessLogsFormat( const int web_server_id ) { - return this->logs_formats.at( web_server_id ).at( 1 ); + return this->logs_formats.at( web_server_id ).at( this->ACCESS_LOGS ); } const FormatOps::LogsFormat& Craplog::getErrorLogsFormat( const int web_server_id ) { - return this->logs_formats.at( web_server_id ).at( 2 ); + return this->logs_formats.at( web_server_id ).at( this->ERROR_LOGS ); } // set the logs format void Craplog::setApacheALF( const std::string& format_string ) { // apache + this->logs_format_strings.at( this->APACHE_ID ).at( this->ACCESS_LOGS ) = format_string; this->logs_formats.at( this->APACHE_ID ).at( this->ACCESS_LOGS ) = this->formatOps.processApacheFormatString( format_string, this->ACCESS_LOGS ); } void Craplog::setNginxALF( const std::string& format_string ) { // nginx + this->logs_format_strings.at( this->NGINX_ID ).at( this->ACCESS_LOGS ) = format_string; this->logs_formats.at( this->NGINX_ID ).at( this->ACCESS_LOGS ) = this->formatOps.processApacheFormatString( format_string, this->ACCESS_LOGS ); } void Craplog::setIisALF( const std::string& format_string, const int log_module ) { // iis + this->logs_format_strings.at( this->IIS_ID ).at( this->ACCESS_LOGS ) = format_string; this->logs_formats.at( this->IIS_ID ).at( this->ACCESS_LOGS ) = this->formatOps.processApacheFormatString( format_string, log_module ); } void Craplog::setApacheELF( const std::string& format_string ) { // apache + this->logs_format_strings.at( this->APACHE_ID ).at( this->ERROR_LOGS ) = format_string; this->logs_formats.at( this->APACHE_ID ).at( this->ERROR_LOGS ) = this->formatOps.processApacheFormatString( format_string, this->ERROR_LOGS ); } void Craplog::setNginxELF( const std::string& format_string ) { // nginx + this->logs_format_strings.at( this->NGINX_ID ).at( this->ERROR_LOGS ) = format_string; this->logs_formats.at( this->NGINX_ID ).at( this->ERROR_LOGS ) = this->formatOps.processApacheFormatString( format_string, this->ERROR_LOGS ); } +const QString Craplog::getLogsFormatSample( const int web_server_id, const int log_type ) +{ + QString sample; + if ( web_server_id == this->APACHE_ID ) { + sample = this->formatOps.getApacheLogSample( this->logs_formats.at( web_server_id ).at( this->APACHE_ID ), log_type ); + } else if ( web_server_id == this->NGINX_ID ) { + sample = this->formatOps.getNginxLogSample( this->logs_formats.at( web_server_id ).at( this->NGINX_ID ), log_type ); + } else if ( web_server_id == this->IIS_ID ) { + if ( log_type == this->ACCESS_LOGS ) { + sample = this->formatOps.getIisLogSample( this->logs_formats.at( web_server_id ).at( this->IIS_ID )/*, log_type*/ ); + } + } else { + // unexpected WebServer + throw (&"Unexpected WebServer: "[web_server_id]); + } + return sample; +} + // set the current Web Server void Craplog::setCurrentWSID( const int web_server_id ) @@ -246,6 +348,17 @@ const FormatOps::LogsFormat& Craplog::getCurrentELF() } +/////////////////// +//// LOGS PATH //// +const std::string& Craplog::getLogsPath( const int web_server, const int log_type ) +{ + return this->logs_paths.at( web_server ).at( log_type ); +} +void Craplog::setLogsPath( const int web_server, const int log_type, const std::string& new_path ) +{ + this->logs_paths.at( web_server ).at( log_type ) = new_path; +} + /////////////////// //// LOGS LIST //// @@ -755,23 +868,6 @@ void Craplog::parseLogLines() } -const bool Craplog::isBlacklistUsed( const int web_server_id, const int log_type, const int log_field_id ) -{ - return this->blacklists.at( this->current_WS ).at( log_type ).at( log_field_id ).used; -} -const bool Craplog::isWarnlistUsed( const int web_server_id, const int log_type, const int log_field_id ) -{ - return this->warnlists.at( this->current_WS ).at( log_type ).at( log_field_id ).used; -} -const std::vector& Craplog::getBlacklist( const int web_server_id, const int log_type, const int log_field_id ) -{ - return this->blacklists.at( this->current_WS ).at( log_type ).at( log_field_id ).list; -} -const std::vector& Craplog::getWarnlist( const int web_server_id, const int log_type, const int log_field_id ) -{ - return this->warnlists.at( this->current_WS ).at( log_type ).at( log_field_id ).list; -} - void Craplog::storeLogLines() { diff --git a/logdoctor/tools/craplog/craplog.h b/logdoctor/tools/craplog/craplog.h index f07b59a5..6b71f56d 100644 --- a/logdoctor/tools/craplog/craplog.h +++ b/logdoctor/tools/craplog/craplog.h @@ -21,18 +21,17 @@ public: void run(); - const int getDialogLevel(); + const int getDialogsLevel(); void setDialogLevel( const int new_level ); + void setChartsTheme( const int new_theme_id ); + const std::string & getStatsDatabasePath(), & getHashesDatabasePath(); void setStatsDatabasePath( const std::string& path ), setHashesDatabasePath( const std::string& path ); - // web servers ID constants - const unsigned int APACHE_ID=11, NGINX_ID=12, IIS_ID=13; - // logs formats web server specific settings void setApacheALF( const std::string& format_string ); void setNginxALF( const std::string& format_string ); @@ -40,16 +39,21 @@ public: void setApacheELF( const std::string& format_string ); void setNginxELF( const std::string& format_string ); //void setIisELF( const std::string& format_string ); + const std::string& getAccessLogsFormatString( const int web_server_id ); + const std::string& getErrorLogsFormatString( const int web_server_id ); const FormatOps::LogsFormat& getAccessLogsFormat( const int web_server_id ); const FormatOps::LogsFormat& getErrorLogsFormat( const int web_server_id ); + const QString getLogsFormatSample( const int web_server_id, const int log_type ); // currently used WS and LF void setCurrentWSID( const int web_server_id ); const int getCurrentWSID(); const FormatOps::LogsFormat& getCurrentALF(); const FormatOps::LogsFormat& getCurrentELF(); - // log type constants - const unsigned int FAILED=0, ACCESS_LOGS=1, ERROR_LOGS=2; + // logs paths + const std::string& getLogsPath( const int web_server, const int log_type ); + void setLogsPath( const int web_server, const int log_type, const std::string& new_path ); + // log file item's infoes struct LogFile { bool selected; @@ -87,9 +91,19 @@ public: }; const bool isBlacklistUsed( const int web_server_id, const int log_type, const int log_field_id ), isWarnlistUsed( const int web_server_id, const int log_type, const int log_field_id ); + void setBlacklistUsed( const int web_server_id, const int log_type, const int log_field_id, const bool used ), + setWarnlistUsed( const int web_server_id, const int log_type, const int log_field_id, const bool used ); const std::vector - & getBlacklist( const int web_server_id, const int log_type, const int log_field_id ), - & getWarnlist( const int web_server_id, const int log_type, const int log_field_id ); + &getBlacklist( const int web_server_id, const int log_type, const int log_field_id ), + &getWarnlist( const int web_server_id, const int log_type, const int log_field_id ); + void blacklistAdd( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ), + warnlistAdd( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ), + blacklistRemove( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ), + warnlistRemove( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ), + blacklistMoveUp( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ), + warnlistMoveUp( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ), + blacklistMoveDown( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ), + warnlistMoveDown( const int web_server_id, const int log_type, const int log_field_id, const std::string& new_item ); // job related const bool checkStuff(); @@ -115,6 +129,11 @@ private: // quantity of informational dialogs to display int dialog_level = 2; // 0: essential, 1: usefull, 2: explanatory + // web servers ID constants + const unsigned int APACHE_ID=11, NGINX_ID=12, IIS_ID=13; + // log type constants + const unsigned int FAILED=0, ACCESS_LOGS=1, ERROR_LOGS=2; + // databases paths std::string db_stats_path, db_hashes_path; @@ -166,8 +185,6 @@ private: std::string configs_path; // control related - bool delete_old_hashes = false; - int old_hashes_months = 12; int warning_size = 1'048'576 +1; //104'857'600; // in Bytes ( => 100 MiB ) // !!! RESTORE !!! // black/warn-list // { web_server_id : { log_type : { log_field_id : BWlist } } } @@ -193,7 +210,7 @@ private: // logs format related FormatOps formatOps; - std::unordered_map> logs_format_stings; + std::unordered_map> logs_format_strings; std::unordered_map> logs_formats; void setCurrentALF(); void setCurrentELF();