From 3158a4e629e2e37f7e5a5796b9ea9628d39ee4db Mon Sep 17 00:00:00 2001 From: Valentino Orlandi Date: Sat, 3 Feb 2024 17:44:28 +0100 Subject: [PATCH] Code improvements and updates Improved Craplog module and its submodules --- logdoctor/modules/craplog/craplog.cpp | 188 ++++++++-------- logdoctor/modules/craplog/modules/formats.cpp | 4 +- logdoctor/modules/craplog/modules/hash.cpp | 210 ++++------------- logdoctor/modules/craplog/modules/hash.h | 10 +- logdoctor/modules/craplog/modules/lib.h | 7 +- logdoctor/modules/craplog/modules/logs.cpp | 13 +- .../modules/craplog/modules/workers/lib.h | 4 + .../craplog/modules/workers/parser.cpp | 211 ++++++++---------- .../modules/craplog/modules/workers/parser.h | 9 +- 9 files changed, 262 insertions(+), 394 deletions(-) diff --git a/logdoctor/modules/craplog/craplog.cpp b/logdoctor/modules/craplog/craplog.cpp index 2cfe07c5..80c773d3 100644 --- a/logdoctor/modules/craplog/craplog.cpp +++ b/logdoctor/modules/craplog/craplog.cpp @@ -186,14 +186,14 @@ void Craplog::warnlistAdd( const WebServer& web_server, const int& log_field_id, void Craplog::blacklistRemove( const WebServer& web_server, const int& log_field_id, const std::string& item ) noexcept { - auto& list = this->blacklists.at( web_server ).at( log_field_id ).list; + auto& list{ this->blacklists.at( web_server ).at( log_field_id ).list }; if ( const auto it{ std::find( list.cbegin(), list.cend(), item ) }; it != list.cend() ) { list.erase( it ); } } void Craplog::warnlistRemove( const WebServer& web_server, const int& log_field_id, const std::string& item ) noexcept { - auto& list = this->warnlists.at( web_server ).at( log_field_id ).list; + auto& list{ this->warnlists.at( web_server ).at( log_field_id ).list }; if ( const auto it{ std::find( list.cbegin(), list.cend(), item ) }; it != list.cend() ) { list.erase( it ); } @@ -201,7 +201,7 @@ void Craplog::warnlistRemove( const WebServer& web_server, const int& log_field_ int Craplog::blacklistMoveUp( const WebServer& web_server, const int& log_field_id, const std::string& item ) noexcept { - auto& list = this->blacklists.at( web_server ).at( log_field_id ).list; + auto& list{ this->blacklists.at( web_server ).at( log_field_id ).list }; if ( auto it{ std::find( std::next(list.begin()), list.end(), item ) }; it != list.cend() ) { const int pos{ static_cast( std::distance(list.begin(), it) ) - 1 }; std::swap( *it, *std::prev(it) ); @@ -211,7 +211,7 @@ int Craplog::blacklistMoveUp( const WebServer& web_server, const int& log_field_ } int Craplog::warnlistMoveUp( const WebServer& web_server, const int& log_field_id, const std::string& item ) noexcept { - auto& list = this->warnlists.at( web_server ).at( log_field_id ).list; + auto& list{ this->warnlists.at( web_server ).at( log_field_id ).list }; if ( auto it{ std::find( std::next(list.begin()), list.end(), item ) }; it != list.cend() ) { const int pos{ static_cast( std::distance(list.begin(), it) ) - 1 }; std::swap( *it, *std::prev(it) ); @@ -222,7 +222,7 @@ int Craplog::warnlistMoveUp( const WebServer& web_server, const int& log_field_i int Craplog::blacklistMoveDown( const WebServer& web_server, const int& log_field_id, const std::string& item ) noexcept { - auto& list = this->blacklists.at( web_server ).at( log_field_id ).list; + auto& list{ this->blacklists.at( web_server ).at( log_field_id ).list }; if ( auto it{ std::find( list.begin(), std::prev(list.end()), item ) }; it != list.cend() ) { const int pos{ static_cast( std::distance(list.begin(), it) ) + 1 }; std::swap( *it, *std::next(it) ); @@ -232,7 +232,7 @@ int Craplog::blacklistMoveDown( const WebServer& web_server, const int& log_fiel } int Craplog::warnlistMoveDown( const WebServer& web_server, const int& log_field_id, const std::string& item ) noexcept { - auto& list = this->warnlists.at( web_server ).at( log_field_id ).list; + auto& list{ this->warnlists.at( web_server ).at( log_field_id ).list }; if ( auto it{ std::find( list.begin(), std::prev(list.end()), item ) }; it != list.cend() ) { const int pos{ static_cast( std::distance(list.begin(), it) ) + 1 }; std::swap( *it, *std::next(it) ); @@ -243,41 +243,41 @@ int Craplog::warnlistMoveDown( const WebServer& web_server, const int& log_field std::string Craplog::sanitizeBWitem( const int& log_field_id, const std::string& new_item ) const { - std::string sanitized_item; switch ( log_field_id ) { case 11: - sanitized_item = StringOps::strip( new_item ); + { + const std::string sanitized_item{ StringOps::strip( new_item ) }; if ( ! StringOps::isAlphabetic( sanitized_item ) ) { // only letters allowed throw BWlistException("Invalid Method"); } - sanitized_item = StringOps::toUpper( sanitized_item ); - break; + return StringOps::toUpper( sanitized_item ); + } case 12: - sanitized_item = StringOps::lstrip( new_item ); + { + const std::string sanitized_item{ StringOps::lstrip( new_item ) }; if ( sanitized_item.empty() ) { throw BWlistException("Invalid URI"); } - sanitized_item = QUrl::toPercentEncoding( + return QUrl::toPercentEncoding( QString::fromStdString( sanitized_item ), "/#&?=+").toStdString(); - break; + } case 20: - sanitized_item = StringOps::strip( new_item ); + { + const std::string sanitized_item{ StringOps::strip( new_item ) }; if ( ! StringOps::isIP( sanitized_item ) ) { // only IPv4/IPv6 allowed throw BWlistException("Invalid Client"); } - break; + return sanitized_item; + } case 21: - sanitized_item = StringOps::replace( new_item, "\"", "\\\"" ); - break; + return StringOps::replace( new_item, "\"", "\\\"" ); default: // shouldn't be here throw GenericException("Unexpected LogField ID: "+std::to_string(log_field_id)); - break; } - return sanitized_item; } @@ -298,55 +298,49 @@ const LogsFormat& Craplog::getLogsFormat(const WebServer& web_server ) const noe // set the logs format bool Craplog::setApacheLogFormat( const std::string& format_string ) noexcept { - // apache - bool success{ true }; try { this->logs_formats.at( WS_APACHE ) = this->formatOps.processApacheFormatString( format_string ); this->logs_format_strings.at( WS_APACHE ) = format_string; } catch ( LogFormatException& e ) { - success &= false; DialogSec::errInvalidLogFormatString( e.what() ); + return false; } catch (...) { - success &= false; DialogSec::errGeneric( DialogSec::tr("An error occured while parsing the format string"), true ); + return false; } - return success; + return true; } bool Craplog::setNginxLogFormat( const std::string& format_string ) noexcept { - // nginx - bool success{ true }; try { this->logs_formats.at( WS_NGINX ) = this->formatOps.processNginxFormatString( format_string ); this->logs_format_strings.at( WS_NGINX ) = format_string; } catch ( LogFormatException& e ) { - success &= false; DialogSec::errInvalidLogFormatString( e.what() ); + return false; } catch (...) { - success &= false; DialogSec::errGeneric( DialogSec::tr("An error occured while parsing the format string"), true ); + return false; } - return success; + return true; } bool Craplog::setIisLogFormat( const std::string& format_string, const int log_module ) noexcept { - // iis - bool success{ true }; try { this->logs_formats.at( WS_IIS ) = this->formatOps.processIisFormatString( format_string, log_module ); this->logs_format_strings.at( WS_IIS ) = format_string; this->changeIisLogsBaseNames( log_module ); } catch ( LogFormatException& e ) { - success &= false; DialogSec::errInvalidLogFormatString( e.what() ); + return false; } catch (...) { - success &= false; DialogSec::errGeneric( DialogSec::tr("An error occured while parsing the format string"), true ); + return false; } - return success; + return true; } QString Craplog::getLogsFormatSample( const WebServer& web_server ) const @@ -437,9 +431,8 @@ const std::vector& Craplog::getLogsList() const noexcept // return the LogFile instance of the file matching the given name const LogFile& Craplog::getLogFileItem( const QString& file_name ) const { - const auto& item{ std::find_if - ( this->logs_list.begin(), this->logs_list.end(), - [&file_name](const LogFile& it){ return it.name()==file_name; } ) }; + const auto item{ std::find_if( this->logs_list.begin(), this->logs_list.end(), + [&file_name](const LogFile& file){ return file.name()==file_name; } ) }; if ( item != this->logs_list.end() ) return *item; // should be unreachable throw GenericException("File item not found"); @@ -449,10 +442,8 @@ const LogFile& Craplog::getLogFileItem( const QString& file_name ) const // set a file as selected bool Craplog::setLogFileSelected( const QString& file_name ) noexcept { - const auto item{ std::find_if - ( this->logs_list.begin(), this->logs_list.end(), - [&file_name]( const LogFile& it ) - { return it.name() == file_name; } ) }; + const auto item{ std::find_if( this->logs_list.begin(), this->logs_list.end(), + [&file_name](const LogFile& file){ return file.name() == file_name; } ) }; if ( item != this->logs_list.end() ) { item->setSelected(); return true; @@ -539,7 +530,6 @@ void Craplog::changeIisLogsBaseNames( const int module_id ) bool Craplog::isFileNameValid( const std::string& name ) const { - bool valid{ true }; if ( ! this->logs_base_names.at( this->current_web_server ).starts.empty() ) { if ( ! StringOps::startsWith( name, this->logs_base_names.at( this->current_web_server ).starts ) ) { return false; @@ -561,13 +551,13 @@ bool Craplog::isFileNameValid( const std::string& name ) const switch ( this->current_web_server ) { case WS_APACHE: [[fallthrough]]; - case WS_NGINX: { + case WS_NGINX: + { // further checks for apache / nginx size_t start, stop; start = name.rfind(".log." ); if ( start == std::string::npos ) { - valid &= false; - break; + return false; } start += 5ul; stop = name.size()-1ul; @@ -577,19 +567,18 @@ bool Craplog::isFileNameValid( const std::string& name ) const // serach for incremental numbers for ( size_t i{start}; i<=stop; ++i ) { if ( ! CharOps::isNumeric( name.at( i ) ) ) { - valid &= false; - break; + return false; } } - }break; + }break; - case WS_IIS: { + case WS_IIS: + { // further checks for iis size_t start, stop; start = name.find( this->logs_base_names.at( WS_IIS ).contains ) + 3ul; if ( start == std::string::npos ) { - valid &= false; - break; + return false; } stop = name.size()-5ul; // removing the finel '.log' extension if ( StringOps::endsWith( name, ".gz" ) ) { @@ -599,34 +588,31 @@ bool Craplog::isFileNameValid( const std::string& name ) const std::string date; for ( size_t i{start}; i<=stop; ++i ) { if ( ! CharOps::isNumeric( name.at( i ) ) ) { - valid &= false; - break; + return false; } date.push_back( name.at( i ) ); } - if ( valid ) { - // check if the file has today's date - time_t t; - time( &t ); - struct tm* tmp = localtime( &t ); - char aux_date[7]; - // using strftime to display time - strftime( aux_date, 7, "%y%m%d", tmp ); - valid &= false; - for ( size_t i{0}; i<6ul; ++i ) { - if ( date.at(i) != aux_date[i] ) { - // different date, valid - valid |= true; - break; - } + // check if the file has today's date + time_t t; + time( &t ); + struct tm* tmp = localtime( &t ); + char aux_date[7]; + // using strftime to display time + strftime( aux_date, 7, "%y%m%d", tmp ); + for ( size_t i{0}; i<6ul; ++i ) { + if ( date.at(i) != aux_date[i] ) { + // different date, valid + return true; + break; } } - }break; + }break; default: throw WebServerException( "Unexpected WebServer: " + toString(this->current_web_server) ); } - return valid; + + return true; } @@ -650,8 +636,6 @@ bool Craplog::checkStuff() size_t logs_size{ 0ul }; for ( const LogFile& file : this->logs_list ) { - if ( ! this->proceed ) { break; } - if ( ! file.isSelected() ) { // not selected, skip continue; @@ -668,7 +652,7 @@ bool Craplog::checkStuff() if ( choice == 0 ) { // choosed to abort all this->proceed &= false; - break; + return false; } else if ( choice == 1 ) { // choosed to discard the file and continue continue; @@ -689,18 +673,18 @@ bool Craplog::checkStuff() } const int choice = DialogSec::choiceDuplicateFile( msg ); if ( choice == 0 ) { - // choosed to abort all - this->proceed &= false; - break; + // choosed to abort all + this->proceed &= false; + return false; } else if ( choice == 1 ) { - // choosed to discard the file and continue - continue; + // choosed to discard the file and continue + continue; } else if ( choice == 2 ) { - // choosed to ignore and use the file anyway - ; + // choosed to ignore and use the file anyway + ; } else { - // shouldn't be here - throw GenericException( "Unexpeced value returned: "+std::to_string(choice) ); + // shouldn't be here + throw GenericException( "Unexpeced value returned: "+std::to_string(choice) ); } } } @@ -711,11 +695,11 @@ bool Craplog::checkStuff() // exceeds the warning size QString msg{ file.name() }; if ( this->dialogs_level >= DL_NORMAL ) { - msg += QString("\n\n%1:\n%2").arg( + msg += QStringLiteral("\n\n%1:\n%2").arg( DialogSec::tr("Size of the file"), PrintSec::printableSize( file.size() ) ); if ( this->dialogs_level == DL_EXPLANATORY ) { - msg += QString("\n\n%1:\n%2").arg( + msg += QStringLiteral("\n\n%1:\n%2").arg( DialogSec::tr("Warning size parameter"), PrintSec::printableSize( this->warning_size ) ); } @@ -724,7 +708,7 @@ bool Craplog::checkStuff() if ( choice == 0 ) { // choosed to abort all this->proceed &= false; - break; + return false; } else if ( choice == 1 ) { // choosed to discard the file and continue continue; @@ -742,12 +726,12 @@ bool Craplog::checkStuff() if ( ! CheckSec::checkCollectionDatabase( this->db_stats_path ) ) { // checks failed, abort this->proceed &= false; - break; + return false; } if ( ! CheckSec::checkHashesDatabase( this->db_hashes_path ) ) { // checks failed, abort this->proceed &= false; - break; + return false; } this->log_files_to_use.push_back( @@ -758,31 +742,33 @@ bool Craplog::checkStuff() } // check if there are enough files to use - if ( this->proceed && this->log_files_to_use.size() == 0ul ) { + if ( this->log_files_to_use.empty() ) { // no files left, abort DialogSec::msgNoFileToParse(); this->proceed &= false; + return false; } // check if the total size of the files do not exceed the available RAM - if ( this->proceed && logs_size >= MemOps::availableMemory() ) { + if ( logs_size >= MemOps::availableMemory() ) { // no files left, abort QString msg; if ( this->dialogs_level >= DL_NORMAL ) { - msg += QString("\n\n%1: %2").arg( + msg += QStringLiteral("\n\n%1: %2").arg( DialogSec::tr("Available memory"), PrintSec::printableSize( MemOps::availableMemory() ) ); if ( this->dialogs_level == DL_EXPLANATORY ) { - msg += QString("\n%1: %2").arg( + msg += QStringLiteral("\n%1: %2").arg( DialogSec::tr("Size of the logs"), PrintSec::printableSize( logs_size ) ); } } DialogSec::msgNotEnoughMemory( msg ); this->proceed &= false; + return false; } - return this->proceed; + return true; } void Craplog::showWorkerDialog( const WorkerDialog dialog_type, const QStringList args ) const noexcept @@ -800,6 +786,18 @@ void Craplog::showWorkerDialog( const WorkerDialog dialog_type, const QStringLis case WorkerDialog::errFailedParsingLogs: DialogSec::errFailedParsingLogs( args.at(0) ); break; + case WorkerDialog::errDatabaseFileNotFound: + DialogSec::errDatabaseNotFound( args.at(0) ); + break; + case WorkerDialog::errDatabaseFileNotFile: + DialogSec::errDatabaseNotFile( args.at(0) ); + break; + case WorkerDialog::errDatabaseFileNotReadable: + DialogSec::errDatabaseNotReadable( args.at(0) ); + break; + case WorkerDialog::errDatabaseFileNotWritable: + DialogSec::errDatabaseNotWritable( args.at(0) ); + break; case WorkerDialog::errDatabaseFailedOpening: if ( args.size() < 2 ) { // do not throw, just print to stderr @@ -886,7 +884,11 @@ void Craplog::stopWorking( const bool successful ) this->db_edited = successful; if ( successful ) { // insert the hashes of the used files - this->hashOps.insertUsedHashes( this->db_hashes_path, this->used_files_hashes, this->current_web_server ); + try { + this->hashOps.insertUsedHashes( this->db_hashes_path, this->used_files_hashes, this->current_web_server ); + } catch (...) { + DialogSec::errFailedInsertUsedHashes(); + } } emit this->finishedWorking(); } diff --git a/logdoctor/modules/craplog/modules/formats.cpp b/logdoctor/modules/craplog/modules/formats.cpp index ba476998..e9737645 100644 --- a/logdoctor/modules/craplog/modules/formats.cpp +++ b/logdoctor/modules/craplog/modules/formats.cpp @@ -20,7 +20,7 @@ namespace /*private*/ \return The number of new lines in a single log line \see LogsFormat, processApacheFormatString(), processNginxFormatString() */ -unsigned countNewLines( std::string_view initial, std::string_view final, const std::vector& separators ) +size_t countNewLines( std::string_view initial, std::string_view final, const std::vector& separators ) { size_t nl{ 0ul }; nl += StringOps::count( initial, '\n' ); @@ -28,7 +28,7 @@ unsigned countNewLines( std::string_view initial, std::string_view final, const for ( const std::string& sep : separators ) { nl += StringOps::count( sep, '\n' ); } - return static_cast(nl); + return nl; } diff --git a/logdoctor/modules/craplog/modules/hash.cpp b/logdoctor/modules/craplog/modules/hash.cpp index 8d62ea7a..8a576a9f 100644 --- a/logdoctor/modules/craplog/modules/hash.cpp +++ b/logdoctor/modules/craplog/modules/hash.cpp @@ -11,14 +11,13 @@ #include "modules/dialogs.h" #include "modules/exceptions.h" +#include "modules/database/database.h" + #include "sha256.h" #include #include -#include -#include -#include void HashOps::setDialogLevel( const DialogsLevel new_level ) noexcept @@ -30,51 +29,29 @@ void HashOps::setDialogLevel( const DialogsLevel new_level ) noexcept // reads the database holding the already used hashes bool HashOps::loadUsedHashesLists( const std::string& db_path ) noexcept { - bool successful{ true }; - const QString db_name{ QString::fromStdString( db_path.substr( db_path.find_last_of( '/' ) + 1ul ) ) }; + DatabaseWrapper db{ DatabaseHandler::get( DatabaseType::Hashes ) }; - QSqlDatabase db{ QSqlDatabase::database(DatabasesNames::hashes) }; - db.setDatabaseName( QString::fromStdString( db_path ) ); + db.open( db_path, this->dialogs_level==DL_EXPLANATORY ); - if ( ! CheckSec::checkDatabaseFile( db_path, db_name ) ) { - successful &= false; + const QString stmt{ QStringLiteral(R"(SELECT "hash" FROM "%1";)") }; - } else if ( ! db.open() ) { - // error opening database - successful &= false; - QString err_msg; - if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg = db.lastError().text(); - } - DialogSec::errDatabaseFailedOpening( db_name, err_msg ); + for ( const auto& [wid,name] : this->ws_names ) { - } else { - QSqlQuery query{ db }; - for ( const auto& [wid,name] : this->ws_names ) { - if ( ! query.exec("SELECT hash FROM "+name+";") ) { - // error querying database - successful &= false; - DialogSec::errDatabaseFailedExecuting( db_name, query.lastQuery(), query.lastError().text() ); - break; - } else { - // iterate over results - while ( query.next() ) { - std::string hash{ query.value(0).toString().toStdString() }; - if ( hash.size() != 64ul ) { - // not a valid sha256 hash - continue; - } - this->hashes.at( wid ).push_back( hash ); - } + QueryWrapper query{ db.getQuery() }; + + query( stmt.arg( name ) ); + + while ( query->next() ) { + const QString hash{ query[0].toString() }; + if ( hash.size() != 64ul ) { + // not a valid sha256 hash + continue; } - if ( ! successful ) { break; } + this->hashes.at( wid ).push_back( hash.toStdString() ); } } - if ( db.isOpen() ) { - db.close(); - } - return successful; + return true; } @@ -93,7 +70,7 @@ void HashOps::digestFile( const std::string& file_path, std::string& hash ) } catch (...) { // failed as gzip, try as text file - if ( content.size() > 0ul ) { + if ( ! content.empty() ) { content.clear(); } IOutils::readFile( file_path, content ); @@ -124,7 +101,6 @@ void HashOps::digestFile( const std::string& file_path, std::string& hash ) SHA256 sha; sha.update( content ); - content.clear(); uint8_t* digest{ sha.digest() }; // return the hex digest hash.append( SHA256::toString(digest) ); @@ -143,133 +119,35 @@ bool HashOps::hasBeenUsed( const std::string &file_hash, const WebServer& web_se } -// insert the given hash/es in the relative list -bool HashOps::insertUsedHash( QSqlQuery& query, const QString& db_name, const std::string& hash, const WebServer& web_server ) noexcept +void HashOps::insertUsedHashes( const std::string& db_path, const std::vector& hashes, const WebServer& web_server ) { - bool successful{ true }; + const bool explain_msg{ this->dialogs_level > DL_ESSENTIAL }; + const bool explain_err{ this->dialogs_level == DL_EXPLANATORY }; + + DatabaseWrapper db{ DatabaseHandler::get( DatabaseType::Hashes ) }; + + db.open( db_path, explain_err ); + + db.startTransaction( explain_msg, explain_err ); + try { - if( ! VecOps::contains( this->hashes.at( web_server ), hash ) ) { - this->hashes.at( web_server ).push_back( hash ); - // insert tnto the database - QString stmt = QString("INSERT INTO %1 ( hash ) VALUES ( '%2' );") - .arg( this->ws_names.at(web_server), QString::fromStdString(hash).replace("'","''") ); - if ( ! query.exec( stmt ) ) { - // error opening database - successful &= false; - QString query_msg, err_msg; - if ( this->dialogs_level > DL_ESSENTIAL ) { - query_msg = "query.exec()"; - if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg = query.lastError().text(); - } - } - DialogSec::errDatabaseFailedExecuting( db_name, query_msg, err_msg ); + + for ( const std::string& hash : hashes ) { + + if ( VecOps::contains( this->hashes.at( web_server ), hash ) ) { + + db.getQuery()( QStringLiteral(R"(INSERT INTO "%1" ( hash ) VALUES ( '%2' );)") + .arg( this->ws_names.at(web_server), QString::fromStdString(hash).replace(QLatin1Char('\''),QLatin1String("''")) ) ); } - }/* else { - // hash already stored - }*/ + } + + db.commitTransaction( explain_msg, explain_err ); + + auto& used_hashes{ this->hashes.at( web_server ) }; + used_hashes.insert( used_hashes.end(), hashes.begin(), hashes.end() ); + } catch (...) { - // failed to insert the hash - successful &= false; + // rollback the transaction + db.rollbackTransaction( explain_msg, explain_err ); } - query.finish(); - return successful; -} - - -bool HashOps::insertUsedHashes( const std::string& db_path, const std::vector& hashes, const WebServer& web_server ) -{ - bool successful{ true }; - - QSqlDatabase db{ QSqlDatabase::database(DatabasesNames::hashes) }; - db.setDatabaseName( QString::fromStdString( db_path ) ); - - const QString db_name{ QString::fromStdString( db_path.substr( db_path.find_last_of( '/' ) + 1ul ) ) }; - - if ( ! CheckSec::checkDatabaseFile( db_path, db_name ) ) { - successful &= false; - - } else if ( ! db.open() ) { - // error opening database - successful &= false; - QString err_msg; - if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg = db.lastError().text(); - } - DialogSec::errDatabaseFailedOpening( db_name, err_msg ); - - } else { - QSqlQuery query{ db }; - if ( ! db.transaction() ) { - // error opening database - successful &= false; - QString stmt_msg, err_msg; - if ( this->dialogs_level > DL_ESSENTIAL ) { - stmt_msg = "db.transaction()"; - if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg = db.lastError().text(); - } - } - DialogSec::errDatabaseFailedExecuting( db_name, stmt_msg, err_msg ); - - } else { - - try { - for ( const std::string& hash : hashes ) { - successful = this->insertUsedHash( query, db_name, hash, web_server ); - if ( ! successful ) { - break; - } - } - query.finish(); - - if ( successful ) { - // commit the transaction - if ( ! db.commit() ) { - // error opening database - successful &= false; - QString stmt_msg, err_msg; - if ( this->dialogs_level > DL_ESSENTIAL ) { - stmt_msg = "db.commit()"; - if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg= db.lastError().text(); - } - } - DialogSec::errDatabaseFailedExecuting( db_name, stmt_msg, err_msg ); - } - } - if ( ! successful ) { - // rollback - throw (std::exception()); - } - - } catch (...) { - // wrongthing w3nt some.,. - successful &= false; - bool err_shown{ false }; - // rollback the transaction - if ( ! db.rollback() ) { - // error rolling back commits - QString stmt_msg, err_msg; - if ( this->dialogs_level > DL_ESSENTIAL ) { - stmt_msg = "db.rollback()"; - if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg = db.lastError().text(); - } - } - DialogSec::errDatabaseFailedExecuting( db_name, stmt_msg, err_msg ); - err_shown |= true; - } - if ( ! err_shown ) { - // show a message - QString msg{ DialogSec::tr("An error occured while working on the database\n\nAborting") }; - DialogSec::errGeneric( msg ); - } - } - } - } - if ( db.isOpen() ) { - db.close(); - } - return successful; } diff --git a/logdoctor/modules/craplog/modules/hash.h b/logdoctor/modules/craplog/modules/hash.h index 43beb2a1..0cfdcb7b 100644 --- a/logdoctor/modules/craplog/modules/hash.h +++ b/logdoctor/modules/craplog/modules/hash.h @@ -30,6 +30,7 @@ public: /*! \param db_path The path of the log files' Hashes database \return Whether the operation has been successful or not + \throw LogDoctorException */ bool loadUsedHashesLists( const std::string& db_path ) noexcept; @@ -55,9 +56,9 @@ public: \param db_path The path of the Hashes database \param hashes The list of hashes to insert \param web_server_id The ID of the Web Server which generated the file - \return Whether the operation has been successful or not + \throw LogDoctorException */ - bool insertUsedHashes( const std::string& db_path, const std::vector& hashes, const WebServer& web_server ); + void insertUsedHashes( const std::string& db_path, const std::vector& hashes, const WebServer& web_server ); private: @@ -79,11 +80,6 @@ private: {WS_IIS, {}} }; - - // Called by insertUsedHashes() - // Inserts a hash in the corresponding database table - bool insertUsedHash( QSqlQuery& query, const QString& db_name, const std::string& hash, const WebServer& web_server ) noexcept; - }; diff --git a/logdoctor/modules/craplog/modules/lib.h b/logdoctor/modules/craplog/modules/lib.h index 9a02ff56..9def5d41 100644 --- a/logdoctor/modules/craplog/modules/lib.h +++ b/logdoctor/modules/craplog/modules/lib.h @@ -28,9 +28,12 @@ enum class LogType //! Holds informations about a log file struct LogFile final { explicit LogFile() noexcept = default; - explicit LogFile - (const bool sel,const bool used,const size_t sz,const QString& nm,const std::string& hs,const std::string& pt) noexcept + explicit LogFile(const bool sel,const bool used,const size_t sz,const QString& nm,const std::string& hs,const std::string& pt) noexcept :selected{sel},used_already{used},size_{sz},name_{nm},hash_{hs},path_{pt}{} + LogFile(LogFile&& other) noexcept = default; + LogFile& operator=(LogFile&& other) noexcept = default; + LogFile(const LogFile& other) noexcept = default; + LogFile& operator=(const LogFile& other) noexcept = default; //! Wheter the file has been selected to be used or not inline bool isSelected() const noexcept { return this->selected; } diff --git a/logdoctor/modules/craplog/modules/logs.cpp b/logdoctor/modules/craplog/modules/logs.cpp index 7c7bcf06..8f31ef36 100644 --- a/logdoctor/modules/craplog/modules/logs.cpp +++ b/logdoctor/modules/craplog/modules/logs.cpp @@ -98,15 +98,10 @@ bool deepTypeCheck( const std::string& line, const LogsFormat& format ) noexcept // add the initial and final seps now n_sep += 2; - // the result is considered ture if more then a half of the seps was found - // and more than a half of the found separators was not blank - bool result{ false }; - if ( n_sep_found >= n_sep-1 - && n_blank_sep <= n_sep_found/2 ) { - result |= true; - } - - return result; + // the line is considered valid if all the seps have been found + // and more than a half of them were not blank + return n_sep_found >= n_sep-1 + && n_blank_sep <= n_sep_found/2; } } // namespace (private) diff --git a/logdoctor/modules/craplog/modules/workers/lib.h b/logdoctor/modules/craplog/modules/workers/lib.h index 9ad8c417..0f96fc54 100644 --- a/logdoctor/modules/craplog/modules/workers/lib.h +++ b/logdoctor/modules/craplog/modules/workers/lib.h @@ -14,6 +14,10 @@ enum class WorkerDialog { errDirNotExists, errFailedDefiningLogType, errFailedParsingLogs, + errDatabaseFileNotFound, + errDatabaseFileNotFile, + errDatabaseFileNotReadable, + errDatabaseFileNotWritable, errDatabaseFailedOpening, errDatabaseFailedExecuting, warnFileNotReadable, diff --git a/logdoctor/modules/craplog/modules/workers/parser.cpp b/logdoctor/modules/craplog/modules/workers/parser.cpp index ca690c22..35b239ad 100644 --- a/logdoctor/modules/craplog/modules/workers/parser.cpp +++ b/logdoctor/modules/craplog/modules/workers/parser.cpp @@ -12,10 +12,10 @@ #include "modules/dialogs.h" #include "modules/exceptions.h" +#include "modules/database/database.h" + #include "modules/craplog/modules/workers/lib.h" -#include -#include #include @@ -84,6 +84,7 @@ void CraplogParser::work() emit this->showDialog( WorkerDialog::errFailedParsingLogs, {e.what()} ); this->proceed &= false; + } // send the final data if ( ! this->proceed ) { @@ -187,11 +188,18 @@ void CraplogParser::joinLogLines() void CraplogParser::parseLogLines() { - const auto parseLine = [this]( const std::string& line, const LogsFormat& logs_format ) { + const auto parseLine{ [this]( const std::string& line, const LogsFormat& logs_format ) { this->data_collection.emplace_back( LogLineData(line, logs_format) ); this->parsed_size += line.size(); ++ this->parsed_lines; - }; + }}; + + const auto signal_emission_gap{ [](const size_t n_lines)->size_t{ + return n_lines>10000ul ? n_lines/1000ul + : n_lines>1000ul ? n_lines/100ul + : n_lines>100ul ? n_lines/10ul + : 10ul; + }}; // parse all the lines @@ -200,20 +208,19 @@ void CraplogParser::parseLogLines() const size_t nl{ this->logs_format.new_lines }; size_t send{ 0ul }; if ( nl == 0ul ) { - const size_t send_gap{ n_lines>1000ul ? n_lines/100 : n_lines>100ul ? n_lines/10 : 10 }; + const size_t send_gap{ signal_emission_gap(n_lines) }; const LogsFormat& lf {this->logs_format}; this->data_collection.reserve( n_lines ); for ( const std::string& line : this->logs_lines ) { parseLine( line, lf ); - if (send == send_gap) { + if (++send == send_gap) { this->sendPerfData(); send = 0ul; } - ++send; } } else { const size_t real_lines{ n_lines / (nl+1ul) }; - const size_t send_gap{ real_lines>1000ul ? real_lines/100 : real_lines>100ul ? real_lines/10 : 10 }; + const size_t send_gap{ signal_emission_gap(real_lines) }; const LogsFormat& lf {this->logs_format}; this->data_collection.reserve( real_lines ); for ( size_t i{0ul}; ilogs_lines.at( i ); } parseLine( line, lf ); - if (send == send_gap) { + if (++send == send_gap) { this->sendPerfData(); send = 0ul; } - ++send; } } this->sendPerfData(); @@ -241,136 +247,116 @@ void CraplogParser::storeLogLines() QString db_path{ QString::fromStdString( this->db_data_path ) }; QString db_name{ QString::fromStdString( this->db_data_path.substr( this->db_data_path.find_last_of( '/' ) + 1ul ) ) }; - QSqlDatabase db{ QSqlDatabase::database(DatabasesNames::data) }; - db.setDatabaseName( db_path ); + DatabaseWrapper db{ DatabaseHandler::get( DatabaseType::Hashes ) }; - if ( ! CheckSec::checkDatabaseFile( this->db_data_path, db_name ) ) { + db->setDatabaseName( db_path ); + + if ( ! this->checkDatabaseFile( db_name ) ) [[unlikely]] { this->proceed &= false; + return; + } - } else if ( ! db.open() ) { - // error opening database + if ( ! db->open() ) [[unlikely]] { this->proceed &= false; QString err_msg; if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg = db.lastError().text(); + err_msg = db->lastError().text(); } emit this->showDialog( WorkerDialog::errDatabaseFailedOpening, - {db_name, err_msg} ); + {db_name, err_msg} ); + return; + } - } else { + try { - try { - // ACID transaction - if ( ! db.transaction() ) { - // error opening database + if ( ! db->transaction() ) [[unlikely]] { + this->proceed &= false; + QString stmt_msg, err_msg; + if ( this->dialogs_level > DL_ESSENTIAL ) { + stmt_msg.append( "db.transaction()" ); + if ( this->dialogs_level == DL_EXPLANATORY ) { + err_msg = db->lastError().text(); + } + } + emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting, + {db_name, stmt_msg, err_msg} ); + return; + + } else if ( this->storeData( db, db_name ) ) [[likely]] { + + if ( ! db->commit() ) [[unlikely]] { this->proceed &= false; QString stmt_msg, err_msg; if ( this->dialogs_level > DL_ESSENTIAL ) { - stmt_msg = "db.transaction()"; + stmt_msg.append( "db.commit()" ); if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg = db.lastError().text(); + err_msg.append( db->lastError().text() ); } } emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting, {db_name, stmt_msg, err_msg} ); } + } - if ( this->proceed ) { - this->proceed &= this->storeData( db ); - } - - if ( this->proceed ) { - // commit the transaction - if ( ! db.commit() ) { - // error opening database - this->proceed &= false; - QString stmt_msg, err_msg; - if ( this->dialogs_level > DL_ESSENTIAL ) { - stmt_msg = "db.commit()"; - if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg= db.lastError().text(); - } - } - emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting, - {db_name, stmt_msg, err_msg} ); - } - } - if ( ! this->proceed ) { - // rollback - throw (std::exception()); - } - - } catch (...) { - // wrongthing w3nt some.,. - this->proceed &= false; - bool err_shown = false; + if ( ! this->proceed ) [[unlikely]] { // rollback the transaction - if ( ! db.rollback() ) { - // error rolling back commits + if ( ! db->rollback() ) { QString stmt_msg, err_msg; if ( this->dialogs_level > DL_ESSENTIAL ) { stmt_msg = "db.rollback()"; if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg = db.lastError().text(); + err_msg = db->lastError().text(); } } emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting, {db_name, stmt_msg, err_msg} ); - err_shown = true; - } - if ( ! err_shown ) { - // show a message - emit this->showDialog( - WorkerDialog::errGeneric, - {QString("%1\n\n%2").arg( - DialogSec::tr("An error occured while working on the database"), - DialogSec::tr("Aborting") )} ); + return; } } - if ( db.isOpen() ) { - db.close(); - } + } catch (...) { + // wrongthing w3nt some.,. + emit this->showDialog( + WorkerDialog::errGeneric, + {QStringLiteral("%1\n\n%2").arg( + DialogSec::tr("An error occured while working on the database"), + DialogSec::tr("Aborting") )} ); } } #define APPEND_TO_QUERY_AS_NUMBER(LOG_FIELD)\ if ( LOG_FIELD ) {\ - query_stmt += QString::fromStdString( *LOG_FIELD ).replace("'","''");\ + stmt.append( QString::fromStdString( *LOG_FIELD ).replace(QLatin1Char('\''),QLatin1String("''")) );\ } else {\ - query_stmt += QStringLiteral("NULL");\ + stmt.append( QStringLiteral("NULL") );\ } #define CONCAT_TO_QUERY_AS_NUMBER(LOG_FIELD)\ - query_stmt += QStringLiteral(", ");\ + stmt.append( QStringLiteral(", ") );\ APPEND_TO_QUERY_AS_NUMBER(LOG_FIELD) #define CONCAT_TO_QUERY_AS_STRING(LOG_FIELD)\ - query_stmt += QStringLiteral(", ");\ + stmt.append( QStringLiteral(", ") );\ if ( LOG_FIELD ) {\ - query_stmt += QString("'%1'").arg( QString::fromStdString( *LOG_FIELD ).replace("'","''") );\ + stmt.append( QString("'%1'").arg( QString::fromStdString( *LOG_FIELD ).replace(QLatin1Char('\''),QLatin1String("''")) ) );\ } else {\ - query_stmt += QStringLiteral("NULL");\ + stmt.append( QStringLiteral("NULL") );\ } // in IIS logs the user-agent is logged with '+' instead of ' ' (whitespace) #define CONCAT_TO_QUERY_USERAGENT(LOG_FIELD)\ - query_stmt += QStringLiteral(", ");\ + stmt.append( QStringLiteral(", ") );\ if ( LOG_FIELD ) {\ if ( this->web_server == WS_IIS ) {\ - query_stmt += QString("'%1'").arg( QString::fromStdString( *LOG_FIELD ).replace("+"," ").replace("'","''") );\ + stmt.append( QStringLiteral("'%1'").arg( QString::fromStdString( *LOG_FIELD ).replace(QLatin1Char('+'),QLatin1Char(' ')).replace(QLatin1Char('\''),QLatin1String("''")) ) );\ } else {\ - query_stmt += QString("'%1'").arg( QString::fromStdString( *LOG_FIELD ).replace("'","''") );\ + stmt.append( QStringLiteral("'%1'").arg( QString::fromStdString( *LOG_FIELD ).replace(QLatin1Char('\''),QLatin1String("''")) ) );\ }\ } else {\ - query_stmt += QStringLiteral("NULL");\ + stmt.append( QStringLiteral("NULL") );\ } -bool CraplogParser::storeData( QSqlDatabase& db ) +bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name ) { - const QString db_name{ QString::fromStdString( - this->db_data_path.substr( - this->db_data_path.find_last_of( '/' ) + 1ul ) ) }; - // get blacklist items const bool check_bl_cli { this->blacklists.at( 20 ).used }; @@ -383,22 +369,25 @@ bool CraplogParser::storeData( QSqlDatabase& db ) QString table; switch ( this->web_server ) { case WS_APACHE: - table += "apache"; + table.append( "apache" ); break; case WS_NGINX: - table += "nginx"; + table.append( "nginx" ); break; case WS_IIS: - table += "iis"; + table.append( "iis" ); break; default: // wrong WebServerID, but should be unreachable because of the previous operations throw WebServerException( "Unexpected WebServer: " + toString(this->web_server) ); } + const QString stmt_template{ + QStringLiteral(R"(INSERT INTO "%1" ("year", "month", "day", "hour", "minute", "second", "protocol", "method", "uri", "query", "response", "time_taken", "bytes_sent", "bytes_received", "referrer", "client", "user_agent", "cookie") )" + "VALUES (") + }; /*int perf_size;*/ - QSqlQuery query{ db }; // parse every row of data for ( const LogLineData& line_data : this->data_collection ) { @@ -410,9 +399,7 @@ bool CraplogParser::storeData( QSqlDatabase& db ) } } - // initialize the SQL statement - QString query_stmt{ "INSERT INTO \""+table+"\" (\"year\", \"month\", \"day\", \"hour\", \"minute\", \"second\", \"protocol\", \"method\", \"uri\", \"query\", \"response\", \"time_taken\", \"bytes_sent\", \"bytes_received\", \"referrer\", \"client\", \"user_agent\", \"cookie\") " - "VALUES (" }; + QString stmt{ stmt_template.arg( table ) }; // complete and execute the statement, binding NULL if not found @@ -441,29 +428,13 @@ bool CraplogParser::storeData( QSqlDatabase& db ) CONCAT_TO_QUERY_USERAGENT(line_data.user_agent) // 21 CONCAT_TO_QUERY_AS_STRING(line_data.cookie) // 22 - query_stmt += ");"; + stmt.append( ");" ); - // encode the statement - if ( ! query.prepare( query_stmt ) ) { - // error opening database - QString query_msg, err_msg; - if ( this->dialogs_level > DL_ESSENTIAL ) { - query_msg = "query.prepare()"; - if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg = query.lastError().text(); - } - } - emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting, - {db_name, query_msg, err_msg} ); - return false; - } - - // finalize this statement - if ( ! query.exec() ) { + if ( QSqlQuery query(*db); !query.exec( stmt ) ) [[unlikely]] { // error finalizing step QString query_msg, err_msg; if ( this->dialogs_level > DL_ESSENTIAL ) { - query_msg = "query.exec()"; + query_msg.append( "query.exec()" ); if ( this->dialogs_level == DL_EXPLANATORY ) { err_msg = query.lastError().text(); } @@ -472,10 +443,26 @@ bool CraplogParser::storeData( QSqlDatabase& db ) {db_name, query_msg, err_msg} ); return false; } - - // reset the statement to prepare for the next one - query.finish(); } return true; } + + +bool CraplogParser::checkDatabaseFile( const QString& db_name ) noexcept +{ + if ( ! IOutils::exists( this->db_data_path ) ) { + emit this->showDialog( WorkerDialog::errDatabaseFileNotFound, {db_name} ); + return false; + } else if ( ! IOutils::isFile( this->db_data_path ) ) { + emit this->showDialog( WorkerDialog::errDatabaseFileNotFile, {db_name} ); + return false; + } else if ( ! IOutils::checkFile( this->db_data_path, true ) ) { + emit this->showDialog( WorkerDialog::errDatabaseFileNotReadable, {db_name} ); + return false; + } else if ( ! IOutils::checkFile( this->db_data_path, false, true ) ) { + emit this->showDialog( WorkerDialog::errDatabaseFileNotWritable, {db_name} ); + return false; + } + return true; +} diff --git a/logdoctor/modules/craplog/modules/workers/parser.h b/logdoctor/modules/craplog/modules/workers/parser.h index e05011e9..6dd18432 100644 --- a/logdoctor/modules/craplog/modules/workers/parser.h +++ b/logdoctor/modules/craplog/modules/workers/parser.h @@ -14,9 +14,9 @@ struct BWlist; struct LogLineData; -enum class WorkerDialog; +class DatabaseWrapper; -class QSqlDatabase; +enum class WorkerDialog; class CraplogParser final : public QObject @@ -86,6 +86,8 @@ private: std::string db_data_path; std::string db_hashes_path; + bool checkDatabaseFile( const QString& db_name ) noexcept; + ////////////////////// //// PERFORMANCES //// @@ -136,10 +138,11 @@ private: //! Stores the data collection in the logs Collection database /*! \param db A database instance, already initizlized + \param db_name The database in use, to be shown in the dialogs \return Whether the operation has been successful or not \throw WebServerException */ - bool storeData( QSqlDatabase& db ); + bool storeData( DatabaseWrapper& db , const QString& db_name ); };