From 7df22305daf8437fbf1239a6e7cc7b7071641655 Mon Sep 17 00:00:00 2001 From: Valentino Orlandi Date: Sun, 4 Feb 2024 21:52:33 +0100 Subject: [PATCH] Improvements and updates Improved Craplog and CraplogParser --- logdoctor/modules/craplog/craplog.cpp | 38 ++++-- logdoctor/modules/craplog/craplog.h | 6 + .../craplog/modules/workers/parser.cpp | 114 ++++++++++-------- .../modules/craplog/modules/workers/parser.h | 34 ++++-- 4 files changed, 120 insertions(+), 72 deletions(-) diff --git a/logdoctor/modules/craplog/craplog.cpp b/logdoctor/modules/craplog/craplog.cpp index d0b71983..ee37455c 100644 --- a/logdoctor/modules/craplog/craplog.cpp +++ b/logdoctor/modules/craplog/craplog.cpp @@ -1,6 +1,8 @@ #include "craplog.h" +#include "globals/db_names.h" + #include "utilities/checks.h" #include "utilities/gzip.h" #include "utilities/io.h" @@ -21,6 +23,7 @@ #include "modules/craplog/modules/workers/parser.h" #include +#include #include #include @@ -97,11 +100,11 @@ const std::string& Craplog::getHashesDatabasePath() const noexcept void Craplog::setStatsDatabasePath( const std::string& path ) noexcept { - this->db_stats_path = path + "/collection.db"; + this->db_stats_path = path + "/" + DatabasesNames::data; } void Craplog::setHashesDatabasePath( const std::string& path ) noexcept { - this->db_hashes_path = path + "/hashes.db"; + this->db_hashes_path = path + "/" + DatabasesNames::hashes; } size_t Craplog::getWarningSize() const noexcept @@ -669,14 +672,19 @@ void Craplog::startWorking( const Blacklists& blacklists ) } void Craplog::hireWorker( const Blacklists& blacklists ) const { + std::vector files; + files.reserve( this->log_files_to_use.size() ); + std::transform( + this->log_files_to_use.cbegin(), this->log_files_to_use.cend(), std::back_inserter(files), + [](const auto& tpl){ return std::get<0>(tpl); } ); + CraplogParser* worker{ new CraplogParser( this->current_web_server, this->dialogs_level, - this->db_stats_path, - this->db_hashes_path, this->logs_formats.at( this->current_web_server ), blacklists.getConst( this->current_web_server ), - this->log_files_to_use + std::move(files), + this->db_stats_path ) }; QThread* worker_thread{ new QThread() }; worker->moveToThread( worker_thread ); @@ -695,6 +703,9 @@ void Craplog::hireWorker( const Blacklists& blacklists ) const // receive chart data, only received when worker has done connect( worker, &CraplogParser::chartData, this, &Craplog::updateChartData ); + // store the files hashes + connect( worker, &CraplogParser::readyStoringData, + this, &Craplog::storeFilesHashes ); // show a dialog connect( worker, &CraplogParser::showDialog, this, &Craplog::showWorkerDialog ); @@ -718,11 +729,6 @@ void Craplog::stopWorking( const bool successful ) this->db_edited = successful; if ( successful ) { // insert the hashes of the used files - try { - this->hashOps.insertUsedHashes( this->db_hashes_path, this->used_files_hashes, this->current_web_server ); - } catch (...) { - DialogSec::errFailedInsertUsedHashes(); - } } emit this->finishedWorking(); } @@ -791,6 +797,18 @@ void Craplog::updateChartData( const size_t total_size, const size_t total_lines this->blacklisted_size = blacklisted_size; } +void Craplog::storeFilesHashes( QWaitCondition* wc, bool* successful) noexcept +{ + try { + this->hashOps.insertUsedHashes( this->db_hashes_path, this->used_files_hashes, this->current_web_server ); + *successful |= true; + } catch (...) { + DialogSec::errFailedInsertUsedHashes(); + *successful &= false; + } + wc->wakeAll(); +} + void Craplog::makeChart( const QChart::ChartTheme& theme, const std::unordered_map& fonts, QChartView* size_chart ) const { diff --git a/logdoctor/modules/craplog/craplog.h b/logdoctor/modules/craplog/craplog.h index 3c6f004d..3c2ad237 100644 --- a/logdoctor/modules/craplog/craplog.h +++ b/logdoctor/modules/craplog/craplog.h @@ -13,6 +13,8 @@ struct Blacklists; +class QWaitCondition; + //! Craplog /*! @@ -279,6 +281,8 @@ signals: void pushLogFile( const LogFile& log_file ); + void doneStoringFilesHashes( const bool successful ); + void finishedRefreshing(); void finishedWorking(); @@ -310,6 +314,8 @@ public slots: void showWorkerDialog( const WorkerDialog dialog_type, const QStringList args ) const noexcept; + void storeFilesHashes( QWaitCondition* wc, bool* successful ) noexcept; + private: diff --git a/logdoctor/modules/craplog/modules/workers/parser.cpp b/logdoctor/modules/craplog/modules/workers/parser.cpp index 71f9a2c4..c9ce40a1 100644 --- a/logdoctor/modules/craplog/modules/workers/parser.cpp +++ b/logdoctor/modules/craplog/modules/workers/parser.cpp @@ -12,26 +12,35 @@ #include "modules/dialogs.h" #include "modules/exceptions.h" -#include "modules/database/database.h" - #include "modules/craplog/modules/workers/lib.h" +#include +#include +#include #include +#include -CraplogParser::CraplogParser( const WebServer web_server, const DialogsLevel dialogs_level, const std::string& db_data_path, const std::string& db_hashes_path, const LogsFormat& logs_format, const Blacklist& blacklist, const worker_files_t& log_files, QObject* parent ) - : QObject { parent } - , web_server { web_server } - , dialogs_level { dialogs_level } - , db_data_path { db_data_path } - , db_hashes_path { db_hashes_path } - , blacklist { blacklist } - , logs_format { logs_format } - , files_to_use { log_files } +CraplogParser::CraplogParser( const WebServer web_server, const DialogsLevel dialogs_level, const LogsFormat& logs_format, const Blacklist& blacklist, worker_files_t&& log_files, const std::string& data_db_path, QObject* parent ) + : QObject { parent } + , web_server { web_server } + , dialogs_level { dialogs_level } + , db_path { data_db_path } + , db_name { DatabasesConnections::data } + , blacklist { blacklist } + , logs_format { logs_format } + , files_to_use { std::move( log_files ) } { } +CraplogParser::~CraplogParser() +{ + if ( QSqlDatabase::contains( this->db_conn_name ) ) { + QSqlDatabase::removeDatabase( this->db_conn_name ); + } +} + void CraplogParser::sendPerfData() noexcept { @@ -68,11 +77,25 @@ void CraplogParser::work() } // clear log lines data this->logs_lines.clear(); + this->proceed = !this->data_collection.empty(); - if ( this->proceed && !this->data_collection.empty() ) [[likely]] { + if ( this->proceed ) [[likely]] { + // tell craplog to store the hashes before to proceed + QWaitCondition wc; + emit this->readyStoringData( &wc, &this->proceed ); + QMutex mutex; mutex.lock(); + wc.wait( &mutex ); + mutex.unlock(); + } + + if ( this->proceed ) [[likely]] { // store the new data - this->storeLogLines(); + auto db{ QSqlDatabase::addDatabase( "QSQLITE", this->db_conn_name ) }; + this->storeLogLines( db ); this->db_edited |= this->proceed; + if ( db.isOpen() ) { + db.close(); + } } } catch ( GenericException& e ) { @@ -119,12 +142,10 @@ void CraplogParser::joinLogLines() std::string aux; std::vector content; - for ( const auto& file : this->files_to_use ) { + for ( const auto& file_path : this->files_to_use ) { if ( ! this->proceed ) { break; } - const std::string& file_path = std::get<0>( file ); - // collect lines try { // try reading @@ -242,74 +263,69 @@ void CraplogParser::parseLogLines() -void CraplogParser::storeLogLines() +void CraplogParser::storeLogLines( QSqlDatabase& db ) { - QString db_path{ QString::fromStdString( this->db_data_path ) }; - QString db_name{ QString::fromStdString( this->db_data_path.substr( this->db_data_path.find_last_of( '/' ) + 1ul ) ) }; + db.setDatabaseName( QString::fromStdString( this->db_path ) ); - DatabaseWrapper db{ DatabaseHandler::get( DatabaseType::Hashes ) }; - - db->setDatabaseName( db_path ); - - if ( ! this->checkDatabaseFile( db_name ) ) [[unlikely]] { + if ( ! this->checkDatabaseFile() ) [[unlikely]] { this->proceed &= false; return; } - if ( ! db->open() ) [[unlikely]] { + if ( ! db.open() ) [[unlikely]] { this->proceed &= false; QString err_msg; if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg = db->lastError().text(); + err_msg = db.lastError().text(); } emit this->showDialog( WorkerDialog::errDatabaseFailedOpening, - {db_name, err_msg} ); + {this->db_name, err_msg} ); return; } try { - if ( ! db->transaction() ) [[unlikely]] { + if ( ! db.transaction() ) [[unlikely]] { this->proceed &= false; QString stmt_msg, err_msg; if ( this->dialogs_level > DL_ESSENTIAL ) { stmt_msg.append( "db.transaction()" ); if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg = db->lastError().text(); + err_msg = db.lastError().text(); } } emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting, - {db_name, stmt_msg, err_msg} ); + {this->db_name, stmt_msg, err_msg} ); return; - } else if ( this->storeData( db, db_name ) ) [[likely]] { + } else if ( this->storeData( db ) ) [[likely]] { - if ( ! db->commit() ) [[unlikely]] { + if ( ! db.commit() ) [[unlikely]] { this->proceed &= false; QString stmt_msg, err_msg; if ( this->dialogs_level > DL_ESSENTIAL ) { stmt_msg.append( "db.commit()" ); if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg.append( db->lastError().text() ); + err_msg.append( db.lastError().text() ); } } emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting, - {db_name, stmt_msg, err_msg} ); + {this->db_name, stmt_msg, err_msg} ); } } if ( ! this->proceed ) [[unlikely]] { // rollback the transaction - if ( ! db->rollback() ) { + if ( ! db.rollback() ) { QString stmt_msg, err_msg; if ( this->dialogs_level > DL_ESSENTIAL ) { stmt_msg = "db.rollback()"; if ( this->dialogs_level == DL_EXPLANATORY ) { - err_msg = db->lastError().text(); + err_msg = db.lastError().text(); } } emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting, - {db_name, stmt_msg, err_msg} ); + {this->db_name, stmt_msg, err_msg} ); return; } } @@ -355,7 +371,7 @@ void CraplogParser::storeLogLines() stmt.append( QStringLiteral("NULL") );\ } -bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name ) +bool CraplogParser::storeData( QSqlDatabase& db ) { // get blacklist items const bool check_bl_cli { this->blacklist.client.used }; @@ -430,7 +446,7 @@ bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name ) stmt.append( ");" ); - if ( QSqlQuery query(*db); !query.exec( stmt ) ) [[unlikely]] { + if ( QSqlQuery query(db); !query.exec( stmt ) ) [[unlikely]] { // error finalizing step QString query_msg, err_msg; if ( this->dialogs_level > DL_ESSENTIAL ) { @@ -440,7 +456,7 @@ bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name ) } } emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting, - {db_name, query_msg, err_msg} ); + {this->db_name, query_msg, err_msg} ); return false; } } @@ -449,19 +465,19 @@ bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name ) } -bool CraplogParser::checkDatabaseFile( const QString& db_name ) noexcept +bool CraplogParser::checkDatabaseFile() noexcept { - if ( ! IOutils::exists( this->db_data_path ) ) { - emit this->showDialog( WorkerDialog::errDatabaseFileNotFound, {db_name} ); + if ( ! IOutils::exists( this->db_path ) ) { + emit this->showDialog( WorkerDialog::errDatabaseFileNotFound, {this->db_name} ); return false; - } else if ( ! IOutils::isFile( this->db_data_path ) ) { - emit this->showDialog( WorkerDialog::errDatabaseFileNotFile, {db_name} ); + } else if ( ! IOutils::isFile( this->db_path ) ) { + emit this->showDialog( WorkerDialog::errDatabaseFileNotFile, {this->db_name} ); return false; - } else if ( ! IOutils::checkFile( this->db_data_path, true ) ) { - emit this->showDialog( WorkerDialog::errDatabaseFileNotReadable, {db_name} ); + } else if ( ! IOutils::checkFile( this->db_path, true ) ) { + emit this->showDialog( WorkerDialog::errDatabaseFileNotReadable, {this->db_name} ); return false; - } else if ( ! IOutils::checkFile( this->db_data_path, false, true ) ) { - emit this->showDialog( WorkerDialog::errDatabaseFileNotWritable, {db_name} ); + } else if ( ! IOutils::checkFile( this->db_path, false, true ) ) { + emit this->showDialog( WorkerDialog::errDatabaseFileNotWritable, {this->db_name} ); return false; } return true; diff --git a/logdoctor/modules/craplog/modules/workers/parser.h b/logdoctor/modules/craplog/modules/workers/parser.h index 809ed4e2..51a7a5d3 100644 --- a/logdoctor/modules/craplog/modules/workers/parser.h +++ b/logdoctor/modules/craplog/modules/workers/parser.h @@ -19,27 +19,31 @@ class DatabaseWrapper; enum class WorkerDialog; +class QWaitCondition; + +class QSqlDatabase; + class CraplogParser final : public QObject { Q_OBJECT - using logs_file_t = std::tuple; - using worker_files_t = std::vector; + using worker_files_t = std::vector; public: explicit CraplogParser( const WebServer web_server, const DialogsLevel dialogs_level, - const std::string& db_data_path, - const std::string& db_hashes_path, const LogsFormat& logs_format, const Blacklist& blacklist, - const worker_files_t& log_files, + worker_files_t&& log_files, + const std::string& data_db_path, QObject* parent=nullptr ); + ~CraplogParser(); + signals: void perfData( @@ -55,6 +59,8 @@ signals: const WorkerDialog dialog_type, const QStringList arg ); + void readyStoringData( QWaitCondition* wc, bool* successful ); + void startedParsing(); void finishedParsing(); @@ -83,10 +89,13 @@ private: //// DATABASES //// bool db_edited{ false }; - std::string db_data_path; - std::string db_hashes_path; - bool checkDatabaseFile( const QString& db_name ) noexcept; + std::string db_path; + + const QString db_name; + const QString db_conn_name{ QStringLiteral("Parser_LogsData") }; + + bool checkDatabaseFile() noexcept; ////////////////////// //// PERFORMANCES //// @@ -130,19 +139,18 @@ private: //! Handles the process of storing data in the database /*! + \param db The database instance, not initialized already \see storeData() */ - void storeLogLines(); + void storeLogLines( QSqlDatabase& db ); //! Stores the data collection in the logs Collection database /*! - \param db A database instance, already initizlized - \param db_name The database in use, to be shown in the dialogs + \param db The database instance, already initialized \return Whether the operation has been successful or not \throw WebServerException */ - bool storeData( DatabaseWrapper& db , const QString& db_name ); - + bool storeData( QSqlDatabase& db ); };