Improvements and updates

Improved Craplog and CraplogParser

logdoctor/modules/craplog/craplog.cpp

@@ -1,6 +1,8 @@
 
 #include "craplog.h"
 
+#include "globals/db_names.h"
+
 #include "utilities/checks.h"
 #include "utilities/gzip.h"
 #include "utilities/io.h"
@@ -21,6 +23,7 @@
 #include "modules/craplog/modules/workers/parser.h"
 
 #include <QPainter>
+#include <QWaitCondition>
 
 #include <filesystem>
 #include <thread>
@@ -97,11 +100,11 @@ const std::string& Craplog::getHashesDatabasePath() const noexcept
 
 void Craplog::setStatsDatabasePath( const std::string& path ) noexcept
 {
-    this->db_stats_path = path + "/collection.db";
+    this->db_stats_path = path + "/" + DatabasesNames::data;
 }
 void Craplog::setHashesDatabasePath( const std::string& path ) noexcept
 {
-    this->db_hashes_path = path + "/hashes.db";
+    this->db_hashes_path = path + "/" + DatabasesNames::hashes;
 }
 
 size_t Craplog::getWarningSize() const noexcept
@@ -669,14 +672,19 @@ void Craplog::startWorking( const Blacklists& blacklists )
 }
 void Craplog::hireWorker( const Blacklists& blacklists ) const
 {
+    std::vector<std::string> files;
+    files.reserve( this->log_files_to_use.size() );
+    std::transform(
+        this->log_files_to_use.cbegin(), this->log_files_to_use.cend(), std::back_inserter(files),
+        [](const auto& tpl){ return std::get<0>(tpl); } );
+
     CraplogParser* worker{ new CraplogParser(
         this->current_web_server,
         this->dialogs_level,
-        this->db_stats_path,
-        this->db_hashes_path,
         this->logs_formats.at( this->current_web_server ),
         blacklists.getConst( this->current_web_server ),
-        this->log_files_to_use
+        std::move(files),
+        this->db_stats_path
     ) };
     QThread* worker_thread{ new QThread() };
     worker->moveToThread( worker_thread );
@@ -695,6 +703,9 @@ void Craplog::hireWorker( const Blacklists& blacklists ) const
     // receive chart data, only received when worker has done
     connect( worker, &CraplogParser::chartData,
              this, &Craplog::updateChartData );
+    // store the files hashes
+    connect( worker, &CraplogParser::readyStoringData,
+             this, &Craplog::storeFilesHashes );
     // show a dialog
     connect( worker, &CraplogParser::showDialog,
              this, &Craplog::showWorkerDialog );
@@ -718,11 +729,6 @@ void Craplog::stopWorking( const bool successful )
     this->db_edited = successful;
     if ( successful ) {
         // insert the hashes of the used files
-        try {
-            this->hashOps.insertUsedHashes( this->db_hashes_path, this->used_files_hashes, this->current_web_server );
-        } catch (...) {
-            DialogSec::errFailedInsertUsedHashes();
-        }
     }
     emit this->finishedWorking();
 }
@@ -791,6 +797,18 @@ void Craplog::updateChartData( const size_t total_size, const size_t total_lines
     this->blacklisted_size = blacklisted_size;
 }
 
+void Craplog::storeFilesHashes( QWaitCondition* wc, bool* successful) noexcept
+{
+    try {
+        this->hashOps.insertUsedHashes( this->db_hashes_path, this->used_files_hashes, this->current_web_server );
+        *successful |= true;
+    } catch (...) {
+        DialogSec::errFailedInsertUsedHashes();
+        *successful &= false;
+    }
+    wc->wakeAll();
+}
+
 
 void Craplog::makeChart( const QChart::ChartTheme& theme, const std::unordered_map<std::string, QFont>& fonts, QChartView* size_chart ) const
 {

logdoctor/modules/craplog/craplog.h

@@ -13,6 +13,8 @@
 
 struct Blacklists;
 
+class QWaitCondition;
+
 
 //! Craplog
 /*!
@@ -279,6 +281,8 @@ signals:
 
     void pushLogFile( const LogFile& log_file );
 
+    void doneStoringFilesHashes( const bool successful );
+
     void finishedRefreshing();
 
     void finishedWorking();
@@ -310,6 +314,8 @@ public slots:
     void showWorkerDialog( const WorkerDialog dialog_type,
                            const QStringList args ) const noexcept;
 
+    void storeFilesHashes( QWaitCondition* wc, bool* successful ) noexcept;
+
 
 private:
 

logdoctor/modules/craplog/modules/workers/parser.cpp

@@ -12,26 +12,35 @@
 #include "modules/dialogs.h"
 #include "modules/exceptions.h"
 
-#include "modules/database/database.h"
-
 #include "modules/craplog/modules/workers/lib.h"
 
+#include <QWaitCondition>
+#include <QSqlDatabase>
+#include <QSqlQuery>
 #include <QSqlError>
+#include <QMutex>
 
 
-CraplogParser::CraplogParser( const WebServer web_server, const DialogsLevel dialogs_level, const std::string& db_data_path, const std::string& db_hashes_path, const LogsFormat& logs_format, const Blacklist& blacklist, const worker_files_t& log_files, QObject* parent )
+CraplogParser::CraplogParser( const WebServer web_server, const DialogsLevel dialogs_level, const LogsFormat& logs_format, const Blacklist& blacklist, worker_files_t&& log_files, const std::string& data_db_path, QObject* parent )
-    : QObject        { parent         }
+    : QObject        { parent                     }
-    , web_server     { web_server     }
+    , web_server     { web_server                 }
-    , dialogs_level  { dialogs_level  }
+    , dialogs_level  { dialogs_level              }
-    , db_data_path   { db_data_path   }
+    , db_path        { data_db_path               }
-    , db_hashes_path { db_hashes_path }
+    , db_name        { DatabasesConnections::data }
-    , blacklist      { blacklist      }
+    , blacklist      { blacklist                  }
-    , logs_format    { logs_format    }
+    , logs_format    { logs_format                }
-    , files_to_use   { log_files      }
+    , files_to_use   { std::move( log_files )     }
 {
 
 }
 
+CraplogParser::~CraplogParser()
+{
+    if ( QSqlDatabase::contains( this->db_conn_name ) ) {
+        QSqlDatabase::removeDatabase( this->db_conn_name );
+    }
+}
+
 
 void CraplogParser::sendPerfData() noexcept
 {
@@ -68,11 +77,25 @@ void CraplogParser::work()
         }
         // clear log lines data
         this->logs_lines.clear();
+        this->proceed = !this->data_collection.empty();
 
-        if ( this->proceed && !this->data_collection.empty() ) [[likely]]  {
+        if ( this->proceed ) [[likely]] {
+            // tell craplog to store the hashes before to proceed
+            QWaitCondition wc;
+            emit this->readyStoringData( &wc, &this->proceed );
+            QMutex mutex; mutex.lock();
+            wc.wait( &mutex );
+            mutex.unlock();
+        }
+
+        if ( this->proceed ) [[likely]] {
             // store the new data
-            this->storeLogLines();
+            auto db{ QSqlDatabase::addDatabase( "QSQLITE", this->db_conn_name ) };
+            this->storeLogLines( db );
             this->db_edited |= this->proceed;
+            if ( db.isOpen() ) {
+                db.close();
+            }
         }
 
     } catch ( GenericException& e ) {
@@ -119,12 +142,10 @@ void CraplogParser::joinLogLines()
 
     std::string aux;
     std::vector<std::string> content;
-    for ( const auto& file : this->files_to_use ) {
+    for ( const auto& file_path : this->files_to_use ) {
 
         if ( ! this->proceed ) { break; }
 
-        const std::string& file_path = std::get<0>( file );
-
         // collect lines
         try {
             // try reading
@@ -242,74 +263,69 @@ void CraplogParser::parseLogLines()
 
 
 
-void CraplogParser::storeLogLines()
+void CraplogParser::storeLogLines( QSqlDatabase& db )
 {
-    QString db_path{ QString::fromStdString( this->db_data_path ) };
+    db.setDatabaseName( QString::fromStdString( this->db_path ) );
-    QString db_name{ QString::fromStdString( this->db_data_path.substr( this->db_data_path.find_last_of( '/' ) + 1ul ) ) };
 
-    DatabaseWrapper db{ DatabaseHandler::get( DatabaseType::Hashes ) };
+    if ( ! this->checkDatabaseFile() ) [[unlikely]] {
-
-    db->setDatabaseName( db_path );
-
-    if ( ! this->checkDatabaseFile( db_name ) ) [[unlikely]] {
         this->proceed &= false;
         return;
     }
 
-    if ( ! db->open() ) [[unlikely]] {
+    if ( ! db.open() ) [[unlikely]] {
         this->proceed &= false;
         QString err_msg;
         if ( this->dialogs_level == DL_EXPLANATORY ) {
-            err_msg = db->lastError().text();
+            err_msg = db.lastError().text();
         }
         emit this->showDialog( WorkerDialog::errDatabaseFailedOpening,
-                              {db_name, err_msg} );
+                              {this->db_name, err_msg} );
         return;
     }
 
     try {
 
-        if ( ! db->transaction() ) [[unlikely]] {
+        if ( ! db.transaction() ) [[unlikely]] {
             this->proceed &= false;
             QString stmt_msg, err_msg;
             if ( this->dialogs_level > DL_ESSENTIAL ) {
                 stmt_msg.append( "db.transaction()" );
                 if ( this->dialogs_level == DL_EXPLANATORY ) {
-                    err_msg = db->lastError().text();
+                    err_msg = db.lastError().text();
                 }
             }
             emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting,
-                                   {db_name, stmt_msg, err_msg} );
+                                   {this->db_name, stmt_msg, err_msg} );
             return;
 
-        } else if ( this->storeData( db, db_name ) ) [[likely]] {
+        } else if ( this->storeData( db ) ) [[likely]] {
 
-            if ( ! db->commit() ) [[unlikely]] {
+            if ( ! db.commit() ) [[unlikely]] {
                 this->proceed &= false;
                 QString stmt_msg, err_msg;
                 if ( this->dialogs_level > DL_ESSENTIAL ) {
                     stmt_msg.append( "db.commit()" );
                     if ( this->dialogs_level == DL_EXPLANATORY ) {
-                        err_msg.append( db->lastError().text() );
+                        err_msg.append( db.lastError().text() );
                     }
                 }
                 emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting,
-                                       {db_name, stmt_msg, err_msg} );
+                                       {this->db_name, stmt_msg, err_msg} );
             }
         }
 
         if ( ! this->proceed ) [[unlikely]] {
             // rollback the transaction
-            if ( ! db->rollback() ) {
+            if ( ! db.rollback() ) {
                 QString stmt_msg, err_msg;
                 if ( this->dialogs_level > DL_ESSENTIAL ) {
                     stmt_msg = "db.rollback()";
                     if ( this->dialogs_level == DL_EXPLANATORY ) {
-                        err_msg = db->lastError().text();
+                        err_msg = db.lastError().text();
                     }
                 }
                 emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting,
-                                       {db_name, stmt_msg, err_msg} );
+                                       {this->db_name, stmt_msg, err_msg} );
                 return;
             }
         }
@@ -355,7 +371,7 @@ void CraplogParser::storeLogLines()
         stmt.append( QStringLiteral("NULL") );\
     }
 
-bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name )
+bool CraplogParser::storeData( QSqlDatabase& db )
 {
     // get blacklist items
     const bool check_bl_cli { this->blacklist.client.used };
@@ -430,7 +446,7 @@ bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name )
 
         stmt.append( ");" );
 
-        if ( QSqlQuery query(*db); !query.exec( stmt ) ) [[unlikely]] {
+        if ( QSqlQuery query(db); !query.exec( stmt ) ) [[unlikely]] {
             // error finalizing step
             QString query_msg, err_msg;
             if ( this->dialogs_level > DL_ESSENTIAL ) {
@@ -440,7 +456,7 @@ bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name )
                 }
             }
             emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting,
-                                   {db_name, query_msg, err_msg} );
+                                   {this->db_name, query_msg, err_msg} );
             return false;
         }
     }
@@ -449,19 +465,19 @@ bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name )
 }
 
 
-bool CraplogParser::checkDatabaseFile( const QString& db_name ) noexcept
+bool CraplogParser::checkDatabaseFile() noexcept
 {
-    if ( ! IOutils::exists( this->db_data_path ) ) {
+    if ( ! IOutils::exists( this->db_path ) ) {
-        emit this->showDialog( WorkerDialog::errDatabaseFileNotFound, {db_name} );
+        emit this->showDialog( WorkerDialog::errDatabaseFileNotFound, {this->db_name} );
         return false;
-    } else if ( ! IOutils::isFile( this->db_data_path ) ) {
+    } else if ( ! IOutils::isFile( this->db_path ) ) {
-        emit this->showDialog( WorkerDialog::errDatabaseFileNotFile, {db_name} );
+        emit this->showDialog( WorkerDialog::errDatabaseFileNotFile, {this->db_name} );
         return false;
-    } else if ( ! IOutils::checkFile( this->db_data_path, true ) ) {
+    } else if ( ! IOutils::checkFile( this->db_path, true ) ) {
-        emit this->showDialog( WorkerDialog::errDatabaseFileNotReadable, {db_name} );
+        emit this->showDialog( WorkerDialog::errDatabaseFileNotReadable, {this->db_name} );
         return false;
-    } else if ( ! IOutils::checkFile( this->db_data_path, false, true ) ) {
+    } else if ( ! IOutils::checkFile( this->db_path, false, true ) ) {
-        emit this->showDialog( WorkerDialog::errDatabaseFileNotWritable, {db_name} );
+        emit this->showDialog( WorkerDialog::errDatabaseFileNotWritable, {this->db_name} );
         return false;
     }
     return true;

logdoctor/modules/craplog/modules/workers/parser.h

@@ -19,27 +19,31 @@ class DatabaseWrapper;
 
 enum class WorkerDialog;
 
+class QWaitCondition;
+
+class QSqlDatabase;
+
 
 class CraplogParser final : public QObject
 {
     Q_OBJECT
 
-    using logs_file_t     = std::tuple<std::string,std::string>;
+    using worker_files_t  = std::vector<std::string>;
-    using worker_files_t  = std::vector<logs_file_t>;
 
 public:
 
     explicit CraplogParser(
         const WebServer web_server,
         const DialogsLevel dialogs_level,
-        const std::string& db_data_path,
-        const std::string& db_hashes_path,
         const LogsFormat& logs_format,
         const Blacklist& blacklist,
-        const worker_files_t& log_files,
+        worker_files_t&& log_files,
+        const std::string& data_db_path,
         QObject* parent=nullptr
     );
 
+    ~CraplogParser();
+
 signals:
 
     void perfData(
@@ -55,6 +59,8 @@ signals:
         const WorkerDialog dialog_type,
         const QStringList arg );
 
+    void readyStoringData( QWaitCondition* wc, bool* successful );
+
     void startedParsing();
 
     void finishedParsing();
@@ -83,10 +89,13 @@ private:
     //// DATABASES ////
 
     bool db_edited{ false };
-    std::string db_data_path;
-    std::string db_hashes_path;
 
-    bool checkDatabaseFile( const QString& db_name ) noexcept;
+    std::string db_path;
+
+    const QString db_name;
+    const QString db_conn_name{ QStringLiteral("Parser_LogsData") };
+
+    bool checkDatabaseFile() noexcept;
 
     //////////////////////
     //// PERFORMANCES ////
@@ -130,19 +139,18 @@ private:
 
     //! Handles the process of storing data in the database
     /*!
+        \param db The database instance, not initialized already
         \see storeData()
     */
-    void storeLogLines();
+    void storeLogLines( QSqlDatabase& db );
 
     //! Stores the data collection in the logs Collection database
     /*!
-        \param db A database instance, already initizlized
+        \param db The database instance, already initialized
-        \param db_name The database in use, to be shown in the dialogs
         \return Whether the operation has been successful or not
         \throw WebServerException
     */
-    bool storeData( DatabaseWrapper& db , const QString& db_name );
+    bool storeData( QSqlDatabase& db );
-
 };