elB4RTO/LogDoctor
elB4RTO
/
LogDoctor
2
1
Fork 0
Code Issues 6 Pull Requests Projects 1 Releases 7 Wiki Activity

Improvements and updates 

Improved Craplog and CraplogParser
This commit is contained in:
Valentino Orlandi 2024-02-04 21:52:33 +01:00
parent 9a1456407e
commit 7df22305da
Signed by: elB4RTO
GPG Key ID: 1719E976DB2D4E71
4 changed files with 120 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
logdoctor/modules/craplog/craplog.cpp
View File

@ -1,6 +1,8 @@
#include "craplog.h"
#include "globals/db_names.h"
#include "utilities/checks.h"
#include "utilities/gzip.h"
#include "utilities/io.h"
@ -21,6 +23,7 @@
#include "modules/craplog/modules/workers/parser.h"
#include <QPainter>
#include <QWaitCondition>
#include <filesystem>
#include <thread>
@ -97,11 +100,11 @@ const std::string& Craplog::getHashesDatabasePath() const noexcept
void Craplog::setStatsDatabasePath( const std::string& path ) noexcept
{
this->db_stats_path = path + "/collection.db";
this->db_stats_path = path + "/" + DatabasesNames::data;
}
void Craplog::setHashesDatabasePath( const std::string& path ) noexcept
{
this->db_hashes_path = path + "/hashes.db";
this->db_hashes_path = path + "/" + DatabasesNames::hashes;
}
size_t Craplog::getWarningSize() const noexcept
@ -669,14 +672,19 @@ void Craplog::startWorking( const Blacklists& blacklists )
}
void Craplog::hireWorker( const Blacklists& blacklists ) const
{
std::vector<std::string> files;
files.reserve( this->log_files_to_use.size() );
std::transform(
this->log_files_to_use.cbegin(), this->log_files_to_use.cend(), std::back_inserter(files),
[](const auto& tpl){ return std::get<0>(tpl); } );
CraplogParser* worker{ new CraplogParser(
this->current_web_server,
this->dialogs_level,
this->db_stats_path,
this->db_hashes_path,
this->logs_formats.at( this->current_web_server ),
blacklists.getConst( this->current_web_server ),
this->log_files_to_use
std::move(files),
this->db_stats_path
) };
QThread* worker_thread{ new QThread() };
worker->moveToThread( worker_thread );
@ -695,6 +703,9 @@ void Craplog::hireWorker( const Blacklists& blacklists ) const
// receive chart data, only received when worker has done
connect( worker, &CraplogParser::chartData,
this, &Craplog::updateChartData );
// store the files hashes
connect( worker, &CraplogParser::readyStoringData,
this, &Craplog::storeFilesHashes );
// show a dialog
connect( worker, &CraplogParser::showDialog,
this, &Craplog::showWorkerDialog );
@ -718,11 +729,6 @@ void Craplog::stopWorking( const bool successful )
this->db_edited = successful;
if ( successful ) {
// insert the hashes of the used files
try {
this->hashOps.insertUsedHashes( this->db_hashes_path, this->used_files_hashes, this->current_web_server );
} catch (...) {
DialogSec::errFailedInsertUsedHashes();
}
}
emit this->finishedWorking();
}
@ -791,6 +797,18 @@ void Craplog::updateChartData( const size_t total_size, const size_t total_lines
this->blacklisted_size = blacklisted_size;
}
void Craplog::storeFilesHashes( QWaitCondition* wc, bool* successful) noexcept
{
try {
this->hashOps.insertUsedHashes( this->db_hashes_path, this->used_files_hashes, this->current_web_server );
*successful |= true;
} catch (...) {
DialogSec::errFailedInsertUsedHashes();
*successful &= false;
}
wc->wakeAll();
}
void Craplog::makeChart( const QChart::ChartTheme& theme, const std::unordered_map<std::string, QFont>& fonts, QChartView* size_chart ) const
{

6
logdoctor/modules/craplog/craplog.h
View File

@ -13,6 +13,8 @@
struct Blacklists;
class QWaitCondition;
//! Craplog
/*!
@ -279,6 +281,8 @@ signals:
void pushLogFile( const LogFile& log_file );
void doneStoringFilesHashes( const bool successful );
void finishedRefreshing();
void finishedWorking();
@ -310,6 +314,8 @@ public slots:
void showWorkerDialog( const WorkerDialog dialog_type,
const QStringList args ) const noexcept;
void storeFilesHashes( QWaitCondition* wc, bool* successful ) noexcept;
private:

114
logdoctor/modules/craplog/modules/workers/parser.cpp
View File

@ -12,26 +12,35 @@
#include "modules/dialogs.h"
#include "modules/exceptions.h"
#include "modules/database/database.h"
#include "modules/craplog/modules/workers/lib.h"
#include <QWaitCondition>
#include <QSqlDatabase>
#include <QSqlQuery>
#include <QSqlError>
#include <QMutex>
CraplogParser::CraplogParser( const WebServer web_server, const DialogsLevel dialogs_level, const std::string& db_data_path, const std::string& db_hashes_path, const LogsFormat& logs_format, const Blacklist& blacklist, const worker_files_t& log_files, QObject* parent )
: QObject { parent }
, web_server { web_server }
, dialogs_level { dialogs_level }
, db_data_path { db_data_path }
, db_hashes_path { db_hashes_path }
, blacklist { blacklist }
, logs_format { logs_format }
, files_to_use { log_files }
CraplogParser::CraplogParser( const WebServer web_server, const DialogsLevel dialogs_level, const LogsFormat& logs_format, const Blacklist& blacklist, worker_files_t&& log_files, const std::string& data_db_path, QObject* parent )
: QObject { parent }
, web_server { web_server }
, dialogs_level { dialogs_level }
, db_path { data_db_path }
, db_name { DatabasesConnections::data }
, blacklist { blacklist }
, logs_format { logs_format }
, files_to_use { std::move( log_files ) }
{
}
CraplogParser::~CraplogParser()
{
if ( QSqlDatabase::contains( this->db_conn_name ) ) {
QSqlDatabase::removeDatabase( this->db_conn_name );
}
}
void CraplogParser::sendPerfData() noexcept
{
@ -68,11 +77,25 @@ void CraplogParser::work()
}
// clear log lines data
this->logs_lines.clear();
this->proceed = !this->data_collection.empty();
if ( this->proceed && !this->data_collection.empty() ) [[likely]] {
if ( this->proceed ) [[likely]] {
// tell craplog to store the hashes before to proceed
QWaitCondition wc;
emit this->readyStoringData( &wc, &this->proceed );
QMutex mutex; mutex.lock();
wc.wait( &mutex );
mutex.unlock();
}
if ( this->proceed ) [[likely]] {
// store the new data
this->storeLogLines();
auto db{ QSqlDatabase::addDatabase( "QSQLITE", this->db_conn_name ) };
this->storeLogLines( db );
this->db_edited |= this->proceed;
if ( db.isOpen() ) {
db.close();
}
}
} catch ( GenericException& e ) {
@ -119,12 +142,10 @@ void CraplogParser::joinLogLines()
std::string aux;
std::vector<std::string> content;
for ( const auto& file : this->files_to_use ) {
for ( const auto& file_path : this->files_to_use ) {
if ( ! this->proceed ) { break; }
const std::string& file_path = std::get<0>( file );
// collect lines
try {
// try reading
@ -242,74 +263,69 @@ void CraplogParser::parseLogLines()
void CraplogParser::storeLogLines()
void CraplogParser::storeLogLines( QSqlDatabase& db )
{
QString db_path{ QString::fromStdString( this->db_data_path ) };
QString db_name{ QString::fromStdString( this->db_data_path.substr( this->db_data_path.find_last_of( '/' ) + 1ul ) ) };
db.setDatabaseName( QString::fromStdString( this->db_path ) );
DatabaseWrapper db{ DatabaseHandler::get( DatabaseType::Hashes ) };
db->setDatabaseName( db_path );
if ( ! this->checkDatabaseFile( db_name ) ) [[unlikely]] {
if ( ! this->checkDatabaseFile() ) [[unlikely]] {
this->proceed &= false;
return;
}
if ( ! db->open() ) [[unlikely]] {
if ( ! db.open() ) [[unlikely]] {
this->proceed &= false;
QString err_msg;
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = db->lastError().text();
err_msg = db.lastError().text();
}
emit this->showDialog( WorkerDialog::errDatabaseFailedOpening,
{db_name, err_msg} );
{this->db_name, err_msg} );
return;
}
try {
if ( ! db->transaction() ) [[unlikely]] {
if ( ! db.transaction() ) [[unlikely]] {
this->proceed &= false;
QString stmt_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
stmt_msg.append( "db.transaction()" );
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = db->lastError().text();
err_msg = db.lastError().text();
}
}
emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting,
{db_name, stmt_msg, err_msg} );
{this->db_name, stmt_msg, err_msg} );
return;
} else if ( this->storeData( db, db_name ) ) [[likely]] {
} else if ( this->storeData( db ) ) [[likely]] {
if ( ! db->commit() ) [[unlikely]] {
if ( ! db.commit() ) [[unlikely]] {
this->proceed &= false;
QString stmt_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
stmt_msg.append( "db.commit()" );
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg.append( db->lastError().text() );
err_msg.append( db.lastError().text() );
}
}
emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting,
{db_name, stmt_msg, err_msg} );
{this->db_name, stmt_msg, err_msg} );
}
}
if ( ! this->proceed ) [[unlikely]] {
// rollback the transaction
if ( ! db->rollback() ) {
if ( ! db.rollback() ) {
QString stmt_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
stmt_msg = "db.rollback()";
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = db->lastError().text();
err_msg = db.lastError().text();
}
}
emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting,
{db_name, stmt_msg, err_msg} );
{this->db_name, stmt_msg, err_msg} );
return;
}
}
@ -355,7 +371,7 @@ void CraplogParser::storeLogLines()
stmt.append( QStringLiteral("NULL") );\
}
bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name )
bool CraplogParser::storeData( QSqlDatabase& db )
{
// get blacklist items
const bool check_bl_cli { this->blacklist.client.used };
@ -430,7 +446,7 @@ bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name )
stmt.append( ");" );
if ( QSqlQuery query(*db); !query.exec( stmt ) ) [[unlikely]] {
if ( QSqlQuery query(db); !query.exec( stmt ) ) [[unlikely]] {
// error finalizing step
QString query_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
@ -440,7 +456,7 @@ bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name )
}
}
emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting,
{db_name, query_msg, err_msg} );
{this->db_name, query_msg, err_msg} );
return false;
}
}
@ -449,19 +465,19 @@ bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name )
}
bool CraplogParser::checkDatabaseFile( const QString& db_name ) noexcept
bool CraplogParser::checkDatabaseFile() noexcept
{
if ( ! IOutils::exists( this->db_data_path ) ) {
emit this->showDialog( WorkerDialog::errDatabaseFileNotFound, {db_name} );
if ( ! IOutils::exists( this->db_path ) ) {
emit this->showDialog( WorkerDialog::errDatabaseFileNotFound, {this->db_name} );
return false;
} else if ( ! IOutils::isFile( this->db_data_path ) ) {
emit this->showDialog( WorkerDialog::errDatabaseFileNotFile, {db_name} );
} else if ( ! IOutils::isFile( this->db_path ) ) {
emit this->showDialog( WorkerDialog::errDatabaseFileNotFile, {this->db_name} );
return false;
} else if ( ! IOutils::checkFile( this->db_data_path, true ) ) {
emit this->showDialog( WorkerDialog::errDatabaseFileNotReadable, {db_name} );
} else if ( ! IOutils::checkFile( this->db_path, true ) ) {
emit this->showDialog( WorkerDialog::errDatabaseFileNotReadable, {this->db_name} );
return false;
} else if ( ! IOutils::checkFile( this->db_data_path, false, true ) ) {
emit this->showDialog( WorkerDialog::errDatabaseFileNotWritable, {db_name} );
} else if ( ! IOutils::checkFile( this->db_path, false, true ) ) {
emit this->showDialog( WorkerDialog::errDatabaseFileNotWritable, {this->db_name} );
return false;
}
return true;

34
logdoctor/modules/craplog/modules/workers/parser.h
View File

@ -19,27 +19,31 @@ class DatabaseWrapper;
enum class WorkerDialog;
class QWaitCondition;
class QSqlDatabase;
class CraplogParser final : public QObject
{
Q_OBJECT
using logs_file_t = std::tuple<std::string,std::string>;
using worker_files_t = std::vector<logs_file_t>;
using worker_files_t = std::vector<std::string>;
public:
explicit CraplogParser(
const WebServer web_server,
const DialogsLevel dialogs_level,
const std::string& db_data_path,
const std::string& db_hashes_path,
const LogsFormat& logs_format,
const Blacklist& blacklist,
const worker_files_t& log_files,
worker_files_t&& log_files,
const std::string& data_db_path,
QObject* parent=nullptr
);
~CraplogParser();
signals:
void perfData(
@ -55,6 +59,8 @@ signals:
const WorkerDialog dialog_type,
const QStringList arg );
void readyStoringData( QWaitCondition* wc, bool* successful );
void startedParsing();
void finishedParsing();
@ -83,10 +89,13 @@ private:
//// DATABASES ////
bool db_edited{ false };
std::string db_data_path;
std::string db_hashes_path;
bool checkDatabaseFile( const QString& db_name ) noexcept;
std::string db_path;
const QString db_name;
const QString db_conn_name{ QStringLiteral("Parser_LogsData") };
bool checkDatabaseFile() noexcept;
//////////////////////
//// PERFORMANCES ////
@ -130,19 +139,18 @@ private:
//! Handles the process of storing data in the database
/*!
\param db The database instance, not initialized already
\see storeData()
*/
void storeLogLines();
void storeLogLines( QSqlDatabase& db );
//! Stores the data collection in the logs Collection database
/*!
\param db A database instance, already initizlized
\param db_name The database in use, to be shown in the dialogs
\param db The database instance, already initialized
\return Whether the operation has been successful or not
\throw WebServerException
*/
bool storeData( DatabaseWrapper& db , const QString& db_name );
bool storeData( QSqlDatabase& db );
};