elB4RTO/LogDoctor
elB4RTO
/
LogDoctor
2
2
Fork 0
Code Issues 6 Pull Requests Projects 1 Releases 8 Wiki Activity

Code improvements and updates 

Improved Craplog module and its submodules
This commit is contained in:
Valentino Orlandi 2024-02-03 17:44:28 +01:00
parent 5c92b71c37
commit 3158a4e629
Signed by: elB4RTO
GPG Key ID: 1719E976DB2D4E71
9 changed files with 262 additions and 394 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
logdoctor/modules/craplog/craplog.cpp
View File

@ -186,14 +186,14 @@ void Craplog::warnlistAdd( const WebServer& web_server, const int& log_field_id,
void Craplog::blacklistRemove( const WebServer& web_server, const int& log_field_id, const std::string& item ) noexcept
{
auto& list = this->blacklists.at( web_server ).at( log_field_id ).list;
auto& list{ this->blacklists.at( web_server ).at( log_field_id ).list };
if ( const auto it{ std::find( list.cbegin(), list.cend(), item ) }; it != list.cend() ) {
list.erase( it );
}
}
void Craplog::warnlistRemove( const WebServer& web_server, const int& log_field_id, const std::string& item ) noexcept
{
auto& list = this->warnlists.at( web_server ).at( log_field_id ).list;
auto& list{ this->warnlists.at( web_server ).at( log_field_id ).list };
if ( const auto it{ std::find( list.cbegin(), list.cend(), item ) }; it != list.cend() ) {
list.erase( it );
}
@ -201,7 +201,7 @@ void Craplog::warnlistRemove( const WebServer& web_server, const int& log_field_
int Craplog::blacklistMoveUp( const WebServer& web_server, const int& log_field_id, const std::string& item ) noexcept
{
auto& list = this->blacklists.at( web_server ).at( log_field_id ).list;
auto& list{ this->blacklists.at( web_server ).at( log_field_id ).list };
if ( auto it{ std::find( std::next(list.begin()), list.end(), item ) }; it != list.cend() ) {
const int pos{ static_cast<int>( std::distance(list.begin(), it) ) - 1 };
std::swap( *it, *std::prev(it) );
@ -211,7 +211,7 @@ int Craplog::blacklistMoveUp( const WebServer& web_server, const int& log_field_
}
int Craplog::warnlistMoveUp( const WebServer& web_server, const int& log_field_id, const std::string& item ) noexcept
{
auto& list = this->warnlists.at( web_server ).at( log_field_id ).list;
auto& list{ this->warnlists.at( web_server ).at( log_field_id ).list };
if ( auto it{ std::find( std::next(list.begin()), list.end(), item ) }; it != list.cend() ) {
const int pos{ static_cast<int>( std::distance(list.begin(), it) ) - 1 };
std::swap( *it, *std::prev(it) );
@ -222,7 +222,7 @@ int Craplog::warnlistMoveUp( const WebServer& web_server, const int& log_field_i
int Craplog::blacklistMoveDown( const WebServer& web_server, const int& log_field_id, const std::string& item ) noexcept
{
auto& list = this->blacklists.at( web_server ).at( log_field_id ).list;
auto& list{ this->blacklists.at( web_server ).at( log_field_id ).list };
if ( auto it{ std::find( list.begin(), std::prev(list.end()), item ) }; it != list.cend() ) {
const int pos{ static_cast<int>( std::distance(list.begin(), it) ) + 1 };
std::swap( *it, *std::next(it) );
@ -232,7 +232,7 @@ int Craplog::blacklistMoveDown( const WebServer& web_server, const int& log_fiel
}
int Craplog::warnlistMoveDown( const WebServer& web_server, const int& log_field_id, const std::string& item ) noexcept
{
auto& list = this->warnlists.at( web_server ).at( log_field_id ).list;
auto& list{ this->warnlists.at( web_server ).at( log_field_id ).list };
if ( auto it{ std::find( list.begin(), std::prev(list.end()), item ) }; it != list.cend() ) {
const int pos{ static_cast<int>( std::distance(list.begin(), it) ) + 1 };
std::swap( *it, *std::next(it) );
@ -243,41 +243,41 @@ int Craplog::warnlistMoveDown( const WebServer& web_server, const int& log_field
std::string Craplog::sanitizeBWitem( const int& log_field_id, const std::string& new_item ) const
{
std::string sanitized_item;
switch ( log_field_id ) {
case 11:
sanitized_item = StringOps::strip( new_item );
{
const std::string sanitized_item{ StringOps::strip( new_item ) };
if ( ! StringOps::isAlphabetic( sanitized_item ) ) {
// only letters allowed
throw BWlistException("Invalid Method");
}
sanitized_item = StringOps::toUpper( sanitized_item );
break;
return StringOps::toUpper( sanitized_item );
}
case 12:
sanitized_item = StringOps::lstrip( new_item );
{
const std::string sanitized_item{ StringOps::lstrip( new_item ) };
if ( sanitized_item.empty() ) {
throw BWlistException("Invalid URI");
}
sanitized_item = QUrl::toPercentEncoding(
return QUrl::toPercentEncoding(
QString::fromStdString( sanitized_item ),
"/#&?=+").toStdString();
break;
}
case 20:
sanitized_item = StringOps::strip( new_item );
{
const std::string sanitized_item{ StringOps::strip( new_item ) };
if ( ! StringOps::isIP( sanitized_item ) ) {
// only IPv4/IPv6 allowed
throw BWlistException("Invalid Client");
}
break;
return sanitized_item;
}
case 21:
sanitized_item = StringOps::replace( new_item, "\"", "\\\"" );
break;
return StringOps::replace( new_item, "\"", "\\\"" );
default:
// shouldn't be here
throw GenericException("Unexpected LogField ID: "+std::to_string(log_field_id));
break;
}
return sanitized_item;
}
@ -298,55 +298,49 @@ const LogsFormat& Craplog::getLogsFormat(const WebServer& web_server ) const noe
// set the logs format
bool Craplog::setApacheLogFormat( const std::string& format_string ) noexcept
{
// apache
bool success{ true };
try {
this->logs_formats.at( WS_APACHE ) =
this->formatOps.processApacheFormatString( format_string );
this->logs_format_strings.at( WS_APACHE ) = format_string;
} catch ( LogFormatException& e ) {
success &= false;
DialogSec::errInvalidLogFormatString( e.what() );
return false;
} catch (...) {
success &= false;
DialogSec::errGeneric( DialogSec::tr("An error occured while parsing the format string"), true );
return false;
}
return success;
return true;
}
bool Craplog::setNginxLogFormat( const std::string& format_string ) noexcept
{
// nginx
bool success{ true };
try {
this->logs_formats.at( WS_NGINX ) =
this->formatOps.processNginxFormatString( format_string );
this->logs_format_strings.at( WS_NGINX ) = format_string;
} catch ( LogFormatException& e ) {
success &= false;
DialogSec::errInvalidLogFormatString( e.what() );
return false;
} catch (...) {
success &= false;
DialogSec::errGeneric( DialogSec::tr("An error occured while parsing the format string"), true );
return false;
}
return success;
return true;
}
bool Craplog::setIisLogFormat( const std::string& format_string, const int log_module ) noexcept
{
// iis
bool success{ true };
try {
this->logs_formats.at( WS_IIS ) =
this->formatOps.processIisFormatString( format_string, log_module );
this->logs_format_strings.at( WS_IIS ) = format_string;
this->changeIisLogsBaseNames( log_module );
} catch ( LogFormatException& e ) {
success &= false;
DialogSec::errInvalidLogFormatString( e.what() );
return false;
} catch (...) {
success &= false;
DialogSec::errGeneric( DialogSec::tr("An error occured while parsing the format string"), true );
return false;
}
return success;
return true;
}
QString Craplog::getLogsFormatSample( const WebServer& web_server ) const
@ -437,9 +431,8 @@ const std::vector<LogFile>& Craplog::getLogsList() const noexcept
// return the LogFile instance of the file matching the given name
const LogFile& Craplog::getLogFileItem( const QString& file_name ) const
{
const auto& item{ std::find_if
( this->logs_list.begin(), this->logs_list.end(),
[&file_name](const LogFile& it){ return it.name()==file_name; } ) };
const auto item{ std::find_if( this->logs_list.begin(), this->logs_list.end(),
[&file_name](const LogFile& file){ return file.name()==file_name; } ) };
if ( item != this->logs_list.end() ) return *item;
// should be unreachable
throw GenericException("File item not found");
@ -449,10 +442,8 @@ const LogFile& Craplog::getLogFileItem( const QString& file_name ) const
// set a file as selected
bool Craplog::setLogFileSelected( const QString& file_name ) noexcept
{
const auto item{ std::find_if
( this->logs_list.begin(), this->logs_list.end(),
[&file_name]( const LogFile& it )
{ return it.name() == file_name; } ) };
const auto item{ std::find_if( this->logs_list.begin(), this->logs_list.end(),
[&file_name](const LogFile& file){ return file.name() == file_name; } ) };
if ( item != this->logs_list.end() ) {
item->setSelected();
return true;
@ -539,7 +530,6 @@ void Craplog::changeIisLogsBaseNames( const int module_id )
bool Craplog::isFileNameValid( const std::string& name ) const
{
bool valid{ true };
if ( ! this->logs_base_names.at( this->current_web_server ).starts.empty() ) {
if ( ! StringOps::startsWith( name, this->logs_base_names.at( this->current_web_server ).starts ) ) {
return false;
@ -561,13 +551,13 @@ bool Craplog::isFileNameValid( const std::string& name ) const
switch ( this->current_web_server ) {
case WS_APACHE:
[[fallthrough]];
case WS_NGINX: {
case WS_NGINX:
{
// further checks for apache / nginx
size_t start, stop;
start = name.rfind(".log." );
if ( start == std::string::npos ) {
valid &= false;
break;
return false;
}
start += 5ul;
stop = name.size()-1ul;
@ -577,19 +567,18 @@ bool Craplog::isFileNameValid( const std::string& name ) const
// serach for incremental numbers
for ( size_t i{start}; i<=stop; ++i ) {
if ( ! CharOps::isNumeric( name.at( i ) ) ) {
valid &= false;
break;
return false;
}
}
}break;
}break;
case WS_IIS: {
case WS_IIS:
{
// further checks for iis
size_t start, stop;
start = name.find( this->logs_base_names.at( WS_IIS ).contains ) + 3ul;
if ( start == std::string::npos ) {
valid &= false;
break;
return false;
}
stop = name.size()-5ul; // removing the finel '.log' extension
if ( StringOps::endsWith( name, ".gz" ) ) {
@ -599,34 +588,31 @@ bool Craplog::isFileNameValid( const std::string& name ) const
std::string date;
for ( size_t i{start}; i<=stop; ++i ) {
if ( ! CharOps::isNumeric( name.at( i ) ) ) {
valid &= false;
break;
return false;
}
date.push_back( name.at( i ) );
}
if ( valid ) {
// check if the file has today's date
time_t t;
time( &t );
struct tm* tmp = localtime( &t );
char aux_date[7];
// using strftime to display time
strftime( aux_date, 7, "%y%m%d", tmp );
valid &= false;
for ( size_t i{0}; i<6ul; ++i ) {
if ( date.at(i) != aux_date[i] ) {
// different date, valid
valid |= true;
break;
}
// check if the file has today's date
time_t t;
time( &t );
struct tm* tmp = localtime( &t );
char aux_date[7];
// using strftime to display time
strftime( aux_date, 7, "%y%m%d", tmp );
for ( size_t i{0}; i<6ul; ++i ) {
if ( date.at(i) != aux_date[i] ) {
// different date, valid
return true;
break;
}
}
}break;
}break;
default:
throw WebServerException( "Unexpected WebServer: " + toString(this->current_web_server) );
}
return valid;
return true;
}
@ -650,8 +636,6 @@ bool Craplog::checkStuff()
size_t logs_size{ 0ul };
for ( const LogFile& file : this->logs_list ) {
if ( ! this->proceed ) { break; }
if ( ! file.isSelected() ) {
// not selected, skip
continue;
@ -668,7 +652,7 @@ bool Craplog::checkStuff()
if ( choice == 0 ) {
// choosed to abort all
this->proceed &= false;
break;
return false;
} else if ( choice == 1 ) {
// choosed to discard the file and continue
continue;
@ -689,18 +673,18 @@ bool Craplog::checkStuff()
}
const int choice = DialogSec::choiceDuplicateFile( msg );
if ( choice == 0 ) {
// choosed to abort all
this->proceed &= false;
break;
// choosed to abort all
this->proceed &= false;
return false;
} else if ( choice == 1 ) {
// choosed to discard the file and continue
continue;
// choosed to discard the file and continue
continue;
} else if ( choice == 2 ) {
// choosed to ignore and use the file anyway
;
// choosed to ignore and use the file anyway
;
} else {
// shouldn't be here
throw GenericException( "Unexpeced value returned: "+std::to_string(choice) );
// shouldn't be here
throw GenericException( "Unexpeced value returned: "+std::to_string(choice) );
}
}
}
@ -711,11 +695,11 @@ bool Craplog::checkStuff()
// exceeds the warning size
QString msg{ file.name() };
if ( this->dialogs_level >= DL_NORMAL ) {
msg += QString("\n\n%1:\n%2").arg(
msg += QStringLiteral("\n\n%1:\n%2").arg(
DialogSec::tr("Size of the file"),
PrintSec::printableSize( file.size() ) );
if ( this->dialogs_level == DL_EXPLANATORY ) {
msg += QString("\n\n%1:\n%2").arg(
msg += QStringLiteral("\n\n%1:\n%2").arg(
DialogSec::tr("Warning size parameter"),
PrintSec::printableSize( this->warning_size ) );
}
@ -724,7 +708,7 @@ bool Craplog::checkStuff()
if ( choice == 0 ) {
// choosed to abort all
this->proceed &= false;
break;
return false;
} else if ( choice == 1 ) {
// choosed to discard the file and continue
continue;
@ -742,12 +726,12 @@ bool Craplog::checkStuff()
if ( ! CheckSec::checkCollectionDatabase( this->db_stats_path ) ) {
// checks failed, abort
this->proceed &= false;
break;
return false;
}
if ( ! CheckSec::checkHashesDatabase( this->db_hashes_path ) ) {
// checks failed, abort
this->proceed &= false;
break;
return false;
}
this->log_files_to_use.push_back(
@ -758,31 +742,33 @@ bool Craplog::checkStuff()
}
// check if there are enough files to use
if ( this->proceed && this->log_files_to_use.size() == 0ul ) {
if ( this->log_files_to_use.empty() ) {
// no files left, abort
DialogSec::msgNoFileToParse();
this->proceed &= false;
return false;
}
// check if the total size of the files do not exceed the available RAM
if ( this->proceed && logs_size >= MemOps::availableMemory() ) {
if ( logs_size >= MemOps::availableMemory() ) {
// no files left, abort
QString msg;
if ( this->dialogs_level >= DL_NORMAL ) {
msg += QString("\n\n%1: %2").arg(
msg += QStringLiteral("\n\n%1: %2").arg(
DialogSec::tr("Available memory"),
PrintSec::printableSize( MemOps::availableMemory() ) );
if ( this->dialogs_level == DL_EXPLANATORY ) {
msg += QString("\n%1: %2").arg(
msg += QStringLiteral("\n%1: %2").arg(
DialogSec::tr("Size of the logs"),
PrintSec::printableSize( logs_size ) );
}
}
DialogSec::msgNotEnoughMemory( msg );
this->proceed &= false;
return false;
}
return this->proceed;
return true;
}
void Craplog::showWorkerDialog( const WorkerDialog dialog_type, const QStringList args ) const noexcept
@ -800,6 +786,18 @@ void Craplog::showWorkerDialog( const WorkerDialog dialog_type, const QStringLis
case WorkerDialog::errFailedParsingLogs:
DialogSec::errFailedParsingLogs( args.at(0) );
break;
case WorkerDialog::errDatabaseFileNotFound:
DialogSec::errDatabaseNotFound( args.at(0) );
break;
case WorkerDialog::errDatabaseFileNotFile:
DialogSec::errDatabaseNotFile( args.at(0) );
break;
case WorkerDialog::errDatabaseFileNotReadable:
DialogSec::errDatabaseNotReadable( args.at(0) );
break;
case WorkerDialog::errDatabaseFileNotWritable:
DialogSec::errDatabaseNotWritable( args.at(0) );
break;
case WorkerDialog::errDatabaseFailedOpening:
if ( args.size() < 2 ) {
// do not throw, just print to stderr
@ -886,7 +884,11 @@ void Craplog::stopWorking( const bool successful )
this->db_edited = successful;
if ( successful ) {
// insert the hashes of the used files
this->hashOps.insertUsedHashes( this->db_hashes_path, this->used_files_hashes, this->current_web_server );
try {
this->hashOps.insertUsedHashes( this->db_hashes_path, this->used_files_hashes, this->current_web_server );
} catch (...) {
DialogSec::errFailedInsertUsedHashes();
}
}
emit this->finishedWorking();
}

4
logdoctor/modules/craplog/modules/formats.cpp
View File

@ -20,7 +20,7 @@ namespace /*private*/
\return The number of new lines in a single log line
\see LogsFormat, processApacheFormatString(), processNginxFormatString()
*/
unsigned countNewLines( std::string_view initial, std::string_view final, const std::vector<std::string>& separators )
size_t countNewLines( std::string_view initial, std::string_view final, const std::vector<std::string>& separators )
{
size_t nl{ 0ul };
nl += StringOps::count( initial, '\n' );
@ -28,7 +28,7 @@ unsigned countNewLines( std::string_view initial, std::string_view final, const
for ( const std::string& sep : separators ) {
nl += StringOps::count( sep, '\n' );
}
return static_cast<unsigned>(nl);
return nl;
}

210
logdoctor/modules/craplog/modules/hash.cpp
View File

@ -11,14 +11,13 @@
#include "modules/dialogs.h"
#include "modules/exceptions.h"
#include "modules/database/database.h"
#include "sha256.h"
#include <ios>
#include <QVariant>
#include <QSqlDatabase>
#include <QSqlQuery>
#include <QSqlError>
void HashOps::setDialogLevel( const DialogsLevel new_level ) noexcept
@ -30,51 +29,29 @@ void HashOps::setDialogLevel( const DialogsLevel new_level ) noexcept
// reads the database holding the already used hashes
bool HashOps::loadUsedHashesLists( const std::string& db_path ) noexcept
{
bool successful{ true };
const QString db_name{ QString::fromStdString( db_path.substr( db_path.find_last_of( '/' ) + 1ul ) ) };
DatabaseWrapper db{ DatabaseHandler::get( DatabaseType::Hashes ) };
QSqlDatabase db{ QSqlDatabase::database(DatabasesNames::hashes) };
db.setDatabaseName( QString::fromStdString( db_path ) );
db.open( db_path, this->dialogs_level==DL_EXPLANATORY );
if ( ! CheckSec::checkDatabaseFile( db_path, db_name ) ) {
successful &= false;
const QString stmt{ QStringLiteral(R"(SELECT "hash" FROM "%1";)") };
} else if ( ! db.open() ) {
// error opening database
successful &= false;
QString err_msg;
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = db.lastError().text();
}
DialogSec::errDatabaseFailedOpening( db_name, err_msg );
for ( const auto& [wid,name] : this->ws_names ) {
} else {
QSqlQuery query{ db };
for ( const auto& [wid,name] : this->ws_names ) {
if ( ! query.exec("SELECT hash FROM "+name+";") ) {
// error querying database
successful &= false;
DialogSec::errDatabaseFailedExecuting( db_name, query.lastQuery(), query.lastError().text() );
break;
} else {
// iterate over results
while ( query.next() ) {
std::string hash{ query.value(0).toString().toStdString() };
if ( hash.size() != 64ul ) {
// not a valid sha256 hash
continue;
}
this->hashes.at( wid ).push_back( hash );
}
QueryWrapper query{ db.getQuery() };
query( stmt.arg( name ) );
while ( query->next() ) {
const QString hash{ query[0].toString() };
if ( hash.size() != 64ul ) {
// not a valid sha256 hash
continue;
}
if ( ! successful ) { break; }
this->hashes.at( wid ).push_back( hash.toStdString() );
}
}
if ( db.isOpen() ) {
db.close();
}
return successful;
return true;
}
@ -93,7 +70,7 @@ void HashOps::digestFile( const std::string& file_path, std::string& hash )
} catch (...) {
// failed as gzip, try as text file
if ( content.size() > 0ul ) {
if ( ! content.empty() ) {
content.clear();
}
IOutils::readFile( file_path, content );
@ -124,7 +101,6 @@ void HashOps::digestFile( const std::string& file_path, std::string& hash )
SHA256 sha;
sha.update( content );
content.clear();
uint8_t* digest{ sha.digest() };
// return the hex digest
hash.append( SHA256::toString(digest) );
@ -143,133 +119,35 @@ bool HashOps::hasBeenUsed( const std::string &file_hash, const WebServer& web_se
}
// insert the given hash/es in the relative list
bool HashOps::insertUsedHash( QSqlQuery& query, const QString& db_name, const std::string& hash, const WebServer& web_server ) noexcept
void HashOps::insertUsedHashes( const std::string& db_path, const std::vector<std::string>& hashes, const WebServer& web_server )
{
bool successful{ true };
const bool explain_msg{ this->dialogs_level > DL_ESSENTIAL };
const bool explain_err{ this->dialogs_level == DL_EXPLANATORY };
DatabaseWrapper db{ DatabaseHandler::get( DatabaseType::Hashes ) };
db.open( db_path, explain_err );
db.startTransaction( explain_msg, explain_err );
try {
if( ! VecOps::contains<std::string>( this->hashes.at( web_server ), hash ) ) {
this->hashes.at( web_server ).push_back( hash );
// insert tnto the database
QString stmt = QString("INSERT INTO %1 ( hash ) VALUES ( '%2' );")
.arg( this->ws_names.at(web_server), QString::fromStdString(hash).replace("'","''") );
if ( ! query.exec( stmt ) ) {
// error opening database
successful &= false;
QString query_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
query_msg = "query.exec()";
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = query.lastError().text();
}
}
DialogSec::errDatabaseFailedExecuting( db_name, query_msg, err_msg );
for ( const std::string& hash : hashes ) {
if ( VecOps::contains( this->hashes.at( web_server ), hash ) ) {
db.getQuery()( QStringLiteral(R"(INSERT INTO "%1" ( hash ) VALUES ( '%2' );)")
.arg( this->ws_names.at(web_server), QString::fromStdString(hash).replace(QLatin1Char('\''),QLatin1String("''")) ) );
}
}/* else {
// hash already stored
}*/
}
db.commitTransaction( explain_msg, explain_err );
auto& used_hashes{ this->hashes.at( web_server ) };
used_hashes.insert( used_hashes.end(), hashes.begin(), hashes.end() );
} catch (...) {
// failed to insert the hash
successful &= false;
// rollback the transaction
db.rollbackTransaction( explain_msg, explain_err );
}
query.finish();
return successful;
}
bool HashOps::insertUsedHashes( const std::string& db_path, const std::vector<std::string>& hashes, const WebServer& web_server )
{
bool successful{ true };
QSqlDatabase db{ QSqlDatabase::database(DatabasesNames::hashes) };
db.setDatabaseName( QString::fromStdString( db_path ) );
const QString db_name{ QString::fromStdString( db_path.substr( db_path.find_last_of( '/' ) + 1ul ) ) };
if ( ! CheckSec::checkDatabaseFile( db_path, db_name ) ) {
successful &= false;
} else if ( ! db.open() ) {
// error opening database
successful &= false;
QString err_msg;
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = db.lastError().text();
}
DialogSec::errDatabaseFailedOpening( db_name, err_msg );
} else {
QSqlQuery query{ db };
if ( ! db.transaction() ) {
// error opening database
successful &= false;
QString stmt_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
stmt_msg = "db.transaction()";
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = db.lastError().text();
}
}
DialogSec::errDatabaseFailedExecuting( db_name, stmt_msg, err_msg );
} else {
try {
for ( const std::string& hash : hashes ) {
successful = this->insertUsedHash( query, db_name, hash, web_server );
if ( ! successful ) {
break;
}
}
query.finish();
if ( successful ) {
// commit the transaction
if ( ! db.commit() ) {
// error opening database
successful &= false;
QString stmt_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
stmt_msg = "db.commit()";
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg= db.lastError().text();
}
}
DialogSec::errDatabaseFailedExecuting( db_name, stmt_msg, err_msg );
}
}
if ( ! successful ) {
// rollback
throw (std::exception());
}
} catch (...) {
// wrongthing w3nt some.,.
successful &= false;
bool err_shown{ false };
// rollback the transaction
if ( ! db.rollback() ) {
// error rolling back commits
QString stmt_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
stmt_msg = "db.rollback()";
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = db.lastError().text();
}
}
DialogSec::errDatabaseFailedExecuting( db_name, stmt_msg, err_msg );
err_shown |= true;
}
if ( ! err_shown ) {
// show a message
QString msg{ DialogSec::tr("An error occured while working on the database\n\nAborting") };
DialogSec::errGeneric( msg );
}
}
}
}
if ( db.isOpen() ) {
db.close();
}
return successful;
}

10
logdoctor/modules/craplog/modules/hash.h
View File

@ -30,6 +30,7 @@ public:
/*!
\param db_path The path of the log files' Hashes database
\return Whether the operation has been successful or not
\throw LogDoctorException
*/
bool loadUsedHashesLists( const std::string& db_path ) noexcept;
@ -55,9 +56,9 @@ public:
\param db_path The path of the Hashes database
\param hashes The list of hashes to insert
\param web_server_id The ID of the Web Server which generated the file
\return Whether the operation has been successful or not
\throw LogDoctorException
*/
bool insertUsedHashes( const std::string& db_path, const std::vector<std::string>& hashes, const WebServer& web_server );
void insertUsedHashes( const std::string& db_path, const std::vector<std::string>& hashes, const WebServer& web_server );
private:
@ -79,11 +80,6 @@ private:
{WS_IIS, {}}
};
// Called by insertUsedHashes()
// Inserts a hash in the corresponding database table
bool insertUsedHash( QSqlQuery& query, const QString& db_name, const std::string& hash, const WebServer& web_server ) noexcept;
};

7
logdoctor/modules/craplog/modules/lib.h
View File

@ -28,9 +28,12 @@ enum class LogType
//! Holds informations about a log file
struct LogFile final {
explicit LogFile() noexcept = default;
explicit LogFile
(const bool sel,const bool used,const size_t sz,const QString& nm,const std::string& hs,const std::string& pt) noexcept
explicit LogFile(const bool sel,const bool used,const size_t sz,const QString& nm,const std::string& hs,const std::string& pt) noexcept
:selected{sel},used_already{used},size_{sz},name_{nm},hash_{hs},path_{pt}{}
LogFile(LogFile&& other) noexcept = default;
LogFile& operator=(LogFile&& other) noexcept = default;
LogFile(const LogFile& other) noexcept = default;
LogFile& operator=(const LogFile& other) noexcept = default;
//! Wheter the file has been selected to be used or not
inline bool isSelected() const noexcept
{ return this->selected; }

13
logdoctor/modules/craplog/modules/logs.cpp
View File

@ -98,15 +98,10 @@ bool deepTypeCheck( const std::string& line, const LogsFormat& format ) noexcept
// add the initial and final seps now
n_sep += 2;
// the result is considered ture if more then a half of the seps was found
// and more than a half of the found separators was not blank
bool result{ false };
if ( n_sep_found >= n_sep-1
&& n_blank_sep <= n_sep_found/2 ) {
result |= true;
}
return result;
// the line is considered valid if all the seps have been found
// and more than a half of them were not blank
return n_sep_found >= n_sep-1
&& n_blank_sep <= n_sep_found/2;
}
} // namespace (private)

4
logdoctor/modules/craplog/modules/workers/lib.h
View File

@ -14,6 +14,10 @@ enum class WorkerDialog {
errDirNotExists,
errFailedDefiningLogType,
errFailedParsingLogs,
errDatabaseFileNotFound,
errDatabaseFileNotFile,
errDatabaseFileNotReadable,
errDatabaseFileNotWritable,
errDatabaseFailedOpening,
errDatabaseFailedExecuting,
warnFileNotReadable,

211
logdoctor/modules/craplog/modules/workers/parser.cpp
View File

@ -12,10 +12,10 @@
#include "modules/dialogs.h"
#include "modules/exceptions.h"
#include "modules/database/database.h"
#include "modules/craplog/modules/workers/lib.h"
#include <QSqlDatabase>
#include <QSqlQuery>
#include <QSqlError>
@ -84,6 +84,7 @@ void CraplogParser::work()
emit this->showDialog( WorkerDialog::errFailedParsingLogs,
{e.what()} );
this->proceed &= false;
}
// send the final data
if ( ! this->proceed ) {
@ -187,11 +188,18 @@ void CraplogParser::joinLogLines()
void CraplogParser::parseLogLines()
{
const auto parseLine = [this]( const std::string& line, const LogsFormat& logs_format ) {
const auto parseLine{ [this]( const std::string& line, const LogsFormat& logs_format ) {
this->data_collection.emplace_back( LogLineData(line, logs_format) );
this->parsed_size += line.size();
++ this->parsed_lines;
};
}};
const auto signal_emission_gap{ [](const size_t n_lines)->size_t{
return n_lines>10000ul ? n_lines/1000ul
: n_lines>1000ul ? n_lines/100ul
: n_lines>100ul ? n_lines/10ul
: 10ul;
}};
// parse all the lines
@ -200,20 +208,19 @@ void CraplogParser::parseLogLines()
const size_t nl{ this->logs_format.new_lines };
size_t send{ 0ul };
if ( nl == 0ul ) {
const size_t send_gap{ n_lines>1000ul ? n_lines/100 : n_lines>100ul ? n_lines/10 : 10 };
const size_t send_gap{ signal_emission_gap(n_lines) };
const LogsFormat& lf {this->logs_format};
this->data_collection.reserve( n_lines );
for ( const std::string& line : this->logs_lines ) {
parseLine( line, lf );
if (send == send_gap) {
if (++send == send_gap) {
this->sendPerfData();
send = 0ul;
}
++send;
}
} else {
const size_t real_lines{ n_lines / (nl+1ul) };
const size_t send_gap{ real_lines>1000ul ? real_lines/100 : real_lines>100ul ? real_lines/10 : 10 };
const size_t send_gap{ signal_emission_gap(real_lines) };
const LogsFormat& lf {this->logs_format};
this->data_collection.reserve( real_lines );
for ( size_t i{0ul}; i<n_lines; ++i ) {
@ -223,11 +230,10 @@ void CraplogParser::parseLogLines()
line += "\n" + this->logs_lines.at( i );
}
parseLine( line, lf );
if (send == send_gap) {
if (++send == send_gap) {
this->sendPerfData();
send = 0ul;
}
++send;
}
}
this->sendPerfData();
@ -241,136 +247,116 @@ void CraplogParser::storeLogLines()
QString db_path{ QString::fromStdString( this->db_data_path ) };
QString db_name{ QString::fromStdString( this->db_data_path.substr( this->db_data_path.find_last_of( '/' ) + 1ul ) ) };
QSqlDatabase db{ QSqlDatabase::database(DatabasesNames::data) };
db.setDatabaseName( db_path );
DatabaseWrapper db{ DatabaseHandler::get( DatabaseType::Hashes ) };
if ( ! CheckSec::checkDatabaseFile( this->db_data_path, db_name ) ) {
db->setDatabaseName( db_path );
if ( ! this->checkDatabaseFile( db_name ) ) [[unlikely]] {
this->proceed &= false;
return;
}
} else if ( ! db.open() ) {
// error opening database
if ( ! db->open() ) [[unlikely]] {
this->proceed &= false;
QString err_msg;
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = db.lastError().text();
err_msg = db->lastError().text();
}
emit this->showDialog( WorkerDialog::errDatabaseFailedOpening,
{db_name, err_msg} );
{db_name, err_msg} );
return;
}
} else {
try {
try {
// ACID transaction
if ( ! db.transaction() ) {
// error opening database
if ( ! db->transaction() ) [[unlikely]] {
this->proceed &= false;
QString stmt_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
stmt_msg.append( "db.transaction()" );
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = db->lastError().text();
}
}
emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting,
{db_name, stmt_msg, err_msg} );
return;
} else if ( this->storeData( db, db_name ) ) [[likely]] {
if ( ! db->commit() ) [[unlikely]] {
this->proceed &= false;
QString stmt_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
stmt_msg = "db.transaction()";
stmt_msg.append( "db.commit()" );
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = db.lastError().text();
err_msg.append( db->lastError().text() );
}
}
emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting,
{db_name, stmt_msg, err_msg} );
}
}
if ( this->proceed ) {
this->proceed &= this->storeData( db );
}
if ( this->proceed ) {
// commit the transaction
if ( ! db.commit() ) {
// error opening database
this->proceed &= false;
QString stmt_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
stmt_msg = "db.commit()";
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg= db.lastError().text();
}
}
emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting,
{db_name, stmt_msg, err_msg} );
}
}
if ( ! this->proceed ) {
// rollback
throw (std::exception());
}
} catch (...) {
// wrongthing w3nt some.,.
this->proceed &= false;
bool err_shown = false;
if ( ! this->proceed ) [[unlikely]] {
// rollback the transaction
if ( ! db.rollback() ) {
// error rolling back commits
if ( ! db->rollback() ) {
QString stmt_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
stmt_msg = "db.rollback()";
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = db.lastError().text();
err_msg = db->lastError().text();
}
}
emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting,
{db_name, stmt_msg, err_msg} );
err_shown = true;
}
if ( ! err_shown ) {
// show a message
emit this->showDialog(
WorkerDialog::errGeneric,
{QString("%1\n\n%2").arg(
DialogSec::tr("An error occured while working on the database"),
DialogSec::tr("Aborting") )} );
return;
}
}
if ( db.isOpen() ) {
db.close();
}
} catch (...) {
// wrongthing w3nt some.,.
emit this->showDialog(
WorkerDialog::errGeneric,
{QStringLiteral("%1\n\n%2").arg(
DialogSec::tr("An error occured while working on the database"),
DialogSec::tr("Aborting") )} );
}
}
#define APPEND_TO_QUERY_AS_NUMBER(LOG_FIELD)\
if ( LOG_FIELD ) {\
query_stmt += QString::fromStdString( *LOG_FIELD ).replace("'","''");\
stmt.append( QString::fromStdString( *LOG_FIELD ).replace(QLatin1Char('\''),QLatin1String("''")) );\
} else {\
query_stmt += QStringLiteral("NULL");\
stmt.append( QStringLiteral("NULL") );\
}
#define CONCAT_TO_QUERY_AS_NUMBER(LOG_FIELD)\
query_stmt += QStringLiteral(", ");\
stmt.append( QStringLiteral(", ") );\
APPEND_TO_QUERY_AS_NUMBER(LOG_FIELD)
#define CONCAT_TO_QUERY_AS_STRING(LOG_FIELD)\
query_stmt += QStringLiteral(", ");\
stmt.append( QStringLiteral(", ") );\
if ( LOG_FIELD ) {\
query_stmt += QString("'%1'").arg( QString::fromStdString( *LOG_FIELD ).replace("'","''") );\
stmt.append( QString("'%1'").arg( QString::fromStdString( *LOG_FIELD ).replace(QLatin1Char('\''),QLatin1String("''")) ) );\
} else {\
query_stmt += QStringLiteral("NULL");\
stmt.append( QStringLiteral("NULL") );\
}
// in IIS logs the user-agent is logged with '+' instead of ' ' (whitespace)
#define CONCAT_TO_QUERY_USERAGENT(LOG_FIELD)\
query_stmt += QStringLiteral(", ");\
stmt.append( QStringLiteral(", ") );\
if ( LOG_FIELD ) {\
if ( this->web_server == WS_IIS ) {\
query_stmt += QString("'%1'").arg( QString::fromStdString( *LOG_FIELD ).replace("+"," ").replace("'","''") );\
stmt.append( QStringLiteral("'%1'").arg( QString::fromStdString( *LOG_FIELD ).replace(QLatin1Char('+'),QLatin1Char(' ')).replace(QLatin1Char('\''),QLatin1String("''")) ) );\
} else {\
query_stmt += QString("'%1'").arg( QString::fromStdString( *LOG_FIELD ).replace("'","''") );\
stmt.append( QStringLiteral("'%1'").arg( QString::fromStdString( *LOG_FIELD ).replace(QLatin1Char('\''),QLatin1String("''")) ) );\
}\
} else {\
query_stmt += QStringLiteral("NULL");\
stmt.append( QStringLiteral("NULL") );\
}
bool CraplogParser::storeData( QSqlDatabase& db )
bool CraplogParser::storeData( DatabaseWrapper& db, const QString& db_name )
{
const QString db_name{ QString::fromStdString(
this->db_data_path.substr(
this->db_data_path.find_last_of( '/' ) + 1ul ) ) };
// get blacklist items
const bool check_bl_cli { this->blacklists.at( 20 ).used };
@ -383,22 +369,25 @@ bool CraplogParser::storeData( QSqlDatabase& db )
QString table;
switch ( this->web_server ) {
case WS_APACHE:
table += "apache";
table.append( "apache" );
break;
case WS_NGINX:
table += "nginx";
table.append( "nginx" );
break;
case WS_IIS:
table += "iis";
table.append( "iis" );
break;
default:
// wrong WebServerID, but should be unreachable because of the previous operations
throw WebServerException( "Unexpected WebServer: " + toString(this->web_server) );
}
const QString stmt_template{
QStringLiteral(R"(INSERT INTO "%1" ("year", "month", "day", "hour", "minute", "second", "protocol", "method", "uri", "query", "response", "time_taken", "bytes_sent", "bytes_received", "referrer", "client", "user_agent", "cookie") )"
"VALUES (")
};
/*int perf_size;*/
QSqlQuery query{ db };
// parse every row of data
for ( const LogLineData& line_data : this->data_collection ) {
@ -410,9 +399,7 @@ bool CraplogParser::storeData( QSqlDatabase& db )
}
}
// initialize the SQL statement
QString query_stmt{ "INSERT INTO \""+table+"\" (\"year\", \"month\", \"day\", \"hour\", \"minute\", \"second\", \"protocol\", \"method\", \"uri\", \"query\", \"response\", \"time_taken\", \"bytes_sent\", \"bytes_received\", \"referrer\", \"client\", \"user_agent\", \"cookie\") "
"VALUES (" };
QString stmt{ stmt_template.arg( table ) };
// complete and execute the statement, binding NULL if not found
@ -441,29 +428,13 @@ bool CraplogParser::storeData( QSqlDatabase& db )
CONCAT_TO_QUERY_USERAGENT(line_data.user_agent) // 21
CONCAT_TO_QUERY_AS_STRING(line_data.cookie) // 22
query_stmt += ");";
stmt.append( ");" );
// encode the statement
if ( ! query.prepare( query_stmt ) ) {
// error opening database
QString query_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
query_msg = "query.prepare()";
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = query.lastError().text();
}
}
emit this->showDialog( WorkerDialog::errDatabaseFailedExecuting,
{db_name, query_msg, err_msg} );
return false;
}
// finalize this statement
if ( ! query.exec() ) {
if ( QSqlQuery query(*db); !query.exec( stmt ) ) [[unlikely]] {
// error finalizing step
QString query_msg, err_msg;
if ( this->dialogs_level > DL_ESSENTIAL ) {
query_msg = "query.exec()";
query_msg.append( "query.exec()" );
if ( this->dialogs_level == DL_EXPLANATORY ) {
err_msg = query.lastError().text();
}
@ -472,10 +443,26 @@ bool CraplogParser::storeData( QSqlDatabase& db )
{db_name, query_msg, err_msg} );
return false;
}
// reset the statement to prepare for the next one
query.finish();
}
return true;
}
bool CraplogParser::checkDatabaseFile( const QString& db_name ) noexcept
{
if ( ! IOutils::exists( this->db_data_path ) ) {
emit this->showDialog( WorkerDialog::errDatabaseFileNotFound, {db_name} );
return false;
} else if ( ! IOutils::isFile( this->db_data_path ) ) {
emit this->showDialog( WorkerDialog::errDatabaseFileNotFile, {db_name} );
return false;
} else if ( ! IOutils::checkFile( this->db_data_path, true ) ) {
emit this->showDialog( WorkerDialog::errDatabaseFileNotReadable, {db_name} );
return false;
} else if ( ! IOutils::checkFile( this->db_data_path, false, true ) ) {
emit this->showDialog( WorkerDialog::errDatabaseFileNotWritable, {db_name} );
return false;
}
return true;
}

9
logdoctor/modules/craplog/modules/workers/parser.h
View File

@ -14,9 +14,9 @@
struct BWlist;
struct LogLineData;
enum class WorkerDialog;
class DatabaseWrapper;
class QSqlDatabase;
enum class WorkerDialog;
class CraplogParser final : public QObject
@ -86,6 +86,8 @@ private:
std::string db_data_path;
std::string db_hashes_path;
bool checkDatabaseFile( const QString& db_name ) noexcept;
//////////////////////
//// PERFORMANCES ////
@ -136,10 +138,11 @@ private:
//! Stores the data collection in the logs Collection database
/*!
\param db A database instance, already initizlized
\param db_name The database in use, to be shown in the dialogs
\return Whether the operation has been successful or not
\throw WebServerException
*/
bool storeData( QSqlDatabase& db );
bool storeData( DatabaseWrapper& db , const QString& db_name );
};