From 423bb187554e7827f0d7884887172cc4be453190 Mon Sep 17 00:00:00 2001 From: Valentino Orlandi Date: Sun, 7 Aug 2022 23:36:47 +0200 Subject: [PATCH] Removed ErrorLogs support + code improvements --- logdoctor/tools/craplog/modules/store.cpp | 229 +++++++--------------- logdoctor/tools/craplog/modules/store.h | 2 +- 2 files changed, 75 insertions(+), 156 deletions(-) diff --git a/logdoctor/tools/craplog/modules/store.cpp b/logdoctor/tools/craplog/modules/store.cpp index b95b4513..21b171b2 100644 --- a/logdoctor/tools/craplog/modules/store.cpp +++ b/logdoctor/tools/craplog/modules/store.cpp @@ -1,6 +1,8 @@ #include "store.h" +#include "modules/exceptions.h" + #include @@ -11,7 +13,7 @@ StoreOps::StoreOps() -bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector>& data, const int log_type ) +bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector>& data ) { bool successful = true; @@ -22,71 +24,46 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector< db_name.append( QString::fromStdString( db_path.substr( db_path.find_last_of( '/' ) + 1 ) ) ); } - // get whitelist/blacklist items - bool check_bl_cli, check_bl_err=false, - check_wl_cli, check_wl_ua=false, check_wl_met=false, check_wl_req=false, check_wl_err=false; - check_bl_cli = craplog.isBlacklistUsed( wsID, log_type, 20 ); - check_wl_cli = craplog.isWarnlistUsed( wsID, log_type, 20 ); - if ( log_type == 1 ) { - check_wl_ua = craplog.isWarnlistUsed( wsID, log_type, 21 ); - check_wl_met = craplog.isWarnlistUsed( wsID, log_type, 11 ); - check_wl_req = craplog.isWarnlistUsed( wsID, log_type, 12 ); - } else { - check_bl_err = craplog.isBlacklistUsed( wsID, log_type, 31 ); - check_wl_err = craplog.isWarnlistUsed( wsID, log_type, 31 ); - } + // get blacklist/warnlist items + bool check_bl_cli, + check_wl_cli, check_wl_ua, check_wl_met, check_wl_req; + check_bl_cli = craplog.isBlacklistUsed( wsID, 20 ); + check_wl_met = craplog.isWarnlistUsed( wsID, 11 ); + check_wl_req = craplog.isWarnlistUsed( wsID, 12 ); + check_wl_cli = craplog.isWarnlistUsed( wsID, 20 ); + check_wl_ua = craplog.isWarnlistUsed( wsID, 21 ); std::vector bl_cli_list, bl_err_list, wl_cli_list, wl_ua_list, wl_met_list, wl_req_list, wl_err_list; if ( check_bl_cli ) { - bl_cli_list = craplog.getBlacklist( wsID, log_type, 20 ); + bl_cli_list = craplog.getBlacklist( wsID, 20 ); + } + if ( check_wl_met ) { + wl_met_list = craplog.getWarnlist( wsID, 11 ); + } + if ( check_wl_req ) { + wl_req_list = craplog.getWarnlist( wsID, 12 ); } if ( check_wl_cli ) { - wl_cli_list = craplog.getWarnlist( wsID, log_type, 20 ); + wl_cli_list = craplog.getWarnlist( wsID, 20 ); } - if ( log_type == 1 ) { - if ( check_wl_ua ) { - wl_ua_list = craplog.getWarnlist( wsID, log_type, 21 ); - } - if ( check_wl_met ) { - wl_met_list = craplog.getWarnlist( wsID, log_type, 11 ); - } - if ( check_wl_req ) { - wl_req_list = craplog.getWarnlist( wsID, log_type, 12 ); - } - } else { - if ( check_bl_err ) { - bl_err_list = craplog.getBlacklist( wsID, log_type, 31 ); - } - if ( check_wl_err ) { - wl_err_list = craplog.getWarnlist( wsID, log_type, 31 ); - } + if ( check_wl_ua ) { + wl_ua_list = craplog.getWarnlist( wsID, 21 ); } // prepare the database related studd - QString table = ""; + QString table; switch ( wsID ) { case 11: - table += "apache_"; + table = "apache"; break; case 12: - table += "nginx_"; + table = "nginx"; break; case 13: - table += "iis_"; + table = "iis"; break; default: // wrong WebServerID, but should be unreachable because of the previous operations - throw( "Unexpected WebServerID: " + std::to_string(wsID) ); - } - switch ( log_type ) { - case 1: - table += "access"; - break; - case 2: - table += "error"; - break; - default: - // wrong LogType, but should be unreachable because of initial checks - throw( "Unexpected LogType: " + std::to_string(log_type) ); + throw WebServerException( "Unexpected WebServerID: " + std::to_string(wsID) ); } @@ -113,27 +90,13 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector< } } } - // check blacklisted error levels - if ( check_bl_err == true && skip == false ) { - if ( row.find( 31 ) != row.end() ) { - // this row do contains this row item, check if they match - std::string target = row.at( 31 ); - for ( const auto& item : bl_err_list ) { - if ( item == target ) { - // match found! skip this line - skip = true; - break; - } - } - } - } if ( skip == true ) { // append every field to ignored size - int ignored_size = 0; + int blacklisted_size = 0; for ( const auto& [ id, str ] : row ) { - ignored_size += str.size(); + blacklisted_size += str.size(); } - craplog.sumBlacklistededSize( ignored_size, log_type ); + craplog.sumBlacklistededSize( blacklisted_size ); skip=false; continue; } @@ -194,37 +157,18 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector< } } } - // check warnlisted error levels - if ( check_wl_err == true && warning == false ) { - if ( row.find( 31 ) != row.end() ) { - // this row do contains this row item, check if they match - std::string target = row.at( 31 ); - for ( const auto& item : wl_err_list ) { - if ( item == target ) { - // match found! skip this line - warning = true; - break; - } - } - } - } // initialize the SQL statement QString query_stmt; - if ( log_type == LogOps::LogType::Access ) { - query_stmt = "INSERT INTO "+table+" (warning, year, month, day, hour, minute, second, protocol, method, uri, query, response, time_taken, bytes_sent, bytes_received, referrer, client, user_agent, cookie) "\ - "VALUES ("; - - } else { - query_stmt = "INSERT INTO "+table+" (warning, year, month, day, hour, minute, second, level, message, source_file, client, port) "\ - "VALUES ("; - } + query_stmt = "INSERT INTO \""+table+"\" (\"warning\", \"year\", \"month\", \"day\", \"hour\", \"minute\", \"second\", \"protocol\", \"method\", \"uri\", \"query\", \"response\", \"time_taken\", \"bytes_sent\", \"bytes_received\", \"referrer\", \"client\", \"user_agent\", \"cookie\") "\ + "VALUES ("; - // complete and execute the statement + // complete and execute the statement, binding NULL if not found perf_size = 0; - // set values to NULL + + // warning if ( row.find( 99 ) == row.end() ) { // no value found in the collection, bind NULL query_stmt += "0"; @@ -234,6 +178,7 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector< query_stmt += QString::fromStdString( row.at( 99 ) ).replace("'","''"); } + // date and time for ( int i=1; i<7; i++ ) { query_stmt += ", "; if ( row.find( i ) == row.end() ) { @@ -246,73 +191,48 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector< } } - if ( log_type == LogOps::LogType::Access ) { - // access logs - for ( int i=10; i<14; i++ ) { - query_stmt += ", "; - if ( row.find( i ) == row.end() ) { - // no value found in the collection, bind NULL - query_stmt += "NULL"; - } else { - // value found, bind it - perf_size += row.at( i ).size(); - query_stmt += QString("'%1'").arg( QString::fromStdString( row.at( i ) ).replace("'","''") ); - } - } - - for ( int i=14; i<18; i++ ) { - query_stmt += ", "; - if ( row.find( i ) == row.end() ) { - // no value found in the collection, bind NULL - query_stmt += "NULL"; - } else { - // value found, bind it - perf_size += row.at( i ).size(); - query_stmt += QString::fromStdString( row.at( i ) ).replace("'","''"); - } - } - - for ( int i : std::vector({18,20,21,22}) ) { - query_stmt += ", "; - if ( row.find( i ) == row.end() ) { - // no value found in the collection, bind NULL - query_stmt += "NULL"; - } else { - // value found, bind it - if ( i == 21 && wsID == 13 ) { - // iis logs the user-agent using '+' instead of ' ' (spaces) - QString str = QString::fromStdString( row.at( i ) ).replace("+"," "); - perf_size += str.size(); - query_stmt += QString("'%1'").arg( str.replace("'","''") ); - } else { - perf_size += row.at( i ).size(); - query_stmt += QString("'%1'").arg( QString::fromStdString( row.at( i ) ).replace("'","''") ); - } - } - } - - } else { - // error logs - for ( int i : std::vector({31,32,33,20}) ) { - query_stmt += ", "; - if ( row.find( i ) == row.end() ) { - // no value found in the collection, bind NULL - query_stmt += "NULL"; - } else { - // value found, bind it - perf_size += row.at( i ).size(); - query_stmt += QString("'%1'").arg( QString::fromStdString( row.at( i ) ).replace("'","''") ); - } - } - + // request + for ( int i=10; i<14; i++ ) { query_stmt += ", "; - if ( row.find( 30 ) == row.end() ) { + if ( row.find( i ) == row.end() ) { // no value found in the collection, bind NULL query_stmt += "NULL"; } else { // value found, bind it - perf_size += row.at( 30 ).size(); - query_stmt += QString::fromStdString( row.at( 30 ) ).replace("'","''"); + perf_size += row.at( i ).size(); + query_stmt += QString("'%1'").arg( QString::fromStdString( row.at( i ) ).replace("'","''") ); + } + } + + for ( int i=14; i<18; i++ ) { + query_stmt += ", "; + if ( row.find( i ) == row.end() ) { + // no value found in the collection, bind NULL + query_stmt += "NULL"; + } else { + // value found, bind it + perf_size += row.at( i ).size(); + query_stmt += QString::fromStdString( row.at( i ) ).replace("'","''"); + } + } + + // client data and referrer + for ( int i : std::vector({18,20,21,22}) ) { + query_stmt += ", "; + if ( row.find( i ) == row.end() ) { + // no value found in the collection, bind NULL + query_stmt += "NULL"; + } else { + // value found, bind it + if ( i == 21 && wsID == 13 ) { + // iis logs the user-agent using '+' instead of ' ' (spaces) + QString str = QString::fromStdString( row.at( i ) ).replace("+"," "); + perf_size += str.size(); + query_stmt += QString("'%1'").arg( str.replace("'","''") ); + } else { + perf_size += row.at( i ).size(); + query_stmt += QString("'%1'").arg( QString::fromStdString( row.at( i ) ).replace("'","''") ); + } } } @@ -333,7 +253,7 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector< } // sum stored data size for the perfs - craplog.sumPerfSize( perf_size, log_type ); + craplog.sumPerfSize( perf_size ); // finalize this statement if ( query.exec() == false ) { @@ -350,10 +270,9 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector< break; } - // reset the statement to prepare for the next + // reset the statement to prepare for the next one query.finish(); } - return successful; } diff --git a/logdoctor/tools/craplog/modules/store.h b/logdoctor/tools/craplog/modules/store.h index 55bd52ac..03094436 100644 --- a/logdoctor/tools/craplog/modules/store.h +++ b/logdoctor/tools/craplog/modules/store.h @@ -9,7 +9,7 @@ class StoreOps public: StoreOps(); - static bool storeData( QSqlDatabase& db, Craplog& caplog, const std::vector>& data, const int log_type ); + static bool storeData( QSqlDatabase& db, Craplog& caplog, const std::vector>& data ); }; #endif // STORE_H