Removed ErrorLogs support + code improvements
This commit is contained in:
parent
cd4a44416a
commit
423bb18755
GPG key ID:
1719E976DB2D4E71
2 changed files with 75 additions and 156 deletions
|
|
@ -1,6 +1,8 @@
|
|||
|
||||
#include "store.h"
|
||||
|
||||
#include "modules/exceptions.h"
|
||||
|
||||
#include <QVariant>
|
||||
|
||||
|
||||
|
|
@ -11,7 +13,7 @@ StoreOps::StoreOps()
|
|||
|
||||
|
||||
|
||||
bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector<std::unordered_map<int, std::string>>& data, const int log_type )
|
||||
bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector<std::unordered_map<int, std::string>>& data )
|
||||
{
|
||||
bool successful = true;
|
||||
|
||||
|
|
@ -22,71 +24,46 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector<
|
|||
db_name.append( QString::fromStdString( db_path.substr( db_path.find_last_of( '/' ) + 1 ) ) );
|
||||
}
|
||||
|
||||
// get whitelist/blacklist items
|
||||
bool check_bl_cli, check_bl_err=false,
|
||||
check_wl_cli, check_wl_ua=false, check_wl_met=false, check_wl_req=false, check_wl_err=false;
|
||||
check_bl_cli = craplog.isBlacklistUsed( wsID, log_type, 20 );
|
||||
check_wl_cli = craplog.isWarnlistUsed( wsID, log_type, 20 );
|
||||
if ( log_type == 1 ) {
|
||||
check_wl_ua = craplog.isWarnlistUsed( wsID, log_type, 21 );
|
||||
check_wl_met = craplog.isWarnlistUsed( wsID, log_type, 11 );
|
||||
check_wl_req = craplog.isWarnlistUsed( wsID, log_type, 12 );
|
||||
} else {
|
||||
check_bl_err = craplog.isBlacklistUsed( wsID, log_type, 31 );
|
||||
check_wl_err = craplog.isWarnlistUsed( wsID, log_type, 31 );
|
||||
}
|
||||
// get blacklist/warnlist items
|
||||
bool check_bl_cli,
|
||||
check_wl_cli, check_wl_ua, check_wl_met, check_wl_req;
|
||||
check_bl_cli = craplog.isBlacklistUsed( wsID, 20 );
|
||||
check_wl_met = craplog.isWarnlistUsed( wsID, 11 );
|
||||
check_wl_req = craplog.isWarnlistUsed( wsID, 12 );
|
||||
check_wl_cli = craplog.isWarnlistUsed( wsID, 20 );
|
||||
check_wl_ua = craplog.isWarnlistUsed( wsID, 21 );
|
||||
std::vector<std::string> bl_cli_list, bl_err_list, wl_cli_list, wl_ua_list, wl_met_list, wl_req_list, wl_err_list;
|
||||
if ( check_bl_cli ) {
|
||||
bl_cli_list = craplog.getBlacklist( wsID, log_type, 20 );
|
||||
bl_cli_list = craplog.getBlacklist( wsID, 20 );
|
||||
}
|
||||
if ( check_wl_met ) {
|
||||
wl_met_list = craplog.getWarnlist( wsID, 11 );
|
||||
}
|
||||
if ( check_wl_req ) {
|
||||
wl_req_list = craplog.getWarnlist( wsID, 12 );
|
||||
}
|
||||
if ( check_wl_cli ) {
|
||||
wl_cli_list = craplog.getWarnlist( wsID, log_type, 20 );
|
||||
wl_cli_list = craplog.getWarnlist( wsID, 20 );
|
||||
}
|
||||
if ( log_type == 1 ) {
|
||||
if ( check_wl_ua ) {
|
||||
wl_ua_list = craplog.getWarnlist( wsID, log_type, 21 );
|
||||
}
|
||||
if ( check_wl_met ) {
|
||||
wl_met_list = craplog.getWarnlist( wsID, log_type, 11 );
|
||||
}
|
||||
if ( check_wl_req ) {
|
||||
wl_req_list = craplog.getWarnlist( wsID, log_type, 12 );
|
||||
}
|
||||
} else {
|
||||
if ( check_bl_err ) {
|
||||
bl_err_list = craplog.getBlacklist( wsID, log_type, 31 );
|
||||
}
|
||||
if ( check_wl_err ) {
|
||||
wl_err_list = craplog.getWarnlist( wsID, log_type, 31 );
|
||||
}
|
||||
if ( check_wl_ua ) {
|
||||
wl_ua_list = craplog.getWarnlist( wsID, 21 );
|
||||
}
|
||||
|
||||
// prepare the database related studd
|
||||
QString table = "";
|
||||
QString table;
|
||||
switch ( wsID ) {
|
||||
case 11:
|
||||
table += "apache_";
|
||||
table = "apache";
|
||||
break;
|
||||
case 12:
|
||||
table += "nginx_";
|
||||
table = "nginx";
|
||||
break;
|
||||
case 13:
|
||||
table += "iis_";
|
||||
table = "iis";
|
||||
break;
|
||||
default:
|
||||
// wrong WebServerID, but should be unreachable because of the previous operations
|
||||
throw( "Unexpected WebServerID: " + std::to_string(wsID) );
|
||||
}
|
||||
switch ( log_type ) {
|
||||
case 1:
|
||||
table += "access";
|
||||
break;
|
||||
case 2:
|
||||
table += "error";
|
||||
break;
|
||||
default:
|
||||
// wrong LogType, but should be unreachable because of initial checks
|
||||
throw( "Unexpected LogType: " + std::to_string(log_type) );
|
||||
throw WebServerException( "Unexpected WebServerID: " + std::to_string(wsID) );
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -113,27 +90,13 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector<
|
|||
}
|
||||
}
|
||||
}
|
||||
// check blacklisted error levels
|
||||
if ( check_bl_err == true && skip == false ) {
|
||||
if ( row.find( 31 ) != row.end() ) {
|
||||
// this row do contains this row item, check if they match
|
||||
std::string target = row.at( 31 );
|
||||
for ( const auto& item : bl_err_list ) {
|
||||
if ( item == target ) {
|
||||
// match found! skip this line
|
||||
skip = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if ( skip == true ) {
|
||||
// append every field to ignored size
|
||||
int ignored_size = 0;
|
||||
int blacklisted_size = 0;
|
||||
for ( const auto& [ id, str ] : row ) {
|
||||
ignored_size += str.size();
|
||||
blacklisted_size += str.size();
|
||||
}
|
||||
craplog.sumBlacklistededSize( ignored_size, log_type );
|
||||
craplog.sumBlacklistededSize( blacklisted_size );
|
||||
skip=false;
|
||||
continue;
|
||||
}
|
||||
|
|
@ -194,37 +157,18 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector<
|
|||
}
|
||||
}
|
||||
}
|
||||
// check warnlisted error levels
|
||||
if ( check_wl_err == true && warning == false ) {
|
||||
if ( row.find( 31 ) != row.end() ) {
|
||||
// this row do contains this row item, check if they match
|
||||
std::string target = row.at( 31 );
|
||||
for ( const auto& item : wl_err_list ) {
|
||||
if ( item == target ) {
|
||||
// match found! skip this line
|
||||
warning = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// initialize the SQL statement
|
||||
QString query_stmt;
|
||||
if ( log_type == LogOps::LogType::Access ) {
|
||||
query_stmt = "INSERT INTO "+table+" (warning, year, month, day, hour, minute, second, protocol, method, uri, query, response, time_taken, bytes_sent, bytes_received, referrer, client, user_agent, cookie) "\
|
||||
"VALUES (";
|
||||
|
||||
} else {
|
||||
query_stmt = "INSERT INTO "+table+" (warning, year, month, day, hour, minute, second, level, message, source_file, client, port) "\
|
||||
"VALUES (";
|
||||
}
|
||||
query_stmt = "INSERT INTO \""+table+"\" (\"warning\", \"year\", \"month\", \"day\", \"hour\", \"minute\", \"second\", \"protocol\", \"method\", \"uri\", \"query\", \"response\", \"time_taken\", \"bytes_sent\", \"bytes_received\", \"referrer\", \"client\", \"user_agent\", \"cookie\") "\
|
||||
"VALUES (";
|
||||
|
||||
|
||||
// complete and execute the statement
|
||||
// complete and execute the statement, binding NULL if not found
|
||||
perf_size = 0;
|
||||
// set values to NULL
|
||||
|
||||
// warning
|
||||
if ( row.find( 99 ) == row.end() ) {
|
||||
// no value found in the collection, bind NULL
|
||||
query_stmt += "0";
|
||||
|
|
@ -234,6 +178,7 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector<
|
|||
query_stmt += QString::fromStdString( row.at( 99 ) ).replace("'","''");
|
||||
}
|
||||
|
||||
// date and time
|
||||
for ( int i=1; i<7; i++ ) {
|
||||
query_stmt += ", ";
|
||||
if ( row.find( i ) == row.end() ) {
|
||||
|
|
@ -246,73 +191,48 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector<
|
|||
}
|
||||
}
|
||||
|
||||
if ( log_type == LogOps::LogType::Access ) {
|
||||
// access logs
|
||||
for ( int i=10; i<14; i++ ) {
|
||||
query_stmt += ", ";
|
||||
if ( row.find( i ) == row.end() ) {
|
||||
// no value found in the collection, bind NULL
|
||||
query_stmt += "NULL";
|
||||
} else {
|
||||
// value found, bind it
|
||||
perf_size += row.at( i ).size();
|
||||
query_stmt += QString("'%1'").arg( QString::fromStdString( row.at( i ) ).replace("'","''") );
|
||||
}
|
||||
}
|
||||
|
||||
for ( int i=14; i<18; i++ ) {
|
||||
query_stmt += ", ";
|
||||
if ( row.find( i ) == row.end() ) {
|
||||
// no value found in the collection, bind NULL
|
||||
query_stmt += "NULL";
|
||||
} else {
|
||||
// value found, bind it
|
||||
perf_size += row.at( i ).size();
|
||||
query_stmt += QString::fromStdString( row.at( i ) ).replace("'","''");
|
||||
}
|
||||
}
|
||||
|
||||
for ( int i : std::vector<int>({18,20,21,22}) ) {
|
||||
query_stmt += ", ";
|
||||
if ( row.find( i ) == row.end() ) {
|
||||
// no value found in the collection, bind NULL
|
||||
query_stmt += "NULL";
|
||||
} else {
|
||||
// value found, bind it
|
||||
if ( i == 21 && wsID == 13 ) {
|
||||
// iis logs the user-agent using '+' instead of ' ' (spaces)
|
||||
QString str = QString::fromStdString( row.at( i ) ).replace("+"," ");
|
||||
perf_size += str.size();
|
||||
query_stmt += QString("'%1'").arg( str.replace("'","''") );
|
||||
} else {
|
||||
perf_size += row.at( i ).size();
|
||||
query_stmt += QString("'%1'").arg( QString::fromStdString( row.at( i ) ).replace("'","''") );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
// error logs
|
||||
for ( int i : std::vector<int>({31,32,33,20}) ) {
|
||||
query_stmt += ", ";
|
||||
if ( row.find( i ) == row.end() ) {
|
||||
// no value found in the collection, bind NULL
|
||||
query_stmt += "NULL";
|
||||
} else {
|
||||
// value found, bind it
|
||||
perf_size += row.at( i ).size();
|
||||
query_stmt += QString("'%1'").arg( QString::fromStdString( row.at( i ) ).replace("'","''") );
|
||||
}
|
||||
}
|
||||
|
||||
// request
|
||||
for ( int i=10; i<14; i++ ) {
|
||||
query_stmt += ", ";
|
||||
if ( row.find( 30 ) == row.end() ) {
|
||||
if ( row.find( i ) == row.end() ) {
|
||||
// no value found in the collection, bind NULL
|
||||
query_stmt += "NULL";
|
||||
} else {
|
||||
// value found, bind it
|
||||
perf_size += row.at( 30 ).size();
|
||||
query_stmt += QString::fromStdString( row.at( 30 ) ).replace("'","''");
|
||||
perf_size += row.at( i ).size();
|
||||
query_stmt += QString("'%1'").arg( QString::fromStdString( row.at( i ) ).replace("'","''") );
|
||||
}
|
||||
}
|
||||
|
||||
for ( int i=14; i<18; i++ ) {
|
||||
query_stmt += ", ";
|
||||
if ( row.find( i ) == row.end() ) {
|
||||
// no value found in the collection, bind NULL
|
||||
query_stmt += "NULL";
|
||||
} else {
|
||||
// value found, bind it
|
||||
perf_size += row.at( i ).size();
|
||||
query_stmt += QString::fromStdString( row.at( i ) ).replace("'","''");
|
||||
}
|
||||
}
|
||||
|
||||
// client data and referrer
|
||||
for ( int i : std::vector<int>({18,20,21,22}) ) {
|
||||
query_stmt += ", ";
|
||||
if ( row.find( i ) == row.end() ) {
|
||||
// no value found in the collection, bind NULL
|
||||
query_stmt += "NULL";
|
||||
} else {
|
||||
// value found, bind it
|
||||
if ( i == 21 && wsID == 13 ) {
|
||||
// iis logs the user-agent using '+' instead of ' ' (spaces)
|
||||
QString str = QString::fromStdString( row.at( i ) ).replace("+"," ");
|
||||
perf_size += str.size();
|
||||
query_stmt += QString("'%1'").arg( str.replace("'","''") );
|
||||
} else {
|
||||
perf_size += row.at( i ).size();
|
||||
query_stmt += QString("'%1'").arg( QString::fromStdString( row.at( i ) ).replace("'","''") );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -333,7 +253,7 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector<
|
|||
}
|
||||
|
||||
// sum stored data size for the perfs
|
||||
craplog.sumPerfSize( perf_size, log_type );
|
||||
craplog.sumPerfSize( perf_size );
|
||||
|
||||
// finalize this statement
|
||||
if ( query.exec() == false ) {
|
||||
|
|
@ -350,10 +270,9 @@ bool StoreOps::storeData( QSqlDatabase& db, Craplog& craplog, const std::vector<
|
|||
break;
|
||||
}
|
||||
|
||||
// reset the statement to prepare for the next
|
||||
// reset the statement to prepare for the next one
|
||||
query.finish();
|
||||
}
|
||||
|
||||
|
||||
return successful;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ class StoreOps
|
|||
public:
|
||||
StoreOps();
|
||||
|
||||
static bool storeData( QSqlDatabase& db, Craplog& caplog, const std::vector<std::unordered_map<int, std::string>>& data, const int log_type );
|
||||
static bool storeData( QSqlDatabase& db, Craplog& caplog, const std::vector<std::unordered_map<int, std::string>>& data );
|
||||
};
|
||||
|
||||
#endif // STORE_H
|
||||
|
|
|
|||
Loading…
Reference in a new issue