first commit
This commit is contained in:
parent
f9465f1b90
commit
ccb6d4f3b0
GPG Key ID:
83289D84AA7C9A54
42
README.md
42
README.md
|
|
@ -1,3 +1,41 @@
|
|||
# logcat2sys
|
||||
# Synopsis
|
||||
|
||||
Logcat to Syslog (RFC5242) converter
|
||||
logcat2sys is a Logcat to Syslog (RFC5242) converter.
|
||||
|
||||
It accepts a stream of logcat-formatted messages, and sends them over
|
||||
TCP or UDP using the RFC5242 format.
|
||||
|
||||
# Usage
|
||||
|
||||
The host and port and mandatory.
|
||||
|
||||
Additionally, you can specify the delay between each packet, and the protocol.
|
||||
|
||||
In order to make reading/aggregating logs easier, you can specify a `log-id`,
|
||||
which is then added to the log line, so you can grep it, aggregate it, configure
|
||||
Promtail to parse it, etc.
|
||||
|
||||
## Example
|
||||
|
||||
```sh
|
||||
adb logcat | logcat2sys --proto TCP --log-id second-run my-promtail 1514
|
||||
```
|
||||
|
||||
## Options
|
||||
|
||||
```
|
||||
usage: logcat2sys.py [-h] [--proto PROTO] [--delay DELAY] [--log-id LOG_ID] [--debug] hostname port
|
||||
|
||||
logcat to syslog conversion.
|
||||
|
||||
positional arguments:
|
||||
hostname hostname to connect to
|
||||
port hostname's port
|
||||
|
||||
optional arguments:
|
||||
-h, --help show this help message and exit
|
||||
--proto PROTO protocol (TCP/UDP, default UDP)
|
||||
--delay DELAY sleep between logs
|
||||
--log-id LOG_ID logcat2sys log id
|
||||
--debug enable debugging
|
||||
```
|
||||
|
|
|
|||
|
|
@ -0,0 +1,291 @@
|
|||
#!/bin/env python
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import IntEnum
|
||||
|
||||
from datetime import datetime
|
||||
import socket
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
#
|
||||
# Logcat format according to:
|
||||
# 01-30 20:40:03.004 213 213 I lowmemorykiller: Using in-kernel low memory killer interface
|
||||
#
|
||||
default_logcat_format = \
|
||||
"([0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}).([0-9]{3})[ ]+" + \
|
||||
"([0-9]+)[ ]+([0-9]+)[ ]+" + \
|
||||
"([IDTVEWF])[ ]+" + \
|
||||
"(.+?):[ ]+" + \
|
||||
"(.*)"
|
||||
|
||||
default_logcat_date_format = "%m-%d %H:%M:%S"
|
||||
|
||||
#
|
||||
# Exceptions & constants
|
||||
#
|
||||
class ParsingException(Exception):
|
||||
pass
|
||||
class NetworkException(Exception):
|
||||
pass
|
||||
|
||||
RFC5425_DATEFMT = "%Y-%m-%dT%H:%M:%S.%fZ"
|
||||
DEFAULT_HOSTNAME = "logcat2sys"
|
||||
DEFAULT_SLEEP_MS = 50
|
||||
|
||||
#
|
||||
# Internal log level
|
||||
#
|
||||
class LogLevel(IntEnum):
|
||||
LOG_EMERG = 0
|
||||
LOG_ALERT = 1
|
||||
LOG_CRIT = 2
|
||||
LOG_ERR = 3
|
||||
LOG_WARNING = 4
|
||||
LOG_NOTICE = 5
|
||||
LOG_INFO = 6
|
||||
LOG_DEBUG = 7
|
||||
|
||||
def from_str(level_str):
|
||||
if level_str == "M":
|
||||
return LogLevel.LOG_EMERG
|
||||
elif level_str == "A":
|
||||
return LogLevel.LOG_ALERT
|
||||
elif level_str == "C":
|
||||
return LogLevel.LOG_CRIT
|
||||
elif level_str == "E" or level_str == "F":
|
||||
return LogLevel.LOG_ERR
|
||||
elif level_str == "W":
|
||||
return LogLevel.LOG_WARNING
|
||||
elif level_str == "N":
|
||||
return LogLevel.LOG_NOTICE
|
||||
elif level_str == "I" or level_str == "V":
|
||||
return LogLevel.LOG_INFO
|
||||
elif level_str == "D":
|
||||
return LogLevel.LOG_DEBUG
|
||||
else:
|
||||
raise ParsingException("unparsable log level: %s" % level_str)
|
||||
|
||||
def to_syslog_str(self):
|
||||
if self == LogLevel.LOG_EMERG:
|
||||
return "emerg"
|
||||
if self == LogLevel.LOG_ALERT:
|
||||
return "alert"
|
||||
if self == LogLevel.LOG_CRIT:
|
||||
return "critical"
|
||||
if self == LogLevel.LOG_ERR:
|
||||
return "error"
|
||||
if self == LogLevel.LOG_WARNING:
|
||||
return "warn"
|
||||
if self == LogLevel.LOG_NOTICE:
|
||||
return "notice"
|
||||
if self == LogLevel.LOG_INFO:
|
||||
return "info"
|
||||
if self == LogLevel.LOG_DEBUG:
|
||||
return "debug"
|
||||
|
||||
#
|
||||
# Internal log record
|
||||
#
|
||||
@dataclass(frozen=True)
|
||||
class LogRecord():
|
||||
timestamp: datetime = datetime.now()
|
||||
level: LogLevel = LogLevel.LOG_INFO
|
||||
pid: int = -1
|
||||
ident: str = "logcat"
|
||||
log_line: str = ""
|
||||
|
||||
def serialize(self, log_id):
|
||||
sep = " "
|
||||
|
||||
priority = "<%d>" % int(self.level)
|
||||
version = 1
|
||||
timestamp = self.timestamp.strftime(RFC5425_DATEFMT)
|
||||
|
||||
hostname = DEFAULT_HOSTNAME
|
||||
|
||||
app_name = self.ident
|
||||
procid = self.pid
|
||||
msgid = "-"
|
||||
|
||||
# structured_data = "-"
|
||||
structured_data = "[logcat2sys logid=\"%s\"]" % log_id
|
||||
|
||||
message = self.log_line
|
||||
|
||||
# final message
|
||||
header = \
|
||||
("%s%d%s%s" + \
|
||||
"%s%s" + \
|
||||
"%s%s%s%d%s%s") % \
|
||||
(priority, version, sep, timestamp,
|
||||
sep, hostname,
|
||||
sep, app_name, sep, procid, sep, msgid)
|
||||
|
||||
syslog_message = "%s%s%s%s%s%s" % (header, sep, structured_data, sep, message, "\n")
|
||||
|
||||
# done
|
||||
return syslog_message
|
||||
|
||||
#
|
||||
# The Logcat decoder
|
||||
#
|
||||
class LogcatDecoder():
|
||||
def __init__(self, logcat_format, logcat_date_format):
|
||||
self.logcat_format = logcat_format
|
||||
self.logcat_date_format = logcat_date_format
|
||||
|
||||
def decode(self, raw_line):
|
||||
matched_line = re.search(self.logcat_format, raw_line)
|
||||
|
||||
try:
|
||||
raw_date = matched_line.group(1)
|
||||
raw_date_milis = matched_line.group(2)
|
||||
raw_pid = matched_line.group(3)
|
||||
raw_level = matched_line.group(5)
|
||||
raw_ident = matched_line.group(6)
|
||||
raw_line = matched_line.group(7)
|
||||
|
||||
parsed_date = datetime.strptime(raw_date, self.logcat_date_format)
|
||||
|
||||
try:
|
||||
parsed_milis = int(raw_date_milis)
|
||||
parsed_date = parsed_date.replace(microsecond = parsed_milis * 1000)
|
||||
except:
|
||||
pass
|
||||
|
||||
if parsed_date.year == 1900:
|
||||
parsed_date = parsed_date.replace(year=2022)
|
||||
|
||||
except AttributeError:
|
||||
raise ParsingException("no match for line")
|
||||
|
||||
return LogRecord(
|
||||
timestamp = parsed_date,
|
||||
pid = int(raw_pid),
|
||||
level = LogLevel.from_str(raw_level),
|
||||
ident = raw_ident.strip(),
|
||||
log_line = raw_line.strip())
|
||||
|
||||
#
|
||||
# The Syslog writer
|
||||
#
|
||||
class SyslogWriter():
|
||||
def __init__(self, host, port, log_id="dummy", proto=socket.SOCK_DGRAM, retries=3):
|
||||
self.host = host
|
||||
self.port = port
|
||||
self.proto = proto
|
||||
self.retries = retries
|
||||
self.log_id = log_id
|
||||
|
||||
logger = logging.Logger("SyslogWriter")
|
||||
logger.setLevel(logging.INFO)
|
||||
|
||||
lh = logging.StreamHandler()
|
||||
lh.setLevel(logging.INFO)
|
||||
|
||||
logger.addHandler(lh)
|
||||
|
||||
logger.info("creating writer")
|
||||
|
||||
self.logger = logger
|
||||
|
||||
self.sock = None
|
||||
self._init_socket()
|
||||
|
||||
def _init_socket(self):
|
||||
self.logger.info("connecting...")
|
||||
|
||||
for i in range(self.retries):
|
||||
self.logger.info("retry no %d" % i)
|
||||
|
||||
new_socket = socket.socket(
|
||||
family = socket.AF_INET,
|
||||
type = self.proto)
|
||||
|
||||
if self.proto == socket.SOCK_STREAM:
|
||||
try:
|
||||
new_socket.connect((self.host, self.port))
|
||||
except Exception as e:
|
||||
self.logger.error("retry failed: %s" % e)
|
||||
continue
|
||||
|
||||
self.sock = new_socket
|
||||
self.logger.info("connection successful!")
|
||||
return
|
||||
|
||||
self.logger.error("could not connect to %s:%s", self.host, self.port)
|
||||
raise NetworkException("could not connect to %s:%s" % (self.host, self.port))
|
||||
|
||||
def _write_raw(self, data):
|
||||
raw_data = data.encode('utf-8')
|
||||
if self.proto == socket.SOCK_STREAM:
|
||||
self.sock.sendall(raw_data)
|
||||
else:
|
||||
self.sock.sendto(raw_data, (self.host, self.port))
|
||||
|
||||
def write(self, log_record):
|
||||
syslog_message = log_record.serialize(self.log_id)
|
||||
self.logger.debug("message: %s", syslog_message)
|
||||
self._write_raw(syslog_message)
|
||||
|
||||
#
|
||||
# The main function
|
||||
#
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description='logcat to syslog conversion.')
|
||||
parser.add_argument('hostname', metavar='hostname', type=str,
|
||||
help='hostname to connect to')
|
||||
parser.add_argument('port', metavar='port', type=int,
|
||||
help="hostname's port")
|
||||
parser.add_argument('--proto', dest='proto', type=str,
|
||||
default="UDP",
|
||||
help="protocol (TCP/UDP, default UDP)")
|
||||
parser.add_argument('--delay', dest='delay', type=int,
|
||||
default=DEFAULT_SLEEP_MS,
|
||||
help="sleep between logs")
|
||||
parser.add_argument('--log-id', dest='log_id', type=str,
|
||||
default="dummy", help="logcat2sys log id")
|
||||
parser.add_argument('--debug', dest='debug',
|
||||
action='store_true', default=False, help='enable debugging')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.proto not in ["TCP", "UDP"]:
|
||||
raise ParsingException("could not parse proto %s" % args.proto)
|
||||
|
||||
prog_proto = socket.SOCK_DGRAM
|
||||
if args.proto == "TCP":
|
||||
prog_proto = socket.SOCK_STREAM
|
||||
|
||||
logger = logging.Logger("logcat2sys")
|
||||
lh = logging.StreamHandler()
|
||||
|
||||
if args.debug:
|
||||
logger.setLevel(logging.DEBUG)
|
||||
lh.setLevel(logging.DEBUG)
|
||||
else:
|
||||
logger.setLevel(logging.INFO)
|
||||
lh.setLevel(logging.INFO)
|
||||
|
||||
logger.addHandler(lh)
|
||||
|
||||
# Actual run
|
||||
decoder = LogcatDecoder(default_logcat_format, default_logcat_date_format)
|
||||
syslog_handler = SyslogWriter(args.hostname, args.port, args.log_id, prog_proto)
|
||||
|
||||
log_input = sys.stdin
|
||||
|
||||
for raw_log_line in log_input:
|
||||
try:
|
||||
lr = decoder.decode(raw_log_line)
|
||||
logger.debug(lr)
|
||||
syslog_handler.write(lr)
|
||||
time.sleep(args.delay/1000)
|
||||
except ParsingException as e:
|
||||
logger.info("failed to parse line '%s': %s", raw_log_line, e)
|
||||
|
||||
Loading…
Reference in New Issue