diff --git a/security/etpro_telemetry/Makefile b/security/etpro_telemetry/Makefile deleted file mode 100644 index 8ccca174a..000000000 --- a/security/etpro_telemetry/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -PLUGIN_NAME= etpro_telemetry -PLUGIN_VERSION= 1.0 -#PLUGIN_REVISION= 1 -PLUGIN_COMMENT= ETPRO telemetry edition -PLUGIN_MAINTAINER= ad@opnsense.org -PLUGIN_WWW= https://docs.opnsense.org/manual/etpro_telemetry.html - -.include "../../Mk/plugins.mk" diff --git a/security/etpro_telemetry/pkg-descr b/security/etpro_telemetry/pkg-descr deleted file mode 100644 index 43a8d3e25..000000000 --- a/security/etpro_telemetry/pkg-descr +++ /dev/null @@ -1,20 +0,0 @@ -Todays cybersecurity engineers need timely and accurate data about eminent threats and how they spread around the globe. -With this data cybersecurity researchers and analysts can improve the detection of malicious network traffic. -The times when we could rely on just firewall rules for our protection are long gone. -Additional layers of security are desperately needed to guard against these attacks. - -With growing risks the need to fortify our security is growing for both big enterprises as for SMEs alike, but often -out of reach for the latter. -An important extra security addition is an Intrusion Detection and Prevention System (IDS/IPS). - -The IDS/IPS available in OPNsense is based on Suricata. -This open source IDS/IPS engine has proven its value in OPNsense, especially in combination with the free Proofpoint ETOpen ruleset. - -The need for valuable threat detection data and the increasing importance of additional network security -has brought Proofpoint and OPNsense together. -Our joined efforts resulted in the ETPro Telemetry edition. - -The ETPro Telemetry edition embraces our vision that sharing knowledge leads to better products. - -When you allow your OPNsense system to share anonymized information about detected threats - the alerts - -you are able to use the ETPro ruleset free of charge. diff --git a/security/etpro_telemetry/src/etc/cron.d/etpro-telemetry.cron b/security/etpro_telemetry/src/etc/cron.d/etpro-telemetry.cron deleted file mode 100644 index bd4d7b1e7..000000000 --- a/security/etpro_telemetry/src/etc/cron.d/etpro-telemetry.cron +++ /dev/null @@ -1,10 +0,0 @@ -# DO NOT EDIT THIS FILE -- OPNsense auto-generated file -# -# User-defined crontab files can be loaded via /etc/cron.d -# or /usr/local/etc/cron.d and follow the same format as -# /etc/crontab, see the crontab(5) manual page. -SHELL=/bin/sh -PATH=/etc:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin -#minute hour mday month wday who command -0,30 * * * * root /usr/local/opnsense/scripts/etpro_telemetry/send_heartbeat.py -* * * * * root /usr/local/opnsense/scripts/etpro_telemetry/send_telemetry.py diff --git a/security/etpro_telemetry/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/ProofpointEtController.php b/security/etpro_telemetry/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/ProofpointEtController.php deleted file mode 100644 index e817ae4f1..000000000 --- a/security/etpro_telemetry/src/opnsense/mvc/app/controllers/OPNsense/Diagnostics/Api/ProofpointEtController.php +++ /dev/null @@ -1,53 +0,0 @@ -sessionClose(); - $backend = new Backend(); - $response = $backend->configdRun('proofpoint et status'); - $activity = json_decode($response, true); - - return $activity; - } -} diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/dump_data.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/dump_data.py deleted file mode 100755 index 884b52bf4..000000000 --- a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/dump_data.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/local/bin/python2.7 - -""" - Copyright (c) 2018-2019 Ad Schellevis - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. -""" - -import argparse -import urllib3 -import datetime -import ujson -import telemetry.log -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - -parser = argparse.ArgumentParser() -parser.add_argument('-l', '--log', help='log directory containing eve.json files', default="/var/log/suricata/") -parser.add_argument('-t', '--time', help='max seconds to read from now()', type=int, default=3600) -parser.add_argument('-p', '--parsed', help='show data as shipped using send_telemetry', - default=False, action="store_true") -parser.add_argument('-L', '--limit', help='limit number of rows', type=int, default=-1) -args = parser.parse_args() - -last_update = datetime.datetime.now() - datetime.timedelta(seconds=float(args.time)) - -event_collector = telemetry.EventCollector() -row_count = 0 -for record in telemetry.log.reader(args.log, last_update=last_update): - if args.parsed: - event_collector.push(record) - else: - print (ujson.dumps(record)) - - row_count += 1 - if args.limit != -1 and row_count >= args.limit: - break - -if args.parsed: - for push_data in event_collector: - print (push_data.strip()) diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/send_heartbeat.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/send_heartbeat.py deleted file mode 100755 index 232edb9ce..000000000 --- a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/send_heartbeat.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/local/bin/python2.7 - -""" - Copyright (c) 2018-2019 Ad Schellevis - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. -""" - -import sys -import argparse -import requests -import syslog -import urllib3 -import telemetry -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - -parser = argparse.ArgumentParser() -parser.add_argument('-e', '--endpoint', - help='Endpoint url to reach', - default="%s/api/v1/sensor" % telemetry.BASE_URL) -parser.add_argument('-i', '--insecure', - help='Insecure, skip certificate validation', - action="store_true", - default=False) -parser.add_argument('-c', '--config', - help='rule downloader configuration', - default="/usr/local/etc/suricata/rule-updater.config" - ) -args = parser.parse_args() - -exit_code = -1 -cnf = telemetry.get_config(args.config) -if cnf.token is not None: - params = {'timeout': 5, 'headers': {'Authorization': 'Bearer %s' % cnf.token}} - if args.insecure: - params['verify'] = False - try: - r = requests.head(args.endpoint, **params) - if r.status_code == 200: - # expected result, set exit code - exit_code = 0 - else: - syslog.syslog(syslog.LOG_ERR, 'unexpected result from %s (http_code %s)' % (args.endpoint, r.status_code)) - except requests.exceptions.ConnectionError: - syslog.syslog(syslog.LOG_ERR, 'connection error sending heardbeat to %s' % args.endpoint) -else: - syslog.syslog(syslog.LOG_ERR, 'telemetry token missing in %s' % args.config) - - -# exit -sys.exit(exit_code) diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/send_telemetry.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/send_telemetry.py deleted file mode 100755 index ece86d1e3..000000000 --- a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/send_telemetry.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/local/bin/python2.7 - -""" - Copyright (c) 2018-2019 Ad Schellevis - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. -""" - -import sys -import os -import argparse -import requests -import time -import syslog -import urllib3 -import ujson -import telemetry.log -import telemetry.state - -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - -parser = argparse.ArgumentParser() -parser.add_argument('-e', '--endpoint', help='Endpoint url to reach', - default="%s/api/v1/event" % telemetry.BASE_URL) -parser.add_argument('-i', '--insecure', help='Insecure, skip certificate validation', - action="store_true", default=False) -parser.add_argument('-c', '--config', help='rule downloader configuration', - default="/usr/local/etc/suricata/rule-updater.config") -parser.add_argument('-l', '--log', help='log directory containing eve.json files', - default="/var/log/suricata/") -parser.add_argument('-s', '--state', help='persistent state (and lock) filename', - default="/usr/local/var/run/et_telemetry.state") -parser.add_argument('-d', '--days', help='Maximum number of days to look back on initial run', type=int, default=2) -args = parser.parse_args() - - -exit_code = -1 -send_start_time = time.time() -telemetry_state = telemetry.state.Telemetry(filename=args.state, init_last_days=args.days) -if not telemetry_state.is_running(): - cnf = telemetry.get_config(args.config) - if cnf.token is not None: - if os.path.isdir(args.log): - last_update = telemetry_state.get_last_update() - event_collector = telemetry.EventCollector() - row_count = 0 - max_timestamp = None - for record in telemetry.log.reader(args.log, last_update): - if max_timestamp is None or record['__timestamp__'] > max_timestamp: - max_timestamp = record['__timestamp__'] - event_collector.push(record) - row_count += 1 - - # data collected, log and push - if row_count > 0 and max_timestamp is not None: - syslog.syslog( - syslog.LOG_NOTICE, - 'telemetry data collected %d records in %.2f seconds @%s' % ( - row_count, time.time() - send_start_time, max_timestamp - ) - ) - for push_data in event_collector: - params = { - 'timeout': 5, - 'headers': {'Authorization': 'Bearer %s' % cnf.token}, - 'data': push_data.strip() - } - if args.insecure: - params['verify'] = False - - r = requests.post(args.endpoint, **params) - if r.status_code != 201: - syslog.syslog( - syslog.LOG_ERR, - 'unexpected result from %s (http_code %s)' % (args.endpoint, r.status_code) - ) - exit_code = -1 - break - else: - try: - ujson.loads(r.text) - exit_code = 0 - except ValueError: - syslog.syslog(syslog.LOG_ERR, 'telemetry unexpected response %s' % r.text[:256]) - exit_code = -1 - break - if exit_code == 0: - # update timestamp, last record processed - telemetry_state.set_last_update(max_timestamp) - else: - # no data - exit_code = 0 - else: - syslog.syslog(syslog.LOG_ERR, 'telemetry token missing in %s' % args.config) - else: - syslog.syslog(syslog.LOG_ERR, 'directory %s missing' % args.log) - - -sys.exit(exit_code) diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/sensor_info.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/sensor_info.py deleted file mode 100755 index bb6abc456..000000000 --- a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/sensor_info.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/local/bin/python2.7 - -""" - Copyright (c) 2018-2019 Ad Schellevis - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. -""" - -import argparse -import requests -import urllib3 -import ujson -import telemetry -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - -parser = argparse.ArgumentParser() -parser.add_argument('-e', '--endpoint', help='Endpoint url to reach', - default="%s/api/v1/sensorinfo" % telemetry.BASE_URL) -parser.add_argument('-i', '--insecure', help='Insecure, skip certificate validation', - action="store_true", default=False) -parser.add_argument('-c', '--config', help='rule downloader configuration', - default="/usr/local/etc/suricata/rule-updater.config") -args = parser.parse_args() - -cnf = telemetry.get_config(args.config) -if cnf.token is not None: - try: - req = requests.get(args.endpoint, headers={'Authorization': 'Bearer %s' % cnf.token}) - if req.status_code == 200: - response = ujson.loads(req.text) - response['status'] = 'ok' - else: - response = {'status': 'failed', 'response': req.text} - except requests.exceptions.SSLError as e: - response = {'status': 'failed', 'response': '%s' % e} - except ValueError: - response = {'status': 'failed', 'response': req.text} -else: - response = {'status': 'unconfigured'} - -print (ujson.dumps(response)) diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/__init__.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/__init__.py deleted file mode 100644 index ee66262f0..000000000 --- a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/__init__.py +++ /dev/null @@ -1,140 +0,0 @@ -""" - Copyright (c) 2018-2019 Ad Schellevis - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. -""" -import os -import subprocess -import tempfile -import collections -try: - from configparser import ConfigParser -except ImportError: - from ConfigParser import ConfigParser -import netaddr -import ujson - - -BASE_URL = 'https://opnsense.emergingthreats.net' - - -def get_config(rule_update_config): - """ - retrieve device token, since we align our telemetry data to the existing rule download feature in OPNsense - it should be safe to assume rule-updater.config contains the token that is used. - :param rule_update_config: path to OPNsense rule update configuration - :return: token id or None if not found - """ - response = collections.namedtuple('sensor', 'token') - if os.path.exists(rule_update_config): - cnf = ConfigParser() - cnf.read(rule_update_config) - if cnf.has_section('__properties__'): - if cnf.has_option('__properties__', 'et_telemetry.token'): - response.token = cnf.get('__properties__', 'et_telemetry.token') - - return response - - -class EventCollector(object): - """ Event collector, responsible for extracting and anonymising from an eve.json stream - """ - def __init__(self): - self._tmp_handle = tempfile.NamedTemporaryFile() - self._local_networks = list() - self._get_local_networks() - - def _get_local_networks(self): - """ collect local attached networks for anonymization purposes - :return: None - """ - with tempfile.NamedTemporaryFile() as output_stream: - subprocess.call(['ifconfig', '-a'], stdout=output_stream, stderr=open(os.devnull, 'wb')) - output_stream.seek(0) - for line in output_stream: - if line.startswith(b'\tinet'): - parts = line.split() - if len(parts) > 3: - if parts[0] == 'inet6' and parts[2] == 'prefixlen': - # IPv6 addresses - self._local_networks.append( - netaddr.IPNetwork("%s/%s" % (parts[1].split('%')[0], parts[3])) - ) - elif parts[0] == 'inet' and len(parts) > 3 and parts[2] == 'netmask': - # IPv4 addresses - mask = int(parts[3], 16) - self._local_networks.append( - netaddr.IPNetwork("%s/%s" % (netaddr.IPAddress(parts[1]), netaddr.IPAddress(mask))) - ) - - def is_local_address(self, address): - """ check if provided address is local for this device - :param address: address (string) - :return: boolean - """ - addr_to_check = netaddr.IPAddress(address) - for local_network in self._local_networks: - if addr_to_check in local_network: - return True - return False - - def push(self, record): - """ cleanup and write record - :param record: parsed eve log record - :return: None - """ - to_push = dict() - for address in ['src_ip', 'dest_ip']: - if address in record: - if self.is_local_address(record[address]): - if record[address].find(':') > -1: - # replace local IPv6 address - to_push[address] = 'xxxx:xxxx:%s' % ':'.join(record[address].split(':')[-2:]) - else: - to_push[address] = 'xxx.xxx.xxx.%s' % record[address].split('.')[-1] - else: - # non local address - to_push[address] = record[address] - - # unfiltered output fields - for attr in ["timestamp", "flow_id", "in_iface", "event_type", - "vlan", "src_port", "dest_port", "proto", "alert"]: - if attr in record: - to_push[attr] = record[attr] - - self._tmp_handle.write(("%s\n" % ujson.dumps(to_push)).encode()) - - def get(self): - """ fetch all data from temp - :return: - """ - self._tmp_handle.seek(0) - return self._tmp_handle.read() - - def __iter__(self): - """ Iterate parsed events - :return: - """ - self._tmp_handle.seek(0) - for line in self._tmp_handle: - yield line diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/log.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/log.py deleted file mode 100644 index 67fa20592..000000000 --- a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/log.py +++ /dev/null @@ -1,105 +0,0 @@ -""" - Copyright (c) 2018-2019 Ad Schellevis - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. -""" -import os -import re -import glob -import datetime -import ujson - - -def reverse_log_reader(filename): - """ read log file in reverse order - :param filename: filename - :return: generator - """ - block_size = 81920 - input_stream = open(filename, 'rU') - input_stream.seek(0, os.SEEK_END) - file_byte_start = input_stream.tell() - - data = '' - while file_byte_start > 0: - if file_byte_start - block_size < 0: - block_size = file_byte_start - file_byte_start = 0 - else: - file_byte_start -= block_size - - input_stream.seek(file_byte_start) - - data = input_stream.read(block_size) + data - bol = data.rfind('\n') - eol = len(data) - - while bol > -1: - yield data[bol:eol] - eol = bol - bol = data.rfind('\n', 0, eol) - - data = data[:eol] if bol == -1 else '' - - if file_byte_start == 0 and bol == -1: - yield data - - -def parse_log_line(line): - """ parse eve log records, add __timestamp__ containing a timezoned datetime object - :param line: raw json string - :return: dict or None if unable to parse - """ - try: - record = ujson.loads(line.strip()) - except ValueError: - record = dict() - if 'timestamp' in record: - # convert timestamp - try: - tmp = list(map(lambda x: int(x), re.split('T|-|\:|\.|\+', record['timestamp']))) - tz = record['timestamp'][-5:] - ts = datetime.datetime(*tmp[:7]) - ts -= datetime.timedelta(hours=int(tz[2:3]), minutes=int(tz[-2:])) * int(tz[0:1] + '1') - record['__timestamp__'] = ts - return record - except ValueError: - pass - - return None - - -def reader(log_directory, last_update): - """ read and parse eve logs until timestamp is reached - :param log_directory: directory to search for eve.json* - :param last_update: datetime of last send event - :return: iterator - """ - for filename in sorted(glob.glob("%s/eve.json*" % log_directory)): - for line in reverse_log_reader(filename=filename): - record = parse_log_line(line) - if record: - if record['__timestamp__'] > last_update: - yield record - else: - return diff --git a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/state.py b/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/state.py deleted file mode 100644 index d5ea56783..000000000 --- a/security/etpro_telemetry/src/opnsense/scripts/etpro_telemetry/telemetry/state.py +++ /dev/null @@ -1,71 +0,0 @@ -""" - Copyright (c) 2018-2019 Ad Schellevis - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, - INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, - OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. -""" -import fcntl -import datetime - - -class Telemetry(object): - def __init__(self, filename, init_last_days=2): - self._filename = filename - self._init_last_days = init_last_days - try: - self._file_handle = open(self._filename, 'a+') - fcntl.flock(self._file_handle, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - # already running - self._file_handle = None - - def is_running(self): - return self._file_handle is None - - def get_last_update(self): - """ return last used timestamp - :return: datetime - """ - self._file_handle.seek(0) - try: - result = datetime.datetime.fromtimestamp(float(self._file_handle.readline())) - except ValueError: - result = datetime.datetime.now() - datetime.timedelta(days=self._init_last_days) - - return result - - def set_last_update(self, stamp): - """ set last timestamp - :param stamp: datetime object - :return: - """ - self._file_handle.seek(0) - self._file_handle.truncate() - self._file_handle.write("%s\n" % stamp.strftime('%s.%f')) - - def __del__(self): - """ close file handle on destruct - :return: - """ - if self._file_handle: - self._file_handle.close() - diff --git a/security/etpro_telemetry/src/opnsense/scripts/suricata/metadata/rules/et-telemetry.xml b/security/etpro_telemetry/src/opnsense/scripts/suricata/metadata/rules/et-telemetry.xml deleted file mode 100644 index 8b6da84fb..000000000 --- a/security/etpro_telemetry/src/opnsense/scripts/suricata/metadata/rules/et-telemetry.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - Bearer %%et_telemetry.token%% - - - - botcc.portgrouped.rules - botcc.rules - ciarmy.rules - compromised.rules - drop.rules - dshield.rules - emerging-activex.rules - emerging-attack_response.rules - emerging-chat.rules - emerging-current_events.rules - emerging-deleted.rules - emerging-dns.rules - emerging-dos.rules - emerging-exploit.rules - emerging-ftp.rules - emerging-games.rules - emerging-icmp.rules - emerging-icmp_info.rules - emerging-imap.rules - emerging-inappropriate.rules - emerging-info.rules - emerging-malware.rules - emerging-misc.rules - emerging-mobile_malware.rules - emerging-netbios.rules - emerging-p2p.rules - emerging-policy.rules - emerging-pop3.rules - emerging-rpc.rules - emerging-scada.rules - emerging-scan.rules - emerging-shellcode.rules - emerging-smtp.rules - emerging-snmp.rules - emerging-sql.rules - emerging-telnet.rules - emerging-tftp.rules - emerging-trojan.rules - emerging-user_agents.rules - emerging-voip.rules - emerging-web_client.rules - emerging-web_server.rules - emerging-web_specific_apps.rules - emerging-worm.rules - tor.rules - - - - - diff --git a/security/etpro_telemetry/src/opnsense/service/conf/actions.d/actions_proofpoint.conf b/security/etpro_telemetry/src/opnsense/service/conf/actions.d/actions_proofpoint.conf deleted file mode 100644 index 41a3c0b48..000000000 --- a/security/etpro_telemetry/src/opnsense/service/conf/actions.d/actions_proofpoint.conf +++ /dev/null @@ -1,5 +0,0 @@ -[et.status] -command:/usr/local/opnsense/scripts/etpro_telemetry/sensor_info.py -parameters: -type:script_output -message:proofpoint ids telemetry status diff --git a/security/etpro_telemetry/src/opnsense/www/img/proofpoint.svg b/security/etpro_telemetry/src/opnsense/www/img/proofpoint.svg deleted file mode 100644 index 564d1adfc..000000000 --- a/security/etpro_telemetry/src/opnsense/www/img/proofpoint.svg +++ /dev/null @@ -1,26 +0,0 @@ - - - - - Asset 1 - - - - - - - - - - - - - - - - - diff --git a/security/etpro_telemetry/src/www/widgets/include/proofpoint_et.inc b/security/etpro_telemetry/src/www/widgets/include/proofpoint_et.inc deleted file mode 100644 index abf77c3bc..000000000 --- a/security/etpro_telemetry/src/www/widgets/include/proofpoint_et.inc +++ /dev/null @@ -1,2 +0,0 @@ - - - - - - - - - - - - - - - - - - - -
- -
- -
-
- - Deciso B.V. -