diff options
| -rw-r--r-- | src/soc_collector/schema.py | 57 | ||||
| -rw-r--r-- | src/soc_collector/soc_collector_cli.py | 8 | ||||
| -rw-r--r-- | tests/data/example_data_1.json | 9 | ||||
| -rw-r--r-- | tests/data/example_data_1_replace_test.json | 9 | ||||
| -rw-r--r-- | tests/data/example_data_3.json | 8 | ||||
| -rw-r--r-- | tests/data/example_data_3_replace_test.json | 8 |
6 files changed, 65 insertions, 34 deletions
diff --git a/src/soc_collector/schema.py b/src/soc_collector/schema.py index 2c2dfb9..7688631 100644 --- a/src/soc_collector/schema.py +++ b/src/soc_collector/schema.py @@ -1,6 +1,10 @@ """Our schema module""" from typing import Any, Dict, Optional, Union -import jsonschema +from jsonschema.exceptions import ValidationError + +# docker-compose require jsonschema < 4 so use Draft7 for now +from jsonschema.validators import Draft7Validator + from bson import ObjectId from bson.errors import InvalidId @@ -8,12 +12,19 @@ from bson.errors import InvalidId # NOTE: Commented out properties are left intentionally, so it is easier to see # what properties are optional. schema = { - "$schema": "http://json-schema.org/schema#", + "$schema": "https://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "document_version": {"type": "integer"}, + "document_version": {"type": "integer", "minimum": 2}, + "tags": { + "type": "array", + "uniqueItems": True, + "items": {"type": "string", "enum": ["dev", "display", "hide", "notify"]}, + "minItems": 1, + "maxItems": 3 # hide is incompatible with notify and/or display + }, "ip": {"type": "string"}, - "port": {"type": "integer"}, + "port": {"type": "integer", "minimum": 1, "maximum": 65535}, "whois_description": {"type": "string"}, "asn": {"type": "string"}, "asn_country_code": {"type": "string"}, @@ -50,34 +61,29 @@ schema = { "display_name": {"type": "string"}, "vulnerable": {"type": "boolean"}, "investigation_needed": {"type": "boolean"}, - "reliability": {"type": "integer"}, + "reliability": {"type": "integer", "minimum": 1, "maximum": 5}, + "severity": {"type": "integer", "minimum": 1, "maximum": 5}, "description": {"type": "string"}, }, "oneOf": [ { - "required": [ - "display_name", - "vulnerable", - # "reliability", # TODO: reliability is required if vulnerable = true - # "description", - ] + "properties": {"investigation_needed": {"const": True}}, + "required": ["display_name", "investigation_needed", "description"] }, { - "required": - [ - "display_name", - "investigation_needed", - # "reliability", # TODO: reliability is required if investigation_needed = true - # "description", - ] - }, + "properties": {"vulnerable": {"type": "boolean"}}, + "if": {"properties": {"vulnerable": {"const": True}}}, + "then": {"required": ["display_name", "vulnerable", "reliability", "severity", "description"]}, + "else": {"required": ["display_name", "vulnerable"]} + } ] - }, - }, - }, + } + } + } }, "required": [ "document_version", + "tags", "ip", "port", "whois_description", @@ -88,9 +94,9 @@ schema = { "domain", "timestamp", "display_name", + "result" # "description", # "custom_data", - "result", ], } @@ -103,8 +109,9 @@ def valid_schema(json_data: Dict[str, Any]) -> bool: """ try: - jsonschema.validate(json_data, schema, format_checker=jsonschema.FormatChecker()) - except jsonschema.exceptions.ValidationError as exc: + # docker-compose require jsonschema < 4 so use Draft7 for now + Draft7Validator(schema).validate(json_data) + except ValidationError as exc: print(f"Validation failed with error: {exc.message}") return False return True diff --git a/src/soc_collector/soc_collector_cli.py b/src/soc_collector/soc_collector_cli.py index 4929655..85afce1 100644 --- a/src/soc_collector/soc_collector_cli.py +++ b/src/soc_collector/soc_collector_cli.py @@ -8,7 +8,7 @@ from sys import exit as app_exit import json import requests -from .schema import object_id_from_data +from .schema import object_id_from_data, valid_schema ROOT_CA_FILE = __file__.replace("soc_collector_cli.py", "data/collector_root_ca.crt") @@ -136,6 +136,9 @@ def replace_action(data: str, api_key: str, base_url: str = "https://collector-d print("ERROR: Valid '_id' key not in data") app_exit(1) + if not valid_schema(json_data): + app_exit(1) + req = requests.put( f"{base_url}/sc/v0", json=json_data, headers={"API-KEY": api_key}, timeout=5, verify=ROOT_CA_FILE ) @@ -166,6 +169,9 @@ def insert_action(data: str, api_key: str, base_url: str = "https://collector-de print("ERROR: '_id' key in data") app_exit(1) + if not valid_schema(json_data): + app_exit(1) + req = requests.post( f"{base_url}/sc/v0", json=json_data, headers={"API-KEY": api_key}, timeout=5, verify=ROOT_CA_FILE ) diff --git a/tests/data/example_data_1.json b/tests/data/example_data_1.json index 69f5d85..b0adb8a 100644 --- a/tests/data/example_data_1.json +++ b/tests/data/example_data_1.json @@ -1,5 +1,6 @@ { - "document_version": 1, + "document_version": 2, + "tags": ["dev", "hide"], "ip": "192.0.2.10", "port": 443, "whois_description": "SOMENET", @@ -35,7 +36,9 @@ "cve_2015_0060": { "display_name": "CVE-2015-0060", "vulnerable": true, - "reliability": 2 + "reliability": 2, + "severity": 2, + "description": "Allows local users to cause a denial of service (system hang) via a crafted application, aka Windows Font Driver Denial of Service Vulnerability." }, "cve_2015_0063": { "display_name": "CVE-2015-0063", @@ -45,12 +48,14 @@ "display_name": "Insecure cryptography", "vulnerable": true, "reliability": 5, + "severity": 3, "description": "Uses RSA instead of elliptic curve." }, "possible_webshell": { "display_name": "Webshells (PST)", "investigation_needed": true, "reliability": 1, + "severity": 3, "description": "A webshell of type PST was confirmed at /test/webshell.php" } } diff --git a/tests/data/example_data_1_replace_test.json b/tests/data/example_data_1_replace_test.json index f56d82c..4d07181 100644 --- a/tests/data/example_data_1_replace_test.json +++ b/tests/data/example_data_1_replace_test.json @@ -1,5 +1,6 @@ { "document_version": 2, + "tags": ["dev", "hide"], "ip": "192.0.2.10", "port": 444, "whois_description": "SOMENET", @@ -9,7 +10,7 @@ "abuse_mail": "abuse@test.soc.sunet.se", "domain": "sunet.se", "timestamp": "2021-06-21T14:06:00Z", - "display_name": "Apache 2.1.3", + "display_name": "Apache 2.1.4", "description": "The Apache HTTP Server is a free and open-source cross-platform web server software, released under the terms of Apache License 2.0.", "custom_data": { "subject_cn": { @@ -35,7 +36,9 @@ "cve_2015_0060": { "display_name": "CVE-2015-0060", "vulnerable": true, - "reliability": 2 + "reliability": 2, + "severity": 3, + "description": "Allows local users to cause a denial of service (system hang) via a crafted application, aka Windows Font Driver Denial of Service Vulnerability." }, "cve_2015_0063": { "display_name": "CVE-2015-0063", @@ -45,12 +48,14 @@ "display_name": "Insecure cryptography", "vulnerable": true, "reliability": 5, + "severity": 2, "description": "Uses RSA instead of elliptic curve." }, "possible_webshell": { "display_name": "Webshells (PST)", "investigation_needed": true, "reliability": 1, + "severity": 1, "description": "A webshell of type PST was confirmed at /test/webshell.php" } } diff --git a/tests/data/example_data_3.json b/tests/data/example_data_3.json index 44d483b..897f9d3 100644 --- a/tests/data/example_data_3.json +++ b/tests/data/example_data_3.json @@ -1,5 +1,6 @@ { - "document_version": 1, + "document_version": 3, + "tags": ["notify", "display"], "ip": "192.0.2.28", "port": 111, "whois_description": "SOMENET", @@ -36,6 +37,7 @@ "display_name": "CVE-2015-0003", "vulnerable": true, "reliability": 2, + "severity": 4, "description": "A carefully crafted request body can cause a read to a random memory area which could cause the process to crash." }, "cve_2015_0004": { @@ -45,7 +47,9 @@ "cve_2015_0005": { "display_name": "CVE-2015-0005", "vulnerable": true, - "reliability": 4 + "severity": 4, + "reliability": 4, + "description": "Allows local users to cause a denial of service (system hang) via a crafted application, aka Windows Font Driver Denial of Service Vulnerability." } } } diff --git a/tests/data/example_data_3_replace_test.json b/tests/data/example_data_3_replace_test.json index 31cc64d..5f09020 100644 --- a/tests/data/example_data_3_replace_test.json +++ b/tests/data/example_data_3_replace_test.json @@ -1,6 +1,7 @@ { "_id": "6370498050845fac09e0fc01", "document_version": 2, + "tags": ["dev"], "ip": "192.0.2.28", "port": 112, "whois_description": "SOMENET", @@ -10,7 +11,7 @@ "abuse_mail": "abuse@test.soc.sunet.se", "domain": "sunet.se", "timestamp": "2021-06-30T15:00:00Z", - "display_name": "VMware ESXi 6.7.0 build-17700523", + "display_name": "VMware ESXi 6.7.1 build-17700523", "description": "VMware ESXi is an enterprise-class, type-1 hypervisor developed by VMware for deploying and serving virtual computers. As a type-1 hypervisor, ESXi is not a software application that is installed on an operating system; instead, it includes and integrates vital OS components, such as a kernel.", "custom_data": { "subject_cn": { @@ -37,6 +38,7 @@ "display_name": "CVE-2015-0003", "vulnerable": true, "reliability": 2, + "severity": 4, "description": "A carefully crafted request body can cause a read to a random memory area which could cause the process to crash." }, "cve_2015_0004": { @@ -46,7 +48,9 @@ "cve_2015_0005": { "display_name": "CVE-2015-0005", "vulnerable": true, - "reliability": 4 + "severity": 5, + "reliability": 4, + "description": "Allows local users to cause a denial of service (system hang) via a crafted application, aka Windows Font Driver Denial of Service Vulnerability." } } } |