{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Usage of Processors with Event Objects\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following example demonstrates the delivery of events to the opensearch output connector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event before processing: {\n",
      "  \"id\": \"912b6720-53e1-4b33-bdc5-9c5f404491ee\",\n",
      "  \"@timestamp\": \"2025-07-28 13:14:36.035533+00:00\",\n",
      "  \"user\": {\n",
      "    \"name\": \"Hubert K. Kabal\",\n",
      "    \"email\": \"kabal@example.com\",\n",
      "    \"id\": 12345\n",
      "  }\n",
      "}\n",
      "DEBUG:Processor:GenericAdder (my generic adder) loaded 1 rules\n",
      "DEBUG:Processor:GenericAdder (my generic adder) processing event LogEvent(data={'id': '912b6720-53e1-4b33-bdc5-9c5f404491ee', '@timestamp': '2025-07-28 13:14:36.035533+00:00', 'user': {'name': 'Hubert K. Kabal', 'email': 'kabal@example.com', 'id': 12345}}, state=received)\n",
      "Event after processing: {\n",
      "  \"id\": \"912b6720-53e1-4b33-bdc5-9c5f404491ee\",\n",
      "  \"@timestamp\": \"2025-07-28 13:14:36.035533+00:00\",\n",
      "  \"user\": {\n",
      "    \"name\": \"Hubert K. Kabal\",\n",
      "    \"email\": \"kabal@example.com\",\n",
      "    \"id\": 12345\n",
      "  },\n",
      "  \"event\": {\n",
      "    \"tags\": \"generic added tag\"\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import uuid\n",
    "from logprep.factory import Factory\n",
    "from logprep.util.time import TimeParser\n",
    "from logprep.ng.event.log_event import LogEvent\n",
    "from logprep.ng.event.event_state import EventStateType\n",
    "import logging\n",
    "import sys\n",
    "\n",
    "# Configure logging\n",
    "logging.basicConfig(level=logging.DEBUG, stream=sys.stdout)\n",
    "\n",
    "document = {\n",
    "    \"id\": f\"{uuid.uuid4()}\",\n",
    "    \"@timestamp\": str(TimeParser.now()),\n",
    "    \"user\": {\n",
    "        \"name\": \"Hubert K. Kabal\",\n",
    "        \"email\": \"kabal@example.com\",\n",
    "        \"id\": 12345,\n",
    "    },\n",
    "}\n",
    "\n",
    "event = LogEvent(document, original=b\"\", state=EventStateType.RECEIVED)\n",
    "\n",
    "print(f\"Event before processing: {json.dumps(event.data, indent=2)}\")\n",
    "\n",
    "# adding a custom field to the event\n",
    "config = {\n",
    "    \"my generic adder\": {\n",
    "        \"type\": \"ng_generic_adder\",\n",
    "        \"rules\": [{\"filter\": \"*\", \"generic_adder\": {\"add\": {\"event.tags\": \"generic added tag\"}}}],\n",
    "    }\n",
    "}\n",
    "processor = Factory.create(config)\n",
    "processor.setup()\n",
    "processor.process(event)\n",
    "print(f\"Event after processing: {json.dumps(event.data, indent=2)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event before processing: {\n",
      "  \"id\": \"912b6720-53e1-4b33-bdc5-9c5f404491ee\",\n",
      "  \"@timestamp\": \"2025-07-28 13:14:36.035533+00:00\",\n",
      "  \"user\": {\n",
      "    \"name\": \"Hubert K. Kabal\",\n",
      "    \"email\": \"kabal@example.com\",\n",
      "    \"id\": 12345\n",
      "  },\n",
      "  \"event\": {\n",
      "    \"tags\": \"generic added tag\"\n",
      "  }\n",
      "}\n",
      "DEBUG:Processor:PreDetector (almighty pre_detector) loaded 1 rules\n",
      "DEBUG:Processor:PreDetector (almighty pre_detector) processing event LogEvent(data={'id': '912b6720-53e1-4b33-bdc5-9c5f404491ee', '@timestamp': '2025-07-28 13:14:36.035533+00:00', 'user': {'name': 'Hubert K. Kabal', 'email': 'kabal@example.com', 'id': 12345}, 'event': {'tags': 'generic added tag'}}, state=received)\n",
      "Event after processing: {\n",
      "  \"id\": \"912b6720-53e1-4b33-bdc5-9c5f404491ee\",\n",
      "  \"@timestamp\": \"2025-07-28 13:14:36.035533+00:00\",\n",
      "  \"user\": {\n",
      "    \"name\": \"Hubert K. Kabal\",\n",
      "    \"email\": \"kabal@example.com\",\n",
      "    \"id\": 12345\n",
      "  },\n",
      "  \"event\": {\n",
      "    \"tags\": \"generic added tag\"\n",
      "  },\n",
      "  \"pre_detection_id\": \"1f686a6f-7f61-46bf-9b60-0481e97521e0\"\n",
      "}\n",
      "len(event.extra_data)=1\n",
      "Event extra data: {\n",
      "  \"description\": \"\",\n",
      "  \"id\": \"RULE_ONE_ID\",\n",
      "  \"title\": \"Rule one\",\n",
      "  \"severity\": \"critical\",\n",
      "  \"mitre\": [\n",
      "    \"attack.something1\",\n",
      "    \"attack.something2\"\n",
      "  ],\n",
      "  \"case_condition\": \"directly\",\n",
      "  \"rule_filter\": \"user.id:\\\"12345\\\"\",\n",
      "  \"pre_detection_id\": \"1f686a6f-7f61-46bf-9b60-0481e97521e0\",\n",
      "  \"creation_timestamp\": \"2025-07-28T13:14:40.312029+00:00\",\n",
      "  \"@timestamp\": \"2025-07-28T13:14:36.035533Z\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(f\"Event before processing: {json.dumps(event.data, indent=2)}\")\n",
    "\n",
    "# Predetection example\n",
    "config = {\n",
    "    \"almighty pre_detector\": {\n",
    "        \"type\": \"ng_pre_detector\",\n",
    "        \"outputs\": [\n",
    "            {\"opensearch\": \"pseudonyms\"}\n",
    "        ],\n",
    "        \"rules\": [\n",
    "            {\n",
    "                \"filter\": 'user.id: 12345',\n",
    "                \"pre_detector\": {\n",
    "                    \"case_condition\": \"directly\",\n",
    "                    \"id\": \"RULE_ONE_ID\",\n",
    "                    \"mitre\": [\n",
    "                        \"attack.something1\",\n",
    "                        \"attack.something2\"\n",
    "                    ],\n",
    "                    \"severity\": \"critical\",\n",
    "                    \"title\": \"Rule one\",\n",
    "                    \"description\": \"Some malicious event.\"\n",
    "                }\n",
    "            }\n",
    "        ],\n",
    "    }\n",
    "}\n",
    "processor = Factory.create(config)\n",
    "processor.setup()\n",
    "processor.process(event)\n",
    "print(f\"Event after processing: {json.dumps(event.data, indent=2)}\")\n",
    "print(f\"{len(event.extra_data)=}\")\n",
    "print(f\"Event extra data: {json.dumps(event.extra_data[0].data, indent=2)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Event before processing: {\n",
      "  \"id\": \"912b6720-53e1-4b33-bdc5-9c5f404491ee\",\n",
      "  \"@timestamp\": \"2025-07-28 13:14:36.035533+00:00\",\n",
      "  \"user\": {\n",
      "    \"name\": \"<pseudonym:811e0bc983ec82c3a44469a243b547db259ba89ce3448efd31dc6568042ed9ff>\",\n",
      "    \"email\": \"kabal@example.com\",\n",
      "    \"id\": 12345\n",
      "  },\n",
      "  \"event\": {\n",
      "    \"tags\": \"generic added tag\"\n",
      "  },\n",
      "  \"pre_detection_id\": \"1f686a6f-7f61-46bf-9b60-0481e97521e0\"\n",
      "}\n",
      "DEBUG:Processor:Pseudonymizer (almighty pseudonymizer) loaded 1 rules\n",
      "DEBUG:Component:Checking health of almighty pseudonymizer\n",
      "DEBUG:Processor:Pseudonymizer (almighty pseudonymizer) processing event LogEvent(data={'id': '912b6720-53e1-4b33-bdc5-9c5f404491ee', '@timestamp': '2025-07-28 13:14:36.035533+00:00', 'user': {'name': '<pseudonym:811e0bc983ec82c3a44469a243b547db259ba89ce3448efd31dc6568042ed9ff>', 'email': 'kabal@example.com', 'id': 12345}, 'event': {'tags': 'generic added tag'}, 'pre_detection_id': '1f686a6f-7f61-46bf-9b60-0481e97521e0'}, state=received)\n",
      "Event after processing: {\n",
      "  \"id\": \"912b6720-53e1-4b33-bdc5-9c5f404491ee\",\n",
      "  \"@timestamp\": \"2025-07-28 13:14:36.035533+00:00\",\n",
      "  \"user\": {\n",
      "    \"name\": \"<pseudonym:811e0bc983ec82c3a44469a243b547db259ba89ce3448efd31dc6568042ed9ff>\",\n",
      "    \"email\": \"kabal@example.com\",\n",
      "    \"id\": 12345\n",
      "  },\n",
      "  \"event\": {\n",
      "    \"tags\": \"generic added tag\"\n",
      "  },\n",
      "  \"pre_detection_id\": \"1f686a6f-7f61-46bf-9b60-0481e97521e0\"\n",
      "}\n",
      "len(event.extra_data)=2\n",
      "Event extra data: {\n",
      "  \"pseudonym\": \"811e0bc983ec82c3a44469a243b547db259ba89ce3448efd31dc6568042ed9ff\",\n",
      "  \"origin\": \"DSsdFzcgxCsnirGibno2ixeuNBn5O5uK9f7BQ169oLE7h8q/d9I4TvlzjVl1Ia5lVCtf5BsqzjRp8WQyWFMMcWn2pyqJXf79H1AlGmRKkg2ahuEvAGv1z26q1cConS4kK+1F4pw2e9WmM+fwqWJUzBPjDZCsmIn82hZfQwjwA18=:liEUJfmrju2FcrhUkj34aw==:r4bNr3wl9es5rL4OmNs3HcZWwevlL7cq3jcVafeQVbny13pxMs2GE23OtYDkD0i7SHZjdk6YtceW26v7BeoIJDhatbMvjHNpk3ZJCSAX4LphSKo/KYYiTD3aTifMjKmc7oi2+1FehJbG6nqSh/dnl4vhOa+QLbzd1bx2G4KDqWCQ552/S1ctg2CfjgsVp4iWe1BV/KByiFTGC6banTbsHLGCO3+7qBK14ToSE/ndGAKL36nfL33rIREV7CYOQyCaC4ZN2uzfsPsF72DI5WkcDKcQhrgLOlyoIu9BCqIyuyZhS6yWmnZHZ7ss7JiW6oyH/uA8hE0exn1iTTQlNt/wXQ==:VX/wHIucunc7QMbBiuaK0w==:lHArIpHhjZoNgMwS572f\",\n",
      "  \"@timestamp\": \"2025-07-28 13:14:36.035533+00:00\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(f\"Event before processing: {json.dumps(event.data, indent=2)}\")\n",
    "\n",
    "# Pseudonymization\n",
    "config = {\n",
    "    \"almighty pseudonymizer\": {\n",
    "        \"type\": \"ng_pseudonymizer\",\n",
    "        \"pubkey_analyst\": \"../../../../../examples/exampledata/rules/pseudonymizer/example_analyst_pub.pem\",\n",
    "        \"pubkey_depseudo\": \"../../../../../examples/exampledata/rules/pseudonymizer/example_depseudo_pub.pem\",\n",
    "        \"regex_mapping\": \"../../../../../examples/exampledata/rules/pseudonymizer/regex_mapping.yml\",\n",
    "        \"hash_salt\": \"a_secret_tasty_ingredient\",\n",
    "        \"outputs\": [\n",
    "            {\"opensearch\": \"pseudonyms\"}\n",
    "        ],\n",
    "        \"rules\": [\n",
    "            {\n",
    "                \"filter\": \"*\",\n",
    "                \"pseudonymizer\": {\n",
    "                    \"mapping\": {\n",
    "                        \"user.name\": \"RE_WHOLE_FIELD\",\n",
    "                    }\n",
    "                }\n",
    "            }\n",
    "        ],\n",
    "        \"max_cached_pseudonyms\": 1000000\n",
    "    }\n",
    "}\n",
    "processor = Factory.create(config)\n",
    "processor.setup()\n",
    "processor.process(event)\n",
    "print(f\"Event after processing: {json.dumps(event.data, indent=2)}\")\n",
    "print(f\"{len(event.extra_data)=}\")\n",
    "print(f\"Event extra data: {json.dumps(event.extra_data[1].data, indent=2)}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}