description: Pipeline for parsing MISP Threat Intel
processors:
  ####################
  # Event ECS fields #
  ####################
  - set:
      field: event.ingested
      value: "{{_ingest.timestamp}}"
  - set:
      field: event.kind
      value: enrichment
  - set:
      field: event.category
      value: threat
  - set:
      field: event.type
      value: indicator

  ######################
  # General ECS fields #
  ######################
  - rename:
      field: json.Event
      target_field: threatintel.misp
      ignore_missing: true
  - set:
      field: threatintel.indicator.provider
      value: misp
      if: ctx?.threatintel?.misp?.Orgc?.local != 'false'
  - set:
      field: threatintel.indicator.provider
      value: "{{misp.Orgc.name}}"
      if: ctx?.threatintel?.misp?.Orgc?.local == 'false'
      ignore_empty_value: true

  # Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event
  - remove:
      field:
        - threatintel.misp.ShadowAttribute
        - threatintel.misp.RelatedEvent
        - threatintel.misp.Galaxy
        - threatintel.misp.Attribute.Galaxy
        - threatintel.misp.Attribute.ShadowAttribute
        - threatintel.misp.Object
        - threatintel.misp.EventReport
        - message
      ignore_missing: true
  - date:
      field: threatintel.misp.timestamp
      formats:
        - UNIX
      ignore_failure: true
  - rename:
      field: threatintel.misp.Attribute
      target_field: threatintel.misp.attribute
      ignore_missing: true
  - rename:
      field: threatintel.misp.Orgc
      target_field: threatintel.misp.orgc
      ignore_missing: true
  - rename:
      field: threatintel.misp.Org
      target_field: threatintel.misp.org
      ignore_missing: true
  - rename:
      field: threatintel.misp.Attribute
      target_field: threatintel.misp.attribute
      ignore_missing: true
  - rename:
      field: threatintel.misp.Tag
      target_field: threatintel.misp.tag
      ignore_missing: true

  #####################
  # Threat ECS Fields #
  #####################
  - rename:
      field: threatintel.misp.attribute.first_seen
      target_field: threatintel.indicator.first_seen
      ignore_missing: true
  - rename:
      field: threatintel.misp.attribute.last_seen
      target_field: threatintel.indicator.last_seen
      ignore_missing: true
  - convert:
      field: threatintel.misp.analysis
      type: long
      target_field: threatintel.indicator.scanner_stats
      ignore_missing: true
  - convert:
      field: threatintel.misp.threat_level_id
      type: long
      ignore_missing: true

  ## File/Hash indicator operations
  - set:
      field: threatintel.indicator.type
      value: file
      if: "['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.threatintel?.misp?.attribute?.type) || ctx.threatintel?.misp?.attribute?.type.startsWith('filename')"
  - rename:
      field: threatintel.misp.attribute.value
      target_field: "threatintel.indicator.file.hash.{{threatintel.misp.attribute.type}}"
      ignore_missing: true
      if: "ctx?.threatintel?.indicator?.type == 'file' && ctx?.threatintel?.misp?.attribute?.type != null && !ctx?.threatintel?.misp?.attribute?.type.startsWith('filename')"
  - rename:
      field: threatintel.misp.attribute.value
      target_field: threatintel.indicator.file.name
      ignore_missing: true
      if: "ctx?.threatintel?.indicator?.type == 'file' && ctx?.threatintel?.misp?.attribute?.type == 'filename'"
  - grok:
      field: threatintel.misp.attribute.type
      patterns:
        - "%{WORD}\\|%{WORD:_tmp.hashtype}"
      ignore_missing: true
      if: ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|')
  - grok:
      field: threatintel.misp.attribute.value
      patterns:
        - "%{DATA:threatintel.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}"
      ignore_missing: true
      if: ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|')
  - set:
      field: threatintel.indicator.file.hash.{{_tmp.hashtype}}
      value: "{{_tmp.hashvalue}}"
      if: "ctx?.threatintel?.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null"

  ## URL/URI indicator operations
  - set:
      field: threatintel.indicator.type
      value: url
      if: "ctx?.threatintel?.misp?.attribute?.type != null && ['url', 'link', 'uri'].contains(ctx?.threatintel?.misp?.attribute?.type)"
  - uri_parts:
      field: threatintel.misp.attribute.value
      target_field: threatintel.indicator.url
      keep_original: true
      remove_if_successful: true
      if: ctx?.threatintel?.indicator?.type == 'url' && ctx?.threatintel?.misp?.attribute?.type != 'uri'

  - set:
      field: threatintel.indicator.url.full
      value: "{{{threatintel.indicator.url.original}}}"
      ignore_empty_value: true
      if: "ctx?.threatintel?.indicator?.type == 'url' && ctx?.threatintel?.misp?.attribute?.type != 'uri'"

  ## Regkey indicator operations
  - set:
      field: threatintel.indicator.type
      value: windows-registry-key
      if: "ctx?.threatintel?.misp?.attribute?.type != null && ctx?.threatintel?.misp?.attribute?.type.startsWith('regkey')"
  - rename:
      field: threatintel.misp.attribute.value
      target_field: threatintel.indicator.registry.key
      ignore_missing: true
      if: "ctx?.threatintel?.indicator?.type == 'windows-registry-key' && ctx?.threatintel?.misp?.attribute?.type == 'regkey'"
  - grok:
      field: threatintel.misp.attribute.value
      patterns:
        - "%{DATA:threatintel.indicator.registry.key}\\|%{DATA:threatintel.indicator.registry.value}"
      ignore_missing: true
      if: "ctx?.threatintel?.misp?.attribute?.type == 'regkey|value'"

  ## AS indicator operations
  - set:
      field: threatintel.indicator.type
      value: autonomous-system
      if: "ctx?.threatintel?.misp?.attribute?.type != null && ctx?.threatintel?.misp?.attribute?.type == 'AS'"
  - convert:
      field: threatintel.misp.attribute.value
      type: long
      target_field: threatintel.indicator.as.number
      ignore_missing: true
      if: ctx?.threatintel?.indicator?.type == 'autonomous-system'

  ## Domain/IP/Port indicator operations
  - set:
      field: threatintel.indicator.type
      value: domain-name
      if: "ctx?.threatintel?.misp?.attribute?.type != null && (ctx?.threatintel?.misp?.attribute?.type == 'hostname' || ctx?.threatintel?.misp?.attribute?.type.startsWith('domain'))"
  - set:
      field: threatintel.indicator.type
      value: ipv4-addr
      if: "ctx?.threatintel?.misp?.attribute?.type != null && ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx?.threatintel?.misp?.attribute?.type)"
  - rename:
      field: threatintel.misp.attribute.value
      target_field: threatintel.indicator.url.domain
      ignore_missing: true
      if: "ctx?.threatintel?.indicator?.type == 'domain-name' && ctx?.threatintel?.misp?.attribute?.type != 'domain|ip' && ctx.threatintel?.indicator?.url?.domain == null"
  - rename:
      field: threatintel.misp.attribute.value
      target_field: threatintel.indicator.ip
      ignore_missing: true
      if: "ctx?.threatintel?.indicator?.type == 'ipv4-addr' && ctx?.threatintel?.misp?.attribute?.type != 'domain|ip' && !['ip-src|port', 'ip-dst|port'].contains(ctx?.threatintel?.misp?.attribute?.type)"
  - grok:
      field: threatintel.misp.attribute.value
      patterns:
        - "%{DATA:threatintel.indicator.url.domain}\\|%{IP:threatintel.indicator.ip}"
      ignore_missing: true
      if: ctx.threatintel?.misp?.attribute?.type == 'domain|ip' && ctx.threatintel?.indicator?.url?.domain == null
  - grok:
      field: threatintel.misp.attribute.value
      patterns:
        - "%{IP:threatintel.indicator.ip}\\|%{NUMBER:threatintel.indicator.port}"
      ignore_missing: true
      if: "['ip-src|port', 'ip-dst|port'].contains(ctx.threatintel?.misp?.attribute?.type)"

  ## Email indicator operations
  # Currently this ignores email-message, except setting the type it will leave the rest of the fields under misp.
  - set:
      field: threatintel.indicator.type
      value: email-addr
      if: "ctx?.threatintel?.misp?.attribute?.type != null && ['email-dst', 'email-src'].contains(ctx.threatintel?.misp?.attribute?.type)"
  - set:
      field: threatintel.indicator.type
      value: email-message
      if: "ctx?.threatintel?.misp?.attribute?.type != null && ctx.threatintel?.misp?.attribute?.type.startsWith('email') && !['email-dst', 'email-src'].contains(ctx.threatintel?.misp?.attribute?.type)"
  - rename:
      field: threatintel.misp.attribute.value
      target_field: threatintel.indicator.email.address
      ignore_missing: true
      if: ctx?.threatintel?.indicator?.type == 'email-addr'

  ## MAC Address indicator operations
  - set:
      field: threatintel.indicator.type
      value: mac-addr
      if: "ctx?.threatintel?.misp?.attribute?.type != null && ['mac-address', 'mac-eui-64'].contains(ctx.threatintel?.misp?.attribute?.type)"
  - rename:
      field: threatintel.misp.attribute.value
      target_field: threatintel.indicator.mac
      ignore_missing: true
      if: ctx?.threatintel?.indicator?.type == 'mac-addr'

  ###################
  # Tags ECS fields #
  ###################
  # Stripping special characters from tags
  - script:
      lang: painless
      if: ctx?.threatintel?.misp?.tag != null
      source: |
        def tags = ctx.threatintel.misp.tag.stream()
           .map(t -> t.name.replace('\\', '').replace('"', ''))
           .collect(Collectors.toList());
        def tlpTags = tags.stream()
           .filter(t -> t.startsWith('tlp:'))
           .map(t -> t.replace('tlp:', ''))
           .collect(Collectors.toList());

        ctx.tags = tags;
        ctx.threatintel.indicator.marking = [ 'tlp': tlpTags ];

  # Setting indicator type to unknown if it does not match anything
  - set:
      field: threatintel.indicator.type
      value: unknown
      if: ctx?.threatintel?.indicator?.type == null

  ######################
  # Cleanup processors #
  ######################
  - script:
      lang: painless
      if: ctx?.threatintel != null
      source: |
        void handleMap(Map map) {
          for (def x : map.values()) {
            if (x instanceof Map) {
                handleMap(x);
            } else if (x instanceof List) {
                handleList(x);
            }
          }
        map.values().removeIf(v -> v == null);
        }
        void handleList(List list) {
          for (def x : list) {
              if (x instanceof Map) {
                  handleMap(x);
              } else if (x instanceof List) {
                  handleList(x);
              }
          }
        }
        handleMap(ctx);
  # Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event
  - remove:
      field:
        - threatintel.misp.attribute.value
      ignore_missing: true
      if: ctx?.threatintel?.indicator?.type != 'unknown'
  - remove:
      field:
        - threatintel.misp.Attribute.timestamp
        - threatintel.misp.timestamp
        - threatintel.misp.tag
        - threatintel.misp.org
        - threatintel.misp.analysis
        - _tmp
        - json
      ignore_missing: true

on_failure:
  - set:
      field: error.message
      value: "{{ _ingest.on_failure_message }}"
