description: Pipeline for parsing Recorded Future threat intel.
processors:
  #
  # Set basic ECS fields.
  #
  - set:
      field: event.ingested
      value: "{{_ingest.timestamp}}"
  - set:
      field: ecs.version
      value: "1.12"
  - set:
      field: event.kind
      value: enrichment
  - set:
      field: event.category
      value: threat
  - set:
      field: event.type
      value: indicator

  - rename:
      field: message
      target_field: event.original
      ignore_missing: true
  - json:
      field: event.original
      target_field: json

  - set:
      field: threat.feed.name
      value: "[Filebeat] RecordedFuture"
  - set:
      field: threat.feed.dashboard_id
      value: "ad9c7430-72de-11eb-a3e3-b3cc7c78a70f"

  #
  # Map itype field to STIX 2.0 Cyber Observable values (threat.indicator.type).
  #
  - script:
      lang: painless
      if: "ctx.json.entity?.type != null"
      description: >
        Map entity.type field to STIX 2.0 Cyber Observable values (threat.indicator.type).
      params:
        IpAddress: ipv4-addr
        InternetDomainName: domain-name
        Hash: file
        URL: url
      source: >
        String mapping = params[ctx.json.entity.type];
        if (mapping != null) {
           ctx["threat_indicator_type"] = mapping;
        }
      on_failure:
        - append:
            field: error.message
            value: 'Unable to determine indicator type from "{{{ json.entity.type }}}": {{{ _ingest.on_failure_message }}}'

  - rename:
      field: threat_indicator_type
      target_field: threat.indicator.type
      ignore_missing: true

  #
  # Detect ipv6 for ipv4-addr types.
  #
  - set:
      field: threat.indicator.type
      value: ipv6-addr
      if: 'ctx.threat?.indicator?.type == "ipv4-addr" && ctx.json.entity.name != null && ctx.json.entity.name.contains(":")'

  #
  # Map first and last seen dates.
  #
  - date:
      field: json.timestamps.firstSeen
      target_field: threat.indicator.first_seen
      formats:
        - ISO8601
      if: "ctx.json.timestamps?.firstSeen != null"
      on_failure:
        - append:
            field: error.message
            value: 'Error parsing firstSeen field value "{{{ json.timestamps.firstSeen }}}": {{{ _ingest.on_failure_message }}}'
  - date:
      field: json.timestamps.lastSeen
      target_field: threat.indicator.last_seen
      formats:
        - ISO8601
      if: "ctx.json.timestamps?.lastSeen != null"
      on_failure:
        - append:
            field: error.message
            value: 'Error parsing lastSeen field value "{{{ json.timestamps.lastSeen }}}": {{{ _ingest.on_failure_message }}}'

  #
  # Map location fields.
  #
  - rename:
      field: json.location.location.city
      target_field: threat.indicator.geo.city_name
      ignore_missing: true
  - rename:
      field: json.location.location.continent
      target_field: threat.indicator.geo.continent_name
      ignore_missing: true
  - rename:
      field: json.location.location.country
      target_field: threat.indicator.geo.country_name
      ignore_missing: true
  - grok:
      field: json.location.asn
      patterns:
        - "^(?:[Aa][Ss])?%{NUMBER:threat.indicator.as.number:long}$"
      ignore_missing: true
      on_failure:
        - append:
            field: error.message
            value: "Cannot parse asn field `{{{ json.location.asn }}}`: {{{ _ingest.on_failure_message }}}"
  - rename:
      field: json.location.organization
      target_field: threat.indicator.as.organization.name
      ignore_missing: true
  - set:
      field: threat.indicator.reference
      value: "{{{ json.intelCard }}}"
      ignore_empty_value: true
  - set:
      field: json.ip_range
      value: "{{{json.entity.name}}}"
      if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && ctx.json.entity.name.contains("/")'
  - set:
      field: json.ip_range
      value: "{{{ json.entity.name }}}/32"
      if: 'ctx.threat?.indicator?.type == "ipv4-addr" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")'
  - set:
      field: json.ip_range
      value: "{{{ json.entity.name }}}/128"
      if: 'ctx.threat?.indicator?.type == "ipv6-addr" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")'
  - set:
      field: json.ip_range
      value: "{{{json.entity.name}}}"
      if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && ctx.json.entity.name.contains("/")'
  - rename:
      field: json.entity.name
      target_field: threat.indicator.ip
      if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")'
  - rename:
      field: json.entity.name
      target_field: threat.indicator.url.domain
      ignore_missing: true
      if: 'ctx.threat?.indicator?.type == "domain-name" && ctx.threat?.indicator?.url?.domain == null'
  - uri_parts:
      field: json.entity.name
      target_field: threat.indicator.url
      keep_original: true
      remove_if_successful: true
      if: 'ctx.threat?.indicator?.type == "url"'
      on_failure:
        - append:
            field: error.message
            value: "Cannot parse url field `{{{ json.entity.name }}}`: {{{ _ingest.on_failure_message }}}"

  # At this point fileHashes may exist if "fileHashes" field is requested.
  - append:
      field: json.fileHashes
      value: "{{{ json.entity.name }}}"
      allow_duplicates: false
      if: 'ctx.threat?.indicator?.type == "file"'
  - remove:
      field: json.entity.name
      if: 'ctx.threat?.indicator?.type == "file"'
  - script:
      lang: painless
      description: >
        Map file hashes.
      if: "ctx.json.fileHashes != null"
      params:
        "4": crc32
        "32": md5
        "40": sha1
        "64": sha256
        "128": sha512
      source: >
        def hashes = new HashMap();
        for (def hash : ctx.json.fileHashes) {
          def algo = params[String.valueOf(hash.length())];
          if (algo != null) {
            hashes[algo] = hash;
          }
        }
        ctx["_hashes"] = hashes;
      on_failure:
        - append:
            field: error.message
            value: "Failed to map fileHashes field: {{ _ingest.on_failure_message }}"
  - rename:
      field: _hashes
      target_field: threat.indicator.file.hash
      ignore_missing: true

  #
  # Map risk.score to event.risk_score.
  #
  - convert:
      field: json.risk.score
      target_field: event.risk_score
      ignore_missing: true
      type: float
      on_failure:
        - append:
            field: error.message
            value: "Risk score `{{{ json.risk.score }}}` cannot be converted to float: {{ _ingest.on_failure_message }}"
  #
  # Remove fields converted to an ECS field.
  #
  - remove:
      field:
        - json.timestamps
        - json.location
        - json.fileHashes
        - message
      ignore_missing: true
  #
  # Save fields without an ECS mapping under `recordedfuture`.
  #
  - rename:
      field: json
      target_field: recordedfuture

  ######################
  # Cleanup processors #
  ######################
  - remove:
      field: event.original
      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
      ignore_failure: true
      ignore_missing: true
  - set:
      field: threat.indicator.type
      value: unknown
      if: ctx.threat?.indicator?.type == null
  - script:
      lang: painless
      if: ctx.recordedfuture != null
      source: |
        void handleMap(Map map) {
          for (def x : map.values()) {
            if (x instanceof Map) {
                handleMap(x);
            } else if (x instanceof List) {
                handleList(x);
            }
          }
        map.values().removeIf(v -> v == null);
        }
        void handleList(List list) {
          for (def x : list) {
              if (x instanceof Map) {
                  handleMap(x);
              } else if (x instanceof List) {
                  handleList(x);
              }
          }
        }
        handleMap(ctx);
on_failure:
  - append:
      field: error.message
      value: "{{ _ingest.on_failure_message }}"
