description: Pipeline for parsing Recorded Future threat intel.
processors:
  #
  # Safeguard against feeding the pipeline with documents other
  # that the ones generated by Filebeat's httpjson input.
  #
  - fail:
      if: "ctx.json == null || !(ctx.json instanceof Map)"
      message: "missing json object in input document"

  #
  # Set basic ECS fields.
  #
  - set:
      field: event.ingested
      value: "{{{ _ingest.timestamp }}}"
  - set:
      field: event.kind
      value: enrichment
  - set:
      field: event.category
      value: threat
  - set:
      field: event.type
      value: indicator

  #
  # Map itype field to STIX 2.0 Cyber Observable values (threatintel.indicator.type).
  #
  - script:
      lang: painless
      if: "ctx.json.entity?.type != null"
      description: >
        Map entity.type field to STIX 2.0 Cyber Observable values (threatintel.indicator.type).
      params:
        IpAddress: ipv4-addr
        InternetDomainName: domain-name
        Hash: file
        URL: url
      source: >
        String mapping = params[ctx.json.entity.type];
        if (mapping != null) {
           ctx["threatintel_indicator_type"] = mapping;
        }
      on_failure:
        - append:
            field: error.message
            value: 'Unable to determine indicator type from "{{{ json.entity.type }}}": {{{ _ingest.on_failure_message }}}'

  - rename:
      field: threatintel_indicator_type
      target_field: threatintel.indicator.type
      ignore_missing: true

  #
  # Detect ipv6 for ipv4-addr types.
  #
  - set:
      field: threatintel.indicator.type
      value: ipv6-addr
      if: 'ctx.threatintel?.indicator?.type == "ipv4-addr" && ctx.json.entity.name != null && ctx.json.entity.name.contains(":")'

  #
  # Map first and last seen dates.
  #
  - date:
      field: json.timestamps.firstSeen
      target_field: threatintel.indicator.first_seen
      formats:
        - ISO8601
      if: "ctx.json.timestamps?.firstSeen != null"
      on_failure:
        - append:
            field: error.message
            value: 'Error parsing firstSeen field value "{{{ json.timestamps.firstSeen }}}": {{{ _ingest.on_failure_message }}}'
  - date:
      field: json.timestamps.lastSeen
      target_field: threatintel.indicator.last_seen
      formats:
        - ISO8601
      if: "ctx.json.timestamps?.lastSeen != null"
      on_failure:
        - append:
            field: error.message
            value: 'Error parsing lastSeen field value "{{{ json.timestamps.lastSeen }}}": {{{ _ingest.on_failure_message }}}'

  #
  # Map location fields.
  #
  - rename:
      field: json.location.location.city
      target_field: threatintel.indicator.geo.city_name
      ignore_missing: true
  - rename:
      field: json.location.location.continent
      target_field: threatintel.indicator.geo.continent_name
      ignore_missing: true
  - rename:
      field: json.location.location.country
      target_field: threatintel.indicator.geo.country_name
      ignore_missing: true
  - grok:
      field: json.location.asn
      patterns:
        - "^(?:[Aa][Ss])?%{NUMBER:threatintel.indicator.as.number:long}$"
      ignore_missing: true
      on_failure:
        - append:
            field: error.message
            value: "Cannot parse asn field `{{{ json.location.asn }}}`: {{{ _ingest.on_failure_message }}}"
  - rename:
      field: json.location.organization
      target_field: threatintel.indicator.as.organization.name
      ignore_missing: true
  - set:
      field: threatintel.indicator.reference
      value: "{{{ json.intelCard }}}"
      ignore_empty_value: true
  - set:
      field: json.ip_range
      value: "{{{json.entity.name}}}"
      if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && ctx.json.entity.name.contains("/")'
  - set:
      field: json.ip_range
      value: "{{{ json.entity.name }}}/32"
      if: 'ctx.threatintel?.indicator?.type == "ipv4-addr" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")'
  - set:
      field: json.ip_range
      value: "{{{ json.entity.name }}}/128"
      if: 'ctx.threatintel?.indicator?.type == "ipv6-addr" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")'
  - set:
      field: json.ip_range
      value: "{{{json.entity.name}}}"
      if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && ctx.json.entity.name.contains("/")'
  - rename:
      field: json.entity.name
      target_field: threatintel.indicator.ip
      if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")'
  - rename:
      field: json.entity.name
      target_field: threatintel.indicator.url.domain
      ignore_missing: true
      if: 'ctx.threatintel?.indicator?.type == "domain-name" && ctx.threatintel?.indicator?.url?.domain == null'
  - uri_parts:
      field: json.entity.name
      target_field: threatintel.indicator.url
      keep_original: true
      remove_if_successful: true
      if: 'ctx.threatintel?.indicator?.type == "url"'
      on_failure:
        - append:
            field: error.message
            value: "Cannot parse url field `{{{ json.entity.name }}}`: {{{ _ingest.on_failure_message }}}"

  # At this point fileHashes may exist if "fileHashes" field is requested.
  - append:
      field: json.fileHashes
      value: "{{{ json.entity.name }}}"
      allow_duplicates: false
      if: 'ctx.threatintel?.indicator?.type == "file"'
  - remove:
      field: json.entity.name
      if: 'ctx.threatintel?.indicator?.type == "file"'
  - script:
      lang: painless
      description: >
        Map file hashes.
      if: "ctx.json.fileHashes != null"
      params:
        "4": crc32
        "32": md5
        "40": sha1
        "64": sha256
        "128": sha512
      source: >
        def hashes = new HashMap();
        for (def hash : ctx.json.fileHashes) {
          def algo = params[String.valueOf(hash.length())];
          if (algo != null) {
            hashes[algo] = hash;
          }
        }
        ctx["_hashes"] = hashes;
      on_failure:
        - append:
            field: error.message
            value: "Failed to map fileHashes field: {{ _ingest.on_failure_message }}"
  - rename:
      field: _hashes
      target_field: threatintel.indicator.file.hash
      ignore_missing: true

  #
  # Map risk.score to event.risk_score.
  #
  - convert:
      field: json.risk.score
      target_field: event.risk_score
      ignore_missing: true
      type: float
      on_failure:
        - append:
            field: error.message
            value: "Risk score `{{{ json.risk.score }}}` cannot be converted to float: {{ _ingest.on_failure_message }}"
  #
  # Remove fields converted to an ECS field.
  #
  - remove:
      field:
        - json.timestamps
        - json.location
        - json.fileHashes
        - message
      ignore_missing: true
  #
  # Save fields without an ECS mapping under `threatintel.recordedfuture`.
  #
  - rename:
      field: json
      target_field: threatintel.recordedfuture
on_failure:
  - append:
      field: error.message
      value: "{{ _ingest.on_failure_message }}"
