## SOURCES: Data sources that Observability Pipelines Worker collects data from.
## For a Datadog use case, the Worker receives data from the Datadog agent.
sources:
  datadog_agent:
    address: 0.0.0.0:8282
    type: datadog_agent
    multiple_outputs: true

transforms:
  ## The Datadog Agent natively encodes its tags as a comma-separated list
  ## of values that are stored in the string `.ddtags`. To work with
  ## and filter off of these tags, you need to parse that string into
  ## more structured data.
  logs_parse_ddtags:
    type: remap
    inputs:
      - datadog_agent.logs
    source: |
      .ddtags = parse_key_value!(.ddtags, key_value_delimiter: ":", field_delimiter: ",")

  ## The `.status` attribute added by the Datadog Agent needs to be deleted, otherwise
  ## your logs can be miscategorized at intake.
  logs_remove_wrong_level:
    type: remap
    inputs:
      - logs_parse_ddtags
    source: |
      del(.status)

  ## This is a placeholder for your own remap (or other transforms)
  ## steps with tags. Datadog recommends these tag assignments.
  ## They show which data has been moved over to OP and what still needs
  ## to be moved.
  logs_enrich:
    type: remap
    inputs:
      - logs_remove_wrong_level
    source: |
      .ddtags.sender = "observability_pipelines_worker"
      .ddtags.opw_aggregator = get_hostname!()

  ## Before sending data to the logs intake, you must re-encode the
  ## tags into the expected format, so that it appears as if the Agent is
  ## sending it directly.
  logs_finish_ddtags:
    type: remap
    inputs:
      - logs_enrich
    source: |
      .ddtags = encode_key_value!(.ddtags, key_value_delimiter: ":", field_delimiter: ",")

  metrics_add_dd_tags:
    type: remap
    inputs:
      - datadog_agent.metrics
    source: |
      .tags.sender = "observability_pipelines_worker"
      .tags.opw_aggregator = get_hostname!()

## This buffer configuration is split into the following, totaling the 288GB
## provisioned above:
## - 240GB buffer for logs
## - 48GB buffer for metrics
##
## This should work for the vast majority of OP Worker deployments and should rarely
## need to be adjusted.
##
## The "${DD_API_KEY}" parameters are replaced with the API key you provisioned
## earlier.
sinks:
  datadog_logs:
    type: datadog_logs
    inputs:
      - logs_finish_ddtags
    default_api_key: "${DD_API_KEY}"
    site: "${DD_SITE}"
    compression: gzip
    buffer:
       type: disk
       max_size: 257698037760
  datadog_metrics:
    type: datadog_metrics
    inputs:
      - metrics_add_dd_tags
    default_api_key: "${DD_API_KEY}"
    site: "${DD_SITE}"
    buffer:
      type: disk
      max_size: 51539607552
  ## This sink writes logs in a Datadog-rehydratable format to an S3 bucket.
  ## Replace ${DD_ARCHIVES_BUCKET} with the name of the S3 bucket
  ## storing your logs and ${DD_ARCHIVES_REGION} with the AWS region of the target
  ## service.
  datadog_archives:
    type: datadog_archives
    inputs:
      - logs_finish_ddtags
    service: aws_s3
    bucket: ${DD_ARCHIVES_BUCKET}
    aws_s3:
      storage_class: "STANDARD"
      region: "${DD_ARCHIVES_REGION}"
