Distributed Tracing

    Consul alone cannot implement distributed tracing for your applications. Each application must propagate the required headers. Typically this is done using a tracing library such as:

    Once your applications have been instrumented with a tracing library, you are ready to configure Consul to add sidecar proxy spans to the trace. Your eventual config will look something like:

    HCL

    • HCL
    • Kubernetes YAML
    • JSON
    2. Name = "global"
    3. Config {
    4.   protocol           = "http"
    5.   envoy_tracing_json = <<EOF
    6. {
    7.   "http":{
    8.     "name":"envoy.tracers.zipkin",
    9.     "typedConfig":{
    10.       "@type":"type.googleapis.com/envoy.config.trace.v3.ZipkinConfig",
    11.       "collector_cluster":"collector_cluster_name",
    12.       "collector_endpoint_version":"HTTP_JSON",
    13.       "collector_endpoint":"/api/v2/spans",
    14.       "shared_span_context":false
    15.     }
    16.   }
    17. }
    18. EOF
    20.   envoy_extra_static_clusters_json = <<EOF
    21. {
    22.   "connect_timeout":"3.000s",
    23.   "dns_lookup_family":"V4_ONLY",
    24.   "lb_policy":"ROUND_ROBIN",
    25.   "load_assignment":{
    26.     "cluster_name":"collector_cluster_name",
    27.     "endpoints":[
    28.       {
    29.         "lb_endpoints":[
    30.           {
    31.             "endpoint":{
    32.               "address":{
    33.                 "socket_address":{
    34.                   "address":"collector-url",
    35.                   "port_value":9411,
    36.                   "protocol":"TCP"
    37.                 }
    38.               }
    39.             }
    40.           }
    41.         ]
    42.       }
    43.     ]
    44.   },
    45.   "name":"collector_cluster_name",
    46.   "type":"STRICT_DNS"
    47. }
    48. EOF
    49. }
    1. apiVersion: consul.hashicorp.com/v1alpha1
    2. kind: ProxyDefaults
    3. metadata:
    4.   name: global
    5. spec:
    6.   config:
    7.     protocol: http
    8.     envoy_tracing_json: |
    9.       {
    10.         "http":{
    11.           "name":"envoy.tracers.zipkin",
    12.           "typedConfig":{
    13.             "@type":"type.googleapis.com/envoy.config.trace.v3.ZipkinConfig",
    14.             "collector_cluster":"collector_cluster_name",
    15.             "collector_endpoint_version":"HTTP_JSON",
    16.             "collector_endpoint":"/api/v2/spans",
    17.             "shared_span_context":false
    18.           }
    19.         }
    20.       }
    21.     envoy_extra_static_clusters_json: |
    22.       {
    23.         "connect_timeout":"3.000s",
    24.         "dns_lookup_family":"V4_ONLY",
    25.         "lb_policy":"ROUND_ROBIN",
    26.         "load_assignment":{
    27.           "endpoints":[
    28.             {
    29.               "lb_endpoints":[
    30.                 {
    32.                     "address":{
    33.                       "socket_address":{
    34.                         "address":"collector-url",
    35.                         "port_value":9411,
    36.                         "protocol":"TCP"
    37.                       }
    38.                     }
    39.                   }
    40.                 }
    41.               ]
    42.             }
    43.           ]
    44.         },
    45.         "name":"collector_cluster_name",
    46.         "type":"STRICT_DNS"
    47.       }
    1. apiVersion: consul.hashicorp.com/v1alpha1
    2. kind: ProxyDefaults
    3. metadata:
    4.   name: global
    5. spec:
    6.   config:
    7.     protocol: http
    8.     envoy_tracing_json: |
    9.       {
    10.         "http":{
    11.           "name":"envoy.tracers.zipkin",
    12.           "typedConfig":{
    13.             "@type":"type.googleapis.com/envoy.config.trace.v3.ZipkinConfig",
    14.             "collector_cluster":"collector_cluster_name",
    15.             "collector_endpoint_version":"HTTP_JSON",
    16.             "collector_endpoint":"/api/v2/spans",
    17.             "shared_span_context":false
    18.           }
    19.         }
    20.       }
    21.     envoy_extra_static_clusters_json: |
    22.       {
    23.         "connect_timeout":"3.000s",
    24.         "dns_lookup_family":"V4_ONLY",
    25.         "lb_policy":"ROUND_ROBIN",
    26.         "load_assignment":{
    27.           "cluster_name":"collector_cluster_name",
    28.           "endpoints":[
    29.             {
    30.               "lb_endpoints":[
    31.                 {
    32.                   "endpoint":{
    33.                     "address":{
    34.                       "socket_address":{
    35.                         "address":"collector-url",
    36.                         "port_value":9411,
    37.                         "protocol":"TCP"
    38.                       }
    39.                     }
    40.                   }
    41.                 }
    42.               ]
    43.             }
    44.           ]
    45.         },
    46.         "name":"collector_cluster_name",
    47.         "type":"STRICT_DNS"
    48.       }
    1. {
    2.   "Kind": "ProxyDefaults",
    3.   "Name": "global",
    4.   "Config": {
    5.     "protocol": "http",
    6.     "envoy_tracing_json": "{\"http\":{\"name\":\"envoy.tracers.zipkin\",\"typedConfig\":{\"@type\":\"type.googleapis.com/envoy.config.trace.v3.ZipkinConfig\",\"collector_cluster\":\"collector_cluster_name\",\"collector_endpoint_version\":\"HTTP_JSON\",\"collector_endpoint\":\"/api/v2/spans\",\"shared_span_context\":false}}}",
    7.     "envoy_extra_static_clusters_json": "{\"connect_timeout\":\"3.000s\",\"dns_lookup_family\":\"V4_ONLY\",\"lb_policy\":\"ROUND_ROBIN\",\"load_assignment\":{\"cluster_name\":\"collector_cluster_name\",\"endpoints\":[{\"lb_endpoints\":[{\"endpoint\":{\"address\":{\"socket_address\":{\"address\":\"collector-url\",\"port_value\":9411,\"protocol\":\"TCP\"}}}}]}]},\"name\":\"collector_cluster_name\",\"type\":\"STRICT_DNS\"}"
    8.   }
    9. }
    1. {
    2.   "Kind": "ProxyDefaults",
    3.   "Name": "global",
    4.   "Config": {
    5.     "protocol": "http",
    6.     "envoy_tracing_json": "{\"http\":{\"name\":\"envoy.tracers.zipkin\",\"typedConfig\":{\"@type\":\"type.googleapis.com/envoy.config.trace.v3.ZipkinConfig\",\"collector_cluster\":\"collector_cluster_name\",\"collector_endpoint_version\":\"HTTP_JSON\",\"collector_endpoint\":\"/api/v2/spans\",\"shared_span_context\":false}}}",
    8. }

    NOTE: This example uses a config entry which will apply to all proxies but you can also apply this config in the proxy service registration (not supported on Kubernetes).

    1. : Sets the tracing configuration for your specific tracing type. See the Envoy tracers documentation for your specific collector’s configuration. This configuration will reference the cluster name defined in envoy_extra_static_clusters_json.
    2. : Defines the address of your tracing collector where Envoy will send its spans. In this example the URL was collector-url:9411.

    This configuration only applies when proxies are restarted since it changes the bootstrap config for Envoy which can only be applied on startup. This means you must restart all your proxies for changes to this config to take effect.

    Note: On Kubernetes this is a matter of restarting your deployments, e.g. kubectl rollout restart deploy/deploy-name.

    1. Distributed tracing is only supported for HTTP and gRPC services. You must specify the protocol either globally via a proxy defaults config entry:

      HCL

      Distributed Tracing - 图2

      • HCL
      • Kubernetes YAML
      • JSON
      1. Kind      = "proxy-defaults"
      2. Name      = "global"
      3. Config {
      4.   protocol = "http"
      5. }
      1. apiVersion: consul.hashicorp.com/v1alpha1
      2. kind: ProxyDefaults
      3. metadata:
      4.   name: global
      5. spec:
      6.   config:
      7.     protocol: http
      1. apiVersion: consul.hashicorp.com/v1alpha1
      2. kind: ProxyDefaults
      3. metadata:
      4.   name: global
      5. spec:
      6.   config:
      7.     protocol: http
      1. {
      2.   "Kind": "proxy-defaults",
      3.   "Name": "global",
      4.   "Config": {
      5.     "protocol": "http"
      6.   }
      7. }
      1. {
      2.   "Kind": "proxy-defaults",
      3.   "Name": "global",
      4.   "Config": {
      5.     "protocol": "http"
      6.   }
      7. }

      HCL

      • HCL
      • Kubernetes YAML
      • JSON
      1. Kind      = "service-defaults"
      2. Name      = "service-name"
      3. Protocol  = "http"
      1. apiVersion: consul.hashicorp.com/v1alpha1
      2. kind: ProxyDefaults
      3. metadata:
      4.   name: service-name
      5. spec:
      6.   protocol: http
      1. apiVersion: consul.hashicorp.com/v1alpha1
      2. kind: ProxyDefaults
      3. metadata:
      4.   name: service-name
      5. spec:
      6.   protocol: http
      1. {
      2.   "Kind": "service-defaults",
      3.   "Name": "service-name",
      4.   "Protocol": "http"
      5. }
      1. {
      2.   "Kind": "service-defaults",
      3.   "Name": "service-name",
      4.   "Protocol": "http"
      5. }

    2. Requests through Ingress Gateways will not be traced unless the header x-client-trace-id: 1 is set (see ).

    3. Consul does not currently support interoperation with OpenTelemetry libraries due to Envoy not yet having support.

    4. Tracing is only supported with Envoy proxies, not the built-in proxy.

      1. {
      2.   "http": {
      3.     "name": "envoy.tracers.zipkin",
      4.     "typedConfig": {
      5.       "@type": "type.googleapis.com/envoy.config.trace.v3.ZipkinConfig",
      6.       "collector_cluster": "zipkin",
      7.       "collector_endpoint_version": "HTTP_JSON",
      8.       "collector_endpoint": "/api/v2/spans",
      9.       "shared_span_context": false,
      10.       "trace_id_128bit": true
      11.     }
      12.   }
      13. }