Go, OpenTelemetry で AWS にログ・トレースを計装してみる

package main

import (
	"context"
	"log"
	"net/http"
	"os"

	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
	"go.opentelemetry.io/otel"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
	"go.opentelemetry.io/otel/propagation"
	"go.opentelemetry.io/otel/sdk/resource"
	"go.opentelemetry.io/otel/sdk/trace"
	semconv "go.opentelemetry.io/otel/semconv/v1.10.0"
)

func initTracer() {
        ctx := context.Background()

        collectorAddress := os.Getenv("OTEL_COLLECTOR_ADDRESS")
        if collectorAddress == "" {
                collectorAddress = "otel-collector:4317"
        }

        // gRPCを使用してOpenTelemetry Collectorと通信するクライアントを作成します
        traceClient := otlptracegrpc.NewClient(
                otlptracegrpc.WithEndpoint(collectorAddress),
                otlptracegrpc.WithInsecure(),
        )

        // 作成したクライアントを使用して
        // トレースデータをエクスポートするエクスポーターを作成します
        traceExporter, err := otlptrace.New(ctx, traceClient)
        if err != nil {
                log.Fatalf("Failed to create trace exporter: %v", err)
        }

        // 新しいTracerProviderを作成します
        // これはトレースデータを生成するTracerを作成するためのものです
        tp := trace.NewTracerProvider(
                trace.WithBatcher(traceExporter),
                trace.WithResource(newResource()),
        )

        // 作成したTracerProviderをグローバルなTracerProviderとして設定します
        otel.SetTracerProvider(tp)
        // TraceContextを使用したPropagatorを設定します。
        // これはトレースコンテキストを伝播させるためのものです。
        otel.SetTextMapPropagator(propagation.TraceContext{})
}


func main() {
	initTracer()

    // http ハンドラ
	http.Handle("/", otelhttp.NewHandler(http.HandlerFunc(home), "Home"))

	log.Println("Server is running on port 8080...")
	log.Fatal(http.ListenAndServe(":8080", nil))
}

func home(w http.ResponseWriter, r *http.Request) {
    // Span の設定
	ctx := r.Context()
	tr := otel.GetTracerProvider().Tracer("example")
	_, span := tr.Start(ctx, "about")
	defer span.End()

    // Otel 形式の ID を X-Ray 形式に変換
	traceID := IdOtel2Xray(span.SpanContext().TraceID().String())

	log.Printf("TraceID: %s, About us page", traceID)
	w.Write([]byte("Welcome to the Home Page!"))
}

// OpenTelemetry Exporter が初期化の際に毎回用いられる OpenTelemetry リソース作成
// の際にリソース属性として Cloudwatch Log Group を指定
func newResource() *resource.Resource {
	LogGroupNames := []string{"otel-test"}
	return resource.NewWithAttributes(
		semconv.SchemaURL,
		semconv.AWSLogGroupNamesKey.StringSlice(LogGroupNames),
	)
}

func IdOtel2Xray(OtelId string) string {
	xrayId := "1-" + OtelId[0:8] + "-" + OtelId[8:]
	return xrayId
}

version: '3.8'

services:
  go-app:
    build: .
    ports:
      - "8080:8080"
    environment:
      - OTEL_COLLECTOR_ADDRESS=otel-collector:4317
    depends_on:
      - otel-collector

  otel-collector:
    image: public.ecr.aws/aws-observability/aws-otel-collector:latest
    command: ["--config=/etc/otel-agent-config.yaml"]
    volumes:
      - ./otel-agent-config.yaml:/etc/otel-agent-config.yaml
    environment:
      - AWS_ACCESS_KEY_ID=****
      - AWS_SECRET_ACCESS_KEY=*****
      - AWS_REGION=ap-northeast-1
    ports:
      - "4317:4317"

  fluent-bit:
    image: fluent/fluent-bit:latest
    volumes:
      - /var/lib/docker/containers:/var/lib/docker/containers
      - /var/run/docker.sock:/var/run/docker.sock
      - ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
    environment:
      - AWS_ACCESS_KEY_ID=****
      - AWS_SECRET_ACCESS_KEY=****
      - AWS_REGION=ap-northeast-1

FROM golang:1.21 AS builder

WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download

COPY . .
RUN CGO_ENABLED=0 go build -o myapp .

FROM alpine:latest

WORKDIR /root/
COPY --from=builder /app/myapp .

CMD ["./myapp"]

[SERVICE]
    Flush        1
    Daemon       Off
    Log_Level    info
    Parsers_File parsers.conf

[INPUT]
    Name              tail
    Tag               docker.*
    Path              /var/lib/docker/containers/*/*.log
    Parser            docker
    DB                /var/log/flb_docker.db
    Mem_Buf_Limit     5MB
    Skip_Long_Lines   On
    Refresh_Interval  10

[OUTPUT]
    Name              cloudwatch_logs
    Match             *
    region            ap-northeast-1
    log_group_name    otel-test
    log_stream_prefix go-app-
    auto_create_group On

receivers:
  otlp:
    protocols:
      grpc:
      http:

processors:
  batch:

exporters:
  awsxray:
    region: 'ap-northeast-1'

service:
  telemetry:
    logs:
      level: "DEBUG"
  pipelines:
    traces:
      receivers:
        - otlp
      exporters:
        - awsxray

func home(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	tr := otel.GetTracerProvider().Tracer("example")
	_, span := tr.Start(ctx, "about")
	defer span.End()

	traceID := IdOtel2Xray(span.SpanContext().TraceID().String())

	log.Printf("TraceID: %s, About us page", traceID)
	w.Write([]byte("Welcome to the Home Page!"))
}