Skip to content

feat: add tracing for sql #1610

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 17 commits into from
May 20, 2022
19 changes: 14 additions & 5 deletions cli/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,13 @@ import (
"github.com/pion/webrtc/v3"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/spf13/cobra"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"golang.org/x/oauth2"
xgithub "golang.org/x/oauth2/github"
"golang.org/x/xerrors"
"google.golang.org/api/idtoken"
"google.golang.org/api/option"

sdktrace "go.opentelemetry.io/otel/sdk/trace"

"cdr.dev/slog"
"cdr.dev/slog/sloggers/sloghuman"
"github.com/coder/coder/cli/cliflag"
Expand Down Expand Up @@ -103,8 +102,11 @@ func server() *cobra.Command {
logger = logger.Leveled(slog.LevelDebug)
}

var tracerProvider *sdktrace.TracerProvider
var err error
var (
tracerProvider *sdktrace.TracerProvider
err error
sqlDriver = "postgres"
)
if trace {
tracerProvider, err = tracing.TracerProvider(cmd.Context(), "coderd")
if err != nil {
Expand All @@ -116,6 +118,13 @@ func server() *cobra.Command {
defer cancel()
_ = tracerProvider.Shutdown(ctx)
}()

d, err := tracing.PostgresDriver(tracerProvider, "coderd.database")
if err != nil {
logger.Warn(cmd.Context(), "failed to start postgres tracing driver", slog.Error(err))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

potential bug: If tracing.PostgresDriver encounters an error here, sqlDriver will be the empty string; I believe that would cause us to fail to connect to the DB. We should probably fall back to the original driver name in this case and degrade gracefully.

Suggested change
logger.Warn(cmd.Context(), "failed to start postgres tracing driver", slog.Error(err))
logger.Warn(cmd.Context(), "failed to start postgres tracing driver", slog.Error(err))
sqlDriver = "postgres"

} else {
sqlDriver = d
}
}
}

Expand Down Expand Up @@ -245,7 +254,7 @@ func server() *cobra.Command {
_, _ = fmt.Fprintln(cmd.ErrOrStderr())

if !dev {
sqlDB, err := sql.Open("postgres", postgresURL)
sqlDB, err := sql.Open(sqlDriver, postgresURL)
if err != nil {
return xerrors.Errorf("dial postgres: %w", err)
}
Expand Down
36 changes: 11 additions & 25 deletions coderd/tracing/httpmw.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,24 +6,23 @@ import (

"github.com/go-chi/chi/middleware"
"github.com/go-chi/chi/v5"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.10.0"
)

// HTTPMW adds tracing to http routes.
func HTTPMW(tracerProvider *sdktrace.TracerProvider, name string) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
// do not trace if exporter has not be initialized
if tracerProvider == nil {
next.ServeHTTP(rw, r)
return
}

// start span with default span name. Span name will be updated to "method route" format once request finishes.
_, span := tracerProvider.Tracer(name).Start(r.Context(), fmt.Sprintf("%s %s", r.Method, r.RequestURI))
ctx, span := tracerProvider.Tracer(name).Start(r.Context(), fmt.Sprintf("%s %s", r.Method, r.RequestURI))
defer span.End()
r = r.WithContext(ctx)

wrw := middleware.NewWrapResponseWriter(rw, r.ProtoMajor)

Expand All @@ -35,34 +34,21 @@ func HTTPMW(tracerProvider *sdktrace.TracerProvider, name string) func(http.Hand
if route != "" {
span.SetName(fmt.Sprintf("%s %s", r.Method, route))
}
span.SetAttributes(attribute.KeyValue{
Key: "http.method",
Value: attribute.StringValue(r.Method),
})
span.SetAttributes(attribute.KeyValue{
Key: "http.route",
Value: attribute.StringValue(route),
})
span.SetAttributes(attribute.KeyValue{
Key: "http.path",
Value: attribute.StringValue(r.URL.EscapedPath()),
})
span.SetName(fmt.Sprintf("%s %s", r.Method, route))
span.SetAttributes(semconv.NetAttributesFromHTTPRequest("tcp", r)...)
span.SetAttributes(semconv.EndUserAttributesFromHTTPRequest(r)...)
span.SetAttributes(semconv.HTTPServerAttributesFromHTTPRequest("", route, r)...)
span.SetAttributes(semconv.HTTPRouteKey.String(route))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice change with the semconv package!


// set the status code
status := wrw.Status()
// 0 status means one has not yet been sent in which case net/http library will write StatusOK
if status == 0 {
status = http.StatusOK
}
span.SetAttributes(attribute.KeyValue{
Key: "http.status_code",
Value: attribute.IntValue(status),
})

// if 5XX we set the span to "error" status
if status >= 500 {
span.SetStatus(codes.Error, fmt.Sprintf("%d: %s", status, http.StatusText(status)))
}
span.SetAttributes(semconv.HTTPStatusCodeKey.Int(status))
spanStatus, spanMessage := semconv.SpanStatusFromHTTPStatusCode(status)
span.SetStatus(spanStatus, spanMessage)
})
}
}
45 changes: 45 additions & 0 deletions coderd/tracing/postgres.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package tracing

import (
"context"
"fmt"
"strings"

"github.com/nhatthm/otelsql"
semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
"go.opentelemetry.io/otel/trace"
"golang.org/x/xerrors"
)

// Postgres driver will register a new tracing sql driver and return the driver name.
func PostgresDriver(tp trace.TracerProvider, service string) (string, error) {
// Register the otelsql wrapper for the provided postgres driver.
driverName, err := otelsql.Register("postgres",
otelsql.WithDefaultAttributes(
semconv.ServiceNameKey.String(service),
),
otelsql.TraceQueryWithoutArgs(),
otelsql.WithSystem(semconv.DBSystemPostgreSQL),
otelsql.WithTracerProvider(tp),
otelsql.WithSpanNameFormatter(formatPostgresSpan),
)
if err != nil {
return "", xerrors.Errorf("registering postgres tracing driver: %w", err)
}

return driverName, nil
}

func formatPostgresSpan(ctx context.Context, op string) string {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we add the sql: prefix (or coder-db:), like in https://pkg.go.dev/github.com/nhatthm/otelsql#readme-span-name-formatter? Wondering if it'd help with narrowing down the spans.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm just copying the format I've seen before with other database tracing tools (New Relic) and also mimicing the format of the http spans. Since an http span is METHOD route, I followed a similar format of OP name so the spans look like QUERY GetAllUsersById.

I don't love the prefix because I already can see where the span is coming from by the instrumentation library (see the screenshots in description), but I also don't know if this is the case on other tools like Jaeger, which I'd like to setup eventually. I'm down to change it if we find it hard to parse as we use it, does that sound ok?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, that sounds perfectly reasonable. 👍🏻

const qPrefix = "-- name: "
q := otelsql.QueryFromContext(ctx)
if q == "" || !strings.HasPrefix(q, qPrefix) {
return strings.ToUpper(op)
}

// Remove the qPrefix and then grab the method name.
// We expect the first line of the query to be in
// the format "-- name: GetAPIKeyByID :one".
s := strings.SplitN(strings.TrimPrefix(q, qPrefix), " ", 2)[0]
return fmt.Sprintf("%s %s", strings.ToUpper(op), s)
}
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,7 @@ require (
github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
github.com/muesli/reflow v0.3.0 // indirect
github.com/muesli/termenv v0.11.1-0.20220212125758-44cd13922739 // indirect
github.com/nhatthm/otelsql v0.3.0
github.com/niklasfasching/go-org v1.6.2 // indirect
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
Expand Down Expand Up @@ -240,8 +241,9 @@ require (
go.opentelemetry.io/otel v1.7.0
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0
go.opentelemetry.io/otel/metric v0.30.0 // indirect
go.opentelemetry.io/otel/sdk v1.7.0
go.opentelemetry.io/otel/trace v1.7.0 // indirect
go.opentelemetry.io/otel/trace v1.7.0
go.opentelemetry.io/proto/otlp v0.16.0 // indirect
golang.org/x/time v0.0.0-20220224211638-0e9765cccd65 // indirect
google.golang.org/appengine v1.6.7 // indirect
Expand Down
13 changes: 13 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI=
github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
Expand Down Expand Up @@ -228,6 +229,7 @@ github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqO
github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
github.com/bool64/shared v0.1.4 h1:zwtb1dl2QzDa9TJOq2jzDTdb5IPf9XlxTGKN8cySWT0=
github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
github.com/bshuster-repo/logrus-logstash-hook v1.0.0/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
Expand Down Expand Up @@ -896,6 +898,7 @@ github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87/go.mod h1:CtWFDAQg
github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/iancoleman/orderedmap v0.2.0 h1:sq1N/TFpYH++aViPcaKjys3bDClUEU7s5B+z6jq8pNA=
github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
Expand Down Expand Up @@ -1190,6 +1193,8 @@ github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+
github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA=
github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM=
github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba/go.mod h1:ncO5VaFWh0Nrt+4KT4mOZboaczBZcLuHrG+/sUeP8gI=
github.com/nhatthm/otelsql v0.3.0 h1:BvqFgk6FkkmlY2KrtSyxILkPJL5oI2Bzny/s7d134N8=
github.com/nhatthm/otelsql v0.3.0/go.mod h1:6OmgQmHfKwLqNQp+nNh5xHOrMl19y8n4v44FLRZWYlQ=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/niklasfasching/go-org v1.6.2 h1:kQBIZlfL4oRNApJCrBgaeNBfzxWzP6XlC7/b744Polk=
github.com/niklasfasching/go-org v1.6.2/go.mod h1:wn76Xgu4/KRe43WZhsgZjxYMaloSrl3BSweGV74SwHs=
Expand Down Expand Up @@ -1490,6 +1495,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/swaggest/assertjson v1.6.8 h1:1O/9UI5M+2OJI7BeEWKGj0wTvpRXZt5FkOJ4nRkY4rA=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
Expand Down Expand Up @@ -1554,6 +1560,8 @@ github.com/xtaci/lossyconn v0.0.0-20200209145036-adba10fffc37/go.mod h1:HpMP7DB2
github.com/yashtewari/glob-intersection v0.1.0 h1:6gJvMYQlTDOL3dMsPF6J0+26vwX9MB8/1q3uAdhmTrg=
github.com/yashtewari/glob-intersection v0.1.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCOA=
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down Expand Up @@ -1615,8 +1623,12 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.6.3/go.mod h1
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0 h1:MFAyzUPrTwLOwCi+cltN0ZVyy4phU41lwH+lyMyQTS4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0/go.mod h1:E+/KKhwOSw8yoPxSSuUHG6vKppkvhN+S1Jc7Nib3k3o=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE=
go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.30.0 h1:2glg1ZFVVZf47zFuX0iwBPPid4tqzBYYWTVVu0pc+us=
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.7.0 h1:8hPcgCg0rUJiKE6VWahRvjgLUrNl7rW2hffUEPKXVEM=
go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
go.opentelemetry.io/otel/metric v0.28.0/go.mod h1:TrzsfQAmQaB1PDcdhBauLMk7nyyg9hm+GoQq/ekE9Iw=
go.opentelemetry.io/otel/metric v0.30.0 h1:Hs8eQZ8aQgs0U49diZoaS6Uaxw3+bBE3lcMUKBFIk3c=
go.opentelemetry.io/otel/metric v0.30.0/go.mod h1:/ShZ7+TS4dHzDFmfi1kSXMhMVubNoP0oIaBp70J6UXU=
go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw=
go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs=
Expand All @@ -1625,6 +1637,7 @@ go.opentelemetry.io/otel/sdk v1.7.0 h1:4OmStpcKVOfvDOgCt7UriAPtKolwIhxpnSNI/yK+1
go.opentelemetry.io/otel/sdk v1.7.0/go.mod h1:uTEOTwaqIVuTGiJN7ii13Ibp75wJmYUDe374q6cZwUU=
go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
go.opentelemetry.io/otel/sdk/metric v0.30.0 h1:XTqQ4y3erR2Oj8xSAOL5ovO5011ch2ELg51z4fVkpME=
go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk=
go.opentelemetry.io/otel/trace v1.6.0/go.mod h1:qs7BrU5cZ8dXQHBGxHMOxwME/27YH2qEp4/+tZLLwJE=
Expand Down