Skip to content

chore: Implement workspace proxy going away (graceful shutdown) #7459

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
May 11, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 30 additions & 2 deletions coderd/apidoc/docs.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

28 changes: 26 additions & 2 deletions coderd/apidoc/swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions codersdk/workspaceproxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,9 @@ import (
type ProxyHealthStatus string

const (
// ProxyReachable means the proxy access url is reachable and returns a healthy
// ProxyHealthy means the proxy access url is reachable and returns a healthy
// status code.
ProxyReachable ProxyHealthStatus = "reachable"
ProxyHealthy ProxyHealthStatus = "ok"
// ProxyUnreachable means the proxy access url is not responding.
ProxyUnreachable ProxyHealthStatus = "unreachable"
// ProxyUnhealthy means the proxy access url is responding, but there is some
Expand Down
10 changes: 5 additions & 5 deletions docs/api/enterprise.md
Original file line number Diff line number Diff line change
Expand Up @@ -1192,7 +1192,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaceproxies \
"errors": ["string"],
"warnings": ["string"]
},
"status": "reachable"
"status": "ok"
},
"updated_at": "2019-08-24T14:15:22Z",
"url": "string",
Expand Down Expand Up @@ -1234,7 +1234,7 @@ Status Code **200**

| Property | Value |
| -------- | -------------- |
| `status` | `reachable` |
| `status` | `ok` |
| `status` | `unreachable` |
| `status` | `unhealthy` |
| `status` | `unregistered` |
Expand Down Expand Up @@ -1289,7 +1289,7 @@ curl -X POST http://coder-server:8080/api/v2/workspaceproxies \
"errors": ["string"],
"warnings": ["string"]
},
"status": "reachable"
"status": "ok"
},
"updated_at": "2019-08-24T14:15:22Z",
"url": "string",
Expand Down Expand Up @@ -1342,7 +1342,7 @@ curl -X GET http://coder-server:8080/api/v2/workspaceproxies/{workspaceproxy} \
"errors": ["string"],
"warnings": ["string"]
},
"status": "reachable"
"status": "ok"
},
"updated_at": "2019-08-24T14:15:22Z",
"url": "string",
Expand Down Expand Up @@ -1453,7 +1453,7 @@ curl -X PATCH http://coder-server:8080/api/v2/workspaceproxies/{workspaceproxy}
"errors": ["string"],
"warnings": ["string"]
},
"status": "reachable"
"status": "ok"
},
"updated_at": "2019-08-24T14:15:22Z",
"url": "string",
Expand Down
8 changes: 4 additions & 4 deletions docs/api/schemas.md
Original file line number Diff line number Diff line change
Expand Up @@ -3476,7 +3476,7 @@ Parameter represents a set value for the scope.
## codersdk.ProxyHealthStatus

```json
"reachable"
"ok"
```

### Properties
Expand All @@ -3485,7 +3485,7 @@ Parameter represents a set value for the scope.

| Value |
| -------------- |
| `reachable` |
| `ok` |
| `unreachable` |
| `unhealthy` |
| `unregistered` |
Expand Down Expand Up @@ -5361,7 +5361,7 @@ Parameter represents a set value for the scope.
"errors": ["string"],
"warnings": ["string"]
},
"status": "reachable"
"status": "ok"
},
"updated_at": "2019-08-24T14:15:22Z",
"url": "string",
Expand Down Expand Up @@ -5393,7 +5393,7 @@ Parameter represents a set value for the scope.
"errors": ["string"],
"warnings": ["string"]
},
"status": "reachable"
"status": "ok"
}
```

Expand Down
1 change: 1 addition & 0 deletions enterprise/cli/proxyserver.go
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,7 @@ func (*RootCmd) proxyServer() *clibase.Cmd {
if err != nil {
return xerrors.Errorf("create workspace proxy: %w", err)
}
closers.Add(func() { _ = proxy.Close() })

shutdownConnsCtx, shutdownConns := context.WithCancel(ctx)
defer shutdownConns()
Expand Down
3 changes: 2 additions & 1 deletion enterprise/coderd/coderd.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ func New(ctx context.Context, options *Options) (*API, error) {
)
r.Post("/issue-signed-app-token", api.workspaceProxyIssueSignedAppToken)
r.Post("/register", api.workspaceProxyRegister)
r.Post("/goingaway", api.workspaceProxyGoingAway)
})
r.Route("/{workspaceproxy}", func(r chi.Router) {
r.Use(
Expand Down Expand Up @@ -239,7 +240,7 @@ func New(ctx context.Context, options *Options) (*API, error) {
if api.AGPL.Experiments.Enabled(codersdk.ExperimentMoons) {
// Proxy health is a moon feature.
api.ProxyHealth, err = proxyhealth.New(&proxyhealth.Options{
Interval: time.Minute * 1,
Interval: options.ProxyHealthInterval,
DB: api.Database,
Logger: options.Logger.Named("proxyhealth"),
Client: api.HTTPClient,
Expand Down
2 changes: 2 additions & 0 deletions enterprise/coderd/coderdenttest/coderdenttest.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ type Options struct {
EntitlementsUpdateInterval time.Duration
SCIMAPIKey []byte
UserWorkspaceQuota int
ProxyHealthInterval time.Duration
}

// New constructs a codersdk client connected to an in-memory Enterprise API instance.
Expand All @@ -74,6 +75,7 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c
Options: oop,
EntitlementsUpdateInterval: options.EntitlementsUpdateInterval,
Keys: Keys,
ProxyHealthInterval: options.ProxyHealthInterval,
})
assert.NoError(t, err)
setHandler(coderAPI.AGPL.RootHandler)
Expand Down
38 changes: 32 additions & 6 deletions enterprise/coderd/workspaceproxy.go
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,7 @@ func (api *API) regions(rw http.ResponseWriter, r *http.Request) {
continue
}

health, ok := proxyHealth[proxy.ID]
if !ok {
health.Status = proxyhealth.Unknown
}

health := proxyHealth[proxy.ID]
regions = append(regions, codersdk.Region{
ID: proxy.ID,
Name: proxy.Name,
Expand Down Expand Up @@ -423,7 +419,7 @@ func (api *API) workspaceProxyRegister(rw http.ResponseWriter, r *http.Request)
// Log: api.Logger,
// Request: r,
// Action: database.AuditActionWrite,
//})
// })
)
// aReq.Old = proxy
// defer commitAudit()
Expand Down Expand Up @@ -473,6 +469,33 @@ func (api *API) workspaceProxyRegister(rw http.ResponseWriter, r *http.Request)
go api.forceWorkspaceProxyHealthUpdate(api.ctx)
}

// workspaceProxyGoingAway is used to tell coderd that the workspace proxy is
// shutting down and going away. The main purpose of this function is for the
// health status of the workspace proxy to be more quickly updated when we know
// that the proxy is going to be unhealthy. This does not delete the workspace
// or cause any other side effects.
// If the workspace proxy comes back online, even without a register, it will
// be found healthy again by the normal checks.
// @Summary Workspace proxy going away
// @ID workspace-proxy-going-away
// @Security CoderSessionToken
// @Produce json
// @Tags Enterprise
// @Success 201 {object} codersdk.Response
// @Router /workspaceproxies/me/goingaway [post]
// @x-apidocgen {"skip": true}
func (api *API) workspaceProxyGoingAway(rw http.ResponseWriter, r *http.Request) {
ctx := r.Context()

// Force a health update to happen immediately. The proxy should
// not return a successful response if it is going away.
go api.forceWorkspaceProxyHealthUpdate(api.ctx)

httpapi.Write(ctx, rw, http.StatusOK, codersdk.Response{
Message: "OK",
})
}

// reconnectingPTYSignedToken issues a signed app token for use when connecting
// to the reconnecting PTY websocket on an external workspace proxy. This is set
// by the client as a query parameter when connecting.
Expand Down Expand Up @@ -588,6 +611,9 @@ func convertProxies(p []database.WorkspaceProxy, statuses map[uuid.UUID]proxyhea
}

func convertProxy(p database.WorkspaceProxy, status proxyhealth.ProxyStatus) codersdk.WorkspaceProxy {
if status.Status == "" {
status.Status = proxyhealth.Unknown
}
return codersdk.WorkspaceProxy{
ID: p.ID,
Name: p.Name,
Expand Down
64 changes: 64 additions & 0 deletions enterprise/coderd/workspaceproxy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"net/http/httputil"
"net/url"
"testing"
"time"

"github.com/google/uuid"
"github.com/moby/moby/pkg/namesgenerator"
Expand Down Expand Up @@ -172,6 +173,69 @@ func TestRegions(t *testing.T) {
require.Error(t, err)
require.Empty(t, regions)
})

t.Run("GoingAway", func(t *testing.T) {
t.Parallel()

dv := coderdtest.DeploymentValues(t)
dv.Experiments = []string{
string(codersdk.ExperimentMoons),
"*",
}

db, pubsub := dbtestutil.NewDB(t)

ctx := testutil.Context(t, testutil.WaitLong)

client, closer, api := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
Options: &coderdtest.Options{
AppHostname: appHostname,
Database: db,
Pubsub: pubsub,
DeploymentValues: dv,
},
// The interval is set to 1 hour so the proxy health
// check will never happen manually. All checks will be
// forced updates.
ProxyHealthInterval: time.Hour,
})
t.Cleanup(func() {
_ = closer.Close()
})
_ = coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
Features: license.Features{
codersdk.FeatureWorkspaceProxy: 1,
},
})

const proxyName = "testproxy"
proxy := coderdenttest.NewWorkspaceProxy(t, api, client, &coderdenttest.ProxyOptions{
Name: proxyName,
})
_ = proxy

require.Eventuallyf(t, func() bool {
proxy, err := client.WorkspaceProxyByName(ctx, proxyName)
if err != nil {
// We are testing the going away, not the initial healthy.
// Just force an update to change this to healthy.
_ = api.ProxyHealth.ForceUpdate(ctx)
return false
}
return proxy.Status.Status == codersdk.ProxyHealthy
}, testutil.WaitShort, testutil.IntervalFast, "proxy never became healthy")

_ = proxy.Close()
// The proxy should tell the primary on close that is is no longer healthy.
require.Eventuallyf(t, func() bool {
proxy, err := client.WorkspaceProxyByName(ctx, proxyName)
if err != nil {
return false
}
return proxy.Status.Status == codersdk.ProxyUnhealthy
}, testutil.WaitShort, testutil.IntervalFast, "proxy never became unhealthy after close")
})
}

func TestWorkspaceProxyCRUD(t *testing.T) {
Expand Down
Loading