Skip to content

feat: add auto-locking/deleting workspace based on template config #8240

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Jul 3, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion coderd/apidoc/docs.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion coderd/apidoc/swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

125 changes: 107 additions & 18 deletions coderd/autobuild/lifecycle_executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -160,23 +160,65 @@ func (e *Executor) runOnce(t time.Time) Stats {
return nil
}

builder := wsbuilder.New(ws, nextTransition).
SetLastWorkspaceBuildInTx(&latestBuild).
SetLastWorkspaceBuildJobInTx(&latestJob).
Reason(reason)

if _, _, err := builder.Build(e.ctx, tx, nil); err != nil {
log.Error(e.ctx, "workspace build error",
slog.F("transition", nextTransition),
slog.Error(err),
if nextTransition != "" {
builder := wsbuilder.New(ws, nextTransition).
SetLastWorkspaceBuildInTx(&latestBuild).
SetLastWorkspaceBuildJobInTx(&latestJob).
Reason(reason)

if _, _, err := builder.Build(e.ctx, tx, nil); err != nil {
log.Error(e.ctx, "unable to transition workspace",
slog.F("transition", nextTransition),
slog.Error(err),
)
return nil
}
}

// Lock the workspace if it has breached the template's
// threshold for inactivity.
if reason == database.BuildReasonAutolock {
err = tx.UpdateWorkspaceLockedAt(e.ctx, database.UpdateWorkspaceLockedAtParams{
ID: ws.ID,
LockedAt: sql.NullTime{
Time: database.Now(),
Valid: true,
},
})
if err != nil {
log.Error(e.ctx, "unable to lock workspace",
slog.F("transition", nextTransition),
slog.Error(err),
)
return nil
}

log.Info(e.ctx, "locked workspace",
slog.F("last_used_at", ws.LastUsedAt),
slog.F("inactivity_ttl", templateSchedule.InactivityTTL),
slog.F("since_last_used_at", time.Since(ws.LastUsedAt)),
)
}

if reason == database.BuildReasonAutodelete {
log.Info(e.ctx, "deleted workspace",
slog.F("locked_at", ws.LockedAt.Time),
slog.F("locked_ttl", templateSchedule.LockedTTL),
)
}

if nextTransition == "" {
return nil
}

statsMu.Lock()
stats.Transitions[ws.ID] = nextTransition
statsMu.Unlock()

log.Info(e.ctx, "scheduling workspace transition", slog.F("transition", nextTransition))
log.Info(e.ctx, "scheduling workspace transition",
slog.F("transition", nextTransition),
slog.F("reason", reason),
)

return nil

Expand All @@ -199,6 +241,12 @@ func (e *Executor) runOnce(t time.Time) Stats {
return stats
}

// getNextTransition returns the next eligible transition for the workspace
// as well as the reason for why it is transitioning. It is possible
// for this function to return a nil error as well as an empty transition.
// In such cases it means no provisioning should occur but the workspace
// may be "transitioning" to a new state (such as an inactive, stopped
// workspace transitioning to the locked state).
func getNextTransition(
ws database.Workspace,
latestBuild database.WorkspaceBuild,
Expand All @@ -211,12 +259,23 @@ func getNextTransition(
error,
) {
switch {
case isEligibleForAutostop(latestBuild, latestJob, currentTick):
case isEligibleForAutostop(ws, latestBuild, latestJob, currentTick):
return database.WorkspaceTransitionStop, database.BuildReasonAutostop, nil
case isEligibleForAutostart(ws, latestBuild, latestJob, templateSchedule, currentTick):
return database.WorkspaceTransitionStart, database.BuildReasonAutostart, nil
case isEligibleForFailedStop(latestBuild, latestJob, templateSchedule):
case isEligibleForFailedStop(latestBuild, latestJob, templateSchedule, currentTick):
return database.WorkspaceTransitionStop, database.BuildReasonAutostop, nil
case isEligibleForLockedStop(ws, templateSchedule, currentTick):
// Only stop started workspaces.
if latestBuild.Transition == database.WorkspaceTransitionStart {
return database.WorkspaceTransitionStop, database.BuildReasonAutolock, nil
}
// We shouldn't transition the workspace but we should still
// lock it.
return "", database.BuildReasonAutolock, nil

case isEligibleForDelete(ws, templateSchedule, currentTick):
return database.WorkspaceTransitionDelete, database.BuildReasonAutodelete, nil
default:
return "", "", xerrors.Errorf("last transition not valid for autostart or autostop")
}
Expand All @@ -225,7 +284,12 @@ func getNextTransition(
// isEligibleForAutostart returns true if the workspace should be autostarted.
func isEligibleForAutostart(ws database.Workspace, build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool {
// Don't attempt to autostart failed workspaces.
if !job.CompletedAt.Valid || job.Error.String != "" {
if db2sdk.ProvisionerJobStatus(job) == codersdk.ProvisionerJobFailed {
return false
}

// If the workspace is locked we should not autostart it.
if ws.LockedAt.Valid {
return false
}

Expand Down Expand Up @@ -253,9 +317,13 @@ func isEligibleForAutostart(ws database.Workspace, build database.WorkspaceBuild
}

// isEligibleForAutostart returns true if the workspace should be autostopped.
func isEligibleForAutostop(build database.WorkspaceBuild, job database.ProvisionerJob, currentTick time.Time) bool {
// Don't attempt to autostop failed workspaces.
if !job.CompletedAt.Valid || job.Error.String != "" {
func isEligibleForAutostop(ws database.Workspace, build database.WorkspaceBuild, job database.ProvisionerJob, currentTick time.Time) bool {
if db2sdk.ProvisionerJobStatus(job) == codersdk.ProvisionerJobFailed {
return false
}

// If the workspace is locked we should not autostop it.
if ws.LockedAt.Valid {
return false
}

Expand All @@ -266,14 +334,35 @@ func isEligibleForAutostop(build database.WorkspaceBuild, job database.Provision
!currentTick.Before(build.Deadline)
}

// isEligibleForLockedStop returns true if the workspace should be locked
// for breaching the inactivity threshold of the template.
func isEligibleForLockedStop(ws database.Workspace, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool {
// Only attempt to lock workspaces not already locked.
return !ws.LockedAt.Valid &&
// The template must specify an inactivity TTL.
templateSchedule.InactivityTTL > 0 &&
// The workspace must breach the inactivity TTL.
currentTick.Sub(ws.LastUsedAt) > templateSchedule.InactivityTTL
}

func isEligibleForDelete(ws database.Workspace, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool {
// Only attempt to delete locked workspaces.
return ws.LockedAt.Valid &&
// Locked workspaces should only be deleted if a locked_ttl is specified.
templateSchedule.LockedTTL > 0 &&
// The workspace must breach the locked_ttl.
currentTick.Sub(ws.LockedAt.Time) > templateSchedule.LockedTTL
}

// isEligibleForFailedStop returns true if the workspace is eligible to be stopped
// due to a failed build.
func isEligibleForFailedStop(build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions) bool {
func isEligibleForFailedStop(build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool {
// If the template has specified a failure TLL.
return templateSchedule.FailureTTL > 0 &&
// And the job resulted in failure.
db2sdk.ProvisionerJobStatus(job) == codersdk.ProvisionerJobFailed &&
build.Transition == database.WorkspaceTransitionStart &&
// And sufficient time has elapsed since the job has completed.
job.CompletedAt.Valid && database.Now().Sub(job.CompletedAt.Time) > templateSchedule.FailureTTL
job.CompletedAt.Valid &&
currentTick.Sub(job.CompletedAt.Time) > templateSchedule.FailureTTL
}
63 changes: 54 additions & 9 deletions coderd/autobuild/lifecycle_executor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"github.com/coder/coder/codersdk"
"github.com/coder/coder/provisioner/echo"
"github.com/coder/coder/provisionersdk/proto"
"github.com/coder/coder/testutil"
)

func TestExecutorAutostartOK(t *testing.T) {
Expand Down Expand Up @@ -651,8 +650,9 @@ func TestExecutorAutostartTemplateDisabled(t *testing.T) {
assert.Len(t, stats.Transitions, 0)
}

// TesetExecutorFailedWorkspace tests that failed workspaces that breach
// their template failed_ttl threshold trigger a stop job.
// TestExecutorFailedWorkspace test AGPL functionality which mainly
// ensures that autostop actions as a result of a failed workspace
// build do not trigger.
// For enterprise functionality see enterprise/coderd/workspaces_test.go
func TestExecutorFailedWorkspace(t *testing.T) {
t.Parallel()
Expand Down Expand Up @@ -693,12 +693,57 @@ func TestExecutorFailedWorkspace(t *testing.T) {
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status)
require.Eventually(t,
func() bool {
return database.Now().Sub(*build.Job.CompletedAt) > failureTTL
},
testutil.IntervalMedium, testutil.IntervalFast)
ticker <- time.Now()
ticker <- build.Job.CompletedAt.Add(failureTTL * 2)
stats := <-statCh
// Expect no transitions since we're using AGPL.
require.Len(t, stats.Transitions, 0)
})
}

// TestExecutorInactiveWorkspace test AGPL functionality which mainly
// ensures that autostop actions as a result of an inactive workspace
// do not trigger.
// For enterprise functionality see enterprise/coderd/workspaces_test.go
func TestExecutorInactiveWorkspace(t *testing.T) {
t.Parallel()

// Test that an AGPL TemplateScheduleStore properly disables
// functionality.
t.Run("OK", func(t *testing.T) {
t.Parallel()

var (
ticker = make(chan time.Time)
statCh = make(chan autobuild.Stats)
logger = slogtest.Make(t, &slogtest.Options{
// We ignore errors here since we expect to fail
// builds.
IgnoreErrors: true,
})
inactiveTTL = time.Millisecond

client = coderdtest.New(t, &coderdtest.Options{
Logger: &logger,
AutobuildTicker: ticker,
IncludeProvisionerDaemon: true,
AutobuildStats: statCh,
TemplateScheduleStore: schedule.NewAGPLTemplateScheduleStore(),
})
)
user := coderdtest.CreateFirstUser(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
ProvisionApply: echo.ProvisionComplete,
})
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
ctr.InactivityTTLMillis = ptr.Ref[int64](inactiveTTL.Milliseconds())
})
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status)
ticker <- ws.LastUsedAt.Add(inactiveTTL * 2)
stats := <-statCh
// Expect no transitions since we're using AGPL.
require.Len(t, stats.Transitions, 0)
Expand Down
24 changes: 22 additions & 2 deletions coderd/database/dbfake/dbfake.go
Original file line number Diff line number Diff line change
Expand Up @@ -3495,12 +3495,17 @@ func (q *fakeQuerier) GetWorkspacesEligibleForTransition(ctx context.Context, no
return nil, err
}

if build.Transition == database.WorkspaceTransitionStart && !build.Deadline.IsZero() && build.Deadline.Before(now) {
if build.Transition == database.WorkspaceTransitionStart &&
!build.Deadline.IsZero() &&
build.Deadline.Before(now) &&
!workspace.LockedAt.Valid {
workspaces = append(workspaces, workspace)
continue
}

if build.Transition == database.WorkspaceTransitionStop && workspace.AutostartSchedule.Valid {
if build.Transition == database.WorkspaceTransitionStop &&
workspace.AutostartSchedule.Valid &&
!workspace.LockedAt.Valid {
workspaces = append(workspaces, workspace)
continue
}
Expand All @@ -3513,6 +3518,19 @@ func (q *fakeQuerier) GetWorkspacesEligibleForTransition(ctx context.Context, no
workspaces = append(workspaces, workspace)
continue
}

template, err := q.GetTemplateByID(ctx, workspace.TemplateID)
if err != nil {
return nil, xerrors.Errorf("get template by ID: %w", err)
}
if !workspace.LockedAt.Valid && template.InactivityTTL > 0 {
workspaces = append(workspaces, workspace)
continue
}
if workspace.LockedAt.Valid && template.LockedTTL > 0 {
workspaces = append(workspaces, workspace)
continue
}
}

return workspaces, nil
Expand Down Expand Up @@ -4702,6 +4720,7 @@ func (q *fakeQuerier) UpdateTemplateScheduleByID(_ context.Context, arg database
tpl.MaxTTL = arg.MaxTTL
tpl.FailureTTL = arg.FailureTTL
tpl.InactivityTTL = arg.InactivityTTL
tpl.LockedTTL = arg.LockedTTL
q.templates[idx] = tpl
return tpl.DeepCopy(), nil
}
Expand Down Expand Up @@ -5245,6 +5264,7 @@ func (q *fakeQuerier) UpdateWorkspaceLockedAt(_ context.Context, arg database.Up
continue
}
workspace.LockedAt = arg.LockedAt
workspace.LastUsedAt = database.Now()
q.workspaces[index] = workspace
return nil
}
Expand Down
5 changes: 4 additions & 1 deletion coderd/database/dump.sql

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
-- It's not possible to delete enum values.
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
BEGIN;
ALTER TYPE build_reason ADD VALUE IF NOT EXISTS 'autolock';
ALTER TYPE build_reason ADD VALUE IF NOT EXISTS 'failedstop';
ALTER TYPE build_reason ADD VALUE IF NOT EXISTS 'autodelete';
COMMIT;
17 changes: 13 additions & 4 deletions coderd/database/models.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading