coder · kylecarbs · Feb 3, 2022 · Feb 1, 2022 · Feb 1, 2022 · Feb 1, 2022
diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml
@@ -151,7 +151,6 @@ jobs:
       - run: go install gotest.tools/gotestsum@latest
 
       - uses: hashicorp/setup-terraform@v1
-        if: runner.os == 'Linux'
         with:
           terraform_version: 1.1.2
           terraform_wrapper: false

diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -23,5 +23,32 @@
       }
     ]
   },
-  "cSpell.words": ["coderd", "coderdtest", "codersdk", "httpmw", "oneof", "stretchr", "xerrors"]
+  "cSpell.words": [
+    "coderd",
+    "coderdtest",
+    "codersdk",
+    "drpc",
+    "drpcconn",
+    "drpcmux",
+    "drpcserver",
+    "goleak",
+    "hashicorp",
+    "httpmw",
+    "moby",
+    "nhooyr",
+    "nolint",
+    "nosec",
+    "oneof",
+    "protobuf",
+    "provisionerd",
+    "provisionersdk",
+    "retrier",
+    "sdkproto",
+    "stretchr",
+    "tfexec",
+    "tfstate",
+    "unconvert",
+    "xerrors",
+    "yamux"
+  ]
 }
diff --git a/coderd/cmd/root.go b/coderd/cmd/root.go
@@ -11,6 +11,7 @@ import (
 	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/sloghuman"
 	"github.com/coder/coder/coderd"
+	"github.com/coder/coder/database"
 	"github.com/coder/coder/database/databasefake"
 )
 
@@ -24,6 +25,7 @@ func Root() *cobra.Command {
 			handler := coderd.New(&coderd.Options{
 				Logger:   slog.Make(sloghuman.Sink(os.Stderr)),
 				Database: databasefake.New(),
+				Pubsub:   database.NewPubsubInMemory(),
 			})
 
 			listener, err := net.Listen("tcp", address)

diff --git a/coderd/coderd.go b/coderd/coderd.go
@@ -64,6 +64,10 @@ func New(options *Options) http.Handler {
 					r.Route("/history", func(r chi.Router) {
 						r.Get("/", api.projectHistoryByOrganization)
 						r.Post("/", api.postProjectHistoryByOrganization)
+						r.Route("/{projecthistory}", func(r chi.Router) {
+							r.Use(httpmw.ExtractProjectHistoryParam(api.Database))
+							r.Get("/", api.projectHistoryByOrganizationAndName)
+						})
 					})
 				})
 			})
@@ -84,11 +88,19 @@ func New(options *Options) http.Handler {
 					r.Route("/history", func(r chi.Router) {
 						r.Post("/", api.postWorkspaceHistoryByUser)
 						r.Get("/", api.workspaceHistoryByUser)
-						r.Get("/latest", api.latestWorkspaceHistoryByUser)
+						r.Route("/{workspacehistory}", func(r chi.Router) {
+							r.Use(httpmw.ExtractWorkspaceHistoryParam(options.Database))
+							r.Get("/", api.workspaceHistoryByName)
+						})
 					})
 				})
 			})
 		})
+
+		r.Route("/provisioners/daemons", func(r chi.Router) {
+			r.Get("/", api.provisionerDaemons)
+			r.Get("/serve", api.provisionerDaemonsServe)
+		})
 	})
 	r.NotFound(site.Handler().ServeHTTP)
 	return r

diff --git a/coderd/coderdtest/coderdtest.go b/coderd/coderdtest/coderdtest.go
@@ -3,20 +3,27 @@ package coderdtest
 import (
 	"context"
 	"database/sql"
+	"io"
 	"net/http/httptest"
 	"net/url"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/require"
 
+	"cdr.dev/slog"
 	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/coderd"
 	"github.com/coder/coder/codersdk"
 	"github.com/coder/coder/cryptorand"
 	"github.com/coder/coder/database"
 	"github.com/coder/coder/database/databasefake"
 	"github.com/coder/coder/database/postgres"
+	"github.com/coder/coder/provisioner/terraform"
+	"github.com/coder/coder/provisionerd"
+	"github.com/coder/coder/provisionersdk"
+	"github.com/coder/coder/provisionersdk/proto"
 )
 
 // Server represents a test instance of coderd.
@@ -57,11 +64,46 @@ func (s *Server) RandomInitialUser(t *testing.T) coderd.CreateInitialUserRequest
 	return req
 }
 
+// AddProvisionerd launches a new provisionerd instance!
+func (s *Server) AddProvisionerd(t *testing.T) io.Closer {
+	tfClient, tfServer := provisionersdk.TransportPipe()
+	ctx, cancelFunc := context.WithCancel(context.Background())
+	t.Cleanup(func() {
+		_ = tfClient.Close()
+		_ = tfServer.Close()
+		cancelFunc()
+	})
+	go func() {
+		err := terraform.Serve(ctx, &terraform.ServeOptions{
+			ServeOptions: &provisionersdk.ServeOptions{
+				Listener: tfServer,
+			},
+			Logger: slogtest.Make(t, nil).Named("terraform-provisioner").Leveled(slog.LevelDebug),
+		})
+		require.NoError(t, err)
+	}()
+
+	closer := provisionerd.New(s.Client.ProvisionerDaemonClient, &provisionerd.Options{
+		Logger:         slogtest.Make(t, nil).Named("provisionerd").Leveled(slog.LevelDebug),
+		PollInterval:   50 * time.Millisecond,
+		UpdateInterval: 50 * time.Millisecond,
+		Provisioners: provisionerd.Provisioners{
+			string(database.ProvisionerTypeTerraform): proto.NewDRPCProvisionerClient(provisionersdk.Conn(tfClient)),
+		},
+		WorkDirectory: t.TempDir(),
+	})
+	t.Cleanup(func() {
+		_ = closer.Close()
+	})
+	return closer
+}
+
 // New constructs a new coderd test instance. This returned Server
 // should contain no side-effects.
 func New(t *testing.T) Server {
 	// This can be hotswapped for a live database instance.
 	db := databasefake.New()
+	pubsub := database.NewPubsubInMemory()
 	if os.Getenv("DB") != "" {
 		connectionURL, close, err := postgres.Open()
 		require.NoError(t, err)
@@ -74,11 +116,18 @@ func New(t *testing.T) Server {
 		err = database.Migrate(sqlDB)
 		require.NoError(t, err)
 		db = database.New(sqlDB)
+
+		pubsub, err = database.NewPubsub(context.Background(), sqlDB, connectionURL)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			_ = pubsub.Close()
+		})
 	}
 
 	handler := coderd.New(&coderd.Options{
 		Logger:   slogtest.Make(t, nil),
 		Database: db,
+		Pubsub:   pubsub,
 	})
 	srv := httptest.NewServer(handler)
 	serverURL, err := url.Parse(srv.URL)

diff --git a/coderd/coderdtest/coderdtest_test.go b/coderd/coderdtest/coderdtest_test.go
@@ -16,4 +16,5 @@ func TestNew(t *testing.T) {
 	t.Parallel()
 	server := coderdtest.New(t)
 	_ = server.RandomInitialUser(t)
+	_ = server.AddProvisionerd(t)
 }
diff --git a/coderd/projecthistory.go b/coderd/projecthistory.go
@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"database/sql"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"net/http"
@@ -12,6 +13,7 @@ import (
 	"github.com/go-chi/render"
 	"github.com/google/uuid"
 	"github.com/moby/moby/pkg/namesgenerator"
+	"golang.org/x/xerrors"
 
 	"github.com/coder/coder/database"
 	"github.com/coder/coder/httpapi"
@@ -26,6 +28,7 @@ type ProjectHistory struct {
 	UpdatedAt     time.Time                     `json:"updated_at"`
 	Name          string                        `json:"name"`
 	StorageMethod database.ProjectStorageMethod `json:"storage_method"`
+	Import        ProvisionerJob                `json:"import"`
 }
 
 // CreateProjectHistoryRequest enables callers to create a new Project Version.
@@ -50,12 +53,33 @@ func (api *api) projectHistoryByOrganization(rw http.ResponseWriter, r *http.Req
 	}
 	apiHistory := make([]ProjectHistory, 0)
 	for _, version := range history {
-		apiHistory = append(apiHistory, convertProjectHistory(version))
+		job, err := api.Database.GetProvisionerJobByID(r.Context(), version.ImportJobID)
+		if err != nil {
+			httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
+				Message: fmt.Sprintf("get provisioner job: %s", err),
+			})
+			return
+		}
+		apiHistory = append(apiHistory, convertProjectHistory(version, job))
 	}
 	render.Status(r, http.StatusOK)
 	render.JSON(rw, r, apiHistory)
 }
 
+// Return a single project history by organization and name.
+func (api *api) projectHistoryByOrganizationAndName(rw http.ResponseWriter, r *http.Request) {
+	projectHistory := httpmw.ProjectHistoryParam(r)
+	job, err := api.Database.GetProvisionerJobByID(r.Context(), projectHistory.ImportJobID)
+	if err != nil {
+		httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
+			Message: fmt.Sprintf("get provisioner job: %s", err),
+		})
+		return
+	}
+	render.Status(r, http.StatusOK)
+	render.JSON(rw, r, convertProjectHistory(projectHistory, job))
+}
+
 // Creates a new version of the project. An import job is queued to parse
 // the storage method provided. Once completed, the import job will specify
 // the version as latest.
@@ -82,37 +106,71 @@ func (api *api) postProjectHistoryByOrganization(rw http.ResponseWriter, r *http
 		return
 	}
 
+	apiKey := httpmw.APIKey(r)
 	project := httpmw.ProjectParam(r)
-	history, err := api.Database.InsertProjectHistory(r.Context(), database.InsertProjectHistoryParams{
-		ID:            uuid.New(),
-		ProjectID:     project.ID,
-		CreatedAt:     database.Now(),
-		UpdatedAt:     database.Now(),
-		Name:          namesgenerator.GetRandomName(1),
-		StorageMethod: createProjectVersion.StorageMethod,
-		StorageSource: createProjectVersion.StorageSource,
-		// TODO: Make this do something!
-		ImportJobID: uuid.New(),
+
+	var provisionerJob database.ProvisionerJob
+	var projectHistory database.ProjectHistory
+	err := api.Database.InTx(func(db database.Store) error {
+		projectHistoryID := uuid.New()
+		input, err := json.Marshal(projectImportJob{
+			ProjectHistoryID: projectHistoryID,
+		})
+		if err != nil {
+			return xerrors.Errorf("marshal import job: %w", err)
+		}
+
+		provisionerJob, err = db.InsertProvisionerJob(r.Context(), database.InsertProvisionerJobParams{
+			ID:          uuid.New(),
+			CreatedAt:   database.Now(),
+			UpdatedAt:   database.Now(),
+			InitiatorID: apiKey.UserID,
+			Provisioner: project.Provisioner,
+			Type:        database.ProvisionerJobTypeProjectImport,
+			ProjectID:   project.ID,
+			Input:       input,
+		})
+		if err != nil {
+			return xerrors.Errorf("insert provisioner job: %w", err)
+		}
+
+		projectHistory, err = api.Database.InsertProjectHistory(r.Context(), database.InsertProjectHistoryParams{
+			ID:            projectHistoryID,
+			ProjectID:     project.ID,
+			CreatedAt:     database.Now(),
+			UpdatedAt:     database.Now(),
+			Name:          namesgenerator.GetRandomName(1),
+			StorageMethod: createProjectVersion.StorageMethod,
+			StorageSource: createProjectVersion.StorageSource,
+			ImportJobID:   provisionerJob.ID,
+		})
+		if err != nil {
+			return xerrors.Errorf("insert project history: %s", err)
+		}
+		return nil
 	})
 	if err != nil {
 		httpapi.Write(rw, http.StatusInternalServerError, httpapi.Response{
-			Message: fmt.Sprintf("insert project history: %s", err),
+			Message: err.Error(),
 		})
 		return
 	}
 
-	// TODO: A job to process the new version should occur here.
-
 	render.Status(r, http.StatusCreated)
-	render.JSON(rw, r, convertProjectHistory(history))
+	render.JSON(rw, r, convertProjectHistory(projectHistory, provisionerJob))
 }
 
-func convertProjectHistory(history database.ProjectHistory) ProjectHistory {
+func convertProjectHistory(history database.ProjectHistory, job database.ProvisionerJob) ProjectHistory {
 	return ProjectHistory{
 		ID:        history.ID,
 		ProjectID: history.ProjectID,
 		CreatedAt: history.CreatedAt,
 		UpdatedAt: history.UpdatedAt,
 		Name:      history.Name,
+		Import:    convertProvisionerJob(job),
 	}
 }
+
+func projectHistoryLogsChannel(projectHistoryID uuid.UUID) string {
+	return fmt.Sprintf("project-history-logs:%s", projectHistoryID)
+}
diff --git a/coderd/projecthistory_test.go b/coderd/projecthistory_test.go
@@ -25,7 +25,7 @@ func TestProjectHistory(t *testing.T) {
 			Provisioner: database.ProvisionerTypeTerraform,
 		})
 		require.NoError(t, err)
-		versions, err := server.Client.ProjectHistory(context.Background(), user.Organization, project.Name)
+		versions, err := server.Client.ListProjectHistory(context.Background(), user.Organization, project.Name)
 		require.NoError(t, err)
 		require.Len(t, versions, 0)
 	})
@@ -48,14 +48,17 @@ func TestProjectHistory(t *testing.T) {
 		require.NoError(t, err)
 		_, err = writer.Write(make([]byte, 1<<10))
 		require.NoError(t, err)
-		_, err = server.Client.CreateProjectHistory(context.Background(), user.Organization, project.Name, coderd.CreateProjectHistoryRequest{
+		history, err := server.Client.CreateProjectHistory(context.Background(), user.Organization, project.Name, coderd.CreateProjectHistoryRequest{
 			StorageMethod: database.ProjectStorageMethodInlineArchive,
 			StorageSource: buffer.Bytes(),
 		})
 		require.NoError(t, err)
-		versions, err := server.Client.ProjectHistory(context.Background(), user.Organization, project.Name)
+		versions, err := server.Client.ListProjectHistory(context.Background(), user.Organization, project.Name)
 		require.NoError(t, err)
 		require.Len(t, versions, 1)
+
+		_, err = server.Client.ProjectHistory(context.Background(), user.Organization, project.Name, history.Name)
+		require.NoError(t, err)
 	})
 
 	t.Run("CreateHistoryArchiveTooBig", func(t *testing.T) {