Skip to content

Commit 3bcc6fa

Browse files
ammarioammario
committed
Stream template upload
Resolves #5718
1 parent 89bf8dd commit 3bcc6fa

File tree

13 files changed

+55
-43
lines changed

13 files changed

+55
-43
lines changed

‎cli/root_test.go

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,6 @@ var updateGoldenFiles = flag.Bool("update", false, "update .golden files")
3030

3131
//nolint:tparallel,paralleltest // These test sets env vars.
3232
func TestCommandHelp(t *testing.T) {
33-
t.Parallel()
34-
3533
commonEnv := map[string]string{
3634
"CODER_CONFIG_DIR": "/tmp/coder-cli-test-config",
3735
}

‎cli/templateinit.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package cli
22

33
import (
4+
"bytes"
45
"fmt"
56
"os"
67
"path/filepath"
@@ -70,7 +71,7 @@ func templateInit() *cobra.Command {
7071
if err != nil {
7172
return err
7273
}
73-
err = provisionersdk.Untar(directory, archive)
74+
err = provisionersdk.Untar(directory, bytes.NewReader(archive))
7475
if err != nil {
7576
return err
7677
}

‎cli/templatepush.go

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,11 @@ func (pf *templateUploadFlags) stdin() bool {
3434

3535
func (pf *templateUploadFlags) upload(cmd *cobra.Command, client *codersdk.Client) (*codersdk.UploadResponse, error) {
3636
var (
37-
content []byte
37+
content io.Reader
3838
err error
3939
)
4040
if pf.stdin() {
41-
content, err = io.ReadAll(cmd.InOrStdin())
41+
content = cmd.InOrStdin()
4242
} else {
4343
prettyDir := prettyDirectoryPath(pf.directory)
4444
_, err = cliui.Prompt(cmd, cliui.PromptOptions{
@@ -50,7 +50,11 @@ func (pf *templateUploadFlags) upload(cmd *cobra.Command, client *codersdk.Clien
5050
return nil, err
5151
}
5252

53-
content, err = provisionersdk.Tar(pf.directory, provisionersdk.TemplateArchiveLimit)
53+
pipeReader, pipeWriter := io.Pipe()
54+
err = provisionersdk.Tar(pipeWriter, pf.directory, provisionersdk.TemplateArchiveLimit)
55+
defer pipeReader.Close()
56+
defer pipeWriter.Close()
57+
content = pipeReader
5458
}
5559
if err != nil {
5660
return nil, xerrors.Errorf("read tar: %w", err)

‎coderd/coderdtest/authorize.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package coderdtest
22

33
import (
4+
"bytes"
45
"context"
56
"fmt"
67
"io"
@@ -378,7 +379,7 @@ func NewAuthTester(ctx context.Context, t *testing.T, client *codersdk.Client, a
378379
template := CreateTemplate(t, client, admin.OrganizationID, version.ID)
379380
workspace := CreateWorkspace(t, client, admin.OrganizationID, template.ID)
380381
AwaitWorkspaceBuildJob(t, client, workspace.LatestBuild.ID)
381-
file, err := client.Upload(ctx, codersdk.ContentTypeTar, make([]byte, 1024))
382+
file, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(make([]byte, 1024)))
382383
require.NoError(t, err, "upload file")
383384
workspace, err = client.Workspace(ctx, workspace.ID)
384385
require.NoError(t, err, "workspace resources")

‎coderd/coderdtest/coderdtest.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -525,7 +525,7 @@ func CreateTemplateVersion(t *testing.T, client *codersdk.Client, organizationID
525525
t.Helper()
526526
data, err := echo.Tar(res)
527527
require.NoError(t, err)
528-
file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, data)
528+
file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, bytes.NewReader(data))
529529
require.NoError(t, err)
530530
templateVersion, err := client.CreateTemplateVersion(context.Background(), organizationID, codersdk.CreateTemplateVersionRequest{
531531
FileID: file.ID,
@@ -572,7 +572,7 @@ func CreateTemplate(t *testing.T, client *codersdk.Client, organization uuid.UUI
572572
func UpdateTemplateVersion(t *testing.T, client *codersdk.Client, organizationID uuid.UUID, res *echo.Responses, templateID uuid.UUID) codersdk.TemplateVersion {
573573
data, err := echo.Tar(res)
574574
require.NoError(t, err)
575-
file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, data)
575+
file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, bytes.NewReader(data))
576576
require.NoError(t, err)
577577
templateVersion, err := client.CreateTemplateVersion(context.Background(), organizationID, codersdk.CreateTemplateVersionRequest{
578578
TemplateID: templateID,

‎coderd/files_test.go

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package coderd_test
22

33
import (
4+
"bytes"
45
"context"
56
"net/http"
67
"testing"
@@ -23,7 +24,7 @@ func TestPostFiles(t *testing.T) {
2324
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
2425
defer cancel()
2526

26-
_, err := client.Upload(ctx, "bad", []byte{'a'})
27+
_, err := client.Upload(ctx, "bad", bytes.NewReader([]byte{'a'}))
2728
require.Error(t, err)
2829
})
2930

@@ -35,7 +36,7 @@ func TestPostFiles(t *testing.T) {
3536
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
3637
defer cancel()
3738

38-
_, err := client.Upload(ctx, codersdk.ContentTypeTar, make([]byte, 1024))
39+
_, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(make([]byte, 1024)))
3940
require.NoError(t, err)
4041
})
4142

@@ -48,9 +49,9 @@ func TestPostFiles(t *testing.T) {
4849
defer cancel()
4950

5051
data := make([]byte, 1024)
51-
_, err := client.Upload(ctx, codersdk.ContentTypeTar, data)
52+
_, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(data))
5253
require.NoError(t, err)
53-
_, err = client.Upload(ctx, codersdk.ContentTypeTar, data)
54+
_, err = client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(data))
5455
require.NoError(t, err)
5556
})
5657
}
@@ -79,7 +80,7 @@ func TestDownload(t *testing.T) {
7980
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
8081
defer cancel()
8182

82-
resp, err := client.Upload(ctx, codersdk.ContentTypeTar, make([]byte, 1024))
83+
resp, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(make([]byte, 1024)))
8384
require.NoError(t, err)
8485
data, contentType, err := client.Download(ctx, resp.ID)
8586
require.NoError(t, err)

‎coderd/templateversions_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package coderd_test
22

33
import (
4+
"bytes"
45
"context"
56
"net/http"
67
"testing"
@@ -108,7 +109,7 @@ func TestPostTemplateVersionsByOrganization(t *testing.T) {
108109
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
109110
defer cancel()
110111

111-
file, err := client.Upload(ctx, codersdk.ContentTypeTar, data)
112+
file, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(data))
112113
require.NoError(t, err)
113114
version, err := client.CreateTemplateVersion(ctx, user.OrganizationID, codersdk.CreateTemplateVersionRequest{
114115
Name: "bananas",
@@ -895,7 +896,7 @@ func TestPaginatedTemplateVersions(t *testing.T) {
895896
templateVersionIDs := make([]uuid.UUID, total)
896897
data, err := echo.Tar(nil)
897898
require.NoError(t, err)
898-
file, err := client.Upload(egCtx, codersdk.ContentTypeTar, data)
899+
file, err := client.Upload(egCtx, codersdk.ContentTypeTar, bytes.NewReader(data))
899900
require.NoError(t, err)
900901
for i := 0; i < total; i++ {
901902
i := i

‎codersdk/client.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -109,18 +109,20 @@ func (c *Client) Request(ctx context.Context, method, path string, body interfac
109109

110110
var r io.Reader
111111
if body != nil {
112-
if data, ok := body.([]byte); ok {
112+
switch data := body.(type) {
113+
case io.Reader:
114+
r = data
115+
case []byte:
113116
r = bytes.NewReader(data)
114-
} else {
115-
// Assume JSON if not bytes.
117+
default:
118+
// Assume JSON in all other cases.
116119
buf := bytes.NewBuffer(nil)
117120
enc := json.NewEncoder(buf)
118121
enc.SetEscapeHTML(false)
119122
err = enc.Encode(body)
120123
if err != nil {
121124
return nil, xerrors.Errorf("encode body: %w", err)
122125
}
123-
124126
r = buf
125127
}
126128
}

‎codersdk/files.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ type UploadResponse struct {
2121

2222
// Upload uploads an arbitrary file with the content type provided.
2323
// This is used to upload a source-code archive.
24-
func (c *Client) Upload(ctx context.Context, contentType string, content []byte) (UploadResponse, error) {
25-
res, err := c.Request(ctx, http.MethodPost, "/api/v2/files", content, func(r *http.Request) {
24+
func (c *Client) Upload(ctx context.Context, contentType string, rd io.Reader) (UploadResponse, error) {
25+
res, err := c.Request(ctx, http.MethodPost, "/api/v2/files", rd, func(r *http.Request) {
2626
r.Header.Set("Content-Type", contentType)
2727
})
2828
if err != nil {

‎enterprise/coderd/provisionerdaemons_test.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package coderd_test
22

33
import (
4+
"bytes"
45
"context"
56
"net/http"
67
"testing"
@@ -119,7 +120,7 @@ func TestProvisionerDaemonServe(t *testing.T) {
119120
}},
120121
})
121122
require.NoError(t, err)
122-
file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, data)
123+
file, err := client.Upload(context.Background(), codersdk.ContentTypeTar, bytes.NewReader(data))
123124
require.NoError(t, err)
124125

125126
version, err := client.CreateTemplateVersion(context.Background(), user.OrganizationID, codersdk.CreateTemplateVersionRequest{

0 commit comments

Comments
 (0)