Skip to content

Commit 42fd1c1

Browse files
johnstcnjohnstcn
authored
ci: cache embedded postgres downloaded binaries (#18477)
Updates CI job definitions to cache downloaded binaries for embedded-postgres.
1 parent c4e4fe8 commit 42fd1c1

File tree

5 files changed

+158
-10
lines changed

5 files changed

+158
-10
lines changed

‎.github/actions/embedded-pg-cache/download/action.yml

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
name: "Download Embedded Postgres Cache"
2+
description: |
3+
Downloads the embedded postgres cache and outputs today's cache key.
4+
A PR job can use a cache if it was created by its base branch, its current
5+
branch, or the default branch.
6+
https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#restrictions-for-accessing-a-cache
7+
outputs:
8+
cache-key:
9+
description: "Today's cache key"
10+
value: ${{ steps.vars.outputs.cache-key }}
11+
inputs:
12+
key-prefix:
13+
description: "Prefix for the cache key"
14+
required: true
15+
cache-path:
16+
description: "Path to the cache directory"
17+
required: true
18+
runs:
19+
using: "composite"
20+
steps:
21+
- name: Get date values and cache key
22+
id: vars
23+
shell: bash
24+
run: |
25+
export YEAR_MONTH=$(date +'%Y-%m')
26+
export PREV_YEAR_MONTH=$(date -d 'last month' +'%Y-%m')
27+
export DAY=$(date +'%d')
28+
echo "year-month=$YEAR_MONTH" >> $GITHUB_OUTPUT
29+
echo "prev-year-month=$PREV_YEAR_MONTH" >> $GITHUB_OUTPUT
30+
echo "cache-key=${{ inputs.key-prefix }}-${YEAR_MONTH}-${DAY}" >> $GITHUB_OUTPUT
31+
32+
# By default, depot keeps caches for 14 days. This is plenty for embedded
33+
# postgres, which changes infrequently.
34+
# https://depot.dev/docs/github-actions/overview#cache-retention-policy
35+
- name: Download embedded Postgres cache
36+
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
37+
with:
38+
path: ${{ inputs.cache-path }}
39+
key: ${{ steps.vars.outputs.cache-key }}
40+
# > If there are multiple partial matches for a restore key, the action returns the most recently created cache.
41+
# https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/caching-dependencies-to-speed-up-workflows#matching-a-cache-key
42+
# The second restore key allows non-main branches to use the cache from the previous month.
43+
# This prevents PRs from rebuilding the cache on the first day of the month.
44+
# It also makes sure that once a month, the cache is fully reset.
45+
restore-keys: |
46+
${{ inputs.key-prefix }}-${{ steps.vars.outputs.year-month }}-
47+
${{ github.ref != 'refs/heads/main' && format('{0}-{1}-', inputs.key-prefix, steps.vars.outputs.prev-year-month) || '' }}

‎.github/actions/embedded-pg-cache/upload/action.yml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: "Upload Embedded Postgres Cache"
2+
description: Uploads the embedded Postgres cache. This only runs on the main branch.
3+
inputs:
4+
cache-key:
5+
description: "Cache key"
6+
required: true
7+
cache-path:
8+
description: "Path to the cache directory"
9+
required: true
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: Upload Embedded Postgres cache
14+
if: ${{ github.ref == 'refs/heads/main' }}
15+
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
16+
with:
17+
path: ${{ inputs.cache-path }}
18+
key: ${{ inputs.cache-key }}

‎.github/actions/setup-embedded-pg-cache-paths/action.yml

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: "Setup Embedded Postgres Cache Paths"
2+
description: Sets up a path for cached embedded postgres binaries.
3+
outputs:
4+
embedded-pg-cache:
5+
description: "Value of EMBEDDED_PG_CACHE_DIR"
6+
value: ${{ steps.paths.outputs.embedded-pg-cache }}
7+
cached-dirs:
8+
description: "directories that should be cached between CI runs"
9+
value: ${{ steps.paths.outputs.cached-dirs }}
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: Override Go paths
14+
id: paths
15+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7
16+
with:
17+
script: |
18+
const path = require('path');
19+
20+
// RUNNER_TEMP should be backed by a RAM disk on Windows if
21+
// coder/setup-ramdisk-action was used
22+
const runnerTemp = process.env.RUNNER_TEMP;
23+
const embeddedPgCacheDir = path.join(runnerTemp, 'embedded-pg-cache');
24+
core.exportVariable('EMBEDDED_PG_CACHE_DIR', embeddedPgCacheDir);
25+
core.setOutput('embedded-pg-cache', embeddedPgCacheDir);
26+
const cachedDirs = `${embeddedPgCacheDir}`;
27+
core.setOutput('cached-dirs', cachedDirs);
28+
29+
- name: Create directories
30+
shell: bash
31+
run: |
32+
set -e
33+
mkdir -p "$EMBEDDED_PG_CACHE_DIR"

‎.github/workflows/ci.yaml

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -473,6 +473,17 @@ jobs:
473473
with:
474474
key-prefix: test-go-pg-${{ runner.os }}-${{ runner.arch }}
475475

476+
- name: Setup Embedded Postgres Cache Paths
477+
id: embedded-pg-cache
478+
uses: ./.github/actions/setup-embedded-pg-cache-paths
479+
480+
- name: Download Embedded Postgres Cache
481+
id: download-embedded-pg-cache
482+
uses: ./.github/actions/embedded-pg-cache/download
483+
with:
484+
key-prefix: embedded-pg-${{ runner.os }}-${{ runner.arch }}
485+
cache-path: ${{ steps.embedded-pg-cache.outputs.cached-dirs }}
486+
476487
- name: Normalize File and Directory Timestamps
477488
shell: bash
478489
# Normalize file modification timestamps so that go test can use the
@@ -497,12 +508,12 @@ jobs:
497508
# Create a temp dir on the R: ramdisk drive for Windows. The default
498509
# C: drive is extremely slow: https://github.com/actions/runner-images/issues/8755
499510
mkdir -p "R:/temp/embedded-pg"
500-
go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg"
511+
go run scripts/embedded-pg/main.go -path "R:/temp/embedded-pg" -cache "${EMBEDDED_PG_CACHE_DIR}"
501512
elif [ "${{ runner.os }}" == "macOS" ]; then
502513
# Postgres runs faster on a ramdisk on macOS too
503514
mkdir -p /tmp/tmpfs
504515
sudo mount_tmpfs -o noowners -s 8g /tmp/tmpfs
505-
go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg
516+
go run scripts/embedded-pg/main.go -path /tmp/tmpfs/embedded-pg -cache "${EMBEDDED_PG_CACHE_DIR}"
506517
elif [ "${{ runner.os }}" == "Linux" ]; then
507518
make test-postgres-docker
508519
fi
@@ -571,6 +582,14 @@ jobs:
571582
with:
572583
cache-key: ${{ steps.download-cache.outputs.cache-key }}
573584

585+
- name: Upload Embedded Postgres Cache
586+
uses: ./.github/actions/embedded-pg-cache/upload
587+
# We only use the embedded Postgres cache on macOS and Windows runners.
588+
if: runner.OS == 'macOS' || runner.OS == 'Windows'
589+
with:
590+
cache-key: ${{ steps.download-embedded-pg-cache.outputs.cache-key }}
591+
cache-path: "${{ steps.embedded-pg-cache.outputs.embedded-pg-cache }}"
592+
574593
- name: Upload test stats to Datadog
575594
timeout-minutes: 1
576595
continue-on-error: true

‎scripts/embedded-pg/main.go

Lines changed: 39 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,31 +4,43 @@ package main
44
import (
55
"database/sql"
66
"flag"
7+
"log"
78
"os"
89
"path/filepath"
10+
"time"
911

1012
embeddedpostgres "github.com/fergusstrange/embedded-postgres"
1113
)
1214

1315
func main() {
1416
var customPath string
17+
var cachePath string
1518
flag.StringVar(&customPath, "path", "", "Optional custom path for postgres data directory")
19+
flag.StringVar(&cachePath, "cache", "", "Optional custom path for embedded postgres binaries")
1620
flag.Parse()
1721

1822
postgresPath := filepath.Join(os.TempDir(), "coder-test-postgres")
1923
if customPath != "" {
2024
postgresPath = customPath
2125
}
26+
if err := os.MkdirAll(postgresPath, os.ModePerm); err != nil {
27+
log.Fatalf("Failed to create directory %s: %v", postgresPath, err)
28+
}
29+
if cachePath == "" {
30+
cachePath = filepath.Join(postgresPath, "cache")
31+
}
32+
if err := os.MkdirAll(cachePath, os.ModePerm); err != nil {
33+
log.Fatalf("Failed to create directory %s: %v", cachePath, err)
34+
}
2235

2336
ep := embeddedpostgres.NewDatabase(
2437
embeddedpostgres.DefaultConfig().
2538
Version(embeddedpostgres.V16).
2639
BinariesPath(filepath.Join(postgresPath, "bin")).
27-
// Default BinaryRepositoryURL repo1.maven.org is flaky.
2840
BinaryRepositoryURL("https://repo.maven.apache.org/maven2").
2941
DataPath(filepath.Join(postgresPath, "data")).
3042
RuntimePath(filepath.Join(postgresPath, "runtime")).
31-
CachePath(filepath.Join(postgresPath, "cache")).
43+
CachePath(cachePath).
3244
Username("postgres").
3345
Password("postgres").
3446
Database("postgres").
@@ -38,8 +50,27 @@ func main() {
3850
)
3951
err := ep.Start()
4052
if err != nil {
41-
panic(err)
53+
log.Fatalf("Failed to start embedded postgres: %v", err)
54+
}
55+
56+
// Troubleshooting: list files in cachePath
57+
if err := filepath.Walk(cachePath, func(path string, info os.FileInfo, err error) error {
58+
if err != nil {
59+
return err
60+
}
61+
switch {
62+
case info.IsDir():
63+
log.Printf("D: %s", path)
64+
case info.Mode().IsRegular():
65+
log.Printf("F: %s [%s] (%d bytes) %s", path, info.Mode().String(), info.Size(), info.ModTime().Format(time.RFC3339))
66+
default:
67+
log.Printf("Other: %s [%s] %s", path, info.Mode(), info.ModTime().Format(time.RFC3339))
68+
}
69+
return nil
70+
}); err != nil {
71+
log.Printf("Failed to list files in cachePath %s: %v", cachePath, err)
4272
}
73+
4374
// We execute these queries instead of using the embeddedpostgres
4475
// StartParams because it doesn't work on Windows. The library
4576
// seems to have a bug where it sends malformed parameters to
@@ -58,21 +89,21 @@ func main() {
5889
}
5990
db, err := sql.Open("postgres", "postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable")
6091
if err != nil {
61-
panic(err)
92+
log.Fatalf("Failed to connect to embedded postgres: %v", err)
6293
}
6394
for _, query := range paramQueries {
6495
if _, err := db.Exec(query); err != nil {
65-
panic(err)
96+
log.Fatalf("Failed to execute setup query %q: %v", query, err)
6697
}
6798
}
6899
if err := db.Close(); err != nil {
69-
panic(err)
100+
log.Fatalf("Failed to close database connection: %v", err)
70101
}
71102
// We restart the database to apply all the parameters.
72103
if err := ep.Stop(); err != nil {
73-
panic(err)
104+
log.Fatalf("Failed to stop embedded postgres after applying parameters: %v", err)
74105
}
75106
if err := ep.Start(); err != nil {
76-
panic(err)
107+
log.Fatalf("Failed to start embedded postgres after applying parameters: %v", err)
77108
}
78109
}

0 commit comments

Comments
 (0)