dendrite-upgrade-test: tweaks to get it to run under CI in docker (#1905)

* dendrite-upgrade-test: tweaks to get it to run under CI in docker

* Linting
main
kegsay 2021-07-08 12:28:04 +01:00 committed by GitHub
parent 3fb5ee7e1c
commit ef331c52af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 37 additions and 12 deletions

View File

@ -35,9 +35,13 @@ var (
flagFrom = flag.String("from", "HEAD-1", "The version to start from e.g '0.3.1'. If 'HEAD-N' then starts N versions behind HEAD.") flagFrom = flag.String("from", "HEAD-1", "The version to start from e.g '0.3.1'. If 'HEAD-N' then starts N versions behind HEAD.")
flagTo = flag.String("to", "HEAD", "The version to end on e.g '0.3.3'.") flagTo = flag.String("to", "HEAD", "The version to end on e.g '0.3.3'.")
flagBuildConcurrency = flag.Int("build-concurrency", runtime.NumCPU(), "The amount of build concurrency when building images") flagBuildConcurrency = flag.Int("build-concurrency", runtime.NumCPU(), "The amount of build concurrency when building images")
flagHead = flag.String("head", "", "Location to a dendrite repository to treat as HEAD instead of Github")
flagDockerHost = flag.String("docker-host", "localhost", "The hostname of the docker client. 'localhost' if running locally, 'host.docker.internal' if running in Docker.")
alphaNumerics = regexp.MustCompile("[^a-zA-Z0-9]+") alphaNumerics = regexp.MustCompile("[^a-zA-Z0-9]+")
) )
const HEAD = "HEAD"
// Embed the Dockerfile to use when building dendrite versions. // Embed the Dockerfile to use when building dendrite versions.
// We cannot use the dockerfile associated with the repo with each version sadly due to changes in // We cannot use the dockerfile associated with the repo with each version sadly due to changes in
// Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients // Docker versions. Specifically, earlier Dendrite versions are incompatible with newer Docker clients
@ -135,15 +139,36 @@ func downloadArchive(cli *http.Client, tmpDir, archiveURL string, dockerfile []b
// buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error // buildDendrite builds Dendrite on the branchOrTagName given. Returns the image ID or an error
func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, branchOrTagName string) (string, error) { func buildDendrite(httpClient *http.Client, dockerClient *client.Client, tmpDir, branchOrTagName string) (string, error) {
log.Printf("%s: Downloading version %s to %s\n", branchOrTagName, branchOrTagName, tmpDir) var tarball *bytes.Buffer
// pull an archive, this contains a top-level directory which screws with the build context var err error
// which we need to fix up post download // If a custom HEAD location is given, use that, else pull from github. Mostly useful for CI
u := fmt.Sprintf("https://github.com/matrix-org/dendrite/archive/%s.tar.gz", branchOrTagName) // where we want to use the working directory.
tarball, err := downloadArchive(httpClient, tmpDir, u, []byte(Dockerfile)) if branchOrTagName == HEAD && *flagHead != "" {
if err != nil { log.Printf("%s: Using %s as HEAD", branchOrTagName, *flagHead)
return "", fmt.Errorf("failed to download archive %s: %w", u, err) // add top level Dockerfile
err = ioutil.WriteFile(path.Join(*flagHead, "Dockerfile"), []byte(Dockerfile), os.ModePerm)
if err != nil {
return "", fmt.Errorf("Custom HEAD: failed to inject /Dockerfile: %w", err)
}
// now tarball it
var buffer bytes.Buffer
err = compress(*flagHead, &buffer)
if err != nil {
return "", fmt.Errorf("failed to tarball custom HEAD %s : %s", *flagHead, err)
}
tarball = &buffer
} else {
log.Printf("%s: Downloading version %s to %s\n", branchOrTagName, branchOrTagName, tmpDir)
// pull an archive, this contains a top-level directory which screws with the build context
// which we need to fix up post download
u := fmt.Sprintf("https://github.com/matrix-org/dendrite/archive/%s.tar.gz", branchOrTagName)
tarball, err = downloadArchive(httpClient, tmpDir, u, []byte(Dockerfile))
if err != nil {
return "", fmt.Errorf("failed to download archive %s: %w", u, err)
}
log.Printf("%s: %s => %d bytes\n", branchOrTagName, u, tarball.Len())
} }
log.Printf("%s: %s => %d bytes\n", branchOrTagName, u, tarball.Len())
log.Printf("%s: Building version %s\n", branchOrTagName, branchOrTagName) log.Printf("%s: Building version %s\n", branchOrTagName, branchOrTagName)
res, err := dockerClient.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{ res, err := dockerClient.ImageBuild(context.Background(), tarball, types.ImageBuildOptions{
Tags: []string{"dendrite-upgrade"}, Tags: []string{"dendrite-upgrade"},
@ -235,7 +260,7 @@ func calculateVersions(cli *http.Client, from, to string) []string {
semvers = semvers[i:] semvers = semvers[i:]
} }
} }
if to != "" && to != "HEAD" { if to != "" && to != HEAD {
toVer, err := semver.NewVersion(to) toVer, err := semver.NewVersion(to)
if err != nil { if err != nil {
log.Fatalf("invalid --to: %s", err) log.Fatalf("invalid --to: %s", err)
@ -253,8 +278,8 @@ func calculateVersions(cli *http.Client, from, to string) []string {
for _, sv := range semvers { for _, sv := range semvers {
versions = append(versions, sv.Original()) versions = append(versions, sv.Original())
} }
if to == "HEAD" { if to == HEAD {
versions = append(versions, "HEAD") versions = append(versions, HEAD)
} }
return versions return versions
} }
@ -328,7 +353,7 @@ func runImage(dockerClient *client.Client, volumeName, version, imageID string)
if !ok { if !ok {
return "", "", fmt.Errorf("port 8008 not exposed - exposed ports: %v", inspect.NetworkSettings.Ports) return "", "", fmt.Errorf("port 8008 not exposed - exposed ports: %v", inspect.NetworkSettings.Ports)
} }
baseURL := fmt.Sprintf("http://localhost:%s", csapiPortInfo[0].HostPort) baseURL := fmt.Sprintf("http://%s:%s", *flagDockerHost, csapiPortInfo[0].HostPort)
versionsURL := fmt.Sprintf("%s/_matrix/client/versions", baseURL) versionsURL := fmt.Sprintf("%s/_matrix/client/versions", baseURL)
// hit /versions to check it is up // hit /versions to check it is up
var lastErr error var lastErr error