Add admin_api and database logging of runs #9 #10

pipeline_executor/pipeline_executor.go

@@ -10,6 +10,7 @@ import (
 	"path/filepath"
 
 	"git.ohea.xyz/cursorius/server/config"
+	"git.ohea.xyz/cursorius/server/database"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
@@ -21,40 +22,44 @@ import (
 var log = logging.MustGetLogger("cursorius-server")
 
 type PipelineExecution struct {
-	Name string
-	Job  config.Job
-	Ref  string
+	Pipeline database.Pipeline
+	Ref      string
+	Run      database.Run
 }
 
-func ExecutePipeline(pe PipelineExecution, pipelineConf config.PipelineConf) error {
-	jobFolder := filepath.Join(pipelineConf.WorkingDir, pe.Name)
+func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf config.PipelineConf) {
+	jobFolder := filepath.Join(pipelineConf.WorkingDir, pe.Pipeline.Id.String(), pe.Run.Id.String())
 
-	log.Debugf("Job %v configured with URL \"%v\"", pe.Name, pe.Job.URL)
+	log.Debugf("Job %v configured with URL \"%v\"", pe.Pipeline.Name, pe.Pipeline.Url)
 
-	log.Debugf("Job %v configured with folder \"%v\"", pe.Name, jobFolder)
+	log.Debugf("Job %v configured with folder \"%v\"", pe.Pipeline.Name, jobFolder)
 
 	err := os.RemoveAll(jobFolder)
 	if err != nil {
-		return fmt.Errorf("could not delete existing folder %v", jobFolder)
+		log.Errorf("could not delete existing folder %v", jobFolder)
+		return
 	}
 
 	err = os.MkdirAll(jobFolder, 0755)
 	if err != nil {
-		return fmt.Errorf("could not create working directory for job %v: %w", pe.Name, err)
+		log.Errorf("could not create working directory for job %v: %w", pe.Pipeline.Name, err)
+		return
 	}
 
-	log.Infof("Cloning source from URL %v", pe.Job.URL)
+	log.Infof("Cloning source from URL %v", pe.Pipeline.Url)
 	// TODO: should I use go-git here instead of shelling out to raw git?
-	cloneCmd := exec.Command("git", "clone", pe.Job.URL, jobFolder)
+	cloneCmd := exec.Command("git", "clone", pe.Pipeline.Url, jobFolder)
 	output, err := cloneCmd.CombinedOutput()
 	if err != nil {
 		log.Debugf("%s", output)
-		return fmt.Errorf("could not clone source: %w", err)
+		log.Errorf("could not clone source: %w", err)
+		return
 	}
 
 	cli, err := client.NewClientWithOpts(client.FromEnv)
 	if err != nil {
-		return fmt.Errorf("Could not create docker client: %w", err)
+		log.Errorf("Could not create docker client: %w", err)
+		return
 	}
 	log.Info("Source cloned successfully")
 
@@ -65,18 +70,21 @@ func ExecutePipeline(pe PipelineExecution, pipelineConf config.PipelineConf) err
 	log.Infof("Pulling image %v", imageName)
 	pullOutput, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{})
 	if err != nil {
-		return fmt.Errorf("could not pull image %v: %w", imageName, err)
+		log.Errorf("could not pull image %v: %w", imageName, err)
+		return
 	}
 
 	buf, err := io.ReadAll(pullOutput)
 	if err != nil {
-		return fmt.Errorf("could not read from io.ReadCloser:, %w", err)
+		log.Errorf("could not read from io.ReadCloser:, %w", err)
+		return
 	}
 	log.Infof("%s", buf)
 
 	err = pullOutput.Close()
 	if err != nil {
-		return fmt.Errorf("could not close io.ReadCloser: %w", err)
+		log.Errorf("could not close io.ReadCloser: %w", err)
+		return
 	}
 	log.Info("Image pulled sucessfully")
 
@@ -87,10 +95,12 @@ func ExecutePipeline(pe PipelineExecution, pipelineConf config.PipelineConf) err
 	}
 
 	if pipelineConf.MountConf.Type == config.Bind {
+		sourceDir := filepath.Join(pipelineConf.MountConf.Source, pe.Pipeline.Id.String(), pe.Run.Id.String())
+
 		hostConfig.Mounts = append(hostConfig.Mounts,
 			mount.Mount{
 				Type:        mount.TypeBind,
-				Source:      fmt.Sprintf("%v/%v", pipelineConf.MountConf.Source, pe.Name),
+				Source:      sourceDir,
 				Target:      "/cursorius/src",
 				ReadOnly:    false,
 				Consistency: mount.ConsistencyDefault,
@@ -111,9 +121,9 @@ func ExecutePipeline(pe PipelineExecution, pipelineConf config.PipelineConf) err
 	resp, err := cli.ContainerCreate(ctx,
 		&container.Config{
 			Image: imageName,
-			Cmd:   []string{"/launcher.sh"},
 			Tty:   false,
 			Env: []string{
+				fmt.Sprintf("RUNID=%v", pe.Run.Id),
 				"CURSORIUS_SRC_DIR=/cursorius/src",
 				fmt.Sprintf("CUROSRIUS_SERVER_URL=%v", pipelineConf.AccessURL),
 			},
@@ -123,27 +133,39 @@ func ExecutePipeline(pe PipelineExecution, pipelineConf config.PipelineConf) err
 		nil, nil, "",
 	)
 	if err != nil {
-		return fmt.Errorf("could not create container: %w", err)
+		log.Errorf("could not create container: %w", err)
+		return
 	}
 
 	log.Info("Launching container")
 
 	if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
-		return fmt.Errorf("could not start container: %w", err)
+		log.Errorf("could not start container: %v", err)
+		return
 	}
 	statusCh, errCh := cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning)
 	select {
 	case err := <-errCh:
 		if err != nil {
-			return fmt.Errorf("container returned error: %w", err)
+			log.Errorf("container returned error: %v", err)
+			return
+		}
+	case okBody := <-statusCh:
+		if okBody.Error != nil {
+			log.Errorf("Could not wait on container: %v", err)
+			return
+		} else {
+			log.Debugf("Container finished running with return code: %v", okBody.StatusCode)
+			pe.Run.Result = &okBody.StatusCode
 		}
-	case retCode := <-statusCh:
-		log.Debugf("Container finished running with return code: %v", retCode)
 	}
 
+	pe.Run.InProgress = false
+
 	out, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
 	if err != nil {
-		return fmt.Errorf("could not get container logs: %w", err)
+		log.Errorf("could not get container logs: %w", err)
+		return
 	}
 
 	var stdOut bytes.Buffer
@@ -151,8 +173,10 @@ func ExecutePipeline(pe PipelineExecution, pipelineConf config.PipelineConf) err
 
 	stdcopy.StdCopy(&stdOut, &stdErr, out)
 
-	log.Debugf("%s", stdOut.Bytes())
-	log.Debugf("%s", stdErr.Bytes())
+	pe.Run.Stdout = stdOut.Bytes()
+	pe.Run.Stderr = stdErr.Bytes()
 
-	return nil
+	db.UpdateRunResult(pe.Run)
+
+	return
 }