Cleanup docker build output before saving (#23)

This currently contains the logic for restarting updated jobs,
but nothing exercises this logic. The logic for starting polling
for a newly created pipeline is implemented.

admin_api/admin_api.go

@@ -13,7 +13,7 @@ import (
 
 var log = logging.MustGetLogger("cursorius-server")
 
-func createSchema(db database.Database) (graphql.Schema, error) {
+func createSchema(db database.Database, pollChan chan uuid.UUID) (graphql.Schema, error) {
 	runnerType := graphql.NewObject(graphql.ObjectConfig{
 		Name:        "Runner",
 		Description: "A runner available for use inside of a pipeline.",
@@ -454,6 +454,8 @@ func createSchema(db database.Database) (graphql.Schema, error) {
 						return nil, err
 					}
 
+					pollChan <- pipeline.Id
+
 					return pipeline, nil
 				},
 			},
@@ -558,6 +560,72 @@ func createSchema(db database.Database) (graphql.Schema, error) {
 					return runner, nil
 				},
 			},
+			"updatePipeline": &graphql.Field{
+				Type:        pipelineType,
+				Description: "Create a new pipeline",
+				Args: graphql.FieldConfigArgument{
+					"pipelineId": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+					"name": &graphql.ArgumentConfig{
+						Type: graphql.String,
+					},
+					"url": &graphql.ArgumentConfig{
+						Type: graphql.String,
+					},
+					"pollInterval": &graphql.ArgumentConfig{
+						Type: graphql.Int,
+					},
+					"cloneCredentialId": &graphql.ArgumentConfig{
+						Type: graphql.String,
+					},
+				},
+				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
+					pipelineId, err := uuid.Parse(params.Args["pipelineId"].(string))
+					if err != nil {
+						return nil, err
+					}
+
+					var name *string
+					var url *string
+					var interval *int
+
+					if nameVal, ok := params.Args["name"]; ok {
+						nameVal := nameVal.(string)
+						name = &nameVal
+					} else {
+						name = nil
+					}
+
+					if urlVal, ok := params.Args["url"]; ok {
+						urlVal := urlVal.(string)
+						url = &urlVal
+					} else {
+						url = nil
+					}
+
+					if intervalVal, ok := params.Args["pollInterval"]; ok {
+						intervalVal := intervalVal.(int)
+						interval = &intervalVal
+					} else {
+						interval = nil
+					}
+
+					pipeline, err := db.UpdatePipeline(
+						pipelineId,
+						name,
+						url,
+						interval,
+					)
+					if err != nil {
+						return nil, err
+					}
+
+					pollChan <- pipeline.Id
+
+					return pipeline, nil
+				},
+			},
 			"setPipelineCloneCredential": &graphql.Field{
 				Type:        pipelineType,
 				Description: "Set the CloneCredential used by a pipeline to clone the source repo",
@@ -681,9 +749,9 @@ func createSchema(db database.Database) (graphql.Schema, error) {
 	return schema, nil
 }
 
-func CreateHandler(db database.Database, mux *http.ServeMux) error {
+func CreateHandler(db database.Database, pollChan chan uuid.UUID, mux *http.ServeMux) error {
 
-	schema, err := createSchema(db)
+	schema, err := createSchema(db, pollChan)
 	if err != nil {
 		return err
 	}

database/func.go

@@ -77,6 +77,49 @@ RETURNING id, name, url, poll_interval;`
 	return pipeline, nil
 }
 
+func (db *Database) UpdatePipeline(pipelineId uuid.UUID, name *string, url *string, pollInterval *int) (Pipeline, error) {
+	query := `
+UPDATE pipelines
+SET name=$1, url=$2, poll_interval=$3
+WHERE id=$4
+RETURNING name, url, poll_interval, clone_credential;`
+
+	pipeline, err := db.GetPipelineById(pipelineId)
+	if err != nil {
+		return pipeline, err
+	}
+
+	var nameNew string
+	var urlNew string
+	var pollIntervalNew int
+
+	if name != nil {
+		nameNew = *name
+	} else {
+		nameNew = pipeline.Name
+	}
+	if url != nil {
+		urlNew = *url
+	} else {
+		urlNew = pipeline.Url
+	}
+	if pollInterval != nil {
+		pollIntervalNew = *pollInterval
+	} else {
+		pollIntervalNew = pipeline.PollInterval
+	}
+
+	err = db.Conn.QueryRow(context.Background(),
+		query, nameNew, urlNew, pollIntervalNew, pipelineId).Scan(
+		&pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.CloneCredential,
+	)
+	if err != nil {
+		return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err)
+	}
+
+	return pipeline, err
+}
+
 func (db *Database) SetPipelineCloneCredential(pipelineId uuid.UUID, credentialId *uuid.UUID) (Pipeline, error) {
 	query := `
 UPDATE pipelines

listen/listen.go

@@ -10,6 +10,8 @@ import (
 	"git.ohea.xyz/cursorius/server/pipeline_api"
 	"git.ohea.xyz/cursorius/server/runnermanager"
 	"git.ohea.xyz/cursorius/server/webhook"
+
+	"github.com/google/uuid"
 	"github.com/op/go-logging"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
@@ -23,13 +25,14 @@ func setupHTTPServer(
 	conf config.PipelineConf,
 	db database.Database,
 	runnerManagerChans runnermanager.RunnerManagerChans,
+	pollChan chan uuid.UUID,
 ) error {
 
 	webhook.CreateWebhookHandler(db, conf, mux)
 
 	pipeline_api.CreateHandler(runnerManagerChans.Allocation, runnerManagerChans.Release, mux)
 
-	err := admin_api.CreateHandler(db, mux)
+	err := admin_api.CreateHandler(db, pollChan, mux)
 	if err != nil {
 		return fmt.Errorf("Could not create admin api handler: %w", err)
 	}
@@ -52,6 +55,7 @@ func Listen(
 	conf config.PipelineConf,
 	db database.Database,
 	runnerManagerChans runnermanager.RunnerManagerChans,
+	pollChan chan uuid.UUID,
 ) error {
 
 	err := setupHTTPServer(
@@ -59,6 +63,7 @@ func Listen(
 		conf,
 		db,
 		runnerManagerChans,
+		pollChan,
 	)
 	if err != nil {
 		return fmt.Errorf("Could not setup http endpoints: %w", err)

main.go

@@ -46,7 +46,7 @@ func main() {
 		return
 	}
 
-	_ = poll.StartPolling(configData.Config.PipelineConf, db)
+	pollChan := poll.StartPolling(configData.Config.PipelineConf, db)
 
 	mux := http.NewServeMux()
 
@@ -57,5 +57,6 @@ func main() {
 		configData.Config.PipelineConf,
 		db,
 		runnerManagerChans,
+		pollChan,
 	))
 }

pipeline_api/pipeline_api.go

@@ -56,6 +56,12 @@ func (s *ApiServer) GetRunnerFromMap(u uuid.UUID) (*RunnerWrapper, bool) {
 	return runner, ok
 }
 
+func (s *ApiServer) AddRunnerToMap(u uuid.UUID, runner *runnermanager.Runner) {
+	s.allocatedRunnersMutex.Lock()
+	defer s.allocatedRunnersMutex.Unlock()
+	s.allocatedRunners[u] = &RunnerWrapper{runner: runner}
+}
+
 func (s *ApiServer) GetRunner(
 	ctx context.Context,
 	req *connect.Request[apiv2.GetRunnerRequest],
@@ -114,9 +120,7 @@ func (s *ApiServer) GetRunner(
 
 	runnerUuid := uuid.New()
 
-	s.allocatedRunnersMutex.Lock()
-	s.allocatedRunners[runnerUuid] = &RunnerWrapper{runner: response.Runner}
-	s.allocatedRunnersMutex.Unlock()
+	s.AddRunnerToMap(runnerUuid, response.Runner)
 
 	res := connect.NewResponse(&apiv2.GetRunnerResponse{
 		Id: runnerUuid.String(),

pipeline_executor/pipeline_executor.go

@@ -4,7 +4,6 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -141,13 +140,7 @@ func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf co
 		return
 	}
 
-	response, err := ioutil.ReadAll(buildResponse.Body)
-	if err != nil {
-		log.Errorf("could no read build response: %w", err)
-		return
-	}
-
-	err = db.UpdateRunBuildOutput(pe.Run.Id, string(response))
+	err = db.UpdateRunBuildOutput(pe.Run.Id, cleanupBuildOutput(buildResponse.Body))
 	if err != nil {
 		log.Errorf("could not update build output for run: %w", err)
 		return

pipeline_executor/util.go

@@ -0,0 +1,25 @@
+package pipeline_executor
+
+import (
+	"bufio"
+	"encoding/json"
+	"io"
+)
+
+func cleanupBuildOutput(input io.ReadCloser) string {
+	output := ""
+
+	scanner := bufio.NewScanner(input)
+	for scanner.Scan() {
+		var log map[string]any
+		json.Unmarshal(scanner.Bytes(), &log)
+
+		if logVar, ok := log["stream"]; ok {
+			if log, ok := logVar.(string); ok {
+				output += log
+			}
+		}
+	}
+
+	return output
+}

poll/poll.go

@@ -1,17 +1,18 @@
 package poll
 
 import (
+	"context"
 	"time"
 
-	"github.com/google/uuid"
-	"github.com/op/go-logging"
-
 	"git.ohea.xyz/cursorius/server/config"
 	"git.ohea.xyz/cursorius/server/database"
 	"git.ohea.xyz/cursorius/server/pipeline_executor"
+
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/storage/memory"
+	"github.com/google/uuid"
+	"github.com/op/go-logging"
 )
 
 var log = logging.MustGetLogger("cursorius-server")
@@ -26,13 +27,27 @@ type tag struct {
 	commitHash string
 }
 
-func pollJob(pipeline database.Pipeline, pipelineConf config.PipelineConf, db database.Database) {
+func pollJob(ctx context.Context, pipeline database.Pipeline, pipelineConf config.PipelineConf, db database.Database) {
 	firstScan := true
 	for {
 		// Don't sleep on first scan to ease testing
 		// TODO: this should be replaced with a script that mocks a webhook
 		if !firstScan {
-			time.Sleep(time.Duration(pipeline.PollInterval) * time.Second)
+
+			ctx, cancel := context.WithTimeout(ctx, time.Duration(pipeline.PollInterval)*time.Second)
+
+			select {
+			case <-ctx.Done():
+				switch ctx.Err() {
+				case context.Canceled:
+					log.Infof("Polling for pipeline %v canceled, stopping", pipeline.Name)
+					cancel()
+					return
+				}
+			}
+
+			cancel()
+
 			log.Infof("Polling repo %v", pipeline.Name)
 		} else {
 			firstScan = false
@@ -136,11 +151,17 @@ func launchPollJobs(conf config.PipelineConf, db database.Database, pollChan cha
 		return
 	}
 
+	pipelineCancelations := make(map[uuid.UUID]context.CancelFunc)
+
 	for _, pipeline := range pipelines {
 		if pipeline.PollInterval == 0 {
 			continue
 		} else {
-			go pollJob(pipeline, conf, db)
+			ctx, cancel := context.WithCancel(context.Background())
+			pipelineCancelations[pipeline.Id] = cancel
+
+			log.Infof("Starting polling for pipeline %v with id %v", pipeline.Name, pipeline.Id)
+			go pollJob(ctx, pipeline, conf, db)
 		}
 	}
 
@@ -151,8 +172,17 @@ func launchPollJobs(conf config.PipelineConf, db database.Database, pollChan cha
 			log.Errorf("Could not get pipeline with id \"%v\" from database: %v", err)
 			continue
 		}
-		// TODO: stop existing polling process for given uuid
-		go pollJob(pipeline, conf, db)
+
+		// Cancel existing polling job if it exists
+		if cancelFunc, ok := pipelineCancelations[pipeline.Id]; ok {
+			cancelFunc()
+		}
+
+		// Start new polling job
+		log.Infof("Starting polling for pipeline %v with id %v", pipeline.Name, pipeline.Id)
+		ctx, cancel := context.WithCancel(context.Background())
+		pipelineCancelations[pipeline.Id] = cancel
+		go pollJob(ctx, pipeline, conf, db)
 	}
 
 }

runnermanager/runner.go

@@ -45,6 +45,10 @@ func (r *Runner) Id() uuid.UUID {
 
 func (r *Runner) RunCommand(cmd string, args []string) (returnCode int64, stdout string, stderr string, err error) {
 
+	if r.conn == nil {
+		return 0, "", "", fmt.Errorf("runner with id %v has nil conn, THIS IS A BUG", r.id)
+	}
+
 	// Write RunCommand message to client
 	serverToRunnerMsg := &runner_api.ServerToRunnerMsg{
 		Msg: &runner_api.ServerToRunnerMsg_RunCommandMsg{