Cleanup docker build output before saving (#23)

This currently contains the logic for restarting updated jobs,
but nothing exercises this logic. The logic for starting polling
for a newly created pipeline is implemented.

.gitignore

@@ -1,3 +1,4 @@
 _working
 server
 server.toml
+docker/docker-compose.override.yml

admin_api/admin_api.go

@@ -0,0 +1,768 @@
+package admin_api
+
+import (
+	"fmt"
+	"net/http"
+
+	"git.ohea.xyz/cursorius/server/database"
+	"github.com/google/uuid"
+	"github.com/graphql-go/graphql"
+	"github.com/graphql-go/handler"
+	"github.com/op/go-logging"
+)
+
+var log = logging.MustGetLogger("cursorius-server")
+
+func createSchema(db database.Database, pollChan chan uuid.UUID) (graphql.Schema, error) {
+	runnerType := graphql.NewObject(graphql.ObjectConfig{
+		Name:        "Runner",
+		Description: "A runner available for use inside of a pipeline.",
+		Fields: graphql.Fields{
+			"id": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The id of the runner.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if runner, ok := p.Source.(database.Runner); ok {
+						return runner.Id, nil
+					}
+					return nil, nil
+				},
+			},
+			"name": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The name of the runner.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if runner, ok := p.Source.(database.Runner); ok {
+						return runner.Name, nil
+					}
+					return nil, nil
+				},
+			},
+			"token": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The token.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if runner, ok := p.Source.(database.Runner); ok {
+						return runner.Token, nil
+					}
+					return nil, nil
+				},
+			},
+		},
+	})
+
+	secretType := graphql.NewObject(graphql.ObjectConfig{
+		Name:        "Secret",
+		Description: "A secret available for use inside of a pipeline.",
+		Fields: graphql.Fields{
+			"id": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The id of the secret.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if secret, ok := p.Source.(database.Secret); ok {
+						return secret.Id, nil
+					}
+					return nil, nil
+				},
+			},
+			"name": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The name of the secret.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if secret, ok := p.Source.(database.Secret); ok {
+						return secret.Name, nil
+					}
+					return nil, nil
+				},
+			},
+			"secret": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The secret.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if secret, ok := p.Source.(database.Secret); ok {
+						return secret.Secret, nil
+					}
+					return nil, nil
+				},
+			},
+		},
+	})
+
+	cloneCredentialType := graphql.NewObject(graphql.ObjectConfig{
+		Name:        "CloneCredential",
+		Description: "A credential for authenticating with the pipeline source host.",
+		Fields: graphql.Fields{
+			"id": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The id of the credential.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if credential, ok := p.Source.(database.CloneCredential); ok {
+						return credential.Id, nil
+					}
+					return nil, nil
+				},
+			},
+			"name": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The name of the credential.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if credential, ok := p.Source.(database.CloneCredential); ok {
+						return credential.Name, nil
+					}
+					return nil, nil
+				},
+			},
+			"type": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The credential type.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if credential, ok := p.Source.(database.CloneCredential); ok {
+						return credential.Type, nil
+					}
+					return nil, nil
+				},
+			},
+			"username": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The username to user with the credential.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if credential, ok := p.Source.(database.CloneCredential); ok {
+						return credential.Username, nil
+					}
+					return nil, nil
+				},
+			},
+			"secret": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The secret for the credential.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if credential, ok := p.Source.(database.CloneCredential); ok {
+						return credential.Secret, nil
+					}
+					return nil, nil
+				},
+			},
+		},
+	})
+
+	webhookType := graphql.NewObject(graphql.ObjectConfig{
+		Name:        "Webhook",
+		Description: "A webhook for triggering pipelines",
+		Fields: graphql.Fields{
+			"id": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The id of the webhook.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if webhook, ok := p.Source.(database.Webhook); ok {
+						return webhook.Id, nil
+					}
+					return nil, nil
+				},
+			},
+			"serverType": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The format of the webhook.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if webhook, ok := p.Source.(database.Webhook); ok {
+						return webhook.ServerType, nil
+					}
+					return nil, nil
+				},
+			},
+			"secret": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The secret used to validate the webhook.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if webhook, ok := p.Source.(database.Webhook); ok {
+						return webhook.Secret, nil
+					}
+					return nil, nil
+				},
+			},
+		},
+	})
+
+	runType := graphql.NewObject(graphql.ObjectConfig{
+		Name:        "Run",
+		Description: "A pipeline run",
+		Fields: graphql.Fields{
+			"id": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The id of the run.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if run, ok := p.Source.(database.Run); ok {
+						return run.Id, nil
+					}
+					return nil, nil
+				},
+			},
+			"inProgress": &graphql.Field{
+				Type:        graphql.Boolean,
+				Description: "The progress status of the run.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if run, ok := p.Source.(database.Run); ok {
+						return run.InProgress, nil
+					}
+					return nil, nil
+				},
+			},
+			"result": &graphql.Field{
+				// TODO: handle bigint properly here
+				Type:        graphql.Float,
+				Description: "The result of the run.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if run, ok := p.Source.(database.Run); ok {
+						return run.Result, nil
+					}
+					return nil, nil
+				},
+			},
+			"buildOutput": &graphql.Field{
+				Type:        graphql.String,
+				Description: "Logs of the top level container build for the run.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if run, ok := p.Source.(database.Run); ok {
+						return string(run.BuildOutput), nil
+					}
+					return nil, nil
+				},
+			},
+			"stdout": &graphql.Field{
+				Type:        graphql.String,
+				Description: "The stdout used to validate the run.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if run, ok := p.Source.(database.Run); ok {
+						return string(run.Stdout), nil
+					}
+					return nil, nil
+				},
+			},
+			"stderr": &graphql.Field{
+				Type:        graphql.String,
+				Description: "The stderr used to validate the run.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if run, ok := p.Source.(database.Run); ok {
+						return string(run.Stderr), nil
+					}
+					return nil, nil
+				},
+			},
+		},
+	})
+
+	pipelineType := graphql.NewObject(graphql.ObjectConfig{
+		Name:        "Pipeline",
+		Description: "A pipeline for running ci jobs",
+		Fields: graphql.Fields{
+			"id": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The id of the pipeline.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if pipeline, ok := p.Source.(database.Pipeline); ok {
+						return pipeline.Id, nil
+					}
+					return nil, nil
+				},
+			},
+			"name": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The name of the pipeline.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if pipeline, ok := p.Source.(database.Pipeline); ok {
+						return pipeline.Name, nil
+					}
+					return nil, nil
+				},
+			},
+			"url": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The url of the pipeline.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if pipeline, ok := p.Source.(database.Pipeline); ok {
+						return pipeline.Url, nil
+					}
+					return nil, nil
+				},
+			},
+			"pollInterval": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.Int),
+				Description: "The polling interval for the pipeline.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if pipeline, ok := p.Source.(database.Pipeline); ok {
+						return pipeline.PollInterval, nil
+					}
+					return nil, nil
+				},
+			},
+			"cloneCredential": &graphql.Field{
+				Type:        cloneCredentialType,
+				Description: "The configured credential for cloning the pipeline source.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if pipeline, ok := p.Source.(database.Pipeline); ok {
+						if pipeline.CloneCredential != nil {
+							return db.GetCloneCredentialById(*pipeline.CloneCredential)
+						}
+					}
+					return nil, nil
+				},
+			},
+			"secrets": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.NewList(graphql.NewNonNull(secretType))),
+				Description: "The list of secrets for the pipeline.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if pipeline, ok := p.Source.(database.Pipeline); ok {
+						return db.GetSecretsForPipeline(pipeline.Id)
+					}
+					return []database.Secret{}, nil
+				},
+			},
+			"webhooks": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.NewList(graphql.NewNonNull(webhookType))),
+				Description: "The list of webhooks for the pipeline.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if pipeline, ok := p.Source.(database.Pipeline); ok {
+						return db.GetWebhooksForPipeline(pipeline.Id)
+					}
+					return []database.Webhook{}, nil
+				},
+			},
+			"runs": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.NewList(graphql.NewNonNull(runType))),
+				Description: "The list of runs for the pipeline.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if pipeline, ok := p.Source.(database.Pipeline); ok {
+						return db.GetRunsForPipeline(pipeline.Id)
+					}
+					return []database.Webhook{}, nil
+				},
+			},
+		},
+	})
+
+	queryType := graphql.NewObject(graphql.ObjectConfig{
+		Name: "Query",
+		Fields: graphql.Fields{
+			"Pipeline": &graphql.Field{
+				Type: pipelineType,
+				Args: graphql.FieldConfigArgument{
+					"id": &graphql.ArgumentConfig{
+						Type:        graphql.NewNonNull(graphql.String),
+						Description: "The id of the requested pipeline.",
+					},
+				},
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					id, err := uuid.Parse(p.Args["id"].(string))
+					if err != nil {
+						return nil, err
+					}
+					return db.GetPipelineById(id)
+				},
+			},
+			"Pipelines": &graphql.Field{
+				Type: graphql.NewNonNull(graphql.NewList(pipelineType)),
+				Args: graphql.FieldConfigArgument{},
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					return db.GetPipelines()
+				},
+			},
+			"CloneCredential": &graphql.Field{
+				Type: cloneCredentialType,
+				Args: graphql.FieldConfigArgument{
+					"id": &graphql.ArgumentConfig{
+						Type:        graphql.NewNonNull(graphql.String),
+						Description: "The id of the requested credential.",
+					},
+				},
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					id, err := uuid.Parse(p.Args["id"].(string))
+					if err != nil {
+						return nil, err
+					}
+					return db.GetCloneCredentialById(id)
+				},
+			},
+			"CloneCredentials": &graphql.Field{
+				Type: graphql.NewNonNull(graphql.NewList(cloneCredentialType)),
+				Args: graphql.FieldConfigArgument{},
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					return db.GetCredentials()
+				},
+			},
+			"Secrets": &graphql.Field{
+				Type: graphql.NewNonNull(graphql.NewList(secretType)),
+				Args: graphql.FieldConfigArgument{},
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					return db.GetSecrets()
+				},
+			},
+			"Runners": &graphql.Field{
+				Type: graphql.NewNonNull(graphql.NewList(runnerType)),
+				Args: graphql.FieldConfigArgument{},
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					return db.GetRunners()
+				},
+			},
+		},
+	})
+
+	mutationType := graphql.NewObject(graphql.ObjectConfig{
+		Name: "Mutation",
+		Fields: graphql.Fields{
+			"createPipeline": &graphql.Field{
+				Type:        pipelineType,
+				Description: "Create a new pipeline",
+				Args: graphql.FieldConfigArgument{
+					"name": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+					"url": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+					"pollInterval": &graphql.ArgumentConfig{
+						Type: graphql.Int,
+					},
+					"cloneCredentialId": &graphql.ArgumentConfig{
+						Type: graphql.String,
+					},
+				},
+				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
+					var interval int
+					if intervalVal, ok := params.Args["pollInterval"]; ok {
+						interval = intervalVal.(int)
+					} else {
+						interval = 0
+					}
+
+					var credential *uuid.UUID
+					if credentialVal, ok := params.Args["cloneCredentialId"]; ok {
+						id, err := uuid.Parse(credentialVal.(string))
+						if err != nil {
+							return nil, err
+						}
+						credential = &id
+					} else {
+						credential = nil
+					}
+
+					pipeline, err := db.CreatePipeline(
+						params.Args["name"].(string),
+						params.Args["url"].(string),
+						interval,
+						credential,
+					)
+					if err != nil {
+						return nil, err
+					}
+
+					pollChan <- pipeline.Id
+
+					return pipeline, nil
+				},
+			},
+			"createWebhook": &graphql.Field{
+				Type:        webhookType,
+				Description: "Create a new webhook",
+				Args: graphql.FieldConfigArgument{
+					"type": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+					"pipelineId": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+				},
+				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
+
+					id, err := uuid.Parse(params.Args["id"].(string))
+					if err != nil {
+						return nil, err
+					}
+
+					webhook, err := db.CreateWebhook(
+						database.WebhookSender(params.Args["type"].(string)),
+						id,
+					)
+					if err != nil {
+						return nil, err
+					}
+					return webhook, nil
+				},
+			},
+			"createCloneCredential": &graphql.Field{
+				Type:        cloneCredentialType,
+				Description: "Create a new CloneCredential",
+				Args: graphql.FieldConfigArgument{
+					"name": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+					"type": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+					"username": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+					"secret": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+				},
+				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
+
+					credential, err := db.CreateCredential(
+						params.Args["name"].(string),
+						database.CloneCredentialType(params.Args["type"].(string)),
+						params.Args["username"].(string),
+						params.Args["secret"].(string),
+					)
+					if err != nil {
+						return nil, err
+					}
+					return credential, nil
+				},
+			},
+			"createSecret": &graphql.Field{
+				Type:        secretType,
+				Description: "Create a new secret",
+				Args: graphql.FieldConfigArgument{
+					"name": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+					"secret": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+				},
+				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
+
+					secret, err := db.CreateSecret(
+						params.Args["name"].(string),
+						params.Args["secret"].(string),
+					)
+					if err != nil {
+						return nil, err
+					}
+					return secret, nil
+				},
+			},
+			"createRunner": &graphql.Field{
+				Type:        runnerType,
+				Description: "Create a new runner",
+				Args: graphql.FieldConfigArgument{
+					"name": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+				},
+				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
+
+					runner, err := db.CreateRunner(
+						params.Args["name"].(string),
+					)
+					if err != nil {
+						return nil, err
+					}
+					return runner, nil
+				},
+			},
+			"updatePipeline": &graphql.Field{
+				Type:        pipelineType,
+				Description: "Create a new pipeline",
+				Args: graphql.FieldConfigArgument{
+					"pipelineId": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+					"name": &graphql.ArgumentConfig{
+						Type: graphql.String,
+					},
+					"url": &graphql.ArgumentConfig{
+						Type: graphql.String,
+					},
+					"pollInterval": &graphql.ArgumentConfig{
+						Type: graphql.Int,
+					},
+					"cloneCredentialId": &graphql.ArgumentConfig{
+						Type: graphql.String,
+					},
+				},
+				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
+					pipelineId, err := uuid.Parse(params.Args["pipelineId"].(string))
+					if err != nil {
+						return nil, err
+					}
+
+					var name *string
+					var url *string
+					var interval *int
+
+					if nameVal, ok := params.Args["name"]; ok {
+						nameVal := nameVal.(string)
+						name = &nameVal
+					} else {
+						name = nil
+					}
+
+					if urlVal, ok := params.Args["url"]; ok {
+						urlVal := urlVal.(string)
+						url = &urlVal
+					} else {
+						url = nil
+					}
+
+					if intervalVal, ok := params.Args["pollInterval"]; ok {
+						intervalVal := intervalVal.(int)
+						interval = &intervalVal
+					} else {
+						interval = nil
+					}
+
+					pipeline, err := db.UpdatePipeline(
+						pipelineId,
+						name,
+						url,
+						interval,
+					)
+					if err != nil {
+						return nil, err
+					}
+
+					pollChan <- pipeline.Id
+
+					return pipeline, nil
+				},
+			},
+			"setPipelineCloneCredential": &graphql.Field{
+				Type:        pipelineType,
+				Description: "Set the CloneCredential used by a pipeline to clone the source repo",
+				Args: graphql.FieldConfigArgument{
+					"cloneCredentialId": &graphql.ArgumentConfig{
+						Type: graphql.String,
+					},
+					"pipelineId": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+				},
+				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
+
+					pipelineId, err := uuid.Parse(params.Args["pipelineId"].(string))
+					if err != nil {
+						return nil, err
+					}
+
+					if cloneCredentialIdVal, ok := params.Args["cloneCredentialId"]; ok {
+						cloneCredentialId, err := uuid.Parse(cloneCredentialIdVal.(string))
+						if err != nil {
+							return nil, err
+						}
+
+						pipeline, err := db.SetPipelineCloneCredential(pipelineId, &cloneCredentialId)
+						if err != nil {
+							return nil, err
+						}
+						return pipeline, nil
+					} else {
+						pipeline, err := db.SetPipelineCloneCredential(pipelineId, nil)
+						if err != nil {
+							return nil, err
+						}
+						return pipeline, nil
+					}
+
+				},
+			},
+			"addSecretToPipeline": &graphql.Field{
+				Type:        pipelineType,
+				Description: "Allow a secret to be accessed by a pipeline.",
+				Args: graphql.FieldConfigArgument{
+					"secretId": &graphql.ArgumentConfig{
+						Type: graphql.String,
+					},
+					"pipelineId": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+				},
+				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
+
+					secretId, err := uuid.Parse(params.Args["secretId"].(string))
+					if err != nil {
+						return nil, err
+					}
+
+					pipelineId, err := uuid.Parse(params.Args["pipelineId"].(string))
+					if err != nil {
+						return nil, err
+					}
+
+					err = db.AssignSecretToPipeline(pipelineId, secretId)
+					if err != nil {
+						return nil, err
+					}
+
+					pipeline, err := db.GetPipelineById(pipelineId)
+					if err != nil {
+						return nil, err
+					}
+					return pipeline, nil
+				},
+			},
+			"removeSecretFromPipeline": &graphql.Field{
+				Type:        pipelineType,
+				Description: "Remove a pipeline's access to a secret.",
+				Args: graphql.FieldConfigArgument{
+					"secretId": &graphql.ArgumentConfig{
+						Type: graphql.String,
+					},
+					"pipelineId": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+				},
+				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
+
+					secretId, err := uuid.Parse(params.Args["secretId"].(string))
+					if err != nil {
+						return nil, err
+					}
+
+					pipelineId, err := uuid.Parse(params.Args["pipelineId"].(string))
+					if err != nil {
+						return nil, err
+					}
+
+					err = db.RemoveSecretFromPipeline(pipelineId, secretId)
+					if err != nil {
+						return nil, err
+					}
+
+					pipeline, err := db.GetPipelineById(pipelineId)
+					if err != nil {
+						return nil, err
+					}
+					return pipeline, nil
+				},
+			},
+		},
+	})
+
+	schema, err := graphql.NewSchema(graphql.SchemaConfig{
+		Query:    queryType,
+		Mutation: mutationType,
+	})
+	if err != nil {
+		return schema, fmt.Errorf("Could not create schema: %w", err)
+	}
+
+	return schema, nil
+}
+
+func CreateHandler(db database.Database, pollChan chan uuid.UUID, mux *http.ServeMux) error {
+
+	schema, err := createSchema(db, pollChan)
+	if err != nil {
+		return err
+	}
+
+	h := handler.New(&handler.Config{
+		Schema:   &schema,
+		Pretty:   true,
+		GraphiQL: true,
+	})
+
+	mux.Handle("/graphql", h)
+
+	return nil
+}

config/config.go

@@ -6,28 +6,18 @@ import (
 	"git.ohea.xyz/golang/config"
 )
 
-type WebhookSender string
-
-const (
-	Gitea WebhookSender = "gitea"
-)
-
-type Webhook struct {
-	Sender WebhookSender
-	Secret string
-}
-
-type Job struct {
-	URL          string
-	Webhook      *Webhook
-	Cron         *string
-	PollInterval uint64
-}
-
 type Runner struct {
 	Secret string
 }
 
+type DBConfig struct {
+	Address  string
+	Port     int
+	Username string
+	Password string
+	Name     string
+}
+
 type MountType string
 
 const (
@@ -50,8 +40,8 @@ type PipelineConf struct {
 type Config struct {
 	Address      string
 	Port         int
+	DBConfig     DBConfig
 	PipelineConf PipelineConf
-	Jobs         map[string]Job
 	Runners      map[string]Runner
 }
 
@@ -63,6 +53,13 @@ func GetConfig() (config.Config[Config], error) {
 		Config: Config{
 			Address: "127.0.0.1",
 			Port:    45420,
+			DBConfig: DBConfig{
+				Address:  "DB_ADDRESS",
+				Port:     5432,
+				Username: "USERNAME",
+				Password: "PASSWORD",
+				Name:     "cursorius",
+			},
 			PipelineConf: PipelineConf{
 				AccessURL:     "cursorius-server:45420",
 				DockerNetwork: &defaultNetworkName,
@@ -72,7 +69,6 @@ func GetConfig() (config.Config[Config], error) {
 					Source: "/opt/cursorius/working",
 				},
 			},
-			Jobs:    make(map[string]Job),
 			Runners: make(map[string]Runner),
 		},
 	}

database/db.go

@@ -0,0 +1,200 @@
+package database
+
+import (
+	"time"
+
+	"git.ohea.xyz/cursorius/server/config"
+
+	"context"
+	"fmt"
+
+	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/op/go-logging"
+)
+
+var log = logging.MustGetLogger("cursorius-server")
+
+type Database struct {
+	Conn *pgxpool.Pool
+}
+
+func LaunchDB(conf config.DBConfig) (Database, error) {
+
+	dbURL := fmt.Sprintf(
+		"postgres://%v:%v@%v:%v/%v",
+		conf.Username,
+		conf.Password,
+		conf.Address,
+		conf.Port,
+		conf.Name,
+	)
+
+	dbURLNoPasswd := fmt.Sprintf(
+		"postgres://%v:********@%v:%v/%v",
+		conf.Username,
+		conf.Address,
+		conf.Port,
+		conf.Name,
+	)
+
+	db := Database{}
+
+	var err error
+	log.Infof("Connecting to database with URL \"%v\"", dbURLNoPasswd)
+	db.Conn, err = pgxpool.New(context.Background(), dbURL)
+	if err != nil {
+		return db, fmt.Errorf("could not create database pool: %w", err)
+	}
+
+	// sleep until we can sucessfully acquire a connection
+	for i := 0; i < 10; i++ {
+		_, err = db.Conn.Acquire(context.Background())
+		if err == nil {
+			break
+		}
+		time.Sleep(2 * time.Second)
+
+	}
+	if err != nil {
+		return db, fmt.Errorf("Could not open database: %w", err)
+	}
+
+	log.Infof("Database connected sucessfully!")
+
+	versionTableExistsQuery := `
+SELECT EXISTS (
+    SELECT FROM pg_tables
+    WHERE tablename = 'version'
+);`
+
+	var versionTableExists bool
+	err = db.Conn.QueryRow(context.Background(), versionTableExistsQuery).Scan(&versionTableExists)
+	if err != nil {
+		return db, fmt.Errorf("Could not check if database was initalized: %w", err)
+	}
+
+	if versionTableExists {
+		// TODO: migrations
+	} else {
+		log.Info("New database found, initializing....")
+		err = initDB(db.Conn)
+		if err != nil {
+			return db, fmt.Errorf("Could not initalize database: %w", err)
+		}
+	}
+
+	return db, nil
+}
+
+func initDB(conn *pgxpool.Pool) error {
+	createTablesQuery := `
+CREATE EXTENSION IF NOT EXISTS "uuid-ossp";
+
+CREATE TABLE version (
+    version INT NOT NULL
+
+);
+
+CREATE TABLE clone_credentials (
+	id		  UUID PRIMARY KEY,
+	name      TEXT NOT NULL,
+    type      TEXT NOT NULL,
+	username  TEXT NOT NULL,
+	secret    TEXT NOT NULL
+);
+
+CREATE TABLE pipelines (
+    id                    UUID PRIMARY KEY,
+    name                  TEXT NOT NULL,
+    url                   TEXT NOT NULL,
+    poll_interval         INTEGER,
+    clone_credential      UUID DEFAULT NULL,
+
+    CONSTRAINT fk_clone_credential
+        FOREIGN KEY(clone_credential)
+            REFERENCES clone_credentials(id)
+);
+
+CREATE TABLE secrets (
+	id	   UUID PRIMARY KEY,
+    name   TEXT NOT NULL UNIQUE,
+    secret TEXT NOT NULL
+);
+
+CREATE TABLE pipeline_secret_mappings (
+    pipeline    UUID NOT NULL,
+	secret      UUID NOT NULL,
+
+    CONSTRAINT fk_pipeline
+        FOREIGN KEY(pipeline)
+            REFERENCES pipelines(id),
+
+    CONSTRAINT fk_secret
+        FOREIGN KEY(secret)
+            REFERENCES secrets(id)
+);
+
+CREATE TABLE webhooks (
+    id            UUID PRIMARY KEY,
+    server_type	  TEXT,
+    secret        TEXT,
+    pipeline      UUID,
+
+    CONSTRAINT fk_pipeline
+        FOREIGN KEY(pipeline)
+            REFERENCES pipelines(id)
+);
+
+CREATE TABLE runs (
+    id            UUID PRIMARY KEY,
+    pipeline      UUID,
+    in_progress   BOOLEAN DEFAULT NULL,
+	build_output  TEXT DEFAULT NULL,
+    result        BIGINT DEFAULT NULL,
+    stdout        TEXT DEFAULT NULL,
+    stderr        TEXT DEFAULT NULL,
+
+    CONSTRAINT fk_pipeline
+       FOREIGN KEY(pipeline)
+           REFERENCES pipelines(id)
+);
+
+CREATE TABLE command_executions (
+    id          UUID PRIMARY KEY,
+    run_id      UUID,
+    command     TEXT,
+    return_code INT,
+    stdout      TEXT,
+    stderr      TEXT,
+    start_time  TIMESTAMP,
+    end_time    TIMESTAMP,
+
+    CONSTRAINT fk_run_id
+        FOREIGN KEY(run_id)
+            REFERENCES runs(id)
+);
+
+CREATE TABLE runners (
+    id      UUID PRIMARY KEY,
+    name    TEXT NOT NULL UNIQUE,
+    token   TEXT NOT NULL
+);
+
+CREATE TABLE pipeline_refs (
+	name			TEXT PRIMARY KEY NOT NULL,
+	pipeline_id 	UUID NOT NULL,
+	hash			TEXT NOT NULL,
+
+    CONSTRAINT fk_pipeline_id
+        FOREIGN KEY(pipeline_id)
+            REFERENCES pipelines(id)
+);
+`
+
+	_, err := conn.Exec(context.Background(), createTablesQuery)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

database/func.go

@@ -0,0 +1,672 @@
+package database
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+
+	"github.com/google/uuid"
+)
+
+func (db *Database) GetPipelines() ([]Pipeline, error) {
+	query := `
+SELECT id, name, url, poll_interval, clone_credential
+FROM pipelines;`
+
+	pipelines := make([]Pipeline, 0)
+
+	rows, err := db.Conn.Query(context.Background(), query)
+	if err != nil {
+		return pipelines, fmt.Errorf("Could not query database for pipelines: %w", err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var pipeline Pipeline
+		var idStr string
+		if err := rows.Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.CloneCredential); err != nil {
+			return pipelines, err
+		}
+
+		pipeline.Id, err = uuid.Parse(idStr)
+		if err != nil {
+			return pipelines, err
+		}
+		pipelines = append(pipelines, pipeline)
+	}
+
+	return pipelines, nil
+}
+
+func (db *Database) GetPipelineById(id uuid.UUID) (Pipeline, error) {
+	query := `
+SELECT name, url, poll_interval
+FROM pipelines
+WHERE id=$1;`
+
+	pipeline := Pipeline{
+		Id: id,
+	}
+
+	err := db.Conn.QueryRow(context.Background(), query, id).Scan(&pipeline.Name, &pipeline.Url, &pipeline.PollInterval)
+	if err != nil {
+		return pipeline, fmt.Errorf("Could not query database for pipeline with id %v: %w", id.String(), err)
+	}
+
+	return pipeline, nil
+}
+
+func (db *Database) CreatePipeline(name string, url string, pollInterval int, credential *uuid.UUID) (Pipeline, error) {
+	query := `
+INSERT INTO pipelines (id, name, url, poll_interval, clone_credential)
+VALUES (uuid_generate_v4(), $1, $2, $3, $4)
+RETURNING id, name, url, poll_interval;`
+
+	pipeline := Pipeline{}
+	var idStr string
+	err := db.Conn.QueryRow(context.Background(), query, name, url, pollInterval, credential).Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval)
+	if err != nil {
+		return pipeline, fmt.Errorf("Could not create pipeline: %w", err)
+	}
+
+	pipeline.Id, err = uuid.Parse(idStr)
+	if err != nil {
+		return pipeline, fmt.Errorf("Could not parse UUID generated by DB: %w", err)
+	}
+
+	return pipeline, nil
+}
+
+func (db *Database) UpdatePipeline(pipelineId uuid.UUID, name *string, url *string, pollInterval *int) (Pipeline, error) {
+	query := `
+UPDATE pipelines
+SET name=$1, url=$2, poll_interval=$3
+WHERE id=$4
+RETURNING name, url, poll_interval, clone_credential;`
+
+	pipeline, err := db.GetPipelineById(pipelineId)
+	if err != nil {
+		return pipeline, err
+	}
+
+	var nameNew string
+	var urlNew string
+	var pollIntervalNew int
+
+	if name != nil {
+		nameNew = *name
+	} else {
+		nameNew = pipeline.Name
+	}
+	if url != nil {
+		urlNew = *url
+	} else {
+		urlNew = pipeline.Url
+	}
+	if pollInterval != nil {
+		pollIntervalNew = *pollInterval
+	} else {
+		pollIntervalNew = pipeline.PollInterval
+	}
+
+	err = db.Conn.QueryRow(context.Background(),
+		query, nameNew, urlNew, pollIntervalNew, pipelineId).Scan(
+		&pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.CloneCredential,
+	)
+	if err != nil {
+		return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err)
+	}
+
+	return pipeline, err
+}
+
+func (db *Database) SetPipelineCloneCredential(pipelineId uuid.UUID, credentialId *uuid.UUID) (Pipeline, error) {
+	query := `
+UPDATE pipelines
+SET clone_credential=$1
+WHERE id=$2
+RETURNING name, url, poll_interval, clone_credential;`
+
+	pipeline := Pipeline{
+		Id: pipelineId,
+	}
+
+	err := db.Conn.QueryRow(context.Background(),
+		query, credentialId, pipelineId).Scan(
+		&pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.CloneCredential,
+	)
+	if err != nil {
+		return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err)
+	}
+
+	return pipeline, err
+}
+
+func (db *Database) RemovePipelineCredential(pipelineId uuid.UUID) (Pipeline, error) {
+	query := `
+UPDATE pipelines
+SET credential=null
+WHERE id=$1
+RETURNING name, url, poll_interval, clone_credential;`
+
+	pipeline := Pipeline{
+		Id: pipelineId,
+	}
+
+	err := db.Conn.QueryRow(context.Background(),
+		query, pipelineId).Scan(
+		&pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.CloneCredential,
+	)
+	if err != nil {
+		return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err)
+	}
+
+	return pipeline, err
+}
+
+func (db *Database) GetWebhooksForPipeline(id uuid.UUID) ([]Webhook, error) {
+	query := `
+SELECT id, server_type, secret
+FROM webhooks
+WHERE pipeline=$1;`
+
+	webhooks := make([]Webhook, 0)
+
+	rows, err := db.Conn.Query(context.Background(), query, id)
+	if err != nil {
+		return webhooks, fmt.Errorf("Could not get webhooks for pipeline with id \"%v\": %w", id, err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var webhook Webhook
+		var idStr string
+		if err := rows.Scan(&idStr, &webhook.ServerType, &webhook.Secret); err != nil {
+			return webhooks, err
+		}
+
+		webhook.Id, err = uuid.Parse(idStr)
+		if err != nil {
+			return webhooks, err
+		}
+		webhooks = append(webhooks, webhook)
+	}
+
+	return webhooks, nil
+}
+
+func (db *Database) GetWebhookById(id uuid.UUID) (Webhook, error) {
+	query := `
+SELECT server_type, secret, pipeline
+FROM webhooks
+WHERE id=$1;`
+
+	webhook := Webhook{
+		Id: id,
+	}
+
+	err := db.Conn.QueryRow(context.Background(), query, id).Scan(&webhook.ServerType, &webhook.Secret, &webhook.Pipeline)
+	if err != nil {
+		return webhook, fmt.Errorf("Could not query database for webhook with id %v: %w", id.String(), err)
+	}
+
+	return webhook, nil
+}
+
+func (db *Database) CreateWebhook(serverType WebhookSender, pipelineId uuid.UUID) (Webhook, error) {
+
+	query := `
+INSERT INTO webhooks (id, server_type, secret, pipeline)
+VALUES (uuid_generate_v4(), $1, (select substr(md5(random()::text), 0, 50)), $2)
+RETURNING id, server_type, secret, pipeline;`
+
+	webhook := Webhook{}
+	var idStr string
+	err := db.Conn.QueryRow(context.Background(), query, string(serverType), pipelineId).Scan(&idStr, &webhook.ServerType, &webhook.Secret, &webhook.Pipeline)
+	if err != nil {
+		return webhook, err
+	}
+
+	id, err := uuid.Parse(idStr)
+	if err != nil {
+		return webhook, err
+	}
+
+	webhook.Id = id
+
+	return webhook, nil
+}
+
+func (db *Database) CreateCredential(name string, credentialtype CloneCredentialType, username string, secret string) (CloneCredential, error) {
+	query := `
+INSERT INTO clone_credentials (id, name, type, username, secret)
+VALUES(uuid_generate_v4(), $1, $2, $3, $4)
+RETURNING id, name, type, username, secret;`
+
+	credential := CloneCredential{}
+	var idStr string
+	err := db.Conn.QueryRow(
+		context.Background(),
+		query,
+		name,
+		string(credentialtype),
+		username,
+		secret,
+	).Scan(&idStr, &credential.Name, &credential.Type, &credential.Username, &credential.Secret)
+	if err != nil {
+		return credential, err
+	}
+
+	id, err := uuid.Parse(idStr)
+	if err != nil {
+		return credential, err
+	}
+
+	credential.Id = id
+
+	return credential, nil
+}
+
+func (db *Database) GetCloneCredentialById(id uuid.UUID) (CloneCredential, error) {
+	query := `
+SELECT name, type, username, secret
+FROM clone_credentials
+WHERE id=$1;`
+
+	log.Debugf("requested credential with id %v", id)
+
+	credential := CloneCredential{
+		Id: id,
+	}
+
+	err := db.Conn.QueryRow(context.Background(), query, id).Scan(&credential.Name, &credential.Type, &credential.Username, &credential.Secret)
+	if err != nil {
+		return credential, fmt.Errorf("Could not query database for credential with id %v: %w", id.String(), err)
+	}
+
+	return credential, nil
+}
+
+func (db *Database) GetCredentials() ([]CloneCredential, error) {
+	query := `
+SELECT id, name, type, username, secret
+FROM clone_credentials;`
+
+	credentials := make([]CloneCredential, 0)
+
+	rows, err := db.Conn.Query(context.Background(), query)
+	if err != nil {
+		return credentials, fmt.Errorf("Could not query database for credentials: %w", err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var credential CloneCredential
+		var idStr string
+		if err := rows.Scan(&idStr, &credential.Name, &credential.Type, &credential.Username, &credential.Secret); err != nil {
+			return credentials, err
+		}
+
+		credential.Id, err = uuid.Parse(idStr)
+		if err != nil {
+			return credentials, err
+		}
+		credentials = append(credentials, credential)
+	}
+
+	return credentials, nil
+}
+
+func (db *Database) CreateRun(pipelineId uuid.UUID) (Run, error) {
+	query := `
+INSERT INTO runs (id, pipeline, in_progress)
+VALUES(uuid_generate_v4(), $1, true)
+RETURNING id, pipeline, in_progress;`
+
+	run := Run{}
+	var idStr string
+	err := db.Conn.QueryRow(context.Background(), query, pipelineId).Scan(&idStr, &run.Pipeline, &run.InProgress)
+	if err != nil {
+		return run, err
+	}
+
+	run.Id, err = uuid.Parse(idStr)
+	if err != nil {
+		return run, err
+	}
+
+	return run, nil
+}
+
+func (db *Database) UpdateRunBuildOutput(runId uuid.UUID, buildResult string) error {
+	query := `
+UPDATE runs
+SET build_output=$1
+WHERE id=$2;`
+
+	_, err := db.Conn.Exec(context.Background(),
+		query, buildResult, runId)
+
+	return err
+
+}
+
+func (db *Database) UpdateRunResult(r Run) error {
+	query := `
+UPDATE runs
+SET in_progress=$1, result=$2, stdout=$3, stderr=$4
+WHERE id=$5;`
+
+	// TODO: does r.Result need a pointer derefrence?
+	_, err := db.Conn.Exec(context.Background(),
+		query, r.InProgress, r.Result, r.Stdout, r.Stderr, r.Id)
+
+	return err
+}
+
+func (db *Database) GetRunsForPipeline(pipelineId uuid.UUID) ([]Run, error) {
+	query := `
+SELECT id, in_progress, result, build_output, stdout, stderr
+FROM runs
+WHERE pipeline=$1;`
+
+	runs := make([]Run, 0)
+
+	rows, err := db.Conn.Query(context.Background(), query, pipelineId)
+	if err != nil {
+		return runs, fmt.Errorf("Could not get runs for pipeline with id \"%v\": %w", pipelineId, err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var run Run
+		var idStr string
+		if err := rows.Scan(
+			&idStr,
+			&run.InProgress,
+			&run.Result,
+			&run.BuildOutput,
+			&run.Stdout,
+			&run.Stderr,
+		); err != nil {
+			return runs, err
+		}
+
+		run.Id, err = uuid.Parse(idStr)
+		if err != nil {
+			return runs, err
+		}
+		runs = append(runs, run)
+	}
+
+	return runs, nil
+}
+
+func (db *Database) GetPipelineRefs(pipelineId uuid.UUID) (map[string]string, error) {
+	query := `
+SELECT name, hash
+FROM pipeline_refs
+WHERE pipeline_id=$1;`
+
+	refsMap := make(map[string]string)
+
+	refs, err := db.Conn.Query(context.Background(), query, pipelineId)
+	if err != nil {
+		return refsMap, fmt.Errorf("Could not get pipeline refs for pipeline with id \"%v\": %w", pipelineId, err)
+	}
+	defer refs.Close()
+
+	for refs.Next() {
+		var name string
+		var hash string
+		if err := refs.Scan(
+			&name,
+			&hash,
+		); err != nil {
+			return refsMap, err
+		}
+
+		refsMap[name] = hash
+	}
+
+	return refsMap, nil
+}
+
+func (db *Database) UpdatePipelineRefs(pipelineId uuid.UUID, refsMap map[string]string) error {
+
+	query := `
+INSERT INTO pipeline_refs(name, pipeline_id, hash)
+VALUES($1, $2, $3)
+ON CONFLICT (name)
+DO
+  UPDATE SET hash=$3;`
+
+	for name, hash := range refsMap {
+		_, err := db.Conn.Exec(context.Background(), query, name, pipelineId, hash)
+
+		return err
+	}
+	return nil
+}
+
+func (db *Database) GetSecrets() ([]Secret, error) {
+	query := `
+SELECT id, name, secret
+FROM secrets;`
+
+	secrets := make([]Secret, 0)
+
+	rows, err := db.Conn.Query(context.Background(), query)
+	if err != nil {
+		return secrets, fmt.Errorf("Could not query database for secrets: %w", err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var secret Secret
+		var idStr string
+		if err := rows.Scan(&idStr, &secret.Name, &secret.Secret); err != nil {
+			return secrets, err
+		}
+
+		secret.Id, err = uuid.Parse(idStr)
+		if err != nil {
+			return secrets, err
+		}
+		secrets = append(secrets, secret)
+	}
+
+	return secrets, nil
+}
+
+func (db *Database) GetSecretById(id uuid.UUID) (Secret, error) {
+	query := `
+SELECT id, name, secret
+FROM secrets
+WHERE id=$1;`
+
+	secret := Secret{
+		Id: id,
+	}
+
+	err := db.Conn.QueryRow(context.Background(), query, id).Scan(&secret.Name, &secret.Secret)
+	if err != nil {
+		return secret, fmt.Errorf("Could not query database for secret with id %v: %w", id.String(), err)
+	}
+
+	return secret, nil
+}
+
+func (db *Database) CreateSecret(name string, secret string) (Secret, error) {
+	s := Secret{}
+
+	// validate that the secret is only A-Z or underscores and less than 256 characters
+	if len(name) > 256 {
+		return s, fmt.Errorf("secret name must be 256 characters or less")
+	}
+
+	validName := regexp.MustCompile(`[A-Z0-9_]+$`)
+	if !validName.MatchString(name) {
+		return s, fmt.Errorf("secren name must be made up of only uppercase letters, numbers, and underscores")
+	}
+
+	query := `
+INSERT INTO secrets (id, name, secret)
+VALUES (uuid_generate_v4(), $1, $2)
+RETURNING id, name, secret;`
+
+	var idStr string
+	err := db.Conn.QueryRow(context.Background(), query, name, secret).Scan(&idStr, &s.Name, &s.Secret)
+	if err != nil {
+		return s, fmt.Errorf("Could not create secret: %w", err)
+	}
+
+	s.Id, err = uuid.Parse(idStr)
+	if err != nil {
+		return s, fmt.Errorf("Could not parse UUID generated by DB: %w", err)
+	}
+
+	return s, nil
+}
+
+func (db *Database) AssignSecretToPipeline(pipelineId uuid.UUID, secretId uuid.UUID) error {
+	query := `
+INSERT INTO pipeline_secret_mappings (pipeline, secret)
+VALUES ($1, $2);`
+
+	_, err := db.Conn.Exec(context.Background(), query, pipelineId, secretId)
+
+	return err
+}
+
+func (db *Database) RemoveSecretFromPipeline(pipelineId uuid.UUID, secretId uuid.UUID) error {
+	// TODO: implement this
+	return fmt.Errorf("Not implemented")
+}
+
+func (db *Database) GetSecretsForPipeline(pipelineId uuid.UUID) ([]Secret, error) {
+	query := `
+SELECT
+    secrets.id, secrets.name, secrets.secret
+FROM
+    secrets INNER JOIN pipeline_secret_mappings
+    ON secrets.id = pipeline_secret_mappings.secret
+WHERE
+    pipeline_secret_mappings.pipeline=$1
+;`
+
+	secrets := make([]Secret, 0)
+
+	rows, err := db.Conn.Query(context.Background(), query, pipelineId)
+	if err != nil {
+		return secrets, fmt.Errorf("Could not get secrets for pipeline with id \"%v\": %w", pipelineId, err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var secret Secret
+		var idStr string
+		if err := rows.Scan(
+			&idStr,
+			&secret.Name,
+			&secret.Secret,
+		); err != nil {
+			return secrets, err
+		}
+
+		secret.Id, err = uuid.Parse(idStr)
+		if err != nil {
+			return secrets, err
+		}
+		secrets = append(secrets, secret)
+	}
+
+	return secrets, nil
+}
+
+func (db *Database) GetRunners() ([]Runner, error) {
+	query := `
+SELECT id, name, token
+FROM runners;`
+
+	runners := make([]Runner, 0)
+
+	rows, err := db.Conn.Query(context.Background(), query)
+	if err != nil {
+		return runners, fmt.Errorf("Could not query database for runners: %w", err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var runner Runner
+		var idStr string
+		if err := rows.Scan(&idStr, &runner.Name, &runner.Token); err != nil {
+			return runners, err
+		}
+
+		runner.Id, err = uuid.Parse(idStr)
+		if err != nil {
+			return runners, err
+		}
+		runners = append(runners, runner)
+	}
+
+	return runners, nil
+}
+
+func (db *Database) GetRunnerById(id uuid.UUID) (Runner, error) {
+	query := `
+SELECT name, token
+FROM runners
+WHERE id=$1;`
+
+	runner := Runner{
+		Id: id,
+	}
+
+	err := db.Conn.QueryRow(context.Background(), query, id).Scan(&runner.Name, &runner.Token)
+	if err != nil {
+		return runner, fmt.Errorf("Could not query database for runner with id %v: %w", id.String(), err)
+	}
+
+	return runner, nil
+}
+
+func (db *Database) CreateRunner(name string) (Runner, error) {
+	s := Runner{}
+
+	// validate that the runner name is only A-Z or underscores and less than 256 characters
+	if len(name) > 256 {
+		return s, fmt.Errorf("runner name must be 256 characters or less")
+	}
+
+	validName := regexp.MustCompile(`[A-Z0-9_]+$`)
+	if !validName.MatchString(name) {
+		return s, fmt.Errorf("runner name must be made up of only uppercase letters, numbers, and underscores")
+	}
+
+	query := `
+INSERT INTO runners (id, name, token)
+VALUES
+  (
+    uuid_generate_v4(),
+    $1,
+    (
+      SELECT md5(random()::text)
+    )
+)
+RETURNING id, name, token;`
+
+	var idStr string
+	err := db.Conn.QueryRow(context.Background(), query, name).Scan(&idStr, &s.Name, &s.Token)
+	if err != nil {
+		return s, fmt.Errorf("Could not create runner: %w", err)
+	}
+
+	s.Id, err = uuid.Parse(idStr)
+	if err != nil {
+		return s, fmt.Errorf("Could not parse UUID generated by DB: %w", err)
+	}
+
+	return s, nil
+}

database/types.go

@@ -0,0 +1,81 @@
+package database
+
+import (
+	"time"
+
+	"github.com/google/uuid"
+)
+
+type CloneCredentialType string
+
+const (
+	USER_PASS CloneCredentialType = "USER_PASS"
+	SSH_KEY   CloneCredentialType = "SSH_KEY"
+)
+
+type CloneCredential struct {
+	Id       uuid.UUID
+	Name     string
+	Type     CloneCredentialType
+	Username string
+	Secret   string
+}
+
+type Pipeline struct {
+	Id              uuid.UUID
+	Name            string
+	Url             string
+	PollInterval    int
+	CloneCredential *uuid.UUID
+}
+
+type Secret struct {
+	Id     uuid.UUID
+	Name   string
+	Secret string
+}
+
+type PipelineSecretMapping struct {
+	Pipeline uuid.UUID
+	Secret   uuid.UUID
+}
+
+type WebhookSender string
+
+const (
+	Gitea WebhookSender = "gitea"
+)
+
+type Webhook struct {
+	Id         uuid.UUID
+	ServerType WebhookSender
+	Secret     string
+	Pipeline   uuid.UUID
+}
+
+type Run struct {
+	Id          uuid.UUID
+	Pipeline    uuid.UUID
+	InProgress  bool
+	Result      *int64
+	BuildOutput []byte
+	Stdout      []byte
+	Stderr      []byte
+}
+
+type CommandExecution struct {
+	Id         uuid.UUID
+	RunId      uuid.UUID
+	Command    []string
+	ReturnCode int
+	Stdout     string
+	Stderr     string
+	StartTime  time.Time
+	EndTime    time.Time
+}
+
+type Runner struct {
+	Id    uuid.UUID
+	Name  string
+	Token string
+}

docker/Dockerfile

@@ -1,20 +0,0 @@
-FROM golang:1.19-bullseye as builder
-MAINTAINER restitux <restitux@ohea.xyz>
-
-# Install connect proto build deps
-RUN go install github.com/bufbuild/buf/cmd/buf@latest && \
-    go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest && \
-    go install google.golang.org/protobuf/cmd/protoc-gen-go@latest && \
-    go install github.com/bufbuild/connect-go/cmd/protoc-gen-connect-go@latest
-
-COPY . /server
-WORKDIR /server
-RUN go build .
-
-
-FROM debian:bullseye
-RUN apt-get update && apt-get install -y \
-    ca-certificates \
-    && rm -rf /var/lib/apt/lists/*
-COPY --from=builder /server/server /server
-ENTRYPOINT ["/server"]

docker/Dockerfile.dev

@@ -1,13 +0,0 @@
-FROM golang:1.19-bullseye
-MAINTAINER restitux <restitux@ohea.xyz>
-
-# Install connect proto build deps
-RUN GOPATH="/go-bin" go install github.com/bufbuild/buf/cmd/buf@latest && \
-    GOPATH="/go-bin" go install github.com/fullstorydev/grpcurl/cmd/grpcurl@latest && \
-    GOPATH="/go-bin" go install google.golang.org/protobuf/cmd/protoc-gen-go@latest && \
-    GOPATH="/go-bin" go install github.com/bufbuild/connect-go/cmd/protoc-gen-connect-go@latest
-
-RUN apt-get update && apt-get install -y \
-    ca-certificates \
-    && rm -rf /var/lib/apt/lists/*
-ENTRYPOINT ["/build/docker/build-and-run.sh"]

docker/build-and-run.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-
-set -e
-
-cd /build
-go build .
-./server

docker/cursorius/Dockerfile

@@ -0,0 +1,10 @@
+FROM golang:1.19-alpine as builder
+MAINTAINER restitux <restitux@ohea.xyz>
+
+COPY . /server
+WORKDIR /server
+RUN go build .
+
+FROM alpine:latest
+COPY --from=builder /server/server /server
+ENTRYPOINT ["/server"]

docker/cursorius/Dockerfile.dev

@@ -0,0 +1,7 @@
+FROM golang:1.19-bullseye
+MAINTAINER restitux <restitux@ohea.xyz>
+
+RUN apt-get update && apt-get install -y \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+ENTRYPOINT ["/build/server/docker/cursorius/build-and-run.sh"]

docker/cursorius/build-and-run.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+
+set -e
+
+cd /build/server
+go build -buildvcs=false .
+./server

docker/docker-compose.yml

@@ -2,16 +2,48 @@ version: "3.3"
 services:
   cursorius-server:
     build:
-      context: ..
-      dockerfile: docker/Dockerfile.dev
+      context: ".."
+      dockerfile: "docker/cursorius/Dockerfile.dev"
     ports:
       - "0.0.0.0:45420:45420"
+    networks:
+      - cursorius
     volumes:
-      - "..:/build"
-      - "../server.toml:/root/.config/cursorius/server.toml"
+      - "..:/build/server"
+      - "./server.toml:/root/.config/cursorius/server.toml"
       - "/var/run/docker.sock:/var/run/docker.sock"
       - "../_working/go:/go"
       - "../_working/jobs:/cursorius/jobs"
+  cursorius-db:
+    image: postgres:14
+    environment:
+      - POSTGRES_USER=cursorius
+      - POSTGRES_PASSWORD=cursorius
+      - POSTGRES_DB=cursorius
+    volumes:
+      - "../_working/postgres:/var/lib/postgresql/data"
+    networks:
+      - cursorius
+  graphiql:
+    build:
+      context: "graphiql"
+      dockerfile: "Dockerfile.graphiql"
+    ports:
+      - "0.0.0.0:45421:80"
+    networks:
+      - cursorius
+  gitea:
+    image: gitea/gitea:latest
+    profiles: ["gitea"]
+    environment:
+      - GITEA__webhook__ALLOWED_HOST_LIST=cursorius-server, external
+    ports:
+      - "127.0.0.1:2222:22"
+      - "127.0.0.1:3000:3000"
+    networks:
+      - cursorius
+    volumes:
+      - "../_working/gitea:/data"
 
 networks:
   cursorius:

docker/graphiql/Dockerfile.graphiql

@@ -0,0 +1,3 @@
+FROM nginx:latest
+COPY graphiql.html /usr/share/nginx/html/index.html
+COPY graphiql.conf /etc/nginx/conf.d/default.conf

docker/graphiql/graphiql.conf

@@ -0,0 +1,52 @@
+upstream backend {
+         server cursorius-server:45420;
+}
+
+server {
+    listen       80;
+    listen  [::]:80;
+    server_name  localhost;
+
+    #access_log  /var/log/nginx/host.access.log  main;
+
+    location / {
+        root   /usr/share/nginx/html;
+        index  index.html index.htm;
+    }
+
+    location /graphql {
+             proxy_pass http://backend/graphql;
+    }
+
+    #error_page  404              /404.html;
+
+    # redirect server error pages to the static page /50x.html
+    #
+    error_page   500 502 503 504  /50x.html;
+    location = /50x.html {
+        root   /usr/share/nginx/html;
+    }
+
+    # proxy the PHP scripts to Apache listening on 127.0.0.1:80
+    #
+    #location ~ \.php$ {
+    #    proxy_pass   http://127.0.0.1;
+    #}
+
+    # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
+    #
+    #location ~ \.php$ {
+    #    root           html;
+    #    fastcgi_pass   127.0.0.1:9000;
+    #    fastcgi_index  index.php;
+    #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
+    #    include        fastcgi_params;
+    #}
+
+    # deny access to .htaccess files, if Apache's document root
+    # concurs with nginx's one
+    #
+    #location ~ /\.ht {
+    #    deny  all;
+    #}
+}

docker/graphiql/graphiql.html

@@ -0,0 +1,70 @@
+<!--
+ *  Copyright (c) 2021 GraphQL Contributors
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the license found in the
+ *  LICENSE file in the root directory of this source tree.
+-->
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <title>GraphiQL</title>
+    <style>
+      body {
+        height: 100%;
+        margin: 0;
+        width: 100%;
+        overflow: hidden;
+      }
+
+      #graphiql {
+        height: 100vh;
+      }
+    </style>
+
+    <!--
+      This GraphiQL example depends on Promise and fetch, which are available in
+      modern browsers, but can be "polyfilled" for older browsers.
+      GraphiQL itself depends on React DOM.
+      If you do not want to rely on a CDN, you can host these files locally or
+      include them directly in your favored resource bundler.
+    -->
+    <script
+      src="https://unpkg.com/react@17/umd/react.development.js"
+      integrity="sha512-Vf2xGDzpqUOEIKO+X2rgTLWPY+65++WPwCHkX2nFMu9IcstumPsf/uKKRd5prX3wOu8Q0GBylRpsDB26R6ExOg=="
+      crossorigin="anonymous"
+    ></script>
+    <script
+      src="https://unpkg.com/react-dom@17/umd/react-dom.development.js"
+      integrity="sha512-Wr9OKCTtq1anK0hq5bY3X/AvDI5EflDSAh0mE9gma+4hl+kXdTJPKZ3TwLMBcrgUeoY0s3dq9JjhCQc7vddtFg=="
+      crossorigin="anonymous"
+    ></script>
+
+    <!--
+      These two files can be found in the npm module, however you may wish to
+      copy them directly into your environment, or perhaps include them in your
+      favored resource bundler.
+     -->
+    <link rel="stylesheet" href="https://unpkg.com/graphiql/graphiql.min.css" />
+  </head>
+
+  <body>
+    <div id="graphiql">Loading...</div>
+    <script
+      src="https://unpkg.com/graphiql/graphiql.min.js"
+      type="application/javascript"
+    ></script>
+    <script>
+      ReactDOM.render(
+        React.createElement(GraphiQL, {
+          fetcher: GraphiQL.createFetcher({
+            //url: 'https://swapi-graphql.netlify.app/.netlify/functions/index',
+            url: 'http://127.0.0.1:45421/graphql',
+          }),
+          defaultEditorToolsVisibility: true,
+        }),
+        document.getElementById('graphiql'),
+      );
+    </script>
+  </body>
+</html>

docker/publish.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+docker build . -f docker/Dockerfile -t git.ohea.xyz/cursorius/server:latest
+docker push git.ohea.xyz/cursorius/server:latest

docker/run-dev.sh

@@ -2,40 +2,4 @@
 
 set -e
 
-mkdir -p _working/go
-mkdir -p _working/jobs
-
-function stop_containers {
-    docker compose -f docker/docker-compose.yml down
-    docker compose -f docker/docker-compose.yml -f docker/webhook-override.yml down
-}
-
-function show_logs {
-    current_containers="$(cat _working/current_containers)"
-    if [[ "$current_containers" == "default" ]]
-    then
-        docker compose -f docker/docker-compose.yml logs -f
-    elif [[ "$current_containers" == "webhook" ]]
-    then
-        docker compose -f docker/docker-compose.yml -f docker/webhook-override.yml logs -f
-    fi
-}
-
-case $1 in
-    "default")
-        echo "default" >> _working/current_containers
-        docker compose -f docker/docker-compose.yml up --build -d
-        docker compose -f docker/docker-compose.yml logs -f;;
-    "webhook")
-        echo "webhook" >> _working/current_containers
-        stop_containers
-        docker compose -f docker/docker-compose.yml -f docker/webhook-override.yml up --build -d
-        docker compose -f docker/docker-compose.yml -f docker/webhook-override.yml logs -f;;
-    "stop")
-        stop_containers;;
-    "logs")
-        show_logs;;
-    *) echo "ERROR: Unknown param \"$1\"" 2>&1
-       exit 255;;
-esac
-
+go run docker/run.go "$@"

docker/run.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+)
+
+func panicError(errorString string, params ...any) {
+	fmt.Fprintf(os.Stderr, fmt.Sprintf("ERROR: %v\n", errorString), params...)
+	os.Exit(1)
+}
+
+func run(name string, arg ...string) {
+	cmd := exec.Command(name, arg...)
+	if err := cmd.Run(); err != nil {
+		panicError("could not run command %v: %v", name, err)
+	}
+}
+
+func runAttach(name string, arg ...string) {
+	cmd := exec.Command(name, arg...)
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		panicError("could not run command %v: %v", name, err)
+	}
+}
+
+func runCompose(args []string) {
+	runAttach("docker", append([]string{"compose"}, args...)...)
+}
+
+func createDirs() {
+	run("mkdir", "-p", "_working/go")
+	run("mkdir", "-p", "_working/jobs")
+}
+
+func currentContainers() string {
+	bytes, err := os.ReadFile("_working/current_containers")
+	if err != nil {
+		panicError("could not read current containers: %v", err)
+	}
+	return string(bytes)
+}
+
+func composeFlags() []string {
+	containers := currentContainers()
+	flags := []string{"-f", "docker/docker-compose.yml"}
+	switch containers {
+	case "gitea":
+		flags = append(flags, "--profile", "gitea")
+	}
+	return flags
+}
+
+func runContainers(containers string) {
+	err := os.WriteFile("_working/current_containers", []byte(containers), 0633)
+	if err != nil {
+		panicError("could not write current_containers file: %v", err)
+	}
+	runCompose(append(composeFlags(), "up", "--build", "-d"))
+	runCompose(append(composeFlags(), "logs", "-f"))
+}
+
+func main() {
+	if len(os.Args) < 2 {
+		panicError("not enough arguments passed")
+	}
+
+	createDirs()
+
+	switch os.Args[1] {
+	case "default", "gitea":
+		runContainers(os.Args[1])
+	case "runprev":
+		runContainers(currentContainers())
+	case "stop":
+		runCompose(append(composeFlags(), "down"))
+	case "dbshell":
+		runCompose(append(composeFlags(), "exec", "cursorius-db", "psql", "--user=cursorius"))
+	case "logs":
+		runCompose(append(composeFlags(), "logs", "-f"))
+	case "ps":
+		runCompose(append(composeFlags(), "ps"))
+	case "help":
+		fmt.Println("commands: default, gitea, runprev, stop, dbshell, logs, ps, help")
+	default:
+		panicError("Unknown subcommand: %v", os.Args[1])
+
+	}
+}

docker/webhook-override.yml

@@ -1,24 +0,0 @@
-version: "3.3"
-services:
-  cursorius-server:
-    networks:
-      - gitea
-  gitea:
-    image: gitea/gitea:latest
-    environment:
-      - GITEA__webhook__ALLOWED_HOST_LIST=cursorius-server, external
-    ports:
-      - "127.0.0.1:2222:22"
-      - "127.0.0.1:3000:3000"
-    networks:
-      - gitea
-    volumes:
-      - gitea-data:/data
-
-
-volumes:
-  gitea-data:
-
-networks:
-  gitea:
-    external: false

go.mod

@@ -3,15 +3,21 @@ module git.ohea.xyz/cursorius/server
 go 1.19
 
 require (
+	git.ohea.xyz/cursorius/pipeline-api/go/api/v2 v2.0.0-20230405234139-34d8875b72f4
+	git.ohea.xyz/cursorius/runner-api/go/api/v2 v2.0.0-20230109074922-e20285fe6cf2
 	git.ohea.xyz/golang/config v0.0.0-20220915224621-b9debd233173
 	github.com/bufbuild/connect-go v1.4.1
 	github.com/docker/docker v20.10.22+incompatible
 	github.com/go-git/go-git/v5 v5.4.3-0.20220529141257-bc1f419cebcf
 	github.com/go-playground/webhooks/v6 v6.0.1
 	github.com/google/uuid v1.3.0
+	github.com/graphql-go/graphql v0.8.0
+	github.com/graphql-go/handler v0.2.3
+	github.com/jackc/pgx/v5 v5.2.0
+	github.com/jhoonb/archivex v0.0.0-20201016144719-6a343cdae81d
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	golang.org/x/net v0.2.0
-	google.golang.org/protobuf v1.28.1
+	google.golang.org/protobuf v1.30.0
 	nhooyr.io/websocket v1.8.7
 )
 
@@ -30,6 +36,9 @@ require (
 	github.com/gobwas/ws v1.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/puddle/v2 v2.1.2 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
@@ -44,9 +53,12 @@ require (
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/stretchr/testify v1.8.1 // indirect
 	github.com/xanzy/ssh-agent v0.3.2 // indirect
+	go.uber.org/atomic v1.10.0 // indirect
 	golang.org/x/crypto v0.2.1-0.20221112162523-6fad3dfc1891 // indirect
 	golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
+	golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 // indirect
 	golang.org/x/sys v0.2.0 // indirect
 	golang.org/x/text v0.4.0 // indirect
 	golang.org/x/tools v0.1.12 // indirect

go.sum

@@ -78,6 +78,10 @@ contrib.go.opencensus.io/exporter/stackdriver v0.13.5/go.mod h1:aXENhDJ1Y4lIg4EU
 contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE=
 contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+git.ohea.xyz/cursorius/pipeline-api/go/api/v2 v2.0.0-20230405234139-34d8875b72f4 h1:kKQQEg1nmWnqiNOqtUHteEuacyfy0NdxyDj6HPjbA2c=
+git.ohea.xyz/cursorius/pipeline-api/go/api/v2 v2.0.0-20230405234139-34d8875b72f4/go.mod h1:D7GGcFIH421mo6KuRaXXXmlXPwWeEsemTZG/BOZA/4o=
+git.ohea.xyz/cursorius/runner-api/go/api/v2 v2.0.0-20230109074922-e20285fe6cf2 h1:G1XQEqhj1LZPQbH7avzvT7QL9Wfbb4CXMm0nLL39eDc=
+git.ohea.xyz/cursorius/runner-api/go/api/v2 v2.0.0-20230109074922-e20285fe6cf2/go.mod h1:F9y5Ck4Wchsaj5amSX2eDRUlQ/iYP1VNLFduvjNwmLc=
 git.ohea.xyz/cursorius/webhooks/v6 v6.0.2-0.20221224221147-a2bdbf1756ed h1:gsK15m4Npow74+R6OfZKwwAg1sl7QWQCRXOeE2QLUco=
 git.ohea.xyz/cursorius/webhooks/v6 v6.0.2-0.20221224221147-a2bdbf1756ed/go.mod h1:64JKTmG3kupV+3+ZYJYPB/rGPEKw/diihhIj8lut4UA=
 git.ohea.xyz/golang/config v0.0.0-20220915224621-b9debd233173 h1:dhq/W6sa5KkLHVBwwgcNIPWcO4YK2/ecFTTln2W+1n8=
@@ -705,6 +709,10 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA
 github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/graphql-go/graphql v0.8.0 h1:JHRQMeQjofwqVvGwYnr8JnPTY0AxgVy1HpHSGPLdH0I=
+github.com/graphql-go/graphql v0.8.0/go.mod h1:nKiHzRM0qopJEwCITUuIsxk9PlVlwIiiI8pnJEhordQ=
+github.com/graphql-go/handler v0.2.3 h1:CANh8WPnl5M9uA25c2GBhPqJhE53Fg0Iue/fRNla71E=
+github.com/graphql-go/handler v0.2.3/go.mod h1:leLF6RpV5uZMN1CdImAxuiayrYYhOk33bZciaUGaXeU=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-middleware v1.2.2/go.mod h1:EaizFBKfUKtMIF5iaDEhniwNedqGo9FuLFzppDr3uwI=
@@ -799,6 +807,7 @@ github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bY
 github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
 github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c=
 github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
 github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
 github.com/jackc/pgproto3 v1.1.0/go.mod h1:eR5FA3leWg7p9aeAqi37XOTgTIbkABlvcPB3E5rlc78=
 github.com/jackc/pgproto3/v2 v2.0.0-alpha1.0.20190420180111-c116219b62db/go.mod h1:bhq50y+xrl9n5mRYyCBFKkpRVTLYJVWeCc+mEAI3yXA=
@@ -810,6 +819,7 @@ github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwX
 github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
 github.com/jackc/pgproto3/v2 v2.3.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA=
 github.com/jackc/pgservicefile v0.0.0-20200307190119-3430c5407db8/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
 github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
 github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg=
 github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
@@ -831,12 +841,16 @@ github.com/jackc/pgx/v4 v4.11.0/go.mod h1:i62xJgdrtVDsnL3U8ekyrQXEwGNTRoG7/8r+CI
 github.com/jackc/pgx/v4 v4.12.0/go.mod h1:fE547h6VulLPA3kySjfnSG/e2D861g/50JlVUa/ub60=
 github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
 github.com/jackc/pgx/v4 v4.16.1/go.mod h1:SIhx0D5hoADaiXZVyv+3gSm3LCIIINTVO0PficsvWGQ=
+github.com/jackc/pgx/v5 v5.2.0 h1:NdPpngX0Y6z6XDFKqmFQaE+bCtkqzvQIOt1wvBlAqs8=
+github.com/jackc/pgx/v5 v5.2.0/go.mod h1:Ptn7zmohNsWEsdxRawMzk3gaKma2obW+NWTnKa0S4nk=
 github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jackc/puddle/v2 v2.1.2 h1:0f7vaaXINONKTsxYDn4otOAiJanX/BMeAtY//BXqzlg=
+github.com/jackc/puddle/v2 v2.1.2/go.mod h1:2lpufsF5mRHO6SuZkm0fNYxM6SWHfvyFj62KwNzgels=
 github.com/jarcoal/httpmock v0.0.0-20180424175123-9c70cfe4a1da/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4=
 github.com/jarcoal/httpmock v1.0.5/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jaytaylor/html2text v0.0.0-20211105163654-bc68cce691ba/go.mod h1:CVKlgaMiht+LXvHG173ujK6JUhZXKb2u/BQtjPDIvyk=
@@ -844,6 +858,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/jhoonb/archivex v0.0.0-20201016144719-6a343cdae81d h1:q7n+5taxmM+9T2Q7Ydo7YN90FkoDuR5bbzByZwkQqPo=
+github.com/jhoonb/archivex v0.0.0-20201016144719-6a343cdae81d/go.mod h1:GN1Mg/uXQ6qwXA0HypnUO3xlcQJS9/y68EsHNeuuRa4=
 github.com/jhump/protoreflect v1.6.1/go.mod h1:RZQ/lnuN+zqeRVpQigTwO6o0AJUkxbnSnpuG7toUTG4=
 github.com/jhump/protoreflect v1.8.2/go.mod h1:7GcYQDdMU/O/BBrl/cX6PNHpXh6cenjd8pneu5yW7Tg=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -1293,6 +1309,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v0.0.0-20170130113145-4d4bfba8f1d1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
@@ -1302,8 +1319,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
-github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/subosito/gotenv v1.3.0/go.mod h1:YzJjq/33h7nrwdY+iHMhEOEEbW0ovIz0tB6t6PwAXzs=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
@@ -1429,6 +1447,7 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ=
 go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
 go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
@@ -1656,6 +1675,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 h1:ZrnxWX62AgTKOSagEqxvb3ffipvEDX2pl7E1TdqLqIc=
+golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -2108,8 +2129,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w=
 google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

listen/listen.go

@@ -4,9 +4,14 @@ import (
 	"fmt"
 	"net/http"
 
+	"git.ohea.xyz/cursorius/server/admin_api"
 	"git.ohea.xyz/cursorius/server/config"
+	"git.ohea.xyz/cursorius/server/database"
+	"git.ohea.xyz/cursorius/server/pipeline_api"
 	"git.ohea.xyz/cursorius/server/runnermanager"
 	"git.ohea.xyz/cursorius/server/webhook"
+
+	"github.com/google/uuid"
 	"github.com/op/go-logging"
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
@@ -15,29 +20,59 @@ import (
 
 var log = logging.MustGetLogger("cursorius-server")
 
-func setupHTTPServer(mux *http.ServeMux, registerCh chan runnermanager.RunnerRegistration,
-	conf config.Config) {
+func setupHTTPServer(
+	mux *http.ServeMux,
+	conf config.PipelineConf,
+	db database.Database,
+	runnerManagerChans runnermanager.RunnerManagerChans,
+	pollChan chan uuid.UUID,
+) error {
+
+	webhook.CreateWebhookHandler(db, conf, mux)
+
+	pipeline_api.CreateHandler(runnerManagerChans.Allocation, runnerManagerChans.Release, mux)
+
+	err := admin_api.CreateHandler(db, pollChan, mux)
+	if err != nil {
+		return fmt.Errorf("Could not create admin api handler: %w", err)
+	}
 
-	webhook.CreateWebhookHandler(conf, mux)
 	mux.HandleFunc("/runner", func(w http.ResponseWriter, r *http.Request) {
 		conn, err := websocket.Accept(w, r, nil)
 		if err != nil {
 			log.Errorf("Could not upgrade runner connection to websocket: %v", err)
 			return
 		}
-		go runnermanager.RegisterRunner(conn, registerCh)
+		go runnermanager.RegisterRunner(conn, runnerManagerChans.Registration)
 	})
+	return nil
 }
 
-func Listen(mux *http.ServeMux, address string, port int, registerCh chan runnermanager.RunnerRegistration, conf config.Config) {
+func Listen(
+	mux *http.ServeMux,
+	address string,
+	port int,
+	conf config.PipelineConf,
+	db database.Database,
+	runnerManagerChans runnermanager.RunnerManagerChans,
+	pollChan chan uuid.UUID,
+) error {
 
-	setupHTTPServer(mux, registerCh, conf)
+	err := setupHTTPServer(
+		mux,
+		conf,
+		db,
+		runnerManagerChans,
+		pollChan,
+	)
+	if err != nil {
+		return fmt.Errorf("Could not setup http endpoints: %w", err)
+	}
 
 	connect_string := fmt.Sprintf("%v:%v", address, port)
 	log.Noticef("Launching HTTP server on %v\n", connect_string)
-	log.Fatal(http.ListenAndServe(
+	return http.ListenAndServe(
 		connect_string,
 		h2c.NewHandler(mux, &http2.Server{}),
-	))
-
+	)
 }

main.go

@@ -5,8 +5,8 @@ import (
 	"os"
 
 	"git.ohea.xyz/cursorius/server/config"
+	"git.ohea.xyz/cursorius/server/database"
 	"git.ohea.xyz/cursorius/server/listen"
-	"git.ohea.xyz/cursorius/server/pipeline_api"
 	"git.ohea.xyz/cursorius/server/poll"
 	"git.ohea.xyz/cursorius/server/runnermanager"
 	"github.com/op/go-logging"
@@ -26,23 +26,37 @@ func main() {
 
 	logging.SetBackend(backendLeveled)
 
-	log.Info("Starting cursorius-server")
+	log.Info("Starting cursorius-server v0.1.0")
 
 	configData, err := config.GetConfig()
 	if err != nil {
 		log.Errorf("Could not get configuration: %v", err)
+		return
 	}
 
-	getRunnerCh, registerCh, err := runnermanager.StartRunnerManager(configData.Config.Runners)
+	db, err := database.LaunchDB(configData.Config.DBConfig)
+	if err != nil {
+		log.Errorf("Could not launch db: %v", err)
+		return
+	}
+
+	runnerManagerChans, err := runnermanager.StartRunnerManager(configData.Config.Runners, db)
 	if err != nil {
 		log.Errorf("Could not start runner: %v", err)
+		return
 	}
 
-	poll.StartPolling(configData.Config)
+	pollChan := poll.StartPolling(configData.Config.PipelineConf, db)
 
 	mux := http.NewServeMux()
 
-	pipeline_api.CreateHandler(mux, getRunnerCh)
-
-	listen.Listen(mux, configData.Config.Address, configData.Config.Port, registerCh, configData.Config)
+	log.Fatal(listen.Listen(
+		mux,
+		configData.Config.Address,
+		configData.Config.Port,
+		configData.Config.PipelineConf,
+		db,
+		runnerManagerChans,
+		pollChan,
+	))
 }

pipeline_api/pipeline_api.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"strings"
 	"sync"
+	"time"
 
-	apiv1 "git.ohea.xyz/cursorius/server/proto/gen/api/v1"
-	"git.ohea.xyz/cursorius/server/proto/gen/api/v1/apiv1connect"
+	apiv2 "git.ohea.xyz/cursorius/pipeline-api/go/api/v2"
+	"git.ohea.xyz/cursorius/pipeline-api/go/api/v2/apiv2connect"
 	"git.ohea.xyz/cursorius/server/runnermanager"
+	"git.ohea.xyz/cursorius/server/util"
 	"github.com/bufbuild/connect-go"
 	"github.com/google/uuid"
 	"github.com/op/go-logging"
@@ -18,72 +19,179 @@ import (
 var log = logging.MustGetLogger("cursorius-server")
 
 type ApiServer struct {
-	getRunnerCh      chan runnermanager.GetRunnerRequest
-	allocatedRunners map[uuid.UUID]RunnerWrapper
+	allocationCh          chan runnermanager.RunnerAllocationRequest
+	releaseCh             chan runnermanager.RunnerReleaseRequest
+	allocatedRunners      map[uuid.UUID]*RunnerWrapper
+	allocatedRunnersMutex sync.RWMutex
 }
 
 type RunnerWrapper struct {
-	runner runnermanager.Runner
+	runner *runnermanager.Runner
 	mutex  sync.Mutex
 }
 
+func (r *RunnerWrapper) RunCommand(cmd string, args []string) (int64, string, string, error) {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
+	return_code, stdout, stderr, err := r.runner.RunCommand(cmd, args)
+
+	return return_code, stdout, stderr, err
+}
+
+func (r *RunnerWrapper) Release(releaseCh chan runnermanager.RunnerReleaseRequest) {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
+	releaseCh <- runnermanager.RunnerReleaseRequest{
+		Runner: r.runner,
+	}
+	r.runner = nil
+}
+
+func (s *ApiServer) GetRunnerFromMap(u uuid.UUID) (*RunnerWrapper, bool) {
+	s.allocatedRunnersMutex.RLock()
+	defer s.allocatedRunnersMutex.RUnlock()
+	runner, ok := s.allocatedRunners[u]
+	return runner, ok
+}
+
+func (s *ApiServer) AddRunnerToMap(u uuid.UUID, runner *runnermanager.Runner) {
+	s.allocatedRunnersMutex.Lock()
+	defer s.allocatedRunnersMutex.Unlock()
+	s.allocatedRunners[u] = &RunnerWrapper{runner: runner}
+}
+
 func (s *ApiServer) GetRunner(
 	ctx context.Context,
-	req *connect.Request[apiv1.GetRunnerRequest],
-) (*connect.Response[apiv1.GetRunnerResponse], error) {
+	req *connect.Request[apiv2.GetRunnerRequest],
+) (*connect.Response[apiv2.GetRunnerResponse], error) {
 
-	respChan := make(chan runnermanager.GetRunnerResponse)
-	s.getRunnerCh <- runnermanager.GetRunnerRequest{
-		Tags:     req.Msg.Tags,
-		RespChan: respChan,
+	var response runnermanager.RunnerAllocationResponse
+	var timeoutCtx *context.Context
+	var retryInterval int64 = 0
+
+	respChan := make(chan runnermanager.RunnerAllocationResponse)
+
+	tagsStr := util.FormatTags(req.Msg.Tags)
+
+	if req.Msg.Options != nil {
+		if req.Msg.Options.Timeout != 0 {
+			ctx, cancel := context.WithTimeout(context.Background(), time.Duration(req.Msg.Options.Timeout)*time.Second)
+			timeoutCtx = &ctx
+			defer cancel()
+		}
+
+		retryInterval = req.Msg.Options.RetryInterval
 	}
 
-	var runnerTagsStr strings.Builder
-	fmt.Fprintf(&runnerTagsStr, "%v", req.Msg.Tags[0])
-	for _, tag := range req.Msg.Tags[1:] {
-		fmt.Fprintf(&runnerTagsStr, " %v", tag)
+	for {
+		s.allocationCh <- runnermanager.RunnerAllocationRequest{
+			Tags:     req.Msg.Tags,
+			RespChan: respChan,
+		}
+
+		response = <-respChan
+		if response.Err == nil {
+			break
+		}
+
+		log.Infof("Could not get runner with tags \"%v\": %v", tagsStr, response.Err)
+
+		// If no timeout is specified, skip after one attempt
+		if timeoutCtx == nil {
+			break
+		}
+
+		// If timeout is expired, stop trying to allocate runner
+		if (*timeoutCtx).Err() != nil {
+			break
+		}
+
+		log.Infof("Sleeping for %v seconds before retry...", retryInterval)
+		time.Sleep(time.Duration(retryInterval) * time.Second)
 	}
 
-	response := <-respChan
-	if response.Err != nil {
-		log.Errorf("Could not get runner with tags \"%v\": %v", runnerTagsStr, response.Err)
+	if response.Runner == nil {
 		return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("Could not get runner"))
 	}
 
-	log.Info("Got runner with tags: %v", runnerTagsStr)
+	log.Infof("Got runner with tags: %v", tagsStr)
 
 	runnerUuid := uuid.New()
 
-	s.allocatedRunners[runnerUuid] = RunnerWrapper{runner: response.Runner}
+	s.AddRunnerToMap(runnerUuid, response.Runner)
 
-	res := connect.NewResponse(&apiv1.GetRunnerResponse{
+	res := connect.NewResponse(&apiv2.GetRunnerResponse{
 		Id: runnerUuid.String(),
 	})
-	res.Header().Set("GetRunner-Version", "v1")
+	res.Header().Set("GetRunner-Version", "v2")
 	return res, nil
 }
 
+func (s *ApiServer) ReleaseRunner(
+	ctx context.Context,
+	req *connect.Request[apiv2.ReleaseRunnerRequest],
+) (*connect.Response[apiv2.ReleaseRunnerResponse], error) {
+	uuid, err := uuid.Parse(req.Msg.Id)
+	if err != nil {
+		return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("Invalid runner id"))
+	}
+
+	log.Infof("Releasing runner with ID \"%v\"", uuid)
+
+	s.allocatedRunnersMutex.Lock()
+	runner := s.allocatedRunners[uuid]
+	delete(s.allocatedRunners, uuid)
+	runner.Release(s.releaseCh)
+	s.allocatedRunnersMutex.Unlock()
+
+	res := connect.NewResponse(&apiv2.ReleaseRunnerResponse{})
+	res.Header().Set("ReleaseRunner-Version", "v2")
+	return res, nil
+
+}
+
 func (s *ApiServer) RunCommand(
 	ctx context.Context,
-	req *connect.Request[apiv1.RunCommandRequest],
-) (*connect.Response[apiv1.RunCommandResponse], error) {
+	req *connect.Request[apiv2.RunCommandRequest],
+) (*connect.Response[apiv2.RunCommandResponse], error) {
 
-	res := connect.NewResponse(&apiv1.RunCommandResponse{
-		ReturnCode: 0,
-		Stdout:     "",
-		Stderr:     "",
+	uuid, err := uuid.Parse(req.Msg.Id)
+	if err != nil {
+		return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("Invalid runner id"))
+	}
+
+	runner, ok := s.GetRunnerFromMap(uuid)
+	if !ok {
+		return nil, connect.NewError(connect.CodeInvalidArgument, fmt.Errorf("Invalid runner id"))
+	}
+
+	returnCode, stdout, stderr, err := runner.RunCommand(req.Msg.Command, req.Msg.Args)
+	if err != nil {
+		log.Errorf("Could not run command on runner \"%v\", %v", runner.runner.Id(), err)
+		return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("Could not run command"))
+	}
+
+	res := connect.NewResponse(&apiv2.RunCommandResponse{
+		ReturnCode: returnCode,
+		Stdout:     stdout,
+		Stderr:     stderr,
 	})
-	res.Header().Set("RunCommand-Version", "v1")
+	res.Header().Set("RunCommand-Version", "v2")
 	return res, nil
 }
 
-func CreateHandler(mux *http.ServeMux, getRunnerCh chan runnermanager.GetRunnerRequest) {
+func CreateHandler(allocationCh chan runnermanager.RunnerAllocationRequest, releaseCh chan runnermanager.RunnerReleaseRequest, mux *http.ServeMux) {
 	api_server := &ApiServer{
-		getRunnerCh:      getRunnerCh,
-		allocatedRunners: make(map[uuid.UUID]RunnerWrapper),
+		allocationCh:     allocationCh,
+		releaseCh:        releaseCh,
+		allocatedRunners: make(map[uuid.UUID]*RunnerWrapper),
 	}
-	path, handler := apiv1connect.NewGetRunnerServiceHandler(api_server)
+	path, handler := apiv2connect.NewGetRunnerServiceHandler(api_server)
 	mux.Handle(path, handler)
-	path, handler = apiv1connect.NewRunCommandServiceHandler(api_server)
+	path, handler = apiv2connect.NewReleaseRunnerServiceHandler(api_server)
+	mux.Handle(path, handler)
+	path, handler = apiv2connect.NewRunCommandServiceHandler(api_server)
 	mux.Handle(path, handler)
 }

pipeline_executor/pipeline_executor.go

@@ -4,81 +4,155 @@ import (
 	"bytes"
 	"context"
 	"fmt"
-	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
+	"strings"
+
+	"github.com/jhoonb/archivex"
 
-	"git.ohea.xyz/cursorius/server/config"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/transport"
+	"github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
 	"github.com/op/go-logging"
+
+	"git.ohea.xyz/cursorius/server/config"
+	"git.ohea.xyz/cursorius/server/database"
 )
 
 var log = logging.MustGetLogger("cursorius-server")
 
 type PipelineExecution struct {
-	Name string
-	Job  config.Job
-	Ref  string
+	Pipeline database.Pipeline
+	Ref      string
+	Run      database.Run
 }
 
-func ExecutePipeline(pe PipelineExecution, pipelineConf config.PipelineConf) error {
-	jobFolder := filepath.Join(pipelineConf.WorkingDir, pe.Name)
+func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf config.PipelineConf) {
+	jobFolder := filepath.Join(pipelineConf.WorkingDir, pe.Pipeline.Id.String(), pe.Run.Id.String())
+	cloneFolder := filepath.Join(jobFolder, "repo")
 
-	log.Debugf("Job %v configured with URL \"%v\"", pe.Name, pe.Job.URL)
+	log.Debugf("Job %v configured with URL \"%v\"", pe.Pipeline.Name, pe.Pipeline.Url)
 
-	log.Debugf("Job %v configured with folder \"%v\"", pe.Name, jobFolder)
+	log.Debugf("Job %v configured with folder \"%v\"", pe.Pipeline.Name, jobFolder)
 
 	err := os.RemoveAll(jobFolder)
 	if err != nil {
-		return fmt.Errorf("could not delete existing folder %v", jobFolder)
+		log.Errorf("could not delete existing folder %v", jobFolder)
+		return
 	}
 
-	err = os.MkdirAll(jobFolder, 0755)
+	err = os.MkdirAll(cloneFolder, 0755)
 	if err != nil {
-		return fmt.Errorf("could not create working directory for job %v: %w", pe.Name, err)
+		log.Errorf("could not create working directory for job %v: %w", pe.Pipeline.Name, err)
+		return
 	}
 
-	log.Infof("Cloning source from URL %v", pe.Job.URL)
-	// TODO: should I use go-git here instead of shelling out to raw git?
-	cloneCmd := exec.Command("git", "clone", pe.Job.URL, jobFolder)
-	output, err := cloneCmd.CombinedOutput()
+	log.Infof("Cloning source from URL %v", pe.Pipeline.Url)
+
+	var auth transport.AuthMethod
+
+	if pe.Pipeline.CloneCredential != nil {
+		credential, err := db.GetCloneCredentialById(*pe.Pipeline.CloneCredential)
+		if err != nil {
+			log.Errorf("could not get credenital from db: %v", err)
+			return
+		}
+
+		switch credential.Type {
+		case "USER_PASS":
+			log.Debugf("job %v configured to use credential %v", pe.Pipeline.Name, credential.Name)
+			auth = transport.AuthMethod(&http.BasicAuth{
+				Username: credential.Username,
+				Password: credential.Secret,
+			})
+		case "SSH_KEY":
+			publicKeys, err := ssh.NewPublicKeys(credential.Username, []byte(credential.Secret), "")
+			if err != nil {
+				log.Errorf("could not parse credential %v", credential.Name)
+				return
+			}
+			auth = transport.AuthMethod(publicKeys)
+		default:
+			log.Errorf("unsupported credential type %v", credential.Type)
+			return
+		}
+	} else {
+		auth = nil
+	}
+
+	_, err = git.PlainClone(cloneFolder, false, &git.CloneOptions{
+		URL:  pe.Pipeline.Url,
+		Auth: auth,
+	})
 	if err != nil {
-		log.Debugf("%s", output)
-		return fmt.Errorf("could not clone source: %w", err)
+		log.Errorf("could not clone repo: %v", err)
+		return
 	}
 
 	cli, err := client.NewClientWithOpts(client.FromEnv)
 	if err != nil {
-		return fmt.Errorf("Could not create docker client: %w", err)
+		log.Errorf("Could not create docker client: %w", err)
+		return
 	}
 	log.Info("Source cloned successfully")
 
 	ctx := context.Background()
 
-	imageName := "git.ohea.xyz/cursorius/cursorius:latest"
+	log.Info("Building container")
 
-	log.Infof("Pulling image %v", imageName)
-	pullOutput, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{})
+	tarFile := filepath.Join(jobFolder, "archive.tar")
+	tar := new(archivex.TarFile)
+	err = tar.Create(tarFile)
 	if err != nil {
-		return fmt.Errorf("could not pull image %v: %w", imageName, err)
+		log.Errorf("could not create tarfile: %w", err)
+		return
 	}
 
-	buf, err := io.ReadAll(pullOutput)
+	err = tar.AddAll(cloneFolder, false)
 	if err != nil {
-		return fmt.Errorf("could not read from io.ReadCloser:, %w", err)
+		log.Errorf("could not add repo to tarfile: %w", err)
+		return
 	}
-	log.Infof("%s", buf)
 
-	err = pullOutput.Close()
+	err = tar.Close()
 	if err != nil {
-		return fmt.Errorf("could not close io.ReadCloser: %w", err)
+		log.Errorf("could not close tarfile: %w", err)
+		return
 	}
-	log.Info("Image pulled sucessfully")
+
+	dockerBuildContext, err := os.Open(tarFile)
+	defer dockerBuildContext.Close()
+
+	imageName := fmt.Sprintf("%v-%v:latest", pe.Pipeline.Id.String(), pe.Run.Id.String())
+
+	buildResponse, err := cli.ImageBuild(context.Background(), dockerBuildContext, types.ImageBuildOptions{
+		Tags:       []string{imageName},
+		Dockerfile: ".cursorius/Dockerfile",
+	})
+	if err != nil {
+		log.Errorf("could not build container: %w", err)
+		return
+	}
+
+	err = db.UpdateRunBuildOutput(pe.Run.Id, cleanupBuildOutput(buildResponse.Body))
+	if err != nil {
+		log.Errorf("could not update build output for run: %w", err)
+		return
+	}
+
+	err = buildResponse.Body.Close()
+	if err != nil {
+		log.Errorf("Could not close build response body: %w", err)
+		return
+	}
+
+	log.Info("Image built sucessfully")
 
 	hostConfig := container.HostConfig{}
 
@@ -87,10 +161,12 @@ func ExecutePipeline(pe PipelineExecution, pipelineConf config.PipelineConf) err
 	}
 
 	if pipelineConf.MountConf.Type == config.Bind {
+		sourceDir := filepath.Join(pipelineConf.MountConf.Source, pe.Pipeline.Id.String(), pe.Run.Id.String())
+
 		hostConfig.Mounts = append(hostConfig.Mounts,
 			mount.Mount{
 				Type:        mount.TypeBind,
-				Source:      fmt.Sprintf("%v/%v", pipelineConf.MountConf.Source, pe.Name),
+				Source:      sourceDir,
 				Target:      "/cursorius/src",
 				ReadOnly:    false,
 				Consistency: mount.ConsistencyDefault,
@@ -108,42 +184,71 @@ func ExecutePipeline(pe PipelineExecution, pipelineConf config.PipelineConf) err
 		)
 	}
 
+	env := make([]string, 0)
+
+	// set cursorius environment variables
+	env = append(env, []string{
+		fmt.Sprintf("CURSORIUS_RUN_ID=%v", pe.Run.Id),
+		"CURSORIUS_SRC_DIR=/cursorius/src",
+		fmt.Sprintf("CURSORIUS_SERVER_URL=%v", pipelineConf.AccessURL),
+	}...)
+
+	// load secrets into environment
+	secrets, err := db.GetSecretsForPipeline(pe.Pipeline.Id)
+	if err != nil {
+		log.Errorf("Could not get secrets for pipeline", err)
+		return
+	}
+
+	for _, secret := range secrets {
+		// the env name is validated to be just uppercase letters, numbers, and underscores on ingestion
+		env = append(env, fmt.Sprintf("%v=%v", strings.ToUpper(secret.Name), secret.Secret))
+	}
+
 	resp, err := cli.ContainerCreate(ctx,
 		&container.Config{
 			Image: imageName,
-			Cmd:   []string{"/launcher.sh"},
 			Tty:   false,
-			Env: []string{
-				"CURSORIUS_SRC_DIR=/cursorius/src",
-				fmt.Sprintf("CUROSRIUS_SERVER_URL=%v", pipelineConf.AccessURL),
-			},
+			Env:   env,
 		},
 		// TODO: fix running the runner in docker (add VolumesFrom to HostConfig)
 		&hostConfig,
 		nil, nil, "",
 	)
 	if err != nil {
-		return fmt.Errorf("could not create container: %w", err)
+		log.Errorf("could not create container: %w", err)
+		return
 	}
 
 	log.Info("Launching container")
 
 	if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
-		return fmt.Errorf("could not start container: %w", err)
+		log.Errorf("could not start container: %v", err)
+		return
 	}
 	statusCh, errCh := cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning)
 	select {
 	case err := <-errCh:
 		if err != nil {
-			return fmt.Errorf("container returned error: %w", err)
+			log.Errorf("container returned error: %v", err)
+			return
+		}
+	case okBody := <-statusCh:
+		if okBody.Error != nil {
+			log.Errorf("Could not wait on container: %v", err)
+			return
+		} else {
+			log.Debugf("Container finished running with return code: %v", okBody.StatusCode)
+			pe.Run.Result = &okBody.StatusCode
 		}
-	case retCode := <-statusCh:
-		log.Debugf("Container finished running with return code: %v", retCode)
 	}
 
+	pe.Run.InProgress = false
+
 	out, err := cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true})
 	if err != nil {
-		return fmt.Errorf("could not get container logs: %w", err)
+		log.Errorf("could not get container logs: %w", err)
+		return
 	}
 
 	var stdOut bytes.Buffer
@@ -151,8 +256,10 @@ func ExecutePipeline(pe PipelineExecution, pipelineConf config.PipelineConf) err
 
 	stdcopy.StdCopy(&stdOut, &stdErr, out)
 
-	log.Debugf("%s", stdOut.Bytes())
-	log.Debugf("%s", stdErr.Bytes())
+	pe.Run.Stdout = stdOut.Bytes()
+	pe.Run.Stderr = stdErr.Bytes()
 
-	return nil
+	db.UpdateRunResult(pe.Run)
+
+	return
 }

pipeline_executor/util.go

@@ -0,0 +1,25 @@
+package pipeline_executor
+
+import (
+	"bufio"
+	"encoding/json"
+	"io"
+)
+
+func cleanupBuildOutput(input io.ReadCloser) string {
+	output := ""
+
+	scanner := bufio.NewScanner(input)
+	for scanner.Scan() {
+		var log map[string]any
+		json.Unmarshal(scanner.Bytes(), &log)
+
+		if logVar, ok := log["stream"]; ok {
+			if log, ok := logVar.(string); ok {
+				output += log
+			}
+		}
+	}
+
+	return output
+}

poll/poll.go

@@ -1,15 +1,18 @@
 package poll
 
 import (
+	"context"
 	"time"
 
-	"github.com/op/go-logging"
-
 	"git.ohea.xyz/cursorius/server/config"
+	"git.ohea.xyz/cursorius/server/database"
 	"git.ohea.xyz/cursorius/server/pipeline_executor"
+
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/storage/memory"
+	"github.com/google/uuid"
+	"github.com/op/go-logging"
 )
 
 var log = logging.MustGetLogger("cursorius-server")
@@ -24,17 +27,43 @@ type tag struct {
 	commitHash string
 }
 
-func pollJob(repoName string, jobConfig config.Job, pipelineConf config.PipelineConf) {
-	prevCommits := make(map[string]string)
+func pollJob(ctx context.Context, pipeline database.Pipeline, pipelineConf config.PipelineConf, db database.Database) {
+	firstScan := true
 	for {
-		time.Sleep(time.Duration(jobConfig.PollInterval) * time.Second)
-		log.Infof("Polling repo %v", repoName)
+		// Don't sleep on first scan to ease testing
+		// TODO: this should be replaced with a script that mocks a webhook
+		if !firstScan {
+
+			ctx, cancel := context.WithTimeout(ctx, time.Duration(pipeline.PollInterval)*time.Second)
+
+			select {
+			case <-ctx.Done():
+				switch ctx.Err() {
+				case context.Canceled:
+					log.Infof("Polling for pipeline %v canceled, stopping", pipeline.Name)
+					cancel()
+					return
+				}
+			}
+
+			cancel()
+
+			log.Infof("Polling repo %v", pipeline.Name)
+		} else {
+			firstScan = false
+		}
+
+		prevRefs, err := db.GetPipelineRefs(pipeline.Id)
+		if err != nil {
+			log.Errorf("Could not get pipeline refs from db: %v", err)
+			return
+		}
 
 		repo, err := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
-			URL: jobConfig.URL,
+			URL: pipeline.Url,
 		})
 		if err != nil {
-			log.Errorf("Could not clone repo %v from url %v: %v", repoName, jobConfig.URL, err)
+			log.Errorf("Could not clone repo %v from url %v: %v", pipeline.Name, pipeline.Url, err)
 			continue
 		}
 		refsToRunFor := []string{}
@@ -42,24 +71,24 @@ func pollJob(repoName string, jobConfig config.Job, pipelineConf config.Pipeline
 		// get branches
 		branches, err := repo.Branches()
 		if err != nil {
-			log.Errorf("Could not enumerate branches in repo %v: %v", repoName, err)
+			log.Errorf("Could not enumerate branches in repo %v: %v", pipeline.Name, err)
 			continue
 		}
 
 		branches.ForEach(func(branch *plumbing.Reference) error {
-			log.Debugf("Processing branch %v from repo %v", branch.Name().String(), repoName)
-			prevRef, ok := prevCommits[branch.Name().String()]
+			log.Debugf("Processing branch %v from repo %v (id: %v)", branch.Name().String(), pipeline.Name, pipeline.Id)
+			prevRef, ok := prevRefs[branch.Name().String()]
 			if ok {
 				if branch.Hash().String() != prevRef {
-					log.Debugf("Queuing job for branch %v in repo %v with hash %v", branch.Name().String(), repoName, branch.Hash().String())
-					prevCommits[branch.Name().String()] = branch.Hash().String()
+					log.Debugf("Queuing job for branch %v in repo %v (id: %v) with hash %v", branch.Name().String(), pipeline.Name, pipeline.Id, branch.Hash().String())
+					prevRefs[branch.Name().String()] = branch.Hash().String()
 					refsToRunFor = append(refsToRunFor, branch.Name().String())
 				} else {
-					log.Debugf("Branch %v in repo %v has hash %v, which matches the previously seen hash of %v", branch.Name().String(), repoName, branch.Hash().String(), prevRef)
+					log.Debugf("Branch %v in repo %v (id: %v) has hash %v, which matches the previously seen hash of %v", branch.Name().String(), pipeline.Name, pipeline.Id, branch.Hash().String(), prevRef)
 				}
 			} else {
-				log.Debugf("Queuing job for newly discovered branch %v in repo %v with hash %v", branch.Name().String(), repoName, branch.Hash().String())
-				prevCommits[branch.Name().String()] = branch.Hash().String()
+				log.Debugf("Queuing job for newly discovered branch %v in repo %v (id: %v) with hash %v", branch.Name().String(), pipeline.Name, pipeline.Id, branch.Hash().String())
+				prevRefs[branch.Name().String()] = branch.Hash().String()
 				refsToRunFor = append(refsToRunFor, branch.Name().String())
 			}
 			return nil
@@ -67,48 +96,99 @@ func pollJob(repoName string, jobConfig config.Job, pipelineConf config.Pipeline
 
 		tags, err := repo.Tags()
 		if err != nil {
-			log.Errorf("Could not enumerate tags in repo %v: %v", repoName, err)
+			log.Errorf("Could not enumerate tags in repo %v: %v", pipeline.Name, err)
 			continue
 		}
 		tags.ForEach(func(tag *plumbing.Reference) error {
-			log.Debugf("Processing tag %v from repo %v", tag.Name().String(), repoName)
-			prevRef, ok := prevCommits[tag.Name().String()]
+			log.Debugf("Processing tag %v from repo %v (id: %v)", tag.Name().String(), pipeline.Name, pipeline.Id)
+			prevRef, ok := prevRefs[tag.Name().String()]
 			if ok {
 				if tag.Hash().String() != prevRef {
-					log.Debugf("Queuing job for tag %v in repo %v with hash %v", tag.Name().String(), repoName, tag.Hash().String())
-					prevCommits[tag.Name().String()] = tag.Hash().String()
+					log.Debugf("Queuing job for tag %v in repo %v (id: %v) with hash %v", tag.Name().String(), pipeline.Name, pipeline.Id, tag.Hash().String())
+					prevRefs[tag.Name().String()] = tag.Hash().String()
 					refsToRunFor = append(refsToRunFor, tag.Name().String())
 				} else {
-					log.Debugf("Tag %v in repo %v has hash %v, which matches the previously seen hash of %v", tag.Name().String(), repoName, tag.Hash().String(), prevRef)
+					log.Debugf("Tag %v in repo %v (id: %v) has hash %v, which matches the previously seen hash of %v", tag.Name().String(), pipeline.Name, pipeline.Id, tag.Hash().String(), prevRef)
 				}
 			} else {
-				log.Debugf("Queuing job for newly discovered tag %v in repo %v with hash %v", tag.Name().String(), repoName, tag.Hash().String())
-				prevCommits[tag.Name().String()] = tag.Hash().String()
+				log.Debugf("Queuing job for newly discovered tag %v in repo %v (id: %v) with hash %v", tag.Name().String(), pipeline.Name, pipeline.Id, tag.Hash().String())
+				prevRefs[tag.Name().String()] = tag.Hash().String()
 				refsToRunFor = append(refsToRunFor, tag.Name().String())
 			}
 			return nil
 		})
 
-		for _, ref := range refsToRunFor {
-			log.Debugf("Dispatching job for ref %v in repo %v", ref, repoName)
+		err = db.UpdatePipelineRefs(pipeline.Id, prevRefs)
+		if err != nil {
+			log.Errorf("Could not update pipeline refs: %v", err)
+			return
+		}
 
-			pe := pipeline_executor.PipelineExecution{
-				Name: repoName,
-				Job:  jobConfig,
-				Ref:  ref,
+		for _, ref := range refsToRunFor {
+			log.Debugf("Dispatching job for ref %v in repo %v (id: %v)", ref, pipeline.Name, pipeline.Id)
+
+			run, err := db.CreateRun(pipeline.Id)
+			if err != nil {
+				log.Errorf("Could not create run for pipeline with id \"%v\": ", pipeline.Id, err)
+				continue
 			}
 
-			pipeline_executor.ExecutePipeline(pe, pipelineConf)
+			pe := pipeline_executor.PipelineExecution{
+				Pipeline: pipeline,
+				Ref:      ref,
+				Run:      run,
+			}
+
+			go pipeline_executor.ExecutePipeline(pe, db, pipelineConf)
 		}
 	}
 }
 
-func StartPolling(conf config.Config) {
-	for jobName, job := range conf.Jobs {
-		if job.PollInterval == 0 {
+func launchPollJobs(conf config.PipelineConf, db database.Database, pollChan chan uuid.UUID) {
+	pipelines, err := db.GetPipelines()
+	if err != nil {
+		log.Errorf("Could not get pipelines from database: %w", err)
+		return
+	}
+
+	pipelineCancelations := make(map[uuid.UUID]context.CancelFunc)
+
+	for _, pipeline := range pipelines {
+		if pipeline.PollInterval == 0 {
 			continue
 		} else {
-			go pollJob(jobName, job, conf.PipelineConf)
+			ctx, cancel := context.WithCancel(context.Background())
+			pipelineCancelations[pipeline.Id] = cancel
+
+			log.Infof("Starting polling for pipeline %v with id %v", pipeline.Name, pipeline.Id)
+			go pollJob(ctx, pipeline, conf, db)
 		}
 	}
+
+	for {
+		jobUUID := <-pollChan
+		pipeline, err := db.GetPipelineById(jobUUID)
+		if err != nil {
+			log.Errorf("Could not get pipeline with id \"%v\" from database: %v", err)
+			continue
+		}
+
+		// Cancel existing polling job if it exists
+		if cancelFunc, ok := pipelineCancelations[pipeline.Id]; ok {
+			cancelFunc()
+		}
+
+		// Start new polling job
+		log.Infof("Starting polling for pipeline %v with id %v", pipeline.Name, pipeline.Id)
+		ctx, cancel := context.WithCancel(context.Background())
+		pipelineCancelations[pipeline.Id] = cancel
+		go pollJob(ctx, pipeline, conf, db)
+	}
+
+}
+
+func StartPolling(conf config.PipelineConf, db database.Database) chan uuid.UUID {
+	pollChan := make(chan uuid.UUID)
+	go launchPollJobs(conf, db, pollChan)
+	return pollChan
 }

proto/api/v1/get_runner.proto

@@ -1,22 +0,0 @@
-syntax = "proto3";
-
-package api.v1;
-
-option go_package = "git.ohea.xyz/cursorius/server/proto/gen/api/v1;apiv1";
-
-message GetRunnerOptions {
-  bool clone_source = 2;
-}
-
-message GetRunnerRequest {
-  repeated string tags = 1;
-  GetRunnerOptions options = 2;
-}
-
-message GetRunnerResponse {
-  string id = 1;
-}
-
-service GetRunnerService {
-  rpc GetRunner(GetRunnerRequest) returns (GetRunnerResponse) {}
-}

proto/api/v1/run_command.proto

@@ -1,20 +0,0 @@
-syntax = "proto3";
-
-package api.v1;
-
-option go_package = "git.ohea.xyz/cursorius/server/proto/gen/api/v1;apiv1";
-
-message RunCommandRequest {
-  int64 runner_id = 1;
-  string command = 2;
-}
-
-message RunCommandResponse {
-  int64 return_code = 1;
-  string stdout = 2;
-  string stderr = 3;
-}
-
-service RunCommandService {
-  rpc RunCommand(RunCommandRequest) returns (RunCommandResponse) {}
-}

proto/buf.gen.yaml

@@ -1,8 +0,0 @@
-version: v1
-plugins:
-  - name: go
-    out: gen
-    opt: paths=source_relative
-  - name: connect-go
-    out: gen
-    opt: paths=source_relative

proto/buf.yaml

@@ -1,7 +0,0 @@
-version: v1
-breaking:
-  use:
-    - FILE
-lint:
-  use:
-    - DEFAULT

proto/gen/api/v1/apiv1connect/get_runner.connect.go

@@ -1,86 +0,0 @@
-// Code generated by protoc-gen-connect-go. DO NOT EDIT.
-//
-// Source: api/v1/get_runner.proto
-
-package apiv1connect
-
-import (
-	context "context"
-	errors "errors"
-	v1 "git.ohea.xyz/cursorius/server/proto/gen/api/v1"
-	connect_go "github.com/bufbuild/connect-go"
-	http "net/http"
-	strings "strings"
-)
-
-// This is a compile-time assertion to ensure that this generated file and the connect package are
-// compatible. If you get a compiler error that this constant is not defined, this code was
-// generated with a version of connect newer than the one compiled into your binary. You can fix the
-// problem by either regenerating this code with an older version of connect or updating the connect
-// version compiled into your binary.
-const _ = connect_go.IsAtLeastVersion0_1_0
-
-const (
-	// GetRunnerServiceName is the fully-qualified name of the GetRunnerService service.
-	GetRunnerServiceName = "api.v1.GetRunnerService"
-)
-
-// GetRunnerServiceClient is a client for the api.v1.GetRunnerService service.
-type GetRunnerServiceClient interface {
-	GetRunner(context.Context, *connect_go.Request[v1.GetRunnerRequest]) (*connect_go.Response[v1.GetRunnerResponse], error)
-}
-
-// NewGetRunnerServiceClient constructs a client for the api.v1.GetRunnerService service. By
-// default, it uses the Connect protocol with the binary Protobuf Codec, asks for gzipped responses,
-// and sends uncompressed requests. To use the gRPC or gRPC-Web protocols, supply the
-// connect.WithGRPC() or connect.WithGRPCWeb() options.
-//
-// The URL supplied here should be the base URL for the Connect or gRPC server (for example,
-// http://api.acme.com or https://acme.com/grpc).
-func NewGetRunnerServiceClient(httpClient connect_go.HTTPClient, baseURL string, opts ...connect_go.ClientOption) GetRunnerServiceClient {
-	baseURL = strings.TrimRight(baseURL, "/")
-	return &getRunnerServiceClient{
-		getRunner: connect_go.NewClient[v1.GetRunnerRequest, v1.GetRunnerResponse](
-			httpClient,
-			baseURL+"/api.v1.GetRunnerService/GetRunner",
-			opts...,
-		),
-	}
-}
-
-// getRunnerServiceClient implements GetRunnerServiceClient.
-type getRunnerServiceClient struct {
-	getRunner *connect_go.Client[v1.GetRunnerRequest, v1.GetRunnerResponse]
-}
-
-// GetRunner calls api.v1.GetRunnerService.GetRunner.
-func (c *getRunnerServiceClient) GetRunner(ctx context.Context, req *connect_go.Request[v1.GetRunnerRequest]) (*connect_go.Response[v1.GetRunnerResponse], error) {
-	return c.getRunner.CallUnary(ctx, req)
-}
-
-// GetRunnerServiceHandler is an implementation of the api.v1.GetRunnerService service.
-type GetRunnerServiceHandler interface {
-	GetRunner(context.Context, *connect_go.Request[v1.GetRunnerRequest]) (*connect_go.Response[v1.GetRunnerResponse], error)
-}
-
-// NewGetRunnerServiceHandler builds an HTTP handler from the service implementation. It returns the
-// path on which to mount the handler and the handler itself.
-//
-// By default, handlers support the Connect, gRPC, and gRPC-Web protocols with the binary Protobuf
-// and JSON codecs. They also support gzip compression.
-func NewGetRunnerServiceHandler(svc GetRunnerServiceHandler, opts ...connect_go.HandlerOption) (string, http.Handler) {
-	mux := http.NewServeMux()
-	mux.Handle("/api.v1.GetRunnerService/GetRunner", connect_go.NewUnaryHandler(
-		"/api.v1.GetRunnerService/GetRunner",
-		svc.GetRunner,
-		opts...,
-	))
-	return "/api.v1.GetRunnerService/", mux
-}
-
-// UnimplementedGetRunnerServiceHandler returns CodeUnimplemented from all methods.
-type UnimplementedGetRunnerServiceHandler struct{}
-
-func (UnimplementedGetRunnerServiceHandler) GetRunner(context.Context, *connect_go.Request[v1.GetRunnerRequest]) (*connect_go.Response[v1.GetRunnerResponse], error) {
-	return nil, connect_go.NewError(connect_go.CodeUnimplemented, errors.New("api.v1.GetRunnerService.GetRunner is not implemented"))
-}

proto/gen/api/v1/apiv1connect/run_command.connect.go

@@ -1,86 +0,0 @@
-// Code generated by protoc-gen-connect-go. DO NOT EDIT.
-//
-// Source: api/v1/run_command.proto
-
-package apiv1connect
-
-import (
-	context "context"
-	errors "errors"
-	v1 "git.ohea.xyz/cursorius/server/proto/gen/api/v1"
-	connect_go "github.com/bufbuild/connect-go"
-	http "net/http"
-	strings "strings"
-)
-
-// This is a compile-time assertion to ensure that this generated file and the connect package are
-// compatible. If you get a compiler error that this constant is not defined, this code was
-// generated with a version of connect newer than the one compiled into your binary. You can fix the
-// problem by either regenerating this code with an older version of connect or updating the connect
-// version compiled into your binary.
-const _ = connect_go.IsAtLeastVersion0_1_0
-
-const (
-	// RunCommandServiceName is the fully-qualified name of the RunCommandService service.
-	RunCommandServiceName = "api.v1.RunCommandService"
-)
-
-// RunCommandServiceClient is a client for the api.v1.RunCommandService service.
-type RunCommandServiceClient interface {
-	RunCommand(context.Context, *connect_go.Request[v1.RunCommandRequest]) (*connect_go.Response[v1.RunCommandResponse], error)
-}
-
-// NewRunCommandServiceClient constructs a client for the api.v1.RunCommandService service. By
-// default, it uses the Connect protocol with the binary Protobuf Codec, asks for gzipped responses,
-// and sends uncompressed requests. To use the gRPC or gRPC-Web protocols, supply the
-// connect.WithGRPC() or connect.WithGRPCWeb() options.
-//
-// The URL supplied here should be the base URL for the Connect or gRPC server (for example,
-// http://api.acme.com or https://acme.com/grpc).
-func NewRunCommandServiceClient(httpClient connect_go.HTTPClient, baseURL string, opts ...connect_go.ClientOption) RunCommandServiceClient {
-	baseURL = strings.TrimRight(baseURL, "/")
-	return &runCommandServiceClient{
-		runCommand: connect_go.NewClient[v1.RunCommandRequest, v1.RunCommandResponse](
-			httpClient,
-			baseURL+"/api.v1.RunCommandService/RunCommand",
-			opts...,
-		),
-	}
-}
-
-// runCommandServiceClient implements RunCommandServiceClient.
-type runCommandServiceClient struct {
-	runCommand *connect_go.Client[v1.RunCommandRequest, v1.RunCommandResponse]
-}
-
-// RunCommand calls api.v1.RunCommandService.RunCommand.
-func (c *runCommandServiceClient) RunCommand(ctx context.Context, req *connect_go.Request[v1.RunCommandRequest]) (*connect_go.Response[v1.RunCommandResponse], error) {
-	return c.runCommand.CallUnary(ctx, req)
-}
-
-// RunCommandServiceHandler is an implementation of the api.v1.RunCommandService service.
-type RunCommandServiceHandler interface {
-	RunCommand(context.Context, *connect_go.Request[v1.RunCommandRequest]) (*connect_go.Response[v1.RunCommandResponse], error)
-}
-
-// NewRunCommandServiceHandler builds an HTTP handler from the service implementation. It returns
-// the path on which to mount the handler and the handler itself.
-//
-// By default, handlers support the Connect, gRPC, and gRPC-Web protocols with the binary Protobuf
-// and JSON codecs. They also support gzip compression.
-func NewRunCommandServiceHandler(svc RunCommandServiceHandler, opts ...connect_go.HandlerOption) (string, http.Handler) {
-	mux := http.NewServeMux()
-	mux.Handle("/api.v1.RunCommandService/RunCommand", connect_go.NewUnaryHandler(
-		"/api.v1.RunCommandService/RunCommand",
-		svc.RunCommand,
-		opts...,
-	))
-	return "/api.v1.RunCommandService/", mux
-}
-
-// UnimplementedRunCommandServiceHandler returns CodeUnimplemented from all methods.
-type UnimplementedRunCommandServiceHandler struct{}
-
-func (UnimplementedRunCommandServiceHandler) RunCommand(context.Context, *connect_go.Request[v1.RunCommandRequest]) (*connect_go.Response[v1.RunCommandResponse], error) {
-	return nil, connect_go.NewError(connect_go.CodeUnimplemented, errors.New("api.v1.RunCommandService.RunCommand is not implemented"))
-}

proto/gen/api/v1/get_runner.pb.go

@@ -1,291 +0,0 @@
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// versions:
-// 	protoc-gen-go v1.28.1
-// 	protoc        (unknown)
-// source: api/v1/get_runner.proto
-
-package apiv1
-
-import (
-	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
-	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
-	reflect "reflect"
-	sync "sync"
-)
-
-const (
-	// Verify that this generated code is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
-	// Verify that runtime/protoimpl is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
-)
-
-type GetRunnerOptions struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	CloneSource bool `protobuf:"varint,2,opt,name=clone_source,json=cloneSource,proto3" json:"clone_source,omitempty"`
-}
-
-func (x *GetRunnerOptions) Reset() {
-	*x = GetRunnerOptions{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_api_v1_get_runner_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *GetRunnerOptions) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*GetRunnerOptions) ProtoMessage() {}
-
-func (x *GetRunnerOptions) ProtoReflect() protoreflect.Message {
-	mi := &file_api_v1_get_runner_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use GetRunnerOptions.ProtoReflect.Descriptor instead.
-func (*GetRunnerOptions) Descriptor() ([]byte, []int) {
-	return file_api_v1_get_runner_proto_rawDescGZIP(), []int{0}
-}
-
-func (x *GetRunnerOptions) GetCloneSource() bool {
-	if x != nil {
-		return x.CloneSource
-	}
-	return false
-}
-
-type GetRunnerRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Tags    []string          `protobuf:"bytes,1,rep,name=tags,proto3" json:"tags,omitempty"`
-	Options *GetRunnerOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"`
-}
-
-func (x *GetRunnerRequest) Reset() {
-	*x = GetRunnerRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_api_v1_get_runner_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *GetRunnerRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*GetRunnerRequest) ProtoMessage() {}
-
-func (x *GetRunnerRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_api_v1_get_runner_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use GetRunnerRequest.ProtoReflect.Descriptor instead.
-func (*GetRunnerRequest) Descriptor() ([]byte, []int) {
-	return file_api_v1_get_runner_proto_rawDescGZIP(), []int{1}
-}
-
-func (x *GetRunnerRequest) GetTags() []string {
-	if x != nil {
-		return x.Tags
-	}
-	return nil
-}
-
-func (x *GetRunnerRequest) GetOptions() *GetRunnerOptions {
-	if x != nil {
-		return x.Options
-	}
-	return nil
-}
-
-type GetRunnerResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"`
-}
-
-func (x *GetRunnerResponse) Reset() {
-	*x = GetRunnerResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_api_v1_get_runner_proto_msgTypes[2]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *GetRunnerResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*GetRunnerResponse) ProtoMessage() {}
-
-func (x *GetRunnerResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_api_v1_get_runner_proto_msgTypes[2]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use GetRunnerResponse.ProtoReflect.Descriptor instead.
-func (*GetRunnerResponse) Descriptor() ([]byte, []int) {
-	return file_api_v1_get_runner_proto_rawDescGZIP(), []int{2}
-}
-
-func (x *GetRunnerResponse) GetId() string {
-	if x != nil {
-		return x.Id
-	}
-	return ""
-}
-
-var File_api_v1_get_runner_proto protoreflect.FileDescriptor
-
-var file_api_v1_get_runner_proto_rawDesc = []byte{
-	0x0a, 0x17, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x67, 0x65, 0x74, 0x5f, 0x72, 0x75, 0x6e,
-	0x6e, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x61, 0x70, 0x69, 0x2e, 0x76,
-	0x31, 0x22, 0x35, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x4f, 0x70,
-	0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x6f, 0x6e, 0x65, 0x5f, 0x73,
-	0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0b, 0x63, 0x6c, 0x6f,
-	0x6e, 0x65, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0x5a, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x52,
-	0x75, 0x6e, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04,
-	0x74, 0x61, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73,
-	0x12, 0x32, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x0b, 0x32, 0x18, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x75,
-	0x6e, 0x6e, 0x65, 0x72, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74,
-	0x69, 0x6f, 0x6e, 0x73, 0x22, 0x23, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x6e, 0x65,
-	0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18,
-	0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x32, 0x56, 0x0a, 0x10, 0x47, 0x65, 0x74,
-	0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x42, 0x0a,
-	0x09, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x12, 0x18, 0x2e, 0x61, 0x70, 0x69,
-	0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65, 0x74, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71,
-	0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x47, 0x65,
-	0x74, 0x52, 0x75, 0x6e, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
-	0x00, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x2e, 0x6f, 0x68, 0x65, 0x61, 0x2e, 0x78, 0x79,
-	0x7a, 0x2f, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x69, 0x75, 0x73, 0x2f, 0x73, 0x65, 0x72, 0x76,
-	0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x70, 0x69,
-	0x2f, 0x76, 0x31, 0x3b, 0x61, 0x70, 0x69, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x33,
-}
-
-var (
-	file_api_v1_get_runner_proto_rawDescOnce sync.Once
-	file_api_v1_get_runner_proto_rawDescData = file_api_v1_get_runner_proto_rawDesc
-)
-
-func file_api_v1_get_runner_proto_rawDescGZIP() []byte {
-	file_api_v1_get_runner_proto_rawDescOnce.Do(func() {
-		file_api_v1_get_runner_proto_rawDescData = protoimpl.X.CompressGZIP(file_api_v1_get_runner_proto_rawDescData)
-	})
-	return file_api_v1_get_runner_proto_rawDescData
-}
-
-var file_api_v1_get_runner_proto_msgTypes = make([]protoimpl.MessageInfo, 3)
-var file_api_v1_get_runner_proto_goTypes = []interface{}{
-	(*GetRunnerOptions)(nil),  // 0: api.v1.GetRunnerOptions
-	(*GetRunnerRequest)(nil),  // 1: api.v1.GetRunnerRequest
-	(*GetRunnerResponse)(nil), // 2: api.v1.GetRunnerResponse
-}
-var file_api_v1_get_runner_proto_depIdxs = []int32{
-	0, // 0: api.v1.GetRunnerRequest.options:type_name -> api.v1.GetRunnerOptions
-	1, // 1: api.v1.GetRunnerService.GetRunner:input_type -> api.v1.GetRunnerRequest
-	2, // 2: api.v1.GetRunnerService.GetRunner:output_type -> api.v1.GetRunnerResponse
-	2, // [2:3] is the sub-list for method output_type
-	1, // [1:2] is the sub-list for method input_type
-	1, // [1:1] is the sub-list for extension type_name
-	1, // [1:1] is the sub-list for extension extendee
-	0, // [0:1] is the sub-list for field type_name
-}
-
-func init() { file_api_v1_get_runner_proto_init() }
-func file_api_v1_get_runner_proto_init() {
-	if File_api_v1_get_runner_proto != nil {
-		return
-	}
-	if !protoimpl.UnsafeEnabled {
-		file_api_v1_get_runner_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*GetRunnerOptions); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_api_v1_get_runner_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*GetRunnerRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_api_v1_get_runner_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*GetRunnerResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
-	type x struct{}
-	out := protoimpl.TypeBuilder{
-		File: protoimpl.DescBuilder{
-			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_api_v1_get_runner_proto_rawDesc,
-			NumEnums:      0,
-			NumMessages:   3,
-			NumExtensions: 0,
-			NumServices:   1,
-		},
-		GoTypes:           file_api_v1_get_runner_proto_goTypes,
-		DependencyIndexes: file_api_v1_get_runner_proto_depIdxs,
-		MessageInfos:      file_api_v1_get_runner_proto_msgTypes,
-	}.Build()
-	File_api_v1_get_runner_proto = out.File
-	file_api_v1_get_runner_proto_rawDesc = nil
-	file_api_v1_get_runner_proto_goTypes = nil
-	file_api_v1_get_runner_proto_depIdxs = nil
-}

proto/gen/api/v1/run_command.pb.go

@@ -1,246 +0,0 @@
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// versions:
-// 	protoc-gen-go v1.28.1
-// 	protoc        (unknown)
-// source: api/v1/run_command.proto
-
-package apiv1
-
-import (
-	protoreflect "google.golang.org/protobuf/reflect/protoreflect"
-	protoimpl "google.golang.org/protobuf/runtime/protoimpl"
-	reflect "reflect"
-	sync "sync"
-)
-
-const (
-	// Verify that this generated code is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion)
-	// Verify that runtime/protoimpl is sufficiently up-to-date.
-	_ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20)
-)
-
-type RunCommandRequest struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	RunnerId int64  `protobuf:"varint,1,opt,name=runner_id,json=runnerId,proto3" json:"runner_id,omitempty"`
-	Command  string `protobuf:"bytes,2,opt,name=command,proto3" json:"command,omitempty"`
-}
-
-func (x *RunCommandRequest) Reset() {
-	*x = RunCommandRequest{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_api_v1_run_command_proto_msgTypes[0]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *RunCommandRequest) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*RunCommandRequest) ProtoMessage() {}
-
-func (x *RunCommandRequest) ProtoReflect() protoreflect.Message {
-	mi := &file_api_v1_run_command_proto_msgTypes[0]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use RunCommandRequest.ProtoReflect.Descriptor instead.
-func (*RunCommandRequest) Descriptor() ([]byte, []int) {
-	return file_api_v1_run_command_proto_rawDescGZIP(), []int{0}
-}
-
-func (x *RunCommandRequest) GetRunnerId() int64 {
-	if x != nil {
-		return x.RunnerId
-	}
-	return 0
-}
-
-func (x *RunCommandRequest) GetCommand() string {
-	if x != nil {
-		return x.Command
-	}
-	return ""
-}
-
-type RunCommandResponse struct {
-	state         protoimpl.MessageState
-	sizeCache     protoimpl.SizeCache
-	unknownFields protoimpl.UnknownFields
-
-	ReturnCode int64  `protobuf:"varint,1,opt,name=return_code,json=returnCode,proto3" json:"return_code,omitempty"`
-	Stdout     string `protobuf:"bytes,2,opt,name=stdout,proto3" json:"stdout,omitempty"`
-	Stderr     string `protobuf:"bytes,3,opt,name=stderr,proto3" json:"stderr,omitempty"`
-}
-
-func (x *RunCommandResponse) Reset() {
-	*x = RunCommandResponse{}
-	if protoimpl.UnsafeEnabled {
-		mi := &file_api_v1_run_command_proto_msgTypes[1]
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		ms.StoreMessageInfo(mi)
-	}
-}
-
-func (x *RunCommandResponse) String() string {
-	return protoimpl.X.MessageStringOf(x)
-}
-
-func (*RunCommandResponse) ProtoMessage() {}
-
-func (x *RunCommandResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_api_v1_run_command_proto_msgTypes[1]
-	if protoimpl.UnsafeEnabled && x != nil {
-		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
-		if ms.LoadMessageInfo() == nil {
-			ms.StoreMessageInfo(mi)
-		}
-		return ms
-	}
-	return mi.MessageOf(x)
-}
-
-// Deprecated: Use RunCommandResponse.ProtoReflect.Descriptor instead.
-func (*RunCommandResponse) Descriptor() ([]byte, []int) {
-	return file_api_v1_run_command_proto_rawDescGZIP(), []int{1}
-}
-
-func (x *RunCommandResponse) GetReturnCode() int64 {
-	if x != nil {
-		return x.ReturnCode
-	}
-	return 0
-}
-
-func (x *RunCommandResponse) GetStdout() string {
-	if x != nil {
-		return x.Stdout
-	}
-	return ""
-}
-
-func (x *RunCommandResponse) GetStderr() string {
-	if x != nil {
-		return x.Stderr
-	}
-	return ""
-}
-
-var File_api_v1_run_command_proto protoreflect.FileDescriptor
-
-var file_api_v1_run_command_proto_rawDesc = []byte{
-	0x0a, 0x18, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x72, 0x75, 0x6e, 0x5f, 0x63, 0x6f, 0x6d,
-	0x6d, 0x61, 0x6e, 0x64, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, 0x61, 0x70, 0x69, 0x2e,
-	0x76, 0x31, 0x22, 0x4a, 0x0a, 0x11, 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64,
-	0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x72, 0x75, 0x6e, 0x6e, 0x65,
-	0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x72, 0x75, 0x6e, 0x6e,
-	0x65, 0x72, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x18,
-	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x22, 0x65,
-	0x0a, 0x12, 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x70,
-	0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x72, 0x65, 0x74, 0x75, 0x72, 0x6e, 0x5f, 0x63,
-	0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x72, 0x65, 0x74, 0x75, 0x72,
-	0x6e, 0x43, 0x6f, 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x18,
-	0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x64, 0x6f, 0x75, 0x74, 0x12, 0x16, 0x0a,
-	0x06, 0x73, 0x74, 0x64, 0x65, 0x72, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73,
-	0x74, 0x64, 0x65, 0x72, 0x72, 0x32, 0x5a, 0x0a, 0x11, 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x6d, 0x6d,
-	0x61, 0x6e, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x52, 0x75,
-	0x6e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x12, 0x19, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76,
-	0x31, 0x2e, 0x52, 0x75, 0x6e, 0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x71, 0x75,
-	0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x61, 0x70, 0x69, 0x2e, 0x76, 0x31, 0x2e, 0x52, 0x75, 0x6e,
-	0x43, 0x6f, 0x6d, 0x6d, 0x61, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
-	0x00, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x2e, 0x6f, 0x68, 0x65, 0x61, 0x2e, 0x78, 0x79,
-	0x7a, 0x2f, 0x63, 0x75, 0x72, 0x73, 0x6f, 0x72, 0x69, 0x75, 0x73, 0x2f, 0x73, 0x65, 0x72, 0x76,
-	0x65, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x61, 0x70, 0x69,
-	0x2f, 0x76, 0x31, 0x3b, 0x61, 0x70, 0x69, 0x76, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
-	0x33,
-}
-
-var (
-	file_api_v1_run_command_proto_rawDescOnce sync.Once
-	file_api_v1_run_command_proto_rawDescData = file_api_v1_run_command_proto_rawDesc
-)
-
-func file_api_v1_run_command_proto_rawDescGZIP() []byte {
-	file_api_v1_run_command_proto_rawDescOnce.Do(func() {
-		file_api_v1_run_command_proto_rawDescData = protoimpl.X.CompressGZIP(file_api_v1_run_command_proto_rawDescData)
-	})
-	return file_api_v1_run_command_proto_rawDescData
-}
-
-var file_api_v1_run_command_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
-var file_api_v1_run_command_proto_goTypes = []interface{}{
-	(*RunCommandRequest)(nil),  // 0: api.v1.RunCommandRequest
-	(*RunCommandResponse)(nil), // 1: api.v1.RunCommandResponse
-}
-var file_api_v1_run_command_proto_depIdxs = []int32{
-	0, // 0: api.v1.RunCommandService.RunCommand:input_type -> api.v1.RunCommandRequest
-	1, // 1: api.v1.RunCommandService.RunCommand:output_type -> api.v1.RunCommandResponse
-	1, // [1:2] is the sub-list for method output_type
-	0, // [0:1] is the sub-list for method input_type
-	0, // [0:0] is the sub-list for extension type_name
-	0, // [0:0] is the sub-list for extension extendee
-	0, // [0:0] is the sub-list for field type_name
-}
-
-func init() { file_api_v1_run_command_proto_init() }
-func file_api_v1_run_command_proto_init() {
-	if File_api_v1_run_command_proto != nil {
-		return
-	}
-	if !protoimpl.UnsafeEnabled {
-		file_api_v1_run_command_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*RunCommandRequest); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-		file_api_v1_run_command_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*RunCommandResponse); i {
-			case 0:
-				return &v.state
-			case 1:
-				return &v.sizeCache
-			case 2:
-				return &v.unknownFields
-			default:
-				return nil
-			}
-		}
-	}
-	type x struct{}
-	out := protoimpl.TypeBuilder{
-		File: protoimpl.DescBuilder{
-			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
-			RawDescriptor: file_api_v1_run_command_proto_rawDesc,
-			NumEnums:      0,
-			NumMessages:   2,
-			NumExtensions: 0,
-			NumServices:   1,
-		},
-		GoTypes:           file_api_v1_run_command_proto_goTypes,
-		DependencyIndexes: file_api_v1_run_command_proto_depIdxs,
-		MessageInfos:      file_api_v1_run_command_proto_msgTypes,
-	}.Build()
-	File_api_v1_run_command_proto = out.File
-	file_api_v1_run_command_proto_rawDesc = nil
-	file_api_v1_run_command_proto_goTypes = nil
-	file_api_v1_run_command_proto_depIdxs = nil
-}

runnermanager/runner.go

@@ -0,0 +1,111 @@
+package runnermanager
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/google/uuid"
+	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/reflect/protoreflect"
+	"nhooyr.io/websocket"
+
+	runner_api "git.ohea.xyz/cursorius/runner-api/go/api/v2"
+)
+
+type RunnerData struct {
+	msgType websocket.MessageType
+	data    []byte
+}
+
+type Runner struct {
+	id          uuid.UUID
+	tags        []string
+	conn        *websocket.Conn
+	receiveChan chan []byte
+}
+
+func (r *Runner) HasTags(requestedTags []string) bool {
+tagIter:
+	for _, requestedTag := range requestedTags {
+		for _, posessedTag := range r.tags {
+			// if we find the tag, move on to search for the next one
+			if posessedTag == requestedTag {
+				continue tagIter
+			}
+		}
+		// if we don't find the tag
+		return false
+	}
+	return true
+}
+
+func (r *Runner) Id() uuid.UUID {
+	return r.id
+}
+
+func (r *Runner) RunCommand(cmd string, args []string) (returnCode int64, stdout string, stderr string, err error) {
+
+	if r.conn == nil {
+		return 0, "", "", fmt.Errorf("runner with id %v has nil conn, THIS IS A BUG", r.id)
+	}
+
+	// Write RunCommand message to client
+	serverToRunnerMsg := &runner_api.ServerToRunnerMsg{
+		Msg: &runner_api.ServerToRunnerMsg_RunCommandMsg{
+			RunCommandMsg: &runner_api.RunCommand{
+				Command: cmd,
+				Args:    args,
+			},
+		},
+	}
+
+	err = r.sendProtoStruct(serverToRunnerMsg)
+	if err != nil {
+		err = fmt.Errorf("Could not send command to client: %w", err)
+		return
+	}
+
+	for {
+		// Read RunCommandFinalResponse message from client
+		data, ok := <-r.receiveChan
+		if !ok {
+			err = fmt.Errorf("Channel is closed on runner")
+			return
+		}
+
+		runnerToServerMsg := &runner_api.RunnerToServerMsg{}
+		if err = proto.Unmarshal(data, runnerToServerMsg); err != nil {
+			err = fmt.Errorf("Could not parse RunCommand response: %w", err)
+			r.conn.Close(websocket.StatusUnsupportedData, "Invalid message")
+			return
+		}
+
+		switch x := runnerToServerMsg.Msg.(type) {
+		case *runner_api.RunnerToServerMsg_RunCommandPartialResponseMsg:
+			stdout += x.RunCommandPartialResponseMsg.Stdout
+			stderr += x.RunCommandPartialResponseMsg.Stderr
+		case *runner_api.RunnerToServerMsg_RunCommandFinalResponseMsg:
+			stdout += x.RunCommandFinalResponseMsg.PartialResponse.Stdout
+			stderr += x.RunCommandFinalResponseMsg.PartialResponse.Stderr
+			returnCode = x.RunCommandFinalResponseMsg.ReturnCode
+			return
+		}
+	}
+}
+
+func (r *Runner) sendProtoStruct(p protoreflect.ProtoMessage) error {
+	protoOut, err := proto.Marshal(p)
+	if err != nil {
+		return fmt.Errorf("Could not marshal proto: %w", err)
+	}
+
+	ctx := context.Background()
+
+	log.Debugf("r.conn: %p", r.conn)
+
+	if err := r.conn.Write(ctx, websocket.MessageBinary, protoOut); err != nil {
+		return fmt.Errorf("Could not send proto to websocket: %w", err)
+	}
+
+	return nil
+}

runnermanager/runnermanager.go

@@ -3,180 +3,204 @@ package runnermanager
 import (
 	"context"
 	"fmt"
-	"strings"
 	"time"
 
-	"git.ohea.xyz/cursorius/server/config"
+	"github.com/google/uuid"
 	"github.com/op/go-logging"
+	"google.golang.org/protobuf/proto"
 	"nhooyr.io/websocket"
-	"nhooyr.io/websocket/wsjson"
+
+	"git.ohea.xyz/cursorius/server/config"
+	"git.ohea.xyz/cursorius/server/database"
+	"git.ohea.xyz/cursorius/server/util"
+
+	runner_api "git.ohea.xyz/cursorius/runner-api/go/api/v2"
 )
 
 var log = logging.MustGetLogger("cursorius-server")
 
-type RunnerRegistration struct {
-	Secret string
-	Id     string
-	Tags   []string
-	conn   *websocket.Conn
+func (r *runnerManager) processRunnerAllocation(req RunnerAllocationRequest) {
+	tagsStr := util.FormatTags(req.Tags)
+	log.Infof("Got request for runner with tags \"%v\"", tagsStr)
+
+	log.Debugf("Finding runner with tags %v", tagsStr)
+
+	foundRunner := false
+
+	runnersToRemove := []int{}
+runnerIter:
+	for i, runner := range r.connectedRunners {
+		// don't allocate runner with closed receiveChan (is defunct)
+		// there should never be messages to read on an inactive runner,
+		// so we aren't losing any data here
+		select {
+		case _, ok := <-runner.receiveChan:
+			if ok {
+				// this should never happen
+				// TODO: should we disconnect the runner?
+				log.Errorf("Recieved data from inactive runner %v, this is a bug", runner.id)
+				continue
+			}
+			log.Noticef("Removing defunct runner \"%v\"", runner.id)
+			runnersToRemove = append(runnersToRemove, i)
+		default:
+			log.Debugf("Checking runner %v for requested tags", runner.id)
+
+			if !runner.HasTags(req.Tags) {
+				continue runnerIter
+			}
+
+			runnersToRemove = append(runnersToRemove, i)
+			foundRunner = true
+			log.Debugf("Runner %v has requested tags, allocating", runner.id)
+			req.RespChan <- RunnerAllocationResponse{
+				Runner: &r.connectedRunners[i],
+				Err:    nil,
+			}
+		}
+
+	}
+	// remove allocated runner plus defunct runners
+	// since we iterate, all the indexes will be in accending order
+	for i, runnerInd := range runnersToRemove {
+		r.connectedRunners[runnerInd-i] = r.connectedRunners[len(r.connectedRunners)-1]
+		r.connectedRunners = r.connectedRunners[0 : len(r.connectedRunners)-1]
+	}
+
+	if foundRunner {
+		return
+	}
+
+	errorMsg := "could not find valid runner"
+	if len(r.connectedRunners) == 0 {
+		errorMsg = "no connected runners"
+	}
+	req.RespChan <- RunnerAllocationResponse{
+		Runner: nil,
+		Err:    fmt.Errorf("Could not allocate runner: %v", errorMsg),
+	}
+
 }
 
-type RunnerData struct {
-	msgType websocket.MessageType
-	data    []byte
-}
-type Runner struct {
-	id          string
-	tags        []string
-	conn        *websocket.Conn
-	receiveChan chan RunnerData
-	running     bool
+func (r *runnerManager) processRunnerRegistration(req RunnerRegistrationRequest) {
+	log.Debugf("New runner appeared with id: %v and secret: %v", req.Id, req.Secret)
+
+	// Get runner with give id from database
+	runnerId, err := uuid.Parse(req.Id)
+	if err != nil {
+		log.Errorf("Disconnecting runner with id: %v, could not parse as UUID: %v", req.Id, err)
+		req.conn.Close(websocket.StatusNormalClosure, "registration invalid")
+		return
+	}
+	dbRunner, err := r.db.GetRunnerById(runnerId)
+	if err != nil {
+		log.Errorf("Disconnecting runner with id: %v, could not find runner in DB: %v", runnerId, err)
+		req.conn.Close(websocket.StatusNormalClosure, "registration invalid")
+		return
+	}
+
+	if req.Secret != dbRunner.Token {
+		log.Errorf("Disconnecting runner with id: %v, invalid secret", runnerId)
+		req.conn.Close(websocket.StatusNormalClosure, "registration invalid")
+		return
+	}
+
+	log.Infof("Registering runner \"%v\" with tags %v", req.Id, req.Tags)
+	runner := Runner{
+		id:          runnerId,
+		tags:        req.Tags,
+		conn:        req.conn,
+		receiveChan: make(chan []byte),
+	}
+	r.connectedRunners = append(r.connectedRunners, runner)
+	// start goroutine to call Read function on websocket connection
+	// this is required to keep the connection functioning
+	go func() {
+		defer log.Noticef("Deregistered runner with id: %v", runner.id)
+		defer close(runner.receiveChan)
+		for {
+			msgType, data, err := req.conn.Read(context.Background())
+			if err != nil {
+				// TODO: this is still racy, since a runner could be allocated between the
+				// connection returning an err and the channel closing
+				// This should probably be handled by sending erroring, but not 100% sure
+				log.Errorf("Could not read from connection: %v", err)
+				return
+			}
+			if msgType != websocket.MessageBinary {
+				close(runner.receiveChan)
+				log.Errorf("Got binary data from connection")
+				return
+			}
+
+			runner.receiveChan <- data
+
+		}
+	}()
 }
 
-type runnerManager struct {
-	getRunnerCh       chan GetRunnerRequest
-	registerCh        chan RunnerRegistration
-	connectedRunners  []Runner
-	configuredRunners map[string]config.Runner
-}
-
-type GetRunnerRequest struct {
-	Tags     []string
-	RespChan chan GetRunnerResponse
-}
-
-type GetRunnerResponse struct {
-	Runner Runner
-	Err    error
-}
-
-type runnerJob struct {
-	Id  string
-	URL string
+func (r *runnerManager) processRunnerRelease(req RunnerReleaseRequest) {
+	r.connectedRunners = append(r.connectedRunners, *req.Runner)
 }
 
 func runRunnerManager(r runnerManager) {
 	for {
-	msgCase:
 		select {
-		case request := <-r.getRunnerCh:
-
-			var runnerTagsStr strings.Builder
-			fmt.Fprintf(&runnerTagsStr, "%v", request.Tags[0])
-			for _, tag := range request.Tags[1:] {
-				fmt.Fprintf(&runnerTagsStr, " %v", tag)
-			}
-			log.Infof("Got request for runner with tags \"%v\"", runnerTagsStr.String())
-
-			log.Debugf("Finding runner with tags %v", runnerTagsStr)
-
-			for i, runner := range r.connectedRunners {
-				// don't allocate runner that is already occupied
-				if runner.running {
-					log.Debugf("Skipping runner %v, as runner is activly running another job", runner.id)
-					continue
-				}
-				// don't allocate runner with closed receiveChan (is defunct)
-				// there should never be messages to read on an inactive runner,
-				// so we aren't losing any data here
-				select {
-				case _, ok := <-runner.receiveChan:
-					if ok {
-						// this should never happen
-						log.Errorf("Recieved data from inactive runner %v, this is a bug", runner.id)
-						continue
-					}
-					log.Noticef("Removing defunct runner \"%v\"", runner.id)
-					// if the receive channel is closed, swap delete the runner as it's defunct
-					r.connectedRunners[i] = r.connectedRunners[len(r.connectedRunners)-1]
-					r.connectedRunners = r.connectedRunners[:len(r.connectedRunners)-1]
-				default:
-					runner.running = true
-					request.RespChan <- GetRunnerResponse{
-						Runner: runner,
-						Err:    nil,
-					}
-					break msgCase
-				}
-			}
-			errorMsg := "could not find valid runner"
-			if len(r.connectedRunners) == 0 {
-				errorMsg = "no connected runners"
-			}
-			log.Errorf("Could not allocate runner with tags \"%v\": %v", runnerTagsStr.String(), errorMsg)
-			request.RespChan <- GetRunnerResponse{
-				Runner: Runner{},
-				Err:    fmt.Errorf("Could not allocate runner: %v", errorMsg),
-			}
-
-		case registration := <-r.registerCh:
-			log.Debugf("New runner appeared with id: %v and secret: %v", registration.Id, registration.Secret)
-			if configuredRunner, doesExist := r.configuredRunners[registration.Id]; doesExist {
-				if configuredRunner.Secret == registration.Secret {
-					log.Infof("Registering runner \"%v\" with tags %v", registration.Id, registration.Tags)
-					runner := Runner{
-						id:          registration.Id,
-						tags:        registration.Tags,
-						conn:        registration.conn,
-						receiveChan: make(chan RunnerData),
-						running:     false,
-					}
-					r.connectedRunners = append(r.connectedRunners, runner)
-					// start goroutine to call Read function on websocket connection
-					// this is required to keep the connection functioning
-					go func() {
-						for {
-							msgType, data, err := registration.conn.Read(context.Background())
-							if err != nil {
-								// TODO: this is still racy, since a runner could be alloctade between the
-								// connection returning an err and the channel closing
-								close(runner.receiveChan)
-								log.Errorf("Could not read from connection: %v", err)
-								log.Noticef("Deregistering runner with id: %v", runner.id)
-
-								return
-							} else {
-								log.Debugf("%v: %v", msgType, data)
-								runner.receiveChan <- RunnerData{msgType: msgType, data: data}
-							}
-						}
-					}()
-
-				} else {
-					log.Errorf("Disconnecting runner with id: %v and invalid secret: %v", registration.Id, registration.Secret)
-					registration.conn.Close(websocket.StatusNormalClosure, "registration invalid")
-				}
-			} else {
-				log.Errorf("Disconnecting runner with invalid id: %v", registration.Id)
-				registration.conn.Close(websocket.StatusNormalClosure, "registration invalid")
-			}
+		case request := <-r.chans.Allocation:
+			r.processRunnerAllocation(request)
+		case release := <-r.chans.Release:
+			r.processRunnerRelease(release)
+		case registration := <-r.chans.Registration:
+			r.processRunnerRegistration(registration)
 		}
 	}
 }
 
-func StartRunnerManager(configuredRunners map[string]config.Runner) (chan GetRunnerRequest, chan RunnerRegistration, error) {
+func StartRunnerManager(configuredRunners map[string]config.Runner, db database.Database) (RunnerManagerChans, error) {
 	scheduler := runnerManager{
-		getRunnerCh:       make(chan GetRunnerRequest),
-		registerCh:        make(chan RunnerRegistration),
+		chans: RunnerManagerChans{
+			Allocation:   make(chan RunnerAllocationRequest),
+			Release:      make(chan RunnerReleaseRequest),
+			Registration: make(chan RunnerRegistrationRequest),
+		},
 		connectedRunners:  make([]Runner, 0),
 		configuredRunners: configuredRunners,
+		db:                db,
 	}
 
 	go runRunnerManager(scheduler)
 
-	return scheduler.getRunnerCh, scheduler.registerCh, nil
+	return scheduler.chans, nil
 }
 
-func RegisterRunner(conn *websocket.Conn, registerCh chan RunnerRegistration) {
+func RegisterRunner(conn *websocket.Conn, registerCh chan RunnerRegistrationRequest) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
 	defer cancel()
 
-	var registration RunnerRegistration
+	var registration RunnerRegistrationRequest
 	registration.conn = conn
-	err := wsjson.Read(ctx, conn, &registration)
+
+	typ, r, err := conn.Read(ctx)
 	if err != nil {
-		log.Errorf("Could not read data from websocket connection: %v", err)
+		log.Errorf("Could not read from runner websocket connection: %v", err)
+		log.Errorf("Disconnecting...")
 		return
 	}
+	if typ != websocket.MessageBinary {
+		log.Error("Got non binary message from runner, disconnecting...")
+		conn.Close(websocket.StatusUnsupportedData, "Requires binary data")
+		return
+	}
+	registration_proto := &runner_api.Register{}
+	if err := proto.Unmarshal(r, registration_proto); err != nil {
+		log.Error("Could not parse registration message from runner, disconnection....")
+		conn.Close(websocket.StatusUnsupportedData, "Invalid message")
+		return
+	}
+
+	registration.Secret = registration_proto.Secret
+	registration.Id = registration_proto.Id
+	registration.Tags = registration_proto.Tags
+
 	registerCh <- registration
 }

runnermanager/types.go

@@ -0,0 +1,49 @@
+package runnermanager
+
+import (
+	"nhooyr.io/websocket"
+
+	"git.ohea.xyz/cursorius/server/config"
+	"git.ohea.xyz/cursorius/server/database"
+)
+
+type RunnerManagerChans struct {
+	Allocation   chan RunnerAllocationRequest
+	Release      chan RunnerReleaseRequest
+	Registration chan RunnerRegistrationRequest
+}
+
+type runnerManager struct {
+	chans               RunnerManagerChans
+	connectedRunners    []Runner
+	numConnectedRunners uint64
+	configuredRunners   map[string]config.Runner
+	db                  database.Database
+}
+
+type RunnerAllocationRequest struct {
+	Tags       []string
+	RespChan   chan RunnerAllocationResponse
+	CancelChan chan string
+}
+
+type RunnerAllocationResponse struct {
+	Runner *Runner
+	Err    error
+}
+
+type RunnerReleaseRequest struct {
+	Runner *Runner
+}
+
+type RunnerRegistrationRequest struct {
+	Secret string
+	Id     string
+	Tags   []string
+	conn   *websocket.Conn
+}
+
+type runnerJob struct {
+	Id  string
+	URL string
+}

util/util.go

@@ -0,0 +1,18 @@
+package util
+
+import (
+	"fmt"
+	"strings"
+)
+
+func FormatTags(tags []string) string {
+	var tagsStr strings.Builder
+	if len(tags) > 0 {
+		fmt.Fprintf(&tagsStr, "[%v", tags[0])
+		for _, tag := range tags[1:] {
+			fmt.Fprintf(&tagsStr, ", %v", tag)
+		}
+		fmt.Fprintf(&tagsStr, "]")
+	}
+	return tagsStr.String()
+}

webhook/webhook.go

@@ -5,79 +5,97 @@ import (
 	"strings"
 
 	"git.ohea.xyz/cursorius/server/config"
+	"git.ohea.xyz/cursorius/server/database"
 	"git.ohea.xyz/cursorius/server/pipeline_executor"
 	"github.com/go-playground/webhooks/v6/gitea"
+	"github.com/google/uuid"
 	"github.com/op/go-logging"
 )
 
 var log = logging.MustGetLogger("cursorius-server")
 
-func CreateWebhookHandler(conf config.Config, mux *http.ServeMux) {
-	mux.HandleFunc("/webhook/", func(w http.ResponseWriter, r *http.Request) {
-		switch r.Method {
-		case "POST":
-			splitUrl := strings.Split(r.URL.Path, "/")
-			if len(splitUrl) != 3 {
-				log.Errorf("Webhook recieved with invalid url \"%v\", ignoring...", r.URL)
-				return
-			}
+func webhookHandler(w http.ResponseWriter, r *http.Request, db database.Database, conf config.PipelineConf) {
+	switch r.Method {
+	case "POST":
+		splitUrl := strings.Split(r.URL.Path, "/")
+		if len(splitUrl) != 4 {
+			log.Errorf("Webhook recieved with invalid url \"%v\", ignoring...", r.URL)
+			return
+		}
 
-			// get URL path after /webhook/
-			// TODO: verify that this handles all valid URL formats
-			webhookJobName := splitUrl[2]
+		// get URL path after /webhook/
+		// TODO: verify that this handles all valid URL formats
+		pipelineUUIDStr := splitUrl[2]
+		webhookUUIDStr := splitUrl[3]
 
-			for jobName, jobConfig := range conf.Jobs {
-				if webhookJobName == jobName {
-					if jobConfig.Webhook == nil {
-						log.Errorf("Matching job does not have webhook configuration, ignoring....")
+		pipelineUUID, err := uuid.Parse(pipelineUUIDStr)
+		if err != nil {
+			log.Errorf("Could not parse pipeline UUID: %v", err)
+			return
+		}
+		webhookUUID, err := uuid.Parse(webhookUUIDStr)
+		if err != nil {
+			log.Errorf("Could not parse webhook UUID: %v", err)
+			return
+		}
+
+		pipeline, err := db.GetPipelineById(pipelineUUID)
+		if err != nil {
+			log.Errorf("Could not get webhooks for pipeline with UUID \"%v\": %v", pipelineUUID, err)
+			return
+		}
+
+		webhooks, err := db.GetWebhooksForPipeline(pipelineUUID)
+		if err != nil {
+			log.Errorf("Could not get webhooks for pipeline with UUID \"%v\": %v", webhookUUID, err)
+			return
+		}
+
+		if len(webhooks) < 1 {
+			log.Errorf("No webhooks configured for pipeline with UUID \"%v\"", webhookUUID)
+			return
+		}
+
+		for _, webhook := range webhooks {
+			if webhook.Id == webhookUUID {
+				switch webhook.ServerType {
+				case database.Gitea:
+					hook, err := gitea.New(gitea.Options.Secret(webhook.Secret))
+					if err != nil {
+						log.Errorf("Could not create Gitea webhook handler: %v", err)
 						return
 					}
-					switch jobConfig.Webhook.Sender {
-					case config.Gitea:
-						hook, err := gitea.New(gitea.Options.Secret(jobConfig.Webhook.Secret))
-						if err != nil {
-							log.Errorf("Could not create Gitea webhook handler: %v", err)
-							return
+					payload, err := hook.Parse(r, gitea.PushEvent)
+					if err != nil {
+						if err == gitea.ErrEventNotFound {
+							log.Warning("Got webhook \"%v\" for unexpected event type, ignoring...", webhookUUID)
+							break
 						}
-						payload, err := hook.Parse(r, gitea.PushEvent)
-						if err != nil {
-							if err == gitea.ErrEventNotFound {
-								log.Info("Got webhook for unexpected event type, ignoring...")
-								break
-							}
-							log.Errorf("Could not parse webhook: %v", err)
-							return
+					}
+
+					switch payload.(type) {
+					case gitea.PushPayload:
+						pushPayload := payload.(gitea.PushPayload)
+
+						pe := pipeline_executor.PipelineExecution{
+							Pipeline: pipeline,
+							Ref:      pushPayload.Ref,
 						}
-						log.Infof("Got webhook with payload %v", payload)
-
-						switch payload.(type) {
-						case gitea.PushPayload:
-							pushPayload := payload.(gitea.PushPayload)
-
-							pe := pipeline_executor.PipelineExecution{
-								Name: webhookJobName,
-								Job:  jobConfig,
-								Ref:  pushPayload.Ref,
-							}
-
-							pipeline_executor.ExecutePipeline(pe, conf.PipelineConf)
-						}
-
-						return
-					default:
-						log.Errorf("Job configured with unknown webhook sender \"%v\", igonring...", jobConfig.Webhook.Sender)
-						return
-
+						go pipeline_executor.ExecutePipeline(pe, db, conf)
 					}
 				}
 			}
-
-			log.Errorf("Not job configured with name \"%v\", required by webhook with url \"%v\", ignoring...",
-				webhookJobName, r.URL)
-
-		default:
-			log.Errorf("Got request with method \"%v\", ignoring...", r.Method)
 		}
-	})
 
+		log.Errorf("No webhook found with ID \"%v\"", webhookUUID)
+
+	default:
+		log.Errorf("Got request with method \"%v\", ignoring...", r.Method)
+	}
+}
+
+func CreateWebhookHandler(db database.Database, conf config.PipelineConf, mux *http.ServeMux) {
+	mux.HandleFunc("/webhook/", func(w http.ResponseWriter, r *http.Request) {
+		webhookHandler(w, r, db, conf)
+	})
 }