Fix secret name validation not checking for beginning of string

admin_api/admin_api.go

@@ -14,6 +14,43 @@ import (
 var log = logging.MustGetLogger("cursorius-server")
 
 func createSchema(db database.Database) (graphql.Schema, error) {
+	runnerType := graphql.NewObject(graphql.ObjectConfig{
+		Name:        "Runner",
+		Description: "A runner available for use inside of a pipeline.",
+		Fields: graphql.Fields{
+			"id": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The id of the runner.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if runner, ok := p.Source.(database.Runner); ok {
+						return runner.Id, nil
+					}
+					return nil, nil
+				},
+			},
+			"name": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The name of the runner.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if runner, ok := p.Source.(database.Runner); ok {
+						return runner.Name, nil
+					}
+					return nil, nil
+				},
+			},
+			"token": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The token.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if runner, ok := p.Source.(database.Runner); ok {
+						return runner.Token, nil
+					}
+					return nil, nil
+				},
+			},
+		},
+	})
+
 	secretType := graphql.NewObject(graphql.ObjectConfig{
 		Name:        "Secret",
 		Description: "A secret available for use inside of a pipeline.",
@@ -348,6 +385,13 @@ func createSchema(db database.Database) (graphql.Schema, error) {
 					return db.GetSecrets()
 				},
 			},
+			"Runners": &graphql.Field{
+				Type: graphql.NewNonNull(graphql.NewList(runnerType)),
+				Args: graphql.FieldConfigArgument{},
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					return db.GetRunners()
+				},
+			},
 		},
 	})
 
@@ -485,6 +529,25 @@ func createSchema(db database.Database) (graphql.Schema, error) {
 					return secret, nil
 				},
 			},
+			"createRunner": &graphql.Field{
+				Type:        runnerType,
+				Description: "Create a new runner",
+				Args: graphql.FieldConfigArgument{
+					"name": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+				},
+				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
+
+					runner, err := db.CreateRunner(
+						params.Args["name"].(string),
+					)
+					if err != nil {
+						return nil, err
+					}
+					return runner, nil
+				},
+			},
 			"setPipelineCloneCredential": &graphql.Field{
 				Type:        pipelineType,
 				Description: "Set the CloneCredential used by a pipeline to clone the source repo",

database/db.go

@@ -175,8 +175,8 @@ CREATE TABLE command_executions (
 
 CREATE TABLE runners (
     id      UUID PRIMARY KEY,
-    name    TEXT,
+    name    TEXT NOT NULL UNIQUE,
-    secret  TEXT
+    token   TEXT NOT NULL
 );
 
 CREATE TABLE pipeline_refs (

database/func.go

@@ -6,7 +6,6 @@ import (
 	"regexp"
 
 	"github.com/google/uuid"
-	"github.com/jackc/pgx/v5"
 )
 
 func (db *Database) GetPipelines() ([]Pipeline, error) {
@@ -301,11 +300,11 @@ func (db *Database) UpdateRunResult(r Run) error {
 	query := `
 UPDATE runs
 SET in_progress=$1, result=$2, stdout=$3, stderr=$4
-WHERE id=$3;`
+WHERE id=$5;`
 
 	// TODO: does r.Result need a pointer derefrence?
 	_, err := db.Conn.Exec(context.Background(),
-		query, r.InProgress, r.Result, r.Stdout, r.Stderr)
+		query, r.InProgress, r.Result, r.Stdout, r.Stderr, r.Id)
 
 	return err
 }
@@ -379,21 +378,18 @@ WHERE pipeline_id=$1;`
 
 func (db *Database) UpdatePipelineRefs(pipelineId uuid.UUID, refsMap map[string]string) error {
 
-	refsSlice := make([][]interface{}, 0)
+	query := `
-	for name, ref := range refsMap {
+INSERT INTO pipeline_refs(name, pipeline_id, hash)
-		refsSlice = append(refsSlice, []interface{}{name, pipelineId, ref})
+VALUES($1, $2, $3)
-	}
+ON CONFLICT (name)
+DO
+  UPDATE SET hash=$3;`
 
-	copyCount, err := db.Conn.CopyFrom(
+	for name, hash := range refsMap {
-		context.Background(),
+		_, err := db.Conn.Exec(context.Background(), query, name, pipelineId, hash)
-		pgx.Identifier{"pipeline_refs"},
+
-		[]string{"name", "pipeline_id", "hash"},
+		return err
-		pgx.CopyFromRows(refsSlice),
-	)
-	if err != nil {
-		return fmt.Errorf("could not insert updated pipeline refs: %w", err)
 	}
-	log.Debugf("copyCount: %v", copyCount)
 	return nil
 }
 
@@ -453,7 +449,7 @@ func (db *Database) CreateSecret(name string, secret string) (Secret, error) {
 		return s, fmt.Errorf("secret name must be 256 characters or less")
 	}
 
-	validName := regexp.MustCompile(`^[A-Z0-9_]+$`)
+	validName := regexp.MustCompile(`[A-Z0-9_]+$`)
 	if !validName.MatchString(name) {
 		return s, fmt.Errorf("secren name must be made up of only uppercase letters, numbers, and underscores")
 	}
@@ -531,3 +527,83 @@ WHERE
 
 	return secrets, nil
 }
+
+func (db *Database) GetRunners() ([]Runner, error) {
+	query := `
+SELECT id, name, token
+FROM runners;`
+
+	runners := make([]Runner, 0)
+
+	rows, err := db.Conn.Query(context.Background(), query)
+	if err != nil {
+		return runners, fmt.Errorf("Could not query database for runners: %w", err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var runner Runner
+		var idStr string
+		if err := rows.Scan(&idStr, &runner.Name, &runner.Token); err != nil {
+			return runners, err
+		}
+
+		runner.Id, err = uuid.Parse(idStr)
+		if err != nil {
+			return runners, err
+		}
+		runners = append(runners, runner)
+	}
+
+	return runners, nil
+}
+
+func (db *Database) GetRunnerById(id uuid.UUID) (Runner, error) {
+	query := `
+SELECT name, token
+FROM runners
+WHERE id=$1;`
+
+	runner := Runner{
+		Id: id,
+	}
+
+	err := db.Conn.QueryRow(context.Background(), query, id).Scan(nil, &runner.Name, &runner.Token)
+	if err != nil {
+		return runner, fmt.Errorf("Could not query database for runner with id %v: %w", id.String(), err)
+	}
+
+	return runner, nil
+}
+
+func (db *Database) CreateRunner(name string) (Runner, error) {
+	s := Runner{}
+
+	// validate that the runner name is only A-Z or underscores and less than 256 characters
+	if len(name) > 256 {
+		return s, fmt.Errorf("runner name must be 256 characters or less")
+	}
+
+	validName := regexp.MustCompile(`[A-Z0-9_]+$`)
+	if !validName.MatchString(name) {
+		return s, fmt.Errorf("secren name must be made up of only uppercase letters, numbers, and underscores")
+	}
+
+	query := `
+INSERT INTO runners (id, name, token)
+VALUES (uuid_generate_v4(), $1, TODO_GENERATE_STRING)
+RETURNING id, name, token;`
+
+	var idStr string
+	err := db.Conn.QueryRow(context.Background(), query, name).Scan(&idStr, &s.Name, &s.Token)
+	if err != nil {
+		return s, fmt.Errorf("Could not create runner: %w", err)
+	}
+
+	s.Id, err = uuid.Parse(idStr)
+	if err != nil {
+		return s, fmt.Errorf("Could not parse UUID generated by DB: %w", err)
+	}
+
+	return s, nil
+}

database/types.go

@@ -74,7 +74,7 @@ type CommandExecution struct {
 }
 
 type Runner struct {
-	Id     uuid.UUID
+	Id    uuid.UUID
-	Name   string
+	Name  string
-	Secret string
+	Token string
 }

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/graphql-go/graphql v0.8.0
 	github.com/graphql-go/handler v0.2.3
 	github.com/jackc/pgx/v5 v5.2.0
+	github.com/jhoonb/archivex v0.0.0-20201016144719-6a343cdae81d
 	github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
 	golang.org/x/net v0.2.0
 	google.golang.org/protobuf v1.28.1

go.sum

@@ -858,6 +858,8 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOl
 github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
+github.com/jhoonb/archivex v0.0.0-20201016144719-6a343cdae81d h1:q7n+5taxmM+9T2Q7Ydo7YN90FkoDuR5bbzByZwkQqPo=
+github.com/jhoonb/archivex v0.0.0-20201016144719-6a343cdae81d/go.mod h1:GN1Mg/uXQ6qwXA0HypnUO3xlcQJS9/y68EsHNeuuRa4=
 github.com/jhump/protoreflect v1.6.1/go.mod h1:RZQ/lnuN+zqeRVpQigTwO6o0AJUkxbnSnpuG7toUTG4=
 github.com/jhump/protoreflect v1.8.2/go.mod h1:7GcYQDdMU/O/BBrl/cX6PNHpXh6cenjd8pneu5yW7Tg=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=

pipeline_executor/pipeline_executor.go

@@ -5,10 +5,13 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
 
+	"github.com/jhoonb/archivex"
+
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
@@ -34,6 +37,7 @@ type PipelineExecution struct {
 
 func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf config.PipelineConf) {
 	jobFolder := filepath.Join(pipelineConf.WorkingDir, pe.Pipeline.Id.String(), pe.Run.Id.String())
+	cloneFolder := filepath.Join(jobFolder, "repo")
 
 	log.Debugf("Job %v configured with URL \"%v\"", pe.Pipeline.Name, pe.Pipeline.Url)
 
@@ -45,7 +49,7 @@ func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf co
 		return
 	}
 
-	err = os.MkdirAll(jobFolder, 0755)
+	err = os.MkdirAll(cloneFolder, 0755)
 	if err != nil {
 		log.Errorf("could not create working directory for job %v: %w", pe.Pipeline.Name, err)
 		return
@@ -84,7 +88,7 @@ func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf co
 		auth = nil
 	}
 
-	_, err = git.PlainClone(jobFolder, false, &git.CloneOptions{
+	_, err = git.PlainClone(cloneFolder, false, &git.CloneOptions{
 		URL:  pe.Pipeline.Url,
 		Auth: auth,
 	})
@@ -102,6 +106,49 @@ func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf co
 
 	ctx := context.Background()
 
+	log.Info("Building container")
+
+	tarFile := filepath.Join(jobFolder, "archive.tar")
+	tar := new(archivex.TarFile)
+	err = tar.Create(tarFile)
+	if err != nil {
+		log.Errorf("could not create tarfile: %w", err)
+		return
+	}
+
+	err = tar.AddAll(cloneFolder, false)
+	if err != nil {
+		log.Errorf("could not add repo to tarfile: %w", err)
+		return
+	}
+
+	err = tar.Close()
+	if err != nil {
+		log.Errorf("could not close tarfile: %w", err)
+		return
+	}
+
+	dockerBuildContext, err := os.Open(tarFile)
+	defer dockerBuildContext.Close()
+
+	buildResponse, err := cli.ImageBuild(context.Background(), dockerBuildContext, types.ImageBuildOptions{
+		Dockerfile: ".cursorius/Dockerfile",
+	})
+	if err != nil {
+		log.Errorf("could not build container: %w", err)
+		return
+	}
+
+	log.Debugf("********* %s **********", buildResponse.OSType)
+	response, err := ioutil.ReadAll(buildResponse.Body)
+	if err != nil {
+		log.Errorf("could no read build response: %w", err)
+		return
+	}
+	log.Debugf("build log: %v", string(response))
+
+	return
+
 	imageName := "git.ohea.xyz/cursorius/pipeline-api/cursorius-pipeline:v2"
 
 	log.Infof("Pulling image %v", imageName)

poll/poll.go

@@ -28,15 +28,15 @@ type tag struct {
 
 func pollJob(pipeline database.Pipeline, pipelineConf config.PipelineConf, db database.Database) {
 	for {
+		time.Sleep(time.Duration(pipeline.PollInterval) * time.Second)
+		log.Infof("Polling repo %v", pipeline.Name)
+
 		prevRefs, err := db.GetPipelineRefs(pipeline.Id)
 		if err != nil {
 			log.Errorf("Could not get pipeline refs from db: %v", err)
 			return
 		}
 
-		time.Sleep(time.Duration(pipeline.PollInterval) * time.Second)
-		log.Infof("Polling repo %v", pipeline.Name)
-
 		repo, err := git.Clone(memory.NewStorage(), nil, &git.CloneOptions{
 			URL: pipeline.Url,
 		})