Add support for access repos with credentials

admin_api/admin_api.go

@@ -14,6 +14,63 @@ import (
 var log = logging.MustGetLogger("cursorius-server")
 
 func createSchema(db database.Database) (graphql.Schema, error) {
+	credentialType := graphql.NewObject(graphql.ObjectConfig{
+		Name:        "Credential",
+		Description: "A credential for authenticating with the pipeline source host.",
+		Fields: graphql.Fields{
+			"id": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The id of the credential.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if credential, ok := p.Source.(database.Credential); ok {
+						return credential.Id, nil
+					}
+					return nil, nil
+				},
+			},
+			"name": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The name of the credential.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if credential, ok := p.Source.(database.Credential); ok {
+						return credential.Name, nil
+					}
+					return nil, nil
+				},
+			},
+			"type": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The credential type.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if credential, ok := p.Source.(database.Credential); ok {
+						return credential.Type, nil
+					}
+					return nil, nil
+				},
+			},
+			"username": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The username to user with the credential.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if credential, ok := p.Source.(database.Credential); ok {
+						return credential.Username, nil
+					}
+					return nil, nil
+				},
+			},
+			"secret": &graphql.Field{
+				Type:        graphql.NewNonNull(graphql.String),
+				Description: "The secret for the credential.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if credential, ok := p.Source.(database.Credential); ok {
+						return credential.Secret, nil
+					}
+					return nil, nil
+				},
+			},
+		},
+	})
+
 	webhookType := graphql.NewObject(graphql.ObjectConfig{
 		Name:        "Webhook",
 		Description: "A webhook for triggering pipelines",
@@ -153,6 +210,16 @@ func createSchema(db database.Database) (graphql.Schema, error) {
 					return nil, nil
 				},
 			},
+			"credentialId": &graphql.Field{
+				Type:        graphql.String,
+				Description: "The configured credential for cloning the pipeline source.",
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					if pipeline, ok := p.Source.(database.Pipeline); ok {
+						return pipeline.Credential, nil
+					}
+					return nil, nil
+				},
+			},
 			"webhooks": &graphql.Field{
 				Type:        graphql.NewNonNull(graphql.NewList(graphql.NewNonNull(webhookType))),
 				Description: "The list of webhooks for the pipeline.",
@@ -202,6 +269,29 @@ func createSchema(db database.Database) (graphql.Schema, error) {
 					return db.GetPipelines()
 				},
 			},
+			"Credential": &graphql.Field{
+				Type: credentialType,
+				Args: graphql.FieldConfigArgument{
+					"id": &graphql.ArgumentConfig{
+						Type:        graphql.NewNonNull(graphql.String),
+						Description: "The id of the requested credential.",
+					},
+				},
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					id, err := uuid.Parse(p.Args["id"].(string))
+					if err != nil {
+						return nil, err
+					}
+					return db.GetCredentialById(id)
+				},
+			},
+			"Credentials": &graphql.Field{
+				Type: graphql.NewNonNull(graphql.NewList(credentialType)),
+				Args: graphql.FieldConfigArgument{},
+				Resolve: func(p graphql.ResolveParams) (interface{}, error) {
+					return db.GetCredentials()
+				},
+			},
 		},
 	})
 
@@ -221,6 +311,9 @@ func createSchema(db database.Database) (graphql.Schema, error) {
 					"pollInterval": &graphql.ArgumentConfig{
 						Type: graphql.Int,
 					},
+					"credentialId": &graphql.ArgumentConfig{
+						Type: graphql.String,
+					},
 				},
 				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
 					var interval int
@@ -229,18 +322,28 @@ func createSchema(db database.Database) (graphql.Schema, error) {
 					} else {
 						interval = 0
 					}
+
+					var credential *uuid.UUID
+					if credentialVal, ok := params.Args["credentialId"]; ok {
+						id, err := uuid.Parse(credentialVal.(string))
+						if err != nil {
+							return nil, err
+						}
+						credential = &id
+					} else {
+						credential = nil
+					}
+
 					pipeline, err := db.CreatePipeline(
 						params.Args["name"].(string),
 						params.Args["url"].(string),
 						interval,
+						credential,
 					)
 					if err != nil {
 						return nil, err
 					}
 
-					log.Infof("Created pipeline with id: %v, name: %v, url: %v, and poll interval: %v",
-						pipeline.Id, pipeline.Name, pipeline.Url, pipeline.PollInterval)
-
 					return pipeline, nil
 				},
 			},
@@ -272,6 +375,76 @@ func createSchema(db database.Database) (graphql.Schema, error) {
 					return webhook, nil
 				},
 			},
+			"createCredential": &graphql.Field{
+				Type:        credentialType,
+				Description: "Create a new credential",
+				Args: graphql.FieldConfigArgument{
+					"name": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+					"type": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+					"username": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+					"secret": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+				},
+				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
+
+					credential, err := db.CreateCredential(
+						params.Args["name"].(string),
+						database.CredentialType(params.Args["type"].(string)),
+						params.Args["username"].(string),
+						params.Args["secret"].(string),
+					)
+					if err != nil {
+						return nil, err
+					}
+					return credential, nil
+				},
+			},
+			"setPipelineCredential": &graphql.Field{
+				Type:        pipelineType,
+				Description: "Add an credential to a pipeline",
+				Args: graphql.FieldConfigArgument{
+					"credentialId": &graphql.ArgumentConfig{
+						Type: graphql.String,
+					},
+					"pipelineId": &graphql.ArgumentConfig{
+						Type: graphql.NewNonNull(graphql.String),
+					},
+				},
+				Resolve: func(params graphql.ResolveParams) (interface{}, error) {
+
+					pipelineId, err := uuid.Parse(params.Args["pipelineId"].(string))
+					if err != nil {
+						return nil, err
+					}
+
+					if credentialIdVal, ok := params.Args["credentialId"]; ok {
+						credentialId, err := uuid.Parse(credentialIdVal.(string))
+						if err != nil {
+							return nil, err
+						}
+
+						pipeline, err := db.SetPipelineCredential(pipelineId, &credentialId)
+						if err != nil {
+							return nil, err
+						}
+						return pipeline, nil
+					} else {
+						pipeline, err := db.SetPipelineCredential(pipelineId, nil)
+						if err != nil {
+							return nil, err
+						}
+						return pipeline, nil
+					}
+
+				},
+			},
 		},
 	})
 

database/db.go

@@ -40,10 +40,15 @@ func LaunchDB(conf config.DBConfig) (Database, error) {
 	db := Database{}
 
 	var err error
+	log.Infof("Connecting to database with URL \"%v\"", dbURLNoPasswd)
+	db.Conn, err = pgxpool.New(context.Background(), dbURL)
+	if err != nil {
+		return db, fmt.Errorf("could not create database pool: %w", err)
+	}
+
+	// sleep until we can sucessfully acquire a connection
 	for i := 0; i < 10; i++ {
-		// TODO: retry logic is broken with pgxpool
+		_, err = db.Conn.Acquire(context.Background())
-		log.Infof("Connecting to database with URL \"%v\" (attempt %v)", dbURLNoPasswd, i)
-		db.Conn, err = pgxpool.New(context.Background(), dbURL)
 		if err == nil {
 			break
 		}
@@ -90,11 +95,24 @@ CREATE TABLE version (
 
 );
 
+CREATE TABLE credentials (
+	id		  UUID PRIMARY KEY,
+	name      TEXT NOT NULL,
+    type      TEXT NOT NULL,
+	username  TEXT NOT NULL,
+	secret    TEXT NOT NULL
+);
+
 CREATE TABLE pipelines (
     id             UUID PRIMARY KEY,
     name           TEXT NOT NULL,
     url            TEXT NOT NULL,
-    poll_interval  INTEGER
+    poll_interval  INTEGER,
+	credential     UUID DEFAULT NULL,
+
+    CONSTRAINT fk_credential
+        FOREIGN KEY(credential)
+            REFERENCES credentials(id)
 );
 
 CREATE TABLE webhooks (
@@ -108,12 +126,6 @@ CREATE TABLE webhooks (
             REFERENCES pipelines(id)
 );
 
-CREATE TABLE runners (
-    id      UUID PRIMARY KEY,
-    name    TEXT,
-    secret  TEXT
-);
-
 CREATE TABLE runs (
     id            UUID PRIMARY KEY,
     pipeline      UUID,
@@ -142,6 +154,11 @@ CREATE TABLE command_executions (
             REFERENCES runs(id)
 );
 
+CREATE TABLE runners (
+    id      UUID PRIMARY KEY,
+    name    TEXT,
+    secret  TEXT
+);
 `
 
 	_, err := conn.Exec(context.Background(), createTablesQuery)

database/func.go

@@ -9,7 +9,7 @@ import (
 
 func (db *Database) GetPipelines() ([]Pipeline, error) {
 	query := `
-SELECT id, name, url, poll_interval
+SELECT id, name, url, poll_interval, credential
 FROM pipelines;`
 
 	pipelines := make([]Pipeline, 0)
@@ -23,7 +23,7 @@ FROM pipelines;`
 	for rows.Next() {
 		var pipeline Pipeline
 		var idStr string
-		if err := rows.Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval); err != nil {
+		if err := rows.Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential); err != nil {
 			return pipelines, err
 		}
 
@@ -55,29 +55,71 @@ WHERE id=$1;`
 	return pipeline, nil
 }
 
-func (db *Database) CreatePipeline(name string, url string, pollInterval int) (Pipeline, error) {
+func (db *Database) CreatePipeline(name string, url string, pollInterval int, credential *uuid.UUID) (Pipeline, error) {
 	query := `
-INSERT INTO pipelines (id, name, url, poll_interval)
+INSERT INTO pipelines (id, name, url, poll_interval, credential)
-VALUES (uuid_generate_v4(), $1, $2, $3)
+VALUES (uuid_generate_v4(), $1, $2, $3, $4)
 RETURNING id, name, url, poll_interval;`
 
 	pipeline := Pipeline{}
 	var idStr string
-	err := db.Conn.QueryRow(context.Background(), query, name, url, pollInterval).Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval)
+	err := db.Conn.QueryRow(context.Background(), query, name, url, pollInterval, credential).Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval)
 	if err != nil {
 		return pipeline, fmt.Errorf("Could not create pipeline: %w", err)
 	}
 
-	id, err := uuid.Parse(idStr)
+	pipeline.Id, err = uuid.Parse(idStr)
 	if err != nil {
 		return pipeline, fmt.Errorf("Could not parse UUID generated by DB: %w", err)
 	}
 
-	pipeline.Id = id
-
 	return pipeline, nil
 }
 
+func (db *Database) SetPipelineCredential(pipelineId uuid.UUID, credentialId *uuid.UUID) (Pipeline, error) {
+	query := `
+UPDATE pipelines
+SET credential=$1
+WHERE id=$2
+RETURNING name, url, poll_interval, credential;`
+
+	pipeline := Pipeline{
+		Id: pipelineId,
+	}
+
+	err := db.Conn.QueryRow(context.Background(),
+		query, credentialId, pipelineId).Scan(
+		&pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential,
+	)
+	if err != nil {
+		return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err)
+	}
+
+	return pipeline, err
+}
+
+func (db *Database) RemovePipelineCredential(pipelineId uuid.UUID) (Pipeline, error) {
+	query := `
+UPDATE pipelines
+SET credential=null
+WHERE id=$1
+RETURNING name, url, poll_interval, credential;`
+
+	pipeline := Pipeline{
+		Id: pipelineId,
+	}
+
+	err := db.Conn.QueryRow(context.Background(),
+		query, pipelineId).Scan(
+		&pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential,
+	)
+	if err != nil {
+		return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err)
+	}
+
+	return pipeline, err
+}
+
 func (db *Database) GetWebhooksForPipeline(id uuid.UUID) ([]Webhook, error) {
 	query := `
 SELECT id, server_type, secret
@@ -151,6 +193,86 @@ RETURNING id, server_type, secret, pipeline;`
 	return webhook, nil
 }
 
+func (db *Database) CreateCredential(name string, credentialtype CredentialType, username string, secret string) (Credential, error) {
+	query := `
+INSERT INTO credentials (id, name, type, username, secret)
+VALUES(uuid_generate_v4(), $1, $2, $3, $4)
+RETURNING id, name, type, username, secret;`
+
+	credential := Credential{}
+	var idStr string
+	err := db.Conn.QueryRow(
+		context.Background(),
+		query,
+		name,
+		string(credentialtype),
+		username,
+		secret,
+	).Scan(&idStr, &credential.Name, &credential.Type, &credential.Username, &credential.Secret)
+	if err != nil {
+		return credential, err
+	}
+
+	id, err := uuid.Parse(idStr)
+	if err != nil {
+		return credential, err
+	}
+
+	credential.Id = id
+
+	return credential, nil
+}
+
+func (db *Database) GetCredentialById(id uuid.UUID) (Credential, error) {
+	query := `
+SELECT name, type, username, secret
+FROM credentials
+WHERE id=$1;`
+
+	log.Debugf("requested credential with id %v", id)
+
+	credential := Credential{
+		Id: id,
+	}
+
+	err := db.Conn.QueryRow(context.Background(), query, id).Scan(&credential.Name, &credential.Type, &credential.Username, &credential.Secret)
+	if err != nil {
+		return credential, fmt.Errorf("Could not query database for credential with id %v: %w", id.String(), err)
+	}
+
+	return credential, nil
+}
+
+func (db *Database) GetCredentials() ([]Credential, error) {
+	query := `
+SELECT id, name, type, username, secret
+FROM credentials;`
+
+	credentials := make([]Credential, 0)
+
+	rows, err := db.Conn.Query(context.Background(), query)
+	if err != nil {
+		return credentials, fmt.Errorf("Could not query database for credentials: %w", err)
+	}
+	defer rows.Close()
+
+	for rows.Next() {
+		var credential Credential
+		var idStr string
+		if err := rows.Scan(&idStr, &credential.Name, &credential.Type, &credential.Username, &credential.Secret); err != nil {
+			return credentials, err
+		}
+
+		credential.Id, err = uuid.Parse(idStr)
+		if err != nil {
+			return credentials, err
+		}
+		credentials = append(credentials, credential)
+	}
+
+	return credentials, nil
+}
+
 func (db *Database) CreateRun(pipelineId uuid.UUID) (Run, error) {
 	query := `
 INSERT INTO runs (id, pipeline, in_progress)
@@ -173,6 +295,7 @@ RETURNING id, pipeline, in_progress;`
 }
 
 func (db *Database) UpdateRunResult(r Run) error {
+	// TODO: the id fiend is the query is broken
 	query := `
 UPDATE runs
 SET in_progress=$1, result=$2, stdout=$3, stderr=$4

database/types.go

@@ -6,11 +6,27 @@ import (
 	"github.com/google/uuid"
 )
 
+type CredentialType string
+
+const (
+	USER_PASS CredentialType = "USER_PASS"
+	SSH_KEY   CredentialType = "SSH_KEY"
+)
+
+type Credential struct {
+	Id       uuid.UUID
+	Name     string
+	Type     CredentialType
+	Username string
+	Secret   string
+}
+
 type Pipeline struct {
 	Id           uuid.UUID
 	Name         string
 	Url          string
 	PollInterval int
+	Credential   *uuid.UUID
 }
 
 type WebhookSender string
@@ -26,12 +42,6 @@ type Webhook struct {
 	Pipeline   uuid.UUID
 }
 
-type Runner struct {
-	Id     uuid.UUID
-	Name   string
-	Secret string
-}
-
 type Run struct {
 	Id         uuid.UUID
 	Pipeline   uuid.UUID
@@ -51,3 +61,9 @@ type CommandExecution struct {
 	StartTime  time.Time
 	EndTime    time.Time
 }
+
+type Runner struct {
+	Id     uuid.UUID
+	Name   string
+	Secret string
+}

pipeline_executor/pipeline_executor.go

@@ -6,17 +6,21 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
 
-	"git.ohea.xyz/cursorius/server/config"
-	"git.ohea.xyz/cursorius/server/database"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/stdcopy"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/transport"
+	"github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
 	"github.com/op/go-logging"
+
+	"git.ohea.xyz/cursorius/server/config"
+	"git.ohea.xyz/cursorius/server/database"
 )
 
 var log = logging.MustGetLogger("cursorius-server")
@@ -47,12 +51,44 @@ func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf co
 	}
 
 	log.Infof("Cloning source from URL %v", pe.Pipeline.Url)
-	// TODO: should I use go-git here instead of shelling out to raw git?
+
-	cloneCmd := exec.Command("git", "clone", pe.Pipeline.Url, jobFolder)
+	var auth transport.AuthMethod
-	output, err := cloneCmd.CombinedOutput()
+
+	if pe.Pipeline.Credential != nil {
+		credential, err := db.GetCredentialById(*pe.Pipeline.Credential)
+		if err != nil {
+			log.Errorf("could not get credenital from db: %v", err)
+			return
+		}
+
+		switch credential.Type {
+		case "USER_PASS":
+			log.Debugf("job %v configured to use credential %v", pe.Pipeline.Name, credential.Name)
+			auth = transport.AuthMethod(&http.BasicAuth{
+				Username: credential.Username,
+				Password: credential.Secret,
+			})
+		case "SSH_KEY":
+			publicKeys, err := ssh.NewPublicKeys(credential.Username, []byte(credential.Secret), "")
+			if err != nil {
+				log.Errorf("could not parse credential %v", credential.Name)
+				return
+			}
+			auth = transport.AuthMethod(publicKeys)
+		default:
+			log.Errorf("unsupported credential type %v", credential.Type)
+			return
+		}
+	} else {
+		auth = nil
+	}
+
+	_, err = git.PlainClone(jobFolder, false, &git.CloneOptions{
+		URL:  pe.Pipeline.Url,
+		Auth: auth,
+	})
 	if err != nil {
-		log.Debugf("%s", output)
+		log.Errorf("could not clone repo: %v", err)
-		log.Errorf("could not clone source: %w", err)
 		return
 	}