From 7a665aa348a87888fc805aed51ef4a54e3d6f7db Mon Sep 17 00:00:00 2001 From: restitux Date: Tue, 7 Feb 2023 21:34:32 -0700 Subject: [PATCH] Add support for access repos with credentials --- admin_api/admin_api.go | 179 ++++++++++++++++++++++++- database/db.go | 37 +++-- database/func.go | 141 +++++++++++++++++-- database/types.go | 28 +++- pipeline_executor/pipeline_executor.go | 52 +++++-- 5 files changed, 401 insertions(+), 36 deletions(-) diff --git a/admin_api/admin_api.go b/admin_api/admin_api.go index 37b2982..a596317 100644 --- a/admin_api/admin_api.go +++ b/admin_api/admin_api.go @@ -14,6 +14,63 @@ import ( var log = logging.MustGetLogger("cursorius-server") func createSchema(db database.Database) (graphql.Schema, error) { + credentialType := graphql.NewObject(graphql.ObjectConfig{ + Name: "Credential", + Description: "A credential for authenticating with the pipeline source host.", + Fields: graphql.Fields{ + "id": &graphql.Field{ + Type: graphql.NewNonNull(graphql.String), + Description: "The id of the credential.", + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + if credential, ok := p.Source.(database.Credential); ok { + return credential.Id, nil + } + return nil, nil + }, + }, + "name": &graphql.Field{ + Type: graphql.NewNonNull(graphql.String), + Description: "The name of the credential.", + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + if credential, ok := p.Source.(database.Credential); ok { + return credential.Name, nil + } + return nil, nil + }, + }, + "type": &graphql.Field{ + Type: graphql.NewNonNull(graphql.String), + Description: "The credential type.", + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + if credential, ok := p.Source.(database.Credential); ok { + return credential.Type, nil + } + return nil, nil + }, + }, + "username": &graphql.Field{ + Type: graphql.NewNonNull(graphql.String), + Description: "The username to user with the credential.", + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + if credential, ok := p.Source.(database.Credential); ok { + return credential.Username, nil + } + return nil, nil + }, + }, + "secret": &graphql.Field{ + Type: graphql.NewNonNull(graphql.String), + Description: "The secret for the credential.", + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + if credential, ok := p.Source.(database.Credential); ok { + return credential.Secret, nil + } + return nil, nil + }, + }, + }, + }) + webhookType := graphql.NewObject(graphql.ObjectConfig{ Name: "Webhook", Description: "A webhook for triggering pipelines", @@ -153,6 +210,16 @@ func createSchema(db database.Database) (graphql.Schema, error) { return nil, nil }, }, + "credentialId": &graphql.Field{ + Type: graphql.String, + Description: "The configured credential for cloning the pipeline source.", + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + if pipeline, ok := p.Source.(database.Pipeline); ok { + return pipeline.Credential, nil + } + return nil, nil + }, + }, "webhooks": &graphql.Field{ Type: graphql.NewNonNull(graphql.NewList(graphql.NewNonNull(webhookType))), Description: "The list of webhooks for the pipeline.", @@ -202,6 +269,29 @@ func createSchema(db database.Database) (graphql.Schema, error) { return db.GetPipelines() }, }, + "Credential": &graphql.Field{ + Type: credentialType, + Args: graphql.FieldConfigArgument{ + "id": &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(graphql.String), + Description: "The id of the requested credential.", + }, + }, + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + id, err := uuid.Parse(p.Args["id"].(string)) + if err != nil { + return nil, err + } + return db.GetCredentialById(id) + }, + }, + "Credentials": &graphql.Field{ + Type: graphql.NewNonNull(graphql.NewList(credentialType)), + Args: graphql.FieldConfigArgument{}, + Resolve: func(p graphql.ResolveParams) (interface{}, error) { + return db.GetCredentials() + }, + }, }, }) @@ -221,6 +311,9 @@ func createSchema(db database.Database) (graphql.Schema, error) { "pollInterval": &graphql.ArgumentConfig{ Type: graphql.Int, }, + "credentialId": &graphql.ArgumentConfig{ + Type: graphql.String, + }, }, Resolve: func(params graphql.ResolveParams) (interface{}, error) { var interval int @@ -229,18 +322,28 @@ func createSchema(db database.Database) (graphql.Schema, error) { } else { interval = 0 } + + var credential *uuid.UUID + if credentialVal, ok := params.Args["credentialId"]; ok { + id, err := uuid.Parse(credentialVal.(string)) + if err != nil { + return nil, err + } + credential = &id + } else { + credential = nil + } + pipeline, err := db.CreatePipeline( params.Args["name"].(string), params.Args["url"].(string), interval, + credential, ) if err != nil { return nil, err } - log.Infof("Created pipeline with id: %v, name: %v, url: %v, and poll interval: %v", - pipeline.Id, pipeline.Name, pipeline.Url, pipeline.PollInterval) - return pipeline, nil }, }, @@ -272,6 +375,76 @@ func createSchema(db database.Database) (graphql.Schema, error) { return webhook, nil }, }, + "createCredential": &graphql.Field{ + Type: credentialType, + Description: "Create a new credential", + Args: graphql.FieldConfigArgument{ + "name": &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(graphql.String), + }, + "type": &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(graphql.String), + }, + "username": &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(graphql.String), + }, + "secret": &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(graphql.String), + }, + }, + Resolve: func(params graphql.ResolveParams) (interface{}, error) { + + credential, err := db.CreateCredential( + params.Args["name"].(string), + database.CredentialType(params.Args["type"].(string)), + params.Args["username"].(string), + params.Args["secret"].(string), + ) + if err != nil { + return nil, err + } + return credential, nil + }, + }, + "setPipelineCredential": &graphql.Field{ + Type: pipelineType, + Description: "Add an credential to a pipeline", + Args: graphql.FieldConfigArgument{ + "credentialId": &graphql.ArgumentConfig{ + Type: graphql.String, + }, + "pipelineId": &graphql.ArgumentConfig{ + Type: graphql.NewNonNull(graphql.String), + }, + }, + Resolve: func(params graphql.ResolveParams) (interface{}, error) { + + pipelineId, err := uuid.Parse(params.Args["pipelineId"].(string)) + if err != nil { + return nil, err + } + + if credentialIdVal, ok := params.Args["credentialId"]; ok { + credentialId, err := uuid.Parse(credentialIdVal.(string)) + if err != nil { + return nil, err + } + + pipeline, err := db.SetPipelineCredential(pipelineId, &credentialId) + if err != nil { + return nil, err + } + return pipeline, nil + } else { + pipeline, err := db.SetPipelineCredential(pipelineId, nil) + if err != nil { + return nil, err + } + return pipeline, nil + } + + }, + }, }, }) diff --git a/database/db.go b/database/db.go index 8ffba24..8838977 100644 --- a/database/db.go +++ b/database/db.go @@ -40,10 +40,15 @@ func LaunchDB(conf config.DBConfig) (Database, error) { db := Database{} var err error + log.Infof("Connecting to database with URL \"%v\"", dbURLNoPasswd) + db.Conn, err = pgxpool.New(context.Background(), dbURL) + if err != nil { + return db, fmt.Errorf("could not create database pool: %w", err) + } + + // sleep until we can sucessfully acquire a connection for i := 0; i < 10; i++ { - // TODO: retry logic is broken with pgxpool - log.Infof("Connecting to database with URL \"%v\" (attempt %v)", dbURLNoPasswd, i) - db.Conn, err = pgxpool.New(context.Background(), dbURL) + _, err = db.Conn.Acquire(context.Background()) if err == nil { break } @@ -90,11 +95,24 @@ CREATE TABLE version ( ); +CREATE TABLE credentials ( + id UUID PRIMARY KEY, + name TEXT NOT NULL, + type TEXT NOT NULL, + username TEXT NOT NULL, + secret TEXT NOT NULL +); + CREATE TABLE pipelines ( id UUID PRIMARY KEY, name TEXT NOT NULL, url TEXT NOT NULL, - poll_interval INTEGER + poll_interval INTEGER, + credential UUID DEFAULT NULL, + + CONSTRAINT fk_credential + FOREIGN KEY(credential) + REFERENCES credentials(id) ); CREATE TABLE webhooks ( @@ -108,12 +126,6 @@ CREATE TABLE webhooks ( REFERENCES pipelines(id) ); -CREATE TABLE runners ( - id UUID PRIMARY KEY, - name TEXT, - secret TEXT -); - CREATE TABLE runs ( id UUID PRIMARY KEY, pipeline UUID, @@ -142,6 +154,11 @@ CREATE TABLE command_executions ( REFERENCES runs(id) ); +CREATE TABLE runners ( + id UUID PRIMARY KEY, + name TEXT, + secret TEXT +); ` _, err := conn.Exec(context.Background(), createTablesQuery) diff --git a/database/func.go b/database/func.go index a28a39c..97cdbd1 100644 --- a/database/func.go +++ b/database/func.go @@ -9,7 +9,7 @@ import ( func (db *Database) GetPipelines() ([]Pipeline, error) { query := ` -SELECT id, name, url, poll_interval +SELECT id, name, url, poll_interval, credential FROM pipelines;` pipelines := make([]Pipeline, 0) @@ -23,7 +23,7 @@ FROM pipelines;` for rows.Next() { var pipeline Pipeline var idStr string - if err := rows.Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval); err != nil { + if err := rows.Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential); err != nil { return pipelines, err } @@ -55,29 +55,71 @@ WHERE id=$1;` return pipeline, nil } -func (db *Database) CreatePipeline(name string, url string, pollInterval int) (Pipeline, error) { +func (db *Database) CreatePipeline(name string, url string, pollInterval int, credential *uuid.UUID) (Pipeline, error) { query := ` -INSERT INTO pipelines (id, name, url, poll_interval) -VALUES (uuid_generate_v4(), $1, $2, $3) +INSERT INTO pipelines (id, name, url, poll_interval, credential) +VALUES (uuid_generate_v4(), $1, $2, $3, $4) RETURNING id, name, url, poll_interval;` pipeline := Pipeline{} var idStr string - err := db.Conn.QueryRow(context.Background(), query, name, url, pollInterval).Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval) + err := db.Conn.QueryRow(context.Background(), query, name, url, pollInterval, credential).Scan(&idStr, &pipeline.Name, &pipeline.Url, &pipeline.PollInterval) if err != nil { return pipeline, fmt.Errorf("Could not create pipeline: %w", err) } - id, err := uuid.Parse(idStr) + pipeline.Id, err = uuid.Parse(idStr) if err != nil { return pipeline, fmt.Errorf("Could not parse UUID generated by DB: %w", err) } - pipeline.Id = id - return pipeline, nil } +func (db *Database) SetPipelineCredential(pipelineId uuid.UUID, credentialId *uuid.UUID) (Pipeline, error) { + query := ` +UPDATE pipelines +SET credential=$1 +WHERE id=$2 +RETURNING name, url, poll_interval, credential;` + + pipeline := Pipeline{ + Id: pipelineId, + } + + err := db.Conn.QueryRow(context.Background(), + query, credentialId, pipelineId).Scan( + &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential, + ) + if err != nil { + return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err) + } + + return pipeline, err +} + +func (db *Database) RemovePipelineCredential(pipelineId uuid.UUID) (Pipeline, error) { + query := ` +UPDATE pipelines +SET credential=null +WHERE id=$1 +RETURNING name, url, poll_interval, credential;` + + pipeline := Pipeline{ + Id: pipelineId, + } + + err := db.Conn.QueryRow(context.Background(), + query, pipelineId).Scan( + &pipeline.Name, &pipeline.Url, &pipeline.PollInterval, &pipeline.Credential, + ) + if err != nil { + return pipeline, fmt.Errorf("Could not add credential to pipeline: %w", err) + } + + return pipeline, err +} + func (db *Database) GetWebhooksForPipeline(id uuid.UUID) ([]Webhook, error) { query := ` SELECT id, server_type, secret @@ -151,6 +193,86 @@ RETURNING id, server_type, secret, pipeline;` return webhook, nil } +func (db *Database) CreateCredential(name string, credentialtype CredentialType, username string, secret string) (Credential, error) { + query := ` +INSERT INTO credentials (id, name, type, username, secret) +VALUES(uuid_generate_v4(), $1, $2, $3, $4) +RETURNING id, name, type, username, secret;` + + credential := Credential{} + var idStr string + err := db.Conn.QueryRow( + context.Background(), + query, + name, + string(credentialtype), + username, + secret, + ).Scan(&idStr, &credential.Name, &credential.Type, &credential.Username, &credential.Secret) + if err != nil { + return credential, err + } + + id, err := uuid.Parse(idStr) + if err != nil { + return credential, err + } + + credential.Id = id + + return credential, nil +} + +func (db *Database) GetCredentialById(id uuid.UUID) (Credential, error) { + query := ` +SELECT name, type, username, secret +FROM credentials +WHERE id=$1;` + + log.Debugf("requested credential with id %v", id) + + credential := Credential{ + Id: id, + } + + err := db.Conn.QueryRow(context.Background(), query, id).Scan(&credential.Name, &credential.Type, &credential.Username, &credential.Secret) + if err != nil { + return credential, fmt.Errorf("Could not query database for credential with id %v: %w", id.String(), err) + } + + return credential, nil +} + +func (db *Database) GetCredentials() ([]Credential, error) { + query := ` +SELECT id, name, type, username, secret +FROM credentials;` + + credentials := make([]Credential, 0) + + rows, err := db.Conn.Query(context.Background(), query) + if err != nil { + return credentials, fmt.Errorf("Could not query database for credentials: %w", err) + } + defer rows.Close() + + for rows.Next() { + var credential Credential + var idStr string + if err := rows.Scan(&idStr, &credential.Name, &credential.Type, &credential.Username, &credential.Secret); err != nil { + return credentials, err + } + + credential.Id, err = uuid.Parse(idStr) + if err != nil { + return credentials, err + } + credentials = append(credentials, credential) + } + + return credentials, nil +} + func (db *Database) CreateRun(pipelineId uuid.UUID) (Run, error) { query := ` INSERT INTO runs (id, pipeline, in_progress) @@ -173,6 +295,7 @@ RETURNING id, pipeline, in_progress;` } func (db *Database) UpdateRunResult(r Run) error { + // TODO: the id fiend is the query is broken query := ` UPDATE runs SET in_progress=$1, result=$2, stdout=$3, stderr=$4 diff --git a/database/types.go b/database/types.go index 387f6d6..2c4b67f 100644 --- a/database/types.go +++ b/database/types.go @@ -6,11 +6,27 @@ import ( "github.com/google/uuid" ) +type CredentialType string + +const ( + USER_PASS CredentialType = "USER_PASS" + SSH_KEY CredentialType = "SSH_KEY" +) + +type Credential struct { + Id uuid.UUID + Name string + Type CredentialType + Username string + Secret string +} + type Pipeline struct { Id uuid.UUID Name string Url string PollInterval int + Credential *uuid.UUID } type WebhookSender string @@ -26,12 +42,6 @@ type Webhook struct { Pipeline uuid.UUID } -type Runner struct { - Id uuid.UUID - Name string - Secret string -} - type Run struct { Id uuid.UUID Pipeline uuid.UUID @@ -51,3 +61,9 @@ type CommandExecution struct { StartTime time.Time EndTime time.Time } + +type Runner struct { + Id uuid.UUID + Name string + Secret string +} diff --git a/pipeline_executor/pipeline_executor.go b/pipeline_executor/pipeline_executor.go index 27b8698..c0e72dc 100644 --- a/pipeline_executor/pipeline_executor.go +++ b/pipeline_executor/pipeline_executor.go @@ -6,17 +6,21 @@ import ( "fmt" "io" "os" - "os/exec" "path/filepath" - "git.ohea.xyz/cursorius/server/config" - "git.ohea.xyz/cursorius/server/database" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/mount" "github.com/docker/docker/client" "github.com/docker/docker/pkg/stdcopy" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/transport" + "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/op/go-logging" + + "git.ohea.xyz/cursorius/server/config" + "git.ohea.xyz/cursorius/server/database" ) var log = logging.MustGetLogger("cursorius-server") @@ -47,12 +51,44 @@ func ExecutePipeline(pe PipelineExecution, db database.Database, pipelineConf co } log.Infof("Cloning source from URL %v", pe.Pipeline.Url) - // TODO: should I use go-git here instead of shelling out to raw git? - cloneCmd := exec.Command("git", "clone", pe.Pipeline.Url, jobFolder) - output, err := cloneCmd.CombinedOutput() + + var auth transport.AuthMethod + + if pe.Pipeline.Credential != nil { + credential, err := db.GetCredentialById(*pe.Pipeline.Credential) + if err != nil { + log.Errorf("could not get credenital from db: %v", err) + return + } + + switch credential.Type { + case "USER_PASS": + log.Debugf("job %v configured to use credential %v", pe.Pipeline.Name, credential.Name) + auth = transport.AuthMethod(&http.BasicAuth{ + Username: credential.Username, + Password: credential.Secret, + }) + case "SSH_KEY": + publicKeys, err := ssh.NewPublicKeys(credential.Username, []byte(credential.Secret), "") + if err != nil { + log.Errorf("could not parse credential %v", credential.Name) + return + } + auth = transport.AuthMethod(publicKeys) + default: + log.Errorf("unsupported credential type %v", credential.Type) + return + } + } else { + auth = nil + } + + _, err = git.PlainClone(jobFolder, false, &git.CloneOptions{ + URL: pe.Pipeline.Url, + Auth: auth, + }) if err != nil { - log.Debugf("%s", output) - log.Errorf("could not clone source: %w", err) + log.Errorf("could not clone repo: %v", err) return }